def import_view(request): """ Gets the existing declared parsers for the current project. This view handles only the file based import parsers. """ choices = [] choices_url = [] render_dict = {} choices, choices_url, classes = discover_available_parsers() form = ImportDatasetFormWithFile(choices, prefix="with-file") form_without_file = ImportDatasetForm( choices_url, prefix="without-file") if request.method == 'POST': if 'upload-file' in request.POST: form = ImportDatasetFormWithFile( choices, request.POST, request.FILES, prefix="with-file") if form.is_valid(): print(request.FILES) uploaded = request.FILES['with-file-zipfile'] destination_dir, destination_file = create_tmp_destination( uploaded.name) with open(destination_file, 'w+') as f: f.write(uploaded.file.read()) zfile = ZipFile(f) for name in zfile.namelist(): try: zfile.extract( name, os.path.dirname(os.path.realpath(f.name))) if name.endswith('shp'): parser = classes[int(form['parser'].value())] import_datas.delay( '/'.join((destination_dir, name)), parser.__name__, parser.__module__ ) continue except Exception: raise if 'import-web' in request.POST: form_without_file = ImportDatasetForm( choices_url, request.POST, prefix="without-file") if form_without_file.is_valid(): parser = classes[int(form_without_file['parser'].value())] import_datas_from_web.delay( parser.__name__, parser.__module__ ) # Hide second form if parser has no web based imports. render_dict['form'] = form if choices_url: render_dict['form_without_file'] = form_without_file return render(request, 'common/import_dataset.html', render_dict)
def _loadShpList(self): """ Loads (extracts) a batch of shapefiles from an archive. Only those that match the given pattern are extracted. """ self.shpList = None shpFiles = list() tmpFiles = list() shpmatch = re.compile('\.shp$') searchRe = re.compile(self.pattern) # Iterate through the list of archives. for i in range(self.zipIndex, len(self.zipList)): f = self.zipList[i] try: # Open the zip file and read the entries. z = ZipFile(f) entries = z.namelist() # Iterate through the entries. If an entry matches the search string, extract # it to the tmp folder. for e in entries: if searchRe.search(e): z.extract(e, self.tmpDir) # Add to the list of extracted files. tmpFiles.append(e) # If the entry is a shapefile, add it to the shp list. if shpmatch.search(e): shpFiles.append(e) except Exception, e: print "Failed to open {0}: {1}.".format(f, e.__str__()) finally:
def extract_subfiles(source_path, dest_path, verbose=False): if os.path.isdir(source_path): for dirpath, dirnames, filenames in os.walk(source_path): relpath = os.path.relpath(dirpath, source_path) new_dir_path = os.path.join(dest_path, relpath) if not os.path.isdir(new_dir_path): os.mkdir(new_dir_path) for filename in filenames: try: source_file_path = os.path.join(dirpath, filename) relpath = os.path.relpath(source_file_path, source_path) dest_file_path = os.path.join(dest_path, relpath) print dest_file_path if dest_file_path.endswith('.cod'): zip = ZipFile(source_file_path) for info in zip.infolist(): if verbose: print ' %s (extracted)' % info.filename dest_unzip_path = os.path.split(dest_file_path)[0] if not os.path.realpath(os.path.join(dest_unzip_path, info.filename)).startswith(os.path.realpath(dest_unzip_path)): raise(Exception('Security exception: zip file %s attempted to extract to a non-local location' % info.filename)) zip.extract(info, path = dest_unzip_path) else: shutil.copyfile(source_file_path, dest_file_path) except Exception, e: if str(e) == 'File is not a zip file': # this is a cod file or some other file shutil.copyfile(source_file_path, dest_file_path) else: if verbose: print >>sys.stderr, 'Error:', print >>sys.stderr, str(e) raise(e)
def main(): parser = argparse.ArgumentParser('Update the icomoon icon font from the provided archive') parser.add_argument('archive', help='Path to .zip file generated by icomoon') args = parser.parse_args() script_dir = os.path.dirname(os.path.abspath(__file__)) vendor_style_dir = script_dir + '/../h/static/styles/vendor' icon_font_archive = ZipFile(args.archive) icon_font_archive.extract('selection.json', vendor_style_dir + '/fonts') icon_font_archive.extract('fonts/h.woff', vendor_style_dir) css_input_file = icon_font_archive.open('style.css') css_output_file = open(vendor_style_dir + '/icomoon.css', 'w') for line in css_input_file: if "format('woff')" in line: # inline the WOFF format file woff_content = icon_font_archive.open('fonts/h.woff').read() woff_src_line = """ /* WARNING - the URL below is inlined * because the CSS asset pipeline is not correctly rebasing * URLs when concatenating files together. * * See issue #2571 */ src:url('data:application/font-woff;base64,%s') format('woff'); """ css_output_file.write(woff_src_line % b64encode(woff_content)) elif "url(" in line: # skip non-WOFF format fonts pass else: css_output_file.write(line)
def download_custom_reports(domain, report_id, report_name): """Downloads custom report data csv zip export from commcare. The extracted csv is placed in data/DOMAIN/latest/Pregnancy Visit.csv. :domain - commcare domain :report_id - custom report id for report """ if not report_id: raise Exception(u"Please ensure the custom report id" u" is configured in CUSTOM_REPORTS settings.") url_path = "/a/%(domain)s/reports/export/custom/%(id)s/download/"\ "?format=csv" % {'domain': domain, 'id': report_id} f = download_from_commcare(url_path) try: z = ZipFile(f, 'r') except BadZipfile: print f.read() else: zdst = os.path.join('data', domain, 'latest') dst = os.path.join(zdst, report_name) if z.NameToInfo.keys(): filename = z.NameToInfo.keys()[0] src = os.path.join(zdst, filename) z.extract(z.NameToInfo[filename], zdst) if os.path.exists(src): os.rename(src, dst) print u"Successfully downloaded %s" % report_name
def main(opts): for filename in opts.files: zip_file = ZipFile(filename) info_list = zip_file.infolist() for entry in info_list: entry.filename = decode_filename(entry.filename, opts.codepage) zip_file.extract(entry, path=opts.dest)
def install(self, sid, data, fname): try: z = ZipFile(io.BytesIO(data)) except zipfile.BadZipfile: showWarning(_("The download was corrupt. Please try again.")) return name = os.path.splitext(fname)[0] # previously installed? meta = self.addonMeta(sid) base = self.addonsFolder(sid) if os.path.exists(base): self.backupUserFiles(sid) self.deleteAddon(sid) os.mkdir(base) self.restoreUserFiles(sid) # extract for n in z.namelist(): if n.endswith("/"): # folder; ignore continue path = os.path.join(base, n) # skip existing user files if os.path.exists(path) and n.startswith("user_files/"): continue z.extract(n, base) # update metadata meta['name'] = name meta['mod'] = intTime() self.writeAddonMeta(sid, meta)
def unzip(filename, match_dir=False, destdir=None): """ Extract all files from a zip archive filename: The path to the zip file match_dir: If True all files in the zip must be contained in a subdirectory named after the archive file with extension removed destdir: Extract the zip into this directory, default current directory return: If match_dir is True then returns the subdirectory (including destdir), otherwise returns destdir or '.' """ if not destdir: destdir = '.' z = ZipFile(filename) unzipped = '.' if match_dir: if not filename.endswith('.zip'): raise FileException('Expected .zip file extension', filename) unzipped = os.path.basename(filename)[:-4] check_extracted_paths(z.namelist(), unzipped) else: check_extracted_paths(z.namelist()) # File permissions, see # http://stackoverflow.com/a/6297838 # http://stackoverflow.com/a/3015466 for info in z.infolist(): log.debug('Extracting %s to %s', info.filename, destdir) z.extract(info, destdir) os.chmod(os.path.join(destdir, info.filename), info.external_attr >> 16 & 4095) return os.path.join(destdir, unzipped)
def extract(zip): """Extrait le fichier '/word/document.xml' d'une archive word dans le répertoire courant """ myzip = ZipFile(zip, 'a') myzip.extract('word/document.xml', './../')
def fetch_fs_peptide(data_home=None, download_if_missing=True): """Loader for the Fs peptide dataset Parameters ---------- data_home : optional, default: None Specify another download and cache folder for the datasets. By default all mixtape data is stored in '~/mixtape_data' subfolders. download_if_missing: optional, True by default If False, raise a IOError if the data is not locally available instead of trying to download the data from the source site. """ data_home = get_data_home(data_home=data_home) if not exists(data_home): makedirs(data_home) data_dir = join(data_home, TARGET_DIRECTORY) if not exists(data_dir): print('downloading fs peptide from %s to %s' % (DATA_URL, data_home)) fhandle = urlopen(DATA_URL) buf = BytesIO(fhandle.read()) zip_file = ZipFile(buf) makedirs(data_dir) for name in zip_file.namelist(): zip_file.extract(name, path=data_dir) top = md.load(join(data_dir, 'fs_peptide.pdb')) trajectories = [] for fn in sorted(glob(join(data_dir, 'trajectory*.xtc'))): print('loading %s...' % basename(fn)) trajectories.append(md.load(fn, top=top)) return Bunch(trajectories=trajectories, DESCR=__doc__)
def unzip(filename): z = ZipFile(filename) names = z.namelist() for path in names: if path.startswith('__MACOSX/'): continue base, name = os.path.split(path) if name.startswith('._') and\ '%s/' % name.replace('._', '', 1) in names: continue double = os.path.join('__MACOSX', base, '._' + name) if double in names: print '=> %s.bin' % path info = z.getinfo(path) bin = MacBinary(name) bin.data = z.open(path, 'r').read() bin.res = z.open(double, 'r').read() modified = datetime.datetime(*info.date_time) bin.modified = time.mktime(modified.timetuple()) bin.created = time.time() if not os.path.exists(base): os.makedirs(base) with open('%s.bin' % path.rstrip('\r'), 'wb') as f: f.write(bin.encode()) else: print '-> %s' % path z.extract(path)
def download(url, dirname, filename): if int(__addon__.getSetting("sub_language1")) == 0: return {} cache_path = os.path.join(CACHE_DIR, "temp.zip") try: urlretrieve(url, cache_path) z = ZipFile(cache_path) except: return None path = None for f in z.namelist(): ext = os.path.splitext(f)[1] if not ext in _subtitles_formats: continue path = os.path.join(dirname, filename + ext) z.extract(f, CACHE_DIR) if os.path.isfile(path): os.unlink(path) os.rename(os.path.join(CACHE_DIR, f), path) break z.close() os.unlink(cache_path) return path
def census2000(summary_file_num, filenum): base_url = 'http://www2.census.gov/census_2000/datasets/' path = os.path.join( 'Summary_File_%s' % summary_file_num, state_name.replace(' ', '_'), '%s000%02d_uf%s.zip' % (state_abbr, filenum, summary_file_num) ) reader = remote_fallback_reader( '../data/census_2000/datasets', base_url, path ) z = ZipFile(reader) n = z.namelist() z.extract(n[0], '/tmp/') filename = '/tmp/%s000%02d.uf%s' % (state_abbr, filenum, summary_file_num) # count the number of columns, to determine how many columns to fill firstline = open(filename).readline() cols = ['fileid', 'stusab', 'chariter', 'cifsn', 'logrecno'] cols.extend(map( lambda c: 'col%s' % str(c+1), range(firstline.count(',')+1-5) # subtract 5 to account for the 5 geo header columns )) cursor.copy_from(open(filename), 'census_row', sep=',', columns=cols) os.unlink(filename)
def extract_from_zip(filename): zp = ZipFile(filename, 'r') namelist = zp.namelist() desired_files = [x for x in namelist if 'Award' in x or 'Summary' in x] for file in desired_files: if not os.path.exists(file): zp.extract(file)
def addCBZFiles(self, filenames): directories = [] tempDir = tempfile.gettempdir() filenames.sort() filenamesListed = [] for i in xrange(0, self.listWidgetFiles.count()): filenamesListed.append(self.listWidgetFiles.item(i).text()) for filename in filenames: folderName = os.path.splitext(basename(str(filename)))[0] path = tempDir + "/" + folderName + "/" cbzFile = ZipFile(str(filename)) for f in cbzFile.namelist(): if f.endswith('/'): try: os.makedirs(path + f) except: pass # the dir exists so we are going to extract the images only. else: cbzFile.extract(f, path) if os.path.isdir(unicode(path)): # Add the directories directories.append(path) self.addImageDirs(directories) # Add the files
def acs(file_num, geo_type_dir): base_url = 'http://www2.census.gov/acs2010_5yr/summaryfile/2006-2010_ACSSF_By_State_By_Sequence_Table_Subset/' path = os.path.join( state_name.replace(' ', ''), geo_type_dir, '20105%s%04d000.zip' % (state_abbr, file_num) ) reader = remote_fallback_reader( '../data/acs2006_2010_5yr/summaryfile/2006-2010_ACSSF_By_State_By_Sequence_Table_Subset', base_url, path ) z = ZipFile(reader) n = z.namelist() z.extractall('/tmp/') files = ['e20105%s%04d000.txt' % (state_abbr, file_num), 'm20105%s%04d000.txt' % (state_abbr, file_num),] for f in files: z.extract(f, '/tmp/') # count the number of columns, to determine how many columns to fill firstline = open('/tmp/' + f).readline() if not firstline: # some files are empty, so just continue to the next os.unlink('/tmp/' + f) continue cols = ['fileid', 'filetype', 'stusab', 'chariter', 'cifsn', 'logrecno'] cols.extend(map( lambda c: 'col%s' % str(c+1), range(firstline.count(',')+1-6) # subtract 6 to account for the 6 geo header columns )) cursor.copy_from(open('/tmp/%s' % f), 'census_row', sep=',', columns=cols) os.unlink('/tmp/' + f)
def unzip(zippath, outdir): """ Unzip the contents of zippath to outdir. Example: unzip("zelda30tribute.zip", "./assets") """ f = open(zippath, 'rb') z = ZipFile(f) ziprootdir = commondir(z.namelist()) fileno = 0 for asset in assets: fileno += 1 outpath = joinpaths(outdir, asset) inpath = joinpaths(ziprootdir, asset) print("Extracting ({}/{}): {} -> {}" .format(fileno, len(assets), inpath, outpath)) z.extract(inpath, outdir) if ziprootdir: # If the assets are stored within a top folder: srcpath = joinpaths(asset_dir, ziprootdir) dstpath = asset_dir print("Replacing top folder: {} -> {}".format(srcpath, dstpath)) topfiles = os.listdir(srcpath) for topfile in topfiles: # Remove if it happens to already be there: try: shutil.rmtree(joinpaths(dstpath, topfile)) except OSError as e: if e.errno != errno.ENOENT: raise shutil.move(joinpaths(srcpath, topfile), dstpath) shutil.rmtree(srcpath)
def base_scrape_alexa(url, browser_args, out_name, add_http): b = Browser(name, **browser_args) r = b.go(url) zip_file = StringIO(r.content) z = ZipFile(zip_file, 'r') csv_file = 'top-1m.csv' z.extract(csv_file, out_dir) file_path = path.join(out_dir, csv_file) lines = strip_open(file_path) out_path = path.join(out_dir, out_name) if add_http: os.remove(file_path) with open(out_path, 'w+') as f: for line in lines: print >> f, 'http://' + line.split(',')[1] else: try: os.rename(file_path, out_path) except Exception as e: pass return len(lines)
def install(self, sid, data, fname): try: z = ZipFile(io.BytesIO(data)) except zipfile.BadZipfile: showWarning(_("The download was corrupt. Please try again.")) return name = os.path.splitext(fname)[0] # previously installed? meta = self.addonMeta(sid) base = self.addonsFolder(sid) if os.path.exists(base): self.deleteAddon(sid) # extract os.mkdir(base) for n in z.namelist(): if n.endswith("/"): # folder; ignore continue # write z.extract(n, base) # update metadata meta['name'] = name meta['mod'] = intTime() self.writeAddonMeta(sid, meta)
def upgrade(self, version, force = False): if not mkdir(self.path): return version if self.updater == "bukkitdev": uver, urlh = self.__bukkitdev_info() elif self.updater == "bukkitdl": uver, urlh = self.__bukkitdl_info() elif self.updater == "github": uver, urlh = self.__github_info() elif self.updater == "jenkins": uver, urlh = self.__jenkins_info() else: log.error("%s: package upgrade failed: invalid updater `%s'", self.package, self.updater) return version if not urlh: log.error("%s: package upgrade failed", self.package) return version out = os.path.join(self.path, "%s.%s" % (self.package, self.type)) if uver and uver == version and not force: log.info("%s: package already up-to-date", self.package) return version if not self.dryrun and not download(urlh, out): return version log.info("%s: package upgraded: %s -> %s", self.package, version, uver) if self.dryrun or (self.type != "zip"): return uver if len(self.extract) < 1: return uver zf = ZipFile(out, "r") nl = zf.namelist() for path in self.extract: if not path.endswith("/") and path in nl: zf.extract(path, self.path) continue for zpath in nl: if zpath.endswith("/"): continue if not zpath.startswith(path): continue zf.extract(zpath, self.path) zf.close() unlink(out) return uver
def extract(upload, check_file): directory = mkdtemp() f = os.path.join(cfg["UPLOAD_FOLDER"], upload) z = ZipFile(f, 'r') z.extract(check_file, directory) z.close() return directory
def get_subtitle(url, path): in_data = urllib2.urlopen(url) temp_file = NamedTemporaryFile() temp_file.write(in_data.read()) in_data.close() temp_file.seek(0) if is_zipfile(temp_file.name): zip_file = ZipFile(temp_file) for name in zip_file.namelist(): # don't unzip stub __MACOSX folders if '.srt' in name and '__MACOSX' not in name: logger.info(' '.join(['Unpacking zipped subtitle', name, 'to', os.path.dirname(path)])) zip_file.extract(name, os.path.dirname(path)) zip_file.close() elif is_rarfile(temp_file.name): rar_path = path + '.rar' logger.info('Saving rared subtitle as %s' % rar_path) with open(rar_path, 'w') as out_file: out_file.write(temp_file.read()) try: import subprocess #extract all .srt in the rared file ret_code = subprocess.call(['unrar', 'e', '-n*srt', rar_path]) if ret_code == 0: logger.info('Unpacking rared subtitle to %s' % os.path.dirname(path)) os.remove(rar_path) except OSError: logger.info('Unpacking rared subtitle failed.' 'Please, install unrar to automate this step.') temp_file.close()
def main(): print "initializing" ap.env.overwriteOutput = True ap.env.workspace = WORKSPACE dirlist = os.listdir(ASCZIPDIR) print dirlist for z in dirlist: if z[-8:]==".asc.zip": zf = ZipFile("%s%s" % (ASCZIPDIR,z)) z0=zf.namelist()[0] print "extracting %s" % z0 zf.extract(z0,UNZIPDIR) dirlist = os.listdir(UNZIPDIR) print dirlist for z in dirlist: if z[-4:]==".asc": targetfile = "%s%s/%s" % (CWD, WORKSPACE, z.split('.')[0]) srcfile = "%s%s%s" % (CWD,UNZIPDIR,z) print "converting %s - %s" % (srcfile, targetfile) if not ap.Exists(targetfile): ap.ASCIIToRaster_conversion(srcfile, targetfile, "FLOAT") ap.DefineProjection_management(targetfile, ap.SpatialReference("WGS 1984")) else: print "%s exists" % targetfile print "complete"
def _create_resource(self, data_set, site, file_): """ Creates a new resource or file associated with its data set :param data_set: :param site: :param file_: """ #content of the zip file zip_file_name = file_.filename zip_path = os.path.join(config.DATA_SETS_DIR, zip_file_name) file_.save(zip_path) sourcezip = ZipFile(zip_path) i = 0 while i < len(sourcezip.namelist()): zip_entry_path = os.path.join(os.path.abspath(os.path.dirname(zip_path)), sourcezip.namelist()[i]) sourcezip.extract(sourcezip.namelist()[i], config.DATA_SETS_DIR) url = self.parser.get_file_name().replace(".zip","") + "_" + str(i) site.action.resource_create(package_id=data_set, upload=open(zip_entry_path), name=sourcezip.namelist()[i], url=url) i += 1 #xml content xml_file_name = self.parser.get_dataset().id + ".xml" path = os.path.join(config.DATA_SETS_DIR, xml_file_name) with open(path, "w") as ttmp: ttmp.write(self._content.encode(encoding="utf-8")) url = xml_file_name site.action.resource_create(package_id=data_set, upload=open(path), name=xml_file_name, url=url)
def extract_archive(name): with open(name, 'rb') as file_handle: zipfile = ZipFile(file_handle) for name in zipfile.namelist(): out_path = os.getcwd() zipfile.extract(name, out_path) zipfile.close()
def get_dmf(): """Downloads DMF, convert to CSV for SQL""" URL = """http://www.fda.gov/drugs/developmentapprovalprocess/ formssubmissionrequirements/drugmasterfilesdmfs/default.htm""" bs = BeautifulSoup(requests.get(URL).content) dmf = [link for link in bs.findAll('a') if 'DMF zip file' in link.renderContents()] ob_link = dmf.pop() ob_link = ob_link.get('href') download_link = urlparse.urljoin(URL, ob_link) def dlfile(url): f = requests.get(url) with open(os.path.basename(url), "wb") as local_file: local_file.write(f.content) return local_file.name dlfile(download_link) zipfiles = dlfile(download_link) zf = ZipFile(zipfiles) zip_contents = zf.namelist() match = [target for target in zip_contents if target.endswith('.xls')] filename = str(match[0]) zf.extract(match.pop()) master = (pd.read_excel(filename, sheetname='Sheet1', skiprows=1, index_col='DMF#', encoding='UTF-8')) nameoutput = 'dmf_to_load_' + strftime('%Y%m%d') master.to_csv(nameoutput + '.csv', encoding='UTF-8', sep='|')
def download_flights(start_date, end_date): pkl_path = 'data/processed/flights/flights{0}.{1}-{2}.{3}.pkl'.format(start_date.year, start_date.month, end_date.year, end_date.month) if Path(pkl_path).exists(): return pd.read_pickle(pkl_path) files, failed = download_range(r'http://tsdata.bts.gov/PREZIP/On_Time_On_Time_Performance_%Y_%-m.zip', 'data/raw/flights', start_date, end_date) dest_path = Path('data/unpacked/flights') dest_path.mkdir(parents=True, exist_ok=True) frames = [] for zip_file in files: zf = ZipFile(str(zip_file)) csv_name = zip_file.with_suffix('.csv').name csv_file = dest_path / csv_name if not csv_file.exists(): print("Extracting", csv_name) zf.extract(csv_name, str(dest_path)) print("...done") else: print(csv_name, "already extracted") df = flightdata.read_csv(str(csv_file)) frames.append(df) all_flights = merge_frames(frames, ignore_index=True) all_flights.to_pickle(pkl_path) return all_flights
def dump_files(zipfile: ZipFile): """Dump packed files to a location. """ dump_folder = CONF['packfile_dump', ''] if not dump_folder: return dump_folder = os.path.abspath(dump_folder) # Delete files in the folder, but don't delete the folder itself. try: files = os.listdir(dump_folder) except FileNotFoundError: return for name in files: name = os.path.join(dump_folder, name) if os.path.isdir(name): try: shutil.rmtree(name) except OSError: # It's possible to fail here, if the window is open elsewhere. # If so, just skip removal and fill the folder. pass else: os.remove(name) for zipinfo in zipfile.infolist(): zipfile.extract(zipinfo, dump_folder)
def open_zip(self, infiles): metadata_extracted = False metadata = None data_files = [] for infile in infiles: try: zip = ZipFile(infile, 'r') for efile in zip.namelist(): if not metadata_extracted and efile.startswith('Stationsmetadaten'): zip.extract(efile, path=self.workdir) metadata = os.path.join(self.workdir, efile) metadata_extracted = True logger.debug("metadata: %s", metadata) if efile.startswith('produkt'): zip.extract(efile, path=self.workdir) data_files.append( os.path.join(self.workdir, efile) ) logger.debug("data: %s", os.path.join(self.workdir, efile)) except BadZipfile as e: logger.error("could not open: {}".format(infile)) raise e return metadata, data_files
def SRTM3_retrieve(lon, lat, data_dir=None): if data_dir is None: dname = os.path.dirname # be more clever in the data directory so that users can define a setting. data_dir = os.path.join(dname(dname(__file__)), "data", "SRTM3") x = "%s%03d" % ("E" if lon > 0 else "W", abs(int(lon))) y = "%s%02d" % ("N" if lat > 0 else "S", abs(int(lat))) filename = "{y}{x}.hgt".format(x=x, y=y) filepath = os.path.join(data_dir, filename) if not os.path.exists(filepath): # download the zip file file_url = _SRTM3_FILE_LOOKUP.get(u"{y}{x}".format(x=x, y=y), None) # no file exists in the SRTM dataset for these coordinates if file_url is None: return None import urllib2 import cStringIO as StringIO from zipfile import ZipFile if not os.path.exists(data_dir): os.makedirs(data_dir) srtm_online = urllib2.urlopen(file_url) zfh = ZipFile(StringIO.StringIO(srtm_online.read()), "r") zfh.extract(filename, data_dir) return filepath
def get_symbols(exchange="polo"): if exchange == "polo": polo = ccxt.poloniex() polo.load_markets() return polo.symbols elif exchange == "stock": url = "https://www.quandl.com/api/v3/databases/WIKI/codes.json?api_key=%s" % QUANDL_APIKEY res = urlopen(url) zipfile = ZipFile(BytesIO(res.read())) zipfile.extract(zipfile.namelist()[0]) df = pd.read_csv(zipfile.namelist()[0], header=None) return df[0].values else: raise NotImplementedError()
def _read_one_data(self, url, params): results = [] # retrying _MAX_RETRY_COUNT for _ in range(1, _MAX_RETRY_COUNT): try: url = self.url.format(**params) tree = ElemTree.fromstring( requests.get(url).text.encode('utf-8')) break except requests.HTTPError: sleep(_SLEEP_TIME) else: raise Exception for el in tree.findall('.//' + self._namespace + 'entry'): updated = el.find(self._namespace + 'updated').text updated = parser.parse(updated, ignoretz=True) if self.start <= updated <= self.end: id = el.find(self._namespace + 'id').text title = el.find(self._namespace + 'title').text docid = el.find(self._namespace + 'docid').text url = el.find(self._namespace + 'link[@type="application/zip"]').attrib['href'] is_yuho = any( [word in title for word in ['有価証券報告書', '四半期報告書']]) is_quarterly = '四半期報告書' in title if self.fetch_xbrl: r = requests.get(url) if r.ok: z = ZipFile(BytesIO(r.content)) for info in z.infolist(): if is_yuho and '.xbrl' in info.filename and 'AuditDoc' not in info.filename: self.xbrl_filename = info.filename.split( '/')[-1] z.extract(info.filename) results.append({ 'id': id, 'title': title, 'docid': docid, 'url': url, 'updated': updated, 'is_yuho': is_yuho, 'is_quarterly': is_quarterly, }) sleep(_SLEEP_TIME) return results
def rename_and_prepare(base_file): """ensure the file(s) have a proper name @hack this should be done in a nicer way, but needs fixing now To fix longer term: if geonode computes a name, the uploader should respect it As it is/was, geonode will compute a name based on the zipfile but the importer will use names as it unpacks the zipfile. Renaming all the various pieces seems a burden on the client Additionally, if a SLD file is present, extract this. """ name, ext = os.path.splitext(os.path.basename(base_file)) dirname = os.path.dirname(base_file) if ext == ".zip": zf = ZipFile(base_file, 'r') rename = False main_file = None for f in zf.namelist(): name, ext = os.path.splitext(os.path.basename(f)) if _clean_string(name) != name: rename = True # @todo other files - need to unify extension handling somewhere if ext.lower() == '.shp': main_file = f elif ext.lower() == '.tif': main_file = f elif ext.lower() == '.csv': main_file = f # if an sld is there, extract so it can be found if ext.lower() == '.sld': zf.extract(f, dirname) if not main_file: raise Exception( 'Could not locate a shapefile or tif file') if rename: # dang, have to unpack and rename zf.extractall(dirname) zf.close() if rename: os.unlink(base_file) base_file = os.path.join(dirname, main_file) for f in os.listdir(dirname): safe = _clean_string(f) if safe != f: os.rename(os.path.join(dirname, f), os.path.join(dirname, safe)) return os.path.join( dirname, _clean_string(os.path.basename(base_file)) )
def make_runtime(capsule, output, licfile=None, platform=None): myzip = ZipFile(capsule, 'r') if 'pytransform.key' in myzip.namelist(): myzip.extract('pytransform.key', output) else: myzip.extract('pyshield.key', output) myzip.extract('pyshield.lic', output) myzip.extract('product.key', output) if licfile is None: myzip.extract('license.lic', output) else: shutil.copy2(licfile, os.path.join(output, 'license.lic')) if platform is None: libname = dll_name + dll_ext libfile = os.path.join(PYARMOR_PATH, libname) if not os.path.exists(libfile): sysname = pytransform.format_platname() libpath = os.path.join(PYARMOR_PATH, 'platforms') libfile = os.path.join(libpath, sysname, libname) shutil.copy2(libfile, output) else: path = os.path.join(PYARMOR_PATH, 'platforms', platform) for x in os.listdir(path): shutil.copy2(os.path.join(path, x), output) shutil.copy2(os.path.join(PYARMOR_PATH, 'pytransform.py'), output)
def unzip(self, infile, path): """ Given a zip file (infile) and path to unzip to (path), unzip the file and return the full path. """ path = path.strip("/") zip = ZipFile(infile) names = zip.namelist() # # When passed PKWare zips with Type 9 compression, we can read the zip header, but # # are unable to do the extraction. If for some reason we have a problem reading # # headers, we can use the system unzip to do so, as commented below: # status = not subprocess.call(["which", "unzip"]) # if status: # out = subprocess.run(["unzip", "-Z1", infile], stdout=subprocess.PIPE) # names = [ i.decode() for i in out.stdout.split()] extractions = [] for name in names: if name.startswith("npidata_pfile_") and \ name.endswith(".csv") and \ not "FileHeader" in name: extractions.append(name) if len(extractions) != 1: print("Did not find exactly one file in {infile}. Exiting.") sys.exit(1) csv_file = extractions[0] try: zip.extract(csv_file, path) except NotImplementedError as e: print( "Python does not support Type 9 compression, trying system unzip..." ) if not subprocess.run(["which", "unzip"]).returncode: out = subprocess.run(["unzip", infile, csv_file, "-d", path], stdout=subprocess.PIPE) if out.returncode: print("Can't unzip this file. Local unzip failed.") raise else: print( "Can't unzip this file. Type 9 compression not supported, and no local unzip." ) raise return f"{path}/{csv_file}"
def import_file(uploaded, parser): destination_dir, destination_file = create_tmp_destination(uploaded.name) with open(destination_file, 'w+') as f: f.write(uploaded.file.read()) zfile = ZipFile(f) for name in zfile.namelist(): try: zfile.extract(name, os.path.dirname(os.path.realpath(f.name))) except Exception: raise if name.endswith('shp'): import_datas.delay(parser.__name__, '/'.join( (destination_dir, name)), parser.__module__)
def extractFile(filepath): filename, file_extension = os.path.splitext(filepath) if file_extension == '.fz': os.rename(filepath, "output.fz") else: with open(filepath, 'rb') as file: z = ZipFile(file) for name in z.namelist(): if '.fzp' not in name and '.fz' in name: z.extract(name) os.rename(name, "output.fz") break
def classify_zip(): if request.method == 'POST': file = request.files['file'] if not file: return "No file sent." if not file.filename.split(".")[-1] == 'zip': return ".zip is the only compression format currently supported" filename = secure_filename(file.filename) zip_file_path = os.path.join(app.config['UPLOAD_FOLDER'], filename) file.save(zip_file_path) zip_file = ZipFile(zip_file_path) zip_file_list = list_zip_files(zip_file_path) all_images = filter_image_files(zip_file_list, ALLOWED_EXTENSIONS) if len(all_images) == 0: return "No image files detected in the zip file" #loop through images start = 0 increment = 500 all_images_len = len(all_images) while start < all_images_len: end = start + increment if end > len(all_images): end = len(all_images) #extract filenames curr_file_list = all_images[start:end] for filename in curr_file_list: zip_file.extract(filename, path=app.config['UPLOAD_FOLDER']) curr_file_list = [ os.path.join(app.config['UPLOAD_FOLDER'], x) for x in curr_file_list ] predictions = predict_multiple(GRAPH, curr_file_list, LABELS) #remove files for curr_file in curr_file_list: os.remove(curr_file) return make_response(json.dumps(predictions, cls=NumpyEncoder)) start = end + 1
def fetch_data(self, i): url = self._data["images"][i]["rawUrl"] r = self._client.get(url) zip = ZipFile(BytesIO(r.content)) images = [] with tempfile.TemporaryDirectory() as tmpdir: for name in zip.namelist(): if name.endswith(".tif"): zip.extract(name, tmpdir) images.append(Image.open(tmpdir + "/" + name)) zip.close() arrays = [np.array(image) for image in images] combined_image = np.dstack(arrays) return combined_image
def __call__(self, data_bytes): zf = ZipFile(io.BytesIO(data_bytes)) uncompress_size = sum((file.file_size for file in zf.infolist())) with tqdm(desc=self.label, total=uncompress_size, unit='B', unit_scale=True, unit_divisor=1024) as bar: for f in zf.infolist(): filename = f.filename zf.extract(f, self.workspace_out_dir) bar.set_postfix(file=filename[-10:], refresh=False) bar.update(f.file_size)
def unzip_fp(f): with tmp_dir() as tmp: zf = ZipFile(f, mode='r') members = zf.namelist() members = [ m for m in members if (('__MACOS' not in m) and ( not m.startswith('/')) and (not '..' in m)) ] for member in members: zf.extract(member, path=tmp) yield tmp
def make_runtime(capsule, output, licfile=None, platforms=None, package=False): if package: output = os.path.join(output, 'pytransform') if not os.path.exists(output): os.makedirs(output) logging.info('Generating runtime files to %s', output) myzip = ZipFile(capsule, 'r') if 'pytransform.key' in myzip.namelist(): logging.info('Extract pytransform.key') myzip.extract('pytransform.key', output) else: logging.info('Extract pyshield.key, pyshield.lic, product.key') myzip.extract('pyshield.key', output) myzip.extract('pyshield.lic', output) myzip.extract('product.key', output) if licfile is None: logging.info('Extract license.lic') myzip.extract('license.lic', output) else: logging.info('Copying %s as license file', licfile) shutil.copy2(licfile, os.path.join(output, 'license.lic')) if not platforms: libfile = pytransform._pytransform._name if not os.path.exists(libfile): libname = dll_name + dll_ext libfile = os.path.join(PYARMOR_PATH, libname) if not os.path.exists(libfile): pname = pytransform.format_platform() libpath = os.path.join(PYARMOR_PATH, 'platforms') libfile = os.path.join(libpath, pname, libname) logging.info('Copying %s', libfile) shutil.copy2(libfile, output) elif len(platforms) == 1: filename = _build_platforms(platforms)[0] logging.info('Copying %s', filename) shutil.copy2(filename, output) else: libpath = os.path.join(output, pytransform.plat_path) logging.info('Create library path to support multiple platforms: %s', libpath) if not os.path.exists(libpath): os.mkdir(libpath) filenames = _build_platforms(platforms) for platid, filename in list(zip(platforms, filenames)): logging.info('Copying %s', filename) path = os.path.join(libpath, *platid.split('.')) logging.info('To %s', path) if not os.path.exists(path): os.makedirs(path) shutil.copy2(filename, path) filename = os.path.join(PYARMOR_PATH, 'pytransform.py') shutil.copy2(filename, os.path.join(output, '__init__.py') if package else output) logging.info('Generate runtime files OK')
def install_forge(name, version, forge_version): '''安装forge''' version_path = os.path.join(g.config['cur_gamepath'], 'versions') game_path = os.path.join(version_path, name) config = json.load(open(os.path.join(game_path, f'{name}.json'))) installerpath = game_path + f'\\installer.jar' # 获取forge的配置 zip = ZipFile(installerpath) zip.extract('version.json', game_path) zip.extract('install_profile.json', game_path) forge_config = json.load(open(os.path.join(game_path, f'version.json'))) install_profile = json.load( open(os.path.join(game_path, f'install_profile.json'))) thread_count = threading.active_count() for i in install_profile['libraries']: Game.analysis_library(zip, i) for i in forge_config['libraries']: Game.analysis_library(zip, i) while threading.active_count() > thread_count: g.logapi.debug(f'剩余线程{threading.active_count()}({thread_count})') time.sleep(1) g.logapi.debug(f'剩余线程{threading.active_count()}({thread_count})') zip.close() # forge-client lib_path = os.path.join(g.config['cur_gamepath'], f'libraries') userdev_path = game_path + f'\\userdev.jar' client_path = os.path.join( lib_path, f'net/minecraftforge/forge/{version}-{forge_version}/forge-{version}-{forge_version}-client.jar' ) # 获取需要的文件 file_subset_list = [] with ZipFile(userdev_path) as userdev: for i in userdev.namelist(): if 'patches' in i and '.' in i: file_subset_list.append(i) Game.stream_conents(userdev_path, client_path, file_subset_list, lambda path: '/'.join(path.split('/')[1:])) # 拼接 Game.splicing(config, forge_config) json.dump(config, open(os.path.join(game_path, f'{name}.json'), mode='w'))
def download_and_unzip(buffer, a, b, path): unzipped = [] from zipfile import ZipFile, BadZipfile try: zip_file = ZipFile(buffer) except BadZipFile: # Often happens with GEE API print("bad_zip") return None files = zip_file.namelist() for i in range(a, b): zip_file.extract(files[i], path + "/tiff/") #print("{} downloaded and unzippped".format(files[i])) unzipped.append(files[i]) return unzipped
def classify_zip(): """Classify all images from a zip file""" if request.method != "POST": return file = request.files["file"] if not file: return "No file sent." if file.filename.split(".")[-1] != "zip": return ".zip is the only compression format currently supported" filename = secure_filename(file.filename) zip_file_path = os.path.join(app.config["UPLOAD_FOLDER"], filename) file.save(zip_file_path) zip_file = ZipFile(zip_file_path) zip_file_list = list_zip_files(zip_file_path) all_images = filter_image_files(zip_file_list, ALLOWED_EXTENSIONS) if len(all_images) == 0: return "No image files detected in the zip file" # loop through images start = 0 increment = 500 all_images_len = len(all_images) while start < all_images_len: end = start + increment if end > len(all_images): end = len(all_images) # extract filenames curr_file_list = all_images[start:end] for filename in curr_file_list: zip_file.extract(filename, path=app.config["UPLOAD_FOLDER"]) curr_file_list = [ os.path.join(app.config["UPLOAD_FOLDER"], x) for x in curr_file_list ] predictions = predict_multiple(curr_file_list) # remove files for curr_file in curr_file_list: os.remove(curr_file) return make_response(jsonify(predictions))
def _parse_connectome_surfaces(self, connectome_surface, warning_message, should_center): """ Parse data from a CSurface object and save it in our internal Surface DataTypes """ surfaces, processed_files = [], [] parser = GIFTIParser(self.storage_path, self.operation_id) for c_surface in connectome_surface: if c_surface.src in processed_files: continue try: # create a meaningful but unique temporary path to extract tmpdir = os.path.join(gettempdir(), c_surface.parent_cfile.get_unique_cff_name()) self.log.debug("Extracting %s[%s] into %s ..." % (c_surface.src, c_surface.name, tmpdir)) _zipfile = ZipFile(c_surface.parent_cfile.src, 'r', ZIP_DEFLATED) gifti_file_1 = _zipfile.extract(c_surface.src, tmpdir) gifti_file_2 = None surface_name, pair_surface = self._find_pair_file(c_surface, connectome_surface) if pair_surface: self.log.debug("Extracting pair %s[%s] into %s ..." % (pair_surface.src, pair_surface.name, tmpdir)) gifti_file_2 = _zipfile.extract(pair_surface.src, tmpdir) surface_type = self._guess_surface_type(c_surface.src.lower()) self.logger.info("We will import surface %s as type %s" % (c_surface.src, surface_type)) surface = parser.parse(gifti_file_1, gifti_file_2, surface_type, should_center) surface.user_tag_1 = surface_name validation_result = surface.validate() if validation_result.warnings: warning_message += validation_result.summary() + "\n" surfaces.append(surface) if pair_surface: processed_files.append(pair_surface.src) processed_files.append(c_surface.src) if os.path.exists(tmpdir): shutil.rmtree(tmpdir) except ParseException: self.logger.exception("Could not import a Surface entity.") warning_message += "Problem when importing Surfaces!! \n" except OSError: self.log.exception("Could not clean up temporary file(s).") return surfaces
def shortcut_download(dataset, compression_type='tar.gz'): """Download and unpack pre-processed dataset""" if compression_type not in ['tar.gz', 'zip']: print('Warning! Wrong compression format. Changing to tar.gz') compression_type = 'tar.gz' if dataset == 'reddit_casual' and compression_type == 'zip': print( 'Warning! Zip format is not supported for reddit casual dataset due to file size. Changing to tar.gz' ) compression_type = 'tar.gz' if not os.path.exists(datasets_dir): os.makedirs(datasets_dir) compressed_url = f'https://affect.media.mit.edu/neural_chat/datasets/{dataset}_preprocessed.{compression_type}' compressed_file_dir = datasets_dir.joinpath(dataset) compressed_file_path = datasets_dir.joinpath( f'{dataset}_preprocessed.{compression_type}') # Prepare Dialog data if not os.path.exists(compressed_file_dir): print(f'Downloading {compressed_url} to {compressed_file_path}') urlretrieve(compressed_url, compressed_file_path) print(f'Successfully downloaded {compressed_file_path}') if compression_type == 'tar.gz': tar_ref = tarfile.open(compressed_file_path, 'r:gz') for member in tar_ref.getmembers(): try: tar_ref.extract(member, path=datasets_dir) print(f'Extracting {member.name}: OK') except Exception as e: print(f'Extracting {member.name}: ERROR - {e}') tar_ref.close() elif compression_type == 'zip': zip_ref = ZipFile(compressed_file_path, mode='r') for member in zip_ref.infolist(): try: zip_ref.extract(member, path=datasets_dir) print(f'Extracting {member}: OK') except Exception as e: print(f'Extracting {member}: ERROR - {e}') zip_ref.close() print(f'Successfully extracted {compressed_file_path}') else: print('Directory already exists. Aborting download.')
def grab_ensdf_decay(build_dir=""): """ Grabs the ENSDF decay data files if not already present. Parameters ---------- build_dir : str Major directory to place html files in. 'ENSDF/' will be appended. """ # Add ENSDF to build_dir build_dir = os.path.join(build_dir, 'ENSDF') try: os.makedirs(build_dir) except OSError: pass # Grab ENSDF files and unzip them. iaea_base_url = 'http://www.nndc.bnl.gov/ensarchivals/distributions/dist19/' cf_base_url = 'https://github.com/pyne/data/raw/master/' ensdf_zip = ['ensdf_191004_099.zip', 'ensdf_191004_199.zip', 'ensdf_191004_300.zip', ] for f in ensdf_zip: fpath = os.path.join(build_dir, f) if f not in os.listdir(build_dir): print(" grabbing {0} and placing it in {1}".format(f, fpath)) conn = urllib.urlopen(iaea_base_url + f) with open(fpath, 'wb') as f: f.write(conn.read()) if os.path.getsize(fpath) < 1048576: print(" could not get {0} from NNDC; trying mirror".format(f)) os.remove(fpath) conn = urllib.urlopen(cf_base_url + f) with open(fpath, 'wb') as f: f.write(conn.read()) # not using ZipFile context manager (with statement for Python 2.6) try: zf = ZipFile(fpath) for name in zf.namelist(): if not os.path.exists(os.path.join(build_dir, name)): print(" extracting {0} from {1}".format(name, fpath)) zf.extract(name, build_dir) finally: zf.close()
def oem_logo_check(): filecheck = FileCheck() LOGOSCALE = { "7寸": ["102ML", "700ML", "750ML", "2070", "2070D", "2070S"], "4.3寸": ["2043T", "2043E", "2043E-N"], "3.5寸": ["2035T"] } logo_md5s = [] filename = "logo.NTB" path_oem_logos = r"E:\Redmine2019\LEVIOEM测试\logo_OEM" path_levi_oem = r"\\192.168.11.20\hmi软件镜像\nuc972(V1.0 ~ V1.4)\组态_LEVI\OEM" filelist = filecheck.lookfolder(path_levi_oem) # 获取文件绝对路径 print(filelist) path_temp = r"E:\Redmine2019\LEVIOEM测试\temp" oem_logo_dic = dict() for fullpath in filelist: # 穷举OEM目录下所有文件 path, name = os.path.split(fullpath) # 分离“productfile.osf” if name == "productfile.osf": # 取productfile.osf文件 osf = ZipFile(fullpath) fileb = osf.open(filename, 'rU') md5 = filecheck.calc_md5(fileb) p, oeminfostr = os.path.split( path) # 分离“NUC972_2070S_OEM泉州科源_6.4.18_2019-01-23” oeminfo = oeminfostr.split("_") if len(oeminfo) >= 4: # 具有标准命名格式的才进行处理 hmitype = oeminfo[1] # 获取2070S oemcomp = ''.join(oeminfo[2:-2]) # 获取OEM泉州科源 if oemcomp not in oem_logo_dic.keys(): # 相同OEM厂家的归类 oem_logo_dic[oemcomp] = {"7寸": [], "4.3寸": [], "3.5寸": []} for scale in LOGOSCALE.keys(): # 按不同尺寸区分 if hmitype in LOGOSCALE[scale]: # 型号归类 if not oem_logo_dic[oemcomp][scale]: # 若未存MD5值 oem_logo_dic[oemcomp][scale].append(md5) depath = os.path.join( path_oem_logos, oemcomp, scale) # 创建目录logos\OEM泉州科源\7寸 if not os.path.exists(depath): os.makedirs(depath) osf.extract(filename, depath) else: # 已存MD5,但值不一致,说明文件不同。 if md5 not in oem_logo_dic[oemcomp][scale]: print("OEM LOGO 文件冲突:{}".format( oeminfostr)) # 同厂家同尺寸屏但LOGO文件不一致 d = json.dumps(oem_logo_dic) print(d) with open( os.path.join(path_oem_logos, "{} md5 dict.json".format(filename)), "w") as f: f.write(d)
def agregar_theme(request): if request.method == "POST": form = ThemeForm(request.POST, request.FILES) if form.is_valid(): ZipFile(request.FILES.get('comprimido'), 'r') nombre_carpeta = "" error_template = False error_static = False error_imagen = False error_json = False mensaje = "" unziped = ZipFile(request.FILES.get('comprimido'), 'r') print unziped.namelist() for file_path in unziped.namelist(): split_directorio = file_path.split("/") if split_directorio[0] != 'template' and split_directorio[ 0] != 'static': if not nombre_carpeta: nombre_carpeta = split_directorio[0] if split_directorio[1] == 'template' or split_directorio[ 0] == 'template': archivo = unziped.extract( file_path, "templates/" + request.user.username + "/" + request.POST.get('theme_titulo')) if not error_template: error_template = True if split_directorio[1] == 'static' or split_directorio[ 0] == 'static': # print "entro" archivo = unziped.extract( file_path, "static/" + request.user.username + "/" + request.POST.get('theme_titulo')) if not error_static: error_static = True if error_static and error_template: theme = form.save() theme.carpeta_titulo = nombre_carpeta theme.save() mensaje = "Se Creo el theme Satisfactoriamente" return redirect(reverse_lazy("configuracion:home")) else: mensaje = "Hubo error posiblemente por que no tiene la carpeta template y static" # file_content = unziped.printdir() # print "----" # print file_path # print unziped.extract(file_path, "carpeta nueva/template") else: form = ThemeForm() return render(request, 'core/agregar_theme.html', locals())
def cache(self): if not exists(self.data_home): makedirs(self.data_home) if not exists(self.data_dir): print('downloading met-enk from %s to %s' % (DATA_URL, self.data_dir)) fhandle = urlopen(DATA_URL) buf = BytesIO(fhandle.read()) zip_file = ZipFile(buf) makedirs(self.data_dir) for name in zip_file.namelist(): zip_file.extract(name, path=self.data_dir) self.cached = True
def unzip(self): try: from zipfile import ZipFile except: raise ImportError("没有zipFile库") file = ZipFile(r"{}".format(self.file)) if not os.path.exists(self.path): os.mkdir(self.path) for file_name in file.namelist(): filename = file_name.encode("cp437").decode("gbk") file.extract(file_name, path=self.path) os.chdir(self.path) os.rename(file_name, filename)
def _extract_certificate(file_name): """Extract the cert of an APK and return the extracted file name.""" apk_zip = ZipFile(file_name) cert_file = None cert_file_name_regexp = re.compile(r'^META-INF\/.*\.[RD]SA$') for zipped_file in apk_zip.filelist: if cert_file_name_regexp.match(zipped_file.filename): cert_file = zipped_file break else: raise CryptoVerificationError( file_name, message='No certificate found in APK. Is it signed?') apk_zip.extract(cert_file, SETTINGS['temp_dir']) return os.path.join(SETTINGS['temp_dir'], cert_file.filename)
def download_and_extract(self, lang): print('Downloading corpus') if path.exists(f'data/{lang}.txt'): return url = f'{self.base_url}/{lang}-eng.zip' request = Request(url, None, {'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.7) ' 'Gecko/2009021910 Firefox/3.0.7'}) archive = urlopen(request) fd = open(f'/tmp/{lang}.zip', 'wb') fd.write(archive.read()) fd.close() zf = ZipFile(f'/tmp/{lang}.zip') zf.extract(member=f'{lang.split("-")[0]}.txt', path=f'data/') return self
def unzip(): print("Unzipping data...") zipFile = ZipFile(zipFolder, "r") uncompressedSize = sum(file.file_size for file in zipFile.infolist()) extractedSize = 0 pbar = ProgressBar(widgets=[Percentage(), Bar()], maxval=100).start() start = datetime.datetime.now() for file in zipFile.infolist(): extractedSize += file.file_size percent = extractedSize * 100 / uncompressedSize pbar.update(percent) zipFile.extract(file, path=zipFolder.rsplit(".", 1)[0]) print("Unzipped in {} s.".format((datetime.datetime.now() - start).seconds)) zipFile.close()
def extract(_p_dir, file='RomDataBu/df_kl.h5.zip'): """ 解压数据 """ data_example_zip = os.path.join(_p_dir, file) try: from zipfile import ZipFile zip_h5 = ZipFile(data_example_zip, 'r') unzip_dir = os.path.join(_p_dir, 'RomDataBu/') for h5 in zip_h5.namelist(): zip_h5.extract(h5, unzip_dir) zip_h5.close() except Exception as e: print('example env failed! e={}'.format(e))
def __extract_cbz(self, file): """ extract first image in cbz file, cache it in local cache folder""" try: archive_zip = ZipFile(file, 'r') extract_path = self.cache + path.basename(file) if self.mode is 'all': archive_zip.extractall(path=extract_path) else: first_file = archive_zip.namelist()[0] archive_zip.extract(member=first_file, path=extract_path) except BadZipFile as e: raise e finally: archive_zip.close() return extract_path
def extract_overrides(self, pack_archive: zipfile.ZipFile): p = os.path # TODO : Slight chance of 'override' folder being a flexibly named dir # : that is hard referenced within the addon info json from the api for i in pack_archive.namelist(): if i.startswith('overrides'): pack_archive.extract(i, self.out_dir) o_folder = p.join(self.out_dir, 'overrides') # o_temp = p.join(self.out_dir, '_overrides_temp') # TODO : Might be bad idea if overrides folder contains a folder named overrides (:3) copy_tree(o_folder, self.out_dir) remove_tree(o_folder)
def unzip(zip_filename, dest_folder, loading_bar=tqdm): """ Unzipping archive """ zip_file = ZipFile(zip_filename) all_files = zip_file.infolist() uncompressed_size = sum((file.file_size for file in zip_file.infolist())) files = zip_file # Uncompressing with loading_bar(total=uncompressed_size) as pbar: for file in all_files: zip_file.extract(file, dest_folder) pbar.update(file.file_size)
def install(self, data, fname): if fname.endswith(".py"): # .py files go directly into the addon folder path = os.path.join(self.addonsFolder(), fname) open(path, "w").write(data) return # .zip file z = ZipFile(StringIO(data)) base = self.addonsFolder() for n in z.namelist(): if n.endswith("/"): # folder; ignore continue # write z.extract(n, base)