def _download_biogrid_data(url): """Downloads zipped, tab-separated Biogrid data in .tab2 format. Parameters: ----------- url : str URL of the BioGrid zip file. Returns ------- csv.reader A csv.reader object for iterating over the rows (header has already been skipped). """ res = requests.get(biogrid_file_url) if res.status_code != 200: raise Exception('Unable to download Biogrid data: status code %s' % res.status_code) zip_bytes = BytesIO(res.content) zip_file = ZipFile(zip_bytes) zip_info_list = zip_file.infolist() # There should be only one file in this zip archive if len(zip_info_list) != 1: raise Exception('There should be exactly zipfile in BioGrid zip ' 'archive: %s' % str(zip_info_list)) unzipped_bytes = zip_file.read(zip_info_list[0]) # Unzip the file biogrid_str = StringIO(unzipped_bytes.decode('utf8')) # Make file-like obj csv_reader = csv.reader(biogrid_str, delimiter='\t') # Get csv reader next(csv_reader) # Skip the header return csv_reader
def MyUnzipFile(in_file_path, target_root_path): """Unzip file. >>> unzip_file("d:\\zout.zip", "d:\\")""" zin = ZipFile(in_file_path, "r") zin.extractall(target_root_path) zin.close()
def get(self): lines = [] jps = models.JourneyPattern.query().order(models.JourneyPattern.line) for jp in jps: if jp.line not in lines: logging.info(jp.line) lines.append(jp.line) logging.info([s.get().name for s in jp.stops]) return # return taskqueue.add(url='/timetable', queue_name='default', params={'file_name': 'tfl_1-BAK_-390106-y05.xml'}) if self.is_local(): taskqueue.add(url='/timetable', queue_name='default') else: bucket_name = os.environ.get('BUCKET_NAME', app_identity.get_default_gcs_bucket_name()) # bucket_name = 'jeg376-tm470.appspot.com' filename = '/%s/stream.zip' % bucket_name gcs_file = cloudstorage.open(filename) tfl_zip = ZipFile(gcs_file, 'r') tfl_data = tfl_zip.read('LULDLRRiverTramCable.zip') tfl_data = ZipFile(StringIO.StringIO(tfl_data), 'r') tube_file_matcher = re.compile('^tfl_\d-\w{3}_.*\.xml$') for file_name in tfl_data.namelist(): if tube_file_matcher.match(file_name): taskqueue.add(url='/timetable', queue_name='default', params={'file_name': file_name})
def get_info(in_stream): """ Return the version and submitter strings from zipfile byte stream. """ arch = ZipFile(in_stream, 'r') try: return unpack_info(arch.read('__INFO__')) finally: arch.close()
class UnZip(): """Unzip the given file into a temporary directory.""" def __init__(self, input_file): self.input_file = input_file def unzip(self): self.zip_file = ZipFile(self.input_file, 'r') self.tempdir = tempfile.mkdtemp() self.zip_file.extractall(self.tempdir) return self.tempdir def cleanup(self): self.zip_file.close() for root, dirs, files in os.walk(self.tempdir, topdown=False): for name in files: os.remove(os.path.join(root, name)) for name in dirs: os.rmdir(os.path.join(root, name)) os.rmdir(self.tempdir) def __enter__(self): return self.unzip() def __exit__(self, *args): self.cleanup() return False
def temp_shapefile_from_zip(zip_path): """ Given a path to a ZIP file, unpack it into a temp dir and return the path to the shapefile that was in there. Doesn't clean up after itself unless there was an error. If you want to cleanup later, you can derive the temp dir from this path. """ zf = ZipFile(zip_path) tempdir = mkdtemp() shape_path = None # Copy the zipped files to a temporary directory, preserving names. for name in zf.namelist(): data = zf.read(name) outfile = os.path.join(tempdir, name) if name.endswith('.shp'): shape_path = outfile f = open(outfile, 'w') f.write(data) f.close() if shape_path is None: log.warn("No shapefile, cleaning up") # Clean up after ourselves. for file in os.listdir(tempdir): os.unlink(os.path.join(tempdir, file)) os.rmdir(tempdir) raise ValueError("No shapefile found in zip") return shape_path
def extract(zip): """Extrait le fichier '/word/document.xml' d'une archive word dans le répertoire courant """ myzip = ZipFile(zip, 'a') myzip.extract('word/document.xml', './../')
def _render_zip(self, req, filename, repos, diff): """ZIP archive with all the added and/or modified files.""" new_rev = diff.new_rev req.send_response(200) req.send_header('Content-Type', 'application/zip') req.send_header('Content-Disposition', 'attachment;' 'filename=%s.zip' % filename) from zipfile import ZipFile, ZipInfo, ZIP_DEFLATED buf = StringIO() zipfile = ZipFile(buf, 'w', ZIP_DEFLATED) for old_node, new_node, kind, change in repos.get_changes(**diff): if kind == Node.FILE and change != Changeset.DELETE: assert new_node zipinfo = ZipInfo() zipinfo.filename = new_node.path.encode('utf-8') # Note: unicode filenames are not supported by zipfile. # UTF-8 is not supported by all Zip tools either, # but as some does, I think UTF-8 is the best option here. zipinfo.date_time = time.gmtime(new_node.last_modified)[:6] zipinfo.compress_type = ZIP_DEFLATED zipfile.writestr(zipinfo, new_node.get_content().read()) zipfile.close() buf.seek(0, 2) # be sure to be at the end req.send_header("Content-Length", buf.tell()) req.end_headers() req.write(buf.getvalue())
def import_view(request): """ Gets the existing declared parsers for the current project. This view handles only the file based import parsers. """ choices = [] choices_url = [] render_dict = {} choices, choices_url, classes = discover_available_parsers() form = ImportDatasetFormWithFile(choices, prefix="with-file") form_without_file = ImportDatasetForm( choices_url, prefix="without-file") if request.method == 'POST': if 'upload-file' in request.POST: form = ImportDatasetFormWithFile( choices, request.POST, request.FILES, prefix="with-file") if form.is_valid(): print(request.FILES) uploaded = request.FILES['with-file-zipfile'] destination_dir, destination_file = create_tmp_destination( uploaded.name) with open(destination_file, 'w+') as f: f.write(uploaded.file.read()) zfile = ZipFile(f) for name in zfile.namelist(): try: zfile.extract( name, os.path.dirname(os.path.realpath(f.name))) if name.endswith('shp'): parser = classes[int(form['parser'].value())] import_datas.delay( '/'.join((destination_dir, name)), parser.__name__, parser.__module__ ) continue except Exception: raise if 'import-web' in request.POST: form_without_file = ImportDatasetForm( choices_url, request.POST, prefix="without-file") if form_without_file.is_valid(): parser = classes[int(form_without_file['parser'].value())] import_datas_from_web.delay( parser.__name__, parser.__module__ ) # Hide second form if parser has no web based imports. render_dict['form'] = form if choices_url: render_dict['form_without_file'] = form_without_file return render(request, 'common/import_dataset.html', render_dict)
def acquire_resource(self, target_path, format_dict): """ Downloads the zip file and extracts the files listed in :meth:`zip_file_contents` to the target path. """ import cStringIO as StringIO from zipfile import ZipFile target_dir = os.path.dirname(target_path) if not os.path.isdir(target_dir): os.makedirs(target_dir) url = self.url(format_dict) shapefile_online = self._urlopen(url) zfh = ZipFile(StringIO.StringIO(shapefile_online.read()), 'r') for member_path in self.zip_file_contents(format_dict): ext = os.path.splitext(member_path)[1] target = os.path.splitext(target_path)[0] + ext member = zfh.getinfo(member_path) with open(target, 'wb') as fh: fh.write(zfh.open(member).read()) shapefile_online.close() zfh.close() return target_path
def aqcuire_all_resources(self, format_dict): import cStringIO as StringIO from zipfile import ZipFile # Download archive. url = self.url(format_dict) shapefile_online = self._urlopen(url) zfh = ZipFile(StringIO.StringIO(shapefile_online.read()), 'r') shapefile_online.close() # Iterate through all scales and levels and extract relevant files. modified_format_dict = dict(format_dict) scales = ('c', 'l', 'i', 'h', 'f') levels = (1, 2, 3, 4) for scale, level in itertools.product(scales, levels): modified_format_dict.update({'scale': scale, 'level': level}) target_path = self.target_path(modified_format_dict) target_dir = os.path.dirname(target_path) if not os.path.isdir(target_dir): os.makedirs(target_dir) for member_path in self.zip_file_contents(modified_format_dict): ext = os.path.splitext(member_path)[1] target = os.path.splitext(target_path)[0] + ext member = zfh.getinfo(member_path) with open(target, 'wb') as fh: fh.write(zfh.open(member).read()) zfh.close()
def get_data_famafrench(name, start=None, end=None): start, end = _sanitize_dates(start, end) # path of zip files zipFileURL = "http://mba.tuck.dartmouth.edu/pages/faculty/ken.french/ftp/" url = urllib.urlopen(zipFileURL + name + ".zip") zipfile = ZipFile(StringIO(url.read())) data = zipfile.open(name + ".txt").readlines() file_edges = np.where(np.array([len(d) for d in data]) == 2)[0] datasets = {} for i in range(len(file_edges) - 1): dataset = [d.split() for d in data[(file_edges[i] + 1): file_edges[i + 1]]] if(len(dataset) > 10): ncol = np.median(np.array([len(d) for d in dataset])) header_index = np.where( np.array([len(d) for d in dataset]) == (ncol - 1))[0][-1] header = dataset[header_index] # to ensure the header is unique header = [str(j + 1) + " " + header[j] for j in range(len(header))] index = np.array( [d[0] for d in dataset[(header_index + 1):]], dtype=int) dataset = np.array( [d[1:] for d in dataset[(header_index + 1):]], dtype=float) datasets[i] = DataFrame(dataset, index, columns=header) return datasets
def save_pickle_in_cfile(self, local_fname, networkref): """ Creates a pickled version of the graph and stores it in the cfile Parameters ---------- local_fname: string The filename used in the Pickle folder to store networkref: NetworkX Graph instance The NetworkX graph to pickle """ logger.info('Write a generated graph pickle to the connectome file.') picklefilepath = os.path.join(tempfile.gettempdir(),local_fname) from networkx import write_gpickle # add nodekeys, edgekeys, graphid to helpernode 'n0' before storage helperdict = {'nodekeys': networkref.nodekeys.copy(), \ 'edgekeys': networkref.edgekeys.copy(), \ 'graphid' : networkref.networkid } networkref.graph.add_node('n0') networkref.graph.node['n0'] = helperdict write_gpickle(networkref.graph, picklefilepath) networkref.graph.remove_node('n0') from zipfile import ZipFile, ZIP_DEFLATED tmpzipfile = ZipFile(self.data.fullpathtofile, 'a', ZIP_DEFLATED) # store it in the zip file tmpzipfile.write(picklefilepath, 'Pickle/' + local_fname) tmpzipfile.close() # remove pickle file from system logger.debug('Unlink: %s' % picklefilepath) os.unlink(picklefilepath)
def do_export(self, REQUEST=None): """ """ if REQUEST and not self.getParentNode().checkPermissionView(): raise Unauthorized errors = [] my_container = self.getParentNode() objects_to_archive = self.gather_objects(my_container) file_like_object = StringIO() zip_file = ZipFile(file_like_object, 'w') archive_files = [] try: for obj in objects_to_archive: added_path = None if self.is_exportable(obj): added_path = self.add_object_to_zip(obj, zip_file) if added_path: archive_files.append((obj.title_or_id(), getattr(obj, 'meta_label', obj.meta_type), added_path)) self.add_index(zip_file, archive_files) zip_file.close() except Exception, e: errors.append(e)
def getTranslations(type, localesDir, defaultLocale, projectName, key): result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/export?key=%s' % (projectName, key)).read() if result.find('<success') < 0: raise Exception('Server indicated that the operation was not successful\n' + result) result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/download/all.zip?key=%s' % (projectName, key)).read() zip = ZipFile(StringIO(result)) dirs = {} for info in zip.infolist(): if not info.filename.endswith('.json'): continue dir, file = os.path.split(info.filename) if not re.match(r'^[\w\-]+$', dir) or dir == defaultLocale: continue if type == 'chrome' and file.count('.') == 1: origFile = file else: origFile = re.sub(r'\.json$', '', file) if type == 'gecko' and not origFile.endswith('.dtd') and not origFile.endswith('.properties'): continue mapping = langMappingChrome if type == 'chrome' else langMappingGecko for key, value in mapping.iteritems(): if value == dir: dir = key if type == 'chrome': dir = dir.replace('-', '_') data = zip.open(info.filename).read() if data == '[]': continue if not dir in dirs: dirs[dir] = set() dirs[dir].add(origFile) path = os.path.join(localesDir, dir, origFile) if not os.path.exists(os.path.dirname(path)): os.makedirs(os.path.dirname(path)) if type == 'chrome' and origFile.endswith('.json'): postprocessChromeLocale(path, data) elif type == 'chrome': data = json.loads(data) if origFile in data: fileHandle = codecs.open(path, 'wb', encoding='utf-8') fileHandle.write(data[origFile]['message']) fileHandle.close() else: fromJSON(path, data) # Remove any extra files for dir, files in dirs.iteritems(): baseDir = os.path.join(localesDir, dir) if not os.path.exists(baseDir): continue for file in os.listdir(baseDir): path = os.path.join(baseDir, file) if os.path.isfile(path) and (file.endswith('.json') or file.endswith('.properties') or file.endswith('.dtd')) and not file in files: os.remove(path)
def layer_type(filename): """Finds out if a filename is a Feature or a Vector returns a gsconfig resource_type string that can be either 'featureType' or 'coverage' """ base_name, extension = os.path.splitext(filename) shp_exts = ['.shp',] cov_exts = ['.tif', '.tiff', '.geotiff', '.geotif'] csv_exts = ['.csv'] kml_exts = ['.kml'] if extension.lower() == '.zip': zf = ZipFile(filename) # ZipFile doesn't support with statement in 2.6, so don't do it try: for n in zf.namelist(): b, e = os.path.splitext(n.lower()) if e in shp_exts or e in cov_exts or e in csv_exts: base_name, extension = b,e finally: zf.close() if extension.lower() in shp_exts + csv_exts + kml_exts: return FeatureType.resource_type elif extension.lower() in cov_exts: return Coverage.resource_type else: msg = ('Saving of extension [%s] is not implemented' % extension) raise GeoNodeException(msg)
def pack(self, output_dir, devel=False, force=False, keep_temp=False): self.output_dir = os.path.realpath(output_dir) if not os.path.exists(self.output_dir): os.makedirs(self.output_dir) self.force = force self.keep_temp = keep_temp paths = [] self.merge_modules = {} # create runtime package p = self._create_msi_installer(PackageType.RUNTIME) paths.append(p) # create devel package if devel and not isinstance(self.package, App): p = self._create_msi_installer(PackageType.DEVEL) paths.append(p) # create zip with merge modules self.package.set_mode(PackageType.RUNTIME) zipf = ZipFile(os.path.join(self.output_dir, '%s-merge-modules.zip' % self._package_name()), 'w') for p in self.merge_modules[PackageType.RUNTIME]: zipf.write(p) zipf.close() if not keep_temp: for msms in self.merge_modules.values(): for p in msms: os.remove(p) return paths
def get_results(self): """Get analysis results. @return: data. """ root = self._get_root(container="cuckoo", create=False) if not os.path.exists(root): return False zip_data = StringIO() zip_file = ZipFile(zip_data, "w", ZIP_DEFLATED) root_len = len(os.path.abspath(root)) for root, dirs, files in os.walk(root): archive_root = os.path.abspath(root)[root_len:] for name in files: path = os.path.join(root, name) archive_name = os.path.join(archive_root, name) zip_file.write(path, archive_name, ZIP_DEFLATED) zip_file.close() data = xmlrpclib.Binary(zip_data.getvalue()) zip_data.close() return data
def zip_layer_folder(dir_path, layer_name): """ Create a zip archive with the content of the folder located at `dir_path` and name it with `layer_name`. Parameters ---------- dir_path: str The path to the temporary folder in which are located the files to be zipped. layer_name: str The name of the concerned layer (will be used as file name for the zip archive). Returns ------- raw_content: str The zip archive archive_name: str The name of the archive (used later in the header of the response). """ filenames = os.listdir(dir_path) zip_stream = BytesIO() myZip = ZipFile(zip_stream, "w", compression=ZIP_DEFLATED) for filename in filenames: if not filename.endswith('.geojson'): f_name = path_join(dir_path, filename) myZip.write(f_name, filename, ZIP_DEFLATED) myZip.close() zip_stream.seek(0) return zip_stream.read(), ''.join([layer_name, ".zip"])
def load_otto_group(): """ Loads and returns several variables for the data set from Kaggle's Otto Group Product Classification competition. Link: https://www.kaggle.com/c/otto-group-product-classification-challenge Returns ---------- data : array-like Pandas data frame containing the entire data set. X : array-like Training input samples. y : array-like Target values. """ file_location = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data', 'otto_group.zip') z = ZipFile(file_location) data = pd.read_csv(z.open('train.csv')) data = data.set_index('id') # move the label to the first position cols = data.columns.tolist() cols = cols[-1:] + cols[0:-1] data = data[cols] X = data.iloc[:, 1:].values y = data.iloc[:, 0].values # transform the labels from strings to integers encoder = LabelEncoder() y = encoder.fit_transform(y) return data, X, y
def download_file(url, name, root_destination='~/data/', zipfile=False, replace=False): """Download a file from dropbox, google drive, or a URL. This will download a file and store it in a '~/data/` folder, creating directories if need be. It will also work for zip files, in which case it will unzip all of the files to the desired location. Parameters ---------- url : string The url of the file to download. This may be a dropbox or google drive "share link", or a regular URL. If it is a share link, then it should point to a single file and not a folder. To download folders, zip them first. name : string The name / path of the file for the downloaded file, or the folder to zip the data into if the file is a zipfile. root_destination : string The root folder where data will be downloaded. zipfile : bool Whether the URL points to a zip file. If yes, it will be unzipped to root_destination + name. replace : bool If True and the URL points to a single file, overwrite the old file if possible. """ # Make sure we have directories to dump files home = op.expanduser('~') tmpfile = home + '/tmp/tmp' if not op.isdir(home + '/data/'): print('Creating data folder...') os.makedirs(home + '/data/') if not op.isdir(home + '/tmp/'): print('Creating tmp folder...') os.makedirs(home + '/tmp/') download_path = _convert_url_to_downloadable(url) # Now save to the new destination out_path = root_destination.replace('~', home) + name if not op.isdir(op.dirname(out_path)): print('Creating path {} for output data'.format(out_path)) os.makedirs(op.dirname(out_path)) if zipfile is True: _fetch_file(download_path, tmpfile) myzip = ZipFile(tmpfile) myzip.extractall(out_path) os.remove(tmpfile) else: if len(name) == 0: raise ValueError('Cannot overwrite the root data directory') if replace is False and op.exists(out_path): raise ValueError('Path {} exists, use `replace=True` to ' 'overwrite'.format(out_path)) _fetch_file(download_path, out_path) print('Successfully moved file to {}'.format(out_path))
def load_property_inspection(): """ Loads and returns several variables for the data set from Kaggle's Property Inspection Prediction competition. Link: https://www.kaggle.com/c/liberty-mutual-group-property-inspection-prediction Returns ---------- data : array-like Pandas data frame containing the entire data set. X : array-like Training input samples. y : array-like Target values. """ file_location = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data', 'property_inspection.zip') z = ZipFile(file_location) data = pd.read_csv(z.open('train.csv')) data = data.set_index('Id') X = data.iloc[:, 1:].values y = data.iloc[:, 0].values # transform the categorical variables from strings to integers encoder = CategoryEncoder() X = encoder.fit_transform(X) return data, X, y
def parse_template(template_name): """Resolve template name into absolute path to the template and boolean if absolute path is temporary directory. """ if template_name.startswith('http'): if '#' in template_name: url, subpath = template_name.rsplit('#', 1) else: url = template_name subpath = '' with tempfile.NamedTemporaryFile() as tmpfile: urlretrieve(url, tmpfile.name) if not is_zipfile(tmpfile.name): raise ConfigurationError("Not a zip file: %s" % tmpfile) zf = ZipFile(tmpfile) try: path = tempfile.mkdtemp() zf.extractall(path) return os.path.join(path, subpath), True finally: zf.close() registry = TemplatesRegistry() if registry.has_template(template_name): path = registry.path_of_template(template_name) elif ':' in template_name: path = resolve_dotted_path(template_name) else: path = os.path.realpath(template_name) if not os.path.isdir(path): raise ConfigurationError('Template directory does not exist: %s' % path) return path, False
def download_unzip(input_zip): url = urllib.urlopen(input_zip) unzipped_string = '' zipfile = ZipFile(StringIO(url.read())) for name in zipfile.namelist(): unzipped_string += zipfile.open(name).read() return unzipped_string
def export_zip(self, paths): stringio = StringIO() archive = ZipFile(stringio, mode='w') def _add_resource(resource): for filename in resource.get_files_to_archive(True): if filename.endswith('.metadata'): continue path = Path(self.handler.key).get_pathto(filename) archive.writestr(str(path), resource.handler.to_str()) for path in paths: child = self.get_resource(path, soft=True) if child is None: continue # A Folder => we add its content if isinstance(child, Folder): for subchild in child.traverse_resources(): if subchild is None or isinstance(subchild, Folder): continue _add_resource(subchild) else: _add_resource(child) archive.close() return stringio.getvalue()
def get_sightings_from_atlas(uri, species_ids): # Create a dict of sightings # Each species ID will have a list of sightings with [lat, long] sightings = dict() for species_id in species_ids: sightings[species_id] = [] # The CSV headers LONG = 0 LAT = 1 LSID = 2 # Download API call and unzip url = urlopen(uri) zipfile = ZipFile(StringIO(url.read())) # Skip the header row using [1:] for line in zipfile.open("data.csv").readlines()[1:]: sighting_record = line.split(",") sightings[sighting_record[LSID][1:-2]].append([sighting_record[LAT][1:-1],sighting_record[LONG][1:-1]]) for species_id in species_ids: # Don't return too many sightings for a single species sightings[species_id] = sightings[species_id][0:species_sighting_limit] # Prune any empty entries if sightings[species_id] == []: del sightings[species_id] return sightings
def load_forest_cover(): """ Loads and returns several variables for the data set from Kaggle's Forest Cover Type Prediction competition. Link: https://www.kaggle.com/c/forest-cover-type-prediction Returns ---------- data : array-like Pandas data frame containing the entire data set. X : array-like Training input samples. y : array-like Target values. """ file_location = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data', 'forest_cover.zip') z = ZipFile(file_location) data = pd.read_csv(z.open('train.csv')) data = data.set_index('Id') # move the label to the first position cols = data.columns.tolist() cols = cols[-1:] + cols[0:-1] data = data[cols] X = data.iloc[:, 1:].values y = data.iloc[:, 0].values return data, X, y
def createDevEnv(baseDir, type): fileBuffer = StringIO() createBuild(baseDir, type=type, outFile=fileBuffer, devenv=True, releaseBuild=True) from zipfile import ZipFile zip = ZipFile(StringIO(fileBuffer.getvalue()), 'r') zip.extractall(os.path.join(baseDir, 'devenv')) zip.close() print 'Development environment created, waiting for connections from active extensions...' metadata = readMetadata(baseDir, type) connections = [0] import SocketServer, time, thread class ConnectionHandler(SocketServer.BaseRequestHandler): def handle(self): connections[0] += 1 self.request.sendall('HTTP/1.0 OK\nConnection: close\n\n%s' % metadata.get('general', 'basename')) server = SocketServer.TCPServer(('localhost', 43816), ConnectionHandler) def shutdown_server(server): time.sleep(10) server.shutdown() thread.start_new_thread(shutdown_server, (server,)) server.serve_forever() if connections[0] == 0: print 'Warning: No incoming connections, extension probably not active in the browser yet' else: print 'Handled %i connection(s)' % connections[0]
def upload_analyzer(self): """Upload analyzer to guest. @return: operation status. """ zip_data = StringIO() zip_file = ZipFile(zip_data, "w", ZIP_DEFLATED) root = os.path.join("analyzer", self.platform) root_len = len(os.path.abspath(root)) if not os.path.exists(root): log.error("No valid analyzer found at path: %s" % root) return False for root, dirs, files in os.walk(root): archive_root = os.path.abspath(root)[root_len:] for name in files: path = os.path.join(root, name) archive_name = os.path.join(archive_root, name) zip_file.write(path, archive_name, ZIP_DEFLATED) zip_file.close() data = xmlrpclib.Binary(zip_data.getvalue()) zip_data.close() log.debug("Uploading analyzer to guest (ip=%s)" % self.ip) self.server.add_analyzer(data)
def extract_zip(input_zip): input_zip = ZipFile(input_zip) return {name: input_zip.read(name) for name in input_zip.namelist()}
def make_capsule(rootdir=None, filename='project.zip'): '''Generate all the files used by running encrypted scripts, pack all of them to a zip file. rootdir pyarmor root dir, where you can find license files, auxiliary library and pyshield extension module. filename output filename, the default value is project.zip Return True if sucess, otherwise False or raise exception ''' try: if rootdir is None: rootdir = sys.rootdir except AttributeError: rootdir = os.path.dirname(os.path.abspath(sys.argv[0])) logging.info('Rootdir is %s', rootdir) filelist = 'public.key', 'pyimcore.py', 'pytransform.py' for x in filelist: src = os.path.join(rootdir, x) if not os.path.exists(src): raise RuntimeError('No %s found in the rootdir' % src) licfile = os.path.join(rootdir, 'license.lic') if not os.path.exists(licfile): raise RuntimeError('Missing license file %s' % licfile) logging.info('Generating project key ...') pri, pubx, capkey, lic = pytransform.generate_project_capsule(licfile) logging.info('Generating project OK.') logging.info('Writing capsule to %s ...', filename) myzip = ZipFile(filename, 'w') try: myzip.write(os.path.join(rootdir, 'public.key'), 'pyshield.key') myzip.writestr('pyshield.lic', capkey) myzip.write(os.path.join(rootdir, 'pyimcore.py'), 'pyimcore.py') myzip.write(os.path.join(rootdir, 'pytransform.py'), 'pytransform.py') myzip.writestr('private.key', pri) myzip.writestr('product.key', pubx) myzip.writestr('license.lic', lic) finally: myzip.close() logging.info('Write project capsule OK.')
import os, sys, wget, csv, redis from datetime import date, timedelta from zipfile import ZipFile url = 'https://www.bseindia.com/download/BhavCopy/Equity/EQ' + ( date.today() - timedelta(1)).strftime('%d%m%y') + '_CSV.ZIP' try: wget.download(url, os.getcwd()) except: sys.exit() file = 'EQ' + (date.today() - timedelta(1)).strftime('%d%m%y') zip = ZipFile(file + '_CSV.ZIP').extractall() file = open(file + '.csv') file = csv.reader(file) r = redis.Redis(host='localhost', port=6379, db=0, password=None) r.set("total", 1) for i in file: if file.line_num == 1: continue hash_key = "member:" + str(file.line_num - 1) r.hmset( hash_key, { "code": i[0], "name": i[1], "open": i[4], "high": i[5], "low": i[6], "close": i[7] })
def model_config(fuel_blob): with ZipFile(BytesIO(fuel_blob)) as model_file: with model_file.open("model.config", "r") as sdf_file: xml_string = sdf_file.read().decode("utf-8") return xml_string
def model_sdf(fuel_blob): with ZipFile(BytesIO(fuel_blob)) as model_file: with model_file.open("model.sdf", "r") as sdf_file: sdf_string = sdf_file.read().decode("utf-8") return sdf_string
from zipfile import ZipFile import os caminho = r'C:\Users\desen\Downloads\serie2' with ZipFile('arquivo.zip', 'w') as zip: for file in os.listdir( caminho): # Só os arquivos da pasta, não entra nas subpastas caminho_completo = os.path.join(caminho, file) zip.write( caminho_completo, file ) # o arquivo que quero salvar o nome que desejar ficar no arquivo with ZipFile('arquivo.zip', 'r') as zip: for arquivo in zip.namelist(): print(arquivo) with ZipFile('arquivo.zip', 'r') as zip: zip.extractall('descompactado')
def do_encrypt(argv): '''Usage: pyarmor encrypt [OPTIONS] [File Patterns or @Filename] Encrpty the files list in the command line, you can use a specified pattern according to the rules used by the Unix shell. No tilde expansion is done, but *, ?, and character ranges expressed with [] will be correctly matched. You can either list file patterns in one file, one pattern one line, then add a prefix '@' to the filename. All the files will be encrypted and saved as orginal file name plus 'e'. By default, the encrypted scripts and all the auxiliary files used to run the encrypted scripts are save in the path "dist". Available options: -O, --output=DIR Output path for runtime files and encrypted files (if no --in-place) The default value is "build". -C, --with-capsule=FILENAME Specify the filename of capsule generated before. The default value is "project.zip". -i, --in-place [option], the encrypted scripts will be saved in the original path (same as source). Otherwise, save to --output specified. -s, --src=DIR [option], the source path of python scripts. The default value is current path. -p, --plat-name [option] platform name to run encrypted scripts. Only used when encrypted scripts will be run in different platform. -m, --main=NAME Generate wrapper file to run encrypted script -e, --mode=MODE Encrypt mode, available value: 0 Encrypt both source and bytecode 1 Encrypt bytecode only. 2 Encrypt source code only. 3 Obfuscate bytecodes. 5 Obfuscate code object of module. 6 Combine mode 3 and 4 7 Obfuscate code object of module, output wrapper scripts 8 Obfuscate both code object and bytecode, output wrapper scripts Mode 0, 1, 2 is deprecated from v3.2.0, this option can be ignored in general. -d, --clean Clean output path at start. --manifest FILENAME Write file list to FILENAME For examples: - Encrypt a.py and b.py as a.pyx and b.pyx, saved in the path "dist": pyarmor encrypt a.py b.py - Use file pattern to specify files: pyarmor encrypt a.py *.py src/*.py lib/*.pyc - Save encrypted files in the directory "/tmp/build" other than "dist": pyarmor encrypt --output=/tmp/build a.py - Encrypt python scripts by project capsule "project.zip" in the current directory: pyarmor encrypt --with-capsule=project.zip src/*.py - Encrypt python scripts to run in different platform: pyarmor encrypt --plat-name=linux_x86_64 a.py b.py Use MANIFEST.in to list files pyarmor encrypt --with-capsule=project.zip @myproject/MANIFEST.in It's Following the Distutils’ own manifest template ''' opts, args = getopt.getopt( argv, 'C:de:im:O:p:s:', ['in-place', 'output=', 'src=', 'with-capsule=', 'plat-name=', 'main=', 'clean', 'mode=', 'manifest='] ) output = 'build' srcpath = None capsule = 'project.zip' inplace = False pname = None extfile = None mainname = [] clean = False mode = 8 manifest = None for o, a in opts: if o in ('-O', '--output'): output = a elif o in ('-s', '--src'): srcpath = a elif o in ('-i', '--in-place'): inplace = True elif o in ('-C', '--with-capsule'): capsule = a elif o in ('-p', '--plat-name'): pname = a elif o in ('-d', '--clean'): clean = True elif o in ('-e', '--mode'): if a not in ('0', '1', '2', '3', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14'): raise RuntimeError('Invalid mode "%s"' % a) mode = int(a) elif o in ('-m', '--main'): mainname.append(a) elif o in ('--manifest', ): manifest = a if srcpath is not None and not os.path.exists(srcpath): raise RuntimeError('No found specified source path "%s"' % srcpath) if capsule is None or not os.path.exists(capsule): raise RuntimeError('No found capsule file %s' % capsule) # Maybe user specify an empty path if output == '': output = 'build' logging.info('Output path is %s' % output) if os.path.exists(output) and clean: logging.info('Removing output path %s', output) shutil.rmtree(output) logging.info('Remove output path OK.') if not os.path.exists(output): logging.info('Make output path %s', output) os.makedirs(output) if pname is None: extfile = os.path.join(sys.rootdir, dll_name + dll_ext) else: logging.info("Cross publish, target platform is %s", pname) name = dll_name + ('.so' if pname.startswith('linux') else '.dll') extfile = os.path.join(sys.rootdir, 'platforms', pname, name) if not os.path.exists(extfile): # Need to download platforms/... from pyarmor homepage logging.info('You need download prebuilt library files ' 'from pyarmor homepage first.') raise RuntimeError('Missing cross platform library %s' % extfile) logging.info('Copy %s to %s' % (extfile, output)) shutil.copy(extfile, output) logging.info('Extract capsule %s ...', capsule) ZipFile(capsule).extractall(path=output) logging.info('Extract capsule to %s OK.', output) # Fix bootstrap restrict issue from v5.7.0 make_license(capsule, os.path.join(output, 'license.lic'), '*FLAGS:A*CODE:PyArmor') if mode >= 3: logging.info('Encrypt mode: %s', mode) with open(os.path.join(output, 'pyimcore.py'), 'w') as f: lines = 'from pytransform import old_init_runtime', \ 'old_init_runtime(0, 0, 0, 0)', '' f.write('\n'.join(lines)) elif mode: logging.info('Encrypt mode: %s', mode) with open(os.path.join(output, 'pyimcore.py'), 'r') as f: lines = f.read() with open(os.path.join(output, 'pyimcore.py'), 'w') as f: i = lines.rfind('\n\n') if i == -1: raise RuntimeError('Invalid pyimcore.py') f.write(lines[:i]) if mode == 1: f.write('\n\nold_init_runtime()\n') elif mode == 2: f.write('\n\nsys.meta_path.append(PyshieldImporter())\n' 'old_init_runtime(0, 0, 0, 0)\n') prikey = os.path.join(output, 'private.key') if os.path.exists(prikey): logging.info('Remove private key %s in the output', prikey) os.remove(prikey) if mode not in (7, 8, 9, 10, 11, 12, 13, 14): for name in mainname: n = name.find(':') if n == -1: script = os.path.join(output, name + '.py') else: script = os.path.join(output, name[n+1:]) name = name[:n] logging.info('Writing script wrapper %s ...', script) ch = 'c' if mode == 1 or mode == 3 else ext_char with open(script, 'w') as f: f.write(wrap_runner % (name + '.py' + ch)) logging.info('Write script wrapper OK.') filelist = _parse_file_args(args, srcpath=srcpath) if manifest is not None: logging.info('Write filelist to %s', manifest) with open(manifest, 'w') as fp: fp.write('\n'.join([x[0] for x in filelist])) if len(filelist[:1]) == 0: logging.info('Generate extra files OK.') else: prokey = os.path.join(output, 'product.key') if not os.path.exists(prokey): raise RuntimeError('Missing project key %s' % prokey) logging.info('Encrypt files ...') encrypt_files(filelist, prokey, mode, None if inplace else output) if mode in (7, 8, 9, 10, 11, 12, 13, 14): for name in mainname: script = os.path.join( output, name + ('' if name.endswith('.py') else '.py')) with open(script, 'r') as f: source = f.read() logging.info('Patch entry script %s.', script) with open(script, 'w') as f: f.write('import pyimcore\n') f.write(source) logging.info('Encrypt files OK.')
def _getAllObjectsData(self, objects_listing, context_path): """ Returns the data in all files with a content object to be placed in a zipfile """ # Use temporary IO object instead of writing to filesystem. out = StringIO() zipFile = ZipFile(out, 'w', ZIP_DEFLATED) for obj in objects_listing: object_path = str(obj.virtual_url_path()) if self._objImplementsInterface( obj, interfaces.IATFile) or self._objImplementsInterface( obj, interfaces.IATImage): file_data = str(obj.data) object_path = object_path.replace(context_path + '/', '') elif self._objImplementsInterface(obj, interfaces.IATDocument): if "text/html" == obj.Format(): file_data = obj.getText() if object_path[-5:] != ".html" and object_path[ -4:] != ".htm": object_path += ".html" elif "text/x-rst" == obj.Format(): file_data = obj.getRawText() if object_path[-4:] != ".rst": object_path += ".rst" elif "text/structured" == obj.Format(): file_data = obj.getRawText() if object_path[-4:] != ".stx": object_path += ".stx" elif "text/plain" == obj.Format(): file_data = obj.getRawText() if object_path[-4:] != ".txt": object_path += ".txt" else: file_data = obj.getRawText() object_path = object_path.replace(context_path + '/', '') elif self._objImplementsInterface(obj, interfaces.IATFolder): if hasattr(obj, 'getRawText'): file_data = obj.getRawText() if object_path == context_path: object_path = object_path.split("/")[-1] else: object_path = object_path.replace( context_path + '/', '') if object_path[-5:] != ".html" and object_path[ -4:] != ".htm": object_path += ".html" else: continue # start point for object path, adding 1 removes the initial '/' object_path = self.generateSafeFileName(object_path) if object_path: zipFile.writestr(object_path, file_data) zipFile.close() out.seek(0) content = out.read() out.close() return content
_author_ = "aman" from zipfile import ZipFile for i in range(17): print("{0:>2} in binary is {0:>08b}".format(i)) with ZipFile('C:\\Users\\aman.raj\\Documents\\GST\\GST_C_20200229.zip', 'r') as zipObj: zipObj.extractall('C:\\Users\\aman.raj\\Documents\\GST\\temp')
def importContent(self, file, context=None, description=None, contributors=None, categories=None, overwrite=False): """ Import content from a zip file, creating the folder structure within a ZODB hierarchy. """ self.bad_folders = [] zf = ZipFile(file, 'r') files = [file.filename for file in zf.filelist] if len(files) < 1: return ('failure', 'The zip file was empty') for current_file in files: # If the current file is a folder move to the next file. if current_file[-1] == '/': continue if current_file[0] == '/': path_as_list = current_file[1:].split('/') else: path_as_list = current_file.split('/') file_name = path_as_list[-1] # Checks to make sure that the file path does not contain any previouslsy found bad folders. if not self._checkFilePath(current_file, path_as_list): continue folder = self._createFolderStructure(path_as_list, context) # no folder to add to? Then move on to next object. if not folder: continue code, msg = context.checkid(id=file_name, required=1, contained_by=folder) # Create an object if everything looks good if not code or (overwrite and 'existing' == code): fdata = zf.read(current_file) if "existing" == code: folder.manage_delObjects([file_name]) obj = self._createObject(file_name, fdata, folder) try: obj.setFilename(file_name) except: pass #do nothing if hasattr(obj, 'description') and description: obj.setDescription(description) if hasattr(obj, 'contributors') and contributors: obj.setContributors(contributors) if hasattr(obj, 'subject') and categories: obj.setSubject(categories) obj.reindexObject() zf.close()
def processAlgorithm(self, parameters, context, feedback): self.parameters = parameters self.context = context filename = self.parameterAsFile(parameters, self.PrmInput, context) f, extension = os.path.splitext(filename) extension = extension.lower() try: if extension == '.kmz': kmz = ZipFile(filename, 'r') kml = kmz.open('doc.kml', 'r') elif extension == '.kml': kml = open(filename, encoding="utf-8", errors="backslashreplace") else: msg = "Invalid extension: Should be kml or kmz" feedback.reportError(msg) raise QgsProcessingException(msg) except: msg = "Failed to open file" feedback.reportError(msg) raise QgsProcessingException(msg) skipPt = True if self.PrmPointOutputLayer not in parameters or parameters[ self.PrmPointOutputLayer] is None else False skipline = True if self.PrmLineOutputLayer not in parameters or parameters[ self.PrmLineOutputLayer] is None else False skipPoly = True if self.PrmPolygonOutputLayer not in parameters or parameters[ self.PrmPolygonOutputLayer] is None else False self.cntPt = 0 self.cntLine = 0 self.cntPoly = 0 parser = xml.sax.make_parser() # Do a pre-pass through the KML to see if there are any extended data fields preprocess = PreProcessHandler() parser.setContentHandler(preprocess) try: input_source = xml.sax.xmlreader.InputSource() input_source.setByteStream(kml) input_source.setEncoding('utf-8') parser.parse(input_source) except: preprocess.endDocument() # Reset the KML/KMZ file to the beginning. The handler seems to automatically close the file # which is why we are reopening it. if extension == '.kmz': kmz.close() else: kml.close() if extension == '.kmz': kmz = ZipFile(filename, 'r') kml = kmz.open('doc.kml', 'r') else: kml = open(filename, encoding="utf-8", errors="backslashreplace") # Set up the handler for doing the main processing self.extData = preprocess.getExtendedDataFields() self.extData.sort() self.extDataMap = {} index = 0 for item in self.extData: self.extDataMap[item] = index index += 1 handler = PlacemarkHandler(skipPt, skipline, skipPoly, self.extDataMap, feedback) handler.addpoint.connect(self.addpoint) handler.addline.connect(self.addline) handler.addpolygon.connect(self.addpolygon) parser.setContentHandler(handler) try: input_source = xml.sax.xmlreader.InputSource() input_source.setByteStream(kml) input_source.setEncoding('utf-8') parser.parse(input_source) except: '''s = traceback.format_exc() feedback.pushInfo(s)''' feedback.pushInfo( tr('Failure in kml extraction - May return partial results.')) handler.endDocument() if extension == '.kmz': kmz.close() else: kml.close() feedback.pushInfo('{} points extracted'.format(self.cntPt)) feedback.pushInfo('{} lines extracted'.format(self.cntLine)) feedback.pushInfo('{} polygons extracted'.format(self.cntPoly)) r = {} if self.cntPt > 0: r[self.PrmPointOutputLayer] = self.dest_id_pt if self.cntLine > 0: r[self.PrmLineOutputLayer] = self.dest_id_line if self.cntPoly > 0: r[self.PrmPolygonOutputLayer] = self.dest_id_poly return (r)
def source_dataset(): bea_dataset_name = os.getenv('BEA_DATASET_NAME', 'regional') table_name = os.getenv('BEA_TABLE_NAME', 'CAINC1') source_dataset_url = "https://apps.bea.gov/{}/zip/{}.zip".format(bea_dataset_name, table_name) response = None retries = 5 for attempt in range(retries): try: response = urlopen(source_dataset_url) except HTTPError as e: if attempt == retries: raise Exception('HTTPError: ', e.code) time.sleep(0.2 * attempt) except URLError as e: if attempt == retries: raise Exception('URLError: ', e.reason) time.sleep(0.2 * attempt) else: break if response == None: raise Exception('There was an issue downloading the dataset') data_set_name = os.environ['DATA_SET_NAME'] data_dir = '/tmp' if not os.path.exists(data_dir): os.mkdir(data_dir) zip_location = os.path.join(data_dir, data_set_name+'.zip') with open(zip_location, 'wb') as f: f.write(response.read()) with ZipFile(zip_location, 'r') as z: z.extractall(data_dir) os.remove(zip_location) s3_bucket = os.environ['S3_BUCKET'] s3 = boto3.client('s3') unzipped_name = os.listdir(data_dir)[0] s3_uploads = [] asset_list = [] for r, d, f in os.walk(data_dir): for filename in f: obj_name = os.path.join(r, filename).split('/', 3).pop().replace(' ', '_').lower() file_location = os.path.join(r, filename) new_s3_key = data_set_name + '/dataset/' + obj_name has_changes = md5_compare(s3, s3_bucket, new_s3_key, file_location) if has_changes: s3.upload_file(file_location, s3_bucket, new_s3_key) print('Uploaded: ' + filename) else: print('No changes in: ' + filename) asset_source = {'Bucket': s3_bucket, 'Key': new_s3_key} s3_uploads.append({'has_changes': has_changes, 'asset_source': asset_source}) count_updated_data = sum(upload['has_changes'] == True for upload in s3_uploads) if count_updated_data > 0: asset_list = list(map(lambda upload: upload['asset_source'], s3_uploads)) if len(asset_list) == 0: raise Exception('Something went wrong when uploading files to s3') # asset_list is returned to be used in lamdba_handler function # if it is empty, lambda_handler will not republish return asset_list
''' ZipFile-向已有的zip压缩包中添加文件(一) ''' from zipfile import ZipFile file_name= 'new.zip' file_list=['Snipaste.png','test_fun.py'] with ZipFile(file=file_name,mode='w') as myzip: #注意w会删除压缩包内已有的文件 for f in file_list: myzip.write(f) myzip.printdir()
def parseM3U(infile):#Traite les m3u local oGui = cGui() oInputParameterHandler = cInputParameterHandler() sUrl = oInputParameterHandler.getValue('siteUrl') if 'iptv4sat' in sUrl or '.zip' in sUrl: sHtmlContent = getHtml(sUrl) from zipfile import ZipFile import io zip_file = ZipFile(io.BytesIO(sHtmlContent)) files = zip_file.namelist() with zip_file.open(files[0]) as f: sHtmlContent = [] for line in f: sHtmlContent.append(line) inf = sHtmlContent elif not '#EXTM3U' in sUrl: site= infile user_agent = 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/48.0.2564.116 Chrome/48.0.2564.116 Safari/537.36' headers = {'User-Agent': user_agent} #req = urllib2.Request(site,headers=headers) #inf = urllib2.urlopen(req) oRequestHandler = cRequestHandler(sUrl) oRequestHandler.addHeaderEntry('User-Agent',user_agent) inf = oRequestHandler.request() inf = inf.split('\n') else: inf = infile try: line = inf.readline() except: pass #cConfig().log(str(line)) #if not line.startswith('#EXTM3U'): #return playlist=[] song=track(None,None,None,None) ValidEntry = False for line in inf: line=line.strip() if line.startswith('#EXTINF:'): length,title=line.split('#EXTINF:')[1].split(',',1) try: licon = line.split('#EXTINF:')[1].partition('tvg-logo=')[2] icon = licon.split('"')[1] except: icon = "tv.png" ValidEntry = True song=track(length,title,None,icon) elif (len(line) != 0): if (ValidEntry) and (not (line.startswith('!') or line.startswith('#'))): ValidEntry = False song.path=line playlist.append(song) #VSlog(playlist) song=track(None,None,None,None) try: inf.close() except: pass return playlist
class XPCShellRemote(xpcshell.XPCShellTests, object): def __init__(self, devmgr, options, args, log=None): xpcshell.XPCShellTests.__init__(self, log) # Add Android version (SDK level) to mozinfo so that manifest entries # can be conditional on android_version. androidVersion = devmgr.shellCheckOutput(['getprop', 'ro.build.version.sdk']) mozinfo.info['android_version'] = androidVersion self.localLib = options.localLib self.localBin = options.localBin self.options = options self.device = devmgr self.pathMapping = [] self.remoteTestRoot = self.device.getTestRoot("xpcshell") # remoteBinDir contains xpcshell and its wrapper script, both of which must # be executable. Since +x permissions cannot usually be set on /mnt/sdcard, # and the test root may be on /mnt/sdcard, remoteBinDir is set to be on # /data/local, always. self.remoteBinDir = "/data/local/xpcb" # Terse directory names are used here ("c" for the components directory) # to minimize the length of the command line used to execute # xpcshell on the remote device. adb has a limit to the number # of characters used in a shell command, and the xpcshell command # line can be quite complex. self.remoteTmpDir = remoteJoin(self.remoteTestRoot, "tmp") self.remoteScriptsDir = self.remoteTestRoot self.remoteComponentsDir = remoteJoin(self.remoteTestRoot, "c") self.remoteModulesDir = remoteJoin(self.remoteTestRoot, "m") self.remoteMinidumpDir = remoteJoin(self.remoteTestRoot, "minidumps") self.profileDir = remoteJoin(self.remoteTestRoot, "p") self.remoteDebugger = options.debugger self.remoteDebuggerArgs = options.debuggerArgs self.testingModulesDir = options.testingModulesDir self.env = {} if self.options.objdir: self.xpcDir = os.path.join(self.options.objdir, "_tests/xpcshell") elif os.path.isdir(os.path.join(here, 'tests')): self.xpcDir = os.path.join(here, 'tests') else: print >> sys.stderr, "Couldn't find local xpcshell test directory" sys.exit(1) if options.localAPK: self.localAPKContents = ZipFile(options.localAPK) if options.setup: self.setupUtilities() self.setupModules() self.setupTestDir() self.setupMinidumpDir() self.remoteAPK = None if options.localAPK: self.remoteAPK = remoteJoin(self.remoteBinDir, os.path.basename(options.localAPK)) self.setAppRoot() # data that needs to be passed to the RemoteXPCShellTestThread self.mobileArgs = { 'device': self.device, 'remoteBinDir': self.remoteBinDir, 'remoteScriptsDir': self.remoteScriptsDir, 'remoteComponentsDir': self.remoteComponentsDir, 'remoteModulesDir': self.remoteModulesDir, 'options': self.options, 'remoteDebugger': self.remoteDebugger, 'pathMapping': self.pathMapping, 'profileDir': self.profileDir, 'remoteTmpDir': self.remoteTmpDir, 'remoteMinidumpDir': self.remoteMinidumpDir, } if self.remoteAPK: self.mobileArgs['remoteAPK'] = self.remoteAPK def setLD_LIBRARY_PATH(self): self.env["LD_LIBRARY_PATH"] = self.remoteBinDir def pushWrapper(self): # Rather than executing xpcshell directly, this wrapper script is # used. By setting environment variables and the cwd in the script, # the length of the per-test command line is shortened. This is # often important when using ADB, as there is a limit to the length # of the ADB command line. localWrapper = tempfile.mktemp() f = open(localWrapper, "w") f.write("#!/system/bin/sh\n") for envkey, envval in self.env.iteritems(): f.write("export %s=%s\n" % (envkey, envval)) f.write("cd $1\n") f.write("echo xpcw: cd $1\n") f.write("shift\n") f.write("echo xpcw: xpcshell \"$@\"\n") f.write("%s/xpcshell \"$@\"\n" % self.remoteBinDir) f.close() remoteWrapper = remoteJoin(self.remoteBinDir, "xpcw") self.device.pushFile(localWrapper, remoteWrapper) os.remove(localWrapper) self.device.chmodDir(self.remoteBinDir) def buildEnvironment(self): self.buildCoreEnvironment() self.setLD_LIBRARY_PATH() self.env["MOZ_LINKER_CACHE"] = self.remoteBinDir if self.options.localAPK and self.appRoot: self.env["GRE_HOME"] = self.appRoot self.env["XPCSHELL_TEST_PROFILE_DIR"] = self.profileDir self.env["TMPDIR"] = self.remoteTmpDir self.env["HOME"] = self.profileDir self.env["XPCSHELL_TEST_TEMP_DIR"] = self.remoteTmpDir self.env["XPCSHELL_MINIDUMP_DIR"] = self.remoteMinidumpDir if self.options.setup: self.pushWrapper() def setAppRoot(self): # Determine the application root directory associated with the package # name used by the Fennec APK. self.appRoot = None packageName = None if self.options.localAPK: try: packageName = self.localAPKContents.read("package-name.txt") if packageName: self.appRoot = self.device.getAppRoot(packageName.strip()) except Exception as detail: print "unable to determine app root: " + str(detail) pass return None def setupUtilities(self): if (not self.device.dirExists(self.remoteBinDir)): # device.mkDir may fail here where shellCheckOutput may succeed -- see bug 817235 try: self.device.shellCheckOutput(["mkdir", self.remoteBinDir]); except devicemanager.DMError: # Might get a permission error; try again as root, if available self.device.shellCheckOutput(["mkdir", self.remoteBinDir], root=True); self.device.shellCheckOutput(["chmod", "777", self.remoteBinDir], root=True); remotePrefDir = remoteJoin(self.remoteBinDir, "defaults/pref") if (self.device.dirExists(self.remoteTmpDir)): self.device.removeDir(self.remoteTmpDir) self.device.mkDir(self.remoteTmpDir) if (not self.device.dirExists(remotePrefDir)): self.device.mkDirs(remoteJoin(remotePrefDir, "extra")) if (not self.device.dirExists(self.remoteScriptsDir)): self.device.mkDir(self.remoteScriptsDir) if (not self.device.dirExists(self.remoteComponentsDir)): self.device.mkDir(self.remoteComponentsDir) local = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'head.js') remoteFile = remoteJoin(self.remoteScriptsDir, "head.js") self.device.pushFile(local, remoteFile) # The xpcshell binary is required for all tests. Additional binaries # are required for some tests. This list should be similar to # TEST_HARNESS_BINS in testing/mochitest/Makefile.in. binaries = ["xpcshell", "ssltunnel", "certutil", "pk12util", "BadCertServer", "OCSPStaplingServer", "GenerateOCSPResponse"] for fname in binaries: local = os.path.join(self.localBin, fname) if os.path.isfile(local): print >> sys.stderr, "Pushing %s.." % fname remoteFile = remoteJoin(self.remoteBinDir, fname) self.device.pushFile(local, remoteFile) else: print >> sys.stderr, "*** Expected binary %s not found in %s!" % (fname, self.localBin) local = os.path.join(self.localBin, "components/httpd.js") remoteFile = remoteJoin(self.remoteComponentsDir, "httpd.js") self.device.pushFile(local, remoteFile) local = os.path.join(self.localBin, "components/httpd.manifest") remoteFile = remoteJoin(self.remoteComponentsDir, "httpd.manifest") self.device.pushFile(local, remoteFile) local = os.path.join(self.localBin, "components/test_necko.xpt") remoteFile = remoteJoin(self.remoteComponentsDir, "test_necko.xpt") self.device.pushFile(local, remoteFile) if self.options.localAPK: remoteFile = remoteJoin(self.remoteBinDir, os.path.basename(self.options.localAPK)) self.device.pushFile(self.options.localAPK, remoteFile) self.pushLibs() def pushLibs(self): pushed_libs_count = 0 if self.options.localAPK: try: dir = tempfile.mkdtemp() szip = os.path.join(self.localBin, '..', 'host', 'bin', 'szip') if not os.path.exists(szip): # Tinderbox builds must run szip from the test package szip = os.path.join(self.localBin, 'host', 'szip') if not os.path.exists(szip): # If the test package doesn't contain szip, it means files # are not szipped in the test package. szip = None for info in self.localAPKContents.infolist(): if info.filename.endswith(".so"): print >> sys.stderr, "Pushing %s.." % info.filename remoteFile = remoteJoin(self.remoteBinDir, os.path.basename(info.filename)) self.localAPKContents.extract(info, dir) file = os.path.join(dir, info.filename) if szip: out = subprocess.check_output([szip, '-d', file], stderr=subprocess.STDOUT) self.device.pushFile(os.path.join(dir, info.filename), remoteFile) pushed_libs_count += 1 finally: shutil.rmtree(dir) return pushed_libs_count for file in os.listdir(self.localLib): if (file.endswith(".so")): print >> sys.stderr, "Pushing %s.." % file if 'libxul' in file: print >> sys.stderr, "This is a big file, it could take a while." remoteFile = remoteJoin(self.remoteBinDir, file) self.device.pushFile(os.path.join(self.localLib, file), remoteFile) pushed_libs_count += 1 # Additional libraries may be found in a sub-directory such as "lib/armeabi-v7a" localArmLib = os.path.join(self.localLib, "lib") if os.path.exists(localArmLib): for root, dirs, files in os.walk(localArmLib): for file in files: if (file.endswith(".so")): print >> sys.stderr, "Pushing %s.." % file remoteFile = remoteJoin(self.remoteBinDir, file) self.device.pushFile(os.path.join(root, file), remoteFile) pushed_libs_count += 1 return pushed_libs_count def setupModules(self): if self.testingModulesDir: self.device.pushDir(self.testingModulesDir, self.remoteModulesDir) def setupTestDir(self): print 'pushing %s' % self.xpcDir try: self.device.pushDir(self.xpcDir, self.remoteScriptsDir, retryLimit=10) except TypeError: # Foopies have an older mozdevice ver without retryLimit self.device.pushDir(self.xpcDir, self.remoteScriptsDir) def setupMinidumpDir(self): if self.device.dirExists(self.remoteMinidumpDir): self.device.removeDir(self.remoteMinidumpDir) self.device.mkDir(self.remoteMinidumpDir) def buildTestList(self): xpcshell.XPCShellTests.buildTestList(self) uniqueTestPaths = set([]) for test in self.alltests: uniqueTestPaths.add(test['here']) for testdir in uniqueTestPaths: abbrevTestDir = os.path.relpath(testdir, self.xpcDir) remoteScriptDir = remoteJoin(self.remoteScriptsDir, abbrevTestDir) self.pathMapping.append(PathMapping(testdir, remoteScriptDir))
else: box = None fLOG('----------------------') for date, mail in rev: fLOG(mail["Date"], mail['From']) if 'att' in mail: att = (mail['name'], mail['att']) else: for att_ in mail.enumerate_attachments(): att = att_ break if ".zip" in att[0]: content = att[1] st = io.BytesIO(content) with ZipFile(st) as myzip: infos = myzip.infolist() if len(infos) != 1: fLOG("skip mail '{0}' from '{1}'".format(i, mail['From'])) else: name = infos[0].filename data = myzip.read(name) datas = data.decode('ascii') rec = compute_metrics(datas) rec["a_sender"] = mail['From'] perf_both.append(rec) rec['a_date'] = mail['Date'] elif ".csv" in att[0]: datas = att[1].decode('ascii', errors='ignore') try: df = pandas.read_csv(io.StringIO(datas), sep=";")
def _extract_seq_results(order, session): """ Given an order (full metadata), saves the ABI trace files into a temporary zip file and returns some information about the order. Args: order: a dictionary containing the keys 'id', 'orderType', 'orderName', and 'orderStatus' representing a certain GeneWiz order. session: a Requests session to download files with Returns: A tuple encoding (string_summary, zip_file_path). String_summary is a message fit for a Slack message summarizing the results and zip_file_path is a filesystem path to a temporary zip file containing all trace files. """ if order['orderType'] != 'Sanger Sequencing': raise RuntimeError('Order is not a Sanger sequencing result') if order['orderStatus'] != 'Completed': raise RuntimeError('Selected order is not completed!') details_html = session.get('https://clims4.genewiz.com/SangerSequencing/ViewResults', data={'OrderId': order['id']}) details_model_match = re.search("window.GWZ.CLIMS.model = (.*);", details_html.text) details = json.loads(details_model_match.group(1)) reaction_list = list(details['OrdersResults'].values())[0] # Extract each sequencing reaction information ab1_files = [] tempdir = tempfile.mkdtemp() names = [] qualities = [] lengths = [] for reaction in reaction_list: quality = reaction['QS'] length = reaction['CRL'] order_id = order['id'] sequencer = reaction['Sequencer'] folder = reaction['ActualPlateFolder'] file_name = reaction['AB1FileName'] labwell = '_' + reaction['LabWellFormatted'] query_string = {'orderId': order_id, 'sequencer': sequencer, 'folder': folder, 'fileName': file_name[:-4] + labwell + '.ab1', 'labwell': labwell} names.append(reaction['SamplePrimerName']) qualities.append(quality) lengths.append(length) ab1_file = session.get('https://clims4.genewiz.com/SangerSequencing/DownloadResult', data=query_string) # Assert that we actually got an AB1 files assert ab1_file.content[0:4] == b'ABIF' # Save the file into our temp directory new_ab1 = os.path.join(tempdir, reaction['SamplePrimerName'] + '.ab1') ab1_files.append(new_ab1) with open(new_ab1, 'wb') as outfile: outfile.write(ab1_file.content) # Zip all files together order_name = reaction_list[0]['OrderName'] if order_name is None: order_name = 'no_order_name' zip_filename = os.path.join(tempdir, order_name + '.zip') with ZipFile(zip_filename, 'w') as outzip: for ab1 in ab1_files: outzip.write(ab1, arcname = os.path.basename(ab1)) # Cleanup for filename in ab1_files: os.remove(filename) # Built output string: url = '<https://clims4.genewiz.com/SangerSequencing/ViewResults?OrderID={}|Sequencing results>'.format(order['id']) order_str = 'Order: {}'.format(order_name) table = '' for tup in zip(names, lengths, qualities): table += '\n\t{}: Length {}, quality {}'.format(*tup) return (order_str + '\n' + url + table, zip_filename)
def __init__(self, devmgr, options, args, log=None): xpcshell.XPCShellTests.__init__(self, log) # Add Android version (SDK level) to mozinfo so that manifest entries # can be conditional on android_version. androidVersion = devmgr.shellCheckOutput(['getprop', 'ro.build.version.sdk']) mozinfo.info['android_version'] = androidVersion self.localLib = options.localLib self.localBin = options.localBin self.options = options self.device = devmgr self.pathMapping = [] self.remoteTestRoot = self.device.getTestRoot("xpcshell") # remoteBinDir contains xpcshell and its wrapper script, both of which must # be executable. Since +x permissions cannot usually be set on /mnt/sdcard, # and the test root may be on /mnt/sdcard, remoteBinDir is set to be on # /data/local, always. self.remoteBinDir = "/data/local/xpcb" # Terse directory names are used here ("c" for the components directory) # to minimize the length of the command line used to execute # xpcshell on the remote device. adb has a limit to the number # of characters used in a shell command, and the xpcshell command # line can be quite complex. self.remoteTmpDir = remoteJoin(self.remoteTestRoot, "tmp") self.remoteScriptsDir = self.remoteTestRoot self.remoteComponentsDir = remoteJoin(self.remoteTestRoot, "c") self.remoteModulesDir = remoteJoin(self.remoteTestRoot, "m") self.remoteMinidumpDir = remoteJoin(self.remoteTestRoot, "minidumps") self.profileDir = remoteJoin(self.remoteTestRoot, "p") self.remoteDebugger = options.debugger self.remoteDebuggerArgs = options.debuggerArgs self.testingModulesDir = options.testingModulesDir self.env = {} if self.options.objdir: self.xpcDir = os.path.join(self.options.objdir, "_tests/xpcshell") elif os.path.isdir(os.path.join(here, 'tests')): self.xpcDir = os.path.join(here, 'tests') else: print >> sys.stderr, "Couldn't find local xpcshell test directory" sys.exit(1) if options.localAPK: self.localAPKContents = ZipFile(options.localAPK) if options.setup: self.setupUtilities() self.setupModules() self.setupTestDir() self.setupMinidumpDir() self.remoteAPK = None if options.localAPK: self.remoteAPK = remoteJoin(self.remoteBinDir, os.path.basename(options.localAPK)) self.setAppRoot() # data that needs to be passed to the RemoteXPCShellTestThread self.mobileArgs = { 'device': self.device, 'remoteBinDir': self.remoteBinDir, 'remoteScriptsDir': self.remoteScriptsDir, 'remoteComponentsDir': self.remoteComponentsDir, 'remoteModulesDir': self.remoteModulesDir, 'options': self.options, 'remoteDebugger': self.remoteDebugger, 'pathMapping': self.pathMapping, 'profileDir': self.profileDir, 'remoteTmpDir': self.remoteTmpDir, 'remoteMinidumpDir': self.remoteMinidumpDir, } if self.remoteAPK: self.mobileArgs['remoteAPK'] = self.remoteAPK
class MetadataLookup(GObject.GObject): """ Look up the localized activity name and size of a bundle. This is useful when no previous version of the activity is installed, and there is no local source of the activity's name. """ __gsignals__ = { 'complete': (GObject.SignalFlags.RUN_FIRST, None, (object, object, object)), } def __init__(self, url): GObject.GObject.__init__(self) self._url = url self._icon_file_name = None self._size = None def run(self): # Perform the name lookup, catch any exceptions, and report the result. try: name = self._do_name_lookup() self._complete(name) except Exception as e: self._complete(e) def _do_name_lookup(self): fd = httprange.open(self._url) self._size = fd.size() return self._name_from_fd(fd) def _name_from_fd(self, fd): self._zf = ZipFile(fd) self._namelist = self._zf.namelist() self._prefix = None have_activity_info = False have_library_info = False for path in self._namelist: if path.count('/') != 2: continue if path.endswith('/activity/activity.info'): have_activity_info = True self._prefix = path.split('/', 1)[0] break if path.endswith('/library/library.info'): have_library_info = True self._prefix = path.split('/', 1)[0] break if self._prefix is None: raise Exception("Couldn't find activity prefix") # To find the localized name, we first look for an activity.linfo file # in the current locale. # We fall back on pulling the non-localized name from activity.info, # if there is one. # The final fallback is pulling the name from library.info; in the # case of a content bundle, that name is expected to be already # localized according to the content. name = self._locale_data_lookup() icon_path = None if not name and have_activity_info: name = self._activity_info_lookup('name') if not name and have_library_info: name = self._library_info_lookup('name') # get icondata if have_activity_info: icon_name = self._activity_info_lookup('icon') if not name and have_library_info: icon_name = self._library_info_lookup('icon') icon_path = os.path.join(self._prefix, 'activity', '%s.svg' % icon_name) if icon_path is not None and icon_path in self._namelist: icon_data = self._zf.read(icon_path) # save the icon to a temporary file with NamedTemporaryFile(mode='w', suffix='.svg', delete=False) as iconfile: iconfile.write(icon_data) self._icon_file_name = iconfile.name return name def _locale_data_lookup(self): lang = locale.getdefaultlocale()[0] if lang is None: return None for f in ('locale/%s/activity.linfo' % lang, 'locale/%s/activity.linfo' % lang[:2]): filename = os.path.join(self._prefix, f) if filename not in self._namelist: continue cp = ConfigParser() cp.readfp(StringIO(self._zf.read(filename))) return cp.get('Activity', 'name') return None def _activity_info_lookup(self, parameter): filename = os.path.join(self._prefix, 'activity', 'activity.info') cp = ConfigParser() cp.readfp(StringIO(self._zf.read(filename))) if cp.has_option('Activity', parameter): return cp.get('Activity', parameter) else: return '' def _library_info_lookup(self, parameter): filename = os.path.join(self._prefix, 'library', 'library.info') cp = ConfigParser() cp.readfp(StringIO(self._zf.read(filename))) if cp.has_option('Library', parameter): return cp.get('Library', parameter) else: return '' def _complete(self, result): GLib.idle_add(self.emit, 'complete', result, self._size, self._icon_file_name)
def makezipfile(data): output_filename = '{}_zipped.zip'.format(data) with ZipFile(output_filename, "w") as z: z.write(data) return output_filename
def extractPdf(filePath): with ZipFile(filePath, 'r') as zipobject: listoffile = zipobject.namelist() for filename in listoffile: if filename.endswith('.pdf'): zipobject.extract(filename, 'Extracted_pdf')
def create_submission(model, loader, int_to_label, config, pred_zip=None, tta=False, base_h=1536, base_w=1536): if pred_zip is not None and os.path.exists(pred_zip): os.remove(pred_zip) sub = [] model.eval() image_ids, labels = [], [] with th.no_grad(): for ret in tqdm(loader): Xs = ret[:-1] fns = ret[-1] hm_preds, classes_embeddings = 0., 0. # scale TTA for X in Xs: X = X.to(config.device).float() hm_pred, classes_embedding = model(X, return_embeddings=True) hm_pred = th.nn.functional.interpolate(hm_pred, size=(base_h // 4, base_w // 4), mode='bilinear', align_corners=True) classes_embedding = th.nn.functional.interpolate( classes_embedding, size=(base_h // 4, base_w // 4), mode='bilinear', align_corners=True) hm_pred = th.sigmoid(hm_pred) hm_preds += hm_pred classes_embeddings += classes_embedding hm_pred = hm_preds / len(Xs) classes_embeddings /= len(Xs) hm_pred = nms(hm_pred).cpu().numpy() for j in range(len(hm_pred)): img_id = KuzushijiDataset.fn_to_id(fns[j]) w, h = Image.open(fns[j]).size pad_top, pad_bottom, pad_left, pad_right = \ KuzushijiDataset.get_paddings(h, w, ratio=1.5) h = h + pad_top + pad_bottom w = w + pad_left + pad_right image_ids.append(img_id) hmp = hm_pred[j].squeeze(0) if pred_zip is not None: pred_fn = img_id + '.png' yp_img = np.uint8(hmp * 255) img_bytes = cv2.imencode('.png', yp_img)[1].tobytes() with ZipFile(pred_zip, 'a') as f: f.writestr(pred_fn, img_bytes) hmp_thresholded = hmp > config.p_letter ys, xs = np.where(hmp_thresholded) if len(ys) == 0: labels.append('') continue centers = th.stack( [th.tensor(xs), th.tensor(ys)], -1).unsqueeze(0) gathered_embeddings = model.gather_embeddings( classes_embeddings[j:j + 1], centers) classes_preds = model.classes( gathered_embeddings.unsqueeze(-1).unsqueeze(-1)).squeeze( -1).squeeze(-1) # set prob for extra letters to zero classes_preds = classes_preds[:, :4212] classes_preds = th.nn.functional.softmax(classes_preds, 1).cpu().numpy() per_image_labels = [] for center_ind in range(len(ys)): x, y = xs[center_ind], ys[center_ind] pred_probs = classes_preds[center_ind] # scale to (padded) image coords x, y = int(round(x * 4 * (w / base_w))), int( round(y * 4 * (h / base_h))) # undo padding x -= pad_left y -= pad_top # pred_classes = np.where(pred_probs > config.p_class)[0] pred_classes = [pred_probs.argmax()] # print(len(pred_classes)) for pred_class in pred_classes: ll = int_to_label[pred_class] per_image_labels.extend([ll, str(x), str(y)]) image_label_str = ' '.join(per_image_labels) labels.append(image_label_str) # img = cv2.imread('test_images/%s.jpg' % (img_id), 1) # plt.subplot(2, 1, 1) # plt.imshow(img) # plt.subplot(2, 1, 2) # plt.imshow(hmp.squeeze()) # plt.show() sub = pd.DataFrame({'image_id': image_ids, 'labels': labels}) return sub
def __init__(self, *args, **kwargs): """ Create a :class:`.ZipFileArchiver` instance. We create a new :class:`zipfile.ZipFile` and store it to the ``zipfile`` member. """ self.zipfile = ZipFile(*args, **kwargs)
def getZipNameInMemory(zipfile_obj: zipfile.ZipFile): files_in_zip = zipfile_obj.namelist() if len(files_in_zip) == 0: return None zip_name = zipfile_obj.filename if bool(zipfile_obj.filename) else 'nameless_file.zip' return pathlib.Path(zip_name).stem
if not os.path.exists(APPENGINE_TARGET_DIR): print('Downloading the AppEngine SDK...') #First try and get it from the 'featured' folder sdk_file = urlopen(FEATURED_SDK_REPO + APPENGINE_SDK_FILENAME) if sdk_file.getcode() == 404: #Failing that, 'deprecated' sdk_file = urlopen(DEPRECATED_SDK_REPO + APPENGINE_SDK_FILENAME) #Handle other errors if sdk_file.getcode() >= 299: raise Exception( 'App Engine SDK could not be found. {} returned code {}.'. format(sdk_file.geturl(), sdk_file.getcode())) zipfile = ZipFile(StringIO(sdk_file.read())) zipfile.extractall(TARGET_DIR) #Make sure the dev_appserver and appcfg are executable for module in ("dev_appserver.py", "appcfg.py"): app = os.path.join(APPENGINE_TARGET_DIR, module) st = os.stat(app) os.chmod(app, st.st_mode | stat.S_IEXEC) else: print( 'Not updating SDK as it exists. Remove {} and re-run to get the latest SDK' .format(APPENGINE_TARGET_DIR)) print("Running pip...") args = [ "pip", "install", "--no-deps", "-r", REQUIREMENTS_FILE, "-t",
def main(args): if len(args) < 1: sys.stderr.write("Error - one required argument: <data directory>\n") sys.exit(-1) working_dir = args[0] print("Reading data...") Y, outcome_map, outcome_list, X, feature_alphabet = ctk_io.read_multitask_token_sequence_data(working_dir) # ('data_testing/multitask_assertion/train_and_test') print("Shape of X is %s and Y is %s" % (str(X.shape), str(Y.shape))) num_examples, dimension = X.shape num_y_examples, num_labels = Y.shape assert num_examples == num_y_examples #print("Data has %d examples and dimension %d" % (num_examples, dimension) ) #print("Output has %d dimensions" % (num_labels) ) #X = np.reshape(X, (num_examples, 11, dimension / 11)) Y_adj, indices = ctk_io.flatten_outputs(Y) stopper = nn_models.get_early_stopper() output_dims_list = [] y_list = [] for i in range(len(indices)-1): label_dims = indices[i+1] - indices[i] output_dims_list.append(label_dims) if label_dims == 1: y_list.append(Y_adj[:, indices[i]]) else: y_list.append(Y_adj[:, indices[i]:indices[i+1]]) print("Dimensions of label %d are %s" % (i, str(y_list[-1].shape) ) ) model = nn_models.get_multitask_cnn(X.shape, len(feature_alphabet), output_dims_list, conv_layers=filters, fc_layers=layers, embed_dim=embed_dim, filter_widths=widths) #model = nn_models.get_multitask_mlp(X.shape, len(feature_alphabet), output_dims_list, fc_layers=layers, embed_dim=embed_dim) model.fit(X, y_list, nb_epoch=nb_epoch, batch_size=batch_size, verbose=1, validation_split=0.2, callbacks=[stopper]) model.summary() json_string = model.to_json() open(os.path.join(working_dir, 'model_0.json'), 'w').write(json_string) model.save_weights(os.path.join(working_dir, 'model_0.h5'), overwrite=True) #script_dir = os.path.dirname(os.path.realpath(sys.argv[0])) fn = open(os.path.join(working_dir, 'alphabets.pkl'), 'w') pickle.dump( (feature_alphabet, outcome_map, outcome_list), fn) fn.close() with ZipFile(os.path.join(working_dir, 'script.model'), 'w') as myzip: myzip.write(os.path.join(working_dir, 'model_0.json'), 'model_0.json') myzip.write(os.path.join(working_dir, 'model_0.h5'), 'model_0.h5') myzip.write(os.path.join(working_dir, 'alphabets.pkl'), 'alphabets.pkl')
def get_liste_pays(): with ZipFile('asia.zip', 'r') as fichier: return fichier.namelist()
def __init__(self, archpath): super(ZIPArchive, self).__init__(ZipFile(archpath))
def get_zipfile_attachment(self, email, attachment_file_name, zipfile_file_name): attachment = self.get_attachment(email, attachment_file_name) zipfile = ZipFile(attachment, 'r') return zipfile.open(zipfile_file_name, 'r')
return True else: return False genrecounts = dict() volumesread = dict() for folder in folderlist: thispath = os.path.join(rootpath, folder) filelist = os.listdir(thispath) for afile in filelist: if afile.endswith("maps.zip"): filepath = os.path.join(thispath, afile) with ZipFile(filepath, mode='r') as zf: for member in zf.infolist(): if not member.filename.endswith( '/') and not member.filename.endswith( "_Store") and not member.filename.startswith( "_"): datafile = ZipFile.open(zf, name=member, mode='r') filelines = datafile.readlines() filelines[0] = filelines[0].rstrip() htid = filelines[0].decode(encoding="UTF-8") thismap = list() counter = 0 for line in filelines[1:]: line = line.decode(encoding="UTF-8") line = line.rstrip()
def extractAll(zip): """ Extrait tous les fichiers d'une archive word dans un répertoire temporaire. """ myzip = ZipFile(zip, 'a') myzip.extractall('./../tmp/')