def extract_zip(infile): """ Extract the required parts of the QuinCe export ZIP """ manifest = None content = None if not zipfile.is_zipfile(infile): raise zipfile.BadZipFile('Supplied file is not a ZIP file') basename = os.path.splitext(os.path.basename(infile))[0] with zipfile.ZipFile(infile) as in_zip: manifest_path = zipfile.Path(in_zip, f'{basename}/manifest.json') if not manifest_path.exists(): raise KeyError('ZIP file is missing manifest.json') manifest = json.loads(manifest_path.read_text()) dataset_name = manifest['manifest']['metadata']['name'] dataset_filename = f'{basename}/dataset/SOCAT/{dataset_name}.tsv' dataset_path = zipfile.Path(in_zip, dataset_filename) if not dataset_path.exists(): raise KeyError('ZIP file does not contain SOCAT export') content = in_zip.read(dataset_filename) return manifest, content
def create(args): """Create a CMake project according to the provided information """ path = args.path root_exists = os.path.exists(path) if root_exists and os.path.isdir(path) and len(os.listdir(path)) != 0: print( f"Error - directory exists and is not empty:\n{path}", file=sys.stderr, ) exit(1) if args.flags_used: with contextlib.redirect_stdout(io.StringIO()): d = get_substitutes(args, os.path.basename(path)) else: d = get_substitutes(args, os.path.basename(path)) mkdir(path) mapping = {"s": "shared/", "e": "executable/", "h": "header/"} write_dir(path, d, zipfile.Path(zip, "templates/" + mapping[d["type_id"]])) write_dir(path, d, zipfile.Path(zip, "templates/common/")) git_init(path) print_tips(d) print("""\ Now make sure you have at least CMake 3.19 installed for local development, to make use of all the nice Quality-of-Life improvements in newer releases: https://cmake.org/download/ For more tips, like integration with package managers, please see the Wiki: http://github.com/friendlyanon/cmake-init/wiki You are all set. Have fun programming and create something awesome!""")
def test_1(act: Action, fdb_112_file: Path, fbk_file: Path): zipped_fdb_file = zipfile.Path(act.files_dir / 'core_6023-ods-11_2-fdb.zip', at='core_6023-ods-11_2.fdb') fdb_112_file.write_bytes(zipped_fdb_file.read_bytes()) # Change permissions fdb_112_file.chmod(16895) # Ensure that we really have deal with .fdb file with old ODS. act.expected_stderr = expected_stderr act.gstat(switches=['-h', str(fdb_112_file)], connect_db=False) assert act.clean_stderr == act.clean_expected_stderr # Backup work database and restore over extracted db act.reset() act.gbak(switches=['-b', act.db.dsn, str(fbk_file)]) act.reset() act.gbak(switches=['-rep', str(fbk_file), act.get_dsn(fdb_112_file)]) # act.reset() act.expected_stdout = expected_stdout act.isql( switches=['-q', act.get_dsn(fdb_112_file)], connect_db=False, input= 'set list on; select sign(current_connection) as restore_with_replace_result from rdb$database;' ) assert act.clean_stdout == act.clean_expected_stdout
def zip_children(self): zip_path = zipfile.Path(self.root) names = zip_path.root.namelist() self.joinpath = zip_path.joinpath return dict.fromkeys( child.split(posixpath.sep, 1)[0] for child in names)
def test_1(act: Action, blob_src: Path): zipped_blob_file = zipfile.Path(act.files_dir / 'core_5618.zip', at='core_5618.bin') blob_src.write_bytes(zipped_blob_file.read_bytes()) # with act.db.connect() as con: c = con.cursor() with open(blob_src, mode='rb') as blob_handle: c.execute('insert into test (b) values (?)', [blob_handle]) c.close() con.execute_immediate('drop table test') con.commit() # log_before = act.get_firebird_log() # Run full validation (this is what 'gfix -v -full' does) with act.connect_server() as srv: srv.database.repair(database=act.db.db_path, flags=SrvRepairFlag.FULL | SrvRepairFlag.VALIDATE_DB) assert srv.readlines() == [] # log_after = act.get_firebird_log() log_diff = [ line.strip().upper() for line in unified_diff(log_before, log_after) if line.startswith('+') and 'WARNING' in line.upper() ] assert log_diff == [ '+\tVALIDATION FINISHED: 0 ERRORS, 0 WARNINGS, 0 FIXED' ]
def maybe_extract_from_zipfile(zip_file): """ Extract files needed for Promtimer to run if necessary. Files needed by Promtimer are: * everything under the stats_snapshot directory; nothing is extracted if the stats_snapshot directory is already present * couchbase.log: extracted if not present """ root = zipfile.Path(zip_file) for p in root.iterdir(): if is_cbcollect_dir(p): stats_snapshot_exists = snapshot_dir_exists(pathlib.Path(p.name)) logging.debug("{}/stats_snapshot exists: {}".format( p.name, stats_snapshot_exists)) extracting = False for item in zip_file.infolist(): item_path = path.join(*item.filename.split('/')) should_extract = False if is_stats_snapshot_file(item.filename): should_extract = not stats_snapshot_exists elif item.filename.endswith(COUCHBASE_LOG): should_extract = not path.exists(item_path) if should_extract: logging.debug("zipfile item:{}, exists:{}".format( item_path, path.exists(item_path))) if not extracting: extracting = True logging.info( 'extracting stats, couchbase.log from cbcollect zip:{}' .format(zip_file.filename)) zip_file.extract(item)
def open(ifc_path: str) -> ifcopenshell.file: ext = os.path.splitext(ifc_path)[1].lower() if ext == ".ifc": return ifcopenshell.open(ifc_path) if ext == ".ifcxml": # TODO: How to do this as ifcopenshell.ifcopenshell_wrapper has no parse_ifcxml ? raise NotImplementedError("No support for .ifcXML yet") if ext in (".ifczip", ".zip"): try: # python 3.8+ zip_path = zipfile.Path(ifc_path) for member in zip_path.iterdir(): zipped_ext = os.path.splitext(member.name)[1].lower() if zipped_ext == ".ifc": return ifcopenshell.file.from_string( member.read_text()) if zipped_ext == ".ifcxml": raise NotImplementedError("No support for .ifcXML yet") except AttributeError as python36_zip_error: # python 3.6 with zipfile.ZipFile(ifc_path) as zip_file: for member in zip_file.namelist(): zipped_ext = os.path.splitext(member)[1].lower() if zipped_ext == ".ifc": with zip_file.open(member) as ifc_file: return ifcopenshell.file.from_string( ifc_file.read().decode()) if zipped_ext == ".ifcxml": raise NotImplementedError( "No support for .ifcXML yet" ) from python36_zip_error raise NotImplementedError("""Supported files : - unzipped : *.ifc | *.ifcXML - zipped : *.ifczip | *.zip containing un unzipped type""")
def set_zip(self, path): if Path(path).is_dir(): self.zip = ZipLikeFolder(path) spath = Path(path) self._is_folder = True else: self.zip = zipfile.ZipFile(path) # Find the root folder: spath = zipfile.Path(self.zip, "/") self._is_folder = False root = None for entry in spath.iterdir(): if root is None: if entry.is_dir(): root = entry.name else: # the first entry we found is not a dir root = None break else: # We found more than one entry in the root dir root = None break if root is None or self._is_folder: self.root = "" else: self.root = root + "/" if not self._is_folder: # Workaround for a weird bug where zipfile object considers itself closed # after the above zipfile.Path call self.zip = zipfile.ZipFile(path)
def _get_zpath(pudl_table, pudl_settings): """Given a table and pudl_settings, return a Path to the requested file.""" return zipfile.Path( pathlib.Path(pudl_settings["data_dir"], "local/ferc714/ferc714.zip"), TABLE_FNAME[pudl_table] )
def vcpkg(d): """Generate a vcpkg port in the ports folder, to make writing custom ports easier for dependencies on GitHub that are not present among the vcpkg provided ports """ path = os.path.join("ports", d["name"]) if os.path.isdir(path): print(f"""'{d["name"]}' already exists""", file=sys.stderr) exit(1) mkdir(path) write_dir(path, d, False, zipfile.Path(zip, "templates/vcpkg/")) vcpkg_root = "%VCPKG_ROOT:\=/%" if is_windows else "$VCPKG_ROOT" pwd = "%cd:\=/%" if is_windows else "$PWD" print(f"""\ The port has been generated in: {os.path.realpath(path)} Please navigate there and edit the placeholders that are surrounded with <angle brackets> and follow the instructions in the comments. After that's done, make sure you add the port to your vcpkg manifest for easy integration. When configuring your project, you must pass some extra variables: cmake --preset=dev -D "CMAKE_TOOLCHAIN_FILE={vcpkg_root}/scripts/buildsystems/vcpkg.cmake" -D "VCPKG_OVERLAY_PORTS={pwd}/ports" Make sure you always keep up-to-date with vcpkg documentation: https://github.com/microsoft/vcpkg See the following example for how integration with vcpkg should look like: https://github.com/friendlyanon/cmake-init-vcpkg-example""")
def test_1(act: Action, fbk_file: Path, db_tmp: Database, capsys): zipped_fbk_file = zipfile.Path(act.files_dir / 'core_5965.zip', at='core_5965.fbk') fbk_file.write_bytes(zipped_fbk_file.read_bytes()) # with act.connect_server() as srv: srv.database.restore(backup=fbk_file, database=db_tmp.db_path) srv.wait() # Test with db_tmp.connect() as con: c1 = con.cursor() c2 = con.cursor() c1.execute( "select 1 from opt_test where clid = 23 and cust_type = 1 and cust_id = 73 order by order_no desc" ) print(c1.statement.plan) # c2.execute( "select 2 from opt_test where sysid = 1 and clid = 23 and cust_type = 1 and cust_id = 73 order by order_no desc" ) print(c2.statement.plan) # Check act.expected_stdout = expected_stdout act.stdout = capsys.readouterr().out assert act.clean_stdout == act.clean_expected_stdout
def _read_metadata_resources(self): if self.dst_path.endswith(".charm"): metadata_path = zipfile.Path(self.dst_path) / "metadata.yaml" else: metadata_path = Path(self.dst_path) / "metadata.yaml" metadata = yaml.safe_load(metadata_path.read_text()) return metadata.get("resources", {})
def download(self, fname): """Fetch single file from associated store/charm/channel.""" if not self.full_entity: return None elif self.store == "cs": entity_p = self.full_entity.lstrip("cs:") url = f"https://api.jujucharms.com/charmstore/v5/{entity_p}/archive/{fname}" self.echo(f"Downloading {fname} from {url}") resp = requests.get(url) if resp.ok: return yaml.safe_load(resp.content.decode()) self.echo( f"Failed to read {fname} due to {resp.status_code} - {resp.content}" ) elif self.store == "ch": name, channel = self.full_entity.rsplit(":") info = _CharmHub.info( name, channel=channel, fields="default-release.revision.download.url") try: url = info["default-release"]["revision"]["download"]["url"] except (KeyError, TypeError): self.echo(f"Failed to find in charmhub.io \n{info}") return None self.echo(f"Downloading {fname} from {url}") resp = requests.get(url, stream=True) if resp.ok: yaml_file = zipfile.Path(BytesIO(resp.content)) / fname return yaml.safe_load(yaml_file.read_text()) self.echo( f"Failed to read {fname} due to {resp.status_code} - {resp.content}" )
async def get_translation(file_q,res_q): while True: lang_name, file = await file_q.get() with zipfile.ZipFile(file) as zip_file: # path = browser/chrome/{language abbreviation}/locale/browser/browser.properties path = zipfile.Path(zip_file,"browser/chrome/") lang_code = list(path.iterdir())[0].name # middle folder new_path = f"browser/chrome/{lang_code}/locale/browser/browser.properties" with zip_file.open(new_path) as browser_properties: translations = re.findall( r""" # userContextPersonal.label = Pa ngat moni userContext (Personal|Work|Banking|Shopping|None) \. label \ = \ (.*?) \n """, browser_properties.read().decode("utf-8"), flags=re.X ) logging.info(f"Got translations for {lang_name}.") await res_q.put((lang_name, lang_code, translations)) file_q.task_done()
def __parse_canopus(self): self.filename = None filename = None if self.maxZodiac: filename = self.maxZodiac elif self.maxSirius: filename = self.maxSirius self.topFormula = filename[3] self.topAdduct = filename[1] if self.maxZodiac is not None: self.zodiacScore = self.maxZodiac[2] else: self.zodiacScore = None canopusPath = Path(self.dirname, "canopus", filename[0] + "_" + filename[1] + ".fpt") canopusZipPath = Path(self.dirname, "canopus") if canopusPath.exists(): self.canopusfp = np.loadtxt(canopusPath) self.filename = canopusPath.name elif canopusZipPath.exists(): with zipfile.ZipFile(canopusZipPath) as myzip: path = zipfile.Path(myzip, at=filename[0] + "_" + filename[1] + ".fpt") if path.exists(): with path.open("r") as myfile: self.canopusfp = np.loadtxt(myfile) self.filename = canopusPath.name else: self.canopusfp = None else: self.canopusfp = None
def test_1(act: Action, sec_fbk: Path, sec_fdb: Path): zipped_fbk_file = zipfile.Path(act.files_dir / 'core_5637.zip', at='core5637-security3.fbk') sec_fbk.write_bytes(zipped_fbk_file.read_bytes()) # log_before = act.get_firebird_log() # Restore security database with act.connect_server() as srv: srv.database.restore(database=sec_fdb, backup=sec_fbk, flags=SrvRestoreFlag.REPLACE) restore_log = srv.readlines() # log_after = act.get_firebird_log() # srv.database.validate(database=sec_fdb) validation_log = srv.readlines() srv.database.shutdown(database=sec_fdb, mode=ShutdownMode.FULL, method=ShutdownMethod.FORCED, timeout=0) # # assert [line for line in restore_log if 'ERROR' in line.upper()] == [] assert [line for line in validation_log if 'ERROR' in line.upper()] == [] assert list(unified_diff(log_before, log_after)) == []
def _get_zpath(ferc714_table, pudl_settings): """ Given a table name and pudl_settings, return a Path to the corresponding file. Args: ferc714_table """ return zipfile.Path(get_ferc714(pudl_settings), TABLE_FNAME[ferc714_table])
def list_files_in_zip_file_from_a_directory( zip_file_path: str, path_inside_of_zipfile: str = '', filenames_to_ignore: Optional[List[str]] = None) -> Set[Path]: """ Return a list of files from a directory within a zipfile. The list of files returned is for a single directory within the zipfile. An empty string for path_inside_of_zipfile represents the root of the zip file. Sub directories are access by passing a string of the path inside the the zip file. 'sub_folder' will return files in a directory called sub_folder that is in the root of the zip file. 'sub_folder/sub_sub_folder' would return the files with in the sub_sub_folder directory of the zip file. If the directory does not exist an empty set is returned. The list of files paths returned can be further filtered by passing a list strings of names to ignore. Only exact matches are ignored. Passing ['file1.txt, 'file2.txt'] will exclude those files form the returned list if they exist. Parameters ---------- zip_file_path : str Path to a zip file. path_inside_of_zipfile : str default is empty string which returns files in the root of the zip file. if a non-empty string representing a path in the zip file is provided files in a directory matching that string will be returned. String must be posix format. filenames_to_ignore : Optional list[str] Optional list of stings of file names that can be ignored. Returns ------- list[Path] list containing paths to files in the zip file """ if not filenames_to_ignore: filenames_to_ignore = [] files_in_zip = set() try: zip_root_path = zipfile.Path(zip_file_path) folder_to_search = zip_root_path if path_inside_of_zipfile: folder_to_search = zip_root_path.joinpath(path_inside_of_zipfile) if not folder_to_search.exists(): logger.warning( f'Directory "{folder_to_search}" does not exist in zip file "{zip_file_path}"' ) return set() for item in folder_to_search.iterdir(): if item.is_file() and item.name not in filenames_to_ignore: files_in_zip.add(item) return files_in_zip except Exception as e: _error_handling(e, '', zip_file_path, '')
def test_package_zip(tmpdir): path = Path(tmpdir.mkdir("package")) (path / "foo.txt").write_text("foo") (path / "bar").mkdir() (path / "bar" / "baz.txt").write_text("baz") (path / ".qux").touch() pack_file = package_zip(path / "foo.txt") with zipfile.ZipFile(pack_file.name, "r") as zip_file: assert zipfile.Path(zip_file, "foo.txt").is_file() pack_dir = package_zip(path) h1 = gethash(Path(pack_dir.name)) with zipfile.ZipFile(pack_dir.name, "r") as zip_file: assert zipfile.Path(zip_file, "foo.txt").is_file() assert zipfile.Path(zip_file, "bar/").is_dir() assert zipfile.Path(zip_file, "bar/baz.txt").is_file() assert not zipfile.Path(zip_file, ".qux").exists() time.sleep(1) (path / "foo.txt").touch() pack_dir = package_zip(path) h2 = gethash(Path(pack_dir.name)) assert h1 == h2 pack_dir = package_zip(path, [path / "bar" / "baz.txt"]) with zipfile.ZipFile(pack_dir.name, "r") as zip_file: assert not zipfile.Path(zip_file, str(path / "foo.txt")).exists() assert zipfile.Path(zip_file, str(path / "bar" / "baz.txt")).is_file()
def fallback_resources(spec): package_directory = pathlib.Path(spec.origin).parent try: archive_path = spec.loader.archive rel_path = package_directory.relative_to(archive_path) return zipfile.Path(archive_path, str(rel_path) + '/') except Exception: pass return package_directory
def write_gui_documents(gui_document_by_path: Dict[str, bytes]) -> None: for path, gui_document_contents in gui_document_by_path.items(): gui_document_path = zipfile.Path(path, at='GuiDocument.xml') if gui_document_path.exists(): continue with ZipFile(path, 'a', ZIP_DEFLATED) as fcstd: member = ZipInfo('GuiDocument.xml', time.localtime()[:6]) member.compress_type = ZIP_DEFLATED fcstd.writestr(member, gui_document_contents)
def _local(cls, root='.'): from pep517 import build, meta system = build.compat_system(root) builder = functools.partial( meta.build, source_dir=root, system=system, ) return PathDistribution(zipfile.Path(meta.build_as_zip(builder)))
def zip_children(self): zip_path = zipfile.Path(self.root) names = zip_path.root.namelist() self.joinpath = zip_path.joinpath return ( posixpath.split(child)[0] for child in names )
def html_zip(self, tmp_path, html_path): if not hasattr(zipfile, 'Path'): pytest.skip("Needs zipfile.Path") zip_path = tmp_path / 'qutebrowser.zip' with zipfile.ZipFile(zip_path, 'w') as zf: for path in html_path.iterdir(): zf.write(path, path.relative_to(tmp_path)) yield zipfile.Path(zip_path) / 'qutebrowser'
def test_importlib_resource_load_zip_path() -> None: config_source = ImportlibResourcesConfigSource(provider="foo", path="pkg://bar") conf = config_source._read_config( zipfile.Path( # type: ignore "hydra/test_utils/configs/conf.zip", "config.yaml", )) assert conf.config == {"foo": "bar"} assert conf.header == {"package": None}
def test_1(act: Action, fbk_file: Path, fdb_file: Path): zipped_fbk_file = zipfile.Path(act.files_dir / 'core_5078.zip', at='tmp_core_5078.fbk') fbk_file.write_bytes(zipped_fbk_file.read_bytes()) with act.connect_server() as srv: srv.database.restore(database=fdb_file, backup=fbk_file) srv.wait() # This should execute without errors act.isql(switches=[str(fdb_file)], input='set list on; select * from do_changeTxStatus;', connect_db=False)
def describe_all_csvs_in_zips_0(fs): for zip_name in get_zips(fs): my_zip = zipfile.ZipFile("/tmp/dl.zip") print(zip_name) for zip_info in my_zip.infolist(): if not zip_info.filename.endswith(".csv"): continue print(zip_info.filename) my_zip_open = zipfile.ZipFile("/tmp/dl.zip") df = pd.read_csv( zipfile.Path(my_zip_open, zip_info.filename).open()) print(df.describe())
def __parse_npc(self): self.canopusnpc = None if self.filename: npcPath = Path(self.dirname, "canopus_npc", self.filename) npcZipPath = Path(self.dirname, "canopus_npc") if npcPath.exists(): self.canopusnpc = np.loadtxt(npcPath) elif npcZipPath.exists(): with zipfile.ZipFile(npcZipPath) as myzip: path = zipfile.Path(myzip, at=self.filename) if path.exists(): with path.open("r") as myfile: self.canopusnpc = np.loadtxt(myfile)
def files(package): spec = import_module(package).__spec__ if spec.submodule_search_locations is None: raise TypeError("{!r} is not a package".format(package)) package_directory = pathlib.Path(spec.origin).parent try: archive_path = spec.loader.archive rel_path = package_directory.relative_to(archive_path) return zipfile.Path(archive_path, str(rel_path) + "/") except Exception: pass return package_directory
def _read(self, dirs, filename, binary=False): ''' Return text/binary contents of a file, None if file does not exist. ''' dirpath = '/'.join(dirs) + '/' + filename if len(dirs) else '' path = zipfile.Path(self._zipfile, dirpath + filename) if path.exists(): if binary: return path.read_bytes() else: with path.open() as f: return '\n'.join(f.readlines()) else: return None