def test_convert_pdf_page_to_web(tmp_fixture_dir): """Convert single page pdf to web version""" pdf = Path(tmp_fixture_dir) / 'PDF' / PAGE_ONE # Make new file opt = convert_pdf_to_web(pdf) # Default file location assert opt == pdf.parent / 'WEB' / PAGE_ONE assert opt.exists() # Conversion changes file content assert opt.stat().st_size != pdf.stat().st_size # Calling again should not create new file mtime = opt.stat().st_mtime opt2 = convert_pdf_to_web(pdf) assert opt2.stat().st_mtime == mtime # If pdf if changed, convert should rerun pdf.touch() opt3 = convert_pdf_to_web(pdf) assert opt3.stat().st_mtime > mtime # Nonexisting file should raise error false_pdf = pdf.with_name('nonexisting') assert not false_pdf.is_file() with pytest.raises(FileNotFoundError): convert_pdf_to_web(false_pdf)
def test_open(w): mask = IN_OPEN | IN_CLOSE w.add('testfile', mask) watch = w._paths[P('testfile')] assert len(watch.links) == 2 assert watch.path == P('testfile') assert watch.watcher == w assert watch.mask == mask link1 = watch.links[0] assert link1.idx == 0 assert link1.path == str(P.cwd()) assert link1.rest == 'testfile' assert link1.mask == IN_UNMOUNT | IN_ONLYDIR | IN_EXCL_UNLINK | IN_IGNORED | IN_MOVE | IN_DELETE | IN_CREATE assert link1.watch == watch wd = link1.wd assert wd.callbacks['testfile'] == [link1] assert wd.mask == link1.mask assert wd.watcher == w watchdesc = wd.wd assert w._watchdescriptors[watchdesc] == wd link2 = watch.links[1] assert link2.idx == 1 assert link2.path == str(P.cwd()['testfile']) assert link2.rest == '.' assert link2.mask == IN_OPEN | IN_CLOSE assert link2.watch == watch wd = link2.wd assert wd.callbacks[None] == [link2] assert wd.mask == link2.mask assert wd.watcher == w watchdesc = wd.wd assert w._watchdescriptors[watchdesc] == wd open('testfile').close() evts = w.read(block=False) ev1, ev2 = evts assert ev1.open assert ev2.close assert ev2.close_nowrite os.remove('testfile') ev3 = w.read(block=False)[0] assert ev3.path_delete and ev3.path_changed assert ev3.path == 'testfile' assert P(ev3.name).parts[-1] == 'testfile' w.close()
def key_file(self, value): if self.connected: raise SFTPSetOnConnectedError('key_file', value) kfile = None if value is None: kfile = DEFAULT_SSH_KEY else: if isinstance(value, Path): kfile = value else: kfile = PosixPath(str(value)) if str(kfile).startswith('~'): kfile = kfile.expanduser() self._key_file = kfile
def test_multi(w): open('file2', 'w').close() os.symlink('file2', 'link2') os.symlink(str(P.cwd()['link2']), 'link3') os.symlink('testfile', 'link4') w.add('link3', IN_OPEN) w.add('link4', IN_OPEN) open('file2').close() evts = w.read() assert len(evts) == 1 e = evts[0] assert e.open assert e.path == 'link3' open('testfile').close() evts = w.read() assert len(evts) == 1 e = evts[0] assert e.open assert e.path == 'link4' assert len(w._watchdescriptors) == 5 os.remove('link3') os.symlink('link4', 'link3') evts = w.read() open('testfile').close() evts.extend(w.read()) assert len(evts) == 4
def _naive_relative_to(path: PosixPath, root: PosixPath) -> PurePosixPath: """ Compute relative PurePosixPath, result may include '..' parts. Both arguments must be absolute PurePosixPath's and lack '..' parts. Possibility of symlinks is ignored, i. e. arguments are interpreted as resolved paths. """ if not path.is_absolute(): raise ValueError(path) if not root.is_absolute(): raise ValueError(root) if '..' in path.parts: raise ValueError(path) if '..' in root.parts: raise ValueError(root) upstairs = 0 while root not in path.parents: parent = root.parent assert parent != root root = parent upstairs += 1 return PurePosixPath( * (('..',) * upstairs), path.relative_to(root) )
def search_dir(d: PosixPath, basepath, files, extension): """Search directories recursively for repo files.""" scripts = [] for path in d.iterdir(): string_path = str(path).replace(basepath, '')[1:] if path.is_dir(): recu_scripts = search_dir(path, basepath, files, extension) if recu_scripts: scripts.extend(recu_scripts) if string_path in files and path.suffix == extension: scripts.extend([string_path]) return scripts
def local_dir(self, value): if value is None: raise ValueError("The local directory may not be set to None.") ldir = None if isinstance(value, Path): ldir = value else: ldir = PosixPath(str(value)) if str(ldir).startswith('~'): ldir = ldir.__class__(os.path.expanduser(str(ldir))) if self.connected: if not ldir.is_dir(): msg = "Directory %r does not exists or is not a directory." % (ldir) raise SFTPLocalPathError(msg) os.chdir(str(ldir)) self.base_dir = str(ldir) self._local_dir = ldir
def main(): # type: () -> None import argparse parser = argparse.ArgumentParser( description="Windows Explorer" ) parser.add_argument( "posixpath", nargs='?', default='.', ) args = parser.parse_args() call_explorer( posixpath=PosixPath(args.posixpath) )
def test_report_summary_no_warnings(capsys): """ Given: - Lint manager dictionary with one pack which has warnings. When: - Creating summary of the lint. Then: - Ensure that there are no warnings printed in the summary and all passed. """ from demisto_sdk.commands.lint import lint_manager lint_status = { 'fail_packs_flake8': [], 'fail_packs_XSOAR_linter': [], 'fail_packs_bandit': [], 'fail_packs_mypy': [], 'fail_packs_vulture': [], 'fail_packs_pylint': [], 'fail_packs_pytest': [], 'fail_packs_pwsh_analyze': [], 'fail_packs_pwsh_test': [], 'fail_packs_image': [], 'warning_packs_flake8': [], 'warning_packs_XSOAR_linter': [], 'warning_packs_bandit': [], 'warning_packs_mypy': [], 'warning_packs_vulture': [], 'warning_packs_pylint': [], 'warning_packs_pytest': [], 'warning_packs_pwsh_analyze': [], 'warning_packs_pwsh_test': [], 'warning_packs_image': [] } pkg = [ PosixPath( '/Users/test_user/dev/demisto/content/Packs/Maltiverse/Integrations/Maltiverse' ) ] lint_manager.LintManager.report_summary(pkg=pkg, lint_status=lint_status) captured = capsys.readouterr() assert "Packages PASS: \x1b[32m1\x1b[0m" in captured.out assert "Packages WARNING (can either PASS or FAIL): \x1b[33m0\x1b[0m" in captured.out assert "Packages FAIL: [31m0[0m" in captured.out
def installNeoVim(args): print("Checking for Vim directory") vimDir = PosixPath("~/.config/nvim/").expanduser() if not vimDir.exists(): print("Neovim config dir not found, need to install...") else: copyfile(PosixPath("./shared/init.vim"), PosixPath("~/.config/nvim/init.vim").expanduser()) copyfile(PosixPath("./shared/plug.vim"), PosixPath("~/.config/nvim/autoload/plug.vim").expanduser()) os.system("python3 -m pip install --user --upgrade pynvim") print("Additional Notes:") print("================") print("Install universal-ctags...") print("https://github.com/universal-ctags/ctags") print("Install the coc-json and coc-tsserver in nvim:") print(":CocInstall coc-json coc-tsserver")
def run(self): if not self.user_authorize('system', 'manage'): return while True: task = self.task_q.get() logging.info(task) if task[0] == 'UPLOAD': p = PosixPath(task[1]) if p.is_dir(): target_dir = os.path.join(task[2], p.name) if self.do_mkdir(p.name, task[2]): children = [d for d in p.iterdir()] children.sort() for f in children: self.task_q.put(['UPLOAD', str(f), target_dir]) else: self.do_upload(str(p), task[2]) elif task[0] == 'DOWNLOAD': R = self._stat(task[1]) if R["fileType"]==1: #dir p = PosixPath(task[2]).joinpath(R["name"]=="" and "ROOT" or R["name"]) p.mkdir() target_dir = os.path.join(task[2], p.name) for r in self._stat2(task[1]): self.task_q.put(['DOWNLOAD', r["path"], target_dir]) else: self.do_download(task[1], task[2]) elif task[0] == 'MKDIR': self.do_mkdir(os.path.basename(task[1]), os.path.dirname(task[1])) elif task[0] == 'STAT': self.do_stat(task[1]) elif task[0] == 'LS': self.do_ls(task[1]) elif task[0] == 'LS-R': self.do_ls_r(task[1]) elif task[0] == 'RENAME': self.do_rename(task[1], task[2]) elif task[0] == 'REMOVE': self.do_delete(task[1]) elif task[0] == 'EXIT': self.close() logging.info("Exit.") self.task_q.task_done() return else: msg = "Invalid task: %s" % str(task) logging.error(msg) TestClient.ERRORS.append(msg) self.task_q.task_done()
def get(self, resource: Text, path: Text, location_type: Optional[DataLocationType] = None ) -> Set[DataLocation]: resource = resource or LOCAL_RESOURCE node = self._filesystems.get(resource) if not node: return set() path = Path(path) if resource == LOCAL_RESOURCE else PosixPath(path) for token in path.parts: if token in node.children: node = node.children[token] else: return set() return ({ loc for loc in node.locations if loc.location_type == location_type } if location_type else node.locations)
def __init__( self, path: PosixPath, text: str, *, build_dir_node: DirectoryNode, name: Optional[str] = None, needs: Iterable[Node] = (), ) -> None: if path.parent != build_dir_node.path: raise RuntimeError(path) var_name = '{name}.{hash}'.format( name=path.name, hash=text_hash(text) ) var_text_node = VarTextNode( path=path.with_name(var_name), text=text, name='{}:var'.format(name), needs=(build_dir_node,) ) if isinstance(build_dir_node, BuildDirectoryNode): build_dir_node.register_node(var_text_node) super().__init__( source=var_text_node, path=path, name=name, needs=(*needs, build_dir_node) ) self.text = text
def resolve_path(path): '''Resolve the symlinks in path, yielding all filesystem locations that are traversed. The yielded value is a tuple, of which the first element is a symlink-free path, and the second element is a path relative to the first element that has not yet been traversed. This second element may contain more symlinks. The resolution implementation will follow an unbounded number of symlinks but will still detect symlink loops if they prevent a path from resolving. path can be given as a string or as a pathlib object. The yielded values are pathlib.PosixPath objects. ''' linkcache = {} linkcounter = [0] for p in resolve_symlink(_curdir, PosixPath(path), set(), linkcache, linkcounter): yield p
def test_experiment_output(self) -> None: config = Config(LABBY_CONFIG_YAML) sequence = ExperimentSequence("./sequences/seq.yaml", SEQUENCE_YAML) runner = ExperimentRunner(config, sequence) with patch_time("2020-08-08"), patch_file_contents( "output/seq/000.csv") as output, patch( "os.makedirs") as makedirs: runner.start() runner.join() makedirs.assert_called_with(PosixPath("output/seq/"), exist_ok=True) self.assertEqual(len(output.write.call_args_list), 4) output.write.assert_has_calls([ call("seconds,voltage\n"), call("0.0,15.0\n"), call("0.5,15.0\n"), call("1.0,15.0\n"), ])
def select_model_by_facet_value( facet_value: str, root: PosixPath = Path("/kbdata/Processed/Models/")) -> dict: """Select models over time by a specific facet value. Arguments: facet_value (str): selected facet value, e.g. 'Katholiek' root (PosixPath): the folder where all models are stored Returns: a dictionary that maps year to a path """ models = root.glob(f"*-{facet_value}.w2v.model") out = {} for m in models: start = m.stem.lstrip("FT-").split('-')[0] out[int(start)] = m return out
def test_generate_snap_graph_no_speckle_filter(safe_file): params = {"mask": ["sea"], "tcorrection": False, "speckle_filter": False} SNAPPolarimetry(params).generate_snap_graph( safe_file.feature, "VV", "/tmp/input/S1B_IW_GRDH_1SDV_" "20190220T050359_20190220T050424_015025_01C12F_4EA4.SAFE_vv", ) graph_xml_file = PosixPath( "/tmp/S1B_IW_GRDH_1SDV_" "20190220T050359_20190220T050424_015025_01C12F_4EA4.SAFE_VV.xml") tree = ET.parse(str(graph_xml_file)) all_nodes = tree.findall("node") node_id_list = [graph_node.attrib["id"] for graph_node in all_nodes] assert "Speckle-Filter" not in node_id_list assert "LinearToFromdB" in node_id_list
class API: section = 'api' TITLE = config.get(section, 'title', fallback='TensorHive API') URL_SCHEMA = config.get(section, 'url_schema', fallback='http') URL_HOSTNAME = config.get(section, 'url_hostname', fallback='0.0.0.0') URL_PORT = config.get(section, 'url_port', fallback='1111') URL_PREFIX = config.get(section, 'url_prefix', fallback='api') SPEC_FILE = config.get(section, 'spec_file', fallback='api_specification.yml') IMPL_LOCATION = config.get(section, 'impl_location', fallback='tensorhive.api.controllers') import yaml respones_file_path = str( PosixPath(__file__).parent / 'controllers/responses.yml') with open(respones_file_path, 'r') as file: RESPONSES = yaml.safe_load(file)
def test_nested_album_art_album_layout_case_2a(album_layout_case_2a): """Find nested album art of case 2a.""" for instance in album_layout_case_2a: with mock.patch("os.walk", return_value=instance): album_path = instance[0][0] directories_in_album_directory = instance[0][1] with mock.patch("os.listdir", return_value=directories_in_album_directory): with mock.patch("os.path.isfile", return_value=False): result = teeb.find.nested_album_art(album_path) expected = { "case1": [], "case2": [{ "art_dir": f"{album_path}/album_art", "art_files": ["cover.jpg", "back.jpg"], "parent_dir": PosixPath(album_path), }], } assert result == expected
def __init__(self, path: pathlib.PosixPath): self.name = path.name self.image_paths = [] self.preview_path = path.parent / "preview.jpg" for image_path in sorted([ x for x in path.glob("./*") if x.is_file() and any( str(x).endswith(ext) for ext in [".png", ".jpg", ".jpeg"]) ]): if image_path.name.startswith("preview"): self.preview_path = image_path else: self.image_paths.append(image_path) self.num_items = len(self.image_paths) self.image_width, self.image_height = self._validate_image_dimensions() self.images_per_row = math.ceil(math.sqrt(self.num_items)) self.images_per_column = (math.floor( self.num_items / self.images_per_row) if self.num_items % self.images_per_row == 0 else math.ceil( self.num_items / self.images_per_row))
def train_fold(scandir, datadir, use_tptrw, n_splits): """Perform a folded training based on a hyperparameter scan result.""" from tdub.ml_train import folded_training, prepare_from_root from tdub.data import quick_files scandir = PosixPath(scandir).resolve() summary_file = scandir / "best" / "summary.json" outdir = scandir / "foldres" if outdir.exists(): log.warn(f"fold result already exists for {scandir}, exiting") return 0 summary = None with summary_file.open("r") as f: summary = json.load(f) nlo_method = summary["nlo_method"] best_iteration = summary["best_iteration"] if best_iteration > 0: summary["all_params"]["n_estimators"] = best_iteration region = summary["region"] branches = summary["features"] selection = summary["selection_used"] qf = quick_files(datadir) df, y, w = prepare_from_root( qf[f"tW_{nlo_method}"], qf["ttbar"], region, override_selection=selection, branches=branches, weight_mean=1.0, use_tptrw=use_tptrw, ) folded_training( df, y, w, summary["all_params"], {"verbose": 10}, str(outdir), summary["region"], kfold_kw={"n_splits": n_splits, "shuffle": True}, ) return 0
def download_audioset_sound_files(path_to_audioset_folder, data_type, num_classes): """ :param path_to_audioset_csv_folder: :param data_type: :return: """ csv_data = pd.read_csv( os.path.join(path_to_audioset_folder, 'csv', '{}_segments_{}.csv'.format(data_type, num_classes))) # Create the folder in which we store the sound files sound_folder = Path(path_to_audioset_folder) / PosixPath(data_type) os.makedirs(os.path.join(sound_folder, 'wav'), exist_ok=True) for idx in range(len(csv_data)): video_id = csv_data['YTID'][idx] start_time = csv_data['start_seconds'][ idx] + 1.0 # remove sound files borders end_time = csv_data['end_seconds'][ idx] - 1.0 # remove sound files borders sys.stdout.write('\r Video {}/{} information: {} {} {}'.format( idx, len(csv_data), video_id, start_time, end_time)) os.system( 'youtube-dl "{id}" --quiet --extract-audio --audio-format wav ' '--output "{folder}/wav/{outname}"'.format( id='https://youtube.com/watch?v=' + video_id, outname=video_id + '.%(ext)s', folder=sound_folder)) os.system( 'ffmpeg -loglevel quiet -i "{folder}/wav/{outname}.wav" ' '-ar "44100" -ss "{start}" -to "{end}" "{folder}/wav/{outname}_out.wav"' .format(outname=video_id, start=start_time, end=end_time, folder=sound_folder)) os.system( 'mv "{folder}/wav/{outname}_out.wav" "{folder}/wav/{outname}.wav"'. format(outname=video_id, folder=sound_folder))
def test_get_stats(self): project = Project(project_name, workspace=workspace, epsg=epsg) pointcloud = project.add_new_pointcloud(point_cloud_name, file_format='las') names_polygons = misc.get_names_and_polygons_in_workspace(workspace, settings={'step': 25, 'x_pos': 3, 'y_pos': 4}) for data in names_polygons: pointcloud.add_tile(data['name'], data['polygon']) stats = project.get_stats() true_values = {'name': 'Test', 'num_pointclouds': 1, 'workspace': PosixPath('test_data'), 'pointclouds': {'p1': {'area': 186.82999999999998, 'class_frequency': {2: 17171, 3: 4760}, 'num_points': 21931, 'density': 117.38, 'tiles': 2}}} self.assertEqual(true_values, stats)
def test_put_updates_package_version_data(self, app, client, test_media_directory): n = Namespace(name='Hello') p = Package.create(name='Dog Bog', namespace=n) version = '1.0.0' bytes1 = b'not a zip' response = client.put( f"/api/v1/namespaces/{n.slug}/{p.slug}/{version}", data={"file": (BytesIO(bytes1), "workstation.zip")}) assert response.status_code == 200 response = client.put( f"/api/v1/namespaces/{n.slug}/{p.slug}/{version}", json={"local": "C:/bobo/"}) assert response.status_code == 200 version = PackageVersion.query.one() assert version.local == PosixPath("C:/bobo/")
def find_jupyter_command(proc: str) -> Optional[JupyterCommand]: log = logging.getLogger(__name__).getChild("auto") parts = shlex.split(proc) if parts[0] in ("pgrep", "xargs"): return None python = parts[0] log.debug("Python candidate = {!r}".format(parts)) for p in parts[1:]: if p.endswith("jupyter-notebook"): jupyter = p break if p.endswith("jupyter-lab"): jupyter = str(PosixPath(p).parent / "jupyter-notebook") break else: log.debug("Can't find jupyter notebook in candidate = {}".format(proc)) return None return JupyterCommand(python, jupyter)
def load_dataset(dataset, training_chip_size, bs): """ Load a dataset, create batches and augmentation """ path = PosixPath(dataset) label_path = path / 'label-chips' image_path = path / 'image-chips' image_files = get_image_files(image_path) label_files = get_image_files(label_path) get_y_fn = lambda x: label_path / f'{x.stem}{x.suffix}' codes = np.array(LABELS) src = SegmentationItemList.from_folder(image_path).split_by_fname_file( '../valid.txt').label_from_func(get_y_fn, classes=codes) # some data augmentation here data = src.transform(get_transforms(flip_vert=True, max_warp=0., max_zoom=0., max_rotate=180.), size=training_chip_size, tfm_y=True).databunch(bs=bs) return data
async def test_get_log_by_name(self, client, logs_path): log_filepath = Mock() log_filepath.open = mock_open() log_filepath.is_file.return_value = True log_filepath.stat.return_value = MagicMock() log_filepath.stat.st_size = 1024 filepath = Mock() filepath.name = '190801-13-21-56.log' filepath.open = mock_open() filepath.with_name.return_value = log_filepath with patch.object(package_log, '_get_logs_dir', return_value=logs_path): with patch('os.walk'): with patch("aiohttp.web.FileResponse", return_value=web.FileResponse(path=filepath)) as f_res: resp = await client.get('/foglamp/package/log/{}'.format(filepath.name)) assert 200 == resp.status assert 'OK' == resp.reason args, kwargs = f_res.call_args assert {'path': PosixPath(pathlib.Path("{}/{}".format(logs_path, filepath.name)))} == kwargs assert 1 == f_res.call_count
def apksign(p_apk: PosixPath, key_path: str, key_alias: str, key_pass: str, ks_pass: str) -> PosixPath: try: signed_apk_name = p_apk.name.replace("-zipaligned.apk", "-signed.apk") psigned_apk = p_apk.parent.joinpath(signed_apk_name) key_cmd = [ '--ks', key_path, '--ks-key-alias', key_alias, '--ks-pass', 'pass:{}'.format(ks_pass), '--key-pass', 'pass:{}'.format(key_pass) ] cmd = [config.apksigner, 'sign'] + key_cmd + [ '--out', str(psigned_apk.resolve()), str(p_apk.resolve()) ] r = check_output(cmd) except Exception as e: print("apk signing error: " + str(e)) return False return psigned_apk
def gen_private_key(key_size: int, key_out: pathlib.PosixPath) -> rsa.RSAPrivateKey: # noqa # check if the private key file already exists if key_out.exists(): emsg = 'File %s already exists' % key_out raise Exception(emsg) # generate private key key = rsa.generate_private_key(public_exponent=65537, key_size=key_size) # write to file with open(key_out, "wb") as fp: fp.write( key.private_bytes( encoding=serialization.Encoding.PEM, format=serialization.PrivateFormat.TraditionalOpenSSL, encryption_algorithm=serialization.NoEncryption())) fp.close() return key
def create_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description="Generate the Pants reference markdown files.") parser.add_argument( "--sync", action="store_true", default=False, help="Whether to sync the generated reference docs to the docsite. " "If unset, will generate markdown files to the path in --output " "instead. If set, --api-key must be set.", ) parser.add_argument( "--output", default=PosixPath(os.path.sep) / "tmp" / "pants_docs" / "help" / "option", type=Path, help="Path to a directory under which we generate the markdown files. " "Useful for viewing the files locally when testing and debugging " "the renderer.", ) parser.add_argument("--api-key", help="The readme.io API key to use. Required for --sync.") return parser
def read_id_path(cls, classification_run: ClassificationRunId, read_id: ReadId) -> str: """Path to the group that contains the classification results for a given readId. Parameters ---------- classification_run : ClassificationRunId A unique identifier for the classification run that generated these results (e.g. "my_classication_run_04"). read_id : ReadId The readId of the read we want to know the classification results for. Returns ------- str Path to the group that contains the classification results for a given readId. """ CLASSICATION_RUN_PATH = cls.for_classification_run(classification_run) path = str(PosixPath(CLASSICATION_RUN_PATH, f"{read_id}")) return path
def job_run_ks2(): # Look for flag files in /mnt/s0/Data and sort them in order of date they were created flag_files = list(Path(root_path).glob('**/sort_me.flag')) flag_files.sort(key=os.path.getmtime) # Start with the oldest flag session_path = flag_files[0].parent session = str(PosixPath(*session_path.parts[4:])) flag_files[0].unlink() # Instantiate one one = ONE(cache_rest=None) # sync the probes status, sync_files = sync_probes.sync(session_path) if not status: _logger.error(f'{session}: Could not sync probes') return else: _logger.info(f'{session}: Probes successfully synced') # run ks2 task = ephys.SpikeSorting(session_path, one=one) status = task.run() if status != 0: _logger.error(f'{session}: Could not run ks2') return else: _logger.info(f'{session}: ks2 successfully completed') # Run the cell qc # qc_file = [] # Register and upload files to FTP Patcher outfiles = task.outputs ftp_patcher = FTPPatcher(one=one) ftp_patcher.create_dataset(path=outfiles, created_by=one._par.ALYX_LOGIN)
def test_run_sequence(self) -> None: SEQUENCE_CONTENTS = """ --- sequence: - experiment_type: labby.tests.test_server.TestExperiment - experiment_type: labby.tests.test_server.TestExperiment """ with patch_file_contents( "sequence/test.yml", SEQUENCE_CONTENTS ), patch_file_contents("output/test/000.csv") as output_0, patch_file_contents( "output/test/001.csv" ) as output_1, patch( "os.makedirs" ) as makedirs, patch_time( "2020-08-08" ): self.client.run_sequence("sequence/test.yml") while True: sequence_status = self.client.experiment_status().sequence_status if sequence_status and sequence_status.is_finished(): break time.sleep(0) makedirs.assert_called_with(PosixPath("output/test/"), exist_ok=True) self.assertEqual(len(output_0.write.call_args_list), 4) output_0.write.assert_has_calls( [ call("seconds,voltage\n"), call("0.0,15.0\n"), call("0.5,15.0\n"), call("1.0,15.0\n"), ] ) output_1.write.assert_has_calls( [ call("seconds,voltage\n"), call("0.0,15.0\n"), call("0.5,15.0\n"), call("1.0,15.0\n"), ] )
def volume_dict_pairs_to_str(args, keyval_arr): key = keyval_arr[0] val = keyval_arr[1] if key == 'read_only': return 'readonly=' + str(val) if key == 'source': if val.startswith('~'): path = PosixPath(str(val)).expanduser() return 'source=' + str(os.path.normpath(path.absolute())) if val.startswith('.'): dirname = os.path.dirname(args.composefile) path = PosixPath(dirname, str(val)) return 'source=' + str(os.path.normpath(path.absolute())) return 'source=' + str(val) return key + '=' + str(val)
def condor_submit(workspace: str | os.PathLike) -> None: """Execute condor_submit on the condor.sub file in a workspace. Parameters ---------- workspace : str or os.PathLike the workspace containing the condor.sub file """ ws = PosixPath(workspace).resolve() proc = subprocess.Popen( ["condor_submit", str(ws / "condor.sub")], stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) out, err = proc.communicate() try: log_out = out.decode("utf-8") except AttributeError: pass log.info(log_out)
def _init_skeleton( self, input_files: List[str], name: Optional[str] = None, tree_name: str = "WtLoop_nominal", weight_name: str = "weight_nominal", label: Optional[int] = None, auxlabel: Optional[int] = None, TeXlabel: Optional[str] = None, ) -> None: """Default initialization - should only be called by internal staticmethods ``from_root``, ``from_pytables``, ``from_h5`` Parameters ---------- input_files: List of input files name: Name of the dataset (if none use first file name) tree_name: Name of tree which this dataset originated from weight_name: Name of the weight branch label: Give dataset an integer based label auxlabel: Give dataset an integer based auxiliary label TeXlabel: LaTeX form label """ self._files = [PosixPath(f).resolve(strict=True) for f in input_files] if name is None: self.name = str(self.files[0].parts[-1]) else: self.name = name self.weight_name = weight_name self.tree_name = tree_name self.label = label self.auxlabel = auxlabel self.TeXlabel = TeXlabel
def qc_report_run_info(run_path: Path, qc_report_glob='*_QCreport*.csv'): """get general run info from GT's QCreport file""" try: current_app.logger.info('Parsing QCreport file') qc_info = {} qc_report_fieldnames = [ 'Project', # 'Application', 'Sequence Protocol', 'Sample Size', 'Fastq Files', 'Date Report', ] # N.B. line formats of this report are: "Project: 18-weinstock-005,,,,," qc_report_list = list(run_path.glob(qc_report_glob)) # current_app.logger.debug('qc_report_list: %s', qc_report_list) qc_report_csv = qc_report_list[0] qcr_lines = read_file_text(qc_report_csv) qc_info = {'GT Project': None} # first item in display qcr_rows = [r.split(',') for r in qcr_lines] # current_app.logger.debug('length qcr_rows: %s', len(qcr_rows)) for row in qcr_rows: # current_app.logger.debug('qcr_row: %s', str(row)) for fld in qc_report_fieldnames: if fld in row[0]: # current_app.logger.debug('fld: %s', fld) [f1, f2] = row[0].replace(',', '').split(': ') # current_app.logger.debug('f1,f2: %s, %s', f1, f2) qc_info.update({f1: f2}) qc_info['GT Project'] = qc_info.pop('Project', None) current_app.logger.debug('qc_info: %s', qc_info) except Exception as e: current_app.logger.exception('reading from run' 's QCreport csv file!') finally: return qc_info
def on_epoch_end(self, epoch, logs=None): """ Note: We immediately increment epoch from index-from-0 to index-from-1 to match the TensorFlow output. Normally, ckpts/best is the best saved state, and ckpts/last is the last saved state. Procedure: 1. Write current state to ckpts/work 2. Rename ckpts/work to ckpts/epoch/NNN 3. If best, link ckpts/best to ckpts/epoch/NNN 4. Link ckpts/last to ckpts/epoch/NNN 5. Clean up old ckpts according to keep policy """ epoch += 1 dir_root = PosixPath(self.ckpt_directory).resolve() dir_work = dir_root / "ckpts/work" dir_best = dir_root / "ckpts/best" # a soft link dir_last = dir_root / "ckpts/last" # a soft link dir_epochs = dir_root / "ckpts/epochs" dir_this = dir_epochs / ("%03i" % epoch) if not self.save_check(logs, epoch): return if os.path.exists(dir_this): self.debug("remove: '%s'" % self.relpath(dir_this)) shutil.rmtree(dir_this) os.makedirs(dir_epochs, exist_ok=True) os.makedirs(dir_work, exist_ok=True) self.write_model(dir_work, epoch) self.debug("rename: '%s' -> '%s'" % (self.relpath(dir_work), self.relpath(dir_this))) os.rename(dir_work, dir_this) self.epochs.append(epoch) if self.epoch_best == epoch: self.symlink(dir_this, dir_best) self.symlink(dir_this, dir_last) self.clean(epoch)
def files_on_index( index_url: str, project_name: str) -> Iterator[Tuple[str, Optional[Tuple[str, str]]]]: """Iterate files available on an index for a given project name.""" project_name = project_name.replace("_", "-") base_url = urljoin(index_url, project_name + "/") r = requests.get(base_url) if r.status_code == 404: # project not found on this index return r.raise_for_status() parser = etree.HTMLParser() tree = etree.parse(StringIO(r.text), parser) for a in tree.iterfind("//a"): parsed_url = urlparse(a.get("href")) p = PosixPath(parsed_url.path) if parsed_url.fragment: hash_type, hash_value = parsed_url.fragment.split("=", 2)[:2] yield p.name, (hash_type, hash_value) else: yield p.name, None
def relative_to(self, *other): # produce a PosixPath object again result = super(PosixPath, self).relative_to(*other) return PosixPath._from_parsed_parts(result._drv, result._root, result._parts)
def pt_run_dir(tmp_path: pathlib.PosixPath): dir_from = pathlib.Path('tests/test-data/spc-and-methanol-run') tmp_path = tmp_path.joinpath('spc-and-methanol-run') # str needed for Python 3.5 shutil.copytree(str(dir_from), str(tmp_path)) return tmp_path
def broken_image_file(): img = Path(__file__).parent / 'fixtureimage.png.broken' assert img.exists(), 'image not found' return img
def png_file(): img = Path(__file__).parent / 'fixtureimage.png' assert img.exists(), 'image not found' return img