def parse_files(self, verbosity=1): me = 'parse_files' file_count = 0 total_file_rows = 0 log_file = self.log_file if verbosity > 0: print("{}:Starting with input_folders={},globs={}".format( me, self.input_file_folders, self.input_file_globs), file=log_file) paths = sequence_paths(input_folders=self.input_file_folders, input_path_globs=self.input_file_globs) total_inserts = 0 total_exceptions = 0 for path in paths: file_count += 1 input_file_name = path.resolve() n_rows, n_inserts, n_exceptions = self.import_file( input_file_name=input_file_name, verbosity=verbosity) total_file_rows += n_rows total_inserts += n_inserts total_exceptions += n_exceptions # end for path in paths if verbosity > 0: print("{}:STAR Files - Ending with {} files found and parsed.". format(me, file_count), file=log_file) return file_count, total_file_rows, total_inserts, total_exceptions
def set_teleport(name, target): '''Sets a (new) teleport path for the currently active profile.''' path = Path(os.path.expanduser(target)) if not path.is_dir(): raise StorageException('{} is not a directory'.format(target)) if not name: raise StorageException('You must provide a name') target = str(path.resolve()) data = get_active_profile() data[name] = target update_active_profile(data)
def short_relative_path(path, base='.'): path = Path(path) base = Path(base) common = Path(os.path.commonpath([base.resolve(), path.resolve()])) if common == path.root: return path elif common == Path.home(): up = Path('~') elif common == base: up = Path('.') else: n = len(base.parts) - len(common.parts) up = Path('../' * n) rel = str(up / path.resolve().relative_to(common)) if len(rel) < len(str(path)): return rel else: return str(path)
def parse_files(self, verbosity=1): me = 'parse_files' file_count = 0 total_file_rows = 0 log_file = self.log_file gpaths = sequence_paths(input_folders=self.input_file_folders, input_path_globs=self.input_file_globs) paths = [] total_inserts = 0 total_exceptions = 0 for path in gpaths: if path in paths: # gpaths could have duplicates when mulitple globs # were used to generate the gpaths, so skip dups # If carefully chosen to guarantee the globs have no dups, # one can bypass this checking continue #Store this path to reject future duplicates in the sequence paths.append(path) file_count += 1 input_file_name = path.resolve() if verbosity > 0: print("{}: parsing input file '{}'".format( me, input_file_name), flush=True, file=log_file) n_rows, n_inserts, n_exceptions = self.import_file( input_file_name=input_file_name, verbosity=verbosity) total_file_rows += n_rows total_inserts += n_inserts total_exceptions += n_exceptions #l_rows = ['one'] if verbosity > 0: print("{}: Parsed file {}={} with {} 'readings' rows".format( me, file_count, input_file_name, n_rows), file=log_file) # end for path in paths if verbosity > 0: print("{}:Diver Files - Ending with {} files found and parsed.". format(me, file_count), file=log_file) return file_count, total_file_rows, total_inserts, total_exceptions
async def loadFactorOneFile(self, year: int, upCode: str, factor: str) -> pd.DataFrame: if "pytest" in sys.modules: path = Path('../app/static/factors/' + str(year) + '/' + upCode + '_' + factor + '.xlsx') else: path = Path('app/static/factors/' + str(year) + '/' + upCode + '_' + factor + '.xlsx') print(path.resolve()) print(os.path.dirname(__file__)) name = path.resolve() # 'static/factors/'+str(year)+'/'+upCode+'_'+factor+'.xlsx' df = pd.read_excel(name, sheet_name='계정별 기업 비교 - 특정계정과목', skiprows=8) coli = list(df.iloc[0]) print(coli) for i in range(len(coli)): if i >= 4: coli[i] = float(coli[i]) df.columns = coli df = df.drop([0]) df["데이터명"] = factor return df
def build(self): src = Path(self.source_folder) dst = Path(self.build_folder) vtddir = dst / "VTD.2.2" libdir = vtddir / "bundled" def extract_archive(archive): print(f"Extracting: {archive}") tools.untargz(src / archive, dst) extract_archive(self._archive_base) libdir.mkdir() if self.options.with_osi: extract_archive(self._archive_osi) if self.options.with_road_designer: extract_archive(self._archive_rod) # Patch RPATH of several critical binaries. patch_rpath( vtddir / "Runtime/Core/ModuleManager/moduleManager.4.7.0", ["$ORIGIN/../Framework/lib", "$ORIGIN/lib"], ) patch_rpath(vtddir / "Runtime/Core/ParamServer/paramServer-1.1.0", ["$ORIGIN/../Lib"]) patch_rpath( vtddir / "Runtime/Core/Traffic/ghostdriver.2.2.0.837", ["$ORIGIN/../Lib"], ) # Patch RPATH of all the binaries with the path to our bundled system libraries: for file in find_binary_files(): try: patch_rpath(file, [ f"$ORIGIN/{os.path.relpath(libdir, (dst / file).parent)}" ]) except: # Not all files can be set, but even if this happens it doesn't appear # to be a big deal. print(f"Error: cannot set RPATH of {file}", file=sys.stderr) # Bundle libraries that we need at runtime so that this package is portable: with Path(src / "libdeps_pruned.txt").open() as file: libraries = [Path(x.strip()) for x in file.read().splitlines()] for path in libraries: print(f"Bundling system library: {path}") if path.is_symlink(): shutil.copy(path, libdir, follow_symlinks=False) path = path.resolve(strict=True) assert not path.is_symlink() shutil.copy(path, libdir) patch_rpath(libdir / path.name, ["$ORIGIN"])
def createCheckpointPath(self, GLOVE, CNN_LAYER, POOLING_LAYER, GRU_LAYER, BiLSTM_Layer, LSTM_Layer, DENSE_LAYER): modelFolderName = Logging(self.args).createModelName( GLOVE=GLOVE, CNN_LAYER=CNN_LAYER, POOLING_LAYER=POOLING_LAYER, GRU_LAYER=GRU_LAYER, BiLSTM_Layer=BiLSTM_Layer, LSTM_Layer=LSTM_Layer, DENSE_LAYER=DENSE_LAYER) path = Path(Paths.MODEL_CHECKPOINTS, modelFolderName) path.mkdir(parents=True, exist_ok=True) return str(path.resolve())
def resolve_path_all(path: pathlib.Path, resolve: bool = True) -> pathlib.Path: """ Expend environment variables and resolve path. Args: path (pathlib.Path): [description] resolve (bool, optional): [description]. Defaults to True. Returns: pathlib.Path: [description] """ if path: path = pathlib.Path(os.path.expandvars(path)).expanduser() if resolve: path = path.resolve() return path
def get_files(dirname, extensions=['.png', '.tif', '.jpg']): """ Gathers list of files for processing from path recursively. :param dirname: path to parse :param extensions: extensions to match :return: list of files matching extensions """ dir_path = Path(dirname) files = dir_path.glob('**/*') files = [path.resolve() for path in files] match = [f for f in files if f.suffix in extensions] return match
def normalize_path(path_string, current_folder=None): path_string = os.path.expanduser(path_string) if os.path.isabs(path_string): return path_string if current_folder: path = pathlib.Path(normalize_path(current_folder)) path = path.joinpath(path_string) else: path = pathlib.Path(path_string) if path.exists(): path = path.resolve() return str(path)
def ensure_overwritable(*paths): """ This is a (nasty) workaround for the permissions issues we hit when switching between running the job-runner inside Docker and running it natively on Windows. The issue is that the Docker process creates files which the Windows native process then doesn't have permission to delete or replace. We work around this here by using Docker to delete the offending files for us. Note for the potentially confused: Windows permissions work nothing like POSIX permissions. We can create new files in directories created by Docker, we just can't modify or delete existing files. """ if not config.ENABLE_PERMISSIONS_WORKAROUND: return non_writable = [] for path in paths: path = Path(path) if path.exists(): # It would be nice to have a read-only way of determining if we # have write access but I can't seem to find one that works on # Windows try: path.touch() except PermissionError: non_writable.append(path.resolve()) if not non_writable: return root = os.path.commonpath([f.parent for f in non_writable]) rel_paths = [f.relative_to(root) for f in non_writable] rel_posix_paths = [str(f).replace(os.path.sep, "/") for f in rel_paths] subprocess_run( [ "docker", "run", "--rm", "--volume", f"{root}:/workspace", "--workdir", "/workspace", docker.MANAGEMENT_CONTAINER_IMAGE, "rm", *rel_posix_paths, ], check=True, capture_output=True, )
def test_reader_provenance(path, creation_time): with patch("datetime.datetime", MagicMock()): datetime.datetime.now.return_value = creation_time reader = ADASReader(path, MagicMock()) path = Path(path) openadas = path == Path("") if openadas: path = Path.home() / adas.CACHE_DIR / "adas" prov_id = hash_vals(path=path) reader.session.prov.agent.assert_called_once_with(prov_id) assert reader.agent is reader.session.prov.agent.return_value reader.session.prov.entity.assert_called_once_with( prov_id, {"path": str(path.resolve())}) assert reader.entity is reader.session.prov.entity.return_value reader.session.prov.delegation.assert_called_once_with( reader.session.agent, reader.agent) reader.session.prov.generation.assert_called_once_with( reader.entity, reader.session.session, time=creation_time) reader.session.prov.attribution.assert_called_once_with( reader.entity, reader.session.agent)
def relative_path(path, start=Path('.')): """This is a :class:`pathlib.Path` compatible way of building relative paths. ``Path.relative_to()`` does exist, but does not correctly apply sibling descendent relationships (aka ``../``). ``os.path.relpath`` does what is wanted, so we use it, but with ``pathlib`` objects. Note, this function requires ``path`` and ``start`` to be resolvable. Or in other words, they must exist, so that we can obtain an absolute path. Directly use ``os.path.relpath`` if this is a problem. :param path: the path to make relative :type path: :class:`pathlib.Path` :param start: starting location to make the given ``path`` relative to :type start: :class:`pathlib.Path` :return: relative path :rtype: :class:`pathlib.Path` """ p = str(path.resolve()) s = str(start.resolve()) return Path(os.path.relpath(p, start=s))
def cmdfunc_include(args, cfg, targets): path = pathlib.Path(args.path) try: path = path.resolve() except FileNotFoundError: pass t, relpath = get_target_from_path(targets, path) if t is None: print("error: {!r} is not in any target".format(str(path)), file=sys.stderr) sys.exit(1) state = t.get_state(relpath) if state == target.State.INCLUDED: print("error: {!r} is already included".format(relpath), file=sys.stderr) sys.exit(1) t.include(relpath) t.prune() if args.summon: cmd = rsync_invocation_base(cfg, verbosity=args.verbosity, delete=False) if args.dry_run: apply_dry_run_mode(cmd, args.dry_run) cmd.extend(args.rsync_opts) cmd.append("--ignore-existing") cmd.append(os.path.join(t.src, relpath[1:])+"/") cmd.append(str(t.dest / relpath[1:])) subprocess.check_call(cmd) if not args.dry_run: write_targets(get_targets_path(), targets)
( "imported", { Source.create(os.path.join(TEST_DIRECTORY, "__init__.py")), Source.create( os.path.join(TEST_DIRECTORY, "dependency_example.py")), Source.create( os.path.join(TEST_DIRECTORY, "foo", "__init__.py")), Source.create(os.path.join(TEST_DIRECTORY, "foo", "bar.py")), }, ), ( "dir", { # This list would be too long to explicitly insert here Source.create(str(path.resolve())) for path in Path(TEST_DIRECTORY).rglob("*.py") }, ), ( "none", { Source.create( os.path.join(TEST_DIRECTORY, "dependency_example.py")), }, ), # "sys" is not tested here because it depends on the environment. Can't # make it work consistently on my local machine and azure ], ) def test_gather_sources_and_dependencies(discover_sources, expected_sources):
for unit in manager.ListUnits(): unit_name = str(unit[0]) unit_match = re.search('minecraft@(.*).service', unit_name) if unit_match: world = unit_match.group(1) service = sysbus.get_object( 'org.freedesktop.systemd1', object_path=manager.GetUnit(unit_name)) interface = dbus.Interface( service, dbus_interface='org.freedesktop.DBus.Properties') working_directory = interface.Get('org.freedesktop.systemd1.Service', 'WorkingDirectory') root_directory = Path(os.path.join(working_directory, 'logs')) for path in root_directory.glob('*.gz'): match = re.search('(\d+-\d+-\d+)-\d+.log.gz',path.name) if match: # date_str = match.group(1) p = path.resolve() with gzip.open(p, 'r') as f: for line in f: line = line.decode() if 'logged in with entity id' in line: # logged_in_time = datetime.datetime.strptime(date_str + ' ' + line.split(' ')[0],'%Y-%m-%d [%H:%M:%S]') logged_in_time = datetime.datetime.strptime(line.split(' ')[0],'[%H:%M:%S]') logged_in_time = logged_in_time.replace(minute=0,second=0,microsecond=0) history[logged_in_time][world] += 1 data_table = gviz_api.DataTable(('time', 'timeofday'): [('creative','number'), ('survival', 'number'), 'rainbow', 'number']) for key, value in history.items(): data_table.AppendData([(key, value['creative'], value['creative'], value['rainbow'])]) with open(os.path.join('/tmp', 'activity.json'),'w') as f: f.write(data_table.ToJSon())
def _file_descriptor_is_open_for(path): process = psutil.Process() return any( (f.path == str(path.resolve())) for f in process.open_files())
def find_mount_point(path: Path) -> Path: path = path.resolve() while not os.path.ismount(path): assert str(path) != '/' path = path.parent return path
def find_my_cnf(locations): for location in locations: path = pathlib.Path(os.path.expanduser(location)) if path.exists(): return path.resolve()
def mets_paths_backup_optional(backup_folder=None, input_folders=None, file_globs=None, log_file=None, verbosity=None): ''' mets_paths_backup_optional() Given input_folders and file_globs: (1) visit a sequence of contained files, (2) save only unique paths in paths[] (3) and return the paths[] of the selected files. Optionally, if backup_folder is not None, back up each mets file in the backup folder too ASSUMPTION - these are mets files couched in UFDC resources key-pair directory hierarchy, and each mets.xml file is supposed to be unique. So, destinations filenames are copied directly into one flat backup folder. Consider an option later to 'flatten' the parent directory name into a backup folder subdirectory, eg "AA12345678_12345", to ensure that no duplicate mets file names will be 'lost' due to overwriting in this routine. ''' me = 'mets_paths_backup_optional' if verbosity > 0: utc_now = datetime.datetime.utcnow() utc_secs_z = utc_now.strftime("%Y-%m-%dT%H:%M:%SZ") msg = ("{}:for input_folders={},\n" " globs={},\nlog='{}', getting sequence paths at {}..." .format(me,input_folders,self.mets_glob,log_file.name, utc_secs_z)) print(msg) print(msg,file=log_file) sys.stdout.flush() log_file.flush() gpaths = sequence_paths(log_file=log_file, input_folders=input_folders, input_path_globs=[self.mets_glob], verbosity=verbosity) if verbosity > 0: utc_now = datetime.datetime.utcnow() utc_secs_z = utc_now.strftime("%Y-%m-%dT%H:%M:%SZ") msg = ("{}:Checking the input paths sequence for dups at utc time = {}" .format(me, utc_secs_z)) print(msg) print(msg,file=log_file) sys.stdout.flush() log_file.flush() paths = [] n_path = 0 for path in gpaths: if path in paths: # gpaths could have duplicates when mulitple globs # were used to generate the gpaths, so skip dups # If carefully chosen to guarantee the globs have no dups, # one can bypass this checking continue # Store this path to return and will also use it to check for future # duplicates in the sequence n_path += 1 paths.append(path) #Copy the file to backup location if backup_folder is not None: copy(path.resolve(), backup_folder) if verbosity > 0: utc_now = datetime.datetime.utcnow() utc_secs_z = utc_now.strftime("%Y-%m-%dT%H:%M:%SZ") msg = ("{}: Found paths for {} mets files at utc time = {}" .format(me, n_path, utc_secs_z)) if backup_folder is not None: msg += ', and backed them up under {}'.format(backup_folder) print(msg) print(msg,file=log_file) sys.stdout.flush() log_file.flush() return paths
def xxprocess_files( backup_folder=None, input_folders=None, file_globs=None, log_file=None, parent_tag_name=None, replace_children=False, progress_count=100, verbosity=1, ): me = 'process_files' n_files = 0 test_one = 0 total_file_lines = 0 log_file = log_file # First call mets_paths_backup_optional() to # collect the mets file paths and copy the mets files to backup # location. paths = mets_paths_backup_optional(backup_folder=backup_folder, input_folders=input_folders, file_globs=[self.mets_glob], log_file=log_file, verbosity=verbosity ) # We now loop through the paths[] and apply the edits. n_files = 0 n_unchanged = 0 n_changed = 0 for path in paths: input_file_name = path.resolve() if verbosity > 0: msg=("{}:Got path.resolve()='{}'".format(me, input_file_name)) print(msg, file=log_file) n_files += 1 #Test limits min_file_index = 0 # NOTE: set max_file_index 0 to mean unlimited max_file_index = 0 if n_files < min_file_index: continue if max_file_index > 0 and n_files > max_file_index: return n_files, n_changed, n_unchanged # Start processing a file if verbosity > 0: msg=("{}:Processing input_file_name='{}'" .format(me, input_file_name)) print(msg, file=log_file) print(msg) log_file.flush() sys.stdout.flush() rv = mets_xml_add_or_replace_subjects( log_file=log_file, bib=bib, vid=vid, item_text=item_text, input_file_name=input_file_name, replace_subjects=replace_children, verbosity=verbosity) period = progress_count if n_files % period == 0: utc_now = datetime.datetime.utcnow() utc_secs_z = utc_now.strftime("%Y-%m-%dT%H:%M:%SZ") msg = ("{}: Processed thru file index {}, name {} as of {}" .format(me, n_files, input_file_name, utc_secs_z)) print(msg) print(msg, file=log_file) sys.stdout.flush() log_file.flush() if rv is None or rv <= 0: n_unchanged += 1 else: n_changed +=1 if verbosity > 0: print( "{}: Processed file {}={} with rv={}." .format(me, n_files, input_file_name, rv) ,file=log_file) if test_one == 1: print("Test_one - test break") break # end for path in paths if verbosity > 0: utc_now = datetime.datetime.utcnow() utc_secs_z = utc_now.strftime("%Y-%m-%dT%H:%M:%SZ") msg = ("{}: Processed thru file index {}, name {} as of {}" .format(me, n_files, input_file_name, utc_secs_z)) print(msg) print(msg, file=log_file) sys.stdout.flush() log_file.flush() print("{}: Ending with {} files processed." .format(me,n_files,), file=log_file) return n_files, n_changed, n_unchanged
def abs_path(path): # type: (Path, ) -> Path """Checks if a path is an actual directory""" return path.resolve()
time.sleep(20) except ValueError: return 0.0 analysis_start = time.time() while (time.time() - analysis_start) < cuckoo_timeout: try: return requests.get(report_url + str(task_id), headers=header_settings).json()["info"]["score"] except KeyError: time.sleep(20) return 0.0 while True: files = [] submitted_tasks = [] for path in Path(folder_to_analyze).glob('**/*'): files.append(str(path.resolve())) for file in files: # Get the full file name without other path values file_name = os.path.basename(file) # File extension is split from file_name to determine what kind of file we are dealing with file_extension = file_name.split(".", 1)[1] if file_extension not in disallowed_files: if debug == 1: print("File name is " + file_name + " with an extension of " + file_extension) sha256hash = get_hash() if debug == 1: print("File has a sha256 hash of : " + sha256hash) task_id = file_unique(sha256hash) submitted_tasks.append(task_id)
def path2str(path: Path) -> str: if isinstance(path, Path): string = str(path.resolve()) else: string = path return string
def findyaml(basedir): """Return a list of absolute paths to yaml files recursively discovered by walking the directory tree rooted at basedir""" return [str(path.resolve()) for path in pathlib.Path(basedir).rglob('*.yml')]
def findvideo(basedir): """Return a list of absolute paths to json files recursively discovered by walking the directory tree rooted at basedir""" return [str(path.resolve()) for path in pathlib.Path(basedir).rglob('*') if isvideo(str(path.resolve()))]
def rpath(d, f = ''): path = pathlib.Path(d) / pathlib.Path(f) return str(path.resolve())
#!/usr/bin/python3 import collections import copy import datetime import os.path import pathlib import subprocess import sys path = pathlib.Path(__file__).parent / ".." # noqa path = path.resolve().as_posix() # noqa sys.path.append(path) # noqa import filelock import jinja2 import yaml import mirror MIRROR_DATADIR = mirror.MIRROR_DATADIR #MIRROR_DATA_LOCK = mirror.MIRROR_DATA_LOCK ROOT = pathlib.Path(__file__).parent / ".." env = jinja2.Environment(loader=jinja2.FileSystemLoader( (ROOT / 'template').as_posix()), autoescape=jinja2.select_autoescape(['html', 'xml'])) outdir = ROOT / "output" title = "IFCA repository mirror (repo.ifca.es)"