def parse_files(self, verbosity=1):
me = 'parse_files'
file_count = 0
total_file_rows = 0
log_file = self.log_file
if verbosity > 0:
print("{}:Starting with input_folders={},globs={}".format(
me, self.input_file_folders, self.input_file_globs),
file=log_file)
paths = sequence_paths(input_folders=self.input_file_folders,
input_path_globs=self.input_file_globs)
total_inserts = 0
total_exceptions = 0
for path in paths:
file_count += 1
input_file_name = path.resolve()
n_rows, n_inserts, n_exceptions = self.import_file(
input_file_name=input_file_name, verbosity=verbosity)
total_file_rows += n_rows
total_inserts += n_inserts
total_exceptions += n_exceptions
# end for path in paths
if verbosity > 0:
print("{}:STAR Files - Ending with {} files found and parsed.".
format(me, file_count),
file=log_file)
return file_count, total_file_rows, total_inserts, total_exceptions
def set_teleport(name, target):
'''Sets a (new) teleport path for the currently active profile.'''
path = Path(os.path.expanduser(target))
if not path.is_dir():
raise StorageException('{} is not a directory'.format(target))
if not name:
raise StorageException('You must provide a name')
target = str(path.resolve())
data = get_active_profile()
data[name] = target
update_active_profile(data)
def short_relative_path(path, base='.'):
path = Path(path)
base = Path(base)
common = Path(os.path.commonpath([base.resolve(), path.resolve()]))
if common == path.root:
return path
elif common == Path.home():
up = Path('~')
elif common == base:
up = Path('.')
else:
n = len(base.parts) - len(common.parts)
up = Path('../' * n)
rel = str(up / path.resolve().relative_to(common))
if len(rel) < len(str(path)):
return rel
else:
return str(path)
def parse_files(self, verbosity=1):
me = 'parse_files'
file_count = 0
total_file_rows = 0
log_file = self.log_file
gpaths = sequence_paths(input_folders=self.input_file_folders,
input_path_globs=self.input_file_globs)
paths = []
total_inserts = 0
total_exceptions = 0
for path in gpaths:
if path in paths:
# gpaths could have duplicates when mulitple globs
# were used to generate the gpaths, so skip dups
# If carefully chosen to guarantee the globs have no dups,
# one can bypass this checking
continue
#Store this path to reject future duplicates in the sequence
paths.append(path)
file_count += 1
input_file_name = path.resolve()
if verbosity > 0:
print("{}: parsing input file '{}'".format(
me, input_file_name),
flush=True,
file=log_file)
n_rows, n_inserts, n_exceptions = self.import_file(
input_file_name=input_file_name, verbosity=verbosity)
total_file_rows += n_rows
total_inserts += n_inserts
total_exceptions += n_exceptions
#l_rows = ['one']
if verbosity > 0:
print("{}: Parsed file {}={} with {} 'readings' rows".format(
me, file_count, input_file_name, n_rows),
file=log_file)
# end for path in paths
if verbosity > 0:
print("{}:Diver Files - Ending with {} files found and parsed.".
format(me, file_count),
file=log_file)
return file_count, total_file_rows, total_inserts, total_exceptions
async def loadFactorOneFile(self, year: int, upCode: str,
factor: str) -> pd.DataFrame:
if "pytest" in sys.modules:
path = Path('../app/static/factors/' + str(year) + '/' + upCode +
'_' + factor + '.xlsx')
else:
path = Path('app/static/factors/' + str(year) + '/' + upCode +
'_' + factor + '.xlsx')
print(path.resolve())
print(os.path.dirname(__file__))
name = path.resolve()
# 'static/factors/'+str(year)+'/'+upCode+'_'+factor+'.xlsx'
df = pd.read_excel(name, sheet_name='계정별 기업 비교 - 특정계정과목', skiprows=8)
coli = list(df.iloc[0])
print(coli)
for i in range(len(coli)):
if i >= 4:
coli[i] = float(coli[i])
df.columns = coli
df = df.drop([0])
df["데이터명"] = factor
return df
def build(self):
src = Path(self.source_folder)
dst = Path(self.build_folder)
vtddir = dst / "VTD.2.2"
libdir = vtddir / "bundled"
def extract_archive(archive):
print(f"Extracting: {archive}")
tools.untargz(src / archive, dst)
extract_archive(self._archive_base)
libdir.mkdir()
if self.options.with_osi:
extract_archive(self._archive_osi)
if self.options.with_road_designer:
extract_archive(self._archive_rod)
# Patch RPATH of several critical binaries.
patch_rpath(
vtddir / "Runtime/Core/ModuleManager/moduleManager.4.7.0",
["$ORIGIN/../Framework/lib", "$ORIGIN/lib"],
)
patch_rpath(vtddir / "Runtime/Core/ParamServer/paramServer-1.1.0",
["$ORIGIN/../Lib"])
patch_rpath(
vtddir / "Runtime/Core/Traffic/ghostdriver.2.2.0.837",
["$ORIGIN/../Lib"],
)
# Patch RPATH of all the binaries with the path to our bundled system libraries:
for file in find_binary_files():
try:
patch_rpath(file, [
f"$ORIGIN/{os.path.relpath(libdir, (dst / file).parent)}"
])
except:
# Not all files can be set, but even if this happens it doesn't appear
# to be a big deal.
print(f"Error: cannot set RPATH of {file}", file=sys.stderr)
# Bundle libraries that we need at runtime so that this package is portable:
with Path(src / "libdeps_pruned.txt").open() as file:
libraries = [Path(x.strip()) for x in file.read().splitlines()]
for path in libraries:
print(f"Bundling system library: {path}")
if path.is_symlink():
shutil.copy(path, libdir, follow_symlinks=False)
path = path.resolve(strict=True)
assert not path.is_symlink()
shutil.copy(path, libdir)
patch_rpath(libdir / path.name, ["$ORIGIN"])
def createCheckpointPath(self, GLOVE, CNN_LAYER, POOLING_LAYER, GRU_LAYER,
BiLSTM_Layer, LSTM_Layer, DENSE_LAYER):
modelFolderName = Logging(self.args).createModelName(
GLOVE=GLOVE,
CNN_LAYER=CNN_LAYER,
POOLING_LAYER=POOLING_LAYER,
GRU_LAYER=GRU_LAYER,
BiLSTM_Layer=BiLSTM_Layer,
LSTM_Layer=LSTM_Layer,
DENSE_LAYER=DENSE_LAYER)
path = Path(Paths.MODEL_CHECKPOINTS, modelFolderName)
path.mkdir(parents=True, exist_ok=True)
return str(path.resolve())
def resolve_path_all(path: pathlib.Path, resolve: bool = True) -> pathlib.Path:
""" Expend environment variables and resolve path.
Args:
path (pathlib.Path): [description]
resolve (bool, optional): [description]. Defaults to True.
Returns:
pathlib.Path: [description]
"""
if path:
path = pathlib.Path(os.path.expandvars(path)).expanduser()
if resolve:
path = path.resolve()
return path
def get_files(dirname, extensions=['.png', '.tif', '.jpg']):
"""
Gathers list of files for processing from path recursively.
:param dirname: path to parse
:param extensions: extensions to match
:return: list of files matching extensions
"""
dir_path = Path(dirname)
files = dir_path.glob('**/*')
files = [path.resolve() for path in files]
match = [f for f in files if f.suffix in extensions]
return match
def normalize_path(path_string, current_folder=None):
path_string = os.path.expanduser(path_string)
if os.path.isabs(path_string):
return path_string
if current_folder:
path = pathlib.Path(normalize_path(current_folder))
path = path.joinpath(path_string)
else:
path = pathlib.Path(path_string)
if path.exists():
path = path.resolve()
return str(path)
def ensure_overwritable(*paths):
"""
This is a (nasty) workaround for the permissions issues we hit when
switching between running the job-runner inside Docker and running it
natively on Windows. The issue is that the Docker process creates files
which the Windows native process then doesn't have permission to delete or
replace. We work around this here by using Docker to delete the offending
files for us.
Note for the potentially confused: Windows permissions work nothing like
POSIX permissions. We can create new files in directories created by
Docker, we just can't modify or delete existing files.
"""
if not config.ENABLE_PERMISSIONS_WORKAROUND:
return
non_writable = []
for path in paths:
path = Path(path)
if path.exists():
# It would be nice to have a read-only way of determining if we
# have write access but I can't seem to find one that works on
# Windows
try:
path.touch()
except PermissionError:
non_writable.append(path.resolve())
if not non_writable:
return
root = os.path.commonpath([f.parent for f in non_writable])
rel_paths = [f.relative_to(root) for f in non_writable]
rel_posix_paths = [str(f).replace(os.path.sep, "/") for f in rel_paths]
subprocess_run(
[
"docker",
"run",
"--rm",
"--volume",
f"{root}:/workspace",
"--workdir",
"/workspace",
docker.MANAGEMENT_CONTAINER_IMAGE,
"rm",
*rel_posix_paths,
],
check=True,
capture_output=True,
)
def test_reader_provenance(path, creation_time):
with patch("datetime.datetime", MagicMock()):
datetime.datetime.now.return_value = creation_time
reader = ADASReader(path, MagicMock())
path = Path(path)
openadas = path == Path("")
if openadas:
path = Path.home() / adas.CACHE_DIR / "adas"
prov_id = hash_vals(path=path)
reader.session.prov.agent.assert_called_once_with(prov_id)
assert reader.agent is reader.session.prov.agent.return_value
reader.session.prov.entity.assert_called_once_with(
prov_id, {"path": str(path.resolve())})
assert reader.entity is reader.session.prov.entity.return_value
reader.session.prov.delegation.assert_called_once_with(
reader.session.agent, reader.agent)
reader.session.prov.generation.assert_called_once_with(
reader.entity, reader.session.session, time=creation_time)
reader.session.prov.attribution.assert_called_once_with(
reader.entity, reader.session.agent)
def relative_path(path, start=Path('.')):
"""This is a :class:`pathlib.Path` compatible way of building relative
paths. ``Path.relative_to()`` does exist, but does not correctly apply
sibling descendent relationships (aka ``../``). ``os.path.relpath`` does
what is wanted, so we use it, but with ``pathlib`` objects.
Note, this function requires ``path`` and ``start`` to be resolvable.
Or in other words, they must exist, so that we can obtain an absolute
path. Directly use ``os.path.relpath`` if this is a problem.
:param path: the path to make relative
:type path: :class:`pathlib.Path`
:param start: starting location to make the given ``path`` relative to
:type start: :class:`pathlib.Path`
:return: relative path
:rtype: :class:`pathlib.Path`
"""
p = str(path.resolve())
s = str(start.resolve())
return Path(os.path.relpath(p, start=s))
def cmdfunc_include(args, cfg, targets):
path = pathlib.Path(args.path)
try:
path = path.resolve()
except FileNotFoundError:
pass
t, relpath = get_target_from_path(targets, path)
if t is None:
print("error: {!r} is not in any target".format(str(path)),
file=sys.stderr)
sys.exit(1)
state = t.get_state(relpath)
if state == target.State.INCLUDED:
print("error: {!r} is already included".format(relpath),
file=sys.stderr)
sys.exit(1)
t.include(relpath)
t.prune()
if args.summon:
cmd = rsync_invocation_base(cfg,
verbosity=args.verbosity,
delete=False)
if args.dry_run:
apply_dry_run_mode(cmd, args.dry_run)
cmd.extend(args.rsync_opts)
cmd.append("--ignore-existing")
cmd.append(os.path.join(t.src, relpath[1:])+"/")
cmd.append(str(t.dest / relpath[1:]))
subprocess.check_call(cmd)
if not args.dry_run:
write_targets(get_targets_path(), targets)
(
"imported",
{
Source.create(os.path.join(TEST_DIRECTORY, "__init__.py")),
Source.create(
os.path.join(TEST_DIRECTORY, "dependency_example.py")),
Source.create(
os.path.join(TEST_DIRECTORY, "foo", "__init__.py")),
Source.create(os.path.join(TEST_DIRECTORY, "foo", "bar.py")),
},
),
(
"dir",
{
# This list would be too long to explicitly insert here
Source.create(str(path.resolve()))
for path in Path(TEST_DIRECTORY).rglob("*.py")
},
),
(
"none",
{
Source.create(
os.path.join(TEST_DIRECTORY, "dependency_example.py")),
},
),
# "sys" is not tested here because it depends on the environment. Can't
# make it work consistently on my local machine and azure
],
)
def test_gather_sources_and_dependencies(discover_sources, expected_sources):
for unit in manager.ListUnits():
unit_name = str(unit[0])
unit_match = re.search('minecraft@(.*).service', unit_name)
if unit_match:
world = unit_match.group(1)
service = sysbus.get_object(
'org.freedesktop.systemd1', object_path=manager.GetUnit(unit_name))
interface = dbus.Interface(
service, dbus_interface='org.freedesktop.DBus.Properties')
working_directory = interface.Get('org.freedesktop.systemd1.Service', 'WorkingDirectory')
root_directory = Path(os.path.join(working_directory, 'logs'))
for path in root_directory.glob('*.gz'):
match = re.search('(\d+-\d+-\d+)-\d+.log.gz',path.name)
if match:
# date_str = match.group(1)
p = path.resolve()
with gzip.open(p, 'r') as f:
for line in f:
line = line.decode()
if 'logged in with entity id' in line:
# logged_in_time = datetime.datetime.strptime(date_str + ' ' + line.split(' ')[0],'%Y-%m-%d [%H:%M:%S]')
logged_in_time = datetime.datetime.strptime(line.split(' ')[0],'[%H:%M:%S]')
logged_in_time = logged_in_time.replace(minute=0,second=0,microsecond=0)
history[logged_in_time][world] += 1
data_table = gviz_api.DataTable(('time', 'timeofday'): [('creative','number'), ('survival', 'number'), 'rainbow', 'number'])
for key, value in history.items():
data_table.AppendData([(key, value['creative'], value['creative'], value['rainbow'])])
with open(os.path.join('/tmp', 'activity.json'),'w') as f:
f.write(data_table.ToJSon())
def _file_descriptor_is_open_for(path):
process = psutil.Process()
return any(
(f.path == str(path.resolve())) for f in process.open_files())
def find_mount_point(path: Path) -> Path:
path = path.resolve()
while not os.path.ismount(path):
assert str(path) != '/'
path = path.parent
return path
def find_my_cnf(locations):
for location in locations:
path = pathlib.Path(os.path.expanduser(location))
if path.exists():
return path.resolve()
def mets_paths_backup_optional(backup_folder=None, input_folders=None,
file_globs=None, log_file=None, verbosity=None):
'''
mets_paths_backup_optional()
Given input_folders and file_globs:
(1) visit a sequence of contained files,
(2) save only unique paths in paths[]
(3) and return the paths[] of the selected files.
Optionally, if backup_folder is not None,
back up each mets file in the backup folder too
ASSUMPTION - these are mets files couched in UFDC resources key-pair
directory hierarchy, and each mets.xml file is supposed to be unique.
So, destinations filenames are copied directly into one flat backup folder.
Consider an option later to 'flatten' the parent directory name into a
backup folder subdirectory, eg "AA12345678_12345", to ensure that no
duplicate mets file names will be 'lost' due to overwriting in this routine.
'''
me = 'mets_paths_backup_optional'
if verbosity > 0:
utc_now = datetime.datetime.utcnow()
utc_secs_z = utc_now.strftime("%Y-%m-%dT%H:%M:%SZ")
msg = ("{}:for input_folders={},\n"
" globs={},\nlog='{}', getting sequence paths at {}..."
.format(me,input_folders,self.mets_glob,log_file.name, utc_secs_z))
print(msg)
print(msg,file=log_file)
sys.stdout.flush()
log_file.flush()
gpaths = sequence_paths(log_file=log_file, input_folders=input_folders,
input_path_globs=[self.mets_glob], verbosity=verbosity)
if verbosity > 0:
utc_now = datetime.datetime.utcnow()
utc_secs_z = utc_now.strftime("%Y-%m-%dT%H:%M:%SZ")
msg = ("{}:Checking the input paths sequence for dups at utc time = {}"
.format(me, utc_secs_z))
print(msg)
print(msg,file=log_file)
sys.stdout.flush()
log_file.flush()
paths = []
n_path = 0
for path in gpaths:
if path in paths:
# gpaths could have duplicates when mulitple globs
# were used to generate the gpaths, so skip dups
# If carefully chosen to guarantee the globs have no dups,
# one can bypass this checking
continue
# Store this path to return and will also use it to check for future
# duplicates in the sequence
n_path += 1
paths.append(path)
#Copy the file to backup location
if backup_folder is not None:
copy(path.resolve(), backup_folder)
if verbosity > 0:
utc_now = datetime.datetime.utcnow()
utc_secs_z = utc_now.strftime("%Y-%m-%dT%H:%M:%SZ")
msg = ("{}: Found paths for {} mets files at utc time = {}"
.format(me, n_path, utc_secs_z))
if backup_folder is not None:
msg += ', and backed them up under {}'.format(backup_folder)
print(msg)
print(msg,file=log_file)
sys.stdout.flush()
log_file.flush()
return paths
def xxprocess_files(
backup_folder=None,
input_folders=None,
file_globs=None,
log_file=None,
parent_tag_name=None,
replace_children=False,
progress_count=100,
verbosity=1,
):
me = 'process_files'
n_files = 0
test_one = 0
total_file_lines = 0
log_file = log_file
# First call mets_paths_backup_optional() to
# collect the mets file paths and copy the mets files to backup
# location.
paths = mets_paths_backup_optional(backup_folder=backup_folder,
input_folders=input_folders, file_globs=[self.mets_glob],
log_file=log_file, verbosity=verbosity
)
# We now loop through the paths[] and apply the edits.
n_files = 0
n_unchanged = 0
n_changed = 0
for path in paths:
input_file_name = path.resolve()
if verbosity > 0:
msg=("{}:Got path.resolve()='{}'".format(me, input_file_name))
print(msg, file=log_file)
n_files += 1
#Test limits
min_file_index = 0
# NOTE: set max_file_index 0 to mean unlimited
max_file_index = 0
if n_files < min_file_index:
continue
if max_file_index > 0 and n_files > max_file_index:
return n_files, n_changed, n_unchanged
# Start processing a file
if verbosity > 0:
msg=("{}:Processing input_file_name='{}'"
.format(me, input_file_name))
print(msg, file=log_file)
print(msg)
log_file.flush()
sys.stdout.flush()
rv = mets_xml_add_or_replace_subjects(
log_file=log_file,
bib=bib, vid=vid, item_text=item_text,
input_file_name=input_file_name,
replace_subjects=replace_children,
verbosity=verbosity)
period = progress_count
if n_files % period == 0:
utc_now = datetime.datetime.utcnow()
utc_secs_z = utc_now.strftime("%Y-%m-%dT%H:%M:%SZ")
msg = ("{}: Processed thru file index {}, name {} as of {}"
.format(me, n_files, input_file_name, utc_secs_z))
print(msg)
print(msg, file=log_file)
sys.stdout.flush()
log_file.flush()
if rv is None or rv <= 0:
n_unchanged += 1
else:
n_changed +=1
if verbosity > 0:
print(
"{}: Processed file {}={} with rv={}."
.format(me, n_files, input_file_name, rv)
,file=log_file)
if test_one == 1:
print("Test_one - test break")
break
# end for path in paths
if verbosity > 0:
utc_now = datetime.datetime.utcnow()
utc_secs_z = utc_now.strftime("%Y-%m-%dT%H:%M:%SZ")
msg = ("{}: Processed thru file index {}, name {} as of {}"
.format(me, n_files, input_file_name, utc_secs_z))
print(msg)
print(msg, file=log_file)
sys.stdout.flush()
log_file.flush()
print("{}: Ending with {} files processed."
.format(me,n_files,), file=log_file)
return n_files, n_changed, n_unchanged
def abs_path(path):
# type: (Path, ) -> Path
"""Checks if a path is an actual directory"""
return path.resolve()
time.sleep(20)
except ValueError:
return 0.0
analysis_start = time.time()
while (time.time() - analysis_start) < cuckoo_timeout:
try:
return requests.get(report_url + str(task_id), headers=header_settings).json()["info"]["score"]
except KeyError:
time.sleep(20)
return 0.0
while True:
files = []
submitted_tasks = []
for path in Path(folder_to_analyze).glob('**/*'):
files.append(str(path.resolve()))
for file in files:
# Get the full file name without other path values
file_name = os.path.basename(file)
# File extension is split from file_name to determine what kind of file we are dealing with
file_extension = file_name.split(".", 1)[1]
if file_extension not in disallowed_files:
if debug == 1:
print("File name is " + file_name + " with an extension of " + file_extension)
sha256hash = get_hash()
if debug == 1:
print("File has a sha256 hash of : " + sha256hash)
task_id = file_unique(sha256hash)
submitted_tasks.append(task_id)
def path2str(path: Path) -> str:
if isinstance(path, Path): string = str(path.resolve())
else: string = path
return string
def find_my_cnf(locations):
for location in locations:
path = pathlib.Path(os.path.expanduser(location))
if path.exists():
return path.resolve()
def findyaml(basedir):
"""Return a list of absolute paths to yaml files recursively discovered by walking the directory tree rooted at basedir"""
return [str(path.resolve()) for path in pathlib.Path(basedir).rglob('*.yml')]
def findvideo(basedir):
"""Return a list of absolute paths to json files recursively discovered by walking the directory tree rooted at basedir"""
return [str(path.resolve()) for path in pathlib.Path(basedir).rglob('*') if isvideo(str(path.resolve()))]
def rpath(d, f = ''):
path = pathlib.Path(d) / pathlib.Path(f)
return str(path.resolve())
#!/usr/bin/python3
import collections
import copy
import datetime
import os.path
import pathlib
import subprocess
import sys
path = pathlib.Path(__file__).parent / ".." # noqa
path = path.resolve().as_posix() # noqa
sys.path.append(path) # noqa
import filelock
import jinja2
import yaml
import mirror
MIRROR_DATADIR = mirror.MIRROR_DATADIR
#MIRROR_DATA_LOCK = mirror.MIRROR_DATA_LOCK
ROOT = pathlib.Path(__file__).parent / ".."
env = jinja2.Environment(loader=jinja2.FileSystemLoader(
(ROOT / 'template').as_posix()),
autoescape=jinja2.select_autoescape(['html', 'xml']))
outdir = ROOT / "output"
title = "IFCA repository mirror (repo.ifca.es)"
