def project_closed(self, project):
"""
Called when a project is closed.
:param project: Project instance
"""
yield from super().project_closed(project)
# delete useless Dynamips files
project_dir = project.module_working_path(self.module_name.lower())
files = glob.glob(os.path.join(glob.escape(project_dir), "*.ghost"))
files += glob.glob(os.path.join(glob.escape(project_dir), "*_lock"))
files += glob.glob(os.path.join(glob.escape(project_dir), "ilt_*"))
files += glob.glob(os.path.join(glob.escape(project_dir), "c[0-9][0-9][0-9][0-9]_i[0-9]*_rommon_vars"))
files += glob.glob(os.path.join(glob.escape(project_dir), "c[0-9][0-9][0-9][0-9]_i[0-9]*_log.txt"))
for file in files:
try:
log.debug("Deleting file {}".format(file))
if file in self._ghost_files:
self._ghost_files.remove(file)
yield from wait_run_in_executor(os.remove, file)
except OSError as e:
log.warn("Could not delete file {}: {}".format(file, e))
continue
# Release the dynamips ids if we want to reload the same project
# later
if project.id in self._dynamips_ids:
del self._dynamips_ids[project.id]
def __init__(self, file_name, command=None, show_string=None, match_string=None):
#if not os.path.isfile(file_name):
if not glob.escape(glob.escape(file_name)):
raise FileNotFoundError('%s does not exist' % file_name)
self.file_name=file_name
if show_string:
self.show_string = show_string
else:
self.show_string = file_name
if match_string:
self.show_string = match_string
else:
self.match_string = self.show_string
userpath = os.path.expanduser('~')
if self.show_string[:len(userpath)] == userpath:
self.show_string = '~'+self.show_string[len(userpath):]
if command:
self.command=command
else:
self.command=choose_program(file_name)
self.rating = 1
def _convert_2_0_0_beta_2(topo, topo_path):
"""
Convert topologies from GNS3 2.0.0 beta 2 to beta 3.
Changes:
* Node id folders for dynamips
"""
topo_dir = os.path.dirname(topo_path)
topo["revision"] = 7
for node in topo.get("topology", {}).get("nodes", []):
if node["node_type"] == "dynamips":
node_id = node["node_id"]
dynamips_id = node["properties"]["dynamips_id"]
dynamips_dir = os.path.join(topo_dir, "project-files", "dynamips")
node_dir = os.path.join(dynamips_dir, node_id)
try:
os.makedirs(os.path.join(node_dir, "configs"), exist_ok=True)
for path in glob.glob(os.path.join(glob.escape(dynamips_dir), "*_i{}_*".format(dynamips_id))):
shutil.move(path, os.path.join(node_dir, os.path.basename(path)))
for path in glob.glob(os.path.join(glob.escape(dynamips_dir), "configs", "i{}_*".format(dynamips_id))):
shutil.move(path, os.path.join(node_dir, "configs", os.path.basename(path)))
except OSError as e:
raise aiohttp.web.HTTPConflict(text="Can't convert project {}: {}".format(topo_path, str(e)))
return topo
def _rename_nvram_file(self):
"""
Before starting the VM, rename the nvram and vlan.dat files with the correct IOU application identifier.
"""
destination = self._nvram_file()
for file_path in glob.glob(os.path.join(glob.escape(self.working_dir), "nvram_*")):
shutil.move(file_path, destination)
destination = os.path.join(self.working_dir, "vlan.dat-{:05d}".format(self.application_id))
for file_path in glob.glob(os.path.join(glob.escape(self.working_dir), "vlan.dat-*")):
shutil.move(file_path, destination)
def addMissingImage(self, filename, server, vm_type):
"""
Add a missing image to the queue of images require to be upload on remote server
:param filename: Filename of the image
:param server: Server where image should be uploaded
:param vm_type: Type of the image
"""
if self._asked_for_this_image.setdefault(server.id(), {}).setdefault(filename, False):
return
self._asked_for_this_image[server.id()][filename] = True
if server.isLocal():
return
path = os.path.join(self.getDirectoryForType(vm_type), filename)
if os.path.exists(path):
if self._askForUploadMissingImage(filename, server):
if filename.endswith(".vmdk"):
# A vmdk file could be split in multiple vmdk file
search = glob.escape(path).replace(".vmdk", "-*.vmdk")
for file in glob.glob(search):
self._uploadImageToRemoteServer(file, server, vm_type)
self._uploadImageToRemoteServer(path, server, vm_type)
del self._asked_for_this_image[server.id()][filename]
def find_file(file_dir, file_name):
glob_pattern = '*{}*'.format(glob.escape(file_name))
glob_dir = glob.escape(file_dir)
glob_path = os.path.join(glob_dir, glob_pattern)
debug_print(1, 'Glob pattern: {}'.format(glob_path))
file_list = glob.glob(glob_path)
if len(file_list) > 1:
print('Found more than one file matching pattern {}'\
.format(glob_path))
return None
elif len(file_list) < 1:
print('Could not find any files matching pattern {}'\
.format(glob_path))
return None
else:
return file_list[0]
def load_scripts(script_directory, entry_point, logger=None):
if logger is None:
logger = log
entry_points = []
found_errors = False
glob_pattern = os.path.join(glob.escape(script_directory),
"[0-9][0-9]*.py")
for filename in sorted(glob.glob(glob_pattern)):
try:
with open(filename, "r") as scriptfile:
my_globals = {}
exec(compile(scriptfile.read(), filename, "exec"),
my_globals)
entry = my_globals.get(entry_point)
if entry is not None:
entry_points.append(Script(filename, entry))
else:
found_errors = True
logger.error("Cannot find entry point %r in %r",
entry_point, filename)
except Exception:
found_errors = True
logger.exception("Exception while trying to find entry point %r in %r",
entry_point, filename)
if found_errors:
raise RuntimeError("Errors found while loading scripts."
" See log file for details")
return entry_points
def execute_repair(self): repairedfiles=[] recreatedfiles=[] if self.len_verified_actions>0: for f,retcode in self.verifiedfiles_repairable: yield f retval = self.runpar([self.par_cmd,"r",f]) if retval == 0: if not self.keep_old and os.path.isfile(f+".1"): send2trash(f+".1") repairedfiles.append([ f , retval ]) for f,retcode in self.verifiedfiles_err: yield f pars = glob.glob(glob.escape(f)+'*.par2') for p in pars: send2trash(p) recreatedfiles.append([ f , self.runpar([self.par_cmd,"c","-r"+self.percentage,"-n"+self.nr_parfiles,f]) ]) self.recreate = sorted(recreatedfiles) self.recreate_err = sorted([f for f,err in recreatedfiles if err !=0]) self.fixes = sorted([f for f,err in repairedfiles if err ==0]) self.fixes_err = sorted([f for f,err in repairedfiles if err !=0]) self.len_all_err = self.len_all_err + len(self.recreate_err) + len(self.fixes_err) return
def get_albumart(song):
albumArt = None
if(song != "STOPPED"):
aaDir = re.sub(r"[^/]*$", "", song["file"])
for albumArt in glob.glob(glob.escape(MUSICDIR + aaDir) + PATTERN):
break
return(albumArt)
def complete_path(prefix, line, start, end, ctx, cdpath=True, filtfunc=None):
"""Completes based on a path name."""
# string stuff for automatic quoting
path_str_start = ''
path_str_end = ''
append_end = True
p = _path_from_partial_string(line, end)
lprefix = len(prefix)
if p is not None:
lprefix = len(p[0])
prefix = p[1]
path_str_start = p[2]
path_str_end = p[3]
if len(line) >= end + 1 and line[end] == path_str_end:
append_end = False
tilde = '~'
paths = set()
env = builtins.__xonsh_env__
csc = env.get('CASE_SENSITIVE_COMPLETIONS')
glob_sorted = env.get('GLOB_SORTED')
prefix = glob.escape(prefix)
for s in xt.iglobpath(prefix + '*', ignore_case=(not csc),
sort_result=glob_sorted):
paths.add(s)
if len(paths) == 0 and env.get('SUBSEQUENCE_PATH_COMPLETION'):
# this block implements 'subsequence' matching, similar to fish and zsh.
# matches are based on subsequences, not substrings.
# e.g., ~/u/ro completes to ~/lou/carcolh
# see above functions for details.
p = _splitpath(os.path.expanduser(prefix))
if len(p) != 0:
if p[0] == '':
basedir = ('', )
p = p[1:]
else:
basedir = None
matches_so_far = {basedir}
for i in p:
matches_so_far = _expand_one(matches_so_far, i, csc)
paths |= {_joinpath(i) for i in matches_so_far}
if len(paths) == 0 and env.get('FUZZY_PATH_COMPLETION'):
threshold = env.get('SUGGEST_THRESHOLD')
for s in xt.iglobpath(os.path.dirname(prefix) + '*',
ignore_case=(not csc),
sort_result=glob_sorted):
if xt.levenshtein(prefix, s, threshold) < threshold:
paths.add(s)
if tilde in prefix:
home = os.path.expanduser(tilde)
paths = {s.replace(home, tilde) for s in paths}
if cdpath:
_add_cdpaths(paths, prefix)
paths = set(filter(filtfunc, paths))
paths, _ = _quote_paths({_normpath(s) for s in paths},
path_str_start,
path_str_end,
append_end)
paths.update(filter(filtfunc, _dots(prefix)))
paths.update(filter(filtfunc, _env(prefix)))
return paths, lprefix
def update_srcdir(self, db, source, repo):
need_update = True
# Ensure that every file that Git touches has a timestamp
# which is strictly greater than this, even if filesystem
# timestamps are imprecise.
before_update = time.time()
time.sleep(2)
if not os.path.exists(source):
sys.stderr.write("Cloning {} into {}...\n"
.format(repo, source))
subprocess.check_call(["git", "clone", repo, source])
need_update = False
if not os.path.isdir(os.path.join(source, ".git")):
raise RuntimeError("{!r} exists but is not a Git checkout"
.format(source))
if not os.path.isdir(os.path.join(source, "csv")):
raise RuntimeError("{!r} exists but its contents are "
"not as expected"
.format(source))
if need_update:
sys.stderr.write("Updating {}...\n".format(source))
subprocess.check_call(["git", "pull"], cwd=source)
to_import = []
with db, db.cursor() as cur:
# for f in ${source}/csv/*.csv
for f in glob.iglob(os.path.join(glob.escape(source),
"csv", "*.csv")):
b = os.path.basename(f)
if b.startswith("00-LEGEND-"):
continue
country_code = os.path.splitext(b)[0].upper()
# for the database's sake, we replace 'global' and
# 'cis' with two-letter codes in the "user-assigned"
# ISO 3166-1a2 space.
if country_code == "GLOBAL": country_code = "ZZ"
elif country_code == "CIS": country_code = "XC"
elif len(country_code) != 2:
sys.stderr.write("{!r}: name doesn't contain a 2-letter "
"country code or recognized exception\n"
.format(f))
self.delayed_failure = True
continue
cur.execute("SELECT 1 AS one FROM urls_citizenlab "
"WHERE country = %s "
"LIMIT 1", (country_code,))
prev_import = (cur.fetchone() is not None)
st = os.lstat(f)
if (stat.S_ISREG(st.st_mode) and
(not prev_import or st.st_mtime_ns > before_update)):
to_import.append((f, country_code))
return to_import
def update_file_index(self, filename, index):
pattern = self.dir_to_dl + '*{}.*'.format(glob.escape(filename))
file_to_rename = glob.glob(pattern)[0]
count_len = len(str(index))
new_index = '0'*(6-count_len) + str(index)
old_index = file_to_rename.rpartition('\\')[2][:6]
new_filename = file_to_rename[:].replace(old_index, new_index)
os.rename(file_to_rename, new_filename)
def revisions(self):
revision_to_timestamp = collections.OrderedDict()
prefix = self._path_revision('')
for filename in sorted(glob.glob(glob.escape(prefix) + '*Z')):
revision = filename[len(prefix):]
timestamp = revision[:-1]
revision_to_timestamp[revision] = timestamp
return revision_to_timestamp
def find_fixtures(self, fixture_label):
"""Find fixture files for a given label."""
if fixture_label == READ_STDIN:
return [(READ_STDIN, None, READ_STDIN)]
fixture_name, ser_fmt, cmp_fmt = self.parse_name(fixture_label)
databases = [self.using, None]
cmp_fmts = list(self.compression_formats) if cmp_fmt is None else [cmp_fmt]
ser_fmts = serializers.get_public_serializer_formats() if ser_fmt is None else [ser_fmt]
if self.verbosity >= 2:
self.stdout.write("Loading '%s' fixtures..." % fixture_name)
if os.path.isabs(fixture_name):
fixture_dirs = [os.path.dirname(fixture_name)]
fixture_name = os.path.basename(fixture_name)
else:
fixture_dirs = self.fixture_dirs
if os.path.sep in os.path.normpath(fixture_name):
fixture_dirs = [os.path.join(dir_, os.path.dirname(fixture_name))
for dir_ in fixture_dirs]
fixture_name = os.path.basename(fixture_name)
suffixes = (
'.'.join(ext for ext in combo if ext)
for combo in product(databases, ser_fmts, cmp_fmts)
)
targets = {'.'.join((fixture_name, suffix)) for suffix in suffixes}
fixture_files = []
for fixture_dir in fixture_dirs:
if self.verbosity >= 2:
self.stdout.write("Checking %s for fixtures..." % humanize(fixture_dir))
fixture_files_in_dir = []
path = os.path.join(fixture_dir, fixture_name)
for candidate in glob.iglob(glob.escape(path) + '*'):
if os.path.basename(candidate) in targets:
# Save the fixture_dir and fixture_name for future error messages.
fixture_files_in_dir.append((candidate, fixture_dir, fixture_name))
if self.verbosity >= 2 and not fixture_files_in_dir:
self.stdout.write("No fixture '%s' in %s." %
(fixture_name, humanize(fixture_dir)))
# Check kept for backwards-compatibility; it isn't clear why
# duplicates are only allowed in different directories.
if len(fixture_files_in_dir) > 1:
raise CommandError(
"Multiple fixtures named '%s' in %s. Aborting." %
(fixture_name, humanize(fixture_dir)))
fixture_files.extend(fixture_files_in_dir)
if not fixture_files:
raise CommandError("No fixture named '%s' found." % fixture_name)
return fixture_files
def remove_level_tiles_before(self, level, timestamp):
level_cache = self._get_level(level)
if timestamp == 0:
level_cache.cleanup()
os.unlink(level_cache.mbtile_file)
for file in glob.glob("%s-*" % glob.escape(level_cache.mbtile_file)):
os.unlink(file)
return True
else:
return level_cache.remove_level_tiles_before(level, timestamp)
def iter_paths(paths):
"""Yield paths, expanding globbing patterns if present"""
for path in paths:
if path == escape(path):
# path does not contain globbing patterns
yield path
else:
for filepath in iglob(path, recursive=True):
if not isdir(filepath):
yield filepath
def _parse_line(self, line):
m = re.match(r'.*\[info\] GalleryDownloader: Finished download of gallery: (.+)\n', line)
if not m:
return
name = m.group(1)
paths = self._download_path.glob('{0}*'.format(glob.escape(name)))
paths = list(paths)
if len(paths) != 1:
ERROR('acdul') << '(hah)' << name << 'has multiple target' << paths
return
self._loop.add_callback(self._upload, paths[0])
def sizeChecker(folderSize):
oldFolderSize = folderSize
print("Folder size (old size):" , oldFolderSize , "Elements" )
time.sleep(10) #in seconds
newFolderSize = len([file for file in glob.glob(glob.escape(pathToSearchIn) + '/**/*', recursive=True)])
print("Folder size (new size):" , newFolderSize , "Elements")
if newFolderSize > oldFolderSize:
print ("Changes detected")
reOrderFiles()
else:
print ("No changes")
sizeChecker(newFolderSize) #run again the function after time set
def _load_from_data(self, irc, channel, m):
base_path = os.path.join(conf.supybot.directories.data(), 'Markovgen', channel)
if not os.path.isdir(base_path):
return
for extracter_name in os.listdir(base_path):
extracter = get_extracter(extracter_name)
path = os.path.join(base_path, extracter_name)
path = glob.escape(path)
filenames = rec_list_files(path)
for filename in filenames:
with open(filename, 'rb') as fd:
m.feed_from_file(fd, extracter)
def find_species(output, specie_spec):
if not specie_spec:
return output.model.species()
have_globs = any(glob.escape(pat) != pat for pat in specie_spec)
if not have_globs:
# preserve the specified order
return specie_spec
# use the order in the file (globs can match more than once, so glob order is not useful)
matches = [sp for sp in output.model.species()
if any(fnmatch.fnmatchcase(sp, pat)
for pat in specie_spec)]
if not matches:
raise ValueError('no species matched by {}'.format(specie_spec))
return matches
def get_mp3_files(path):
"""Finds all music files in a directory non-recursively.
:param path: The path in which to look for music files.
:type path: str
:returns: A list of all music files that were found.
:rtype: list
"""
# TODO: Account for bug in glob module in Python versions < 3.4.
# - Before Python 3.4, the glob module didn't automatically escape metacharacters. Escape ?, *, [, and ]. Simply
# replace with [?], [*], [[], []]. When using regex to make a substitution/replacement, make sure not to replace
# the brackets added by the prior substitutions with escaped brackets. That would result in "double escaped"
# brackets.
return glob.glob(glob.escape(path) + "/*.mp3")
def _randomize_id(self, topology):
"""
Iterate on all keys and replace the uuid by a new one.Use by save as
for create new topology. It's also rename the VM folder on disk.
"""
topology["project_id"] = str(uuid.uuid4())
if "nodes" in topology["topology"]:
for key, node in enumerate(topology["topology"]["nodes"]):
old_uuid = topology["topology"]["nodes"][key].get("vm_id", None)
new_uuid = str(uuid.uuid4())
topology["topology"]["nodes"][key]["vm_id"] = new_uuid
if old_uuid:
for path in glob.glob(os.path.join(glob.escape(self.project.filesDir()), "project-files", "*", old_uuid)):
new_path = path.replace(old_uuid, new_uuid)
shutil.move(path, new_path)
return topology
def rglob(dirname, pattern, dirs=False, sort=True):
"""recursive glob, gets all files that match the pattern within the directory tree"""
fns = []
if os.path.isdir(dirname):
fns = glob(os.path.join(dirname, pattern))
dns = [fn for fn
in [escape(os.path.join(dirname, fn))
for fn in os.listdir(dirname)]
if os.path.isdir(fn)]
if dirs==True:
fns += dns
for d in dns:
if os.path.isdir(d):
fns += rglob(d, pattern)
if sort==True:
fns.sort()
return fns
def _kasp_import_keys(self, keydir, bind_keydir, zone_name):
Keymgr.run(keydir, "init")
# add zone if not exists
exitcode, _, _ = Keymgr.run(keydir, "zone", "show", zone_name)
if exitcode != 0:
Keymgr.run_check(keydir, "zone", "add", zone_name)
# retrieve existing keys
_, stdout, _ = Keymgr.run_check(keydir, "zone", "key", "list", zone_name)
tags = [int(re.search(r'\bkeytag\s+(\d+)\b', x).group(1)) for x in stdout.splitlines()]
# import new keys, ignore existing (compare keytag)
assert(zone_name.endswith("."))
for pkey_path in glob.glob("%s/K*.private" % glob.escape(bind_keydir)):
pkey = os.path.basename(pkey_path)
m = re.match(r'K(?P<name>[^+]+)\+(?P<algo>\d+)\+(?P<tag>\d+)\.private', pkey)
if m and m.group("name") == zone_name.lower() and int(m.group("tag")) not in tags:
Keymgr.run_check(keydir, "zone", "key", "import", zone_name, pkey_path)
def _load(self):
"""Load text to memory"""
corpus_directory = glob.escape(self.corpus_directory)
file_list = sorted(glob.glob(os.path.join(corpus_directory, "*.txt")))
for path in file_list:
with open(path, "r", encoding="utf8") as text:
# Read content from text file
content = text.read()
# Preprocessing
content = self._preprocessing(content)
# Create text instance
text = Text(path, os.path.basename(path), content)
# Add text to corpus
self.__corpus.append(text)
def glob_escape(pathname):
"""
Escape all special chars for glob.
For Python after 3.4 we use the glob.escape method.
:returns: Escaped path
"""
if sys.version_info < (3, 4):
# Extracted from Python 3.4 source code
# Escaping is done by wrapping any of "*?[" between square brackets.
# Metacharacters do not work in the drive part and shouldn't be escaped.
magic_check = re.compile('([*?[])')
magic_check_bytes = re.compile(b'([*?[])')
drive, pathname = os.path.splitdrive(pathname)
if isinstance(pathname, bytes):
pathname = magic_check_bytes.sub(br'[\1]', pathname)
else:
pathname = magic_check.sub(r'[\1]', pathname)
return drive + pathname
else:
return glob.escape(pathname)
def check_state(self):
#set that shit if changed in gui
for k, v in self.args.items():
setattr(self, k, v)
self.percentage = str(self.percentage)
if self.runpar([self.par_cmd]) == 200:
return 200
#if 200, then par2 doesnt exist.
allfiles = [
f for f in glob.glob(os.path.join(self.directory, "**", "*"),
recursive=True) if os.path.isfile(f)
] #not sure why required, but glob may introduce paths...
if 'root' in self.excludes:
allfiles = [
f for f in allfiles if os.path.dirname(f) != self.directory
]
self.excludes.remove('root')
for excl in self.excludes:
allfiles = [
f for f in allfiles
if not f.startswith(os.path.join(self.directory, excl))
]
for ext in self.extexcludes:
allfiles = [f for f in allfiles if not f.endswith(ext)]
parrables = [f for f in allfiles if not f.endswith(".par2")]
pattern = '.+vol[0-9]+\+[0-9]+\.par2'
par2corrfiles = [f for f in allfiles if re.search(pattern, f)]
par2files = [
f for f in allfiles
if f.endswith(".par2") and not re.search(pattern, f)
]
par2errcopies = [
f for f in allfiles if f.endswith(".1") or f.endswith(".2")
] #remove copies with errors fixed previously by par.
create = []
verify = []
incomplete = []
#print("Checking files for parrability ...")
for f in parrables:
# check if both or one of the par files is missing
ispar = os.path.isfile(f + ".par2")
isvolpar = len(glob.glob(glob.escape(f) + ".vol*.par2")) > 0
if self.overwrite:
create.append(f)
elif not ispar and not isvolpar:
#both missing
create.append(f)
elif not self.noverify and ispar and isvolpar:
#both present
verify.append(f)
elif self.noverify and ispar and isvolpar:
#both present, but noverify is on, so no action
pass
else:
#one of them is missing but not both
incomplete.append(f)
unused = []
if not self.keep_old:
#print("Checking for unused par2 files ...")
for f in par2files:
if not os.path.isfile(f[:-5]):
unused.append(f)
for f in par2corrfiles:
if not os.path.isfile(f.split('.vol')[0]):
unused.append(f)
self.create = sorted(create)
self.incomplete = sorted(incomplete)
self.verify = sorted(verify)
self.unused = sorted(unused)
self.par2errcopies = sorted(par2errcopies)
self.parrables = sorted(parrables)
self.par2corrfiles = sorted(par2corrfiles)
self.par2files = sorted(par2files)
self.len_all_actions = len(create) + len(incomplete) + len(
verify) + len(unused) + len(par2errcopies)
return
def read_ensemble_preds(self):
"""
reading predictions on ensemble building data set;
populates self.read_preds
"""
self.logger.debug("Read ensemble data set predictions")
if self.y_true_ensemble is None:
try:
self.y_true_ensemble = self.backend.load_targets_ensemble()
except FileNotFoundError:
self.logger.debug(
"Could not find true targets on ensemble data set: %s",
traceback.format_exc(),
)
return False
# no validation predictions so far -- no dir
if not os.path.isdir(self.dir_ensemble):
self.logger.debug("No ensemble dataset prediction directory found")
return False
if self.shared_mode is False:
pred_path = os.path.join(
glob.escape(self.dir_ensemble),
'predictions_ensemble_%s_*.npy' % self.seed,
)
# pSMAC
else:
pred_path = os.path.join(
glob.escape(self.dir_ensemble),
'predictions_ensemble_*_*.npy',
)
y_ens_files = glob.glob(pred_path)
# no validation predictions so far -- no files
if len(y_ens_files) == 0:
self.logger.debug("Found no prediction files on ensemble data set:"
" %s" % pred_path)
return False
n_read_files = 0
for y_ens_fn in y_ens_files:
if self.read_at_most and n_read_files >= self.read_at_most:
# limit the number of files that will be read
# to limit memory consumption
break
if not y_ens_fn.endswith(".npy"):
self.logger.info('Error loading file (not .npy): %s', y_ens_fn)
continue
match = self.model_fn_re.search(y_ens_fn)
_seed = int(match.group(1))
_num_run = int(match.group(2))
if not self.read_preds.get(y_ens_fn):
self.read_preds[y_ens_fn] = {
"ens_score": -1,
"mtime_ens": 0,
"mtime_valid": 0,
"mtime_test": 0,
"seed": _seed,
"num_run": _num_run,
Y_ENSEMBLE: None,
Y_VALID: None,
Y_TEST: None,
# Lazy keys so far:
# 0 - not loaded
# 1 - loaded and in memory
# 2 - loaded but dropped again
"loaded": 0
}
if self.read_preds[y_ens_fn]["mtime_ens"] == os.path.getmtime(
y_ens_fn):
# same time stamp; nothing changed;
continue
# actually read the predictions and score them
try:
with open(y_ens_fn, 'rb') as fp:
y_ensemble = self._read_np_fn(fp=fp)
score = calculate_score(
solution=self.
y_true_ensemble, # y_ensemble = y_true for ensemble set
prediction=y_ensemble,
task_type=self.task_type,
metric=self.metric,
all_scoring_functions=False)
if self.read_preds[y_ens_fn]["ens_score"] > -1:
self.logger.critical(
'Changing ensemble score for file %s from %f to %f '
'because file modification time changed? %f - %f',
y_ens_fn,
self.read_preds[y_ens_fn]["ens_score"],
score,
self.read_preds[y_ens_fn]["mtime_ens"],
os.path.getmtime(y_ens_fn),
)
self.read_preds[y_ens_fn]["ens_score"] = score
self.read_preds[y_ens_fn][Y_ENSEMBLE] = y_ensemble
self.read_preds[y_ens_fn]["mtime_ens"] = os.path.getmtime(
y_ens_fn)
self.read_preds[y_ens_fn]["loaded"] = 1
n_read_files += 1
except:
self.logger.warning(
'Error loading %s: %s',
y_ens_fn,
traceback.format_exc(),
)
self.read_preds[y_ens_fn]["ens_score"] = -1
self.logger.debug(
'Done reading %d new prediction files. Loaded %d predictions in '
'total.', n_read_files,
np.sum([pred["loaded"] > 0 for pred in self.read_preds.values()]))
return True
def parse_directory(
directory: str,
verbose: bool,
excluded_files: List[str],
tree_arg: int,
short: bool,
mode: int,
oldparser: bool,
) -> int:
if tree_arg:
assert mode == PARSE, "Mode should be 1 (parse), when comparing the generated trees"
if oldparser and tree_arg:
print("Cannot specify tree argument with the cpython parser.", file=sys.stderr)
return 1
# For a given directory, traverse files and attempt to parse each one
# - Output success/failure for each file
errors = 0
files = []
trees = {} # Trees to compare (after everything else is done)
total_seconds = 0
for file in sorted(glob(os.path.join(escape(directory), f"**/*.py"), recursive=True)):
# Only attempt to parse Python files and files that are not excluded
if any(PurePath(file).match(pattern) for pattern in excluded_files):
continue
with tokenize.open(file) as f:
source = f.read()
try:
result, dt = parse_file(source, file, mode, oldparser)
total_seconds += dt
if tree_arg:
trees[file] = result
report_status(succeeded=True, file=file, verbose=verbose, short=short)
except SyntaxError as error:
if is_parsing_failure(source):
print(f"File {file} cannot be parsed by either parser.")
else:
report_status(
succeeded=False, file=file, verbose=verbose, error=error, short=short
)
errors += 1
files.append(file)
t1 = time.time()
generate_time_stats(files, total_seconds)
if short:
print_memstats()
if errors:
print(f"Encountered {errors} failures.", file=sys.stderr)
# Compare trees (the dict is empty unless -t is given)
compare_trees_errors = 0
for file, tree in trees.items():
if not short:
print("Comparing ASTs for", file)
if compare_trees(tree, file, verbose, tree_arg >= 2) == 1:
compare_trees_errors += 1
if errors or compare_trees_errors:
return 1
return 0
def test_glob_escape(self):
self.assertEqual(glob.escape('S01E01 - Show Name [SickChill].avi'), 'S01E01 - Show Name [[]SickChill].avi')
self.assertEqual(glob.escape('S01E01 - Show Name [SickChill].avi'), 'S01E01 - Show Name [[]SickChill].avi')
self.assertEqual(glob.escape('S01E01 - Show Name [SickChill].avi'), 'S01E01 - Show Name [[]SickChill].avi')
self.assertEqual(glob.escape('S01E01 - Show Name [SickChill].avi'), 'S01E01 - Show Name [[]SickChill].avi')
def scan(path, name):
pattern = os.path.join(glob.escape(path), '**', name)
return sorted(glob.glob(pattern, recursive=True))
def escapePathWildcards(path): """Escape wildcards in the path""" # TODO: Implement this manually if not supported by the current vresion of Python. # Though, it is not very important, because occurs extremely seldom return glob.escape(path) if hasattr(glob, 'escape') else path
def complete_path(prefix, line, start, end, ctx, cdpath=True, filtfunc=None):
"""Completes based on a path name."""
# string stuff for automatic quoting
path_str_start = ""
path_str_end = ""
append_end = True
p = _path_from_partial_string(line, end)
lprefix = len(prefix)
if p is not None:
lprefix = len(p[0])
prefix = p[1]
path_str_start = p[2]
path_str_end = p[3]
if len(line) >= end + 1 and line[end] == path_str_end:
append_end = False
tilde = "~"
paths = set()
env = builtins.__xonsh_env__
csc = env.get("CASE_SENSITIVE_COMPLETIONS")
glob_sorted = env.get("GLOB_SORTED")
prefix = glob.escape(prefix)
for s in xt.iglobpath(prefix + "*",
ignore_case=(not csc),
sort_result=glob_sorted):
paths.add(s)
if len(paths) == 0 and env.get("SUBSEQUENCE_PATH_COMPLETION"):
# this block implements 'subsequence' matching, similar to fish and zsh.
# matches are based on subsequences, not substrings.
# e.g., ~/u/ro completes to ~/lou/carcolh
# see above functions for details.
p = _splitpath(os.path.expanduser(prefix))
if len(p) != 0:
if p[0] == "":
basedir = ("", )
p = p[1:]
else:
basedir = None
matches_so_far = {basedir}
for i in p:
matches_so_far = _expand_one(matches_so_far, i, csc)
paths |= {_joinpath(i) for i in matches_so_far}
if len(paths) == 0 and env.get("FUZZY_PATH_COMPLETION"):
threshold = env.get("SUGGEST_THRESHOLD")
for s in xt.iglobpath(
os.path.dirname(prefix) + "*",
ignore_case=(not csc),
sort_result=glob_sorted,
):
if xt.levenshtein(prefix, s, threshold) < threshold:
paths.add(s)
if tilde in prefix:
home = os.path.expanduser(tilde)
paths = {s.replace(home, tilde) for s in paths}
if cdpath and cd_in_command(line):
_add_cdpaths(paths, prefix)
paths = set(filter(filtfunc, paths))
paths, _ = _quote_paths({_normpath(s)
for s in paths}, path_str_start, path_str_end,
append_end)
paths.update(filter(filtfunc, _dots(prefix)))
paths.update(filter(filtfunc, _env(prefix)))
return paths, lprefix
def list_all_models(self, seed: int) -> List[str]:
runs_directory = self.get_runs_directory()
model_files = glob.glob(
os.path.join(glob.escape(runs_directory), '%d_*' % seed,
'%s.*.*.model' % seed))
return model_files
def _cleanup(self):
for tmp in glob.iglob(glob.escape(self._path_vid) + '*.img*~*'):
_remove_file(tmp)
_remove_file(self._path_import)
def test_prefix_cleanup():
# regular
assert wr.s3._list._prefix_cleanup("foo*") == "foo"
assert wr.s3._list._prefix_cleanup("*foo") == ""
assert wr.s3._list._prefix_cleanup("foo*boo") == "foo"
assert wr.s3._list._prefix_cleanup("foo?") == "foo"
assert wr.s3._list._prefix_cleanup("?foo") == ""
assert wr.s3._list._prefix_cleanup("foo?boo") == "foo"
assert wr.s3._list._prefix_cleanup("[]foo") == ""
assert wr.s3._list._prefix_cleanup("foo[]") == "foo"
assert wr.s3._list._prefix_cleanup("foo[]boo") == "foo"
# escaped
assert wr.s3._list._prefix_cleanup(glob.escape("foo*")) == "foo"
assert wr.s3._list._prefix_cleanup(glob.escape("*foo")) == ""
assert wr.s3._list._prefix_cleanup(glob.escape("foo*boo")) == "foo"
assert wr.s3._list._prefix_cleanup(glob.escape("foo?")) == "foo"
assert wr.s3._list._prefix_cleanup(glob.escape("?foo")) == ""
assert wr.s3._list._prefix_cleanup(glob.escape("foo?boo")) == "foo"
assert wr.s3._list._prefix_cleanup(glob.escape("[]foo")) == glob.escape("")
assert wr.s3._list._prefix_cleanup(
glob.escape("foo[]")) == glob.escape("foo")
assert wr.s3._list._prefix_cleanup(
glob.escape("foo[]boo")) == glob.escape("foo")
def init():
folderSize = len([
file for file in glob.glob(glob.escape(pathToSearchIn) + '/**/*',
recursive=True)
])
return folderSize
def get_valid_test_preds(self, selected_keys: list):
"""
get valid and test predictions from disc
and store them in self.read_preds
Parameters
---------
selected_keys: list
list of selected keys of self.read_preds
Return
------
success_keys:
all keys in selected keys for which we could read the valid and test predictions
"""
success_keys_valid = []
success_keys_test = []
for k in selected_keys:
valid_fn = glob.glob(
os.path.join(
glob.escape(self.dir_valid),
'predictions_valid_%d_%d.npy' %
(self.read_preds[k]["seed"],
self.read_preds[k]["num_run"])))
test_fn = glob.glob(
os.path.join(
glob.escape(self.dir_test), 'predictions_test_%d_%d.npy' %
(self.read_preds[k]["seed"],
self.read_preds[k]["num_run"])))
# TODO don't read valid and test if not changed
if len(valid_fn) == 0:
# self.logger.debug("Not found validation prediction file "
# "(although ensemble predictions available): "
# "%s" % valid_fn)
pass
else:
valid_fn = valid_fn[0]
if self.read_preds[k]["mtime_valid"] == os.path.getmtime(valid_fn) \
and self.read_preds[k][Y_VALID] is not None:
success_keys_valid.append(k)
continue
try:
with open(valid_fn, 'rb') as fp:
y_valid = self._read_np_fn(fp)
self.read_preds[k][Y_VALID] = y_valid
success_keys_valid.append(k)
self.read_preds[k]["mtime_valid"] = os.path.getmtime(
valid_fn)
except Exception as e:
self.logger.warning('Error loading %s: %s', valid_fn,
traceback.format_exc())
if len(test_fn) == 0:
# self.logger.debug("Not found test prediction file (although "
# "ensemble predictions available):%s" %
# test_fn)
pass
else:
test_fn = test_fn[0]
if self.read_preds[k]["mtime_test"] == \
os.path.getmtime(test_fn) \
and self.read_preds[k][Y_TEST] is not None:
success_keys_test.append(k)
continue
try:
with open(test_fn, 'rb') as fp:
y_test = self._read_np_fn(fp)
self.read_preds[k][Y_TEST] = y_test
success_keys_test.append(k)
self.read_preds[k]["mtime_test"] = os.path.getmtime(
test_fn)
except Exception as e:
self.logger.warning('Error loading %s: %s', test_fn,
traceback.format_exc())
return success_keys_valid, success_keys_test
def _remove_incomplete_files(self):
for tmp in glob.iglob(glob.escape(self._path_vid) + '*.img*~*'):
_remove_file(tmp)
_remove_file(self._path_import)
def execute(self):
create = self.create
incomplete = self.incomplete
verify = self.verify
unused = self.unused
create.extend(incomplete)
unused.extend(self.par2errcopies)
errorcodes = {
0: "Succes.", #can mean no error, but also succesfully repaired!
1: "Repairable damage found.",
2: "Irreparable damage found.",
3: "Invalid commandline arguments.",
4: "Parity file unusable.",
5: "Repair failed.",
6: "IO error.",
7: "Internal error",
8: "Out of memory.",
100: "os.remove succeeded.",
101: "os.remove did not succeed.",
200: "par2 command not found."
}
createdfiles = []
createdfiles_err = []
if len(create) > 0:
#print('Creating ...')
for f in create:
yield f
pars = glob.glob(glob.escape(f) + '*.par2')
for p in pars:
os.remove(p)
createdfiles.append([
f,
self.runpar([
self.par_cmd, "c", "-r" + self.percentage,
"-n" + self.nr_parfiles, f
])
])
createdfiles_err = [[i, j] for i, j in createdfiles
if j != 0 and j != 100]
verifiedfiles = []
verifiedfiles_succes = []
verifiedfiles_err = []
verifiedfiles_repairable = []
if not self.noverify and not self.overwrite and len(verify) > 0:
#print('Verifying ...')
for f in verify:
yield f
verifiedfiles.append([f, self.runpar([self.par_cmd, "v", f])])
verifiedfiles_err = [[i, j] for i, j in verifiedfiles
if j != 0 and j != 100 and j != 1]
verifiedfiles_repairable = [[i, j] for i, j in verifiedfiles
if j == 1]
verifiedfiles_succes = [[i, j] for i, j in verifiedfiles if j == 0]
removedfiles = []
removedfiles_err = []
if not self.keep_old and len(unused) > 0:
#print('Removing ...')
for f in unused:
yield f
if os.path.isfile(
f): # so os.remove always succeeds and returns None
os.remove(f)
removedfiles.append([f, 100])
else:
removedfiles.append([f, 101])
removedfiles_err = [[i, j] for i, j in removedfiles
if j != 0 and j != 100]
self.createdfiles = createdfiles
self.verifiedfiles_succes = verifiedfiles_succes
self.removedfiles = removedfiles
self.createdfiles_err = createdfiles_err
self.verifiedfiles_err = verifiedfiles_err
self.verifiedfiles_repairable = verifiedfiles_repairable
self.removedfiles_err = removedfiles_err
self.len_all_err = len(self.createdfiles_err) + len(
self.verifiedfiles_err) + len(self.verifiedfiles_repairable) + len(
self.removedfiles_err)
self.len_verified_actions = len(self.verifiedfiles_err) + len(
self.verifiedfiles_repairable)
return
def getExcelLinks(args):
print("Getting .xlsx Links")
# Handle arguments and flags
parser = argparse.ArgumentParser(usage=instructions, add_help=False)
parser.add_argument("--help", "-h", action="store_true")
parser.add_argument("-r", action="store_true")
parser.add_argument("-l", action="store_true")
parser.add_argument("-o", action="store")
parser.add_argument("file_names", nargs="*")
args = parser.parse_args(args)
# Replace arguments with wildcards with their expansion.
# If a string does not contain a wildcard, glob will return it as is.
# Mostly important if we run this on Windows systems.
file_names = []
for name in args.file_names:
file_names += glob.glob(glob.escape(name))
# If the filenames don't exist, say so and quit.
if file_names == []:
sys.exit("No file or directory found by that name.")
# Don't run the script on itself.
if sys.argv[0] in file_names:
file_names.remove(sys.argv[0])
if args.help:
sys.exit(instructions)
filecount = 0
linklist = []
target_is_folder = False
for name in file_names:
# Make sure single files exist.
assert os.path.exists(name), "File or directory not found."
# If it's just a file...
if os.path.isfile(name):
# Make sure this is an Excel file (just check extension)
if name.lower().endswith(".xlsx"):
# Get the links from that file.
newlinks = getLinks(name, args, False)
if newlinks != []:
linklist.extend()
filecount += 1
# If it's a directory:
if os.path.isdir(name):
target_is_folder = True
# Recursive version using os.walk for all levels.
if args.r:
for dirpath, dirnames, files in os.walk(name):
for eachfile in files:
# Get links from every file in that directory.
if eachfile.lower().endswith(".xlsx"):
linklist.extend(getLinks(eachfile, args, dirpath))
filecount += 1
# Non-recursive version breaks os.walk after the first level.
else:
topfiles = []
for (dirpath, dirnames, files) in os.walk(name):
topfiles.extend(files)
break
for eachfile in topfiles:
if eachfile.lower().endswith(".xlsx"):
linklist.extend(getLinks(eachfile, args, dirpath))
filecount += 1
if args.l:
# When asked to return a list, quietly return one and stop.
return linklist
else:
# Otherwise, create output file as sibling to the original target of the script.
outFileName = args.o if args.o else "Excel_Doc_Links.csv"
if target_is_folder:
outFileFolder = os.path.abspath(
os.path.join(file_names[0], os.pardir))
outFilePath = os.path.join(outFileFolder, outFileName)
else:
outFilePath = os.path.join(os.path.dirname(file_names[0]),
outFileName)
writeFile(linklist, filecount, outFileName, outFilePath, args)
def revisions(self):
prefix = self._path_clean + '.'
paths = glob.iglob(glob.escape(prefix) + '*@*Z')
items = (path[len(prefix):-1].split('@') for path in paths)
return collections.OrderedDict(
sorted(items, key=lambda item: int(item[0])))
def check_dimension(self):
if os.path.getsize(self.file_name) > 50 * 1024 * 1023:
os.system('split -b 49M "{0}" "{1}"'.format(
self.file_name, self.file_name))
os.remove(self.file_name)
return glob(escape(self.file_name) + '*')
def check_escape(self, arg, expected):
self.assertEqual(glob.escape(arg), expected)
self.assertEqual(glob.escape(os.fsencode(arg)), os.fsencode(expected))
def _load(self,
names=None,
variables=None,
iterations=None,
repetition=None,
sample_index=None,
configs=None,
aliases=None,
use_alias=True,
concat_config=False,
drop_columns=True,
**kwargs):
self.configs = []
for filename in glob.glob(os.path.join(self.path, 'configs', '*')):
with open(filename, 'rb') as f:
self.configs.append(dill.load(f))
if len(kwargs) > 0:
if configs is None:
configs = kwargs
else:
configs.update(kwargs)
if configs is not None:
self._filter_configs(config=configs, repetition=repetition)
elif aliases is not None:
self._filter_configs(alias=aliases, repetition=repetition)
elif repetition is not None:
self._filter_configs(repetition=repetition)
if names is None:
names = list(self.description['executables'].keys())
if variables is None:
variables = [
variable for unit in self.description['executables'].values()
for variable in unit['variables']
]
names = self._get_list(names)
variables = self._get_list(variables)
iterations = self._get_list(iterations)
all_results = []
for config_alias in self.configs:
alias_str = config_alias.alias(as_string=True)
_repetition = config_alias.pop_config('repetition')
_update = config_alias.pop_config('update')
path = os.path.join(self.path, 'results', alias_str)
for unit in names:
sample_folders = glob.glob(
os.path.join(glob.escape(path), sample_index or '*'))
for sample_folder in sample_folders:
files = glob.glob(
glob.escape(os.path.join(sample_folder, unit)) +
'_[0-9]*')
files = self._sort_files(files, iterations)
if len(files) != 0:
res = []
for filename, iterations_to_load in files.items():
with open(filename, 'rb') as file:
res.append(
self._slice_file(dill.load(file),
iterations_to_load,
variables))
res = self._concat(res, variables)
self._fix_length(res)
config_alias.pop_config('_dummy')
if concat_config:
res['config'] = config_alias.alias(as_string=True)
if use_alias:
if not concat_config or not drop_columns:
res.update(config_alias.alias(as_string=False))
else:
res.update(config_alias.config())
res.update(
{'repetition': _repetition.config()['repetition']})
res.update({'update': _update.config()['update']})
all_results.append(pd.DataFrame({'name': unit, **res}))
return pd.concat(all_results, sort=False).reset_index(
drop=True) if len(all_results) > 0 else pd.DataFrame(None)
def safe_glob(self, pathname):
return glob.iglob(glob.escape(pathname) + '*')
def delete_files():
# we don't know the precise name the underlying database uses
# so we use glob to locate all names
for f in glob.glob(glob.escape(_fname) + "*"):
os_helper.unlink(f)
def check_escape(self, arg, expected):
self.assertEqual(glob.escape(arg), expected)
self.assertEqual(glob.escape(os.fsencode(arg)), os.fsencode(expected))
def run(self):
logger = self.get_logger()
with self._appimage.mount() as mountpoint:
# find desktop file, get name of icon and look for it in AppDir root
desktop_files = glob.glob(op.join(mountpoint, "*.desktop"))
# we can of course check the validity of all icon files we find, but there's always one main icon that is
# referenced from the desktop file
main_icon_name = None
if not desktop_files:
logger.error("Could not find desktop file in root directory")
else:
logger.debug("Found desktop files: %s", desktop_files)
desktop_file = desktop_files[0]
logger.info("Extracting icon name from desktop file: %s",
desktop_file)
with open(desktop_file) as f:
# find Icon= entry and get the name of the icon file to look for
# we don't need to check things like "is there just one Icon entry" etc., that's the job of another
# test
desktop_file_contents = f.read()
# note for self: Python's re doesn't do multiline unless explicitly asked for with re.MULTILINE
match = re.search(r"Icon=(.+)", desktop_file_contents)
if not match:
logger.error(
"Could not find Icon= entry in desktop file")
else:
main_icon_name = match.group(1)
# to be able to filter out non-icon files with the same prefix in the AppDir root
known_image_exts = ("png", "xpm", "svg", "jpg")
# assuming test broke
# now prove me wrong!
root_icon_valid = False
if main_icon_name is not None:
if "/" in main_icon_name:
logger.error(
"main icon name is a path, not a filename (contains /)"
)
else:
# properly escape some "magic" characters in the original filename so they won't be interpreted by glob
fixed_main_icon_name = glob.escape(main_icon_name)
# build glob pattern
pattern = "{}.*".format(fixed_main_icon_name)
logger.debug(
"Trying to find main icon in AppDir root, pattern: {}".
format(repr(pattern)))
appdir_root_icons = glob.glob(op.join(mountpoint, pattern))
if not appdir_root_icons:
logger.error(
"Could not find suitable icon for desktop file's Icon= entry"
)
else:
# filter out all files with a not-well-known extension
appdir_root_icons = [
i for i in appdir_root_icons if op.splitext(i)
[-1].lstrip(".") in known_image_exts
]
if len(appdir_root_icons) > 1:
logger.warning(
"Multiple matching icons found in AppDir root, checking all"
)
main_icon_check_results = []
for icon in appdir_root_icons:
valid = self._check_icon_for_valid_resolution(icon)
main_icon_check_results.append(valid)
# if only one of the checks failed, we can't guarantee a working root icon
root_icon_valid = all(main_icon_check_results)
yield TestResult(root_icon_valid, "icons.valid_appdir_root_icon",
"Valid icon in AppDir root")
# next, check that .DirIcon is available and valid
dotdiricon_valid = self._check_icon_for_valid_resolution(
op.join(mountpoint, ".DirIcon"))
yield TestResult(dotdiricon_valid, "icons.valid_dotdiricon",
"Valid icon file in .DirIcon")
# now check all remaining icons in usr/share/icons/...
other_icons_root_path = op.join(mountpoint,
"usr/share/icons/**/*.*")
other_icons = glob.glob(other_icons_root_path, recursive=True)
# assume everything works
# prove me wrong!
other_icons_checks_success = True
for abs_path in other_icons:
# check if this icon even belongs to here
rel_path = op.relpath(abs_path,
op.join(mountpoint, "usr/share/icons"))
filename = op.basename(abs_path)
split_fname = op.splitext(filename)
# not an error, but means we don't have to process that file any further
if split_fname[0] != main_icon_name:
logger.warning(
"Icon found whose file name doesn't match the Icon= entry in desktop file: %s",
rel_path)
else:
# also just a warning
if split_fname[1].lstrip(".") not in known_image_exts:
logger.warning("Icon has invalid extension: %s",
split_fname[1])
logger.debug(
"checking whether icon has good resolution in general")
if not self._check_icon_for_valid_resolution(abs_path):
logger.warning("icon %s has invalid resolution",
abs_path)
other_icons_checks_success = False
logger.debug(
"checking whether icon is in correct location")
# split path into the interesting components: icon theme, resolution and actual filename
split_path = rel_path.split("/")
# find resolution component in split path
path_res = None
def extract_res_from_path_component(s):
if s == "scalable":
return s
return tuple([int(i) for i in s.split("x")])
if len(split_path) != 4 or split_path[2] != "apps":
logger.warning("Icon %s is in non-standard location",
rel_path)
else:
try:
path_res = extract_res_from_path_component(
split_path[1])
except:
pass
if not path_res:
# something's definitely broken
other_icons_checks_success = False
logger.warning(
"Could not find icon resolution at expected position in path, "
"trying to guess from entire path")
for comp in split_path:
try:
path_res = extract_res_from_path_component(
comp)
except:
pass
else:
break
if not path_res:
other_icons_checks_success = False
logger.error(
"Could not extract resolution from icon path,"
"should be usr/share/icons/<theme>/<res>/apps/<name>.<ext>"
)
else:
# make sure extracted resolution corresponds to the file's resolution
actual_res = self._get_icon_res(abs_path)
if actual_res != path_res:
other_icons_checks_success = False
logger.error(
"Icon resolution doesn't match resolution in path: %s (file resolution is %s)",
path_res, actual_res)
if not other_icons_checks_success:
logger.warning("no other icons found")
yield TestResult(other_icons_checks_success,
"icons.valid_other_icons",
"Other integration icons valid")
return
yaml_data = yaml_data['settings']
for k, v in dict_attrs(yaml_data, autoconfig=autoconfig):
if xresources and isinstance(v, str):
v = v.format_map(xresources)
yield k, v
config.load_autoconfig(True)
xresources = read_xresources('*')
config_files = (
fn
for fn in glob.iglob(os.path.join(glob.escape(config.configdir), '*.yml')))
config_data = {}
for filename in config_files:
for k, v in read_yml(filename, xresources=xresources):
if k in CONFIG_KEYS_DICT:
if k not in config_data:
config_data[k] = config.get(k)
if all(isinstance(x, dict) for x in config_data[k].values()):
v = {
subkey: {
**config_data[k].get(subkey, {}),
**subval
}
for subkey, subval in v.items()
dot are special cases that are not matched by '*' and '?'
patterns.
If recursive is true, the pattern '**' will match any files and
zero or more directories and subdirectories.
DATA
__all__ = ['glob', 'iglob', 'escape']
FILE
c:\python37\lib\glob.py
'''
print(dir(glob))
'''
'escape', 'fnmatch', 'glob', 'glob0', 'glob1', 'has_magic', 'iglob', 'magic_check', 'magic_check_bytes', 'os', 're']
'''
glob.escape()
'''
유니스 셀이 사용하는 규칙에 따라 지정된 패턴과 일치하는 모든 경로명을 찾는다.
결과는 무순서로 반환된다.
*, ?, []로 표시되는 문자 범위는 올바르게 일치한다.
리터럴 일치를 위해서는 대괄호 안에 메타문자를 넣는다. 예를 들면 '[?]'는 '?'
문자와 일치한다.
glob.glob(pathname, *, recursive=False) :
경로 지정을 포함하는 문자열인 pathname에 일치하는 경로 이름의 비어있을 수 있는 리스트를 반환한다.
pathname은 절대경로와 상대경로를 지정할 수 있다.
glob.iglob(pathname, *, recursive=False)
glob()과 같은 값을 산출하지만 이터레이터를 반환한다.
def on_build_gen(
self, cxx_gen_dir, missing_reporter: Optional[MissingReporter] = None
):
if not self.cfg.generate:
return
cxx_gen_dir = join(cxx_gen_dir, self.name)
if missing_reporter:
report_only = True
else:
report_only = False
missing_reporter = MissingReporter()
thisdir = abspath(dirname(__file__))
hppoutdir = join(self.rpy_incdir, "rpygen")
tmpl_dir = join(thisdir, "templates")
cpp_tmpl = join(tmpl_dir, "gen_pybind11.cpp.j2")
hpp_tmpl = join(tmpl_dir, "gen_cls_trampoline.hpp.j2")
classdeps_tmpl = join(tmpl_dir, "gen_classdeps.json.j2")
pp_includes = self._all_includes(False)
# TODO: only regenerate files if the generated files
# have changed
if not report_only:
if self.dev_config.only_generate is None:
shutil.rmtree(cxx_gen_dir, ignore_errors=True)
shutil.rmtree(hppoutdir, ignore_errors=True)
os.makedirs(cxx_gen_dir, exist_ok=True)
os.makedirs(hppoutdir, exist_ok=True)
per_header = False
data_fname = self.cfg.generation_data
if self.cfg.generation_data:
datapath = join(self.setup_root, normpath(self.cfg.generation_data))
per_header = isdir(datapath)
if not per_header:
data = self._load_generation_data(datapath)
else:
data = HooksDataYaml()
sources = self.cfg.sources[:]
pp_defines = [self._cpp_version] + self.platform.defines + self.cfg.pp_defines
casters = self._all_casters()
# These are written to file to make it easier for dev mode to work
classdeps = {}
processor = ConfigProcessor(tmpl_dir)
if self.dev_config.only_generate is not None:
only_generate = {n: True for n in self.dev_config.only_generate}
else:
only_generate = None
generation_search_path = [self.root] + self._all_includes(False)
for gen in self.cfg.generate:
for name, header in gen.items():
header = normpath(header)
for path in generation_search_path:
header_path = join(path, header)
if exists(header_path):
break
else:
import pprint
pprint.pprint(generation_search_path)
raise ValueError("could not find " + header)
if report_only:
templates = []
class_templates = []
else:
cpp_dst = join(cxx_gen_dir, f"{name}.cpp")
sources.append(cpp_dst)
classdeps_dst = join(cxx_gen_dir, f"{name}.json")
classdeps[name] = classdeps_dst
hpp_dst = join(
hppoutdir,
"{{ cls['namespace'] | replace(':', '_') }}__{{ cls['name'] }}.hpp",
)
templates = [
{"src": cpp_tmpl, "dst": cpp_dst},
{"src": classdeps_tmpl, "dst": classdeps_dst},
]
class_templates = [{"src": hpp_tmpl, "dst": hpp_dst}]
if only_generate is not None and not only_generate.pop(name, False):
continue
if per_header:
data_fname = join(datapath, name + ".yml")
if not exists(data_fname):
print("WARNING: could not find", data_fname)
data = HooksDataYaml()
else:
data = self._load_generation_data(data_fname)
# for each thing, create a h2w configuration dictionary
cfgd = {
# generation code depends on this being just one header!
"headers": [header_path],
"templates": templates,
"class_templates": class_templates,
"preprocess": True,
"pp_retain_all_content": False,
"pp_include_paths": pp_includes,
"pp_defines": pp_defines,
"vars": {"mod_fn": name},
}
cfg = Config(cfgd)
cfg.validate()
cfg.root = self.incdir
hooks = Hooks(data, casters)
processor.process_config(cfg, data, hooks)
hooks.report_missing(data_fname, missing_reporter)
if only_generate:
unused = ", ".join(sorted(only_generate))
# raise ValueError(f"only_generate specified unused headers! {unused}")
# TODO: make this a warning
if not report_only:
for name, contents in missing_reporter.as_yaml():
print("WARNING: some items not in generation yaml for", basename(name))
print(contents)
# generate an inline file that can be included + called
if not report_only:
self._write_wrapper_hpp(cxx_gen_dir, classdeps)
gen_includes = [cxx_gen_dir]
else:
gen_includes = []
# Add the root to the includes (but only privately)
root_includes = [self.root]
# update the build extension so that build_ext works
self.extension.sources = sources
# use normpath to get rid of .. otherwise gcc is weird
self.extension.include_dirs = [
normpath(p)
for p in (self._all_includes(True) + gen_includes + root_includes)
]
self.extension.library_dirs = self._all_library_dirs()
self.extension.libraries = self._all_library_names()
self.extension.extra_objects = self._all_extra_objects()
for f in glob.glob(join(glob.escape(hppoutdir), "*.hpp")):
self._add_generated_file(f)
def __init__(self):
self._handler = Handler()
# See ydl_opts['forcejson']
self._expect_info_dict_json = False
self._info_dict = None
self._allow_authentication_request = True
self._skip_authentication = False
self._skipped_count = 0
self.ydl_opts = {
'logger': self,
'logtostderr': True,
'no_color': True,
'progress_hooks': [self._on_progress],
'fixup': 'detect_or_warn',
'ignoreerrors': True, # handled via logger error callback
'retries': 10,
'fragment_retries': 10,
'postprocessors': [{
'key': 'XAttrMetadata'
}]
}
url = self._handler.get_url()
download_dir = os.path.abspath(self._handler.get_download_dir())
with tempfile.TemporaryDirectory() as temp_dir:
self.ydl_opts['cookiefile'] = os.path.join(temp_dir, 'cookies')
# Collect info without downloading videos
testplaylist_dir = os.path.join(temp_dir, 'testplaylist')
noplaylist_dir = os.path.join(temp_dir, 'noplaylist')
fullplaylist_dir = os.path.join(temp_dir, 'fullplaylist')
for path in [testplaylist_dir, noplaylist_dir, fullplaylist_dir]:
os.mkdir(path)
self.ydl_opts['writeinfojson'] = True
self.ydl_opts['writethumbnail'] = True
self.ydl_opts['skip_download'] = True
self.ydl_opts['playlistend'] = 2
self.ydl_opts['outtmpl'] = '%(autonumber)s.%(ext)s'
# Test playlist
info_testplaylist, skipped_testplaylist = self._load_playlist(
testplaylist_dir, url)
self.ydl_opts['noplaylist'] = True
if len(info_testplaylist) + skipped_testplaylist > 1:
info_noplaylist, skipped_noplaylist = self._load_playlist(
noplaylist_dir, url)
else:
info_noplaylist = info_testplaylist
skipped_noplaylist = skipped_testplaylist
del self.ydl_opts['noplaylist']
del self.ydl_opts['playlistend']
if (len(info_testplaylist) + skipped_testplaylist >
len(info_noplaylist) + skipped_noplaylist):
self.ydl_opts['noplaylist'] = (
not self._handler.on_playlist_request())
if not self.ydl_opts['noplaylist']:
info_playlist, _ = self._load_playlist(
fullplaylist_dir, url)
else:
info_playlist = info_noplaylist
elif len(info_testplaylist) + skipped_testplaylist > 1:
info_playlist, _ = self._load_playlist(fullplaylist_dir, url)
else:
info_playlist = info_testplaylist
# Download videos
self._allow_authentication_request = False
del self.ydl_opts['writeinfojson']
del self.ydl_opts['writethumbnail']
del self.ydl_opts['skip_download']
# Include id and format_id in outtmpl to prevent youtube-dl
# from continuing wrong file
self.ydl_opts['outtmpl'] = '%(id)s.%(format_id)s.%(ext)s'
# Output info_dict as JSON handled via logger debug callback
self.ydl_opts['forcejson'] = True
mode = self._handler.get_mode()
if mode == 'audio':
resolution = MAX_RESOLUTION
prefer_mpeg = False
self.ydl_opts['format'] = 'bestaudio/best'
self.ydl_opts['postprocessors'].insert(
0, {
'key': 'FFmpegExtractAudio',
'preferredcodec': 'mp3',
'preferredquality': '192'
})
self.ydl_opts['postprocessors'].insert(
1, {
'key': 'EmbedThumbnail',
'already_have_thumbnail': True
})
else:
resolution = self._handler.get_resolution()
prefer_mpeg = self._handler.get_prefer_mpeg()
try:
os.makedirs(download_dir, exist_ok=True)
except OSError as e:
self._handler.on_error(
'ERROR: Failed to create download folder: %s' % e)
raise
for i, info_path in enumerate(info_playlist):
with open(info_path) as f:
info = json.load(f)
title = info.get('title') or info.get('id') or 'video'
output_title = self._get_output_title(title)
thumbnail_paths = list(
filter(
lambda p: os.path.splitext(p)[1][1:] != 'json',
glob.iglob(
glob.escape(info_path[:-len('info.json')]) + '*')))
thumbnail_path = thumbnail_paths[0] if thumbnail_paths else ''
if thumbnail_path:
# Convert thumbnail to JPEG and limit resolution
new_thumbnail_path = thumbnail_path + '-converted.jpg'
try:
subprocess.run([
FFMPEG_EXE, '-i',
os.path.abspath(thumbnail_path), '-vf',
('scale=\'min({0},iw):min({0},ih):'
'force_original_aspect_ratio=decrease\''
).format(MAX_THUMBNAIL_RESOLUTION),
os.path.abspath(new_thumbnail_path)
],
check=True,
stdin=subprocess.DEVNULL,
stdout=sys.stderr)
except FileNotFoundError:
self._handler.on_error('ERROR: %r not found' %
FFMPEG_EXE)
raise
except subprocess.CalledProcessError:
traceback.print_exc(file=sys.stderr)
sys.stderr.flush()
new_thumbnail_path = ''
# No longer needed
os.remove(thumbnail_path)
thumbnail_path = new_thumbnail_path
self._handler.on_progress_start(i, len(info_playlist), title,
thumbnail_path)
for thumbnail in info.get('thumbnails') or []:
thumbnail['filename'] = thumbnail_path
sort_formats(
info.get('formats') or [], resolution, prefer_mpeg)
with open(info_path, 'w') as f:
json.dump(info, f)
# Check if we already got the file
existing_filename = self._find_existing_download(
download_dir, output_title, mode)
if existing_filename is not None:
self._handler.on_progress_end(existing_filename)
continue
# Download into separate directory because youtube-dl generates
# many temporary files
temp_download_dir = os.path.join(download_dir,
output_title + '.part')
# Lock download directory to prevent other processes from
# writing to the same files
temp_download_dir_cm = contextlib.ExitStack()
try:
temp_download_dir_cm.enter_context(
self._create_and_lock_dir(temp_download_dir))
except OSError as e:
self._handler.on_error(
'ERROR: Failed to lock download folder: %s' % e)
raise
with temp_download_dir_cm:
# Check if the file got downloaded in the meantime
existing_filename = self._find_existing_download(
download_dir, output_title, mode)
if existing_filename is not None:
filename = existing_filename
else:
# See ydl_opts['forcejson']
self._expect_info_dict_json = True
self._load_playlist(temp_download_dir,
info_path=info_path)
if self._expect_info_dict_json:
raise RuntimeError('info_dict not received')
# Move finished download from download to target dir
temp_filename = self._info_dict['_filename']
if mode == 'audio':
temp_filename = (
os.path.splitext(temp_filename)[0] + '.mp3')
output_ext = os.path.splitext(temp_filename)[1]
filename = output_title + output_ext
try:
os.replace(
os.path.join(temp_download_dir, temp_filename),
os.path.join(download_dir, filename))
except OSError as e:
self._handler.on_error(
('ERROR: Falied to move finished download to '
'download folder: %s') % e)
raise
# Delete download directory
with contextlib.suppress(OSError):
shutil.rmtree(temp_download_dir)
self._handler.on_progress_end(filename)
self._info_dict = None
def find_fixtures(self, fixture_label):
"""Find fixture files for a given label."""
if fixture_label == READ_STDIN:
return [(READ_STDIN, None, READ_STDIN)]
fixture_name, ser_fmt, cmp_fmt = self.parse_name(fixture_label)
databases = [self.using, None]
cmp_fmts = list(self.compression_formats) if cmp_fmt is None else [cmp_fmt]
ser_fmts = (
serializers.get_public_serializer_formats()
if ser_fmt is None
else [ser_fmt]
)
if self.verbosity >= 2:
self.stdout.write("Loading '%s' fixtures..." % fixture_name)
if os.path.isabs(fixture_name):
fixture_dirs = [os.path.dirname(fixture_name)]
fixture_name = os.path.basename(fixture_name)
else:
fixture_dirs = self.fixture_dirs
if os.path.sep in os.path.normpath(fixture_name):
fixture_dirs = [
os.path.join(dir_, os.path.dirname(fixture_name))
for dir_ in fixture_dirs
]
fixture_name = os.path.basename(fixture_name)
suffixes = (
".".join(ext for ext in combo if ext)
for combo in product(databases, ser_fmts, cmp_fmts)
)
targets = {".".join((fixture_name, suffix)) for suffix in suffixes}
fixture_files = []
for fixture_dir in fixture_dirs:
if self.verbosity >= 2:
self.stdout.write("Checking %s for fixtures..." % humanize(fixture_dir))
fixture_files_in_dir = []
path = os.path.join(fixture_dir, fixture_name)
for candidate in glob.iglob(glob.escape(path) + "*"):
if os.path.basename(candidate) in targets:
# Save the fixture_dir and fixture_name for future error messages.
fixture_files_in_dir.append((candidate, fixture_dir, fixture_name))
if self.verbosity >= 2 and not fixture_files_in_dir:
self.stdout.write(
"No fixture '%s' in %s." % (fixture_name, humanize(fixture_dir))
)
# Check kept for backwards-compatibility; it isn't clear why
# duplicates are only allowed in different directories.
if len(fixture_files_in_dir) > 1:
raise CommandError(
"Multiple fixtures named '%s' in %s. Aborting."
% (fixture_name, humanize(fixture_dir))
)
fixture_files.extend(fixture_files_in_dir)
if not fixture_files:
raise CommandError("No fixture named '%s' found." % fixture_name)
return fixture_files
print(ruta)
# glob con un patrón que utiliza recursividad, Se obtienen los directorios y archivos coincidentes
raiz = '/home/ant/img/2015/**'
rutas = glob.glob(raiz, recursive=True)
print('\nrutas:', len(rutas))
for ruta in rutas:
print(ruta)
glob.iglob (pathname, recursive=False) # devuelve iterador en lugar de lista lo que permite procesarlo directamente sin asignarlo antes.
raiz = '/home/ant/img/2015/**'
for ruta in glob.iglob(raiz, recursive=True):
print(ruta)
glob.escape(pathname) # (desde python 3.4) Permite en pathname carácteres especiales, coincidentes con los comodines.
# glob.escape('/Viajar?.txt') equivale a glob.glob(/Viajar[?].txt)
# MÓDULO FNMATCH. busca coincidencias de un archivo con un patrón.
# fnmatch.fnmatch(archivo, patrón) > Devuelve True si coincide con patrón, diferencia mayúsculas o minúsculas si el sistema operativo lo hace. '''
# ejemplo:
import os, fnmatch
for archivo in os.listdir('.'):
if fnmatch.fnmatch(archivo.upper(), '*.MD'): # interpreta el archivo en mayúsculas.
print(archivo)
# fnmatch.fnmatchcase(filename, pattern) > True si coincide. Diferencia mayúsculas y minúsculas en cualquier caso.
# fnmatch.traslate('patrón') > traduce los comodines a una expresión regular.
num = int(num)
# e.g. ("SSL_R_BAD_DATA", ("ERR_LIB_SSL", "BAD_DATA", 390))
codes.append((code, (libcode, name, num)))
assert codes, f"no codes found in {h_file}"
return codes
if __name__ == "__main__":
openssl_inc = sys.argv[1]
outfile = sys.argv[2]
use_stdout = outfile == '-'
f = sys.stdout if use_stdout else open(outfile, "w")
# mnemonic -> (library code, error prefix, header file)
error_libraries = {}
for error_header in glob.glob(
os.path.join(glob.escape(openssl_inc), 'include/openssl/*err.h')):
base = os.path.basename(error_header)
if base in ('buffererr.h', 'objectserr.h', 'storeerr.h'):
# Deprecated in 3.0.
continue
mnemonic = base[:-5].upper()
if mnemonic == "":
# err.h
lib_codes = {
code: num
for (code, (
_, _,
num)) in parse_error_codes(error_header, 'ERR_LIB_', None)
}
else:
error_libraries[mnemonic] = (f'ERR_LIB_{mnemonic}',
def safe_glob(self, pathname):
return glob.iglob(glob.escape(pathname) + '*')
def main():
parser = argparse.ArgumentParser()
parser.add_argument("results_dir", type=str)
parser.add_argument(
"--method_dirs",
type=str,
nargs='+',
help='directories in results_dir (all of them by default)')
parser.add_argument("--method_names",
type=str,
nargs='+',
help='method names for the header')
parser.add_argument("--web_dir",
type=str,
help='default is results_dir/web')
parser.add_argument(
"--sort_by",
type=str,
nargs=2,
help='task and metric name to sort by, e.g. prediction mse')
parser.add_argument("--no_ffmpeg", action='store_true')
parser.add_argument("--batch_size",
type=int,
default=1,
help="number of samples in batch")
parser.add_argument(
"--num_samples",
type=int,
help=
"number of samples for the table of sequence (all of them by default)")
parser.add_argument("--show_se",
action='store_true',
help="show standard error in the table metrics")
parser.add_argument("--only_metrics", action='store_true')
args = parser.parse_args()
if args.web_dir is None:
args.web_dir = os.path.join(args.results_dir, 'web')
webpage = html.HTML(args.web_dir,
'Experiment name = %s' %
os.path.normpath(args.results_dir),
reflesh=1)
webpage.add_header1(os.path.normpath(args.results_dir))
if args.method_dirs is None:
unsorted_method_dirs = os.listdir(args.results_dir)
# exclude web_dir and all directories that starts with web
if args.web_dir in unsorted_method_dirs:
unsorted_method_dirs.remove(args.web_dir)
unsorted_method_dirs = [
method_dir for method_dir in unsorted_method_dirs
if not os.path.basename(method_dir).startswith('web')
]
# put ground_truth and repeat in the front (if any)
method_dirs = []
for first_method_dir in ['ground_truth', 'repeat']:
if first_method_dir in unsorted_method_dirs:
unsorted_method_dirs.remove(first_method_dir)
method_dirs.append(first_method_dir)
method_dirs.extend(sorted(unsorted_method_dirs))
else:
method_dirs = list(args.method_dirs)
if args.method_names is None:
method_names = list(method_dirs)
else:
method_names = list(args.method_names)
method_dirs = [
os.path.join(args.results_dir, method_dir)
for method_dir in method_dirs
]
if args.sort_by:
task_name, metric_name = args.sort_by
sort_criterion = []
for method_id, (method_name,
method_dir) in enumerate(zip(method_names,
method_dirs)):
metric = load_metrics(
os.path.join(method_dir, task_name, 'metrics', metric_name))
sort_criterion.append(np.mean(metric))
sort_criterion, method_ids, method_names, method_dirs = \
zip(*sorted(zip(sort_criterion, range(len(method_names)), method_names, method_dirs)))
webpage.add_header3('sorted by %s, %s' % tuple(args.sort_by))
else:
method_ids = range(len(method_names))
# infer task and metric names from first method
metric_fnames = sorted(
glob.glob('%s/*/metrics/*.csv' % glob.escape(method_dirs[0])))
task_names = []
metric_names = []
for metric_fname in metric_fnames:
head, tail = os.path.split(metric_fname)
task_name = head.split('/')[-2]
metric_name, _ = os.path.splitext(tail)
task_names.append(task_name)
metric_names.append(metric_name)
# save metrics
webpage.add_table()
header_txts = ['']
header_colspans = [2]
for task_name in task_names:
if task_name != header_txts[-1]:
header_txts.append(task_name)
header_colspans.append(2 if args.show_se else
1) # mean and standard error for each task
else:
# group consecutive task names that are the same
header_colspans[-1] += 2 if args.show_se else 1
webpage.add_row(header_txts, header_colspans)
subheader_txts = ['id', 'method']
for task_name, metric_name in zip(task_names, metric_names):
subheader_txts.append('%s (mean)' % metric_name)
if args.show_se:
subheader_txts.append('%s (se)' % metric_name)
webpage.add_row(subheader_txts)
all_metric_means = []
for method_id, method_name, method_dir in zip(method_ids, method_names,
method_dirs):
metric_txts = [method_id, method_name]
metric_means = []
for task_name, metric_name in zip(task_names, metric_names):
metric = load_metrics(
os.path.join(method_dir, task_name, 'metrics', metric_name))
metric_mean = np.mean(metric)
num_samples = len(metric)
metric_se = np.std(metric) / np.sqrt(num_samples)
metric_txts.append('%.4f' % metric_mean)
if args.show_se:
metric_txts.append('%.4f' % metric_se)
metric_means.append(metric_mean)
webpage.add_row(metric_txts)
all_metric_means.append(metric_means)
webpage.save()
if args.only_metrics:
return
# infer task names from first method
outputs_dirs = sorted(
glob.glob('%s/*/outputs' % glob.escape(method_dirs[0])))
task_names = [outputs_dir.split('/')[-2] for outputs_dir in outputs_dirs]
# save image sequences
image_dir = os.path.join(args.web_dir, 'images')
webpage.add_table()
header_txts = ['']
subheader_txts = ['id']
methods_subheader_txts = ['']
header_colspans = [1]
subheader_colspans = [1]
methods_subheader_colspans = [1]
num_samples = args.num_samples or num_samples
for sample_ind in range(num_samples):
if sample_ind % args.batch_size == 0:
print("saving samples from %d to %d" %
(sample_ind, sample_ind + args.batch_size))
ims = [None]
txts = [sample_ind]
links = [None]
colspans = [1]
for task_name in task_names:
# load input images from first method
input_fnames = sorted(
glob.glob('%s/inputs/*_%05d_??.png' % (glob.escape(
os.path.join(method_dirs[0], task_name)), sample_ind)))
input_images = load_images(input_fnames)
# save input images as image sequence
input_fnames = [
os.path.join(task_name, 'inputs',
os.path.basename(input_fname))
for input_fname in input_fnames
]
save_images([
os.path.join(image_dir, input_fname)
for input_fname in input_fnames
], input_images)
# infer output names from first method
output_fnames = sorted(
glob.glob('%s/outputs/*_%05d_??.png' % (glob.escape(
os.path.join(method_dirs[0], task_name)), sample_ind)))
output_names = sorted(
set(
os.path.splitext(os.path.basename(output_fname))[0][:-9]
for output_fname in output_fnames)) # remove _?????_??.png
# load output images
all_output_images = []
for output_name in output_names:
for method_name, method_dir in zip(method_names, method_dirs):
output_fnames = sorted(
glob.glob(
'%s/outputs/%s_%05d_??.png' %
(glob.escape(os.path.join(method_dir, task_name)),
output_name, sample_ind)))
output_images = load_images(output_fnames)
all_output_images.append(output_images)
# concatenate output images of all the methods
all_output_images = concat_images(all_output_images)
# save output images as image sequence or as gif clip
output_fname = os.path.join(
task_name, 'outputs',
'%s_%05d.gif' % ('_'.join(output_names), sample_ind))
if args.no_ffmpeg:
save_gif(os.path.join(image_dir, output_fname),
all_output_images,
fps=4)
else:
ffmpeg_save_gif(os.path.join(image_dir, output_fname),
all_output_images,
fps=4)
if sample_ind == 0:
header_txts.append(task_name)
subheader_txts.extend(['inputs', 'outputs'])
header_colspans.append(
len(input_fnames) + len(method_ids) * len(output_names))
subheader_colspans.extend(
[len(input_fnames),
len(method_ids) * len(output_names)])
method_id_strs = [
'%02d' % method_id for method_id in method_ids
]
methods_subheader_txts.extend([''] + list(
itertools.chain(*[method_id_strs] * len(output_names))))
methods_subheader_colspans.extend(
[len(input_fnames)] + [1] *
(len(method_ids) * len(output_names)))
ims.extend(input_fnames + [output_fname])
txts.extend([None] * (len(input_fnames) + 1))
links.extend(input_fnames + [output_fname])
colspans.extend([1] * len(input_fnames) +
[len(method_ids) * len(output_names)])
if sample_ind == 0:
webpage.add_row(header_txts, header_colspans)
webpage.add_row(subheader_txts, subheader_colspans)
webpage.add_row(methods_subheader_txts, methods_subheader_colspans)
webpage.add_images(ims, txts, links, colspans, height=64, width=None)
if (sample_ind + 1) % args.batch_size == 0:
webpage.save()
webpage.save()
def get_checkpoint_fnames():
for checkpoint_fname in glob.glob(
os.path.join(glob.escape(log_dir), '*.ckpt.pth')):
epoch = int(os.path.basename(checkpoint_fname).split('.')[0])
if epoch < args.num_epochs:
yield epoch, checkpoint_fname
def path_walk(p):
pattern = pathlib.Path(glob.escape(p)) / "**"
paths = glob.iglob(pattern, recursive=True)
return (pathlib.Path(p) for p in paths)
