def url(context, link_url):
"""
Get the path for a page in the Cactus build.
We'll need this because paths can be rewritten with prettifying.
"""
site = context['__CACTUS_SITE__']
page = context['__CACTUS_CURRENT_PAGE__']
url = site.get_url_for_page(link_url)
if url is None:
# See if we're trying to link to an /subdir/index.html with /subdir
link_url_index = os.path.join(link_url, "index.html")
url_link_url_index = site.get_url_for_page(link_url_index)
if url_link_url_index is None:
logger.warn('%s: page resource does not exist: %s', page.link_url, link_url)
url = link_url
if site.prettify_urls:
return relpath(urllib.parse.urljoin(site.url, url.rsplit('index.html', 1)[0]), dirname(page.absolute_final_url))
return relpath(urllib.parse.urljoin(site.url, url), dirname(page.absolute_final_url))
def walk_prefix(prefix, ignore_predefined_files=True, windows_forward_slashes=True):
"""
Return the set of all files in a given prefix directory.
"""
res = set()
prefix = abspath(prefix)
ignore = {'pkgs', 'envs', 'conda-bld', 'conda-meta', '.conda_lock',
'users', 'LICENSE.txt', 'info', 'conda-recipes', '.index',
'.unionfs', '.nonadmin'}
binignore = {'conda', 'activate', 'deactivate'}
if sys.platform == 'darwin':
ignore.update({'python.app', 'Launcher.app'})
for fn in os.listdir(prefix):
if ignore_predefined_files and fn in ignore:
continue
if isfile(join(prefix, fn)):
res.add(fn)
continue
for root, dirs, files in os.walk(join(prefix, fn)):
should_ignore = ignore_predefined_files and root == join(prefix, 'bin')
for fn2 in files:
if should_ignore and fn2 in binignore:
continue
res.add(relpath(join(root, fn2), prefix))
for dn in dirs:
path = join(root, dn)
if islink(path):
res.add(relpath(path, prefix))
if on_win and windows_forward_slashes:
return {path.replace('\\', '/') for path in res}
else:
return res
def configuration(parent_package='', top_path=None):
from numpy.distutils.misc_util import Configuration
config = Configuration('statsmodels', parent_package, top_path)
# these are subpackages because they have Cython code
config.add_subpackage('nonparametric')
config.add_subpackage('tsa')
#TODO: delegate the non-test stuff to subpackages
config.add_data_files('sandbox/panel/test_data.txt')
curdir = os.path.abspath(os.path.dirname(__file__))
extradatafiles = [relpath(os.path.join(r,d),start=curdir)
for r,ds,f in os.walk(os.path.join(curdir, 'datasets'))
for d in f if not os.path.splitext(d)[1] in
['.py', '.pyc']]
for f in extradatafiles:
config.add_data_files(f)
# add all the test and results directories for non *.py files
for root, dirnames, filenames in os.walk(curdir):
for dir_name in dirnames:
if dir_name in ['tests', 'results'] and root != 'sandbox':
config.add_data_dir(relpath(
os.path.join(root, dir_name),
start = curdir)
)
return config
def static(context, link_url):
"""
Get the path for a static file in the Cactus build.
We'll need this because paths can be rewritten with fingerprinting.
"""
#TODO: Support URLS that don't start with `/static/`
site = context['__CACTUS_SITE__']
page = context['__CACTUS_CURRENT_PAGE__']
url = site.get_url_for_static(link_url)
if url is None:
# For the static method we check if we need to add a prefix
helper_keys = [
"/static/" + link_url,
"/static" + link_url,
"static/" + link_url
]
for helper_key in helper_keys:
url_helper_key = site.get_url_for_static(helper_key)
if url_helper_key is not None:
return relpath(urllib.parse.urljoin( site.static_path, helper_key), dirname(page.absolute_final_url))
url = link_url
return relpath(urllib.parse.urljoin( site.static_path, link_url), dirname(page.absolute_final_url))
def __iter__(self):
"""Seek static files and result full and relative paths.
:return generator: Walk files
"""
app_and_blueprints = self.collect.filter(
[self.collect.app] + list(self.collect.blueprints.values()))
destination_list = set()
for bp in app_and_blueprints:
if bp.has_static_folder and op.isdir(bp.static_folder):
for root, _, files in walk(bp.static_folder):
for f in files:
spath = op.join(root, f)
tpath = op.relpath(spath, bp.static_folder.rstrip('/'))
relative = (bp.static_url_path and
self.collect.static_url and
bp.static_url_path.startswith(op.join(
self.collect.static_url, ''))) # noqa
if relative:
tpath = op.join(op.relpath(
bp.static_url_path, self.collect.static_url
), tpath)
if tpath in destination_list:
self.log("{0} already sourced".format(tpath))
continue
destination_list.add(tpath)
yield bp, spath, tpath
def _handle_and_return_installed_items(ds, installed_items, failed_items, save):
if save and ds is not None:
_save_installed_datasets(ds, installed_items)
if failed_items:
msg = ''
for act, l in (("succeeded", installed_items), ("failed", failed_items)):
if not l:
continue
if msg:
msg += ', and '
msg += "%s %s" % (
single_or_plural("dataset", "datasets", len(l),
include_count=True),
act)
if ds:
paths = [relpath(i.path, ds.path)
if hasattr(i, 'path')
else i if not i.startswith(ds.path) else relpath(i, ds.path)
for i in l]
else:
paths = l
msg += " (%s)" % (", ".join(map(str, paths)))
msg += ' to install'
# we were asked for multiple installations
if installed_items or len(failed_items) > 1:
raise IncompleteResultsError(
results=installed_items, failed=failed_items, msg=msg)
else:
raise InstallFailedError(msg=msg)
return installed_items[0] \
if len(installed_items) == 1 else installed_items
def merge_related(self, path_, d, n, merged, content_types):
path_ = path.abspath(path.join('/', path_))
if path_ in merged:
return
head, tail = path.split(path_)
new_path = path.join(head, '%s%s' % (n, tail))
merged[path_] = new_path
try:
os.makedirs(path.dirname(path.join(self.main_dir, path.relpath(new_path, '/'))))
except OSError:
pass
shutil.copy(
path.join(d, path.relpath(path_, '/')),
path.join(self.main_dir, path.relpath(new_path, '/')),
)
rels_path = path.relpath(path.join(head, '_rels', '%s.rels' % tail), '/')
if path.exists(path.join(d, rels_path)):
new_rels_path = path.relpath(path.join(head, '_rels', '%s%s.rels' % (n, tail)), '/')
non_follow_types = [
'http://schemas.openxmlformats.org/officeDocument/2006/relationships/slideMaster',
'http://schemas.openxmlformats.org/officeDocument/2006/relationships/notesMaster',
'http://schemas.openxmlformats.org/officeDocument/2006/relationships/slideLayout',
'http://schemas.openxmlformats.org/officeDocument/2006/relationships/oleObject',
]
rels = etree.parse(path.join(d, rels_path))
for e in rels.getroot():
if e.get('Type') in non_follow_types:
continue
self.merge_related(path.join(path_, '..', e.get('Target')), d, n, merged, content_types)
for e in rels.getroot():
if e.get('Type') in non_follow_types:
continue
new_abs_path = path.join('/', merged[path.normpath(path.join(path_, '..', e.get('Target')))])
ext = e.get('Target').rsplit('.', 1)[-1]
if ext == 'xml' or ext not in self.default_content_types:
for c in content_types.getroot():
if c.get('PartName') == path.abspath(path.join('/', path_, '..', e.get('Target'))):
if not any(c.get('PartName') == new_abs_path for c in self.content_types.tree.getroot()):
c = copy.deepcopy(c)
c.set('PartName', new_abs_path)
self.content_types.tree.getroot().append(c)
break
else:
raise Exception
e.set(
'Target',
new_abs_path,
)
try:
os.makedirs(path.dirname(path.join(self.main_dir, new_rels_path)))
except OSError:
pass
rels.write(path.join(self.main_dir, new_rels_path))
path.join(self.main_dir, new_rels_path)
def mutate(self):
"""
Performs the mutation. Applies mutation operator to each source file,
then stores a diff between the original and mutated file.
# mutants = # source files x # mutation operators
"""
count = 0
start = time()
for src_file in self.project.source_files():
original_path = join(self.project.settings["source_path"], src_file)
mutant_path = join(out_dir, src_file)
mkdir_p(dirname(mutant_path))
for (op, invoke) in self.project.settings["mutants"].items():
if invoke:
p = Popen(["txl", original_path, join("vendor", "conman", "%s.Txl" % op)], stdout=open(mutant_path, "w"), stderr=open("/dev/null"))
self.store.put(diff(relpath(original_path), relpath(mutant_path)), op, src_file)
count += 1
if count % 1000 == 0:
print("Generated %d mutants. Elapsed time %.02f seconds." % (count, (time() - start)))
stop = time()
print("Generated %d mutants in %d seconds." % (count, (stop - start)))
def compile(conf, env):
"""Copy or compile assets to output directory. If an asset is used as template, it
won't be copied to the output directory."""
global __writers, __default
ext_map = dict((cls.ext, cls) for cls in (
SASSWriter, SCSSWriter, LESSWriter, HTMLWriter, XMLWriter
))
other = [(prefix, filelist(prefix, conf['static_ignore'])) for prefix in conf['static']]
other = [((relpath(path, prefix), prefix) for path in generator)
for prefix, generator in other]
files = ((path, conf['theme']) for path in filelist(conf['theme'], conf['theme_ignore']))
files = ((relpath(path, prefix), prefix) for path, prefix in files)
files = ((path, prefix) for path, prefix in files if path not in env.engine.templates)
for path, directory in chain(files, chain(*other)):
# initialize writer for extension if not already there
_, ext = splitext(path)
if ext in ext_map and ext not in __writers:
__writers[ext] = ext_map[ext](conf, env)
src, dest = join(directory, path), join(conf['output_dir'], path)
writer = __writers.get(ext, __defaultwriter)
writer.write(src, dest, force=env.options.force, dryrun=env.options.dryrun)
def vcs_ignore(config, args):
"""Create a VCS ignore file for a wiki."""
log = logging.getLogger('markdoc.vcs-ignore')
log.debug('Creating ignore file for %s' % args.vcs)
wiki_root = config['meta.root'] # shorter local alias.
ignore_file_lines = []
ignore_file_lines.append(p.relpath(config.html_dir, start=wiki_root))
ignore_file_lines.append(p.relpath(config.temp_dir, start=wiki_root))
if args.vcs == 'hg':
ignore_file_lines.insert(0, 'syntax: glob')
ignore_file_lines.insert(1, '')
if args.output == '-':
log.debug('Writing ignore file to stdout')
fp = sys.stdout
else:
if not args.output:
filename = p.join(wiki_root, '.%signore' % args.vcs)
else:
filename = p.join(wiki_root, args.output)
log.info('Writing ignore file to %s' % p.relpath(filename, start=wiki_root))
fp = open(filename, 'w')
try:
fp.write('\n'.join(ignore_file_lines) + '\n')
finally:
if fp is not sys.stdout:
fp.close()
log.debug('Ignore file written.')
def apply(self,
result,
original_file_dict,
file_diff_dict,
colored: bool=True):
'''
Print a diff of the patch that would be applied.
:param colored: Whether or not to use colored output.
'''
printer = ConsolePrinter(colored)
for filename, this_diff in sorted(result.diffs.items()):
to_filename = this_diff.rename if this_diff.rename else filename
to_filename = "/dev/null" if this_diff.delete else to_filename
original_file = original_file_dict[filename]
try:
current_file = file_diff_dict[filename].modified
new_file = (file_diff_dict[filename] + this_diff).modified
except KeyError:
current_file = original_file
new_file = this_diff.modified
if tuple(current_file) != tuple(new_file):
print_beautified_diff(difflib.unified_diff(current_file,
new_file,
fromfile=filename,
tofile=to_filename),
printer)
elif filename != to_filename:
print_from_name(printer, join('a', relpath(filename)))
print_to_name(printer, join('b', relpath(to_filename)))
return file_diff_dict
def get_simple_state_json(self, configuration):
state_data = {
'id': self._id,
'url': self._url,
'dom_path': posixpath.join(
posixpath.join(
*(relpath(
configuration.get_path('dom'),
configuration.get_path('root')
).split(os.sep) ) ), self._id + '.txt' ),
'img_path': posixpath.join(
posixpath.join(
*(relpath(
configuration.get_path('state'),
configuration.get_path('root')
).split(os.sep) ) ), self._id + '.png' ),
'log_path': posixpath.join(
posixpath.join(
*(relpath(
configuration.get_path('log'),
configuration.get_path('root')
).split(os.sep) ) ), self._id ,'browser_'+ str( configuration._browserID)+ '.json' ),
'depth': self._depth
}
return state_data
def _collect_infos(dirname):
""" Utility function used by ExplodedZipFile to generate ZipInfo
entries for all of the files and directories under dirname """
for r, _ds, fs in walk(dirname):
if not islink(r) and r != dirname:
i = ZipInfo()
i.filename = join(relpath(r, dirname), "")
i.file_size = 0
i.compress_size = 0
i.CRC = 0
yield i.filename, i
for f in fs:
df = join(r, f)
relfn = relpath(join(r, f), dirname)
if islink(df):
pass
elif isfile(df):
i = ZipInfo()
i.filename = relfn
i.file_size = getsize(df)
i.compress_size = i.file_size
i.CRC = file_crc32(df)
yield i.filename, i
else:
# TODO: is there any more special treatment?
pass
def _read_rule(self, filename):
filename = path.realpath(filename)
logging.info("Read %s" % path.relpath(filename))
try:
rule = {u"_filename": filename}
isData = False
for line in codecs.open(filename, "r", "utf-8"):
if isData:
if u"content" not in rule:
rule[u"content"] = u""
rule[u"content"] += line
elif line.strip() == "":
isData = True
elif line.startswith("#"):
continue
else:
key, value = line.split(":", 1)
rule[key.lower().strip()] = value.strip()
if u"content" in rule:
rule[u"content"] = rule[u"content"].strip()
self._rules[filename] = rule
self._update_rules_list()
return rule
except Exception, e:
logging.warning("Failed reading {0}: {1}".format(path.relpath(filename), e))
def get_state_json(self, configuration):
state_data = {
'id': self._id,
'url': self._url,
'depth': self._depth,
# output unix style path for website: first unpack dirs in get_path('dom'),
# and then posixpath.join them with the filename
'dom_path': posixpath.join(
posixpath.join(
*(relpath(
configuration.get_path('dom'),
configuration.get_path('root')
).split(os.sep) ) ), self._id, self._id + '.txt' ),
'img_path': posixpath.join(
posixpath.join(
*(relpath(
configuration.get_path('state'),
configuration.get_path('root')
).split(os.sep) ) ), self._id + '.png' ),
'log_path': posixpath.join(
posixpath.join(
*(relpath(
configuration.get_path('log'),
configuration.get_path('root')
).split(os.sep) ) ), self._id, 'browser_'+str(configuration._browserID) + '.json' ),
'clickable': self.get_all_clickables_json(),
'inputs': self.get_all_inputs_json(),
'selects': self.get_all_selects_json(),
'radios': self.get_all_radios_json(),
'checkboxes': self.get_all_checkboxes_json()
}
return state_data
def mk_relative_osx(path):
assert sys.platform == 'darwin' and is_obj(path)
s = macho.install_name_change(path, osx_ch_link)
names = macho.otool(path)
if names:
# Strictly speaking, not all object files have install names (e.g.,
# bundles and executables do not). In that case, the first name here
# will not be the install name (i.e., the id), but it isn't a problem,
# because in that case it will be a no-op (with the exception of stub
# files, which give an error, which is handled below).
args = [
'install_name_tool',
'-id',
join('@rpath', relpath(dirname(path),
join(config.build_prefix, 'lib')), basename(names[0])),
path,
]
print(' '.join(args))
p = Popen(args, stderr=PIPE)
stdout, stderr = p.communicate()
stderr = stderr.decode('utf-8')
if "Mach-O dynamic shared library stub file" in stderr:
print("Skipping Mach-O dynamic shared library stub file %s" % path)
return
else:
print(stderr, file=sys.stderr)
if p.returncode:
raise RuntimeError("install_name_tool failed with exit status %d"
% p.returncode)
# Add an rpath to every executable to increase the chances of it
# being found.
args = [
'install_name_tool',
'-add_rpath',
join('@loader_path', relpath(join(config.build_prefix, 'lib'),
dirname(path)), '').replace('/./', '/'),
path,
]
print(' '.join(args))
p = Popen(args, stderr=PIPE)
stdout, stderr = p.communicate()
stderr = stderr.decode('utf-8')
if "Mach-O dynamic shared library stub file" in stderr:
print("Skipping Mach-O dynamic shared library stub file %s\n" % path)
return
elif "would duplicate path, file already has LC_RPATH for:" in stderr:
print("Skipping -add_rpath, file already has LC_RPATH set")
return
else:
print(stderr, file=sys.stderr)
if p.returncode:
raise RuntimeError("install_name_tool failed with exit status %d"
% p.returncode)
if s:
# Skip for stub files, which have to use binary_has_prefix_files to be
# made relocatable.
assert_relative_osx(path)
def copy_template(src, dst, dvcs, values):
for root, dirs, files in os.walk(src):
for dir in dirs:
src_path = join(root, dir)
dst_path = join(dst, relpath(src_path, src))
if not isdir(dst_path):
os.makedirs(dst_path)
for file in files:
src_path = join(root, file)
dst_path = join(dst, relpath(src_path, src))
try:
with io.open(src_path, 'r', encoding='utf-8') as fd:
content = fd.read()
except UnicodeDecodeError:
# File is some sort of binary file... just copy it
# directly with no template substitution
with io.open(src_path, 'rb') as fd:
content = fd.read()
with io.open(dst_path, 'wb') as fd:
fd.write(content)
else:
content = content.format(**values)
with io.open(dst_path, 'w', encoding='utf-8') as fd:
fd.write(content)
dvcs.add(dst_path)
def configuration(parent_package="", top_path=None):
from numpy.distutils.misc_util import Configuration
config = Configuration("statsmodels", parent_package, top_path)
# these are subpackages because they have Cython code
config.add_subpackage("nonparametric")
config.add_subpackage("tsa")
# TODO: delegate the non-test stuff to subpackages
config.add_data_files("sandbox/panel/test_data.txt")
config.add_data_files("stats/libqsturng/tests/bootleg.dat")
config.add_data_files("stats/libqsturng/CH.r")
config.add_data_files("stats/libqsturng/LICENSE.txt")
curdir = os.path.abspath(os.path.dirname(__file__))
extradatafiles = [
relpath(os.path.join(r, d), start=curdir)
for r, ds, f in os.walk(os.path.join(curdir, "datasets"))
for d in f
if not os.path.splitext(d)[1] in [".py", ".pyc"]
]
for f in extradatafiles:
config.add_data_files(f)
# add all the test and results directories for non *.py files
for root, dirnames, filenames in os.walk(curdir):
for dir_name in dirnames:
if dir_name in ["tests", "results"] and root != "sandbox":
config.add_data_dir(relpath(os.path.join(root, dir_name), start=curdir))
return config
def __iter_alljobs__(self):
""" Goes through all directories with an OUTVAR. """
from os import walk
from os.path import relpath, join
from glob import iglob
from .relax import RelaxExtract
if self.glob is None:
for dirpath, dirnames, filenames in walk( self.rootpath, topdown=True,
followlinks=True ):
if 'crystal.out' not in filenames: continue
if 'relax' in dirnames:
dirnames[:] = [u for u in dirnames if u != 'relax']
try: result = RelaxExtract(join(self.rootpath, dirpath))
except:
try: result = self.__class__.DefaultExtract(join(self.rootpath, dirpath))
except: continue
else:
try: result = self.__class__.DefaultExtract(join(self.rootpath, dirpath))
except: continue
yield join('/', relpath(dirpath, self.rootpath)), result
else:
for dirpath, dirnames, filenames in walk( self.rootpath, topdown=True,
followlinks=True ):
for filename in iglob(join(join(self.rootpath, dirpath), self.glob)):
try: result = self.__class__.DefaultExtract(filename)
except: continue
else: yield join('/',relpath(filename, self.rootpath)), result
def filterTrainUserPrefFile(self):
file_path = path.relpath(self.orig_train_triplet_file_path)
forig = open(file_path, 'r')
goodTrainUsers = []
for line in forig:
if any(x in line for x in self.user_feed_tracks):
goodTrainUsers.append(line.split('\t', 1)[0])
forig.close()
# TODO search in track database for additional echo song ids
#if any(x in line for x in self.user_feed_artists):
# goodTrainUsers.append(line.split('\t', 1)[0])
# self.user_feed_artists_tracks.append(line.split('\t')[1])
goodTrainUsers = set(goodTrainUsers) # make list unique
file_path = path.relpath(self.train_triplet_file_path)
ffiltered = open(file_path, 'w')
file_path = path.relpath(self.orig_train_triplet_file_path)
forig = open(file_path, 'r')
for line in forig:
if any(x in line for x in goodTrainUsers):
userId, song_id_echo, playCount = line.strip().split('\t')
if int(playCount) > 10: # using constant to limit size of train data
ffiltered.write(line)
forig.close()
ffiltered.close()
def get_subdatasets(self, pattern=None, fulfilled=None, absolute=False,
recursive=False, recursion_limit=None, edges=False):
"""DEPRECATED: use `subdatasets()`"""
# TODO wipe this function out completely once we are comfortable
# with it. Internally we don't need or use it anymore.
import inspect
lgr.warning('%s still uses Dataset.get_subdatasets(). RF to use `subdatasets` command', inspect.stack()[1][3])
from datalad.coreapi import subdatasets
if edges:
return [(r['parentpath'] if absolute else relpath(r['parentpath'], start=self.path),
r['path'] if absolute else relpath(r['path'], start=self.path))
for r in subdatasets(
dataset=self,
fulfilled=fulfilled,
recursive=recursive,
recursion_limit=recursion_limit,
bottomup=True)]
else:
return subdatasets(
dataset=self,
fulfilled=fulfilled,
recursive=recursive,
recursion_limit=recursion_limit,
bottomup=True,
result_xfm='{}paths'.format('' if absolute else 'rel'))
def write_makefile_am_from_apps_dir( directory, moc_generated_files ):
'''Makefile writer for an /apps directory'''
makefile = open(P.join(directory,'Makefile.am'), 'w')
# For some reason ISIS repeats a lot of headers and source
# files. I think they actually compile some of their sources
# multiple times.
#
# To save on errors from autogen, we must keep a dictionary of files
sources_thus_far = []
app_names = []
moc_sources = []
xml_files = []
for sdirectory in glob(P.join(directory,'*')): # Loop through all folders in the directory
if not P.isdir( sdirectory ):
continue
app_name = P.relpath( sdirectory, directory ).split(".")[0] # Each folder is an app
# Identify XML files
xml_files.extend( P.relpath(x, directory) for x in glob(P.join(sdirectory,'*.xml')) )
# Write instructions to create an executable from the sources
sourcefiles = glob(P.join(sdirectory,'*.cpp'))
if sourcefiles:
app_names.append( app_name )
print('%s_SOURCES = ' % app_name, file=makefile, end='')
ld_add = []
for source in sourcefiles:
relative_source = P.relpath( source, directory )
filename = P.relpath( source, sdirectory )
if filename in sources_thus_far:
ld_add.append("%s.$(OBJEXT)" % filename.split('.')[0])
else:
sources_thus_far.append(filename)
print(' \\\n %s' % relative_source, file=makefile, end='')
# Identify headers that need to be moc generated
for header in (glob(P.join(sdirectory,'*.h')) + glob(P.join(sdirectory,'*.hpp'))):
for line in open( header ):
if "Q_OBJECT" in line:
moc_sources.append( "%s.dir/%s.moc.cc" % (app_name,P.basename(header).split('.')[0]) )
print(' \\\n %s' % moc_sources[-1], file=makefile, end='')
break
print('\n', file=makefile, end='')
ld_add.append("../src/Core/libisis3.la") # They're referenced by directory path
# Mission specific stuff is DLopened I believe.
print('%s_LDADD = %s' % (app_name, " ".join(ld_add)), file=makefile)
print('%s_CFLAGS = $(AM_CFLAGS)' % app_name, file=makefile)
print('\n', file=makefile, end='')
print('bin_PROGRAMS =', file=makefile, end='')
for app in app_names:
print(' %s' % app, file=makefile, end='')
print('', file=makefile)
# Write out where the XML files should be installed
print('xmlhelpdir = $(bindir)/xml', file=makefile)
print('xmlhelp_DATA = %s' % ' '.join(xml_files), file=makefile)
write_makefile_am_closing( directory, makefile, [], moc_sources, moc_sources, xml_files )
def main():
args = prog_options()
# Iterate through all directories and access their files
for root, dirs, files in walk(args.sample_dir):
if root != args.sample_dir:
print root
# Map primer names to their 16S regions
gene_region = {"519R": "V1-V3", "806R": "V4-V5"}
# Iterate through all files and perform quality filtering
for file in files:
if file.endswith("R-assigned-01.fastq"):
fname = file.split("_")[0] + "_" +\
gene_region[file.split("_")[1][:4]]
cmd = "fastq_quality_filter -v -q {} -p {} -i {} "\
"-o {}_qual_fil.fastq"\
.format(args.min_qual_score, args.min_base_pct,
relpath(join(root, file)),
relpath(join(root, fname)))
kwargs = shlex.split(cmd)
print kwargs, "\n"
out = sp.check_output(kwargs)
print out
with open(relpath(join(root, fname+"_qual_fil.txt")), "w")as fo:
fo.write("{}\n{}".format(fname, out))
return
def move(filename, basedir, newbase):
''' Given a filename and two directory names, this function generates
a new filename which is constructed from the relative path of the
filename and the first directory and the joint of this relative path
with the second directory.
This is useful to generate a new filename in a different directory
based on another. Craftr uses this function to generate object
filenames.
Example:
>>> move('src/main.c', 'src', 'build/obj')
build/obj/main.c
*path* may be an iterable other than a string in which case the
function is applied recursively to all its items and a list is
returned instead of a string. '''
if isinstance(filename, str):
rel = relpath(filename, basedir)
if rel == os.curdir or rel.startswith(os.pardir):
raise ValueError('pathname not a subpath of basedir', filename, basedir)
return join(newbase, relpath(filename, basedir))
elif isinstance(filename, collections.Iterable):
result = []
for item in filename:
result.append(move(item, basedir, newbase))
return result
else:
raise TypeError('move() expected string or iterable')
def page_list_to_dict(mem_cap, val, file_list):
file_number = 0
result_dict = {}
cnt = 0
ele = 0
fil_max = len(file_list)
# loop: if element in queue is 'DONE', no more files to read --> function saves current dict to file & terminates.
# otherwise: converting list of tuples to dict.
for file in file_list:
# reading file & updating dict with tuples from file
tmp_list = (pickle.load(open(file, "rb")))
result_dict.update(dict(tmp_list))
cnt += 1
print("Process ", val, " || Processed files: ", cnt, " of ", fil_max )
# check memory usage: if dict gets to big, save and clear
if memory_usage() >= mem_cap:
print("saving pages dict...")
full_path = path.relpath("page_dict/dict_"+str(val)+str(file_number)+'.pickle')
with open(full_path, 'wb') as pickle_file:
pickle.dump(result_dict, pickle_file, pickle.HIGHEST_PROTOCOL)
result_dict.clear()
file_number += 1
full_path = path.relpath("page_dict/dict_"+str(val)+str(file_number)+'.pickle')
with open(full_path, 'wb') as pickle_file:
print("Process ", val, " || finishing pages dict...")
pickle.dump(result_dict, pickle_file, pickle.HIGHEST_PROTOCOL)
result_dict.clear()
file_number += 1
def translate(old_commit, new_commit, filename, repo_path, inp):
filename = relpath(filename, repo_path)
repo_path = relpath(repo_path)
try:
git = Git(repo_path)
except Exception as e:
stderr.write(str(e))
return
# Load the line numbers and the times they have been looked at
line_counts = loads(sub('\s', '', inp))
# Put those line numbers in a dictionary so we can translate them to the new diff
line_counts = dict((int(k), v) for k, v in line_counts.iteritems())
diff = create_diff(
old_content=git.get_file_content(filename, old_commit),
new_content=git.get_file_content(filename, new_commit))
new_lines = convert_line_numbers(diff, line_counts.keys())
result = {}
for i, v in enumerate(line_counts.values()):
if new_lines[i] is not None:
result[new_lines[i]] = v
return dumps(result)
def compile(conf, env):
"""Copy or compile assets to output directory. If an asset is used as template, it
won't be copied to the output directory. All assets are tracked by the event object
and should not be removed during `acrylamid clean`."""
global __writers, __default
ext_map = {
'.sass': SASSWriter,
'.scss': SCSSWriter,
'.less': LESSWriter,
# '.haml': 'HAMLWriter',
# ...
}
other = [(prefix, readers.filelist(prefix)) for prefix in conf['static']]
other = [((relpath(path, prefix), prefix) for path in generator)
for prefix, generator in other]
files = ((path, conf['theme']) for path in readers.filelist(conf['theme']))
files = ((relpath(path, prefix), prefix) for path, prefix in files)
files = ((path, prefix) for path, prefix in files if path not in env.engine.templates)
for path, directory in chain(files, chain(*other)):
# initialize writer for extension if not already there
_, ext = splitext(path)
if ext in ext_map and ext not in __writers:
__writers[ext] = ext_map[ext]()
src, dest = join(directory, path), join(conf['output_dir'], path)
writer = __writers.get(ext, __defaultwriter)
writer.write(src, dest, force=env.options.force, dryrun=env.options.dryrun)
def get_current_path():
home = expanduser("~")
view = sublime.active_window().active_view()
data = sublime.active_window().project_data()
here = None
if USE_PROJECT_DIR and data and "folders" in data and len(data["folders"]) == 1:
specified_path = data["folders"][0]["path"]
if isabs(specified_path):
here = specified_path+sep
else:
project_file_name = sublime.active_window().project_file_name()
project_file_dir = split(project_file_name)[0]
here = normpath(join(project_file_dir, specified_path))+sep
elif view != None and view.file_name() != None:
here = split(view.file_name())[0]
if here != sep: here += sep
else:
here = "~" + sep
# Return path relative to home if applicable.
if len(commonprefix([home, here])) > 1:
relative_path = relpath(here,home)
if len(relative_path) > 1:
return join("~", relpath(here,home)) + sep
else:
return "~" + sep
else:
return here
def zip_export(file_name, prefix, resources, project_files, inc_repos, notify):
"""Create a zip file from an exported project.
Positional Parameters:
file_name - the file name of the resulting zip file
prefix - a directory name that will prefix the entire zip file's contents
resources - a resources object with files that must be included in the zip
project_files - a list of extra files to be added to the root of the prefix
directory
"""
to_zip_list = list(_inner_zip_export(resources, inc_repos))
total_files = sum(len(to_zip) for _, to_zip in to_zip_list)
total_files += len(project_files)
zipped = 0
with zipfile.ZipFile(file_name, "w") as zip_file:
for prj_file in project_files:
zip_file.write(prj_file, join(prefix, basename(prj_file)))
for loc, to_zip in to_zip_list:
res = resources[loc]
for source in to_zip:
if source:
zip_file.write(
source,
join(prefix, loc,
relpath(source, res.file_basepath[source])))
notify.progress("Zipping", source,
100 * (zipped / total_files))
zipped += 1
for lib, res in resources.items():
for source in res.lib_builds:
target_dir, _ = splitext(source)
dest = join(prefix, loc,
relpath(target_dir, res.file_basepath[source]),
".bld", "bldrc")
zip_file.write(source, dest)
def parseCurrentUserPref(self):
file_path = path.relpath(self.echo_spotify_translation_file_path)
fTrackTranslation = open(file_path, 'r')
trackTranslation = dict()
for line in fTrackTranslation:
song_id_echo, song_id_spotify = line.strip().split(':')
trackTranslation[song_id_spotify] = song_id_echo
fTrackTranslation.close()
file_path = path.relpath(self.user_triplet_file_path)
fUserTrackPref = open(file_path, 'r')
for line in fUserTrackPref:
userId, songIdSpotify, playCount = line.strip().split('\t')
if songIdSpotify in trackTranslation:
echoSongId = trackTranslation[songIdSpotify]
self.user_feed_tracks.append(echoSongId)
# print("Translated %s" % echoSongId)
# if echoSongId in self.echoSongIdToIdxMap:
# print("Found %s" % echoSongId)
# else:
# print("Not found %s in translations" % songIdSpotify)
fUserTrackPref.close()
# add artist for filtering
fArtists = open(path.relpath("user_artist_preferences.txt"), 'r')
for artist in fArtists:
self.user_feed_artists.append(artist.replace("\n", ""))
fArtists.close()
print(self.user_feed_tracks)
def run(arguments, content, options, state_machine, state, lineno):
# The user may provide a filename *or* Python code content, but not both
if arguments and content:
raise RuntimeError("plot:: directive can't have both args and content")
document = state_machine.document
config = document.settings.env.config
nofigs = 'nofigs' in options
options.setdefault('include-source', config.plot_include_source)
keep_context = 'context' in options
context_opt = None if not keep_context else options['context']
rst_file = document.attributes['source']
rst_dir = os.path.dirname(rst_file)
if len(arguments):
if not config.plot_basedir:
source_file_name = os.path.join(setup.app.builder.srcdir,
directives.uri(arguments[0]))
else:
source_file_name = os.path.join(setup.confdir, config.plot_basedir,
directives.uri(arguments[0]))
# If there is content, it will be passed as a caption.
caption = '\n'.join(content)
# If the optional function name is provided, use it
if len(arguments) == 2:
function_name = arguments[1]
else:
function_name = None
with io.open(source_file_name, 'r', encoding='utf-8') as fd:
code = fd.read()
output_base = os.path.basename(source_file_name)
else:
source_file_name = rst_file
code = textwrap.dedent("\n".join(map(str, content)))
counter = document.attributes.get('_plot_counter', 0) + 1
document.attributes['_plot_counter'] = counter
base, ext = os.path.splitext(os.path.basename(source_file_name))
output_base = '%s-%d.py' % (base, counter)
function_name = None
caption = ''
base, source_ext = os.path.splitext(output_base)
if source_ext in ('.py', '.rst', '.txt'):
output_base = base
else:
source_ext = ''
# ensure that LaTeX includegraphics doesn't choke in foo.bar.pdf filenames
output_base = output_base.replace('.', '-')
# is it in doctest format?
is_doctest = contains_doctest(code)
if 'format' in options:
if options['format'] == 'python':
is_doctest = False
else:
is_doctest = True
# determine output directory name fragment
source_rel_name = relpath(source_file_name, setup.confdir)
source_rel_dir = os.path.dirname(source_rel_name)
while source_rel_dir.startswith(os.path.sep):
source_rel_dir = source_rel_dir[1:]
# build_dir: where to place output files (temporarily)
build_dir = os.path.join(os.path.dirname(setup.app.doctreedir),
'plot_directive', source_rel_dir)
# get rid of .. in paths, also changes pathsep
# see note in Python docs for warning about symbolic links on Windows.
# need to compare source and dest paths at end
build_dir = os.path.normpath(build_dir)
if not os.path.exists(build_dir):
os.makedirs(build_dir)
# output_dir: final location in the builder's directory
dest_dir = os.path.abspath(
os.path.join(setup.app.builder.outdir, source_rel_dir))
if not os.path.exists(dest_dir):
os.makedirs(dest_dir) # no problem here for me, but just use built-ins
# how to link to files from the RST file
dest_dir_link = os.path.join(relpath(setup.confdir, rst_dir),
source_rel_dir).replace(os.path.sep, '/')
build_dir_link = relpath(build_dir, rst_dir).replace(os.path.sep, '/')
source_link = dest_dir_link + '/' + output_base + source_ext
# make figures
try:
results = render_figures(code,
source_file_name,
build_dir,
output_base,
keep_context,
function_name,
config,
context_reset=context_opt == 'reset',
close_figs=context_opt == 'close-figs')
errors = []
except PlotError as err:
reporter = state.memo.reporter
sm = reporter.system_message(
2,
"Exception occurred in plotting %s\n from %s:\n%s" %
(output_base, source_file_name, err),
line=lineno)
results = [(code, [])]
errors = [sm]
# Properly indent the caption
caption = '\n'.join(' ' + line.strip()
for line in caption.split('\n'))
# generate output restructuredtext
total_lines = []
for j, (code_piece, images) in enumerate(results):
if options['include-source']:
if is_doctest:
lines = ['']
lines += [row.rstrip() for row in code_piece.split('\n')]
else:
lines = ['.. code-block:: python', '']
lines += [
' %s' % row.rstrip() for row in code_piece.split('\n')
]
source_code = "\n".join(lines)
else:
source_code = ""
if nofigs:
images = []
opts = [
':%s: %s' % (key, val) for key, val in six.iteritems(options)
if key in ('alt', 'height', 'width', 'scale', 'align', 'class')
]
only_html = ".. only:: html"
only_latex = ".. only:: latex"
only_texinfo = ".. only:: texinfo"
# Not-None src_link signals the need for a source link in the generated
# html
if j == 0 and config.plot_html_show_source_link:
src_link = source_link
else:
src_link = None
result = format_template(
config.plot_template or TEMPLATE,
dest_dir=dest_dir_link,
build_dir=build_dir_link,
source_link=src_link,
multi_image=len(images) > 1,
only_html=only_html,
only_latex=only_latex,
only_texinfo=only_texinfo,
options=opts,
images=images,
source_code=source_code,
html_show_formats=config.plot_html_show_formats and not nofigs,
caption=caption)
total_lines.extend(result.split("\n"))
total_lines.extend("\n")
if total_lines:
state_machine.insert_input(total_lines, source=source_file_name)
# copy image files to builder's output directory, if necessary
if not os.path.exists(dest_dir):
cbook.mkdirs(dest_dir)
for code_piece, images in results:
for img in images:
for fn in img.filenames():
destimg = os.path.join(dest_dir, os.path.basename(fn))
if fn != destimg:
shutil.copyfile(fn, destimg)
# copy script (if necessary)
target_name = os.path.join(dest_dir, output_base + source_ext)
with io.open(target_name, 'w', encoding="utf-8") as f:
if source_file_name == rst_file:
code_escaped = unescape_doctest(code)
else:
code_escaped = code
f.write(code_escaped)
return errors
def run(self,
params,
release=False,
dev=False,
android=None,
debug=False,
debugger=None,
browserhtml=False):
env = self.build_env()
env["RUST_BACKTRACE"] = "1"
if android is None:
android = self.config["build"]["android"]
if android:
if debug:
print(
"Android on-device debugging is not supported by mach yet. See"
)
print(
"https://github.com/servo/servo/wiki/Building-for-Android#debugging-on-device"
)
return
script = [
"am force-stop com.mozilla.servo",
"echo servo >/sdcard/servo/android_params"
]
for param in params:
script += [
"echo '%s' >>/sdcard/servo/android_params" %
param.replace("'", "\\'")
]
script += [
"am start com.mozilla.servo/com.mozilla.servo.MainActivity",
"exit"
]
shell = subprocess.Popen(["adb", "shell"], stdin=subprocess.PIPE)
shell.communicate("\n".join(script) + "\n")
return shell.wait()
args = [self.get_binary_path(release, dev)]
if browserhtml:
browserhtml_path = find_dep_path_newest('browserhtml', args[0])
if browserhtml_path is None:
print(
"Could not find browserhtml package; perhaps you haven't built Servo."
)
return 1
if is_macosx():
# Enable borderless on OSX
args = args + ['-b']
elif is_windows():
# Convert to a relative path to avoid mingw -> Windows path conversions
browserhtml_path = path.relpath(browserhtml_path, os.getcwd())
args = args + [
'-w', '--pref', 'dom.mozbrowser.enabled', '--pref',
'dom.forcetouch.enabled', '--pref',
'shell.builtin-key-shortcuts.enabled=false',
path.join(browserhtml_path, 'out', 'index.html')
]
# Borrowed and modified from:
# http://hg.mozilla.org/mozilla-central/file/c9cfa9b91dea/python/mozbuild/mozbuild/mach_commands.py#l883
if debug:
import mozdebug
if not debugger:
# No debugger name was provided. Look for the default ones on
# current OS.
debugger = mozdebug.get_default_debugger_name(
mozdebug.DebuggerSearch.KeepLooking)
self.debuggerInfo = mozdebug.get_debugger_info(debugger)
if not self.debuggerInfo:
print("Could not find a suitable debugger in your PATH.")
return 1
command = self.debuggerInfo.path
if debugger == 'gdb' or debugger == 'lldb':
rustCommand = 'rust-' + debugger
try:
subprocess.check_call([rustCommand, '--version'],
env=env,
stdout=open(os.devnull, 'w'))
except (OSError, subprocess.CalledProcessError):
pass
else:
command = rustCommand
# Prepend the debugger args.
args = ([command] + self.debuggerInfo.args + args + params)
else:
args = args + params
try:
check_call(args, env=env)
except subprocess.CalledProcessError as e:
print("Servo exited with return value %d" % e.returncode)
return e.returncode
except OSError as e:
if e.errno == 2:
print("Servo Binary can't be found! Run './mach build'"
" and try again!")
else:
raise e
def handle(args):
cmd.find_config()
config = cmd.load_config()
assert config.LOCAL_ARCHIVES_BACKUP_DIR, "No LOCAL_ARCHIVES_BACKUP_DIR!"
db = config.archive_db
old_dir = os.getcwd()
os.chdir(config.LOCAL_ARCHIVES_BACKUP_DIR)
missing = set(db.entries.keys())
seen = set()
changes = 0
need_backup = False
for root, dirs, files in os.walk('.'):
for f in files:
if f.startswith('.'): continue
rel_path = relpath(join(root, f), '.')
sha1 = archives.get_sha1(rel_path)
new = archives.StoredArchive(url=config.ARCHIVES_BASE_URL +
rel_path,
sha1=sha1)
existing = db.entries.get(f, None)
if f in seen:
raise SafeException(
"{}: DUPLICATE basename - not allowed!\nFirst:{}\nSecord:{}"
.format(f, existing, new))
seen.add(f)
if existing:
missing.remove(f)
if existing != new:
need_backup = True
changes += 1
print("{}:".format(rel_path))
if existing.sha1 != new.sha1:
print(" Old SHA1: {old}\n New SHA1: {new}".format(
file=rel_path, old=existing.sha1, new=new.sha1))
if existing.url != new.url:
print(" Old URL: {old}\n New URL: {new}".format(
file=rel_path, old=existing.url, new=new.url))
else:
changes += 1
print("{}: added to database: {url}".format(rel_path,
url=new.url))
db.entries[f] = new
if missing:
print(
"These archives were missing (but were not removed from the database)"
)
for m in sorted(missing):
print(" " + m)
os.chdir(old_dir)
if need_backup:
backup_path = db.path + '.old'
print("Old database saved as {}".format(backup_path))
shutil.copyfile(db.path, backup_path)
if changes:
db.save_all()
print("Updated {} (changes: {})".format(db.path, changes))
if need_backup:
print("Run '0repo update' to update public feeds.")
else:
print("No changes found")
def main():
import sys
# Separate the nose params and the pydev params.
pydev_params = []
other_test_framework_params = []
found_other_test_framework_param = None
NOSE_PARAMS = '--nose-params'
PY_TEST_PARAMS = '--py-test-params'
for arg in sys.argv[1:]:
if not found_other_test_framework_param and arg != NOSE_PARAMS and arg != PY_TEST_PARAMS:
pydev_params.append(arg)
else:
if not found_other_test_framework_param:
found_other_test_framework_param = arg
else:
other_test_framework_params.append(arg)
try:
# Convert to the case stored in the filesystem
import win32api
def get_with_filesystem_case(f):
return win32api.GetLongPathName(win32api.GetShortPathName(f))
except:
def get_with_filesystem_case(f):
return f
# Here we'll run either with nose or with the pydev_runfiles.
from _pydev_runfiles import pydev_runfiles
from _pydev_runfiles import pydev_runfiles_xml_rpc
from _pydevd_bundle import pydevd_constants
from pydevd_file_utils import _NormFile
DEBUG = 0
if DEBUG:
sys.stdout.write('Received parameters: %s\n' % (sys.argv, ))
sys.stdout.write('Params for pydev: %s\n' % (pydev_params, ))
if found_other_test_framework_param:
sys.stdout.write('Params for test framework: %s, %s\n' %
(found_other_test_framework_param,
other_test_framework_params))
try:
configuration = pydev_runfiles.parse_cmdline([sys.argv[0]] +
pydev_params)
except:
sys.stderr.write('Command line received: %s\n' % (sys.argv, ))
raise
pydev_runfiles_xml_rpc.initialize_server(
configuration.port
) # Note that if the port is None, a Null server will be initialized.
NOSE_FRAMEWORK = "nose"
PY_TEST_FRAMEWORK = "py.test"
test_framework = None # Default (pydev)
try:
if found_other_test_framework_param:
if found_other_test_framework_param == NOSE_PARAMS:
test_framework = NOSE_FRAMEWORK
import nose
elif found_other_test_framework_param == PY_TEST_PARAMS:
test_framework = PY_TEST_FRAMEWORK
import pytest
else:
raise ImportError('Test framework: %s not supported.' %
(found_other_test_framework_param, ))
else:
raise ImportError()
except ImportError:
if found_other_test_framework_param:
raise
test_framework = None
# Clear any exception that may be there so that clients don't see it.
# See: https://sourceforge.net/tracker/?func=detail&aid=3408057&group_id=85796&atid=577329
if hasattr(sys, 'exc_clear'):
sys.exc_clear()
if not test_framework:
return pydev_runfiles.main(
configuration) # Note: still doesn't return a proper value.
else:
# We'll convert the parameters to what nose or py.test expects.
# The supported parameters are:
# runfiles.py --config-file|-t|--tests <Test.test1,Test2> dirs|files --nose-params xxx yyy zzz
# (all after --nose-params should be passed directly to nose)
# In java:
# --tests = Constants.ATTR_UNITTEST_TESTS
# --config-file = Constants.ATTR_UNITTEST_CONFIGURATION_FILE
# The only thing actually handled here are the tests that we want to run, which we'll
# handle and pass as what the test framework expects.
py_test_accept_filter = {}
files_to_tests = configuration.files_to_tests
if files_to_tests:
# Handling through the file contents (file where each line is a test)
files_or_dirs = []
for file, tests in files_to_tests.items():
if test_framework == NOSE_FRAMEWORK:
for test in tests:
files_or_dirs.append(file + ':' + test)
elif test_framework == PY_TEST_FRAMEWORK:
file = _NormFile(file)
py_test_accept_filter[file] = tests
files_or_dirs.append(file)
else:
raise AssertionError(
'Cannot handle test framework: %s at this point.' %
(test_framework, ))
else:
if configuration.tests:
# Tests passed (works together with the files_or_dirs)
files_or_dirs = []
for file in configuration.files_or_dirs:
if test_framework == NOSE_FRAMEWORK:
for t in configuration.tests:
files_or_dirs.append(file + ':' + t)
elif test_framework == PY_TEST_FRAMEWORK:
file = _NormFile(file)
py_test_accept_filter[file] = configuration.tests
files_or_dirs.append(file)
else:
raise AssertionError(
'Cannot handle test framework: %s at this point.' %
(test_framework, ))
else:
# Only files or dirs passed (let it do the test-loading based on those paths)
files_or_dirs = configuration.files_or_dirs
argv = other_test_framework_params + files_or_dirs
if test_framework == NOSE_FRAMEWORK:
# Nose usage: http://somethingaboutorange.com/mrl/projects/nose/0.11.2/usage.html
# show_stdout_option = ['-s']
# processes_option = ['--processes=2']
argv.insert(0, sys.argv[0])
if DEBUG:
sys.stdout.write('Final test framework args: %s\n' %
(argv[1:], ))
from _pydev_runfiles import pydev_runfiles_nose
PYDEV_NOSE_PLUGIN_SINGLETON = pydev_runfiles_nose.start_pydev_nose_plugin_singleton(
configuration)
argv.append('--with-pydevplugin')
# Return 'not' because it will return 'success' (so, exit == 0 if success)
return not nose.run(argv=argv,
addplugins=[PYDEV_NOSE_PLUGIN_SINGLETON])
elif test_framework == PY_TEST_FRAMEWORK:
if '--coverage_output_dir' in pydev_params and '--coverage_include' in pydev_params:
coverage_output_dir = pydev_params[
pydev_params.index('--coverage_output_dir') + 1]
coverage_include = pydev_params[
pydev_params.index('--coverage_include') + 1]
try:
import pytest_cov
except ImportError:
sys.stderr.write(
'To do a coverage run with pytest the pytest-cov library is needed (i.e.: pip install pytest-cov).\n\n'
)
raise
argv.insert(0, '--cov-append')
argv.insert(1, '--cov-report=')
argv.insert(2, '--cov=%s' % (coverage_include, ))
import time
os.environ['COVERAGE_FILE'] = os.path.join(
coverage_output_dir, '.coverage.%s' % (time.time(), ))
if DEBUG:
sys.stdout.write('Final test framework args: %s\n' % (argv, ))
sys.stdout.write('py_test_accept_filter: %s\n' %
(py_test_accept_filter, ))
def dotted(p):
# Helper to convert path to have dots instead of slashes
return os.path.normpath(p).replace(os.sep,
"/").replace('/', '.')
curr_dir = os.path.realpath('.')
curr_dotted = dotted(curr_dir) + '.'
# Overcome limitation on py.test:
# When searching conftest if we have a structure as:
# /my_package
# /my_package/conftest.py
# /my_package/tests
# /my_package/tests/test_my_package.py
# The test_my_package won't have access to the conftest contents from the
# test_my_package.py file unless the working dir is set to /my_package.
#
# See related issue (for which we work-around below):
# https://bitbucket.org/hpk42/pytest/issue/639/conftest-being-loaded-twice-giving
for path in sys.path:
path_dotted = dotted(path)
if curr_dotted.startswith(path_dotted):
os.chdir(path)
break
remove = []
for i in xrange(len(argv)):
arg = argv[i]
# Workaround bug in py.test: if we pass the full path it ends up importing conftest
# more than once (so, always work with relative paths).
if os.path.isfile(arg) or os.path.isdir(arg):
# Args must be passed with the proper case in the filesystem (otherwise
# python itself may not recognize it).
arg = get_with_filesystem_case(arg)
argv[i] = arg
from os.path import relpath
try:
# May fail if on different drives
arg = relpath(arg)
except ValueError:
pass
else:
argv[i] = arg
elif '<unable to get>' in arg:
remove.append(i)
for i in reversed(remove):
del argv[i]
# To find our runfile helpers (i.e.: plugin)...
d = os.path.dirname(__file__)
if d not in sys.path:
sys.path.insert(0, d)
import pickle, zlib, base64
# Update environment PYTHONPATH so that it finds our plugin if using xdist.
os.environ['PYTHONPATH'] = os.pathsep.join(sys.path)
# Set what should be skipped in the plugin through an environment variable
s = base64.b64encode(
zlib.compress(pickle.dumps(py_test_accept_filter)))
if pydevd_constants.IS_PY3K:
s = s.decode('ascii') # Must be str in py3.
os.environ['PYDEV_PYTEST_SKIP'] = s
# Identifies the main pid (i.e.: if it's not the main pid it has to connect back to the
# main pid to give xml-rpc notifications).
os.environ['PYDEV_MAIN_PID'] = str(os.getpid())
os.environ['PYDEV_PYTEST_SERVER'] = str(configuration.port)
argv.append('-p')
argv.append('_pydev_runfiles.pydev_runfiles_pytest2')
if 'unittest' in sys.modules or 'unittest2' in sys.modules:
sys.stderr.write(
'pydev test runner error: imported unittest before running pytest.main\n'
)
return pytest.main(argv)
else:
raise AssertionError(
'Cannot handle test framework: %s at this point.' %
(test_framework, ))
def __getitem__(self, index, tocache=False):
if tocache and (self.data_format == 'data'):
assert False, 'Caching only enabled for pano format'
# Select the data
rgb_img_fp, labels_color_fp = self.image_list[index]
# Load the data
basename = osp.splitext(osp.basename(rgb_img_fp))[0]
cache_path = osp.join(
self.cache_folder,
osp.splitext(osp.relpath(rgb_img_fp, self.root_path))[0])
if os.path.exists(cache_path):
if tocache:
# If we're trying to cache the data, and it's already there, don't do anything
return
tmp = torch.load(cache_path)
c = tmp.shape[0]
rgb, labels = torch.split(tmp, [c - 1, 1], dim=0)
# Assemble the pano set
pano_data = [rgb, labels, basename]
# Return the set of pano data
return pano_data
# Load the label files using OpenCV
if self.data_format == 'data':
# uint16 image, labels are stored in the last channel
labels = cv2.imread(labels_color_fp, cv2.IMREAD_UNCHANGED)[:, :, 2]
else:
# uint8 image, value=255 introduced for invalid region
labels = cv2.imread(labels_color_fp, cv2.IMREAD_UNCHANGED)
labels[labels == 255] = 0
labels = labels.astype('float32')
# Handle missing classes
unq = np.unique(labels.tolist())
assert not 13 in unq and not 14 in unq
labels[labels == 15] = 13
# Load the RGB image using PIL
rgb = Image.open(rgb_img_fp)
# Preprocess the loaded data for training
rgb = self.preprocess(rgb)
labels = numpy2torch(labels).unsqueeze(0)
# Camera normalize the perspective images
if self.data_format == 'data':
K = torch.tensor(self.K).view(3, 3)
# Compute a random shift (could use either camnorm instance as both have same intrinsic parameters)
shift = self.camnorm_bilinear.compute_random_shift(
rgb.shape[-2:], K)
# Normalize with a random crop
rgb = self.camnorm_bilinear(rgb, K, shift)
labels = self.camnorm_nearest(labels, K, shift)
# Resize the pano images as desired
elif self.data_format == 'pano':
rgb = F.interpolate(rgb.unsqueeze(0),
scale_factor=self.scale_factor,
mode='bilinear',
align_corners=False,
recompute_scale_factor=True).squeeze(0)
labels = F.interpolate(labels.unsqueeze(0),
scale_factor=self.scale_factor,
mode='nearest',
recompute_scale_factor=True).squeeze(0)
# If caching data
if tocache:
cache_dir = osp.dirname(cache_path)
if not osp.exists(cache_dir):
os.makedirs(cache_dir, exist_ok=True)
if os.path.exists(cache_path):
raise IOError('{} already exists'.format(cache_path))
tmp = torch.cat([rgb, labels], 0)
torch.save(tmp, cache_path, pickle_protocol=4)
return None
# Assemble the pano set
pano_data = [rgb, labels, basename]
# Return the set of pano data
return pano_data
class DocumentCache(object):
"""
A high-level document cache for caching the content of files.
This is a read-only cache which uses the OS-reported modification timestamps
for files (via `os.stat()`) to determine cache dirtiness, and then refreshes
its cache behind the scenes when files are requested.
You can access values via `.get()` (which supports several options) or via
simple subscription syntax (i.e. `cache[path]`). The cache is configured
with a 'root' on instantiation, by which all relative paths are resolved.
"""
def __init__(self, base=None, cache=None, encoding='utf-8'):
if cache is None:
cache = {}
self.cache = cache
if base is None:
base = os.getcwd()
self.base = base
self.encoding = encoding
absolute = lambda self, relpath: p.join(self.base, relpath)
relative = lambda self, abspath: p.relpath(abspath, start=self.base)
def has_latest_version(self, path):
"""Determine whether the cache for a path is up to date."""
# No-op for already-absolute paths.
path = self.absolute(path)
if path not in self.cache:
return False
cached_mtime = self.cache[path][0]
return os.stat(path).st_mtime <= cached_mtime
def refresh_cache(self, path, encoding=None):
"""Refresh the cache, no matter what, with an optional encoding."""
path = self.absolute(path)
encoding = encoding or self.encoding
data = read_from(path, encoding=encoding)
mtime = os.stat(path).st_mtime
self.cache[path] = (mtime, data)
def update_to_latest_version(self, path):
"""If necessary, refresh the cache's copy of a file."""
if not self.has_latest_version(path):
self.refresh_cache(path)
def get(self, path, cache=True, encoding=None):
"""Retrieve the data for a given path, optionally using the cache."""
path = self.absolute(path)
if cache:
self.update_to_latest_version(path)
return self.cache[path][1] # (mtime, data)[1]
if not p.isfile(path):
return None
if encoding is None:
encoding = self.encoding
return read_from(path, encoding=encoding)
def __getitem__(self, path):
result = self.get(path)
if result is None:
raise KeyError(path)
return result
def is_safe(self, directory):
"""Make sure the given absolute path does not point above the htroot."""
return p.pardir not in p.relpath(
directory, start=self.config.html_dir).split(p.sep)
def _get_flexible_source_candidates_for_submodule(ds, sm):
"""Assemble candidate locations from where to clone a submodule
The following locations candidates are considered. For each candidate a
cost is given in parenthesis, lower values indicate higher cost:
- URL of any configured superdataset remote that is known to have the
desired submodule commit, with the submodule path appended to it.
There can be more than one candidate (cost 500).
- A URL or absolute path recorded in `.gitmodules` (cost 600).
- In case `.gitmodules` contains a relative path instead of a URL,
the URL of any configured superdataset remote that is known to have the
desired submodule commit, with this relative path appended to it.
There can be more than one candidate (cost 500).
- In case `.gitmodules` contains a relative path as a URL, the absolute
path of the superdataset, appended with this relative path (cost 900).
Additional candidate URLs can be generated based on templates specified as
configuration variables with the pattern
`datalad.get.subdataset-source-candidate-<name>`
where `name` is an arbitrary identifier. If name starts with three digits
(e.g. '400myserver') these will be interpreted as a cost, and the
respective candidate will be sorted into the generated candidate list
according to this cost. If no cost is given, a default of 700
is used.
A template string assigned to such a variable can utilize the Python format
mini language and may reference a number of properties that are inferred
from the parent dataset's knowledge about the target subdataset. Properties
include any submodule property specified in the respective `.gitmodules`
record. For convenience, an existing `datalad-id` record is made available
under the shortened name `id`.
Additionally, the URL of any configured remote that contains the respective
submodule commit is available as `remote-<name>` properties, where `name`
is the configured remote name.
Lastly, all candidates are sorted according to their cost (lower values
first, and duplicate URLs are stripped, while preserving the first item in the
candidate list.
Parameters
----------
ds : Dataset
Parent dataset of to-be-installed subdataset.
sm : dict
Submodule record as produced by `subdatasets()`.
Returns
-------
list of dict
Where each dict has keys 'cost' (int), 'name' (str), 'url' (str).
Names are not unique and either derived from the name of the respective
remote, template configuration variable, or 'local'.
"""
# short cuts
ds_repo = ds.repo
sm_url = sm.get('gitmodule_url', None)
sm_path = op.relpath(sm['path'], start=sm['parentds'])
clone_urls = []
# CANDIDATE: tracking remote of the current branch
tracking_remote, tracking_branch = ds_repo.get_tracking_branch()
candidate_remotes = [tracking_remote] if tracking_remote else []
# if we have a remote, let's check the location of that remote
# for the presence of the desired submodule
last_commit = ds_repo.get_last_commit_hexsha(sm_path)
if last_commit:
# CANDIDATE: any remote that has the commit when the submodule was
# last modified
# ideally should also give preference to the remotes which have
# the same branch checked out I guess
candidate_remotes += list(
_get_remotes_having_commit(ds_repo, last_commit))
# prepare a dict to generate URL candidates from templates
sm_candidate_props = {
k[10:].replace('datalad-id', 'id'): v
for k, v in sm.items() if k.startswith('gitmodule_')
}
for remote in unique(candidate_remotes):
remote_url = ds_repo.get_remote_url(remote, push=False)
# Directly on parent's ds url
if remote_url:
# make remotes and their URLs available to template rendering
sm_candidate_props['remoteurl-{}'.format(remote)] = remote_url
# attempt: submodule checkout at parent remote URL
# We might need to quote sm_path portion, e.g. for spaces etc
if isinstance(RI(remote_url), URL):
sm_path_url = urlquote(sm_path)
else:
sm_path_url = sm_path
clone_urls.extend(
dict(cost=500, name=remote, url=url)
for url in _get_flexible_source_candidates(
# alternate suffixes are tested by `clone` anyways
sm_path_url,
remote_url,
alternate_suffix=False))
# attempt: provided (configured?) submodule URL
# TODO: consider supporting DataLadRI here? or would confuse
# git and we wouldn't want that (i.e. not allow pure git clone
# --recursive)
if sm_url:
clone_urls.extend(
dict(cost=600, name=remote, url=url)
for url in _get_flexible_source_candidates(
sm_url, remote_url, alternate_suffix=False))
cost_candidate_expr = re.compile('[0-9][0-9][0-9].*')
candcfg_prefix = 'datalad.get.subdataset-source-candidate-'
for name, tmpl in [(c[len(candcfg_prefix):], ds_repo.config[c])
for c in ds_repo.config.keys()
if c.startswith(candcfg_prefix)]:
url = tmpl.format(**sm_candidate_props)
# we don't want "flexible_source_candidates" here, this is
# configuration that can be made arbitrarily precise from the
# outside. Additional guesswork can only make it slower
has_cost = cost_candidate_expr.match(name) is not None
clone_urls.append(
# assign a default cost, if a config doesn't have one
dict(
cost=int(name[:3]) if has_cost else 700,
name=name[3:] if has_cost else name,
url=url,
from_config=True,
))
# CANDIDATE: the actual configured gitmodule URL
if sm_url:
clone_urls.extend(
dict(cost=900, name='local', url=url)
for url in _get_flexible_source_candidates(
sm_url, ds.path, alternate_suffix=False)
# avoid inclusion of submodule location itself
if url != sm['path'])
# sort all candidates by their label, thereby allowing a
# candidate provided by configuration to purposefully
# sort before or after automatically generated configuration
clone_urls = sorted(clone_urls, key=lambda x: x['cost'])
# take out any duplicate source candidates
# unique() takes out the duplicated at the tail end
clone_urls = unique(clone_urls, lambda x: x['url'])
return clone_urls
except Exception:
print("\tFile was not GMA.".format(title))
continue
lua = join(ext, "lua")
curs.execute("DELETE FROM files WHERE owner = ?", (wsid, ))
curs.execute("DELETE FROM components WHERE owner = ?", (wsid, ))
curs.execute("DELETE FROM cars WHERE owner = ?", (wsid, ))
curs.execute("DELETE FROM errors WHERE owner = ?", (wsid, ))
for rt, dirs, files in walk(lua):
tld = basename(rt)
for f in files:
pf = join(rt, f)
p = normpath(normcase(relpath(join(rt, f), ext)))
curs.execute(
"SELECT files.path, addons.name FROM files INNER JOIN addons ON files.owner = addons.wsid WHERE path = ? AND owner != ?",
(
p,
wsid,
))
for res in curs:
print("\tADON-FILE: {}, '{}'!".format(*res))
curs.execute("INSERT OR IGNORE INTO files VALUES (?, ?)", (
p,
wsid,
))
if tld == "auto":
def _install_subds_from_flexible_source(ds, sm, **kwargs):
"""Tries to obtain a given subdataset from several meaningful locations
Parameters
----------
ds : Dataset
Parent dataset of to-be-installed subdataset.
sm : dict
Submodule record as produced by `subdatasets()`.
**kwargs
Passed onto clone()
"""
sm_path = op.relpath(sm['path'], start=sm['parentds'])
# compose a list of candidate clone URLs
clone_urls = _get_flexible_source_candidates_for_submodule(ds, sm)
# prevent inevitable exception from `clone`
dest_path = op.join(ds.path, sm_path)
clone_urls_ = [src['url'] for src in clone_urls if src['url'] != dest_path]
if not clone_urls:
# yield error
yield get_status_dict(
action='install',
ds=ds,
status='error',
message=("Have got no candidates to install subdataset %s from.",
sm_path),
logger=lgr,
)
return
for res in clone_dataset(clone_urls_,
Dataset(dest_path),
cfg=ds.config,
**kwargs):
# make sure to fix a detached HEAD before yielding the install success
# result. The resetting of the branch would undo any change done
# to the repo by processing in response to the result
if res.get('action', None) == 'install' and \
res.get('status', None) == 'ok' and \
res.get('type', None) == 'dataset' and \
res.get('path', None) == dest_path:
_fixup_submodule_dotgit_setup(ds, sm_path)
target_commit = sm['gitshasum']
lgr.debug(
"Update cloned subdataset {0} in parent".format(dest_path))
section_name = 'submodule.{}'.format(sm['gitmodule_name'])
# do not use `git-submodule update --init`, it would make calls
# to git-config which will not obey datalad inter-process locks for
# modifying .git/config
sub = GitRepo(res['path'])
# record what branch we were on right after the clone
# TODO instead of the active branch, this should first consider
# a configured branch in the submodule record of the superdataset
sub_orig_branch = sub.get_active_branch()
# if we are on a branch this hexsha will be the tip of that branch
sub_orig_hexsha = sub.get_hexsha()
if sub_orig_hexsha != target_commit:
# make sure we have the desired commit locally
# expensive and possibly error-prone fetch conditional on cheap
# local check
if not sub.commit_exists(target_commit):
try:
sub.fetch(remote='origin', refspec=target_commit)
except CommandError:
pass
# instead of inspecting the fetch results for possible ways
# with which it could failed to produced the desired result
# let's verify the presence of the commit directly, we are in
# expensive-land already anyways
if not sub.commit_exists(target_commit):
res.update(
status='error',
message=
('Target commit %s does not exist in the clone, and '
'a fetch that commit from origin failed',
target_commit[:8]),
)
yield res
# there is nothing we can do about this
# MIH thinks that removing the clone is not needed, as a likely
# next step will have to be a manual recovery intervention
# and not another blind attempt
continue
# checkout the desired commit
sub.call_git(['checkout', target_commit])
# did we detach?
# XXX: This is a less generic variant of a part of
# GitRepo.update_submodule(). It makes use of already available
# information and trusts the existence of the just cloned repo
# and avoids (redoing) some safety checks
if sub_orig_branch and not sub.get_active_branch():
# trace if current state is a predecessor of the branch_hexsha
lgr.debug(
"Detached HEAD after updating submodule %s "
"(original branch: %s)", sub, sub_orig_branch)
if sub.get_merge_base([sub_orig_hexsha,
target_commit]) == target_commit:
# TODO: config option?
# MIH: There is no real need here. IMHO this should all not
# happen, unless the submodule record has a branch
# configured. And Datalad should leave such a record, when
# a submodule is registered.
# we assume the target_commit to be from the same branch,
# because it is an ancestor -- update that original branch
# to point to the target_commit, and update HEAD to point to
# that location -- this readies the subdataset for
# further modification
lgr.info(
"Reset subdataset branch '%s' to %s (from %s) to "
"avoid a detached HEAD", sub_orig_branch,
target_commit[:8], sub_orig_hexsha[:8])
branch_ref = 'refs/heads/%s' % sub_orig_branch
sub.update_ref(branch_ref, target_commit)
sub.update_ref('HEAD', branch_ref, symbolic=True)
else:
lgr.warning(
"%s has a detached HEAD, because the recorded "
"subdataset state %s has no unique ancestor with "
"branch '%s'", sub, target_commit[:8],
sub_orig_branch)
# register the submodule as "active" in the superdataset
ds.config.set(
'{}.active'.format(section_name),
'true',
reload=False,
force=True,
where='local',
)
ds.config.set(
'{}.url'.format(section_name),
# record the actual source URL of the successful clone
# and not a funky prediction based on the parent ds
# like ds.repo.update_submodule() would do (does not
# accept a URL)
res['source']['giturl'],
reload=True,
force=True,
where='local',
)
yield res
subds = Dataset(dest_path)
if not subds.is_installed():
lgr.debug('Desired subdataset %s did not materialize, stopping', subds)
return
# check whether clone URL generators were involved
cand_cfg = [rec for rec in clone_urls if rec.get('from_config', False)]
if cand_cfg:
# get a handle on the configuration that is specified in the
# dataset itself (local and dataset)
super_cfg = ConfigManager(dataset=ds, source='dataset-local')
need_reload = False
for rec in cand_cfg:
# check whether any of this configuration originated from the
# superdataset. if so, inherit the config in the new subdataset
# clone. if not, keep things clean in order to be able to move with
# any outside configuration change
for c in ('datalad.get.subdataset-source-candidate-{}{}'.format(
rec['cost'], rec['name']),
'datalad.get.subdataset-source-candidate-{}'.format(
rec['name'])):
if c in super_cfg.keys():
subds.config.set(c,
super_cfg.get(c),
where='local',
reload=False)
need_reload = True
break
if need_reload:
subds.config.reload(force=True)
def setup(self):
"""Stata reg output from `sysuse auto; reg price mpg`"""
test_path = path.split(path.relpath(__file__))[0]
auto_path = path.join(test_path, 'data', 'auto.dta')
autodata = pd.read_stata(auto_path)
self.df = autodata
def test_temp_directory_respects_dir_arg():
with tempfile.TemporaryDirectory() as parent:
with tempfile.TemporaryDirectory(dir=parent) as child:
assert path.relpath(child, parent) == path.basename(child)
def append_expected_output_pattern(f):
if f == '/':
expected_patterns.append(r'{}{}{}'.format(size, " +", '.' + f))
else:
expected_patterns.append(r'{}{}{}'.format(size, " +",
path_prefix + path.relpath(f, self.mount_a.mountpoint)))
from datalad.config import anything2bool
import hirni_heuristic as heuristic
dataset = Dataset(sys.argv[1])
rel_spec_path = sys.argv[2]
anonymize = anything2bool(sys.argv[3])
subject = sys.argv[4]
location = sys.argv[5]
from mock import patch
from tempfile import mkdtemp
# relative path to heuristic to be recorded by datalad-run
heuristic_path = op.relpath(heuristic.__file__, dataset.path)
# relative path to not-needed-heudiconv output:
rel_trash_path = op.relpath(
mkdtemp(prefix="hirni-tmp-", dir=op.join(dataset.path, ".git")),
dataset.path)
# heudiconv may create README, CHANGES, dataset_description.json if they
# weren't there. So, if those aren't there before, we want to kill them
# afterwards
keep_readme = op.lexists(op.join(dataset.path, "README"))
keep_changes = op.lexists(op.join(dataset.path, "CHANGES"))
keep_description = op.lexists(
op.join(dataset.path, "dataset_description.json"))
# at least narrow down the output target:
def _copy_binaries(source, destination, toolchain, target):
"""
Copy TF-M binaries from source to destination
:param source: directory where TF-M binaries are available
:param destination: directory to which TF-M binaries are copied to
:param toolchain: build toolchain
:param target: target name
"""
if destination.endswith("/"):
output_dir = destination
else:
output_dir = destination + "/"
tfm_secure_axf = join(source, "tfm_s.axf")
logging.info(
"Copying %s to %s" %
(relpath(tfm_secure_axf, mbed_path), relpath(output_dir, mbed_path)))
shutil.copy2(tfm_secure_axf, output_dir)
try:
out_ext = TARGET_MAP[target].TFM_OUTPUT_EXT
except AttributeError:
tfm_secure_bin = join(source, "tfm_s.bin")
logging.info("Copying %s to %s" % (
relpath(tfm_secure_bin, mbed_path),
relpath(output_dir, mbed_path),
))
shutil.copy2(tfm_secure_bin, output_dir)
else:
if out_ext == "hex":
tfm_secure_bin = join(source, "tfm_s.hex")
global TC_DICT
if toolchain == "ARMCLANG":
elf2bin = join(TOOLCHAIN_PATHS[TC_DICT.get(toolchain)],
"fromelf")
cmd = [
elf2bin,
"--i32",
"--output=" + tfm_secure_bin,
tfm_secure_axf,
]
elif toolchain == "GNUARM":
elf2bin = join(
TOOLCHAIN_PATHS[TC_DICT.get(toolchain)],
"arm-none-eabi-objcopy",
)
cmd = [elf2bin, "-O", "ihex", tfm_secure_axf, tfm_secure_bin]
run_cmd_and_return(cmd)
logging.info("Copying %s to %s" % (
relpath(tfm_secure_bin, mbed_path),
relpath(output_dir, mbed_path),
))
shutil.copy2(tfm_secure_bin, output_dir)
if TARGET_MAP[target].tfm_bootloader_supported:
mcu_bin = join(source, "mcuboot.bin")
shutil.copy2(mcu_bin, output_dir)
if "TFM_V8M" in TARGET_MAP[target].extra_labels:
install_dir = abspath(join(source, os.pardir, os.pardir))
tfm_veneer = join(install_dir, "export", "tfm", "veneers",
"s_veneers.o")
shutil.copy2(tfm_veneer, output_dir)
logging.info("Start script")
proj_image_dir = "/share/mikro/IMX/MDC Polina Georgiev/exp-WIDE/"
last_mod_date = getLastModificationInDir(proj_image_dir, '*')
# Get connection to server
conn = getOmeroConn()
# Get all subdirs (these are the top plate dir)
plate_dirs = get_subdirs(proj_image_dir)
logging.debug("plate_dirs" + str(plate_dirs))
for plate_dir in plate_dirs:
plate_subdirs = get_subdirs(plate_dir)
for plate_date_dir in plate_subdirs:
logging.debug("plate_subdir: " + str(plate_date_dir))
rel_plate_date_dir = relpath(plate_date_dir, proj_image_dir)
# Parse filename for metadata (e.g. platename well, site, channet etc.)
metadata = parse_path_plate_date(rel_plate_date_dir)
logging.debug("metadata" + str(metadata))
# Check if plate exists in database (if no - then import folder)
#
# TODO (and aquisition-date?)
#
# TODO create screen?
#
plate_ID = getPlateID(conn, metadata['plate'])
if plate_ID is None:
# import images and create database entries for plate, well, site etc.
import_plate_images_and_meta(plate_date_dir, conn)
def _train_async(model: Model,
data: TrainingData,
checkpoint: Union[str, None],
parameter_checkpoint: Union[str, None],
save_start: bool,
train_params: trainer.TrainParams,
evaluators: List[Evaluator],
out: ModelDir,
notes=None,
dry_run=False,
start_eval=False):
""" Train while encoding batches on a seperate thread and storing them in a tensorflow Queue, can
be much faster then using the feed_dict approach """
train = data.get_train()
eval_datasets = data.get_eval()
loader = data.get_resource_loader()
print("Training on %d batches" % len(train))
print("Evaluation datasets: " + " ".join("%s (%d)" % (name, len(data)) for name, data in eval_datasets.items()))
# spec the model for the given datasets
model.set_inputs([train] + list(eval_datasets.values()), loader)
placeholders = model.get_placeholders()
train_queue = tf.FIFOQueue(train_params.async_encoding, [x.dtype for x in placeholders], name="train_queue")
evaluator_runner = AysncEvaluatorRunner(evaluators, model, train_params.async_encoding)
train_enqeue = train_queue.enqueue(placeholders)
train_close = train_queue.close(True)
is_train = tf.placeholder(tf.bool, ())
input_tensors = tf.cond(is_train, lambda: train_queue.dequeue(),
lambda: evaluator_runner.eval_queue.dequeue())
# tensorfow can't infer the shape for an unsized queue, so set it manually
for input_tensor, pl in zip(input_tensors, placeholders):
input_tensor.set_shape(pl.shape)
bcast = hvd.broadcast_global_variables(0)
print("Init model...")
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
config.gpu_options.visible_device_list = str(hvd.local_rank())
sess = tf.Session(config=config)
with sess.as_default():
pred = model.get_predictions_for(dict(zip(placeholders, input_tensors)))
evaluator_runner.set_input(pred)
if parameter_checkpoint is not None:
print("Restoring parameters from %s" % parameter_checkpoint)
saver = tf.train.Saver()
saver.restore(sess, checkpoint)
saver = None
print("Setting up model prediction / tf...")
all_vars = tf.global_variables()
loss, summary_tensor, train_opt, global_step, weight_ema = _build_train_ops(train_params)
# Pre-compute tensors we need at evaluations time
eval_tensors = []
for ev in evaluators:
eval_tensors.append(ev.tensors_needed(pred))
if train_params.best_weights is not None:
lst = all_vars
if weight_ema is not None:
for x in lst:
v = weight_ema.average(x)
if v is not None:
lst.append(v)
best_weight_saver = tf.train.Saver(var_list=lst, max_to_keep=1)
cur_best = None
else:
best_weight_saver = None
cur_best = None
saver = tf.train.Saver(max_to_keep=train_params.max_checkpoints_to_keep)
summary_writer = tf.summary.FileWriter(out.log_dir)
# Load or initialize the model parameters
if checkpoint is not None:
print("Restoring from checkpoint...")
saver.restore(sess, checkpoint)
print("Loaded checkpoint: " + str(sess.run(global_step)))
else:
print("Initializing parameters...")
sess.run(tf.global_variables_initializer())
sess.run(bcast)
# Make sure no bugs occur that add to the graph in the train loop, that can cause (eventuall) OOMs
tf.get_default_graph().finalize()
if dry_run:
return
on_step = sess.run(global_step)
if save_start:
# summary_writer.add_graph(sess.graph, global_step=on_step)
if hvd.rank() == 0:
trainer.save_train_start(out.dir, data, sess.run(global_step), evaluators, train_params, notes)
def enqueue_train():
try:
# feed data from the dataset iterator -> encoder -> queue
for epoch in range(train_params.num_epochs):
for batch in train.get_epoch():
feed_dict = model.encode(batch, True)
sess.run(train_enqeue, feed_dict)
except tf.errors.CancelledError:
# The queue_close operator has been called, exit gracefully
return
except Exception as e:
# Crashes the main thread with a queue exception
sess.run(train_close)
raise e
train_enqueue_thread = Thread(target=enqueue_train)
train_enqueue_thread.daemon = True # Ensure we exit the program on an excpetion
print("Start training!")
batch_time = 0
train_dict = {is_train: True}
eval_dict = {is_train: False}
try:
train_enqueue_thread.start()
for epoch in range(train_params.num_epochs):
for batch_ix in range(len(train)):
t0 = time.perf_counter()
on_step = sess.run(global_step) + 1
get_summary = on_step % train_params.log_period == 0
if get_summary:
summary, _, batch_loss = sess.run([summary_tensor, train_opt, loss], feed_dict=train_dict)
else:
summary = None
_, batch_loss = sess.run([train_opt, loss], feed_dict=train_dict)
if np.isnan(batch_loss):
raise RuntimeError("NaN loss!")
batch_time += time.perf_counter() - t0
if hvd.rank() == 0:
if summary is not None:
print("on epoch=%d batch=%d step=%d, time=%.3f" %
(epoch, batch_ix + 1, on_step, batch_time))
summary_writer.add_summary(
tf.Summary(value=[tf.Summary.Value(tag="time", simple_value=batch_time)]), on_step)
summary_writer.add_summary(summary, on_step)
batch_time = 0
# occasional saving
if hvd.rank() == 0:
if on_step % train_params.save_period == 0:
print("Checkpointing")
saver.save(sess, join(out.save_dir, "checkpoint-" + str(on_step)), global_step=global_step)
# Occasional evaluation
if (on_step % train_params.eval_period == 0) or start_eval:
print("Running evaluation...")
start_eval = False
t0 = time.perf_counter()
for name, data in eval_datasets.items():
n_samples = train_params.eval_samples.get(name)
evaluation = evaluator_runner.run_evaluators(sess, data, name, n_samples, eval_dict)
if hvd.rank() == 0:
for s in evaluation.to_summaries(name + "-"):
summary_writer.add_summary(s, on_step)
# Maybe save as the best weights
if train_params.best_weights is not None and name == train_params.best_weights[0]:
val = evaluation.scalars[train_params.best_weights[1]]
if cur_best is None or val > cur_best:
print("Save weights with current best weights (%s vs %.5f)" % (
"None" if cur_best is None else ("%.5f" % cur_best), val))
best_weight_saver.save(sess, join(out.best_weight_dir, "best"), global_step=global_step)
cur_best = val
print("Evaluation took: %.3f seconds" % (time.perf_counter() - t0))
finally:
sess.run(train_close) # terminates the enqueue thread with an exception
train_enqueue_thread.join()
saver.save(sess, relpath(join(out.save_dir, "checkpoint-" + str(on_step))), global_step=global_step)
sess.close()
def _setup_files(self, return_path_to_files=False, path_prefix='./'):
dirname = 'dir1'
regfilename = 'regfile'
hlinkname = 'hlink'
slinkname = 'slink1'
slink2name = 'slink2'
dir_abspath = path.join(self.mount_a.mountpoint, dirname)
regfile_abspath = path.join(self.mount_a.mountpoint, regfilename)
hlink_abspath = path.join(self.mount_a.mountpoint, hlinkname)
slink_abspath = path.join(self.mount_a.mountpoint, slinkname)
slink2_abspath = path.join(self.mount_a.mountpoint, slink2name)
self.mount_a.run_shell('mkdir ' + dir_abspath)
self.mount_a.run_shell('touch ' + regfile_abspath)
self.mount_a.run_shell(['ln', regfile_abspath, hlink_abspath])
self.mount_a.run_shell(['ln', '-s', regfile_abspath, slink_abspath])
self.mount_a.run_shell(['ln', '-s', dir_abspath, slink2_abspath])
dir2_name = 'dir2'
dir21_name = 'dir21'
regfile121_name = 'regfile121'
dir2_abspath = path.join(self.mount_a.mountpoint, dir2_name)
dir21_abspath = path.join(dir2_abspath, dir21_name)
regfile121_abspath = path.join(dir21_abspath, regfile121_name)
self.mount_a.run_shell('mkdir -p ' + dir21_abspath)
self.mount_a.run_shell('touch ' + regfile121_abspath)
sudo_write_file(self.mount_a.client_remote, regfile_abspath,
'somedata')
sudo_write_file(self.mount_a.client_remote, regfile121_abspath,
'somemoredata')
# TODO: is there a way to trigger/force update ceph.dir.rbytes?
# wait so that attr ceph.dir.rbytes gets a chance to be updated.
sleep(20)
expected_patterns = []
path_to_files = []
def append_expected_output_pattern(f):
if f == '/':
expected_patterns.append(r'{}{}{}'.format(size, " +", '.' + f))
else:
expected_patterns.append(r'{}{}{}'.format(size, " +",
path_prefix + path.relpath(f, self.mount_a.mountpoint)))
for f in [dir_abspath, regfile_abspath, regfile121_abspath,
hlink_abspath, slink_abspath, slink2_abspath]:
size = humansize(self.mount_a.stat(f, follow_symlinks=
False)['st_size'])
append_expected_output_pattern(f)
# get size for directories containig regfiles within
for f in [dir2_abspath, dir21_abspath]:
size = humansize(self.mount_a.stat(regfile121_abspath,
follow_symlinks=False)['st_size'])
append_expected_output_pattern(f)
# get size for CephFS root
size = 0
for f in [regfile_abspath, regfile121_abspath, slink_abspath,
slink2_abspath]:
size += self.mount_a.stat(f, follow_symlinks=False)['st_size']
size = humansize(size)
append_expected_output_pattern('/')
if return_path_to_files:
for p in [dir_abspath, regfile_abspath, dir2_abspath,
dir21_abspath, regfile121_abspath, hlink_abspath,
slink_abspath, slink2_abspath]:
path_to_files.append(path.relpath(p, self.mount_a.mountpoint))
return (expected_patterns, path_to_files)
else:
return expected_patterns
app_dir = str(os.environ['APP_DIR'])
host="193.2.205.19" #IP address of the server
port=50000
print("Connect to server")
try:
s.connect((host,port))
except socket.gaierror as err:
print("Address-related error connecting to server: ", err)
sys.exit(1)
except socket.error as err:
print("Connection error: ", err)
sys.exit(1)
filename= path.relpath("/root/LOG-a-TEC-testbed/applications/" + app_dir + "/node_results.txt")
f=open(filename, "rb") #with open (filename, "rb") as f:
line=f.read(1024)
while(line):
s.send(line)
#print('sent' , repr(line))
line=f.read(1024)
f.close()
print("Done sending file to server.")
s.close()
def generate_depfile(outpath, out_dir):
reloutpath = path.relpath(outpath, out_dir)
# Just lie and say we depend on build.ninja so we get re-run every gen.
# Despite the lie, this is more or less correct since we want to observe
# every build graph change.
return "%s: build.ninja" % reloutpath
def run(self):
if "skip-manim" in self.state.document.settings.env.app.builder.tags.tags:
node = skip_manim_node()
self.state.nested_parse(StringList(self.content[0]),
self.content_offset, node)
return [node]
from manim import config
global classnamedict
clsname = self.arguments[0]
if clsname not in classnamedict:
classnamedict[clsname] = 1
else:
classnamedict[clsname] += 1
hide_source = "hide_source" in self.options
save_as_gif = "save_as_gif" in self.options
save_last_frame = "save_last_frame" in self.options
assert not (save_as_gif and save_last_frame)
ref_content = (self.options.get("ref_modules", []) +
self.options.get("ref_classes", []) +
self.options.get("ref_functions", []))
if ref_content:
ref_block = f"""
.. admonition:: Example References
:class: example-reference
{' '.join(ref_content)}"""
else:
ref_block = ""
if "quality" in self.options:
quality = f'{self.options["quality"]}_quality'
else:
quality = "example_quality"
frame_rate = QUALITIES[quality]["frame_rate"]
pixel_height = QUALITIES[quality]["pixel_height"]
pixel_width = QUALITIES[quality]["pixel_width"]
qualitydir = f"{pixel_height}p{frame_rate}"
state_machine = self.state_machine
document = state_machine.document
source_file_name = document.attributes["source"]
source_rel_name = relpath(source_file_name, setup.confdir)
source_rel_dir = os.path.dirname(source_rel_name)
while source_rel_dir.startswith(os.path.sep):
source_rel_dir = source_rel_dir[1:]
dest_dir = os.path.abspath(
os.path.join(setup.app.builder.outdir, source_rel_dir))
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
source_block = [
".. code-block:: python",
"",
*[" " + line for line in self.content],
]
source_block = "\n".join(source_block)
config.media_dir = Path(setup.confdir) / "media"
config.images_dir = "{media_dir}/images"
config.video_dir = "{media_dir}/videos/{quality}"
output_file = f"{clsname}-{classnamedict[clsname]}"
config.assets_dir = Path("_static")
config_code = [
f'config["frame_rate"] = {frame_rate}',
f'config["pixel_height"] = {pixel_height}',
f'config["pixel_width"] = {pixel_width}',
f'config["save_last_frame"] = {save_last_frame}',
f'config["save_as_gif"] = {save_as_gif}',
f'config["output_file"] = r"{output_file}"',
]
user_code = self.content
if user_code[0].startswith(
">>> "): # check whether block comes from doctest
user_code = [
line[4:] for line in user_code
if line.startswith((">>> ", "... "))
]
code = [
"from manim import *",
*config_code,
*user_code,
f"{clsname}().render()",
]
exec("\n".join(code), globals())
# copy video file to output directory
if not (save_as_gif or save_last_frame):
filename = f"{output_file}.mp4"
filesrc = config.get_dir("video_dir") / filename
destfile = os.path.join(dest_dir, filename)
shutil.copyfile(filesrc, destfile)
elif save_as_gif:
filename = f"{output_file}.gif"
filesrc = config.get_dir("video_dir") / filename
elif save_last_frame:
filename = f"{output_file}.png"
filesrc = config.get_dir("images_dir") / filename
else:
raise ValueError("Invalid combination of render flags received.")
rendered_template = jinja2.Template(TEMPLATE).render(
clsname=clsname,
clsname_lowercase=clsname.lower(),
hide_source=hide_source,
filesrc_rel=os.path.relpath(filesrc, setup.confdir),
output_file=output_file,
save_last_frame=save_last_frame,
save_as_gif=save_as_gif,
source_block=source_block,
ref_block=ref_block,
)
state_machine.insert_input(rendered_template.split("\n"),
source=document.attributes["source"])
return []
def get_relative_path_from_output_directory(file_to_resolve):
return path.relpath(path.join(os.getcwd(), file_to_resolve), tsconfig_output_directory)
from bokeh.application import Application
from bokeh.application.handlers.function import FunctionHandler
from bokeh.plotting import figure
from bokeh.io.state import curstate
from bokeh.resources import Resources
from functools import partial # by https://stackoverflow.com/questions/41926478/python-bokeh-send-additional-parameters-to-widget-event-handler
import step_1_action
DATA_PATH_NAME = 'static'
USER_FILE_NAME = 'user_definitions.json'
DATA_DIR = join(dirname(__file__), DATA_PATH_NAME) # full abs path
rel_DATA_DIR = relpath(DATA_DIR)
rel_User_fileName = join(rel_DATA_DIR, USER_FILE_NAME)
case_test = True
case_test = False
default_text = """
this is an initialized text widget
"""
default_abs_script_path = join(dirname(__file__))
"""**************************************************"""
def load_JSON_file_to_Dict(fileName):
"""
def rela_path(path_a, path_b):
return relpath(path_a, path_b)
href="mailto:[email protected]">[email protected]</a></b> </p> </td> <td style="vertical-align: top;"> <p class="copyright"><a href="https://creativecommons.org/licenses/by/4.0/" target="_blank"><img src="{BASE_URL}/static/images/CC-BY-icon-80x15.png" alt="CC-BY" /></a></p> </td> </tr> </table> </div> </body> </html> """ DKIP_KEY_PATH = path.join(path.dirname(path.relpath(path.relpath(__file__))), ".keys", "dkim.key") # RQ RQ_REDIS_URL = "redis://redis:6379/0" RQ_QUEUE_CLASS = "orcid_hub.queuing.ThrottledQueue" # rq-dashboard config: RQ_POLL_INTERVAL = 5000 #: Web interface poll period for updates in ms WEB_BACKGROUND = "gray" # Celery CELERY_BROKER_URL = "redis://localhost:6379/0" CELERY_RESULT_BACKEND = "redis://localhost:6379/0" # CELERY_ACCEPT_CONTENT = ["application/x-python-serialize"] # CELERY_TASK_SERIALIZER = "pickle"
def write_state(source, layer, data_source_name, skipped, destination,
log_handler, tests_passed, cache_result, conform_result,
preview_path, slippymap_path, temp_dir):
'''
'''
source_id, _ = splitext(basename(source))
statedir = join(destination, source_id)
if not exists(statedir):
mkdir(statedir)
statedir = join(statedir, layer)
if not exists(statedir):
mkdir(statedir)
statedir = join(statedir, data_source_name)
if not exists(statedir):
mkdir(statedir)
if cache_result.cache:
scheme, _, cache_path1, _, _, _ = urlparse(cache_result.cache)
if scheme in ('file', ''):
cache_path2 = join(statedir,
'cache{1}'.format(*splitext(cache_path1)))
copy(cache_path1, cache_path2)
state_cache = relpath(cache_path2, statedir)
else:
state_cache = cache_result.cache
else:
state_cache = None
if conform_result.path:
_, _, processed_path1, _, _, _ = urlparse(conform_result.path)
processed_path2 = join(statedir,
'out{1}'.format(*splitext(processed_path1)))
copy(processed_path1, processed_path2)
# Write the sample data to a sample.json file
if conform_result.sample:
sample_path = join(statedir, 'sample.json')
with open(sample_path, 'w') as sample_file:
json.dump(conform_result.sample, sample_file, indent=2)
if preview_path:
preview_path2 = join(statedir, 'preview.png')
copy(preview_path, preview_path2)
if slippymap_path:
slippymap_path2 = join(statedir, 'slippymap.mbtiles')
copy(slippymap_path, slippymap_path2)
log_handler.flush()
output_path = join(statedir, 'output.txt')
copy(log_handler.stream.name, output_path)
if skipped:
source_problem = SourceProblem.skip_source
else:
with open(output_path) as file:
log_content = file.read()
if exists(source):
with open(source) as file:
source_data = json.load(file)
else:
source_data = {}
source_problem = find_source_problem(log_content, source_data)
state = [
('source', basename(source)),
('skipped', bool(skipped)),
('cache', state_cache),
('sample', conform_result.sample and relpath(sample_path, statedir)),
('website', conform_result.website),
('license', conform_result.license),
('geometry type', conform_result.geometry_type),
('address count', conform_result.address_count),
('version', cache_result.version),
('fingerprint', cache_result.fingerprint),
('cache time', cache_result.elapsed and str(cache_result.elapsed)),
('processed', conform_result.path
and relpath(processed_path2, statedir)),
('process time', conform_result.elapsed
and str(conform_result.elapsed)),
('output', relpath(output_path, statedir)),
('preview', preview_path and relpath(preview_path2, statedir)),
('slippymap', slippymap_path and relpath(slippymap_path2, statedir)),
('attribution required', boolstr(conform_result.attribution_flag)),
('attribution name', conform_result.attribution_name),
('share-alike', boolstr(conform_result.sharealike_flag)),
('source problem', getattr(source_problem, 'value', None)),
('code version', __version__),
('tests passed', tests_passed),
]
with open(join(statedir, 'index.txt'), 'w', encoding='utf8') as file:
out = csv.writer(file, dialect='excel-tab')
for row in zip(*state):
out.writerow(row)
with open(join(statedir, 'index.json'), 'w') as file:
json.dump(list(zip(*state)), file, indent=2)
_L.info(u'Wrote to state: {}'.format(file.name))
return file.name
def get_include_files(file_tree, base_dir):
include_files = {}
for f in file_tree:
include_files[path.relpath(f, base_dir)] = f
return include_files
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter
)
parser.add_argument('input_dir', help='input annotated directory')
parser.add_argument('output_dir', help='output dataset directory')
parser.add_argument('--labels', help='labels file', required=True)
parser.add_argument(
'--label-encoding',
default='utf-8',
help='file encoding of label files',
)
args = parser.parse_args()
if osp.exists(args.output_dir):
print('Output directory already exists:', args.output_dir)
sys.exit(1)
os.makedirs(args.output_dir)
os.makedirs(osp.join(args.output_dir, 'JPEGImages'))
print('Creating dataset:', args.output_dir)
now = datetime.datetime.now()
data = dict(
info=dict(
description=None,
url=None,
version=None,
year=now.year,
contributor=None,
date_created=now.strftime('%Y-%m-%d %H:%M:%S.%f'),
),
licenses=[dict(
url=None,
id=0,
name=None,
)],
images=[
# license, url, file_name, height, width, date_captured, id
],
type='instances',
annotations=[
# segmentation, area, iscrowd, image_id, bbox, category_id, id
],
categories=[
# supercategory, id, name
],
)
class_name_to_id = {}
for i, line in enumerate(open(args.labels).readlines()):
class_id = i - 1 # starts with -1
class_name = line.strip()
if class_id == -1:
assert class_name == '__ignore__'
continue
class_name_to_id[class_name] = class_id
data['categories'].append(dict(
supercategory=None,
id=class_id,
name=class_name,
))
out_ann_file = osp.join(args.output_dir, 'annotations.json')
label_files = glob.glob(osp.join(args.input_dir, '*.json'))
for image_id, filename in enumerate(label_files):
print('Generating dataset from:', filename)
label_file = labelme.LabelFile(filename=filename,
encoding=args.label_encoding)
base = osp.splitext(osp.basename(filename))[0]
out_img_file = osp.join(
args.output_dir, 'JPEGImages', base + '.jpg'
)
img = labelme.utils.img_data_to_arr(label_file.imageData)
PIL.Image.fromarray(img).convert("RGB").save(out_img_file)
data['images'].append(dict(
license=0,
url=None,
file_name=osp.relpath(out_img_file, osp.dirname(out_ann_file)),
height=img.shape[0],
width=img.shape[1],
date_captured=None,
id=image_id,
))
masks = {} # for area
segmentations = collections.defaultdict(list) # for segmentation
for shape in label_file.shapes:
points = shape['points']
label = shape['label']
group_id = shape.get('group_id')
shape_type = shape.get('shape_type', 'polygon')
mask = labelme.utils.shape_to_mask(
img.shape[:2], points, shape_type
)
if group_id is None:
group_id = uuid.uuid1()
instance = (label, group_id)
if instance in masks:
masks[instance] = masks[instance] | mask
else:
masks[instance] = mask
if shape_type == 'rectangle':
(x1, y1), (x2, y2) = points
x1, x2 = sorted([x1, x2])
y1, y2 = sorted([y1, y2])
points = [x1, y1, x2, y1, x2, y2, x1, y2]
else:
points = np.asarray(points).flatten().tolist()
segmentations[instance].append(points)
segmentations = dict(segmentations)
for instance, mask in masks.items():
cls_name, group_id = instance
if cls_name not in class_name_to_id:
continue
cls_id = class_name_to_id[cls_name]
mask = np.asfortranarray(mask.astype(np.uint8))
mask = pycocotools.mask.encode(mask)
area = float(pycocotools.mask.area(mask))
bbox = pycocotools.mask.toBbox(mask).flatten().tolist()
data['annotations'].append(dict(
id=len(data['annotations']),
image_id=image_id,
category_id=cls_id,
segmentation=segmentations[instance],
area=area,
bbox=bbox,
iscrowd=0,
))
with open(out_ann_file, 'w') as f:
json.dump(data, f)
def is_subdir(suspect_child, suspect_parent):
suspect_child = path.realpath(suspect_child)
suspect_parent = path.realpath(suspect_parent)
relative = path.relpath(suspect_child, start=suspect_parent)
return not relative.startswith(os.pardir)
try:
word = client.DispatchEx("Word.Application")
for files in listdir(getcwd()):
match = 0
if files.endswith(".doc"):
s, match = "doc", 1
elif files.endswith(".docx"):
s, match = "docx", 1
if match:
new_name = files.replace("." + s, r".pdf")
in_file = path.abspath(folder + "\\" + files)
new_file = path.abspath(folder + "\\" + new_name)
doc = word.Documents.Open(in_file)
print(strftime("%H:%M:%S"), " " + s + " -> pdf ",
path.relpath(new_file))
doc.SaveAs(new_file, FileFormat=17)
doc.Close()
except (Exception, e):
print(e)
finally:
word.Quit()
print("\n", strftime("%H:%M:%S"), "Finished converting files.")
# Count the number of pdf files.
num_pdf = count_files(".pdf")
print("\nNumber of pdf files: ", num_pdf)