def getFilesToBackup(self):
filesToBackup = []
for hash,files in self.currentBackupSet.items():
if not self.previousBackupSet.has_key(hash):
filesToBackup.append((hash,files))
else:
previousFiles = self.previousBackupSet[hash]
found = False
equals = []
for previousEntry in previousFiles:
for file in files:
base = file[0][len(path.commonprefix([file[0],self.options.source])):].lstrip("/")
if base in previousEntry[0]:
found = True
else:
equals.append(previousEntry[0])
if not found:
if self.options.verbose:
pass
newFiles = []
for f in files:
newFiles.append(f[0][len(path.commonprefix([f[0],self.options.source])):].lstrip("/"))
print warn("Duplicate:"), "image already backed up under different name: %s == %s" % (newFiles,equals)
if not self.options.only_hash: filesToBackup.append((hash,[[file[0]]]))
return filesToBackup
def longest_common_prefix_suffix(s1, s2):
"""
:param s1:
:param s2:
:return:
"""
return commonprefix([s1, s2[::-1]]), commonprefix([s1[::-1], s2])
def get_output_file(infiles):
prefix = path.commonprefix(infiles)
suffix = path.commonprefix([f[::-1] for f in infiles])[::-1]
num_suffix = ''.join([c for c in itertools.takewhile(lambda x: x.isdigit(), suffix)])
num_prefix = ''.join([c for c in itertools.takewhile(lambda x: x.isdigit(), prefix[::-1])])[::-1]
prefix = prefix[:len(prefix)-len(num_prefix)]
suffix = suffix[len(num_suffix):]
diffs = [s[len(prefix):len(s)-len(suffix)] for s in infiles]
output_file = prefix + min(diffs, key=alphanum_key) + '-' + max(diffs, key=alphanum_key) + suffix
return output_file
def regex_opt_inner(strings, open_paren):
"""Return a regex that matches any string in the sorted list of strings."""
close_paren = open_paren and ')' or ''
# print strings, repr(open_paren)
if not strings:
# print '-> nothing left'
return ''
first = strings[0]
if len(strings) == 1:
# print '-> only 1 string'
return open_paren + escape(first) + close_paren
if not first:
# print '-> first string empty'
return open_paren + regex_opt_inner(strings[1:], '(?:') \
+ '?' + close_paren
if len(first) == 1:
# multiple one-char strings? make a charset
oneletter = []
rest = []
for s in strings:
if len(s) == 1:
oneletter.append(s)
else:
rest.append(s)
if len(oneletter) > 1: # do we have more than one oneletter string?
if rest:
# print '-> 1-character + rest'
return open_paren + regex_opt_inner(rest, '') + '|' \
+ make_charset(oneletter) + close_paren
# print '-> only 1-character'
return make_charset(oneletter)
prefix = commonprefix(strings)
if prefix:
plen = len(prefix)
# we have a prefix for all strings
# print '-> prefix:', prefix
return open_paren + escape(prefix) \
+ regex_opt_inner([s[plen:] for s in strings], '(?:') \
+ close_paren
# is there a suffix?
strings_rev = [s[::-1] for s in strings]
suffix = commonprefix(strings_rev)
if suffix:
slen = len(suffix)
# print '-> suffix:', suffix[::-1]
return open_paren \
+ regex_opt_inner(sorted(s[:-slen] for s in strings), '(?:') \
+ escape(suffix[::-1]) + close_paren
# recurse on common 1-string prefixes
# print '-> last resort'
return open_paren + \
'|'.join(regex_opt_inner(list(group[1]), '')
for group in groupby(strings, lambda s: s[0] == first[0])) \
+ close_paren
def test_that_a_valid_license_notice_exists_in_every_source_file_and_that_global_licensing_information_is_correct(self):
license_notice = compile(r"""(?P<comment_start>#|--|//) This Source Code Form is subject to the terms of the Mozilla Public
(?P=comment_start) License, v\. 2\.0\. If a copy of the MPL was not distributed with this file,
(?P=comment_start) You can obtain one at http://mozilla\.org/MPL/2\.0/\.
(?P=comment_start)
(?P=comment_start) Copyright \(c\) (?P<first_year>20\d\d)(-(?P<last_year>20\d\d))?, Lars Asplund lars\.anders\.asplund@gmail\.com""")
log_date = compile(r'Date:\s*(?P<year>20\d\d)-\d\d-\d\d')
licensed_files = []
repo_root = abspath(join(dirname(__file__), '..'))
for root, dirs, files in walk(repo_root):
for f in files:
if 'preprocessed' in root:
continue
osvvm_directory = abspath(join(repo_root, 'vhdl', 'osvvm'))
if commonprefix([osvvm_directory, abspath(join(root, f))]) == osvvm_directory:
continue
osvvm_integration_example_directory = abspath(join(repo_root, 'examples', 'osvvm_integration', 'src'))
if commonprefix([osvvm_integration_example_directory, abspath(join(root, f))]) == osvvm_integration_example_directory:
continue
if splitext(f)[1] in ['.vhd', '.vhdl', '.py', '.v', '.sv']:
licensed_files.append(join(root, f))
i = 0
min_first_year = None
max_last_year = None
for f in licensed_files:
stdout.write('\r%d/%d' % (i + 1, len(licensed_files)))
stdout.flush()
i += 1
proc = Popen(['git', 'log', '--follow', '--date=short', f], \
bufsize=0, stdout=PIPE, stdin=PIPE, stderr=STDOUT, universal_newlines=True)
out, _ = proc.communicate()
first_year = None
last_year = None
for date in log_date.finditer(out):
first_year = int(date.group('year')) if first_year is None else min(int(date.group('year')), first_year)
last_year = int(date.group('year')) if last_year is None else max(int(date.group('year')), last_year)
min_first_year = first_year if min_first_year is None else min(min_first_year, first_year)
max_last_year = last_year if max_last_year is None else max(max_last_year, last_year)
with open(f) as fp:
code = fp.read()
match = license_notice.search(code)
self.assertIsNotNone(match, "Failed to find license notice in %s" % f)
if first_year == last_year:
self.assertEqual(int(match.group('first_year')), first_year, 'Expected copyright year to be %d in %s' % (first_year, f))
self.assertIsNone(match.group('last_year'), 'Expected no copyright years range in %s' % join(root, f))
else:
self.assertIsNotNone(match.group('last_year'), 'Expected copyright year range %d-%d in %s' % (first_year, last_year, f))
self.assertEqual(int(match.group('first_year')), first_year, 'Expected copyright year range to start with %d in %s' % (first_year, f))
self.assertEqual(int(match.group('last_year')), last_year, 'Expected copyright year range to end with %d in %s' % (last_year, f))
print('\n')
def _check_lookahead(self, inp):
"""
Check a counterexample for lookahead transitions using prefix-closed
queries. If an unknown lookahead is found it is added on the observation
table.
Args:
inp (list): Counterexample input.
"""
# Make a prefix closed membership query and gather the result
prefix = []
prefix_set = [[]]
prefix_set_input = [[]]
for c in inp:
prefix.append(c)
prefix_set_input.append(prefix)
prefix_set.append(self.membership_query(prefix))
for i in xrange(1, len(prefix_set)):
if commonprefix([prefix_set[i], prefix_set[i-1]]) != prefix_set[i-1]:
logging.debug('Lookahead detected at position %s : %s, %s',
i, prefix_set[i-1], prefix_set[i])
la_out = _remove_common_prefix(prefix_set[i], prefix_set[i-1])
j = None
for j in reversed(xrange(i)):
if commonprefix([prefix_set[i], prefix_set[j]]) == prefix_set[j]:
la_inp = inp[j:i]
break
la_out = _remove_common_prefix(prefix_set[i], prefix_set[j])
access_string = self._run_in_hypothesis(inp, j)
out_as = self.membership_query(access_string)
out_complete = self.membership_query(list(access_string)+la_inp)
# If The access string for the lookahead state is wrong, we will
# add the lookahead path once this is fixed in a next iteration.
if _remove_common_prefix(out_complete, out_as) != la_out:
logging.debug('Lookahead detected but access string is '+ \
'wrong, skipping.')
continue
if self.ot.add_lookahead_transition(access_string,
tuple(la_inp),
tuple(la_out)):
# Fill all table entries for the lookahead transition
for col in self.ot.dist_strings:
self._fill_ot_entry(access_string + tuple(la_inp), col)
# New lookahead added, no need for further processing.
break
def _normalize_path(base_dir, path):
"""Helper to check paths passed to methods of this class.
Checks whether `path` is beneath `base_dir` and normalize it.
Additionally paths are converted into relative paths with respect to
`base_dir`, considering PWD in case of relative paths. This
is intended to be used in repository classes, which means that
`base_dir` usually will be the repository's base directory.
Parameters
----------
path: str
path to be normalized
base_dir: str
directory to serve as base to normalized, relative paths
Returns
-------
str:
path, that is a relative path with respect to `base_dir`
"""
if not path:
return path
base_dir = realpath(base_dir)
# path = normpath(path)
# Note: disabled normpath, because it may break paths containing symlinks;
# But we don't want to realpath relative paths, in case cwd isn't the correct base.
if isabs(path):
# path might already be a symlink pointing to annex etc,
# so realpath only its directory, to get "inline" with realpath(base_dir)
# above
path = opj(realpath(dirname(path)), basename(path))
if commonprefix([path, base_dir]) != base_dir:
raise FileNotInRepositoryError(msg="Path outside repository: %s"
% path, filename=path)
else:
pass
elif commonprefix([realpath(getpwd()), base_dir]) == base_dir:
# If we are inside repository, rebuilt relative paths.
path = opj(realpath(getpwd()), path)
else:
# We were called from outside the repo. Therefore relative paths
# are interpreted as being relative to self.path already.
return path
return relpath(path, start=base_dir)
def _apply_meta_attach(meta, ctx):
action = ctx.new_action()
begin = meta.startswith(META_ATTACH_FLAG)
end = meta.endswith(META_ATTACH_FLAG)
if begin:
meta = meta[len(META_ATTACH_FLAG):]
action.prev_attach = True
if end:
meta = meta[:-len(META_ATTACH_FLAG)]
action.next_attach = True
last_word = ctx.last_action.word or ''
if not meta:
# We use an empty connection to indicate a "break" in the
# application of orthography rules. This allows the
# stenographer to tell Plover not to auto-correct a word.
action.orthography = False
elif (
last_word and
not meta.isspace() and
ctx.last_action.orthography and
begin and (not end or _has_word_boundary(meta))
):
new_word = add_suffix(last_word, meta)
common_len = len(commonprefix([last_word, new_word]))
replaced = last_word[common_len:]
action.prev_replace = ctx.last_text(len(replaced))
assert replaced.lower() == action.prev_replace.lower()
last_word = last_word[:common_len]
meta = new_word[common_len:]
action.text = meta
if action.prev_attach:
action.word = _rightmost_word(last_word + meta)
return action
def flush(self):
# FIXME:
# - what about things like emoji zwj sequences?
# - normalize strings to better handle combining characters?
#
# >>> u"C\u0327"
# 'Ç'
# >>> len(u"C\u0327")
# 2
# >>> len(unicodedata.normalize('NFC', u"C\u0327"))
# 1
if len(self.before.replaced_text) > len(self.after.replaced_text):
assert self.before.replaced_text.endswith(self.after.replaced_text)
replaced_text = self.before.replaced_text
else:
assert self.after.replaced_text.endswith(self.before.replaced_text)
replaced_text = self.after.replaced_text
before = replaced_text[:len(replaced_text)-len(self.before.replaced_text)] + self.before.appended_text
after = replaced_text[:len(replaced_text)-len(self.after.replaced_text)] + self.after.appended_text
common_length = len(commonprefix([before, after]))
erased = len(before) - common_length
if erased:
self.output.send_backspaces(erased)
appended = after[common_length:]
if appended:
self.output.send_string(appended)
self.before.reset(self.after.trailing_space)
self.after.reset(self.after.trailing_space)
def _dump_names_from_pathnames(self, pathnames):
"""Given a list of pathnames of this form:
(uuid[.name].dump)+
This function will return a list of just the name part of the path.
in the case where there is no name, it will use the default dump
name from configuration.
example:
['6611a662-e70f-4ba5-a397-69a3a2121129.dump',
'6611a662-e70f-4ba5-a397-69a3a2121129.flash1.dump',
'6611a662-e70f-4ba5-a397-69a3a2121129.flash2.dump',
]
returns
['upload_file_minidump', 'flash1', 'flash2']
"""
prefix = path.commonprefix([path.basename(x) for x in pathnames])
prefix_length = len(prefix)
dump_names = []
for a_pathname in pathnames:
base_name = path.basename(a_pathname)
dump_name = base_name[prefix_length:-len(self.config.dump_suffix)]
if not dump_name:
dump_name = self.config.dump_field
dump_names.append(dump_name)
return dump_names
def __init__(self, files, output_dir, name, format, reference, excludes):
logger.debug('Incoming files: %s', files)
basedir = commonprefix(files)
logger.debug('Prefix is "%s"', basedir)
if isdir(basedir):
self.basedir = basedir
else:
self.basedir = dirname(basedir)
logger.debug('Basedir is "%s"', self.basedir)
self.output_dir = output_dir
logger.debug('Output dir is "%s"', self.output_dir)
globfiles = list(chain.from_iterable(list(map(glob, files))))
logger.debug("Globfiles: %s", globfiles)
for file_or_dir in globfiles:
for walkroot, _, walkfiles in walk(file_or_dir):
for walkfile in walkfiles:
globfiles.append(join(walkroot, walkfile))
logger.debug('Resolved globfiles: %s', globfiles)
for exclude in (excludes or []):
if exclude[-1] is not '*':
exclude += '*'
evicts = fnmatch.filter(globfiles, exclude)
logger.debug("exclude '%s' evicts => %s", exclude, evicts)
globfiles = [
globfile for globfile in globfiles if globfile not in evicts
]
relative_files = [
r for r in
[relpath(globfile, self.basedir) for globfile in globfiles]
if r is not '.'
]
logger.debug('Resolved relative files: %s', relative_files)
self._files = OrderedDict.fromkeys(relative_files)
logger.debug("Initialized map, is now %s", self._files)
self.name = name
self.format = format
if not reference:
self.refdir = self.output_dir
elif not isdir(reference):
self.refdir = dirname(reference)
else:
self.refdir = reference
def parse_type(not_parent):
abs_other = abspath(other)
abs_not_parent = abspath(not_parent)
if abs_not_parent == commonprefix([abs_not_parent, abs_other]):
raise argparse.ArgumentTypeError("{0} may not be a parent directory of {1}".format(not_parent, other))
else:
return not_parent
def _extract_archive(archive_fpath, archive_file, archive_namelist, output_dir,
force_commonprefix=True, prefix=None,
dryrun=False, verbose=not QUIET, overwrite=None):
"""
archive_fpath = zip_fpath
archive_file = zip_file
"""
# force extracted components into a subdirectory if force_commonprefix is
# on return_path = output_diG
# FIXMpathE doesn't work right
if prefix is not None:
output_dir = join(output_dir, prefix)
util_path.ensurepath(output_dir)
archive_basename, ext = split_archive_ext(basename(archive_fpath))
if force_commonprefix and commonprefix(archive_namelist) == '':
# use the archivename as the default common prefix
output_dir = join(output_dir, archive_basename)
util_path.ensurepath(output_dir)
for member in archive_namelist:
(dname, fname) = split(member)
dpath = join(output_dir, dname)
util_path.ensurepath(dpath)
if verbose:
print('[utool] Unarchive ' + fname + ' in ' + dpath)
if not dryrun:
if overwrite is False:
if exists(join(output_dir, member)):
continue
archive_file.extract(member, path=output_dir)
return output_dir
def _update_config(self):
self.config = deepcopy(self.defaults)
logger.debug('>> Searching for config')
for ext in ('.yml', '.yaml'):
f = File(normpath(self.src.path, 'config' + ext))
if f.exists:
logger.debug('.. found: %s', f.path)
try:
self.config.update(Config(f.content))
except ConfigException as e:
raise ConfigException(e.message, 'src: {0}'.format(f.path))
self.config['locale'] = self.opts.get('locale', self.config['locale'])
self.config['assets_url'] = absurl(self.config['assets_url'], '')
self.config['base_url'] = absurl(self.opts.get('base_url', self.config['base_url']), '')
for setting in ('archives_url', 'posts_url', 'tags_url'):
self.config[setting] = absurl(self.config[setting])
for setting in ('archives_url', 'assets_url', 'base_url', 'posts_url', 'tags_url'):
if re.search(r'(?:^\.{2}/|/\.{2}$|/\.{2}/)', self.config[setting]):
raise ConfigException('Invalid config setting.', 'setting: {0}'.format(setting), 'path traversal is not allowed')
for pattern in self.config['include']:
if op.commonprefix((self.src.path, normpath(self.src.path, pattern))) != self.src.path:
raise ConfigException('Invalid include path.', 'path: {0}'.format(pattern), 'path traversal is not allowed')
break
else:
logger.debug('.. no config file found')
def commit(self, filename, basedir="/"):
skip = len(path.commonprefix([filename, basedir]))
sink = self.open(filename[skip:])
source = open(filename, "r")
shutil.copyfileobj(source, sink)
source.close()
sink.close()
def score_model(self, model_txn, txn):
"""Score an existing transaction for its ability to provide a model
for an incomplete transaction.
Args:
model_txn: The transaction to be scored.
txn: The incomplete transaction.
Returns:
A float number representing the score, normalized in [0,1].
"""
def get_description(txn):
return ('{} {}'.format(txn.payee or '', txn.narration or '')).strip()
# If the target transaction does not have a description, there is
# nothing we can do
txn_description = get_description(txn)
n_max = len(txn_description)
if n_max > 1:
# Only consider model transactions whose posting to the target
# account has the same sign as the transaction to be completed
posting = [p for p in model_txn.postings if p.account == self.account][0]
if number.same_sign(posting.units.number, txn.postings[0].units.number):
model_txn_description = get_description(model_txn)
n_match = len(path.commonprefix(
[model_txn_description, txn_description]))
score = float(n_match) / float(n_max)
return score
return 0
def discover(directories, index, filterfunc=lambda filename: True):
"""Import and initialize modules from `directories` list.
:param directories: list of directories
:param index: index function"""
def find(directories, filterfunc):
"""Discover and yield python modules (aka files that endswith .py) if
`filterfunc` returns True for that filename."""
for directory in directories:
for root, dirs, files in os.walk(directory):
for fname in files:
if fname.endswith('.py') and filterfunc(join(root, fname)):
yield join(root, fname)
for filename in find(directories, filterfunc):
modname, ext = os.path.splitext(os.path.basename(rchop(filename, os.sep + '__init__.py')))
fp, path, descr = imp.find_module(modname, directories)
prefix = commonprefix((PATH, filename))
if prefix:
modname = 'acrylamid.'
modname += rchop(filename[len(prefix):].replace(os.sep, '.'), '.py')
try:
mod = sys.modules[modname]
except KeyError:
try:
mod = imp.load_module(modname, fp, path, descr)
except (ImportError, SyntaxError, ValueError) as e:
log.exception('%r %s: %s', modname, e.__class__.__name__, e)
continue
index(mod)
def get_articles(request):
query = request.GET.get('query', '')
tq = parse_tq(request.GET.get('tq', ''))
tqx = parse_tqx(request.GET.get('tqx', ''))
select = Article.select()
select = select.limit(tq.get('limit', 1))
select = select.offset(tq.get('offset', 0))
if query:
select = select.where(Article.subject % ('*%s*' % query))
subjects = [ a.subject for a in select ]
LOG.debug(lcs(subjects))
LOG.debug(commonprefix(subjects))
dt = gviz.DataTable({
'posted': ('datetime', 'Posted'),
'poster': ('string', 'Poster'),
'subject': ('string', 'Subject'),
'message_id': ('string', 'ID')
})
dt.LoadData( a._data for a in select )
dt_order = ['subject', 'posted', 'poster', 'message_id']
gviz_json = dt.ToJSonResponse(req_id=tqx.get('reqId', 0),
columns_order=dt_order)
return itty.Response(gviz_json, content_type='application/json')
def _derive_module_name(self, path):
common = commonprefix([self.working, path])
slice_module_name = slice(len(common) + 1, len(path))
return path[slice_module_name]\
.replace('.py', '')\
.replace('\\', '.')\
.replace('/', '.')
def __contains__(self, path):
p1 = self.path
p2 = path
if self.IS_WINDOWS:
p1 = p1.lower()
p2 = p2.lower()
return commonprefix((p1 + sep, p2)) == p1 + sep
def lca(scores1, scores2, tax):
classdict = dict()
for query, hit in scores1.iteritems():
scr1 = set(hit.keys())
scr2 = set(scores2[query].keys())
# find the common hits of both dictionaries
common = scr1.intersection(scr2)
commonscores = dict()
topscore = 0
for goodhit in common:
score = hit[goodhit] + scores2[query][goodhit]
commonscores[goodhit] = score
if score > topscore:
topscore = score
# remove from common all the scores that aren't at least 95% of topscore
minscore = 0.95 * topscore
topscores = commonscores.copy()
for goodhit in commonscores:
if commonscores[goodhit] < minscore:
del topscores[goodhit]
# get the LCA for these
classify = ""
for tophit in topscores:
if classify == "" and tophit in tax:
classify = str(tax[tophit])
else:
# print "And the common pref is " + commonprefix([classify, str(tax[tophit])])
classify = commonprefix([classify, str(tax[tophit])])
if classify == "" or classify == "[]":
classify = "Unclassified;"
# take longest substr ending in ;
meaningful = re.match(".+;", classify)
classify = meaningful.group()
classdict[query] = classify
return classdict
def lookup(self, index, word_array):
""" Get closest match to word (accepts imperfect matches)
:param list[str] word_array: str
:param int index: index of word in word_array to check
:return: closest match or None if none found
:rtype: str
"""
word = word_array[index]
logging.debug("looking up in indicator dictionary: " + word)
i = bisect_left(self.dict, word)
nearest_matches = self.dict[i - 1: i + 1]
# todo: return length of match as well
for i in range(0, len(nearest_matches)):
split = nearest_matches[i].split()
# require multi-word indicators to match exactly
# todo: after this, it's exact so don't use get_closest_matches
if len(split) > 1 and \
not self.match_multiple_words(split, word_array[index:]):
nearest_matches[i] = ""
match = get_close_matches(word, nearest_matches, n=1,
cutoff=min_indicator_distance)
if not match:
return None
match = match[0]
# todo: arbitrary, essentially checking stem of word
if word != match and len(commonprefix([word, match])) < 3:
return None
logging.debug("Closest match to " + word + " is " + match)
return match
def printStatus(makeLog, makeAllLog, textTestTmp, smtpServer, out):
failed = ""
build = commonprefix([basename(makeLog), basename(makeAllLog)])
print >> out, build,
print >> out, datetime.now().ctime()
print >> out, "--"
print >> out, basename(makeLog)
warnings = 0
errors = 0
svnLocked = False
for l in file(makeLog):
if ("svn: Working copy" in l and "locked" in l) or "svn: Failed" in l:
svnLocked = True
failed += l
if re.search("[Ww]arn[ui]ng[: ]", l):
warnings += 1
if re.search("[Ee]rror[: ]", l) or re.search("[Ff]ehler[: ]", l):
errors += 1
failed += l
if svnLocked:
failed += "svn up failed\n\n"
print >> out, warnings, "warnings"
if errors:
print >> out, errors, "errors"
failed += "make failed\n\n"
print >> out, "--"
for root, dirs, files in os.walk(textTestTmp):
for f in files:
if f.startswith("batchreport"):
b = open(join(root, f))
l = b.readline()
if l.startswith("FAILED") or l.startswith("succeeded") or l.startswith("killed") or l.startswith("known bugs"):
print >> out, f, l,
b.close()
print >> out, "--"
print >> out, basename(makeAllLog)
warnings = 0
errors = 0
for l in file(makeAllLog):
if re.search("[Ww]arn[ui]ng[: ]", l):
warnings += 1
if "error " in l.lower():
errors += 1
failed += l
print >> out, warnings, "warnings"
if errors:
print >> out, errors, "errors"
failed += "make debug failed\n\n"
print >> out, "--"
if failed:
fromAddr = "*****@*****.**"
toAddr = "*****@*****.**"
message = """From: "%s" <%s>
To: %s
Subject: Error occurred while building
%s""" % (build, fromAddr, toAddr, failed)
server = smtplib.SMTP(smtpServer)
server.sendmail(fromAddr, toAddr, message)
server.quit()
def lca(scores1, scores2, percent, tax):
classdict = dict()
for query, hit in scores1.iteritems():
scr1 = set(hit.keys())
scr2 = set(scores2[query].keys())
#find the common hits of both dictionaries
common = scr1.intersection(scr2)
commonscores=dict()
for goodhit in common:
score = hit[goodhit] + scores2[query][goodhit]
commonscores[goodhit] = score
#get the top percent scores of this intersection
topcommon = toppercent(commonscores, percent)
#get the LCA for these
classify = ''
for query, score in topcommon.iteritems():
if classify == '':
classify = tax[query]
else:
classify = commonprefix([classify, tax[query]])
if classify == '':
classify = 'Unclassified;'
#print classify
#take longest substr ending in ;
meaningful = re.match(".+;", classify)
classify = meaningful.group()
classdict[query] = classify
#print query + "\t" + classify
return classdict
def test_check_existing_dirpaths(self):
# Check that returns a list with the paths, in the same order as the
# input comma separated list
tmp_dirpath1 = mkdtemp(prefix='pyqi_tmp_testd_')
tmp_dirpath2 = mkdtemp(prefix='pyqi_tmp_testd_')
self._dirs_to_clean_up = [tmp_dirpath1, tmp_dirpath2]
option = PyqiOption('-d', '--dirs_test', type='existing_dirpaths')
exp = [tmp_dirpath1, tmp_dirpath2]
value = ",".join(exp)
obs = check_existing_dirpaths(option, '-d', value)
self.assertEqual(obs, exp)
# Check that returns a list with the paths when using wildcars
# note that the order is not important now
value = commonprefix(exp) + '*'
obs = check_existing_dirpaths(option, '-d', value)
self.assertEqual(set(obs), set(exp))
# Check that raises an error when the wildcard does not match any path
self.assertRaises(OptionValueError, check_existing_dirpaths, option,
'-f', '/hopefully/a/non/existing/path*')
# Check that raises an error when one of the directories does not exist
value = ",".join([tmp_dirpath1, tmp_dirpath2,
'/hopefully/a/non/existing/path*'])
self.assertRaises(OptionValueError, check_existing_dirpaths, option,
'-f', value)
# Check that raises an error when one of the paths is a file
tmp_f, tmp_path = mkstemp()
self._paths_to_clean_up = [tmp_path]
value = ",".join([tmp_dirpath1, tmp_dirpath2, tmp_path])
self.assertRaises(OptionValueError, check_existing_dirpaths, option,
'-f', value)
def _dialog(self, message="Select Folder", new_directory=True):
"""Creates a directory dialog box for working with
Keyword Arguments:
message (string): Message to display in dialog
new_directory (bool): True if allowed to create new directory
Returns:
A directory to be used for the file operation.
"""
# Wildcard pattern to be used in file dialogs.
if not self.multifile:
mode = "directory"
else:
mode = "files"
dlg = get_filedialog(what=mode, title=message, mustexist=not new_directory)
if len(dlg) != 0:
if not self.multifile:
self.directory = dlg
ret = self.directory
else:
ret = None
else:
self.pattern = [path.basename(name) for name in dlg]
self.directory = path.commonprefix(dlg)
ret = self.directory
return ret
def create_graph_df(vtask_paths, graphs_dir_out):
"""
Creates a frame that maps sourcefiles to networkx digraphs in terms of DOT files
:param source_path_list:
:param dest_dir_path:
:param relabel:
:return:
"""
if not isdir(graphs_dir_out):
raise ValueError('Invalid destination directory.')
data = []
graphgen_times = []
print('Writing graph representations of verification tasks to {}'.format(graphs_dir_out), flush=True)
common_prefix = commonprefix(vtask_paths)
for vtask in tqdm(vtask_paths):
short_prefix = dirname(common_prefix)
path = join(graphs_dir_out, vtask[len(short_prefix):][1:])
if not os.path.exists(dirname(path)):
os.makedirs(dirname(path))
ret_path = path + '.pickle'
# DEBUG
if isfile(ret_path):
data.append(ret_path)
continue
start_time = time.time()
graph_path, node_labels_path, edge_types_path, edge_truth_path, node_depths_path \
= _run_cpachecker(abspath(vtask))
nx_digraph = nx.read_graphml(graph_path)
node_labels = _read_node_labeling(node_labels_path)
nx.set_node_attributes(nx_digraph, 'label', node_labels)
edge_types = _read_edge_labeling(edge_types_path)
parsed_edge_types = _parse_edge(edge_types)
nx.set_edge_attributes(nx_digraph, 'type', parsed_edge_types)
edge_truth = _read_edge_labeling(edge_truth_path)
parsed_edge_truth = _parse_edge(edge_truth)
nx.set_edge_attributes(nx_digraph, 'truth', parsed_edge_truth)
node_depths = _read_node_labeling(node_depths_path)
parsed_node_depths = _parse_node_depth(node_depths)
nx.set_node_attributes(nx_digraph, 'depth', parsed_node_depths)
assert not isfile(ret_path)
assert node_labels and parsed_edge_types and parsed_edge_truth and parsed_node_depths
nx.write_gpickle(nx_digraph, ret_path)
data.append(ret_path)
gg_time = time.time() - start_time
graphgen_times.append(gg_time)
return pd.DataFrame({'graph_representation': data}, index=vtask_paths), graphgen_times
def safejoin(root, subpath):
if not SAFENAME.match(subpath):
raise BadName(u"unsafe path name: %r" % subpath)
path = join(root, subpath)
if commonprefix([root + sep, path]) != root + sep:
raise BadName(u"invalid relative path: %r" % subpath)
return path
def __contains__(self, path):
p1 = self.path
p2 = path
if system() == "Windows":
p1 = p1.lower()
p2 = p2.lower()
return commonprefix((p1 + sep, p2)) == p1 + sep
def validate_absolute_path(self, root, absolute_path):
"""Overrides StaticFileHandler's method to include authentication
"""
# Get the filename (or the base directory) of the result
len_prefix = len(commonprefix([root, absolute_path]))
base_requested_fp = absolute_path[len_prefix:].split(sep, 1)[0]
current_user = self.current_user
# If the user is an admin, then allow access
if current_user.level == 'admin':
return super(ResultsHandler, self).validate_absolute_path(
root, absolute_path)
# otherwise, we have to check if they have access to the requested
# resource
user_id = current_user.id
accessible_filepaths = check_access_to_analysis_result(
user_id, base_requested_fp)
# Turn these filepath IDs into absolute paths
db_files_base_dir = get_db_files_base_dir()
relpaths = filepath_ids_to_rel_paths(accessible_filepaths)
accessible_filepaths = {join(db_files_base_dir, relpath)
for relpath in relpaths.values()}
# check if the requested resource is a file (or is in a directory) that
# the user has access to
if join(root, base_requested_fp) in accessible_filepaths:
return super(ResultsHandler, self).validate_absolute_path(
root, absolute_path)
else:
raise QiitaPetAuthorizationError(user_id, absolute_path)
def _atom_to_action_spaces_after(atom, last_action):
"""Convert an atom into an action.
Arguments:
atom -- A string holding an atom. An atom is an irreducible string that is
either entirely a single meta command or entirely text containing no meta
commands.
last_action -- The context in which the new action takes place.
Returns: An action for the atom.
"""
action = _Action()
last_word = last_action.word
last_glue = last_action.glue
last_attach = last_action.attach
last_capitalize = last_action.capitalize
last_lower = last_action.lower
last_upper = last_action.upper
last_upper_carry = last_action.upper_carry
last_orthography = last_action.orthography
last_space = SPACE if last_action.text.endswith(SPACE) else NO_SPACE
meta = _get_meta(atom)
if meta is not None:
meta = _unescape_atom(meta)
if meta in META_COMMAS:
action.text = meta + SPACE
if last_action.text != '':
action.replace = SPACE
if last_attach:
action.replace = NO_SPACE
elif meta in META_STOPS:
action.text = meta + SPACE
action.capitalize = True
action.lower = False
if last_action.text != '':
action.replace = SPACE
if last_attach:
action.replace = NO_SPACE
elif meta == META_CAPITALIZE:
action = last_action.copy_state()
action.capitalize = True
action.lower = False
elif meta == META_LOWER:
action = last_action.copy_state()
action.lower = True
action.capitalize = False
elif meta == META_UPPER:
action = last_action.copy_state()
action.lower = False
action.upper = True
action.capitalize = False
elif meta == META_RETRO_CAPITALIZE:
action = last_action.copy_state()
action.word = _capitalize(action.word)
if len(last_action.text) < len(last_action.word):
action.replace = last_action.word + SPACE
action.text = _capitalize(last_action.word + SPACE)
else:
action.replace = last_action.text
action.text = _capitalize_nowhitespace(last_action.text)
elif meta == META_RETRO_LOWER:
action = last_action.copy_state()
action.word = _lower(action.word)
if len(last_action.text) < len(last_action.word):
action.replace = last_action.word + SPACE
action.text = _lower(last_action.word + SPACE)
else:
action.replace = last_action.text
action.text = _lower_nowhitespace(last_action.text)
elif meta == META_RETRO_UPPER:
action = last_action.copy_state()
action.word = _upper(action.word)
action.upper_carry = True
if len(last_action.text) < len(last_action.word):
action.replace = last_action.word + SPACE
action.text = _upper(last_action.word + SPACE)
else:
action.replace = last_action.text
action.text = _upper(last_action.text)
elif meta.startswith(META_RETRO_FORMAT):
if (meta.startswith(META_RETRO_FORMAT) and meta.endswith(')')):
dict_format = meta[len(META_RETRO_FORMAT):-len(')')]
action = last_action.copy_state()
action.replace = last_action.word + SPACE
try:
float(last_action.word)
except ValueError:
pass
else:
format = dict_format.replace('c', '{:,.2f}')
cast_input = float(last_action.word)
try:
int(last_action.word)
except ValueError:
pass
else:
format = dict_format.replace('c', '{:,}')
cast_input = int(last_action.word)
action.text = format.format(cast_input) + SPACE
action.word = format.format(cast_input)
elif meta.startswith(META_COMMAND):
action = last_action.copy_state()
action.command = meta[len(META_COMMAND):]
elif meta.startswith(META_GLUE_FLAG):
action.glue = True
text = meta[len(META_GLUE_FLAG):]
if last_capitalize:
text = _capitalize(text)
if last_lower:
text = _lower(text)
action.text = text + SPACE
action.word = _rightmost_word(text)
if last_glue:
action.replace = SPACE
action.word = _rightmost_word(last_word + text)
if last_attach:
action.replace = NO_SPACE
action.word = _rightmost_word(last_word + text)
elif (meta.startswith(META_ATTACH_FLAG) or
meta.endswith(META_ATTACH_FLAG)):
begin = meta.startswith(META_ATTACH_FLAG)
end = meta.endswith(META_ATTACH_FLAG)
if begin:
meta = meta[len(META_ATTACH_FLAG):]
if end and len(meta) >= len(META_ATTACH_FLAG):
meta = meta[:-len(META_ATTACH_FLAG)]
space = NO_SPACE if end else SPACE
replace_space = NO_SPACE if last_attach else SPACE
if end:
action.attach = True
if begin and end and meta == '':
# We use an empty connection to indicate a "break" in the
# application of orthography rules. This allows the stenographer
# to tell plover not to auto-correct a word.
action.orthography = False
if last_action.text != '':
action.replace = replace_space
if (((begin and not end) or (begin and end and ' ' in meta)) and
last_orthography):
new = orthography.add_suffix(last_word.lower(), meta)
common = commonprefix([last_word.lower(), new])
if last_action.text == '':
replace_space = NO_SPACE
action.replace = last_word[len(common):] + replace_space
meta = new[len(common):]
if begin and end:
if last_action.text != '':
action.replace = replace_space
if last_capitalize:
meta = _capitalize(meta)
if last_lower:
meta = _lower(meta)
if last_upper_carry:
meta = _upper(meta)
action.upper_carry = True
action.text = meta + space
action.word = _rightmost_word(
last_word[:len(last_word + last_space)-len(action.replace)] + meta)
if end and not begin and last_space == SPACE:
action.word = _rightmost_word(meta)
elif meta.startswith(META_KEY_COMBINATION):
action = last_action.copy_state()
action.combo = meta[len(META_KEY_COMBINATION):]
else:
text = _unescape_atom(atom)
if last_capitalize:
text = _capitalize(text)
if last_lower:
text = _lower(text)
if last_upper:
text = _upper(text)
action.upper_carry = True
action.text = text + SPACE
action.word = _rightmost_word(text)
return action
def convert():
# Check if app is in another process
from subprocess import check_output
from shutil import copy2
s = check_output('tasklist', shell=True)
if s.count(b"pymanga") > 1:
fatal("Pymanga is already running on another process!")
# Prompt for archive
archives = selectzip("Open archive(s) to use:")
# Extract and read zip file
files, skipped = zip.read(archives)
if len(skipped):
confirm("The following files will not be included:", str(skipped))
# Select output file
from pathlib import Path
from os.path import commonprefix
outFile = path.basename(commonprefix(archives))
if "." in outFile:
outFile = outFile.split(".")[0]
if False: # confirm("Choose format", "Do you want to convert the .cbz to .pdf?") == "yes":
label("Creating .pdf file...")
command = [imagemagick]
command.extend(files)
# Add extension to output file
outFile = path.join(temp, outFile + ".pdf")
command.append(outFile)
# Convert file using ImageMagick
from subprocess import Popen, PIPE, STDOUT
p = Popen(command, stdin=PIPE, stdout=PIPE, stderr=STDOUT, encoding='UTF8')
# Wait for process to finish
from time import sleep
while p.poll() is None:
sleep(1)
response = p.stdout.readline()
if response != "":
fatal("Can't convert to pdf!", response)
else:
label("Creating .cbz file...")
# Add extension to output file
outFile = path.join(temp, outFile + ".cbz")
# Copy all images in order to root of temp folder
order = 0
pages = []
fill = len(str(len(files))) + 1
for file in files:
order += 1
page = str(order).zfill(fill) + file[file.rfind('.'):]
copy2(file, page)
pages.append(page)
# Create .cbz file
zip.create(pages, outFile)
# Save output file to Desktop or open with default editor
outFile2 = path.join(Path.home(), "Desktop", path.basename(outFile))
try:
copy2(outFile, outFile2)
except Exception as e:
fatal("Error copying file!", e=e)
success("File saved to the desktop successfully!")
from os import _exit
_exit(1)
def _best_prefix_transform(set1, target_set2):
"""
Find a way to transform prefixes of items in set1 to match target_set2
Example:
>>> set1 = {'mod.f.0.w',
>>> 'mod.f.1.b',
>>> 'mod.f.1.n',
>>> 'mod.f.1.rm',
>>> 'mod.f.1.rv',}
>>> #
>>> target_set2 = {
>>> 'bar.foo.extra.f.1.b',
>>> 'bar.foo.extra.f.1.n',
>>> 'bar.foo.extra.f.1.w',
>>> 'bar.foo.extra.f.3.w',
>>> }
>>> _best_prefix_transform(set1, target_set2)
>>> target_set2.add('JUNK')
>>> _best_prefix_transform(set1, target_set2)
"""
# probably an efficient way to do this with a trie
# NOTE: In general this is a graph-isomorphism problem or a maximum common
# subgraph problem. However, we can look only at the special case of
# "maximum common subtrees". Given two directory structures (as trees)
# we find the common bits.
# https://perso.ensta-paris.fr/~diam/ro/online/viggo_wwwcompendium/node168.html
# We can approximate to O(log log n / log^2 n)
# Can get algorithm from maximum independent set
# https://arxiv.org/abs/1602.07210
# The most efficient algorithm here would be for solving
# "Maximum common labeled subtrees"
# APX-hard for unordered trees, but polytime solveable for ordered trees
# For directory structures we can induce an order, and hense obtain a
# polytime solution
# #
# On the Maximum Common Embedded Subtree Problem for Ordered Trees
# https://pdfs.semanticscholar.org/0b6e/061af02353f7d9b887f9a378be70be64d165.pdf
from os.path import commonprefix
prefixes1 = commonprefix(list(set1)).split('.')
prefixes2 = commonprefix(list(target_set2)).split('.')
# Remove the trailing prefixes that are the same
num_same = 0
for i in range(1, min(len(prefixes1), len(prefixes2))):
if prefixes1[-i] == prefixes2[-i]:
num_same = i
else:
break
prefixes1 = prefixes1[:-num_same]
prefixes2 = prefixes2[:-num_same]
ALLOW_FUZZY = 1
if ALLOW_FUZZY and len(prefixes2) == 0:
# SUPER HACK FOR CASE WHERE THERE IS JUST ONE SPOILER ELEMENT IN THE
# TARGET SET. THE ALGORITHM NEEDS TO BE RETHOUGHT FOR THAT CASE
possible_prefixes = [k.split('.') for k in target_set2]
prefix_hist = ub.ddict(lambda: 0)
for item in possible_prefixes:
for i in range(1, len(item)):
prefix_hist[tuple(item[0:i])] += 1
prefixes2 = ['.'.join(ub.argmax(prefix_hist))]
def add_prefix(items, prefix):
return {prefix + k for k in items}
def remove_prefix(items, prefix):
return {k[len(prefix):] if k.startswith(prefix) else k for k in items}
import itertools as it
found_cand = []
for i1, i2 in it.product(range(len(prefixes1) + 1),
range(len(prefixes2) + 1)):
if i1 == 0 and i2 == 0:
continue
# Very inefficient, we should be able to do better
prefix1 = '.'.join(prefixes1[:i1])
prefix2 = '.'.join(prefixes2[:i2])
if prefix1:
prefix1 = prefix1 + '.'
if prefix2:
prefix2 = prefix2 + '.'
# We are allowed to remove a prefix from a set, add the other
# prefix to the set, or remove and then add.
set1_cand1 = remove_prefix(set1, prefix1)
set1_cand2 = add_prefix(set1, prefix2)
set1_cand3 = add_prefix(set1_cand1, prefix2)
common1 = set1_cand1 & target_set2
common2 = set1_cand2 & target_set2
common3 = set1_cand3 & target_set2
if common1:
found_cand.append({
'transform': [('remove', prefix1)],
'value': len(common1),
})
if common2:
found_cand.append({
'transform': [('add', prefix2)],
'value': len(common2),
})
if common3:
found_cand.append({
'transform': [('remove', prefix1), ('add', prefix2)],
'value':
len(common3),
})
if len(found_cand):
found = max(found_cand, key=lambda x: x['value'])
else:
found = None
return found
def find_common_path(dict_values):
return commonprefix(dict_values).rsplit('.', 1)[0]
def _atom_to_action_spaces_before(atom, last_action):
"""Convert an atom into an action.
Arguments:
atom -- A string holding an atom. An atom is an irreducible string that is
either entirely a single meta command or entirely text containing no meta
commands.
last_action -- The context in which the new action takes place.
Returns: An action for the atom.
"""
action = _Action(space_char=last_action.space_char, case=last_action.case)
last_word = last_action.word
last_glue = last_action.glue
last_attach = last_action.attach
last_capitalize = last_action.capitalize
last_lower = last_action.lower
last_upper = last_action.upper
last_upper_carry = last_action.upper_carry
last_orthography = last_action.orthography
begin = False # for meta attach
meta = _get_meta(atom)
if meta is not None:
meta = _unescape_atom(meta)
if meta in META_COMMAS:
action.text = meta
elif meta in META_STOPS:
action.text = meta
action.capitalize = True
action.lower = False
action.upper = False
elif meta == META_CAPITALIZE:
action = last_action.copy_state()
action.capitalize = True
action.lower = False
action.upper = False
elif meta == META_LOWER:
action = last_action.copy_state()
action.lower = True
action.upper = False
action.capitalize = False
elif meta == META_UPPER:
action = last_action.copy_state()
action.lower = False
action.upper = True
action.capitalize = False
elif meta == META_RETRO_CAPITALIZE:
action = last_action.copy_state()
action.word = _capitalize(action.word)
if len(last_action.text) < len(last_action.word):
action.replace = last_action.word
action.text = _capitalize(last_action.word)
else:
action.replace = last_action.text
action.text = _capitalize_nowhitespace(last_action.text)
elif meta == META_RETRO_LOWER:
action = last_action.copy_state()
action.word = _lower(action.word)
if len(last_action.text) < len(last_action.word):
action.replace = last_action.word
action.text = _lower(last_action.word)
else:
action.replace = last_action.text
action.text = _lower_nowhitespace(last_action.text)
elif meta == META_RETRO_UPPER:
action = last_action.copy_state()
action.word = _upper(action.word)
action.upper_carry = True
if len(last_action.text) < len(last_action.word):
action.replace = last_action.word
action.text = _upper(last_action.word)
else:
action.replace = last_action.text
action.text = _upper(last_action.text)
elif (meta.startswith(META_CARRY_CAPITALIZATION) or
meta.startswith(META_ATTACH_FLAG + META_CARRY_CAPITALIZATION)):
action = _apply_carry_capitalize(meta, last_action)
elif meta.startswith(META_RETRO_FORMAT):
if meta.startswith(META_RETRO_FORMAT) and meta.endswith(')'):
action = _apply_currency(meta, last_action)
elif meta.startswith(META_COMMAND):
action = last_action.copy_state()
action.command = meta[len(META_COMMAND):]
elif meta.startswith(META_MODE):
action = last_action.copy_state()
action = _change_mode(meta[len(META_MODE):], action)
elif meta.startswith(META_GLUE_FLAG):
action.glue = True
glue = last_glue or last_attach
space = NO_SPACE if glue else SPACE
text = meta[len(META_GLUE_FLAG):]
if last_capitalize:
text = _capitalize(text)
if last_lower:
text = _lower(text)
action.text = space + text
action.word = _rightmost_word(last_word + action.text)
elif (meta.startswith(META_ATTACH_FLAG)
or meta.endswith(META_ATTACH_FLAG)):
begin = meta.startswith(META_ATTACH_FLAG)
end = meta.endswith(META_ATTACH_FLAG)
if begin:
meta = meta[len(META_ATTACH_FLAG):]
if end and len(meta) >= len(META_ATTACH_FLAG):
meta = meta[:-len(META_ATTACH_FLAG)]
space = NO_SPACE if begin or last_attach else SPACE
if end:
action.attach = True
if begin and end and meta == '':
# We use an empty connection to indicate a "break" in the
# application of orthography rules. This allows the
# stenographer to tell plover not to auto-correct a word.
action.orthography = False
if (((begin and not end) or (begin and end and ' ' in meta))
and last_orthography):
new = orthography.add_suffix(last_word.lower(), meta)
common = commonprefix([last_word.lower(), new])
action.replace = last_word[len(common):]
meta = new[len(common):]
if last_capitalize:
meta = _capitalize(meta)
if last_lower:
meta = _lower(meta)
if last_upper_carry:
meta = _upper(meta)
action.upper_carry = True
action.text = space + meta
action.word = _rightmost_word(last_word[:len(last_word) -
len(action.replace)] +
action.text)
elif meta.startswith(META_KEY_COMBINATION):
action = last_action.copy_state()
action.combo = meta[len(META_KEY_COMBINATION):]
else:
text = _unescape_atom(atom)
if last_capitalize:
text = _capitalize(text)
if last_lower:
text = _lower(text)
if last_upper:
text = _upper(text)
action.upper_carry = True
space = NO_SPACE if last_attach else SPACE
action.text = space + text
action.word = _rightmost_word(text)
action.text = _apply_mode(action.text, action.case, action.space_char,
begin, last_attach, last_glue, last_capitalize,
last_upper, last_lower)
return action
def addToTar(self,
tar,
pattern,
exclude=[],
base=None,
proc=None,
verbose=False):
"""The workhorse for the packCase-method"""
if base == None:
base = path.basename(self.name)
if self.parallel and proc is None:
for p in self.processorDirs():
self.addToTar(tar,
path.join(path.dirname(pattern), p,
path.basename(pattern)),
exclude=exclude,
base=base,
verbose=verbose,
proc=p)
for name in glob.glob(pattern):
excluded = False
for e in exclude:
if fnmatch.fnmatch(path.basename(name), e):
excluded = True
if excluded:
continue
if path.isdir(name):
for m in listdir(name):
self.addToTar(tar,
path.join(name, m),
exclude=exclude,
verbose=verbose,
proc=proc,
base=base)
else:
arcname = path.join(base, name[len(self.name) + 1:])
if path.islink(name):
# if the symbolic link points to a file in the case keep it
# otherwise replace with the real file
lPath = path.os.readlink(name)
if not path.isabs(lPath):
rPath = path.realpath(name)
common = path.commonprefix(
[path.abspath(rPath),
path.abspath(base)])
# if the path is shorter than the base it must be outside the case
if len(common) < len(path.abspath(base)):
name = path.abspath(rPath)
else:
# use the abolute path
name = lPath
try:
tar.getmember(arcname)
# don't add ... the file is already there'
except KeyError:
# file not in tar
if verbose:
print_("Adding", name, "to tar")
tar.add(name, arcname=arcname)
def _find_best_prefix(self, rkey):
for prefix in self._subtries.keys():
common = commonprefix([prefix, rkey])
if common:
return prefix, len(common)
return None
def file_endswith(end_pattern):
return splitext(split(list(
filter(lambda x: len(commonprefix([filename, x])) > 0 and x.endswith(end_pattern), files))[0])[1])[0]
def _hosted_zone_matches_domain(self, zone, domain):
reversed_domain = ''.join(reversed(domain))
reversed_hosted_zone_name = ''.join(reversed(zone['Name']))
common_name = commonprefix(
(reversed_hosted_zone_name, reversed_domain))
return len(common_name) == len(reversed_hosted_zone_name)
def seg(self, seg_node, seg_data):
"""
Generate XML for the segment data and matching map node
@param seg_node: Map Node
@type seg_node: L{node<map_if.x12_node>}
@param seg_data: Segment object
@type seg_data: L{segment<segment.Segment>}
"""
if not seg_node.is_segment():
raise EngineError('Node must be a segment')
parent = pop_to_parent_loop(seg_node) # Get enclosing loop
# check path for new loops to be added
cur_path = self._path_list(parent.get_path())
#if seg_node.id == 'GS':
# import ipdb; ipdb.set_trace()
if self.last_path == cur_path and seg_node.is_first_seg_in_loop():
# loop repeat
self.writer.pop()
(xname, attrib) = self._get_loop_info(cur_path[-1])
self.writer.push(xname, attrib)
else:
last_path = self.last_path
match_idx = self._get_path_match_idx(last_path, cur_path)
root_path = self._path_list(
commonprefix(['/'.join(cur_path), '/'.join(last_path)]))
if seg_node.is_first_seg_in_loop() and root_path == cur_path:
match_idx -= 1
for i in range(len(last_path) - 1, match_idx - 1, -1):
self.writer.pop()
for i in range(match_idx, len(cur_path)):
(xname, attrib) = self._get_loop_info(cur_path[i])
self.writer.push(xname, attrib)
seg_node_id = self._get_node_id(seg_node, parent, seg_data)
(xname, attrib) = self._get_seg_info(seg_node_id)
self.writer.push(xname, attrib)
for i in range(len(seg_data)):
child_node = seg_node.get_child_node_by_idx(i)
if child_node.usage == 'N' or seg_data.get('%02i' %
(i + 1)).is_empty():
pass # Do not try to ouput for invalid or empty elements
elif child_node.is_composite():
(xname, attrib) = self._get_comp_info(seg_node_id)
self.writer.push(xname, attrib)
comp_data = seg_data.get('%02i' % (i + 1))
for j in range(len(comp_data)):
subele_node = child_node.get_child_node_by_idx(j)
(xname, attrib) = self._get_subele_info(subele_node.id)
self.writer.elem(xname, comp_data[j].get_value(), attrib)
self.writer.pop() # end composite
elif child_node.is_element():
if seg_data.get_value('%02i' % (i + 1)) == '':
pass
#self.writer.empty(u"ele", attrs={u'id': child_node.id})
else:
(xname, attrib) = self._get_ele_info(child_node.id)
self.writer.elem(xname,
seg_data.get_value('%02i' % (i + 1)),
attrib)
else:
raise EngineError(
'Node must be a either an element or a composite')
self.writer.pop() # end segment
self.last_path = cur_path
def run(self, stdscr):
"""
Cheats selection menu processing..
:param stdscr: screen
"""
# init
Gui.init_colors()
stdscr.clear()
self.height, self.width = stdscr.getmaxyx()
self.max_visible_cheats = self.height - 7
self.cursorpos = 0
while True:
stdscr.refresh()
self.cheats = self.search()
self.draw(stdscr)
c = stdscr.getch()
if c == curses.KEY_ENTER or c == 10 or c == 13:
# Process selected command (if not empty)
if self.selected_cheat() != None:
Gui.cmd = command.Command(self.selected_cheat(),
Gui.arsenalGlobalVars)
# check if arguments are needed
if len(Gui.cmd.args) != 0:
# args needed -> ask
args_menu = ArgslistMenu(self)
curses.endwin()
curses.echo()
wrapper(args_menu.run)
break
elif c == curses.KEY_F10 or c == 27:
Gui.cmd = None
break # Exit the while loop
elif c == 339 or c == curses.KEY_PPAGE:
# Page UP
self.move_page(-1)
elif c == 338 or c == curses.KEY_NPAGE:
# Page DOWN
self.move_page(1)
elif c == curses.KEY_UP:
# Move UP
self.move_position(-1)
elif c == curses.KEY_DOWN:
# Move DOWN
self.move_position(1)
elif c == curses.KEY_BACKSPACE or c == 8:
if self.check_move_cursor(-1):
i = self.xcursor - self.x_init - 1
self.input_buffer = self.input_buffer[:
i] + self.input_buffer[
i + 1:]
self.xcursor -= 1
# new search -> reset position
self.position = 0
self.page_position = 0
elif c == curses.KEY_DC or c == 127:
if self.check_move_cursor(1):
i = self.xcursor - self.x_init - 1
self.input_buffer = self.input_buffer[:i +
1] + self.input_buffer[
i + 2:]
# new search -> reset position
self.position = 0
self.page_position = 0
elif c == curses.KEY_LEFT:
# Move cursor LEFT
if self.check_move_cursor(-1): self.xcursor -= 1
elif c == curses.KEY_RIGHT:
# Move cursor RIGHT
if self.check_move_cursor(1): self.xcursor += 1
elif c == curses.KEY_BEG or c == curses.KEY_HOME:
# Move cursor to the BEGIN
self.xcursor = self.x_init
elif c == curses.KEY_END:
# Move cursor to the END
self.xcursor = self.x_init + len(self.input_buffer)
elif c == 9:
# TAB cmd auto complete
if self.input_buffer != "":
predictions = []
for cheat in self.cheats:
if cheat.command.startswith(self.input_buffer):
predictions.append(cheat.command)
if len(predictions) != 0:
self.input_buffer = commonprefix(predictions)
self.xcursor = self.x_init + len(self.input_buffer)
self.position = 0
self.page_position = 0
elif c >= 20 and c < 127:
i = self.xcursor - self.x_init
self.input_buffer = self.input_buffer[:i] + chr(
c) + self.input_buffer[i:]
self.xcursor += 1
# new search -> reset position
self.position = 0
self.page_position = 0
curses.endwin()
def activate(sandbox_name,
add_sdk_to_path=False,
new_env_vars=None,
app_id=None,
**overrides):
"""Context manager for command-line scripts started outside of dev_appserver.
:param sandbox_name: str, one of 'local', 'remote' or 'test'
:param add_sdk_to_path: bool, optionally adds the App Engine SDK to sys.path
:param options_override: an options structure to pass down to dev_appserver setup
Available sandboxes:
local: Adds libraries specified in app.yaml to the path and initializes local service stubs as though
dev_appserver were running.
remote: Adds libraries specified in app.yaml to the path and initializes remote service stubs.
test: Adds libraries specified in app.yaml to the path and sets up no service stubs. Use this
with `google.appengine.ext.testbed` to provide isolation for tests.
Example usage:
import djangae.sandbox as sandbox
with sandbox.activate('local'):
from django.core.management import execute_from_command_line
execute_from_command_line(sys.argv)
"""
if sandbox_name not in SANDBOXES:
raise RuntimeError('Unknown sandbox "{}"'.format(sandbox_name))
project_root = environment.get_application_root()
# Store our original sys.path before we do anything, this must be tacked
# onto the end of the other paths so we can access globally installed things (e.g. ipdb etc.)
original_path = sys.path[:]
# Setup paths as though we were running dev_appserver. This is similar to
# what the App Engine script wrappers do.
if add_sdk_to_path:
try:
import wrapper_util # Already on sys.path
except ImportError:
sys.path[0:0] = [_find_sdk_from_path()]
import wrapper_util
else:
try:
import wrapper_util
except ImportError:
raise RuntimeError(
"Couldn't find a recent enough Google App Engine SDK, make sure you are using at least 1.9.6"
)
sdk_path = _find_sdk_from_python_path()
_PATHS = wrapper_util.Paths(sdk_path)
project_paths = [] # Paths under the application root
system_paths = [] # All other paths
app_root = environment.get_application_root()
# We need to look at the original path, and make sure that any paths
# which are under the project root are first, then any other paths
# are added after the SDK ones
for path in _PATHS.scrub_path(_SCRIPT_NAME, original_path):
if commonprefix([app_root, path]) == app_root:
project_paths.append(path)
else:
system_paths.append(path)
# We build a list of SDK paths, and add any additional ones required for
# the oauth client
appengine_paths = _PATHS.script_paths(_SCRIPT_NAME)
for path in _PATHS.oauth_client_extra_paths:
if path not in appengine_paths:
appengine_paths.append(path)
# Now, we make sure that paths within the project take precedence, followed
# by the SDK, then finally any paths from the system Python (for stuff like
# ipdb etc.)
sys.path = (project_paths + appengine_paths + system_paths)
# Gotta set the runtime properly otherwise it changes appengine imports, like wepapp
# when you are not running dev_appserver
import yaml
with open(os.path.join(project_root, 'app.yaml'), 'r') as app_yaml:
app_yaml = yaml.load(app_yaml)
os.environ['APPENGINE_RUNTIME'] = app_yaml.get('runtime', '')
# Initialize as though `dev_appserver.py` is about to run our app, using all the
# configuration provided in app.yaml.
import google.appengine.tools.devappserver2.application_configuration as application_configuration
import google.appengine.tools.devappserver2.python.sandbox as sandbox
import google.appengine.tools.devappserver2.devappserver2 as devappserver2
import google.appengine.tools.devappserver2.wsgi_request_info as wsgi_request_info
import google.appengine.ext.remote_api.remote_api_stub as remote_api_stub
import google.appengine.api.apiproxy_stub_map as apiproxy_stub_map
# The argparser is the easiest way to get the default options.
options = devappserver2.PARSER.parse_args([project_root])
options.enable_task_running = False # Disable task running by default, it won't work without a running server
options.skip_sdk_update_check = True
for option in overrides:
if not hasattr(options, option):
raise ValueError("Unrecognized sandbox option: {}".format(option))
setattr(options, option, overrides[option])
if app_id:
configuration = application_configuration.ApplicationConfiguration(
options.config_paths, app_id=app_id)
else:
configuration = application_configuration.ApplicationConfiguration(
options.config_paths)
# Enable built-in libraries from app.yaml without enabling the full sandbox.
module = configuration.modules[0]
for l in sandbox._enable_libraries(module.normalized_libraries):
sys.path.insert(1, l)
# Propagate provided environment variables to the sandbox.
# This is required for the runserver management command settings flag,
# which sets an environment variable needed by Django.
from google.appengine.api.appinfo import EnvironmentVariables
old_env_vars = module.env_variables if module.env_variables else {}
if new_env_vars is None:
new_env_vars = {}
module._app_info_external.env_variables = EnvironmentVariables.Merge(
old_env_vars,
new_env_vars,
)
try:
global _OPTIONS
global _CONFIG
_CONFIG = configuration
_OPTIONS = options # Store the options globally so they can be accessed later
kwargs = dict(
devappserver2=devappserver2,
configuration=configuration,
options=options,
wsgi_request_info=wsgi_request_info,
remote_api_stub=remote_api_stub,
apiproxy_stub_map=apiproxy_stub_map,
)
with SANDBOXES[sandbox_name](**kwargs):
yield
finally:
sys.path = original_path
from os import listdir, stat
from os.path import isfile, join, commonprefix, splitext
thePath = "\\\\etude\\Archiv\\Daten\\Partikel\\FARO\\"
theScript = "U:\\home\\reina\\src\\megamol-dev\\plugins\\pbs\\utils\\cpe2mmpld.lua"
allFiles = [f for f in listdir(thePath) if isfile(join(thePath, f))]
filteredFiles = []
for f in allFiles:
found = False
for f2 in allFiles:
pf = commonprefix([f, f2])
# print("common prefix of %s and %s : %s" % (f, f2, pf))
if pf == splitext(f)[0]:
found = True
break
if not splitext(f)[1] == ".raw":
found = True
size = stat(thePath + f)
if not found and size.st_size > 48:
# print("%s is a leaf" % f)
filteredFiles.append(f)
for f in filteredFiles:
print("mmconsole -p %s -o file %s" % (theScript, splitext(f)[0]))
def reflexive(self, n):
if n is None:
return False
if commonprefix([self.path, n.path]) == n.path: #LCA(self,n)
return True
return False
def df_read_list(dir_sat, filenms, add_word=None):
out_filename = path.commonprefix(filenms)
if add_word is not None:
out_filename = out_filename + add_word
return [df_read(dir_sat, fname)[0] for fname in filenms], out_filename
def tied_name(name1, name2):
common_suffix = commonprefix([name1[::-1], name2[::-1]])[::-1]
return common_suffix.strip(':_')
def from_spec(cls, spec, ids):
"""
Create a schema from a spec
:param spec: This is a list of dictionaries. Each dictionary specifies
a single attribute that's expected to appear in the message. Each
dictionary contains these fields:
- "name": The name of the attribute; this will typically be an
identifier from an enum in a Linux kernel header file. This
value must appear as a key in the ``ids`` param.
- "python_name": The short Python identifier to be used to access
the attribute in parsed values and identify it when building
with kwargs. If absent, defaults to lower-case version of the
unique suffix of "name" among sibling attributes e.g. if
attribute names are "MY_ATTR_FOO" and "MY_ATTR_BAR", Python
names default to
"foo" and "bar".
- "type": The name of the expected type of the attribute. This
determines the way in which the attribute is built and
parsed. Allowed values are:
- "u8", "u16", "s16", "u32", "u64": Integer of relevant size and
signedness
- "str": Null-terminated ASCII string
- "bytes": Byte blob
- "array": Concatenated array of fixed-size
sub-elements. "subelem_type" specifies type of those
sub-elems. An example of this in Linux is
NL80211_ATTR_STA_SUPPORTED_RATES which is an array of u8
values. This is mapped to a Python list.
- "list": Set of sub-attributes, using attribute IDs 1-N to
index sub-elements. This uses the attribute header (which has
a length field) allows the sub-elements to have a variable
size. An example of this is in Linux is
NL80211_ATTR_IFTYPE_EXT_CAPA; this is a nested set of
attributes with IDs 1-N, each of which is a _further_ nested
set of attributes expressing interface capability info. This
is also mapped to a Python list.
- Or, another list of attribute entries, representing a
directly-nested attribute set. An example of this in Linux is
NL80211_ATTR_KEY, where the payload contains further netlink
attributes.
- "subelem_type": For attribute types that represent collections,
this stores the type of the elements of the collection. This can
have all the same values as "type".
- "required": If present and True, :meth:`build` raises an error
if no value is provided for this attribute
:param ids: Mapping from attribute names to numerical IDs.
"""
# In case of direct recursion from this class to itself (i.e. when
# the spec has an attribute that directly embeds another attribute set)
# We pass the full spec in as a dict instead of just the list of
# subattribute specs. Need to convert to that list.
if isinstance(spec, dict):
spec = spec["type"]
common_prefix = commonprefix(list(a["name"] for a in spec))
if not common_prefix.endswith("_"):
common_prefix += "_"
# It would be weird for the ordering to matter semantically, but we use
# OrderedDict to keep stable ordering so that message content can be
# predicted byte-for-byte for testing and debugging.
subattr_schemata = OrderedDict()
name_mapping = {}
required_attrs = []
for field_spec in spec:
schema_cls = get_schema_class(field_spec)
subattr_schema = schema_cls.from_spec(field_spec, ids)
subattr_schemata[field_spec["name"]] = subattr_schema
if "python_name" in field_spec:
name_mapping[field_spec["python_name"]] = field_spec["name"]
else:
python_name = field_spec["name"][len(common_prefix):].lower()
name_mapping[python_name] = field_spec["name"]
if field_spec.get("required", False):
required_attrs.append(field_spec["name"])
return cls(subattr_schemata, ids, required_attrs, name_mapping)
#if it really doesn't match the previous one
#then take everything before this and send to orphan or testdict
if latestname == "":
#if it's the first in the file
thistestname = name
thistest = [line]
latestname = name
latestamnt = amount
else:
#NORMALLY:
#IF it matches the previous one:
if matches(name, amount, latestname, latestamnt, thistestname,
thistest):
#if they at least match B13_XXXX
print 'its a match'
thistestname = commonprefix([thistestname, name])
thistest.append(line)
else:
#IF they're not a match
if len(thistest) == 0:
print "ERROR THIS SHOULD NEVER HAPPEN"
elif len(thistest) == 1:
#EITHER send the orphan to orphans
orphans.append(thistest)
thistest = [line]
latestname = name
latestamnt = amount
thistestname = name
else:
#OR send the previous good test to testdict
def execute(self, mount_strings):
'''
The execute method runs the generated command line
(from either generateRandomParams or readInput)
If docker is specified, it will attempt to use it, instead
of local execution.
After execution, it checks for output file existence.
'''
command, exit_code, con = self.cmdLine[0], None, self.con or {}
# Check for Container image
conType, conImage = con.get('type'), con.get('image'),
conIndex = con.get("index")
conIsPresent = (conImage is not None)
# Export environment variables, if they are specified in the descriptor
envVars = {}
if 'environment-variables' in list(self.desc_dict.keys()):
variables = [(p['name'], p['value'])
for p in self.desc_dict['environment-variables']]
for (envVarName, envVarValue) in variables:
os.environ[envVarName] = envVarValue
envVars[envVarName] = envVarValue
# Container script constant name
# Note that docker/singularity cannot do a local volume
# mount of files starting with a '.', hence this one does not
millitime = int(time.time() * 1000)
dsname = ('temp-' +
str(random.SystemRandom().randint(0, int(millitime))) + "-" +
str(millitime) + '.localExec.boshjob.sh')
dsname = op.realpath(dsname)
# If container is present, alter the command template accordingly
container_location = ""
container_command = ""
if conIsPresent:
if conType == 'docker':
# Pull the docker image
if self._localExecute("docker pull " + str(conImage))[1]:
container_location = "Local copy"
elif conType == 'singularity':
if not conIndex:
conIndex = "shub://"
elif not conIndex.endswith("://"):
conIndex = conIndex + "://"
conName = conImage.replace("/", "-").replace(":",
"-") + ".simg"
if conName not in os.listdir('./'):
pull_loc = "\"{0}\" {1}{2}".format(conName, conIndex,
conImage)
container_location = ("Pulled from {1} ({0} not found "
"in current"
"working director").format(
conName, pull_loc)
# Pull the singularity image
if self._localExecute("singularity pull --name " +
pull_loc)[1]:
raise ExecutorError("Could not pull Singularity image")
else:
container_location = "Local ({0})".format(conName)
conName = op.abspath(conName)
else:
raise ExecutorError('Unrecognized container'
' type: \"%s\"' % conType)
# Generate command script
uname, uid = pwd.getpwuid(os.getuid())[0], str(os.getuid())
# Adds the user to the container before executing
# the templated command line
userchange = '' if not self.changeUser else ("useradd --uid " +
uid + ' ' + uname +
"\n")
# If --changeUser was desired, run with su so that
# any output files are owned by the user instead of root
# Get the supported shell by the docker or singularity
if self.changeUser:
command = 'su ' + uname + ' -c ' + "\"{0}\"".format(command)
cmdString = "#!" + self.shell + " -l\n" + userchange + str(command)
with open(dsname, "w") as scrFile:
scrFile.write(cmdString)
# Ensure the script is executable
self._localExecute("chmod 755 " + dsname)
# Prepare extra environment variables
envString = ""
if envVars:
for (key, val) in list(envVars.items()):
envString += "SINGULARITYENV_{0}='{1}' ".format(key, val)
# Change launch (working) directory if desired
launchDir = self.launchDir
if launchDir is None:
launchDir = op.realpath('./')
launchDir = op.realpath(launchDir)
# Run it in docker
mount_strings = [] if not mount_strings else mount_strings
mount_strings = [
op.realpath(m.split(":")[0]) + ":" + m.split(":")[1]
for m in mount_strings
]
mount_strings.append(op.realpath('./') + ':' + launchDir)
if conType == 'docker':
envString = " "
if envVars:
for (key, val) in list(envVars.items()):
envString += " -e {0}='{1}' ".format(key, val)
# export mounts to docker string
docker_mounts = " -v ".join(m for m in mount_strings)
container_command = ('docker run --entrypoint=' + self.shell +
' --rm' + envString + ' -v ' +
docker_mounts + ' -w ' + launchDir + ' ' +
str(conImage) + ' ' + dsname)
elif conType == 'singularity':
envString = ""
if envVars:
for (key, val) in list(envVars.items()):
envString += "SINGULARITYENV_{0}='{1}' ".format(
key, val)
# TODO: Singularity 2.4.6 default configuration binds: /proc,
# /sys, /dev, ${HOME}, /tmp, /var/tmp, /etc/localtime, and
# /etc/hosts. This means that any path down-stream shouldn't
# be bound on the command-line, as this will currently raise
# an exception. See:
# https://github.com/singularityware/singularity/issues/1469
#
# Previous bind string:
# singularity_mounts = " -B ".join(m for m in mount_strings)
def_mounts = [
"/proc", "/sys", "/dev", "/tmp", "/var/tmp",
"/etc/localtime", "/etc/hosts",
op.realpath(op.expanduser('~')),
op.expanduser('~')
]
# Ensures the set of paths provided has no overlap
compaths = list()
for idxm, m in enumerate(mount_strings):
for n in mount_strings[idxm:]:
if n != m:
tmp = op.dirname(op.commonprefix([n, m]))
if tmp != '/':
compaths += [tmp]
if not any(m.startswith(c) for c in compaths):
compaths += [m]
mount_strings = set(compaths)
# Only adds mount points for those not already included
singularity_mounts = ""
for m in mount_strings:
if not any(d in m for d in def_mounts):
singularity_mounts += "-B {0} ".format(m)
container_command = (envString + 'singularity exec '
'--cleanenv ' + singularity_mounts +
' -W ' + launchDir + ' ' + str(conName) +
' ' + dsname)
else:
raise ExecutorError('Unrecognized container type: '
'\"%s\"' % conType)
(stdout, stderr), exit_code = self._localExecute(container_command)
# Otherwise, just run command locally
else:
(stdout, stderr), exit_code = self._localExecute(command)
time.sleep(0.5) # Give the OS a (half) second to finish writing
# Destroy temporary docker script, if desired.
# By default, keep the script so the dev can look at it.
if conIsPresent and self.destroyTempScripts:
if os.path.isfile(dsname):
os.remove(dsname)
# Check for output files
missing_files = []
output_files = []
all_files = evaluateEngine(self, "output-files")
required_files = evaluateEngine(self, "output-files/optional=False")
optional_files = evaluateEngine(self, "output-files/optional=True")
for f in all_files.keys():
file_name = all_files[f]
fd = FileDescription(f, file_name, False)
if op.exists(file_name):
output_files.append(fd)
else: # file does not exist
if f in required_files.keys():
missing_files.append(fd)
# Set error messages
desc_err = ''
if 'error-codes' in list(self.desc_dict.keys()):
for err_elem in self.desc_dict['error-codes']:
if err_elem['code'] == exit_code:
desc_err = err_elem['description']
break
return ExecutorOutput(stdout, stderr, exit_code, desc_err,
output_files, missing_files, command,
container_command, container_location)
def printStatus(makeLog, makeAllLog, textTestTmp, smtpServer, out):
failed = ""
build = commonprefix([basename(makeLog), basename(makeAllLog)])
print >> out, build,
print >> out, datetime.now().ctime()
print >> out, "--"
print >> out, basename(makeLog)
warnings = 0
errors = 0
svnLocked = False
for l in file(makeLog):
if ("svn: Working copy" in l and "locked" in l) or "svn: Failed" in l:
svnLocked = True
failed += l
if "warning " in l.lower() or "warnung" in l.lower():
warnings += 1
if "error " in l.lower():
errors += 1
failed += l
if svnLocked:
failed += "svn up failed\n\n"
print >> out, warnings, "warnings"
if errors:
print >> out, errors, "errors"
failed += "make failed\n\n"
print >> out, "--"
for root, dirs, files in os.walk(textTestTmp):
for f in files:
if f.startswith("batchreport"):
b = open(join(root, f))
l = b.readline()
if l.startswith("FAILED") or l.startswith("succeeded"):
print >> out, f, l,
b.close()
print >> out, "--"
print >> out, basename(makeAllLog)
warnings = 0
errors = 0
for l in file(makeAllLog):
if "warning " in l.lower() or "warnung" in l.lower():
warnings += 1
if "error " in l.lower():
errors += 1
failed += l
print >> out, warnings, "warnings"
if errors:
print >> out, errors, "errors"
failed += "make debug failed\n\n"
print >> out, "--"
if failed:
fromAddr = "*****@*****.**"
toAddr = "*****@*****.**"
message = """From: "%s" <%s>
To: %s
Subject: Error occurred while building
%s""" % (build, fromAddr, toAddr, failed)
server = smtplib.SMTP(smtpServer)
server.sendmail(fromAddr, toAddr, message)
server.quit()
def run(self, edit, character='^'):
"""Available parameters:
edit (sublime.Edit)
The edit parameter from TextCommand.
character (str) = '^'
The character to insert when suggesting where the test assertions should go.
"""
view = self.view
view.replace(edit, view.sel()[0], character)
insert_at = view.sel()[0].begin()
listener = sublime_plugin.find_view_event_listener(view, SyntaxTestHighlighterListener)
if not listener.header:
return
lines, line = listener.get_details_of_line_being_tested()
end_token = listener.header.comment_end
# don't duplicate the end token if it is on the line but not selected
if end_token and view.sel()[0].end() == lines[0].line_region.end():
end_token = ' ' + end_token
else:
end_token = ''
scopes = []
length = 0
# find the following columns on the line to be tested where the scopes don't change
test_at_start_of_comment = False
col = view.rowcol(insert_at)[1]
assertion_colrange = lines[0].assertion_colrange or (-1, -1)
if assertion_colrange[0] == assertion_colrange[1]:
col = assertion_colrange[1]
test_at_start_of_comment = True
lines = lines[1:]
col_start, col_end = lines[0].assertion_colrange
base_scope = path.commonprefix([
view.scope_name(pos)
for pos in range(line.begin() + col_start, line.begin() + col_end)
])
for pos in range(line.begin() + col, line.end() + 1):
scope = view.scope_name(pos)
if len(scopes) == 0:
scopes.append(scope)
elif not scope.startswith(base_scope):
break
length += 1
if test_at_start_of_comment:
break
# find the unique scopes at each existing assertion position
if lines and not test_at_start_of_comment:
col_start, col_end = lines[0].assertion_colrange
for pos in range(line.begin() + col_start, line.begin() + col_end):
scope = view.scope_name(pos)
if scope not in scopes:
scopes.append(scope)
suggest_suffix = get_setting('syntax_test.suggest_scope_suffix', True)
scope = find_common_scopes(scopes, not suggest_suffix)
# delete the existing selection
if not view.sel()[0].empty():
view.erase(edit, view.sel()[0])
view.insert(edit, insert_at, (character * max(1, length)) + ' ' + scope + end_token)
# move the selection to cover the added scope name,
# so that the user can easily insert another ^ to extend the test
view.sel().clear()
view.sel().add(sublime.Region(
insert_at + length,
insert_at + length + len(' ' + scope + end_token)
))
def _poll(self, host):
self._reconnect_vcenter_if_necessary(host)
vcenter_options = self.vcenters[host]
values = {'clusters': {}, 'datacenters': {}}
service_instance = vcenter_options['service_instance']
nsx_t_clusters = set()
with filter_spec_context(service_instance,
obj_type=vim.HostSystem,
path_set=['name', 'parent', 'config.network.opaqueSwitch']) as filter_spec:
for h in vcu.collect_properties(service_instance, [filter_spec]):
if 'config.network.opaqueSwitch' not in h:
LOG.debug("Broken ESXi host %s detected in cluster %s",
h['name'], h['parent'])
continue
if len(h['config.network.opaqueSwitch']) > 0:
LOG.debug("(Possible) NSX-T switch found on %s", h['name'])
nsx_t_clusters.add(h['parent'])
with filter_spec_context(service_instance) as filter_spec:
availability_zones = set()
cluster_options = None
for cluster in vcu.collect_properties(service_instance, [filter_spec]):
cluster_name = cluster['name']
match = self.CLUSTER_MATCH.match(cluster_name)
if not match:
LOG.debug(
"%s: Ignoring cluster %s "
"not matching naming scheme", host, cluster_name)
continue
bb_name_no_zeroes = f'bb{match.group(1)}'
nsx_t_enabled = cluster['obj'] in nsx_t_clusters
if nsx_t_enabled:
LOG.debug('NSX-T enabled for %s', cluster_name)
parent = cluster['parent']
availability_zone = parent.parent.name.lower()
availability_zones.add(availability_zone)
cluster_options = self.global_options.copy()
cluster_options.update(vcenter_options)
cluster_options.pop('service_instance', None)
cluster_options.update(name=bb_name_no_zeroes,
cluster_name=cluster_name,
availability_zone=availability_zone,
nsx_t_enabled=nsx_t_enabled,
vcenter_name=vcenter_options['name'])
if cluster_options.get('pbm_enabled', 'false') != 'true':
datastores = cluster['datastore']
datastore_names = [datastore.name
for datastore in datastores
if self.EPH_MATCH.match(datastore.name)]
eph = commonprefix(datastore_names)
cluster_options.update(datastore_regex=f"^{eph}.*")
hagroups = set()
for name in datastore_names:
m = self.HAGROUP_MATCH.match(name)
if not m:
continue
hagroups.add(m.group('hagroup').lower())
if {'a', 'b'}.issubset(hagroups):
LOG.debug('ephemeral datastore hagroups enabled for %s', cluster_name)
cluster_options.update(datastore_hagroup_regex=self.HAGROUP_MATCH.pattern)
for network in cluster['network']:
try:
match = self.BR_MATCH.match(network.name)
if match:
cluster_options['bridge'] = match.group(0).lower()
cluster_options['physical'] = match.group(1).lower()
break
except vim.ManagedObjectNotFound:
# sometimes a portgroup might be already deleted when
# we try to query its name here
continue
if 'bridge' not in cluster_options and not nsx_t_enabled:
LOG.warning("%s: Skipping cluster %s, "
"cannot find bridge matching naming scheme",
host, cluster_name)
continue
values['clusters'][cluster_name] = cluster_options
for availability_zone in availability_zones:
cluster_options = self.global_options.copy()
cluster_options.update(vcenter_options)
cluster_options.pop('service_instance', None)
cluster_options.update(availability_zone=availability_zone)
values['datacenters'][availability_zone] = cluster_options
return values
def test_that_a_valid_license_notice_exists_in_every_source_file_and_that_global_licensing_information_is_correct(
self):
license_notice = compile(
r"""(?P<comment_start>#|--|//) This Source Code Form is subject to the terms of the Mozilla Public
(?P=comment_start) License, v\. 2\.0\. If a copy of the MPL was not distributed with this file,
(?P=comment_start) You can obtain one at http://mozilla\.org/MPL/2\.0/\.
(?P=comment_start)
(?P=comment_start) Copyright \(c\) (?P<first_year>20\d\d)(-(?P<last_year>20\d\d))?, Lars Asplund lars\.anders\.asplund@gmail\.com"""
)
log_date = compile(r'Date:\s*(?P<year>20\d\d)-\d\d-\d\d')
licensed_files = []
repo_root = abspath(join(dirname(__file__), '..'))
for root, dirs, files in walk(repo_root):
for f in files:
if 'preprocessed' in root:
continue
osvvm_directory = abspath(join(repo_root, 'vhdl', 'osvvm'))
if commonprefix([osvvm_directory,
abspath(join(root, f))]) == osvvm_directory:
continue
osvvm_integration_example_directory = abspath(
join(repo_root, 'examples', 'osvvm_integration', 'src'))
if commonprefix([
osvvm_integration_example_directory,
abspath(join(root, f))
]) == osvvm_integration_example_directory:
continue
if splitext(f)[1] in ['.vhd', '.vhdl', '.py', '.v', '.sv']:
licensed_files.append(join(root, f))
i = 0
min_first_year = None
max_last_year = None
for f in licensed_files:
stdout.write('\r%d/%d' % (i + 1, len(licensed_files)))
stdout.flush()
i += 1
proc = Popen(['git', 'log', '--follow', '--date=short', f], \
bufsize=0, stdout=PIPE, stdin=PIPE, stderr=STDOUT, universal_newlines=True)
out, _ = proc.communicate()
first_year = None
last_year = None
for date in log_date.finditer(out):
first_year = int(
date.group('year')) if first_year is None else min(
int(date.group('year')), first_year)
last_year = int(
date.group('year')) if last_year is None else max(
int(date.group('year')), last_year)
min_first_year = first_year if min_first_year is None else min(
min_first_year, first_year)
max_last_year = last_year if max_last_year is None else max(
max_last_year, last_year)
with open(f) as fp:
code = fp.read()
match = license_notice.search(code)
self.assertIsNotNone(match,
"Failed to find license notice in %s" % f)
if first_year == last_year:
self.assertEqual(
int(match.group('first_year')), first_year,
'Expected copyright year to be %d in %s' %
(first_year, f))
self.assertIsNone(
match.group('last_year'),
'Expected no copyright years range in %s' %
join(root, f))
else:
self.assertIsNotNone(
match.group('last_year'),
'Expected copyright year range %d-%d in %s' %
(first_year, last_year, f))
self.assertEqual(
int(match.group('first_year')), first_year,
'Expected copyright year range to start with %d in %s'
% (first_year, f))
self.assertEqual(
int(match.group('last_year')), last_year,
'Expected copyright year range to end with %d in %s' %
(last_year, f))
print('\n')
def commonprefix(args, sep='/'):
return path.commonprefix(args).rpartition(sep)[0]
def tabularize_dictionary(load_path, save_path, max_defs=0):
with open(load_path) as f:
dictionary = json.load(f)
lmt = Lemmatizer(LEMMA_INDEX, LEMMA_EXC, LEMMA_RULES)
words = sorted(list(dictionary.keys()))
with open(save_path, 'w') as f:
for word in words:
if not dictionary[word]:
continue
if max_defs == 0:
n = len(dictionary[word])
else:
n = max_defs
pos2defs = defaultdict(list)
for pos, definition in dictionary[word]:
if pos is not None:
pos = pos.lower()
if pos == 'idioms':
continue
pos = POS_MAP.get(pos)
pos2defs[pos].append(definition)
tag = None
for p in ['VERB', 'ADJ']:
lemma = lmt(word, p)[0]
if lemma != word and\
dictionary.get(lemma) != dictionary.get(word):
prefix = commonprefix([lemma, word])
suffix = word[len(prefix):]
if p == 'VERB':
if suffix.endswith('ing'):
tag = 'VBG'
elif suffix.endswith('s'):
tag = 'VBZ'
elif suffix.endswith('d'):
tag = 'VBD'
elif suffix.endswith('n'):
tag = 'VBN'
elif p == 'NOUN':
tag = 'NNS'
elif p == 'ADJ':
if suffix.endswith('t'):
tag = 'JJS'
elif suffix.endswith('r'):
tag = 'JJR'
if dictionary.get(lemma):
for pos, definition in dictionary[lemma]:
if pos is not None:
pos = pos.lower()
if pos == 'idioms':
continue
pos = POS_MAP.get(pos)
if len(definition) == 0:
continue
if pos == p:
f.write("%s\t%s\t%s\t%s\n" % (word, pos, tag, definition))
for pos in pos2defs:
for definition in pos2defs[pos][:n]:
if len(definition) == 0:
continue
tag = None
skip = 0
for p in ['VERB', 'NOUN', 'ADJ']:
lemma = lmt(word, p)[0]
if lemma != word and\
dictionary.get(lemma) == dictionary.get(word):
if p == pos:
prefix = commonprefix([lemma, word])
suffix = word[len(prefix):]
if pos == 'VERB':
if suffix.endswith('ing'):
tag = 'VBG'
elif suffix.endswith('s'):
tag = 'VBZ'
elif suffix.endswith('d'):
tag = 'VBD'
elif suffix.endswith('n'):
tag = 'VBN'
elif pos == 'NOUN':
tag = 'NNS'
elif pos == 'ADJ':
if suffix.endswith('t'):
tag = 'JJS'
elif suffix.endswith('r'):
tag = 'JJR'
if skip == 1:
f.write("%s\t%s\t%s\t%s\n" % (word, pos, tag, definition))
break
else:
skip = 1
if not skip:
f.write("%s\t%s\t%s\t%s\n" % (word, pos, tag, definition))
def __call__(archive,
annex=None,
add_archive_leading_dir=False,
strip_leading_dirs=False,
leading_dirs_depth=None,
leading_dirs_consider=None,
use_current_dir=False,
delete=False,
key=False,
exclude=None,
rename=None,
existing='fail',
annex_options=None,
copy=False,
commit=True,
allow_dirty=False,
stats=None,
drop_after=False,
delete_after=False):
"""
Returns
-------
annex
"""
if exclude:
exclude = assure_tuple_or_list(exclude)
if rename:
rename = assure_tuple_or_list(rename)
# TODO: actually I see possibly us asking user either he wants to convert
# his git repo into annex
archive_path = archive
pwd = getpwd()
if annex is None:
annex = get_repo_instance(pwd, class_=AnnexRepo)
if not isabs(archive):
# if not absolute -- relative to wd and thus
archive_path = normpath(opj(pwd, archive))
# abspath(archive) is not "good" since dereferences links in the path
# archive_path = abspath(archive)
elif not isabs(archive):
# if we are given an annex, then assume that given path is within annex, not
# relative to PWD
archive_path = opj(annex.path, archive)
annex_path = annex.path
# _rpath below should depict paths relative to the top of the annex
archive_rpath = relpath(archive_path, annex_path)
# TODO: somewhat too cruel -- may be an option or smth...
if not allow_dirty and annex.dirty:
# already saved me once ;)
raise RuntimeError(
"You better commit all the changes and untracked files first")
if not key:
# we were given a file which must exist
if not exists(archive_path):
raise ValueError("Archive {} does not exist".format(archive))
# TODO: support adding archives content from outside the annex/repo
origin = 'archive'
key = annex.get_file_key(archive_rpath)
archive_dir = dirname(archive_path)
else:
origin = 'key'
key = archive
archive_dir = None # We must not have anything to do with the location under .git/annex
archive_basename = file_basename(archive)
if not key:
# TODO: allow for it to be under git??? how to reference then?
raise NotImplementedError(
"Provided file %s is not under annex. We don't support yet adding everything "
"straight to git" % archive)
# are we in a subdirectory of the repository?
pwd_under_annex = commonprefix([pwd, annex_path]) == annex_path
# then we should add content under that
# subdirectory,
# get the path relative to the repo top
if use_current_dir:
# if outside -- extract to the top of repo
extract_rpath = relpath(pwd, annex_path) \
if pwd_under_annex \
else None
else:
extract_rpath = relpath(archive_dir, annex_path)
# relpath might return '.' as the relative path to curdir, which then normalize_paths
# would take as instructions to really go from cwd, so we need to sanitize
if extract_rpath == curdir:
extract_rpath = None # no special relpath from top of the repo
# and operate from now on the key or whereever content available "canonically"
try:
key_rpath = annex.get_contentlocation(
key) # , relative_to_top=True)
except:
raise RuntimeError(
"Content of %s seems to be N/A. Fetch it first" % key)
# now we simply need to go through every file in that archive and
lgr.info("Adding content of the archive %s into annex %s", archive,
annex)
from datalad.customremotes.archives import ArchiveAnnexCustomRemote
# TODO: shouldn't we be able just to pass existing AnnexRepo instance?
# TODO: we will use persistent cache so we could just (ab)use possibly extracted archive
annexarchive = ArchiveAnnexCustomRemote(path=annex_path,
persistent_cache=True)
# We will move extracted content so it must not exist prior running
annexarchive.cache.allow_existing = True
earchive = annexarchive.cache[key_rpath]
# TODO: check if may be it was already added
if ARCHIVES_SPECIAL_REMOTE not in annex.get_remotes():
lgr.debug(
"Adding new special remote {}".format(ARCHIVES_SPECIAL_REMOTE))
annex.init_remote(ARCHIVES_SPECIAL_REMOTE, [
'encryption=none', 'type=external',
'externaltype=%s' % ARCHIVES_SPECIAL_REMOTE, 'autoenable=true'
])
else:
lgr.debug("Special remote {} already exists".format(
ARCHIVES_SPECIAL_REMOTE))
try:
old_always_commit = annex.always_commit
annex.always_commit = False
if annex_options:
if isinstance(annex_options, string_types):
annex_options = shlex.split(annex_options)
leading_dir = earchive.get_leading_directory(
depth=leading_dirs_depth, exclude=exclude, consider=leading_dirs_consider) \
if strip_leading_dirs else None
leading_dir_len = len(leading_dir) + len(
opsep) if leading_dir else 0
# we need to create a temporary directory at the top level which would later be
# removed
prefix_dir = basename(tempfile.mkdtemp(prefix=".datalad", dir=annex_path)) \
if delete_after \
else None
# dedicated stats which would be added to passed in (if any)
outside_stats = stats
stats = ActivityStats()
for extracted_file in earchive.get_extracted_files():
stats.files += 1
extracted_path = opj(earchive.path, extracted_file)
if islink(extracted_path):
link_path = realpath(extracted_path)
if not exists(
link_path
): # TODO: config addarchive.symlink-broken='skip'
lgr.warning("Path %s points to non-existing file %s" %
(extracted_path, link_path))
stats.skipped += 1
continue
# TODO: check if points outside of the archive -- warning and skip
# preliminary target name which might get modified by renames
target_file_orig = target_file = extracted_file
# strip leading dirs
target_file = target_file[leading_dir_len:]
if add_archive_leading_dir:
target_file = opj(archive_basename, target_file)
if rename:
target_file = apply_replacement_rules(rename, target_file)
# continue to next iteration if extracted_file in excluded
if exclude:
try: # since we need to skip outside loop from inside loop
for regexp in exclude:
if re.search(regexp, extracted_file):
lgr.debug(
"Skipping {extracted_file} since contains {regexp} pattern"
.format(**locals()))
stats.skipped += 1
raise StopIteration
except StopIteration:
continue
if prefix_dir:
target_file = opj(prefix_dir, target_file)
url = annexarchive.get_file_url(
archive_key=key,
file=extracted_file,
size=os.stat(extracted_path).st_size)
# lgr.debug("mv {extracted_path} {target_file}. URL: {url}".format(**locals()))
if lexists(target_file):
if md5sum(target_file) == md5sum(extracted_path):
# must be having the same content, we should just add possibly a new extra URL
pass
elif existing == 'fail':
raise RuntimeError(
"File {} already exists, but new (?) file {} was instructed "
"to be placed there while overwrite=False".format(
target_file, extracted_file))
elif existing == 'overwrite':
stats.overwritten += 1
# to make sure it doesn't conflict -- might have been a tree
rmtree(target_file)
else:
target_file_orig_ = target_file
# To keep extension intact -- operate on the base of the filename
p, fn = os.path.split(target_file)
ends_with_dot = fn.endswith('.')
fn_base, fn_ext = file_basename(fn, return_ext=True)
if existing == 'archive-suffix':
fn_base += '-%s' % archive_basename
elif existing == 'numeric-suffix':
pass # archive-suffix will have the same logic
else:
raise ValueError(existing)
# keep incrementing index in the suffix until file doesn't collide
suf, i = '', 0
while True:
target_file_new = opj(
p, fn_base + suf +
('.' if
(fn_ext or ends_with_dot) else '') + fn_ext)
if not lexists(target_file_new):
break
lgr.debug("File %s already exists" %
target_file_new)
i += 1
suf = '.%d' % i
target_file = target_file_new
lgr.debug("Original file %s will be saved into %s" %
(target_file_orig_, target_file))
# TODO: should we reserve smth like
# stats.clobbed += 1
if target_file != target_file_orig:
stats.renamed += 1
#target_path = opj(getpwd(), target_file)
if copy:
raise NotImplementedError(
"Not yet copying from 'persistent' cache")
else:
# os.renames(extracted_path, target_path)
# addurl implementation relying on annex'es addurl below would actually copy
pass
lgr.debug(
"Adding %s to annex pointing to %s and with options %r",
target_file, url, annex_options)
target_file_rpath = opj(
extract_rpath,
target_file) if extract_rpath else target_file
out_json = annex.add_url_to_file(target_file_rpath,
url,
options=annex_options,
batch=True)
if 'key' in out_json and out_json[
'key'] is not None: # annex.is_under_annex(target_file, batch=True):
# due to http://git-annex.branchable.com/bugs/annex_drop_is_not___34__in_effect__34___for_load_which_was___34__addurl_--batch__34__ed_but_not_yet_committed/?updated
# we need to maintain a list of those to be dropped files
if drop_after:
annex.drop_key(out_json['key'], batch=True)
stats.dropped += 1
stats.add_annex += 1
else:
lgr.debug(
"File {} was added to git, not adding url".format(
target_file))
stats.add_git += 1
if delete_after:
# forcing since it is only staged, not yet committed
annex.remove(target_file_rpath, force=True) # TODO: batch!
stats.removed += 1
# # chaining 3 annex commands, 2 of which not batched -- less efficient but more bullet proof etc
# annex.add(target_path, options=annex_options)
# # above action might add to git or to annex
# if annex.file_has_content(target_path):
# # if not -- it was added to git, if in annex, it is present and output is True
# annex.add_url_to_file(target_file, url, options=['--relaxed'], batch=True)
# stats.add_annex += 1
# else:
# lgr.debug("File {} was added to git, not adding url".format(target_file))
# stats.add_git += 1
# # TODO: actually check if it is anyhow different from a previous version. If not
# # then it wasn't really added
del target_file # Done with target_file -- just to have clear end of the loop
if delete and archive and origin != 'key':
lgr.debug("Removing the original archive {}".format(archive))
# force=True since some times might still be staged and fail
annex.remove(archive_rpath, force=True)
lgr.info("Finished adding %s: %s" %
(archive, stats.as_str(mode='line')))
if outside_stats:
outside_stats += stats
if commit:
commit_stats = outside_stats if outside_stats else stats
annex.commit(
"Added content extracted from %s %s\n\n%s" %
(origin, archive, commit_stats.as_str(mode='full')),
_datalad_msg=True)
commit_stats.reset()
finally:
# since we batched addurl, we should close those batched processes
annex.precommit()
if delete_after:
prefix_path = opj(annex_path, prefix_dir)
if exists(prefix_path): # probably would always be there
lgr.info(
"Removing temporary directory under which extracted files were annexed: %s",
prefix_path)
rmtree(prefix_path)
annex.always_commit = old_always_commit
# remove what is left and/or everything upon failure
earchive.clean(force=True)
return annex
def svg_heatmap(data,
filename,
row_labels=None,
box_size=4,
index=None,
all_indices=None,
all_colnames=None,
internal_datanames=None,
cmap=ISH,
norm_rows_by=None,
draw_row_labels=False,
color_row_labels=False,
col_sep='',
box_height=None,
total_width=None,
draw_box=False,
draw_name=False,
data_names=None,
make_hyperlinks=False,
progress_bar=False,
max_width=np.inf,
x_min=10,
y_min=10,
spacers=None,
convert=False,
squeeze_rows=None,
cmap_by_prefix=None,
draw_average=False,
draw_average_only=False,
average_scale=1,
split_columns=False,
vspacer=30,
hatch_nan=True,
hatch_size=20,
figure_title=None,
nan_replace=None,
first_col='',
last_col=''):
"""
Draw heatmap as an SVG file stored in filename
*data* can be either a 2D array-like type (list of lists, numpy array,
pandas DataFrame, etc), or a tuple of 2D array-likes, in which case a
separator will be added between each one in the output
*cmap* is a matplotlib-like colormap (i.e. a callable that expects floats
in the range 0.0-1.0.), or an iterable of the same length as the tuple
*data* containing colormaps
*row_labels* can be supplied, otherwise they will detected from the first
item in *data*, if available, and if not they will be blank.
If *total_width* is supplied, width of each dataset in *data* will be
scaled to that constant. If *box_height* is supplied, the height of each
row will be *box_height*, otherwise it will be equal to the width of each
element. If neither are supplied, elements will be squares equal to
*box_size*. IT IS STRONGLY RECOMMENDED that if if supplying *total_width*,
*box_height* also be specified, but this is not enforced.
*draw_row_labels*, if True, will label the rows on the right hand side. As
of 2013/09/03, this won't scale the SVG properly, so including the
resulting file in an html element won't display properly.
*spacers* is the distance between adjacent datasets. Can either be a
number, in which case it will apply to all datasets, or an interable for
different distances. If the iterable is shorter than the number of
datasets, the last value will be repeated.
"""
import svgwrite as svg
try:
import pandas as pd
has_pandas = True
except:
has_pandas = False
assert all_indices
assert all_colnames
if not isinstance(data, tuple):
data = (data, )
if not isinstance(norm_rows_by, tuple):
norm_rows_by = repeat(norm_rows_by)
old_data = data
colname_tuple = repeat(None)
if split_columns and has_pandas:
from Utils import sel_startswith
data = []
new_normers = []
new_cmaps = []
if isinstance(cmap, tuple):
cmaps = cmap
else:
cmaps = repeat(cmap)
for dataset, normer, c_cmap in zip(old_data, norm_rows_by, cmaps):
if dataset is None:
data.append(dataset)
new_normers.append(normer)
new_cmaps.append(c_cmap)
continue
if not isinstance(dataset, pd.DataFrame):
dataset = pd.DataFrame(dataset).T
colnames = list(
sorted({col.split(col_sep)[0]
for col in dataset.columns}))
data.extend(
dataset.select(**sel_startswith(colname))
for colname in colnames)
new_normers.extend(normer for colname in colnames)
new_cmaps.extend(c_cmap for colname in colnames)
data = tuple(data)
norm_rows_by = tuple(new_normers)
cmap = tuple(new_cmaps)
elif split_columns and all_colnames:
colnames = list(sorted({col.split(col_sep)[0]
for col in all_colnames}))
colname = colnames[0]
data = tuple([
data[:,
array([c.startswith(colname) for c in internal_datanames])]
for colname in colnames
])
colname_tuple = tuple(
[c for c in all_colnames if c.startswith(colname)]
for colname in colnames)
elif not split_columns and all_colnames:
colname_tuple = tuple(
[c for c in all_colnames if c.startswith(dataname)]
for dataname in internal_datanames)
rows, cols = np.shape([ds for ds in data if ds is not None][0])
if index is not None:
rows = len(index)
if box_height is None:
box_height = box_size
if row_labels is None:
if index is not None:
row_labels = list(index)
elif hasattr(data[0], 'index'):
row_labels = list(data[0].index)
else:
row_labels = ['' for row in range(rows)]
if total_width is not None and max_width is not np.inf:
boxes_per_row = max_width // (1.1 * total_width)
if ((boxes_per_row + 1) * 1.1 * total_width - .1 * total_width <
max_width):
boxes_per_row += 1
num_plotted_rows = np.ceil(
len(data) / boxes_per_row + (draw_average or draw_average_only))
if figure_title is None:
fig_title_height = 0
elif isinstance(figure_title, tuple):
fig_title_height = len(figure_title)
else:
fig_title_height = 1
dwg = svg.Drawing(
filename,
size=(max_width + 2 * x_min + 200 * draw_row_labels,
2 * y_min + (num_plotted_rows *
(rows) * box_height) + 80 * (fig_title_height) +
80 * draw_name + (num_plotted_rows - 1) * vspacer),
)
elif total_width is not None:
width = len(data) * total_width * 1.1 - .1 * total_width
height = rows * box_height
max_row_label_len = max(len(str(i)) for i in row_labels)
dwg = svg.Drawing(
filename,
size=(width + 2 * x_min + 20 * draw_row_labels * max_row_label_len,
height + 2 * y_min + 80 * draw_name +
(80 * (figure_title is not None))))
else:
dwg = svg.Drawing(filename)
dwg.add(svg.base.Title(path.basename(filename)))
pat = dwg.pattern(id='hatch',
insert=(0, 0),
size=(hatch_size, hatch_size),
patternUnits='userSpaceOnUse')
g = pat.add(dwg.g(style="fill:none; stroke:#B0B0B0; stroke-width:1"))
g.add(dwg.path(('M0,0', 'l{hatch},{hatch}'.format(hatch=hatch_size))))
g.add(
dwg.path(('M{hatch2},0 l{hatch2},{hatch2}'.format(hatch2=hatch_size /
2).split())))
g.add(
dwg.path(('M0,{hatch2} l{hatch2},{hatch2}'.format(hatch2=hatch_size /
2).split())))
dwg.add(pat)
if box_height is None:
box_height = box_size
if not hasattr(cmap, "__len__"):
cmap = [cmap for frame in data]
if data_names is None:
data_names = ["" for frame in data]
if len(cmap) != len(data):
raise ValueError(
"cmap and data should be the same length ({} vs {})".format(
len(cmap), len(data)))
if not hasattr(spacers, "__len__"):
spacers = [spacers]
else:
spacers = list(spacers)
while len(spacers) < len(data):
spacers.append(spacers[-1])
if ((isinstance(norm_rows_by, repeat)
and isinstance(next(norm_rows_by), str)
and next(norm_rows_by).startswith('center0all'))
or (not isinstance(norm_rows_by, repeat)
and isinstance(norm_rows_by[0], str)
and np.any([i.startswith('center0all')
for i in norm_rows_by]))):
all_data = pd.concat(data, axis=1)
if squeeze_rows is not None:
data = [
pd.DataFrame(d.apply(squeeze_rows, axis=1),
columns=[path.commonprefix(list(d.columns))])
for d in data
]
x_start = x_min
y_start = y_min
y_diff = 0
iterator = zip(data, cmap, data_names, norm_rows_by, spacers,
colname_tuple)
if figure_title:
if isinstance(figure_title, tuple):
font_size = '3em'
for title_line in figure_title:
dwg.add(
dwg.text(
title_line, (
x_start,
y_start + 75,
),
style="font-size:{};font-family:sans-serif".format(
font_size)))
y_start += 80
font_size = '1.5em'
else:
dwg.add(
dwg.text(figure_title, (
x_start,
y_start + 75,
),
style="font-size:3em;font-family:sans-serif"))
y_start += 80
if progress_bar:
from progressbar import ProgressBar
pbar = ProgressBar(maxval=len(data) * rows).start()
pbar_val = 0
for frame, c_cmap, name, normer, spacer, colnames in iterator:
if frame is None:
dwg.add(dwg.text(normer, (x_start, y_start + box_height / 2)))
if total_width is not None:
if spacer is None:
x_start += total_width * 1.1
else:
x_start += total_width + spacer
else:
if spacer is None:
x_start += box_size
else:
x_start += spacer
if x_start > max_width:
x_start = x_min
y_start += box_height + vspacer
continue
if has_pandas:
frame = pd.DataFrame(frame)
if index is not None:
if has_pandas:
frame = frame.ix[index]
else:
setix = set(index)
#selector = [i for i, name in enumerate(all_indices) if name in setix]
#frame = frame[selector, :]
if normer is None:
norm_data = array(frame.copy())
elif normer is 'mean':
if has_pandas:
norm_data = array(
frame.divide(frame.dropna(axis=1, how='all').mean(axis=1) +
10,
axis=0))
else:
norm_data = frame / (
frame[:, isfinite(frame[0, :])].mean(axis=1) + 10).reshape(
(rows, 1))
elif normer == 'max':
if has_pandas:
norm_data = array(
frame.divide(frame.dropna(axis=1, how='all').max(axis=1) +
10,
axis=0))
else:
norm_data = frame / (
frame[:, isfinite(frame[0, :])].max(axis=1) + 10).reshape(
(rows, 1))
elif normer == 'maxall':
if has_pandas:
maxall = frame.max(axis=1)
assert len(data) == len(new_normers)
for old_frame, norm_type in zip(data, new_normers):
if norm_type != 'maxall': continue
if old_frame is not None:
old_frame = old_frame.max(
axis=1
).ix[index if index is not None else old_frame.index]
maxall = maxall.where(maxall > old_frame, old_frame)
norm_data = array(frame.divide(maxall + 10, axis=0))
else:
norm_data = frame / (old_data[:, isfinite(old_data[0, :])].max(
axis=1) + 10).reshape((rows, 1))
elif normer == 'fullvar':
norm_data = frame.subtract(
frame.dropna(axis=1, how='all').min(axis=1) - 1e-6, axis=0)
norm_data = array(
norm_data.divide(norm_data.dropna(axis=1,
how='all').max(axis=1),
axis=0))
elif normer == 'center0':
norm_data = array(
0.5 + 0.5 *
frame.divide(frame.dropna(axis=1).abs().max(axis=1), axis=0))
elif isinstance(normer, str) and normer.startswith('center0min'):
min_norm = (frame.dropna(axis=1).abs().max(axis=1).clip(
float(normer[10:]), 1e6))
norm_data = array(0.5 + 0.5 * frame.divide(min_norm, axis=0))
elif isinstance(normer, str) and normer.startswith('center0allmin'):
min_norm = (all_data.dropna(axis=1).abs().max(axis=1).clip(
float(normer[13:]), 1e6))
norm_data = array(0.5 + 0.5 * frame.divide(min_norm, axis=0))
elif normer == 'center0all':
norm_data = array(0.5 + 0.5 * frame.divide(
all_data.dropna(how='all', axis=1).abs().max(axis=1), axis=0))
elif normer == 'center0pre':
norm_data = array(0.5 + 0.5 * frame)
elif isinstance(normer, (int, float)):
norm_data = array(frame / normer)
normer = 'constant'
elif index is not None and hasattr(normer, "ix"):
norm_data = array(frame.divide(normer.ix[index], axis=0))
elif hasattr(normer, "__len__") and len(normer) == rows:
if has_pandas:
norm_data = array(frame.divide(normer, axis=0))
else:
norm_data = array(frame / np.reshape(normer, (rows, -1)))
elif hasattr(normer, "__len__"):
print('\n' * 5)
print(len(normer), normer, normer == 'max')
print(frame.shape)
raise TypeError("norm_rows_by should be the same shape "
"as the number of rows")
else:
norm_data = array(frame / normer)
if not c_cmap or str(c_cmap).lower() == 'default':
c_cmap = ISH
new_rows, new_cols = np.shape(frame)
if hasattr(frame, 'index'):
col_labels = frame.columns
elif colnames:
col_labels = colnames
else:
col_labels = ['' for col in range(new_cols)]
if new_rows != rows:
raise ValueError("All input elements must have the same number of"
" rows (and same row meanings --unchecked)")
if total_width is not None:
box_size = total_width / float(new_cols)
i = 0
if not draw_average_only:
for i in range(rows):
if progress_bar:
pbar.update(pbar_val)
pbar_val += 1
prefix = col_labels[0][:col_labels[0].find(col_sep)]
if cmap_by_prefix:
c_cmap = cmap_by_prefix(prefix)
for j in range(new_cols):
g = dwg.g()
val = frame.ix[i, j] if has_pandas else frame[i, j]
g.add(
svg.base.Title("{}, {}: {:.2f}".format(
row_labels[i], col_labels[j], val)))
hatch = not isfinite(norm_data[i, j])
if hatch and nan_replace is not None:
if isinstance(nan_replace, float):
norm_data[i, j] = nan_replace
else:
if normer.startswith('center0'):
norm_data[i, j] = 0.5
else:
norm_data[i, j] = 0.0
elif hatch:
n = 0
norm_data[i, j] = 0
left = j - 1
while left >= 0:
if isfinite(norm_data[i, left]):
norm_data[i, j] += norm_data[i, left]
n += 1
break
left -= 1
right = j + 1
while right < norm_data.shape[1]:
if isfinite(norm_data[i, right]):
norm_data[i, j] += norm_data[i, right]
n += 1
break
right += 1
if n == 0:
norm_data[i, j] = .5 if 'center' in normer else 0
else:
norm_data[i, j] /= n
g.add(
dwg.rect(
(x_start + box_size * j, y_start + i * box_height),
(box_size, box_height),
style="fill:#{:02x}{:02x}{:02x}".format(*[
int(255 * x) for x in c_cmap(norm_data[i, j])
])))
dwg.add(g)
if hatch_nan and hatch:
g.add(
dwg.rect((x_start + box_size * j,
y_start + i * box_height),
(box_size, box_height),
style="fill:url(#hatch)"))
col_base = col_labels[j][:col_labels[j].find(col_sep)]
if col_base != prefix:
prefix = col_base
if cmap_by_prefix:
c_cmap = cmap_by_prefix(prefix)
g.add(
dwg.line(
(x_start + box_size * j,
y_start + i * box_height),
(x_start + box_size * j, y_start +
(i + 1) * box_height),
style="stroke-width:{}; stroke:#000000".format(
.1 * box_size)))
else:
for j in range(new_cols):
hatch = not isfinite(norm_data[0, j])
if hatch:
n = 0
norm_data[:, j] = 0
if j > 0 and isfinite(norm_data[0, j - 1]):
norm_data[:, j] += norm_data[:, j - 1]
n += 1
if (j + 1 < norm_data.shape[1]
and isfinite(norm_data[0, j + 1])):
norm_data[:, j] += norm_data[:, j + 1]
n += 1
norm_data[:, j] /= n
dwg.add(dwg.text(first_col, (x_start, y_start + (i + 1) * box_height)))
dwg.add(
dwg.text(last_col, (x_start + (new_cols - 1) * box_size, y_start +
(i + 1) * box_height)))
if draw_box and not draw_average_only:
dwg.add(
dwg.rect((x_start, y_start + 0),
(new_cols * box_size, rows * box_height),
style="stroke-width:1; "
"stroke:#000000; fill:none"))
if draw_average or draw_average_only:
avg_frame = norm_data.mean(axis=0)
for j in range(new_cols):
col_base = col_labels[j][:col_labels[j].find(col_sep)]
prefix = col_base
if cmap_by_prefix:
c_cmap = cmap_by_prefix(prefix)
g = dwg.g()
g.add(
svg.base.Title("Average, {}: {:.2f}".format(
col_labels[j], avg_frame[j])))
g.add(
dwg.rect((x_start + box_size * j, y_start +
(i + (not draw_average_only)) * box_height),
(box_size, box_height),
style="fill:#{:02x}{:02x}{:02x}".format(*[
int(255 * x)
for x in c_cmap(average_scale * avg_frame[j])
])))
if not isfinite(norm_data[0, j]) and hatch_nan:
g.add(
dwg.rect((x_start + box_size * j, y_start +
(i + (not draw_average_only)) * box_height),
(box_size, box_height),
style="fill:url(#hatch)"))
dwg.add(g)
dwg.add(
dwg.rect((x_start, y_start +
(i + (not draw_average_only)) * box_height),
(new_cols * box_size, 1 * box_height),
style="stroke-width:1; stroke:#000000; fill:none"))
if draw_name:
if name == "" and split_columns:
name = col_base
xpos = x_start + box_size * new_cols / 2.0
text = dwg.text(
'', (xpos, y_start + box_height * (rows) *
(1 - draw_average_only) + box_height *
(draw_average or draw_average_only) + 13),
style="text-anchor: middle;font-family:sans-serif;")
text.add(dwg.tspan("", dy=["-1.5em"]))
for line in name.split('_'):
text.add(
dwg.tspan(
line,
dy=["1.5em"],
x=[xpos],
style="text-anchor: middle;",
))
dwg.add(text)
if total_width is not None:
if spacer is None:
x_start += total_width * 1.1
else:
x_start += total_width + spacer
else:
if spacer is None:
x_start += new_cols * box_size + box_size
else:
x_start += new_cols * box_size + spacer
#y_diff = new_rows * box_height + vspacer
if x_start + total_width >= max_width:
x_start = x_min
y_start += new_rows * box_height * (
not draw_average_only) + vspacer
y_start += box_height * (draw_average_only or draw_average)
if draw_row_labels and isinstance(row_labels[0], tuple):
lwidths = Counter()
for r in row_labels:
for i, l in enumerate(r):
lwidths[i] = max(lwidths[i], len(str(l)))
cum_len = 0
for i in range(len(lwidths)):
old_width = lwidths[i]
lwidths[i] += cum_len
cum_len += old_width
if draw_row_labels and not draw_average_only:
for i in range(rows):
if color_row_labels:
style = "font-family:sans-serif; font-size: {size}; fill: {color};".format(
size=box_height,
color='red'
if row_labels[i] in color_row_labels else 'black',
)
else:
style = "font-family:sans-serif; font-size: {}".format(
box_height)
if isinstance(row_labels[i], tuple):
labeltext = dwg.g()
for lnum, ltext in enumerate(row_labels[i]):
labeltext.add(
dwg.text(
ltext,
(x_start + lwidths[lnum - 1] * 10 + lnum * 50,
y_start + i * box_height + box_height),
style=style,
))
else:
labeltext = (dwg.text(
row_labels[i],
(x_start, y_start + i * box_height + box_height),
style=style,
))
if make_hyperlinks:
if make_hyperlinks is True:
link = dwg.a(
'http://insitu.fruitfly.org/cgi-bin/ex/report.pl?ftype={}&ftext={}'
.format(
2 if (isinstance(row_labels[i], str) and
(row_labels[i].startswith('FBgn'))) else 1,
row_labels[i]),
target='_replace',
)
else:
link = dwg.a(make_hyperlinks.format(frame.index[i]))
link.add(labeltext)
dwg.add(link)
else:
dwg.add(labeltext)
if progress_bar:
pbar.finish()
dwg.saveas(filename)
if convert:
cmd = [
'convert',
filename,
'-units',
'PixelsPerInch',
'+antialias',
'-density',
'600',
'-background',
'none',
'-transparent',
'white',
filename.replace('svg', 'png'),
]
subprocess.Popen(cmd)
def is_prefix_of(prefix, of_path):
"""
Return True if 'prefix' is a prefix of 'of_path'
"""
return commonprefix([prefix, of_path]) == prefix
def pairspf(pp):
pf = op.basename(op.commonprefix(pp).rstrip("._-"))
if not pf.strip():
pf = op.basename(pp[0])
return pf
def refresh(self):
"""
Refreshing search results panel
"""
title = "'%s' - " % self.search_text
if self.results is None:
text = _('Search canceled')
else:
nb_files = len(self.results)
if nb_files == 0:
text = _('String not found')
else:
text_matches = _('matches in')
text_files = _('file')
if nb_files > 1:
text_files += 's'
text = "%d %s %d %s" % (self.nb, text_matches, nb_files,
text_files)
if self.error_flag:
text += ' (' + self.error_flag + ')'
elif self.results is not None and not self.completed:
text += ' (' + _('interrupted') + ')'
self.set_title(title + text)
self.clear()
self.data = {}
if not self.results: # First search interrupted *or* No result
return
# Directory set
dir_set = set()
for filename in sorted(self.results.keys()):
dirname = osp.abspath(osp.dirname(filename))
dir_set.add(dirname)
# Root path
root_path_list = None
_common = get_common_path(list(dir_set))
if _common is not None:
root_path_list = [_common]
else:
_common = get_common_path(self.pathlist)
if _common is not None:
root_path_list = [_common]
else:
root_path_list = self.pathlist
if not root_path_list:
return
for _root_path in root_path_list:
dir_set.add(_root_path)
# Populating tree: directories
def create_dir_item(dirname, parent):
if dirname not in root_path_list:
displayed_name = osp.basename(dirname)
else:
displayed_name = dirname
item = QTreeWidgetItem(parent, [displayed_name],
QTreeWidgetItem.Type)
item.setIcon(0, get_std_icon('DirClosedIcon'))
return item
dirs = {}
for dirname in sorted(list(dir_set)):
if dirname in root_path_list:
parent = self
else:
parent_dirname = abspardir(dirname)
parent = dirs.get(parent_dirname)
if parent is None:
# This is related to directories which contain found
# results only in some of their children directories
if osp.commonprefix([dirname] + root_path_list):
# create new root path
pass
items_to_create = []
while dirs.get(parent_dirname) is None:
items_to_create.append(parent_dirname)
parent_dirname = abspardir(parent_dirname)
items_to_create.reverse()
for item_dir in items_to_create:
item_parent = dirs[abspardir(item_dir)]
dirs[item_dir] = create_dir_item(item_dir, item_parent)
parent_dirname = abspardir(dirname)
parent = dirs[parent_dirname]
dirs[dirname] = create_dir_item(dirname, parent)
self.root_items = [dirs[_root_path] for _root_path in root_path_list]
# Populating tree: files
for filename in sorted(self.results.keys()):
parent_item = dirs[osp.dirname(filename)]
file_item = QTreeWidgetItem(parent_item, [osp.basename(filename)],
QTreeWidgetItem.Type)
file_item.setIcon(0, get_filetype_icon(filename))
colno_dict = {}
fname_res = []
for lineno, colno, line in self.results[filename]:
if lineno not in colno_dict:
fname_res.append((lineno, colno, line))
colno_dict[lineno] = colno_dict.get(lineno, []) + [str(colno)]
for lineno, colno, line in fname_res:
colno_str = ",".join(colno_dict[lineno])
item = QTreeWidgetItem(
file_item,
["%d (%s): %s" % (lineno, colno_str, line.rstrip())],
QTreeWidgetItem.Type)
item.setIcon(0, get_icon('arrow.png'))
self.data[id(item)] = (filename, lineno)
# Removing empty directories
top_level_items = [
self.topLevelItem(index)
for index in range(self.topLevelItemCount())
]
for item in top_level_items:
if not item.childCount():
self.takeTopLevelItem(self.indexOfTopLevelItem(item))
