def _should_include_path(path, includes, excludes):
"""Return True iff the given path should be included."""
from os.path import basename
from fnmatch import fnmatch
base = basename(path)
if includes:
for include in includes:
if fnmatch(base, include):
try:
log.debug("include `%s' (matches `%s')", path, include)
except (NameError, AttributeError):
pass
break
else:
try:
log.debug("exclude `%s' (matches no includes)", path)
except (NameError, AttributeError):
pass
return False
for exclude in excludes:
if fnmatch(base, exclude):
try:
log.debug("exclude `%s' (matches `%s')", path, exclude)
except (NameError, AttributeError):
pass
return False
return True
def files(self):
if self.is_dir:
if self.recurse:
for root_dir, dirnames, filenames in os.walk(self.scan_path):
if self.filetype_filter != '*':
filenames = fnmatch.filter(filenames, self.filetype_filter)
for filename in filenames:
processing_callbacks = [callback for type_glob_string, callback
in self.filetype_handlers.iteritems()
if fnmatch.fnmatch(filename, type_glob_string)]
yield ScannerJobFile(os.path.join(root_dir, filename), processing_callbacks, self.data_callback, self.context)
else:
file_paths = os.listdir(self.scan_path)
if self.filetype_filter != '*':
file_paths = fnmatch.filter(file_paths, self.filetype_filter)
for file_path in file_paths:
processing_callbacks = [callback for type_glob_string, callback
in self.filetype_handlers.iteritems()
if fnmatch.fnmatch(file_path, type_glob_string)]
yield ScannerJobFile(os.path.join(self.scan_path, file_path), processing_callbacks, self.data_callback, self.context)
else: # single file
processing_callbacks = [callback for type_glob_string, callback
in self.filetype_handlers.iteritems()
if fnmatch.fnmatch(self.scan_path, type_glob_string)]
yield ScannerJobFile(self.scan_path, processing_callbacks, self.data_callback, self.context)
def expect_content(self, name, content, exact=False):
actual = self.__read_file(name, exact)
content = string.replace(content, "$toolset", self.toolset+"*")
matched = False
if exact:
matched = fnmatch.fnmatch(actual, content)
else:
def sorted_(x):
x.sort()
return x
actual_ = map(lambda x: sorted_(x.split()), actual.splitlines())
content_ = map(lambda x: sorted_(x.split()), content.splitlines())
if len(actual_) == len(content_):
matched = map(
lambda x, y: map(lambda n, p: fnmatch.fnmatch(n, p), x, y),
actual_, content_)
matched = reduce(
lambda x, y: x and reduce(
lambda a, b: a and b,
y),
matched)
if not matched:
print "Expected:\n"
print content
print "Got:\n"
print actual
self.fail_test(1)
def build_archive_list(pattern_list, target_dir):
glob_file_list = []
for pattern in pattern_list:
# Extract patterns
subdir = pattern[0]
file_pattern = pattern[1]
date_pattern = pattern[2]
# Build list of files
if len(date_pattern) > 0:
file_list = [ (datetime.strptime(file_name, date_pattern).strftime("%Y-%m-%dT%H:%M:%S.%fZ"), file_name, subdir)
for file_name in os.listdir(target_dir) if fnmatch.fnmatch(file_name, file_pattern) ]
else:
file_list = [ (date.fromtimestamp(tardate(file_name)).strftime("%Y-%m-%dT%H:%M:%S.%fZ"), file_name, subdir)
for file_name in os.listdir(target_dir) if fnmatch.fnmatch(file_name, file_pattern) ]
assert len(file_list) > 0
# Append files to global list
glob_file_list.extend(file_list)
# Sort the global list by date and return it
glob_file_list.sort(key=lambda x: x[0])
return glob_file_list
def add_extra_headers(self, headers, path, url):
if fnmatch.fnmatch(url, '*.xpi'):
headers['Content-Type'] = 'application/x-xpinstall'
if fnmatch.fnmatch(url, '*.rdf'):
headers['Content-Type'] = 'text/rdf'
if fnmatch.fnmatch(url, '*.json'):
headers['Content-Type'] = 'application/json'
def allow(self, include=None, exclude=None):
"""
Given a set of wilcard patterns in the include and exclude arguments,
tests if the patterns allow this item for processing.
The exclude parameter is processed first as a broader filter and then
include is used as a narrower filter to override the results for more
specific files.
Example:
exclude = (".*", "*~")
include = (".htaccess")
"""
if not include:
include = ()
if not exclude:
exclude = ()
if reduce(lambda result,
pattern: result or
fnmatch.fnmatch(self.name, pattern), include, False):
return True
if reduce(lambda result, pattern:
result and not fnmatch.fnmatch(self.name, pattern),
exclude, True):
return True
return False
def _CheckLinks(input_api, output_api, results):
for affected_file in input_api.AffectedFiles():
name = affected_file.LocalPath()
absolute_path = affected_file.AbsoluteLocalPath()
if not os.path.exists(absolute_path):
continue
if (fnmatch.fnmatch(name, '%s*' % PUBLIC_TEMPLATES_PATH) or
fnmatch.fnmatch(name, '%s*' % INTROS_PATH) or
fnmatch.fnmatch(name, '%s*' % ARTICLES_PATH) or
fnmatch.fnmatch(name, '%s*' % API_PATH)):
contents = _ReadFile(absolute_path)
args = []
if input_api.platform == 'win32':
args = [input_api.python_executable]
args.extend([os.path.join('docs', 'server2', 'link_converter.py'),
'-o',
'-f',
absolute_path])
output = input_api.subprocess.check_output(
args,
cwd=input_api.PresubmitLocalPath(),
universal_newlines=True)
if output != contents:
changes = ''
for i, (line1, line2) in enumerate(
zip(contents.split('\n'), output.split('\n'))):
if line1 != line2:
changes = ('%s\nLine %d:\n-%s\n+%s\n' %
(changes, i + 1, line1, line2))
if changes:
results.append(output_api.PresubmitPromptWarning(
'File %s may have an old-style <a> link to an API page. Please '
'run docs/server2/link_converter.py to convert the link[s], or '
'convert them manually.\n\nSuggested changes are: %s' %
(name, changes)))
def compare(list, values, attrName, subattrName=None, otherList=None, linkProperties=None):
if len(values) == 0:
return Exception
# List to return
returnList = []
for value in values:
for item in list:
if otherList is None:
# Get attribut "attrName" at first level
compareName = getattr(item, attrName)
if subattrName is None:
if fnmatch.fnmatch(compareName, value):
returnList.append(item)
# If sub attribute requested, get it
else:
compareName2 = getattr(compareName, subattrName)
if fnmatch.fnmatch(compareName2, value):
returnList.append(item)
else:
for otherItem in otherList:
if getattr(item, linkProperties[0]) == getattr(otherItem, linkProperties[1]):
compareName = getattr(otherItem, attrName)
if subattrName is None:
if fnmatch.fnmatch(compareName, value):
returnList.append(item)
else:
compareName2 = getattr(compareName, subattrName)
if fnmatch.fnmatch(compareName2, value):
returnList.append(item)
return returnList
def find_fit_speed_log(out_file, path):
pattern = "fitlog_time_speed.dat"
fitlog = []
for root, dirs, files in os.walk(path):
for name in files:
if fnmatch.fnmatch(name, pattern):
fitlog.append(os.path.join(root, name))
pattern = "fitlog_time_speed.dat"
fitlog_time_speed = []
for root, dirs, files in os.walk(path):
for name in files:
if fnmatch.fnmatch(name, pattern):
fitlog_time_speed.append(os.path.join(root, name))
string = "plot "
for my_file in fitlog:
string = string + "'" + my_file + "' using ($1/60/60):($2) with lp,"
# for my_file in fitlog_time_speed:
# string=string+"'"+my_file+"' using ($2) axis x1y2 with lp,"
string = string[:-1]
text_file = open(out_file, "w")
text_file.write(string)
text_file.close()
def write_test_for_class(item, out):
all_screenshots = _recursive_glob('*.png', SCREENSHOTS_ROOT)
always_screenshots = []
try_screenshots = []
for filename in all_screenshots:
if any(fnmatch.fnmatch(filename, x) for x in item.screenshots):
always_screenshots.append(filename)
elif any(fnmatch.fnmatch(filename, x) for x in item.try_screenshots):
try_screenshots.append(filename)
if len(always_screenshots) + len(try_screenshots) == 0:
return 0
always_screenshots.sort()
try_screenshots.sort()
out.write(dedent('''\
def auto_selftest_{name}():
r"""
''').format(name=item.name))
for expr in item.expressions:
for s in always_screenshots:
out.write(" >>> %s\n" % expr.format(frame='f("%s")' % s))
for s in try_screenshots:
out.write(" >>> %s # remove-if-false\n" % expr.format(
frame='f("%s")' % s))
out.write(' """\n pass\n')
return len(always_screenshots) + len(try_screenshots)
def visit(self, fil=None, rec=None, ignore=_dummyclass):
""" yields all paths below the current one
fil is a filter (glob pattern or callable), if not matching the
path will not be yielded, defaulting to None (everything is
returned)
rec is a filter (glob pattern or callable) that controls whether
a node is descended, defaulting to None
ignore is an Exception class that is ignoredwhen calling dirlist()
on any of the paths (by default, all exceptions are reported)
"""
if isinstance(fil, str):
fil = fnmatch(fil)
if rec:
if isinstance(rec, str):
rec = fnmatch(fil)
elif not callable(rec):
rec = lambda x: True
reclist = [self]
while reclist:
current = reclist.pop(0)
try:
dirlist = current.listdir()
except ignore:
return
for p in dirlist:
if fil is None or fil(p):
yield p
if p.check(dir=1) and (rec is None or rec(p)):
reclist.append(p)
def recurse_path(root, excludes=None):
if excludes is None:
excludes = []
basepath = os.path.realpath(root)
pathlist = [basepath]
patterns = [e for e in excludes if os.path.sep not in e]
absolute = [e for e in excludes if os.path.isabs(e)]
relative = [e for e in excludes if os.path.sep in e and
not os.path.isabs(e)]
for root, dirs, files in os.walk(basepath, topdown=True):
dirs[:] = [
d for d in dirs
if not any([fnmatch.fnmatch(d, pattern) for pattern in patterns])
if not any([fnmatch.fnmatch(os.path.abspath(os.path.join(root, d)),
path)
for path in absolute])
if not any([fnmatch.fnmatch(os.path.relpath(os.path.join(root, d)),
path)
for path in relative])
]
pathlist.extend([os.path.join(root, path) for path in dirs])
return pathlist
def main(argv):
api_files = rst_files = [rst for rst in os.listdir('doc/api')
if fnmatch.fnmatch(rst, '*.rst')
and rst not in excluded_docs]
cmd = argv.pop(0)
def has(*options):
for opt in options:
if opt in argv:
return argv.pop(argv.index(opt))
if has('-h', '--help'):
usage(cmd)
verbose = has('-v', '--verbose')
only_documented = not has('-a', '--all')
if argv:
given_files = []
for arg in argv:
arg = arg.replace('\\', '/').replace(api_doc + '/', '')
arg = arg.replace('.rst', '') + '.rst'
if '*' in arg: # glob pattern
given_files += [rst for rst in api_files
if fnmatch.fnmatch(rst, arg)]
elif arg in api_files:
given_files.append(arg)
api_files = given_files
rst_basenames = sorted(f[:-4] for f in rst_files)
for rst in api_files:
basename = rst.replace('.rst', '')
if verbose or len(api_files) > 1:
print("== Checking %s ... " % rst)
check_api_doc(basename, verbose, only_documented,
any(f.startswith(basename) and f != basename
for f in rst_basenames))
def write_mol(mol, filename):
"""
Write a molecule to a file based on file extension. For example, anything
ending in a "xyz" is assumed to be a XYZ file. Supported formats include
xyz, Gaussian input (gjf|g03|g09|com|inp), and pymatgen's JSON serialized
molecules.
Args:
mol (Molecule/IMolecule): Molecule to write
filename (str): A filename to write to.
"""
fname = os.path.basename(filename)
if fnmatch(fname.lower(), "*.xyz*"):
return XYZ(mol).write_file(filename)
elif any([fnmatch(fname.lower(), "*.{}*".format(r))
for r in ["gjf", "g03", "g09", "com", "inp"]]):
return GaussianInput(mol).write_file(filename)
elif fnmatch(fname, "*.json*") or fnmatch(fname, "*.mson*"):
with zopen(filename, "wt") as f:
return f.write(str2unicode(json.dumps(mol, cls=MontyEncoder)))
else:
m = re.search("\.(pdb|mol|mdl|sdf|sd|ml2|sy2|mol2|cml|mrv)",
filename.lower())
if m:
return BabelMolAdaptor(mol).write_file(filename, m.group(1))
raise ValueError("Unrecognized file extension!")
def filter_models(request, models, exclude):
"""
Returns (model, perm,) for all models that match models/exclude patterns
and are visible by current user.
"""
items = get_avail_models(request)
included = []
full_name = lambda model: '%s.%s' % (model.__module__, model.__name__)
# I beleive that that implemented
# O(len(patterns)*len(matched_patterns)*len(all_models))
# algorythm is fine for model lists because they are small and admin
# performance is not a bottleneck. If it is not the case then the code
# should be optimized.
if len(models) == 0:
included = items
else:
for pattern in models:
pattern_items = []
for item in items:
model, perms = item
if fnmatch(full_name(model), pattern) and item not in included:
pattern_items.append(item)
pattern_items.sort(key=lambda x:x[0]._meta.verbose_name_plural)
included.extend(pattern_items)
result = included[:]
for pattern in exclude:
for item in included:
model, perms = item
if fnmatch(full_name(model), pattern):
result.remove(item)
return result
def test_run_sql_test_template(self):
test_case = MockSQLConcurrencyTestCaseTemplate('test_template_query02')
test_case.__class__.__unittest_skip__ = False
test_result = unittest.TestResult()
# Cleanup
if os.path.exists(test_case.get_out_dir()):
for file in os.listdir(test_case.get_out_dir()):
if fnmatch.fnmatch(file, 'template_query02**.out'):
os.remove(os.path.join(test_case.get_out_dir(),file))
test_case.run(test_result)
self.assertEqual(test_result.testsRun, 1)
self.assertEqual(len(test_result.errors), 0)
self.assertEqual(len(test_result.skipped), 0)
self.assertEqual(len(test_result.failures), 0)
count = 0
for file in os.listdir(test_case.get_out_dir()):
if fnmatch.fnmatch(file, 'template_query02*.out'):
count = count + 1
self.assertEqual(count, 12)
# Cleanup
if os.path.exists(test_case.get_out_dir()):
for file in os.listdir(test_case.get_out_dir()):
path = os.path.join(test_case.get_out_dir(), file)
if fnmatch.fnmatch(file, 'template_query02*.*'):
os.remove(os.path.join(test_case.get_out_dir(), file))
if fnmatch.fnmatch(file, 'regress_sql_concurrency_test_case'):
shutil.rmtree(path)
def test_run_sql_test_failure_with_gpdiff(self):
test_case = MockSQLConcurrencyTestCase('test_query03')
# As explained above, we want MockSQLTestCase to run if and only if
# it's being invoked by our unit tests. So, it's skipped if discovered
# directly by unit2. Here, bearing in mind that SQLConcurrencyTestCaseTests is itself
# triggered by unit2, we override MockSQLTestCase's skip decorator to allow
# this explicit construction of MockSQLTestCase to proceed.
test_case.__class__.__unittest_skip__ = False
test_result = unittest.TestResult()
current_dir = os.path.dirname(inspect.getfile(test_case.__class__))
i = 1
for file in os.listdir(test_case.get_sql_dir()):
if fnmatch.fnmatch(file, 'query03_part*.sql'):
i += 1
self.assertTrue(test_case.gpdiff)
test_case.run(test_result)
self.assertEqual(test_result.testsRun, 1)
self.assertEqual(len(test_result.errors), 0)
self.assertEqual(len(test_result.skipped), 0)
self.assertEqual(len(test_result.failures), 1)
# Cleanup
if os.path.exists(test_case.get_out_dir()):
for file in os.listdir(test_case.get_out_dir()):
if fnmatch.fnmatch(file, 'query03*.out'):
os.remove(os.path.join(test_case.get_out_dir(),file))
def get_type_by_name(path):
"""Returns type of file by its name, or None if not known"""
if not _cache_uptodate:
_cache_database()
leaf = os.path.basename(path)
if leaf in literals:
return literals[leaf]
lleaf = leaf.lower()
if lleaf in literals:
return literals[lleaf]
ext = leaf
while 1:
p = ext.find('.')
if p < 0: break
ext = ext[p + 1:]
if ext in exts:
return exts[ext]
ext = lleaf
while 1:
p = ext.find('.')
if p < 0: break
ext = ext[p+1:]
if ext in exts:
return exts[ext]
for (glob, mime_type) in globs:
if fnmatch.fnmatch(leaf, glob):
return mime_type
if fnmatch.fnmatch(lleaf, glob):
return mime_type
return None
def included(self, event, base=None):
path = ''
if not base:
if hasattr(event, 'dest_path'):
base = os.path.basename(event.dest_path)
path = self.remove_prefix(self.get_unicode_path(event.dest_path))
else:
base = os.path.basename(event.src_path)
path = self.remove_prefix(self.get_unicode_path(event.src_path))
try:
#logging.info(" 1 : " + str(type(base)) + " " + str(type(path)))
if isinstance(path, str):
#logging.info('ENCODING PATH')
path = unicode(path, 'utf-8')
if isinstance(base, str):
#logging.info('ENCODING BASE')
base = unicode(base, 'utf-8')
#logging.info("2 : " + str(type(base)) + " " + str(type(path)))
#logging.info(base + u" " + path)
except Exception as e:
logging.exception(e)
if path == '.':
return False
for i in self.includes:
if not fnmatch.fnmatch(base, i):
return False
for e in self.excludes:
if fnmatch.fnmatch(base, e):
return False
for e in self.excludes:
if (e.startswith('/') or e.startswith('*/')) and fnmatch.fnmatch(path, e):
return False
return True
def filter(self, value):
""" Sets a filter for items present in the tree. Only shows tree items
that match the specified file extension(s) and hides the others.
value : None, str or list
If None is passed, this clears the filter, making all items present
in the tree visible again.
If a string is passed, it will be used as a single file extension
to compare the items against.
If a list of file extensions is passed, than items will be shown if
they match any of the extensions present in the list.
"""
# Check if supplied a valid value
if not isinstance(value, list) and \
not isinstance(value, basestring) and \
not value is None:
raise ValueError('Supplied filter invalid, needs to be list, string'
' or None')
# Store the filter for later reference
self._filter = value
# Iterate over the items
iterator = QtWidgets.QTreeWidgetItemIterator(self)
while(iterator.value()):
item = iterator.value()
# Check if item is of type 'file'
# Filters are only applicable to files
item_type = item.data(1, QtCore.Qt.DisplayRole)
if item_type == "file":
# If filter is None, it means everything should be
# visible, so set this item to visible and continue.
if self._filter is None:
item.setHidden(False)
iterator += 1
continue
# Check if filter extension is contained in filename
item_data = item.data(0, QtCore.Qt.UserRole)
filename = item_data['attributes']['name']
# Assume no match by default
typematch = False
# If filter is a single string, just check directly
if isinstance(self._filter, basestring):
typematch = fnmatch.fnmatch(filename, self._filter)
# If filter is a list, compare to each item in it
if isinstance(self._filter, list):
for ext in self._filter:
if fnmatch.fnmatch(filename, ext):
typematch = True
break
# Set item's visibility according to value of typematch
if typematch:
item.setHidden(False)
else:
item.setHidden(True)
iterator += 1
def _getLanguage(self, document):
"""Get language name by file path
"""
fileName = document.fileName()
if not fileName:
return
for languageName, fileNameGlobs, firstLineGlobs, iconPath in self.iterLanguages(): # pylint: disable=W0612
for fileNameGlob in fileNameGlobs:
# Empty patterns are ignored
if fileNameGlob and \
fnmatch.fnmatch(fileName, fileNameGlob):
return languageName
firstLine = document.line(0).strip() # first line without \n and other spaces
if firstLine is not None:
for languageName, fileNameGlobs, firstLineGlobs, iconPath in self.iterLanguages():
for firstLineGlob in firstLineGlobs:
# Empty patterns are ignored
if firstLineGlob and \
fnmatch.fnmatch(firstLine, firstLineGlob):
return languageName
return None
def step_back(io_manager):
'''
deletes the output files from the most recent runs
:type io_manager: IOManger
'''
current_iteration = None
target_iteration = None
# Figure out the current iteration and modify the scan specs file appropriately
parameter_scan_specs_xml_file_path = io_manager.parameter_scan_specs_xml_file_path
xml_file = parse(parameter_scan_specs_xml_file_path)
xml_root = xml_file.getroot()
for parameter_element in xml_root.iter('Parameter'):
current_iteration = int(parameter_element.attrib['CurrentIteration'])
target_iteration = current_iteration - 1
print('Stepping up files to redo batch run {}'.format(target_iteration))
parameter_element.set('CurrentIteration', str(target_iteration))
ElementTree(xml_root).write(parameter_scan_specs_xml_file_path)
# Remove screenshots
for root, dirs, files in os.walk(io_manager.screenshot_output_path):
for dir in dirs:
if dir == str(target_iteration):
shutil.rmtree(os.path.join(root, dir))
# Remove most recent .csv and .txt in output folder
for root, dirs, files in os.walk(io_manager.output_folder):
for file in files:
if fnmatch(file, '*output{}.txt'.format(target_iteration)) or fnmatch(file, '*output{}.csv'.format(target_iteration)):
os.remove(os.path.join(root, file))
def print_result_summaries_list(topnum=5):
print('\n<(^_^<)\n')
# Print out some summary of all results you have
hs = ld2.HotSpotter()
hs.load_tables(ld2.DEFAULT)
result_file_list = os.listdir(hs.dirs.result_dir)
sorted_rankres = []
for result_fname in iter(result_file_list):
if fnmatch.fnmatch(result_fname, 'rankres_str*.csv'):
print(result_fname)
with open(join(hs.dirs.result_dir, result_fname), 'r') as file:
metaline = file.readline()
toprint = metaline
# skip 4 metalines
[file.readline() for _ in xrange(4)]
top5line = file.readline()
top1line = file.readline()
toprint += top5line + top1line
line = read_until(file, '# NumData')
num_data = int(line.replace('# NumData', ''))
file.readline() # header
res_data_lines = [file.readline() for _ in xrange(num_data)]
res_data_str = np.array([line.split(',') for line in res_data_lines])
tt_scores = np.array(res_data_str[:, 5], dtype=np.float)
bt_scores = np.array(res_data_str[:, 6], dtype=np.float)
tf_scores = np.array(res_data_str[:, 7], dtype=np.float)
tt_score_sum = sum([score for score in tt_scores if score > 0])
bt_score_sum = sum([score for score in bt_scores if score > 0])
tf_score_sum = sum([score for score in tf_scores if score > 0])
toprint += ('tt_scores = %r; ' % tt_score_sum)
toprint += ('bt_scores = %r; ' % bt_score_sum)
toprint += ('tf_scores = %r; ' % tf_score_sum)
if topnum == 5:
sorted_rankres.append(top5line + metaline)
else:
sorted_rankres.append(top1line + metaline)
print(toprint + '\n')
print('\n(>^_^)>\n')
sorted_mapscore = []
for result_fname in iter(result_file_list):
if fnmatch.fnmatch(result_fname, 'oxsty_map_csv*.csv'):
print(result_fname)
with open(join(hs.dirs.result_dir, result_fname), 'r') as file:
metaline = file.readline()
scoreline = file.readline()
toprint = metaline + scoreline
sorted_mapscore.append(scoreline + metaline)
print(toprint)
print('\n'.join(sorted(sorted_rankres)))
print('\n'.join(sorted(sorted_mapscore)))
print('\n^(^_^)^\n')
def _ignore_patterns(path, names):
ignored_names = []
for pattern in patterns:
for name in names:
if fnmatch.fnmatch(name, pattern) or fnmatch.fnmatch(os.path.join(path, name), pattern):
ignored_names.append(name)
return set(ignored_names)
def filter_instances(specifiers, instances):
targets = set()
for instance in instances:
for specifier in specifiers:
if re.match(RE_INSTANCE_ID, specifier):
if instance.id == specifier:
targets.add(instance)
elif re.match(RE_PRIVATE_IP, specifier):
if instance.private_ip_address == specifier:
targets.add(instance)
elif re.match(RE_PRIVATE_HOSTNAME_1, specifier):
if instance.private_dns_name.startswith(specifier):
targets.add(instance)
elif re.match(RE_PRIVATE_HOSTNAME_2, specifier):
if instance.private_dns_name.startswith(specifier):
targets.add(instance)
else:
name = instance.tags.get('Name', '').lower()
altname = instance.tags.get('altName', '').lower()
if fnmatch(name, specifier.lower()):
targets.add(instance)
elif fnmatch(altname, specifier.lower()):
targets.add(instance)
return list(targets)
def _get(genre):
response = []
for f in os.listdir("%s/%s" % (options.store, genre)):
if not fnmatch.fnmatch(f, '*_thumbnail.jpg') and not fnmatch.fnmatch(f, '*_meta.json'):
response.append(MediaManager._build_response_for(genre, f))
return response
def get_index(self, requestContext):
matches = []
for root, _, files in walk(settings.WHISPER_DIR):
root = root.replace(settings.WHISPER_DIR, '')
for base_name in files:
if fnmatch.fnmatch(base_name, '*.wsp'):
match = join(root, base_name).replace('.wsp', '').replace('/', '.').lstrip('.')
bisect.insort_left(matches, match)
# unlike 0.9.x, we're going to use os.walk with followlinks
# since we require Python 2.7 and newer that supports it
if RRDReader.supported:
for root, _, files in walk(settings.RRD_DIR, followlinks=True):
root = root.replace(settings.RRD_DIR, '')
for base_name in files:
if fnmatch.fnmatch(base_name, '*.rrd'):
absolute_path = join(settings.RRD_DIR, root, base_name)
base_name = splitext(base_name)[0]
metric_path = join(root, base_name)
rrd = RRDReader(absolute_path, metric_path)
for datasource_name in rrd.get_datasources(absolute_path):
match = join(metric_path, datasource_name).replace('.rrd', '').replace('/', '.').lstrip('.')
if match not in matches:
bisect.insort_left(matches, match)
return matches
def findmatch(list, addrs):
"""Determine whether any of the passed e-mail addresses match a
Unix shell-style wildcard pattern contained in list. The
comparison is case-insensitive. Also, return the second half of
the string if it exists (for exp and ext addresses only)."""
for address in addrs:
if address:
address = address.lower()
for p in list:
stringparts = p.split()
p = stringparts[0]
# Handle special @=domain.dom syntax.
try:
at = p.rindex('@')
atequals = p[at+1] == '='
except (ValueError, IndexError):
atequals = None
if atequals:
p1 = p[:at+1] + p[at+2:]
p2 = p[:at+1] + '*.' + p[at+2:]
match = (fnmatch.fnmatch(address,p1)
or fnmatch.fnmatch(address,p2))
else:
match = fnmatch.fnmatch(address,p)
if match:
try:
return stringparts[1]
except IndexError:
return 1
def get_obs(self, bins, excludebin=None):
if isinstance(bins, basestring):
bins = [bins]
if isinstance(excludebin, basestring):
excludebin = [excludebin]
matching_bins = set([])
for binpattern in bins:
for realbin in self.card.exp.keys():
# First check if we explicitly exclude it
excluded = False
if excludebin is not None:
for excludepattern in excludebin:
if fnmatch.fnmatch(realbin, excludepattern):
excluded = True
break
if excluded:
continue
# otherwise check if it matches a desired bin
if fnmatch.fnmatch(realbin, binpattern) or fnmatch.fnmatch(realbin, 'bin' + binpattern):
matching_bins.add(realbin)
obs = 0
for bin in matching_bins:
obs += self.card.obs[bin]
return int(obs)
def filter_out(strings, patterns):
'''Filter out any string that matches any of the specified patterns.'''
for s in strings:
if all(not fnmatch.fnmatch(s, p) for p in patterns):
yield s
def vcftools():
directories = args.pop1 + '/*/Bcftools/' + filename
file_list = glob.glob(directories)
for f in file_list:
cmd1 = ['bcftools', 'index', '-c', '-f', f]
process1 = subprocess.Popen(cmd1, \
stdout=subprocess.PIPE)
while process1.wait() is None:
pass
process1.stdout.close()
directories1 = args.pop2 + '/*/Bcftools/' + filename
file_list1 = glob.glob(directories1)
for f1 in file_list1:
cmd_1 = ['bcftools', 'index', '-c', '-f', f1]
process_1 = subprocess.Popen(cmd_1, \
stdout=subprocess.PIPE)
while process_1.wait() is None:
pass
process_1.stdout.close()
# Make directory for the merged vcf-files
# for population1 and population2.
if not os.path.exists('Populations'):
os.makedirs('Populations')
# Making a list of vcf-files that will be input to bcftools merge
# and then merge population1.
directories2 = args.pop1 + '/*/Bcftools/' + filename
name_list1 = glob.glob(directories2)
myfile = open("name_1_list.txt", "w")
for n1 in name_list1:
myfile.write(add + "%s\n" % n1)
myfile.close()
cmd2 = ['bcftools', 'merge', \
'-l', add+names1, \
'-Oz', '-o', merged_vcf_pop1]
process2 = subprocess.Popen(cmd2, \
stdout=subprocess.PIPE, \
cwd='Populations')
while process2.wait() is None:
pass
process2.stdout.close()
# Making a list of vcf-files that will be input to bcftools merge
# and then merge population2.
directories3 = args.pop2 + '/*/Bcftools/' + filename
name_list2 = glob.glob(directories3)
myfile2 = open("name_2_list.txt", "w")
for n2 in name_list2:
myfile2.write(add + "%s\n" % n2)
myfile2.close()
cmd3 = ['bcftools', 'merge', \
'-l', add+names2, \
'-Oz', '-o', merged_vcf_pop2]
process3 = subprocess.Popen(cmd3, \
stdout=subprocess.PIPE, \
cwd='Populations')
while process3.wait() is None:
pass
process3.stdout.close()
# Making a txt file of the names of the individuals in the populations
# that is needed for vcftools --wei-fst-pop and indexing
# the merged files for population1 and population2.
for file in os.listdir('Populations'):
if fnmatch.fnmatch(file, '*_merged_pop1.vcf.gz'):
cmd4 = ['bcftools', 'index', '-c', '-f', merged_vcf_pop1]
process4 = subprocess.Popen(cmd4, \
stdout=subprocess.PIPE, \
cwd='Populations')
while process4.wait() is None:
pass
process4.stdout.close()
cmd5 = ('bcftools query -l %s > %s') \
% (merged_vcf_pop1, indv_txt_pop1)
process5 = subprocess.Popen(cmd5, \
stdout=subprocess.PIPE, \
shell=True, \
cwd='Populations')
while process5.wait() is None:
pass
process5.stdout.close()
elif fnmatch.fnmatch(file, '*_merged_pop2.vcf.gz'):
cmd6 = ['bcftools', 'index', '-c', '-f', merged_vcf_pop2]
process6 = subprocess.Popen(cmd6, \
stdout=subprocess.PIPE, \
cwd='Populations')
while process6.wait() is None:
pass
process6.stdout.close()
cmd7 = ('bcftools query -l %s > %s') \
% (merged_vcf_pop2, indv_txt_pop2)
process7 = subprocess.Popen(cmd7, \
stdout=subprocess.PIPE, \
shell=True, \
cwd='Populations')
while process7.wait() is None:
pass
process7.stdout.close()
# Making a list of vcf-files that will be input to bcftools merge
# and then merge population1 and population2 to an "all_merged"
# vcf file, this file will be the input file to
# vcftools --weir-fst-pop.
directories4 = 'Populations/*_merged_*.vcf.gz'
pop_list = glob.glob(directories4)
myfile3 = open("pop_list.txt", "w")
for p in pop_list:
myfile3.write(add + "%s\n" % p)
myfile3.close()
cmd8 = ['bcftools', 'merge', \
'-l', add+population_list, \
'-Oz', '-o', all_pop_merged]
process8 = subprocess.Popen(cmd8, \
stdout=subprocess.PIPE, \
cwd='Populations')
while process8.wait() is None:
pass
process8.stdout.close()
def build_site_packages():
"""Use PEX to resolve dependencies in a virtual environment,
with some customizations to reduce the size of our build.
https://www.pantsbuild.org/pex.html
"""
# Remove flywheel_cli from cache
# If you skip this step, it doesn't automatically update the python code
if os.path.isdir(PEX_BUILD_CACHE_DIR):
for name in os.listdir(PEX_BUILD_CACHE_DIR):
if fnmatch.fnmatch(name, 'flywheel_cli*.whl'):
path = os.path.join(PEX_BUILD_CACHE_DIR, name)
print('Removing {} from cache...'.format(name))
os.remove(path)
# Read ignore list
# See package-ignore.txt, largely we're removing test files and
# Multi-megabyte dicoms from the dicom folder
ignore_patterns = read_ignore_patterns()
# Create resolver
# Loosely based on: https://github.com/pantsbuild/pex/blob/982cb9a988949ffff3348b9bca98ae72a0bf8847/pex/bin/pex.py#L577
resolver_option_builder = ResolverOptionsBuilder()
resolvables = [
Resolvable.get('flywheel-cli=={}'.format(PYTHON_CLI_VERSION),
resolver_option_builder)
]
resolver = CachingResolver(PEX_BUILD_CACHE_DIR, None)
# Effectively we resolve (possibly from cache) The source and all of the dependency packages
# Then create the virtual environment, which contains those files
print('Resolving distributions')
resolved = resolver.resolve(resolvables)
print('Building package lists')
builder = PEXBuilder()
for dist in resolved:
builder.add_distribution(dist)
builder.add_requirement(dist.as_requirement())
# After this point, builder.chroot contains a full list of the files
print('Compiling package')
builder.freeze(bytecode_compile=False)
site_packages_path = os.path.join(BUILD_DIR, 'site-packages.zip')
# Create an uncompressed site-packages.zip and add all of the discovered files
# (Except those that are filtered out)
with open(site_packages_path, 'wb') as f:
added_files = set()
with zipfile.ZipFile(f, 'w') as zf:
for filename in sorted(builder.chroot().files()):
if is_ignored_file(ignore_patterns, filename):
continue
if not filename.startswith('.deps'):
continue
# Determine new path
src_path = os.path.join(builder.chroot().chroot, filename)
dst_path = '/'.join(filename.split('/')[2:])
# Optionally, compile the file
_, ext = os.path.splitext(src_path)
if ext == '.py':
cfile_path = src_path + 'c'
dst_path += 'c'
print('Compiling: {}'.format(dst_path))
py_compile.compile(src_path,
cfile=cfile_path,
dfile=dst_path,
optimize=1)
src_path = cfile_path
if not dst_path in added_files:
zf.write(src_path, dst_path)
added_files.add(dst_path)
return site_packages_path
def wildcard_match(word, wildcard):
return word and fnmatch.fnmatch(word, wildcard)
def main():
try:
usage = "[>] ./ioshook %prog [options] arg\n\n\r[>] Example for spawn or attach app with -s(--script) options:\n./ioshook -p com.apple.AppStore / [-n 'App Store'] -s trace_class.js\n\n\r[>] Example for spawn or attach app with -m(--method) options:\n./ioshook -p com.apple.AppStore / [-n 'App Store'] -m app-static\n\n\r[>] Example dump decrypt ipa with -d(--dump) and -o(--output) options:\n./ioshook -p com.apple.AppStore / [-n 'App Store'] -d -o App_dump_name\n\n\r[>] Example dump memory of application with --dump-memory and -s(--string) options:\n./ioshook -n 'App Store' --dump-memory '-s(--string)'\n\n\r[>] Example Scan IPA with file task:\n./ioshook --hexbyte-scan 'scan AppStore.ipa' -t /hexbyscan-tasks/openssl_hook.json"
parser = optparse.OptionParser(usage, add_help_option=False)
info = optparse.OptionGroup(parser, "Information")
quick = optparse.OptionGroup(parser, "Quick Method")
dump = optparse.OptionGroup(parser, "Dump decrypt IPA")
hexscan = optparse.OptionGroup(parser, "HexByte Scan IPA")
dumpmemory = optparse.OptionGroup(parser, "Dump memory of Application")
parser.add_option('-h',
"--help",
action="help",
dest="help",
help="Show basic help message and exit")
#Using options -p(--package) for spawn application and load script
parser.add_option("-p",
"--package",
dest="package",
help="Identifier of the target app",
metavar="PACKAGE",
action="store",
type="string")
#Using options -n(--name) for attach script to application is running
parser.add_option("-n",
"--name",
dest="name",
help="Name of the target app",
metavar="NAME",
action="store",
type="string")
parser.add_option("-s",
"--script",
dest="script",
help="Frida Script Hooking",
metavar="SCIPRT.JS")
parser.add_option("-c",
"--check-version",
action="store_true",
help="Check iOS hook for the newest version",
dest="checkversion")
parser.add_option("-u",
"--update",
action="store_true",
help="Update iOS hook to the newest version",
dest="update")
quick.add_option(
"-m",
"--method",
dest="method",
type="choice",
choices=[
'app-static', 'bypass-jb', 'bypass-ssl', 'i-url-req',
'i-crypto'
],
help=
"__app-static: Static Ananlysis Application(-n)\n\n\r\r__bypass-jb: Bypass Jailbreak Detection(-p)\n\n\r\r\r\r\r\r__bypass-ssl: Bypass SSL Pinning(-p)\n\n\n\n\n\n\n\n\n\r\r\r\r\r\r__i-url-req: Intercept URLRequest in App(-n)\n\n\n\n\n\n\n\n\n\r\r\r\r\r\r__i-crypto: Intercept Crypto in App(-p)",
metavar="app-static / bypass-jb / bypass-ssl / i-url-req / i-crypto"
)
#Some options to get info from device and applications
info.add_option("--list-devices",
action="store_true",
help="List All Devices",
dest="listdevices")
#Listapp option using the code of the AloneMonkey's repo frida-ios-dump - Link: https://github.com/AloneMonkey/frida-ios-dump
info.add_option("--list-apps",
action="store_true",
help="List The Installed apps",
dest="listapps")
info.add_option("--list-appinfo",
action="store_true",
help="List Info of Apps on Itunes",
dest="listappinfo")
info.add_option("--list-scripts",
action="store_true",
help="List All Scripts",
dest="listscripts")
#Dump decrypt IPA using the code of the AloneMonkey's repo frida-ios-dump - Link: https://github.com/AloneMonkey/frida-ios-dump
dump.add_option("-d",
"--dump",
action="store_true",
help="Dump decrypt application.ipa",
dest="dumpapp")
dump.add_option("-o",
"--output",
action="store",
dest="output_ipa",
help="Specify name of the decrypted IPA",
metavar="OUTPUT_IPA",
type="string")
#Dump memory of application using the code of Nightbringer21's repo fridump - Link: https://github.com/Nightbringer21/fridump
dumpmemory.add_option("--dump-memory",
action="store",
help="Dump memory of application",
dest="dumpmemory")
#Hexbytescan of application using the code of karek314's repo hexbytescanner - Link: https://github.com/karek314/hexbytescanner
hexscan.add_option("--hexbyte-scan",
action="store",
help="Scan or Patch IPA with byte patterns",
dest="hexscan")
hexscan.add_option("-t",
"--task",
action="store",
help="Task for hexbytescan",
dest="task")
parser.add_option_group(dump)
parser.add_option_group(dumpmemory)
parser.add_option_group(hexscan)
parser.add_option_group(info)
parser.add_option_group(quick)
options, args = parser.parse_args()
methods = [
"methods/ios_list_apps.js", #0
"methods/static_analysis.js", #1
"methods/bypass_ssl.js", #2
"methods/bypass_jailbreak.js", #3
"methods/intercept_url_request.js", #4
"methods/intercept_crypto.js", #5
"methods/dump.js" #6
]
utils = [
"core/utils/dump.py" #0
]
if options.listdevices:
logger.info('[*] List All Devices: ')
os.system('frida-ls-devices')
elif options.listapps:
logger.info('[*] List All Apps on Devies: ')
device = get_usb_iphone()
list_applications(device)
elif options.listappinfo:
method = methods[0]
if os.path.isfile(method):
logger.info('[*] List Info of Apps on Itunes: ')
process = 'itunesstored'
os.system('frida -U -n ' + process + ' -l ' + method)
#sys.stdin.read()
else:
logger.error('[?] Script not found!')
elif options.listscripts:
path = 'frida-scripts/'
if os.path.exists(path):
logger.info('[*] List All Scripts: ')
for file_name in os.listdir(path):
if fnmatch.fnmatch(file_name, '*.js'):
print('[*] ' + file_name)
else:
logger.error('[?] Path frida-script not exists!')
#Spawning application and load script
elif options.package and options.script:
if os.path.isfile(options.script):
logger.info('[*] Spawning: ' + options.package)
logger.info('[*] Script: ' + options.script)
time.sleep(2)
pid = frida.get_usb_device().spawn(options.package)
session = frida.get_usb_device().attach(pid)
hook = open(options.script, 'r')
script = session.create_script(hook.read())
script.load()
frida.get_usb_device().resume(pid)
sys.stdin.read()
else:
logger.error('[?] Script not found!')
#Spawning application and load script with output
#Attaching script to application
elif options.name and options.script:
if os.path.isfile(options.script):
logger.info('[*] Attaching: ' + options.name)
logger.info('[*] Script: ' + options.script)
time.sleep(2)
process = frida.get_usb_device().attach(options.name)
hook = open(options.script, 'r')
script = process.create_script(hook.read())
script.load()
sys.stdin.read()
else:
logger.error('[?] Script not found!')
#Static Analysis Application
elif options.name and options.method == "app-static":
method = methods[1]
if os.path.isfile(method):
logger.info('[*] Attaching: ' + options.name)
logger.info('[*] Method: ' + options.method)
time.sleep(2)
process = frida.get_usb_device().attach(options.name)
method = open(method, 'r')
script = process.create_script(method.read())
script.load()
sys.stdin.read()
else:
logger.error('[?] Script not found!')
#Bypass jailbreak
elif options.package and options.method == "bypass-jb":
method = methods[3]
if os.path.isfile(method):
logger.info('[*] Bypass Jailbreak: ')
logger.info('[*] Spawning: ' + options.package)
logger.info('[*] Script: ' + method)
time.sleep(2)
pid = frida.get_usb_device().spawn(options.package)
session = frida.get_usb_device().attach(pid)
hook = open(method, 'r')
script = session.create_script(hook.read())
script.load()
frida.get_usb_device().resume(pid)
sys.stdin.read()
else:
logger.error('[?] Script for method not found!')
#Bypass SSL Pinning
elif options.package and options.method == "bypass-ssl":
method = methods[2]
if os.path.isfile(method):
logger.info('[*] Bypass SSL Pinning: ')
logger.info('[*] Spawning: ' + options.package)
logger.info('[*] Script: ' + method)
os.system('frida -U -f ' + options.package + ' -l ' + method +
' --no-pause')
#sys.stdin.read()
else:
logger.error('[?] Script for method not found!')
#Intercept url request in app
elif options.name and options.method == "i-url-req":
method = methods[4]
if os.path.isfile(method):
logger.info('[*] Intercept UrlRequest: ')
logger.info('[*] Attaching: ' + options.name)
logger.info('[*] Script: ' + method)
time.sleep(2)
process = frida.get_usb_device().attach(options.name)
method = open(method, 'r')
script = process.create_script(method.read())
script.load()
sys.stdin.read()
else:
logger.error('[?] Script for method not found!')
#Intercept Crypto Operations
elif options.package and options.method == "i-crypto":
method = methods[5]
if os.path.isfile(method):
logger.info('[*] Intercept Crypto Operations: ')
logger.info('[*] Spawning: ' + options.package)
logger.info('[*] Script: ' + method)
os.system('frida -U -f ' + options.package + ' -l ' + method +
' --no-pause')
#sys.stdin.read()
else:
logger.error('[?] Script for method not found!')
#check newversion
elif options.checkversion:
logger.info('[*] Checking for updates...')
is_newest = check_version(speak=True)
# if not is_newest:
# logger.info('[*] There is an update available for iOS hook')
#update newversion
elif options.update:
logger.info('[*] Update in progress...')
cmd = shlex.split("git reset --hard & git pull origin master")
subprocess.call(cmd)
sys.exit(0)
#dump decrypt application
elif (options.package or options.name) and options.dumpapp:
logger.info('[*] Dumping...')
util = utils[0]
if options.name is None:
if options.output_ipa is None:
cmd = shlex.split("python3 " + util + " " +
options.package)
else:
cmd = shlex.split("python3 " + util + " " +
options.package + " -o " +
options.output_ipa)
else:
if options.output_ipa is None:
cmd = shlex.split("python3 " + util + " " + "'" +
options.name + "'")
else:
cmd = shlex.split("python3 " + util + " " + "'" +
options.name + "'" + " -o " +
options.output_ipa)
subprocess.call(cmd)
sys.exit(0)
#dump memory application
elif options.name and options.dumpmemory:
dump_memory(options.dumpmemory, options.name)
#hexbytescan ipa
elif options.hexscan:
hexbyte_scan(options.hexscan, options.task)
else:
logger.warning("[!] Specify the options. use (-h) for more help!")
# sys.exit(0)
#EXCEPTION FOR FRIDA
except frida.ServerNotRunningError:
logger.error("Frida server is not running.")
except frida.TimedOutError:
logger.error("Timed out while waiting for device to appear.")
except frida.TransportError:
logger.error("[x_x] The application may crash or lose connection.")
except (frida.ProcessNotFoundError, frida.InvalidOperationError):
logger.error("[x_x] Unable to find process with name " + options.name +
". You need run app first.!!")
#EXCEPTION FOR OPTIONPARSING
#EXCEPTION FOR SYSTEM
except Exception as e:
logger.error(
"[x_x] Something went wrong, please check your error message.\n Message - {0}"
.format(e))
except KeyboardInterrupt:
logger.info("Bye bro!!")
return self.distribution.headers or []
def get_outputs(self):
return self.outfiles
def find_files(pattern, root):
"""Return all the files matching pattern below root dir."""
for dirpath, _, files in os.walk(root):
for filename in fnmatch.filter(files, pattern):
yield os.path.join(dirpath, filename)
so_lib_paths = [
i for i in os.listdir('.')
if os.path.isdir(i) and fnmatch.fnmatch(i, '_solib_*')
]
matches = []
for path in so_lib_paths:
matches.extend(
['../' + x for x in find_files('*', path) if '.py' not in x]
)
if os.name == 'nt':
EXTENSION_NAME = 'python/_pywrap_tensorflow_internal.pyd'
else:
EXTENSION_NAME = 'python/_pywrap_tensorflow_internal.so'
headers = (
list(find_files('*.h', 'tensorflow/core')) + list(
def handle_request(proxied_request):
"""Proxy the given request to the URL in the Forward-Host header with an
Authorization header set using an OIDC bearer token for the Cloud
Function's service account. If the header is not present, return a 400
error.
"""
host = TARGET_HOST
scheme = proxied_request.headers.get('X-Forwarded-Proto', 'https')
url = '{}://{}{}'.format(scheme, host, proxied_request.path)
headers = dict(proxied_request.headers)
# Check path against whitelist.
path = proxied_request.path
if not path:
path = '/'
if '*' not in _whitelist:
_is_match = False
for entry in _whitelist:
if fnmatch.fnmatch(path, entry):
_is_match = True
break
if not _is_match:
logging.warn('Rejected {} {}, not in whitelist'.format(
proxied_request.method, url))
return 'Requested path {} not in whitelist'.format(path), 403
global _oidc_token
if not _oidc_token or _oidc_token.is_expired():
_oidc_token = _get_google_oidc_token()
logging.info('Renewed OIDC bearer token for {}'.format(
_adc_credentials.service_account_email))
# Add the Authorization header with the OIDC token.
headers['Authorization'] = 'Bearer {}'.format(_oidc_token)
webhook_body = proxied_request.data
gitlab_secret = proxied_request.headers.get(SECRET_HEADER, '')
if SECRET_KEY:
try:
_body = json.loads(webhook_body)
# Add the Gitlab secret token into the request body
_body[SECRET_KEY] = gitlab_secret
webhook_body = json.dumps(_body)
except json.JSONDecodeError as e:
return 'Failed to decode webhook body', 500
print('(%s)' % webhook_body)
# We don't want to forward the Host header.
headers.pop('Host', None)
request = Request(proxied_request.method,
url,
headers=headers,
data=webhook_body)
# Send the proxied request.
prepped = request.prepare()
logging.info('{} {}'.format(prepped.method, prepped.url))
resp = _session.send(prepped)
# Strip hop-by-hop headers and Content-Encoding.
headers = _strip_hop_by_hop_headers(resp.headers)
headers.pop('Content-Encoding', None)
return resp.content, resp.status_code, headers.items()
def sliding_window():
# If using sliding window.
for file in os.listdir('Populations'):
if fnmatch.fnmatch(file, 'all_pop_merged.vcf.gz'):
cmda = ['vcftools', \
'--gzvcf', all_pop_merged, \
'--weir-fst-pop', indv_txt_pop1, \
'--weir-fst-pop', indv_txt_pop2, \
'--fst-window-size', args.window, \
'--fst-window-step', args.step, \
'--out', fst_out]
processa = subprocess.Popen(cmda, \
stdout=subprocess.PIPE, \
cwd='Populations')
while processa.wait() is None:
pass
processa.stdout.close()
for file in os.listdir('Populations'):
if fnmatch.fnmatch(file, '*.weir.fst'):
cmdb = ('cat %s | grep -v "nan" > %s') \
% (fst_out_window_in, fst_out_flt)
processb = subprocess.Popen(cmdb, \
stdout=subprocess.PIPE, \
shell=True, \
cwd='Fst_stats')
while processb.wait() is None:
pass
processb.stdout.close()
# Removing the results below zero.
for file in os.listdir('Fst_stats'):
if fnmatch.fnmatch(file, '*flt.table'):
cmdc = ("awk '{if ($6 >0) print}' %s > %s") \
% (fst_out_flt, fst_out_flt_results)
processc = subprocess.Popen(cmdc, \
stdout=subprocess.PIPE, \
shell=True, \
cwd='Fst_stats')
while processc.wait() is None:
pass
processc.stdout.close()
# Calculating the midpoint position when using sliding window,
# column (2+3)/2 and the output in column 2.
cmde = ('''awk '{a=int(($2+$3)/2); $2=a; print}' %s > %s''') \
% (fst_out_flt_results, fst_out_flt2_results)
processe = subprocess.Popen(cmde, \
stdout=subprocess.PIPE, \
shell=True, \
cwd='Fst_stats')
while processe.wait() is None:
pass
processe.stdout.close()
# Rearrange columns (if needed) and keep midpoint value in column 2.
cmdd = ('''awk '{print $1 "\\t" $2 "\\t" $6}' %s > %s''') \
% (fst_out_flt2_results, fst_calculated)
processd = subprocess.Popen(cmdd, \
stdout=subprocess.PIPE, \
shell=True, \
cwd='Fst_stats')
while processd.wait() is None:
pass
processd.stdout.close()
# Remove the header provided when using sliding window.
cmdf = ('echo "$(tail -n +2 %s)" > %s') \
% (fst_calculated, rm_headers)
processf = subprocess.Popen(cmdf, \
stdout=subprocess.PIPE, \
shell=True, \
cwd='Fst_stats')
while processf.wait() is None:
pass
processf.stdout.close()
# Add headers.
cmdg = ('echo -e "CHROM\\tPOS\\tWEIR_AND_COCKERHAM_FST" \
| cat - %s > %s' ) \
% (rm_headers, fst_headers)
processg = subprocess.Popen(cmdg, \
stdout=subprocess.PIPE, \
shell=True, \
cwd='Fst_stats')
while processg.wait() is None:
pass
processg.stdout.close()
# Sorting the POS column (needed for x-axis in highcharts).
cmdi = ("cat %s | sort -n > %s") \
% (fst_headers, fst_results_sorted)
processi = subprocess.Popen(cmdi, \
stdout=subprocess.PIPE, \
shell=True, \
cwd='Fst_stats')
while processi.wait() is None:
pass
processi.stdout.close()
# Making a csv file.
cmdj = ('cat %s | tr "\\t" "," > %s') \
% (fst_results_sorted, fst_results_sorted_csv)
processj = subprocess.Popen(cmdj, \
stdout=subprocess.PIPE, \
shell=True, \
cwd='Fst_stats')
while processj.wait() is None:
pass
processj.stdout.close()
except subprocess.TimeoutExpired:
stats.timeout += 1
fail("Timeout")
continue
except subprocess.CalledProcessError:
stats.crashed += 1
fail("Crashed")
continue
if "expect_sha1" in cmd:
actual = hashlib.sha1(output).hexdigest()
if actual != cmd['expect_sha1']:
fail(f"Actual sha1: {actual}")
if "expect_pattern" in cmd:
if not fnmatch.fnmatch(output.decode("utf-8"), cmd['expect_pattern']):
fail(f"Output does not match pattern")
print()
pprint(stats)
if stats.failed:
print(f"{ansi.FAIL}=======================")
print(f" FAILED: {stats.failed}/{stats.total_run}")
print(f"======================={ansi.ENDC}")
sys.exit(1)
else:
print(f"{ansi.OKGREEN}=======================")
print(f" All {stats.total_run} tests OK")
def plot():
for file in os.listdir('Fst_stats'):
if fnmatch.fnmatch(file, 'pop1_pop2_flt_results_sorted.csv'):
# Import csv file with Fst results.
gl = pd.read_csv('Fst_stats/pop1_pop2_flt_results_sorted.csv')
# Optimize memory usage.
gl_int = gl.select_dtypes(include=['int'])
converted_int = gl_int.apply(pd.to_numeric, downcast='unsigned')
gl_float = gl.select_dtypes(include=['float'])
converted_float = gl_float.apply(pd.to_numeric, downcast='float')
optimized_gl = gl.copy()
optimized_gl[converted_int.columns] = converted_int
optimized_gl[converted_float.columns] = converted_float
# Convert CHROM column from object to category.
gl_obj = gl.select_dtypes(include=['object']).copy()
chrom = gl_obj.CHROM
chrom_cat = chrom.astype('category')
converted_obj = pd.DataFrame()
# If unique values are more than 50% of the data do not
# convert to category, it will not optimize memory usage.
for col in gl_obj.columns:
num_unique_values = len(gl_obj[col].unique())
num_total_values = len(gl_obj[col])
if num_unique_values / num_total_values < 0.5:
converted_obj.loc[:, col] = gl_obj[col].astype('category')
else:
converted_obj.loc[:, col] = gl_obj[col]
# Apply on the csv file.
optimized_gl[converted_obj.columns] = converted_obj
dtypes_col = optimized_gl.dtypes.index
dtypes_type = [i.name for i in optimized_gl.dtypes.values]
column_types = dict(zip(dtypes_col, dtypes_type))
read_and_optimized = pd.read_csv('Fst_stats/pop1_pop2_flt_results_sorted.csv', \
dtype=column_types)
# Rename the read and optimized csv file
# from the Fst analysis to "df".
df = read_and_optimized
df['code'] = chrom_cat.cat.codes
df['ind'] = range(len(df))
df_grouped = df.groupby(('code'))
# Dict for the contig names and index number.
names = dict(enumerate(df['CHROM'].cat.categories))
# Make plot of data.
fig = plt.figure(figsize=(80, 20))
ax = fig.add_subplot(111)
colors = ['green', 'turquoise', \
'blue', 'purple', \
'red', 'orange', \
'yellow']
x_labels = []
x_labels_pos = []
for num, (name, group) in enumerate(df_grouped):
group.plot(kind='scatter', x='ind', y='WEIR_AND_COCKERHAM_FST', \
color=colors[num % len(colors)], ax=ax)
x_labels.append(name)
x_labels_pos.append((group['ind'].iloc[-1] \
- (group['ind'].iloc[-1] - group['ind'].iloc[0])/2))
ax.set_xticks(x_labels_pos)
ax.set_xticklabels(x_labels, rotation='vertical', fontsize=10)
ax.set_xlim([0, len(df)])
ax.set_ylim([0, 1])
ax.set_xlabel('contigs', fontsize=24)
ax.set_ylabel('Fst value', fontsize=24)
ax.set_title('Weir and Cockerham Fst', fontsize=40)
plt.tick_params(axis='x', length=0.01)
# Add legend with key values paired with the name of the contig.
legend_list = []
for key, value in names.items():
temp = [key, value]
legend_list.append(temp)
plt.legend(legend_list,bbox_to_anchor=(1.01, 1), \
ncol=5, \
borderaxespad=0)
plt.tight_layout(pad=7)
# Save plot as pdf.
plt.savefig("Fst_stats/Fst_plot_vcftools.pdf")
def get_files_recursive(dir_name, exclude_mask=''):
for root, dirs, files in os.walk(dir_name):
for _file in files:
if not fnmatch.fnmatch(_file, exclude_mask):
yield os.path.join(root, _file)
def fst():
# Fst_statistics using vcftools --weir-fst-pop, input files are a
# vcf file with all merged populations one txt file with names
# of the individuals from population1 and one txt file with
# names of the individulas from population2, output is a table
# of Fst values and a log file of the results.
for file in os.listdir('Populations'):
if fnmatch.fnmatch(file, 'all_pop_merged.vcf.gz'):
cmd9 = ['vcftools', \
'--gzvcf', all_pop_merged, \
'--weir-fst-pop', indv_txt_pop1, \
'--weir-fst-pop', indv_txt_pop2, \
'--out', fst_out]
process9 = subprocess.Popen(cmd9, \
stdout=subprocess.PIPE, \
cwd='Populations')
while process9.wait() is None:
pass
process9.stdout.close()
# Filtering the resulting files from vcftools and making a new
# directory called 'Fst_stats' with the resulting files,
# output is a csv file and a tab separated table, the csv file
# will be the input file to pandas, matplotlib and highcharts.
for file in os.listdir('Populations'):
if fnmatch.fnmatch(file, '*.weir.fst'):
cmd10 = ('cat %s | grep -v "nan" > %s') \
% (fst_out_in, fst_out_flt)
process10 = subprocess.Popen(cmd10, \
stdout=subprocess.PIPE, \
shell=True, \
cwd='Fst_stats')
while process10.wait() is None:
pass
process10.stdout.close()
# Removing the results below zero.
for file in os.listdir('Fst_stats'):
if fnmatch.fnmatch(file, '*flt.table'):
cmd11 = ("awk '{if ($3 >0) print}' %s > %s") \
% (fst_out_flt, fst_out_flt_results)
process11 = subprocess.Popen(cmd11, \
stdout=subprocess.PIPE, \
shell=True, \
cwd='Fst_stats')
while process11.wait() is None:
pass
process11.stdout.close()
# Rearrange columns (if needed).
cmd12 = ('''awk '{print $1 "\\t" $2 "\\t" $3}' %s > %s''') \
% (fst_out_flt_results, fst_out_flt2_results)
process12 = subprocess.Popen(cmd12, \
stdout=subprocess.PIPE, \
shell=True, \
cwd='Fst_stats')
while process12.wait() is None:
pass
process12.stdout.close()
# Sorting the POS column (needed for x-axis in highcharts).
cmd13 = ("cat %s | sort -n > %s") \
% (fst_out_flt2_results, fst_results_sorted)
process13 = subprocess.Popen(cmd13, \
stdout=subprocess.PIPE, \
shell=True, \
cwd='Fst_stats')
while process13.wait() is None:
pass
process13.stdout.close()
# Making a csv file.
cmd14 = ('cat %s | tr "\\t" "," > %s') \
% (fst_results_sorted, fst_results_sorted_csv)
process14 = subprocess.Popen(cmd14, \
stdout=subprocess.PIPE, \
shell=True, \
cwd='Fst_stats')
while process14.wait() is None:
pass
process14.stdout.close()
print("Successfully sent email")
except Exception:
print("Error: unable to send email")
walz_failed_files = []
print("Processing...")
if not os.path.exists(corrupt_archive_path):
os.makedirs(corrupt_archive_path)
if not os.path.exists(pipe_delimiter_archive_path):
os.makedirs(pipe_delimiter_archive_path)
if not os.path.exists(duplicates_archive_path):
os.makedirs(duplicates_archive_path)
for f in os.listdir(search_dir):
if fnmatch.fnmatch(f, '*.zip'):
single_file = {}
single_file['Filename'] = f
single_file['Reason'] = ''
single_file['Content'] = ''
walz_failed_files.append(single_file)
del (single_file)
print("No. of failed files:", len(walz_failed_files))
#Search reason from outlook
outlook = win32com.client.Dispatch("Outlook.Application").GetNamespace("MAPI")
search_folder = "App Trace"
outlook = outlook.Folders
#Fetch messages from outlook
def is_compressable(filename, gzip_globs):
"""
Determine if a filename is a gzippable type
by comparing to a known list.
"""
return any([fnmatch(filename, glob) for glob in gzip_globs])
#!/usr/bin/python2
#
# This is a simple script that we use to check for files in git
# and not in the distribution. It was previously written in cinnamon
# and inlined in the Makefile.am, but 'git ls-files --exclude=<pattern>'
# was changed to no longer do anything useful, which made that
# too challenging to be worthwhile.
import fnmatch, os, subprocess, sys
srcdir=sys.argv[1]
distdir=sys.argv[2]
excludes=sys.argv[3:]
os.chdir(srcdir)
status=0
for f in subprocess.Popen(["git", "ls-files"], stdout=subprocess.PIPE).stdout:
f = f.strip()
if (not os.path.exists(os.path.join(distdir, f)) and
not any((fnmatch.fnmatch(f, p) for p in excludes))):
print "File missing from distribution:", f
status=1
sys.exit(status)
def find_package_data():
"""
Find package_data.
"""
# This is not enough for these things to appear in an sdist.
# We need to muck with the MANIFEST to get this to work
# exclude components and less from the walk;
# we will build the components separately
excludes = [
pjoin('static', 'components'),
pjoin('static', '*', 'less'),
pjoin('static', '*', 'node_modules')
]
# walk notebook resources:
cwd = os.getcwd()
os.chdir('notebook')
static_data = []
for parent, dirs, files in os.walk('static'):
if any(fnmatch(parent, pat) for pat in excludes):
# prevent descending into subdirs
dirs[:] = []
continue
for f in files:
static_data.append(pjoin(parent, f))
# for verification purposes, explicitly add main.min.js
# so that installation will fail if they are missing
for app in ['auth', 'edit', 'notebook', 'terminal', 'tree']:
static_data.append(pjoin('static', app, 'js', 'main.min.js'))
components = pjoin("static", "components")
# select the components we actually need to install
# (there are lots of resources we bundle for sdist-reasons that we don't actually use)
static_data.extend([
pjoin(components, "backbone", "backbone-min.js"),
pjoin(components, "bootstrap", "js", "bootstrap.min.js"),
pjoin(components, "bootstrap-tour", "build", "css",
"bootstrap-tour.min.css"),
pjoin(components, "bootstrap-tour", "build", "js",
"bootstrap-tour.min.js"),
pjoin(components, "font-awesome", "css", "*.css"),
pjoin(components, "es6-promise", "*.js"),
pjoin(components, "font-awesome", "fonts", "*.*"),
pjoin(components, "google-caja", "html-css-sanitizer-minified.js"),
pjoin(components, "jed", "jed.js"),
pjoin(components, "jquery", "jquery.min.js"),
pjoin(components, "jquery-typeahead", "dist",
"jquery.typeahead.min.js"),
pjoin(components, "jquery-typeahead", "dist",
"jquery.typeahead.min.css"),
pjoin(components, "jquery-ui", "ui", "minified", "jquery-ui.min.js"),
pjoin(components, "jquery-ui", "themes", "smoothness",
"jquery-ui.min.css"),
pjoin(components, "jquery-ui", "themes", "smoothness", "images", "*"),
pjoin(components, "marked", "lib", "marked.js"),
pjoin(components, "preact", "index.js"),
pjoin(components, "preact-compat", "index.js"),
pjoin(components, "proptypes", "index.js"),
pjoin(components, "requirejs", "require.js"),
pjoin(components, "requirejs-plugins", "src", "json.js"),
pjoin(components, "requirejs-text", "text.js"),
pjoin(components, "underscore", "underscore-min.js"),
pjoin(components, "moment", "moment.js"),
pjoin(components, "moment", "min", "*.js"),
pjoin(components, "xterm.js", "dist", "xterm.js"),
pjoin(components, "xterm.js", "dist", "xterm.css"),
pjoin(components, "text-encoding", "lib", "encoding.js"),
])
# Ship all of Codemirror's CSS and JS
for parent, dirs, files in os.walk(pjoin(components, 'codemirror')):
for f in files:
if f.endswith(('.js', '.css')):
static_data.append(pjoin(parent, f))
# Trim mathjax
mj = lambda *path: pjoin(components, 'MathJax', *path)
static_data.extend([
mj('MathJax.js'),
mj('config', 'TeX-AMS-MML_HTMLorMML-full.js'),
mj('config', 'Safe.js'),
])
trees = []
mj_out = mj('jax', 'output')
if os.path.exists(mj_out):
for output in os.listdir(mj_out):
path = pjoin(mj_out, output)
static_data.append(pjoin(path, '*.js'))
autoload = pjoin(path, 'autoload')
if os.path.isdir(autoload):
trees.append(autoload)
for tree in trees + [
mj('localization'), # limit to en?
mj('fonts', 'HTML-CSS', 'STIX-Web', 'woff'),
mj('extensions'),
mj('jax', 'input', 'TeX'),
mj('jax', 'output', 'HTML-CSS', 'fonts', 'STIX-Web'),
mj('jax', 'output', 'SVG', 'fonts', 'STIX-Web'),
mj('jax', 'element', 'mml'),
]:
for parent, dirs, files in os.walk(tree):
for f in files:
static_data.append(pjoin(parent, f))
os.chdir(os.path.join('tests', ))
js_tests = glob('*.js') + glob('*/*.js')
os.chdir(cwd)
package_data = {
'notebook': ['templates/*'] + static_data,
'notebook.tests':
js_tests,
'notebook.bundler.tests':
['resources/*', 'resources/*/*', 'resources/*/*/.*'],
'notebook.services.api': ['api.yaml'],
}
return package_data
def search_file(pattern, f):
"""
Function to searach a single file for a single search pattern.
"""
fn_matched = False
contents_matched = False
# Use mimetypes to exclude binary files where possible
(ftype, encoding) = mimetypes.guess_type(os.path.join(f['root'], f['fn']))
if encoding is not None:
return False
if ftype is not None and ftype.startswith('image'):
return False
# Search pattern specific filesize limit
if pattern.get('max_filesize') is not None and 'filesize' in f:
if f['filesize'] > pattern.get('max_filesize'):
return False
# Search by file name (glob)
if pattern.get('fn') is not None:
if fnmatch.fnmatch(f['fn'], pattern['fn']):
fn_matched = True
if pattern.get('contents') is None and pattern.get(
'contents_re') is None:
return True
# Search by file name (regex)
if pattern.get('fn_re') is not None:
if re.match(pattern['fn_re'], f['fn']):
fn_matched = True
if pattern.get('contents') is None and pattern.get(
'contents_re') is None:
return True
# Search by file contents
if pattern.get('contents') is not None or pattern.get(
'contents_re') is not None:
if pattern.get('contents_re') is not None:
repattern = re.compile(pattern['contents_re'])
try:
with io.open(os.path.join(f['root'], f['fn']),
"r",
encoding='utf-8') as f:
l = 1
for line in f:
# Search by file contents (string)
if pattern.get('contents') is not None:
if pattern['contents'] in line:
contents_matched = True
if pattern.get('fn') is None and pattern.get(
'fn_re') is None:
return True
break
# Search by file contents (regex)
elif pattern.get('contents_re') is not None:
if re.search(repattern, line):
contents_matched = True
if pattern.get('fn') is None and pattern.get(
'fn_re') is None:
return True
break
# Break if we've searched enough lines for this pattern
if pattern.get(
'num_lines') and l >= pattern.get('num_lines'):
break
l += 1
except (IOError, OSError, ValueError, UnicodeDecodeError):
if config.report_readerrors:
logger.debug(
"Couldn't read file when looking for output: {}".format(
f['fn']))
return False
return fn_matched and contents_matched
def find_package_data():
"""
Find IPython's package_data.
"""
# This is not enough for these things to appear in an sdist.
# We need to muck with the MANIFEST to get this to work
# exclude components and less from the walk;
# we will build the components separately
excludes = [
pjoin('static', 'components'),
pjoin('static', '*', 'less'),
]
# walk notebook resources:
cwd = os.getcwd()
os.chdir(os.path.join('IPython', 'html'))
static_data = []
for parent, dirs, files in os.walk('static'):
if any(fnmatch(parent, pat) for pat in excludes):
# prevent descending into subdirs
dirs[:] = []
continue
for f in files:
static_data.append(pjoin(parent, f))
components = pjoin("static", "components")
# select the components we actually need to install
# (there are lots of resources we bundle for sdist-reasons that we don't actually use)
static_data.extend([
pjoin(components, "backbone", "backbone-min.js"),
pjoin(components, "bootstrap", "bootstrap", "js", "bootstrap.min.js"),
pjoin(components, "bootstrap-tour", "build", "css",
"bootstrap-tour.min.css"),
pjoin(components, "bootstrap-tour", "build", "js",
"bootstrap-tour.min.js"),
pjoin(components, "font-awesome", "font", "*.*"),
pjoin(components, "google-caja", "html-css-sanitizer-minified.js"),
pjoin(components, "highlight.js", "build", "highlight.pack.js"),
pjoin(components, "jquery", "jquery.min.js"),
pjoin(components, "jquery-ui", "ui", "minified", "jquery-ui.min.js"),
pjoin(components, "jquery-ui", "themes", "smoothness",
"jquery-ui.min.css"),
pjoin(components, "marked", "lib", "marked.js"),
pjoin(components, "requirejs", "require.js"),
pjoin(components, "underscore", "underscore-min.js"),
])
# Ship all of Codemirror's CSS and JS
for parent, dirs, files in os.walk(pjoin(components, 'codemirror')):
for f in files:
if f.endswith(('.js', '.css')):
static_data.append(pjoin(parent, f))
os.chdir(os.path.join('tests', ))
js_tests = glob('*.js') + glob('*/*.js')
os.chdir(os.path.join(cwd, 'IPython', 'nbconvert'))
nbconvert_templates = [
os.path.join(dirpath, '*.*') for dirpath, _, _ in os.walk('templates')
]
os.chdir(cwd)
package_data = {
'IPython.config.profile': ['README*', '*/*.py'],
'IPython.core.tests': ['*.png', '*.jpg'],
'IPython.lib.tests': ['*.wav'],
'IPython.testing.plugin': ['*.txt'],
'IPython.html': ['templates/*'] + static_data,
'IPython.html.tests':
js_tests,
'IPython.qt.console': ['resources/icon/*.svg'],
'IPython.nbconvert':
nbconvert_templates + ['tests/files/*.*', 'exporters/tests/files/*.*'],
'IPython.nbconvert.filters': ['marked.js'],
'IPython.nbformat': ['tests/*.ipynb']
}
return package_data
print("----This is the layer info template ----")
print(layer_info)
except:
print("Unexpected error:", sys.exc_info()[0])
#-------------------------------------------------------
# Get the list of all available csv files in wide format
#-------------------------------------------------------
wide_files = []
listOfFiles = os.listdir(data_dir)
pattern = '*_wide.csv'
for entry in listOfFiles:
if fnmatch.fnmatch(entry, pattern):
wide_files.append(entry)
#============================================
# START PROCESSING EACH DATA SERIES
#============================================
# Set parameters:
property_update_only = False
update_symbology = True
wide_files = ['1.1.1-SI_POV_DAY1_wide.csv']
for i in range(len(wide_files)):
#----------------------------------------
def get_filelist(run_module_names):
"""
Go through all supplied search directories and assembly a master
list of files to search. Then fire search functions for each file.
"""
# Prep search patterns
spatterns = [{}, {}, {}, {}, {}, {}, {}]
epatterns = [{}, {}]
ignored_patterns = []
for key, sps in config.sp.items():
mod_name = key.split('/', 1)[0]
if mod_name.lower() not in [m.lower() for m in run_module_names]:
ignored_patterns.append(key)
continue
files[key] = list()
if not isinstance(sps, list):
sps = [sps]
# Warn if we have any unrecognised search pattern keys
expected_sp_keys = [
'fn', 'fn_re', 'contents', 'contents_re', 'num_lines', 'shared',
'max_filesize', 'exclude_fn', 'exclude_fn_re', 'exclude_contents',
'exclude_contents_re'
]
unrecognised_keys = [
y for x in sps for y in x.keys() if y not in expected_sp_keys
]
if len(unrecognised_keys) > 0:
logger.warn("Unrecognised search pattern keys for '{}': {}".format(
key, ', '.join(unrecognised_keys)))
# Split search patterns according to speed of execution.
if any([x for x in sps if 'contents_re' in x]):
if any([x for x in sps if 'num_lines' in x]):
spatterns[4][key] = sps
elif any([x for x in sps if 'max_filesize' in x]):
spatterns[5][key] = sps
else:
spatterns[6][key] = sps
elif any([x for x in sps if 'contents' in x]):
if any([x for x in sps if 'num_lines' in x]):
spatterns[1][key] = sps
elif any([x for x in sps if 'max_filesize' in x]):
spatterns[2][key] = sps
else:
spatterns[3][key] = sps
else:
spatterns[0][key] = sps
if len(ignored_patterns) > 0:
logger.debug(
"Ignored search patterns as didn't match running modules: {}".
format(', '.join(ignored_patterns)))
def add_file(fn, root):
"""
Function applied to each file found when walking the analysis
directories. Runs through all search patterns and returns True
if a match is found.
"""
f = {'fn': fn, 'root': root}
# Check that this is a file and not a pipe or anything weird
if not os.path.isfile(os.path.join(root, fn)):
return None
# Check that we don't want to ignore this file
i_matches = [
n for n in config.fn_ignore_files if fnmatch.fnmatch(fn, n)
]
if len(i_matches) > 0:
logger.debug(
"Ignoring file as matched an ignore pattern: {}".format(fn))
return None
# Limit search to small files, to avoid 30GB FastQ files etc.
try:
f['filesize'] = os.path.getsize(os.path.join(root, fn))
except (IOError, OSError, ValueError, UnicodeDecodeError):
logger.debug(
"Couldn't read file when checking filesize: {}".format(fn))
else:
if f['filesize'] > config.log_filesize_limit:
return False
# Test file for each search pattern
for patterns in spatterns:
for key, sps in patterns.items():
for sp in sps:
if search_file(sp, f):
# Check that we shouldn't exclude this file
if not exclude_file(sp, f):
# Looks good! Remember this file
files[key].append(f)
# Don't keep searching this file for other modules
if not sp.get('shared', False):
return
# Don't look at other patterns for this module
else:
break
# Go through the analysis directories and get file list
for path in config.analysis_dir:
if os.path.islink(path) and config.ignore_symlinks:
continue
elif os.path.isfile(path):
searchfiles.append([os.path.basename(path), os.path.dirname(path)])
elif os.path.isdir(path):
for root, dirnames, filenames in os.walk(
path, followlinks=(not config.ignore_symlinks),
topdown=True):
bname = os.path.basename(root)
# Skip any sub-directories matching ignore params
orig_dirnames = dirnames[:]
for n in config.fn_ignore_dirs:
dirnames[:] = [
d for d in dirnames
if not fnmatch.fnmatch(d, n.rstrip(os.sep))
]
if len(orig_dirnames) != len(dirnames):
removed_dirs = [
os.path.join(root, d)
for d in set(orig_dirnames).symmetric_difference(
set(dirnames))
]
logger.debug(
"Ignoring directory as matched fn_ignore_dirs: {}".
format(", ".join(removed_dirs)))
orig_dirnames = dirnames[:]
for n in config.fn_ignore_paths:
dirnames[:] = [
d for d in dirnames if not fnmatch.fnmatch(
os.path.join(root, d), n.rstrip(os.sep))
]
if len(orig_dirnames) != len(dirnames):
removed_dirs = [
os.path.join(root, d)
for d in set(orig_dirnames).symmetric_difference(
set(dirnames))
]
logger.debug(
"Ignoring directory as matched fn_ignore_paths: {}"
.format(", ".join(removed_dirs)))
# Skip *this* directory if matches ignore params
d_matches = [
n for n in config.fn_ignore_dirs
if fnmatch.fnmatch(bname, n.rstrip(os.sep))
]
if len(d_matches) > 0:
logger.debug(
"Ignoring directory as matched fn_ignore_dirs: {}".
format(bname))
continue
p_matches = [
n for n in config.fn_ignore_paths
if fnmatch.fnmatch(root, n.rstrip(os.sep))
]
if len(p_matches) > 0:
logger.debug(
"Ignoring directory as matched fn_ignore_paths: {}".
format(root))
continue
# Search filenames in this directory
for fn in filenames:
searchfiles.append([fn, root])
# Search through collected files
with click.progressbar(searchfiles,
label="Searching {} files..".format(
len(searchfiles))) as sfiles:
for sf in sfiles:
add_file(sf[0], sf[1])
def assemble_files_to_ship(complete_file_list):
"""
This looks for all files which should be shipped in the sdist
"""
# All files which are in the repository except these:
ignore_patterns = (
# Developer-only tools
'.github/*',
'.github/*/*',
'changelogs/fragments/*',
'hacking/backport/*',
'hacking/shippable/*',
'hacking/tests/*',
'hacking/ticket_stubs/*',
'test/sanity/code-smell/botmeta.*',
'test/utils/*',
'test/utils/*/*',
'test/utils/*/*/*',
'.git*',
)
ignore_files = frozenset((
# Developer-only tools
'changelogs/config.yaml',
'changelogs/.changes.yaml',
'hacking/README.md',
'hacking/ansible-profile',
'hacking/cgroup_perf_recap_graph.py',
'hacking/create_deprecated_issues.py',
'hacking/deprecated_issue_template.md',
'hacking/fix_test_syntax.py',
'hacking/get_library.py',
'hacking/metadata-tool.py',
'hacking/report.py',
'hacking/return_skeleton_generator.py',
'hacking/test-module',
'hacking/test-module.py',
'test/support/README.md',
'.cherry_picker.toml',
'.mailmap',
# Possibly should be included
'examples/scripts/uptime.py',
'examples/DOCUMENTATION.yml',
'examples/play.yml',
'examples/hosts.yaml',
'examples/hosts.yml',
'examples/inventory_script_schema.json',
'examples/plugin_filters.yml',
'hacking/env-setup',
'hacking/env-setup.fish',
'MANIFEST',
))
# These files are generated and then intentionally added to the sdist
# Manpages
manpages = ['docs/man/man1/ansible.1']
for dirname, dummy, files in os.walk('bin'):
for filename in files:
path = os.path.join(dirname, filename)
if os.path.islink(path):
if os.readlink(path) == 'ansible':
manpages.append('docs/man/man1/%s.1' % filename)
# Misc
misc_generated_files = [
'SYMLINK_CACHE.json',
'PKG-INFO',
]
shipped_files = manpages + misc_generated_files
for path in complete_file_list:
if path not in ignore_files:
for ignore in ignore_patterns:
if fnmatch.fnmatch(path, ignore):
break
else:
shipped_files.append(path)
return shipped_files
def include(module):
return not any(fnmatch.fnmatch(module, pat)
for pat in excluded_modules)
def ignore_file_filter(path):
for exc in ignore_file_pattern:
if fnmatch.fnmatch(path, exc):
return False
return True
def __init__(self):
BASE_DIR = "/sys/bus/usb/drivers/robotic_arm/"
for file in os.listdir(BASE_DIR):
if fnmatch.fnmatch(file, '*:*'):
self.file = file
self.base_motor = open("%s%s/basemotor" % (BASE_DIR, self.file), "w")
self.arm_motor = open("%s%s/motor4" % (BASE_DIR, self.file), "w")
self.elbow_motor = open("%s%s/motor3" % (BASE_DIR, self.file), "w")
self.wrist_motor = open("%s%s/motor2" % (BASE_DIR, self.file), "w")
self.jaw_motor = open("%s%s/gripmotor" % (BASE_DIR, self.file), "w")
self.led_device = open("%s%s/led" % (BASE_DIR, self.file), "w")
Gtk.Window.__init__(self, title="Robotic Arm Manual Control Program")
self.connect("delete_event", self.delete_event)
self.connect("destroy", self.destroy)
self.set_border_width(10)
self.base_label = Gtk.Label("Base: ")
self.base_label.set_alignment(0, 0)
self.base_label.show()
self.base_clockwise = Gtk.Button("Clockwise")
self.base_clockwise.connect("clicked", self.init_base_clockwise, None)
self.base_clockwise.show()
self.base_anticlockwise = Gtk.Button("AntiClockwise")
self.base_anticlockwise.connect("clicked",
self.init_base_anticlockwise, None)
self.base_anticlockwise.show()
self.base_stop = Gtk.Button("Stop")
self.base_stop.connect("clicked", self.init_base_stop, None)
self.base_stop.show()
self.arm_label = Gtk.Label("Arm: ")
self.arm_label.set_alignment(0, 0)
self.arm_label.show()
self.arm_up = Gtk.Button("Up")
self.arm_up.connect("clicked", self.init_arm_up, None)
self.arm_up.show()
self.arm_down = Gtk.Button("Down")
self.arm_down.connect("clicked", self.init_arm_down, None)
self.arm_down.show()
self.arm_stop = Gtk.Button("Stop")
self.arm_stop.connect("clicked", self.init_arm_stop, None)
self.arm_stop.show()
self.elbow_label = Gtk.Label("Elbow: ")
self.elbow_label.set_alignment(0, 0)
self.elbow_label.show()
self.elbow_up = Gtk.Button("Up")
self.elbow_up.connect("clicked", self.init_elbow_up, None)
self.elbow_up.show()
self.elbow_down = Gtk.Button("Down")
self.elbow_down.connect("clicked", self.init_elbow_down, None)
self.elbow_down.show()
self.elbow_stop = Gtk.Button("Stop")
self.elbow_stop.connect("clicked", self.init_elbow_stop, None)
self.elbow_stop.show()
self.wrist_label = Gtk.Label("Wrist: ")
self.wrist_label.set_alignment(0, 0)
self.wrist_label.show()
self.wrist_up = Gtk.Button("Up")
self.wrist_up.connect("clicked", self.init_wrist_up, None)
self.wrist_up.show()
self.wrist_down = Gtk.Button("Down")
self.wrist_down.connect("clicked", self.init_wrist_down, None)
self.wrist_down.show()
self.wrist_stop = Gtk.Button("Stop")
self.wrist_stop.connect("clicked", self.init_wrist_stop, None)
self.wrist_stop.show()
self.jaw_label = Gtk.Label("Jaw: ")
self.jaw_label.set_alignment(0, 0)
self.jaw_label.show()
self.jaw_open = Gtk.Button("Open")
self.jaw_open.connect("clicked", self.init_jaw_open, None)
self.jaw_open.show()
self.jaw_close = Gtk.Button("Close")
self.jaw_close.connect("clicked", self.init_jaw_close, None)
self.jaw_close.show()
self.jaw_stop = Gtk.Button("Stop")
self.jaw_stop.connect("clicked", self.init_jaw_stop, None)
self.jaw_stop.show()
self.led_label = Gtk.Label("LED: ")
self.led_label.set_alignment(0, 0)
self.led_label.show()
self.led = Gtk.Button("Off")
self.led.connect("clicked", self.switch_led, None)
self.led.show()
self.controls = Gtk.Grid()
self.controls.attach(self.base_label, 0, 0, 1, 1)
self.controls.attach(self.base_clockwise, 1, 0, 1, 1)
self.controls.attach(self.base_anticlockwise, 2, 0, 1, 1)
self.controls.attach(self.base_stop, 3, 0, 1, 1)
self.controls.attach(self.arm_label, 0, 1, 1, 1)
self.controls.attach(self.arm_up, 1, 1, 1, 1)
self.controls.attach(self.arm_down, 2, 1, 1, 1)
self.controls.attach(self.arm_stop, 3, 1, 1, 1)
self.controls.attach(self.elbow_label, 0, 2, 1, 1)
self.controls.attach(self.elbow_up, 1, 2, 1, 1)
self.controls.attach(self.elbow_down, 2, 2, 1, 1)
self.controls.attach(self.elbow_stop, 3, 2, 1, 1)
self.controls.attach(self.wrist_label, 0, 3, 1, 1)
self.controls.attach(self.wrist_up, 1, 3, 1, 1)
self.controls.attach(self.wrist_down, 2, 3, 1, 1)
self.controls.attach(self.wrist_stop, 3, 3, 1, 1)
self.controls.attach(self.jaw_label, 0, 4, 1, 1)
self.controls.attach(self.jaw_open, 1, 4, 1, 1)
self.controls.attach(self.jaw_close, 2, 4, 1, 1)
self.controls.attach(self.jaw_stop, 3, 4, 1, 1)
self.controls.attach(self.led_label, 0, 5, 1, 1)
self.controls.attach(self.led, 1, 5, 3, 1)
self.controls.show()
self.add(self.controls)
self.show()
def prune_final_dir_for_clang_tidy(final_dir, osx_cross_compile):
# Make sure we only have what we expect.
dirs = [
"bin",
"include",
"lib",
"lib32",
"libexec",
"msbuild-bin",
"share",
"tools",
]
if is_linux():
dirs.append("x86_64-unknown-linux-gnu")
for f in glob.glob("%s/*" % final_dir):
if os.path.basename(f) not in dirs:
raise Exception("Found unknown file %s in the final directory" % f)
if not os.path.isdir(f):
raise Exception("Expected %s to be a directory" % f)
kept_binaries = [
"clang-apply-replacements",
"clang-format",
"clang-tidy",
"clangd",
"clang-query",
]
re_clang_tidy = re.compile(r"^(" + "|".join(kept_binaries) + r")(\.exe)?$",
re.I)
for f in glob.glob("%s/bin/*" % final_dir):
if re_clang_tidy.search(os.path.basename(f)) is None:
delete(f)
# Keep include/ intact.
# Remove the target-specific files.
if is_linux():
if os.path.exists(os.path.join(final_dir, "x86_64-unknown-linux-gnu")):
shutil.rmtree(os.path.join(final_dir, "x86_64-unknown-linux-gnu"))
# In lib/, only keep lib/clang/N.M.O/include and the LLVM shared library.
re_ver_num = re.compile(r"^\d+\.\d+\.\d+$", re.I)
for f in glob.glob("%s/lib/*" % final_dir):
name = os.path.basename(f)
if name == "clang":
continue
if osx_cross_compile and name in [
"libLLVM.dylib", "libclang-cpp.dylib"
]:
continue
if is_linux() and (fnmatch.fnmatch(name, "libLLVM*.so")
or fnmatch.fnmatch(name, "libclang-cpp.so*")):
continue
delete(f)
for f in glob.glob("%s/lib/clang/*" % final_dir):
if re_ver_num.search(os.path.basename(f)) is None:
delete(f)
for f in glob.glob("%s/lib/clang/*/*" % final_dir):
if os.path.basename(f) != "include":
delete(f)
# Completely remove libexec/, msbuild-bin and tools, if it exists.
shutil.rmtree(os.path.join(final_dir, "libexec"))
for d in ("msbuild-bin", "tools"):
d = os.path.join(final_dir, d)
if os.path.exists(d):
shutil.rmtree(d)
# In share/, only keep share/clang/*tidy*
re_clang_tidy = re.compile(r"format|tidy", re.I)
for f in glob.glob("%s/share/*" % final_dir):
if os.path.basename(f) != "clang":
delete(f)
for f in glob.glob("%s/share/clang/*" % final_dir):
if re_clang_tidy.search(os.path.basename(f)) is None:
delete(f)
def __init__(self, settings, ncpu=None):
self.settings = settings
self.logger = logging.getLogger(__name__)
self.stats = defaultdict(int)
self.init_pool(ncpu)
check_or_create_dir(settings['destination'])
# Build the list of directories with images
albums = self.albums = {}
src_path = self.settings['source']
ignore_dirs = settings['ignore_directories']
ignore_files = settings['ignore_files']
progressChars = cycle(["/", "-", "\\", "|"])
show_progress = (self.logger.getEffectiveLevel() >= logging.WARNING and
os.isatty(sys.stdout.fileno()))
self.progressbar_target = None if show_progress else Devnull()
for path, dirs, files in os.walk(src_path, followlinks=True,
topdown=False):
if show_progress:
print("\rCollecting albums " + next(progressChars), end="")
relpath = os.path.relpath(path, src_path)
# Test if the directory match the ignore_dirs settings
if ignore_dirs and any(fnmatch.fnmatch(relpath, ignore)
for ignore in ignore_dirs):
self.logger.info('Ignoring %s', relpath)
continue
# Remove files that match the ignore_files settings
if ignore_files:
files_path = {join(relpath, f) for f in files}
for ignore in ignore_files:
files_path -= set(fnmatch.filter(files_path, ignore))
self.logger.debug('Files before filtering: %r', files)
files = [os.path.split(f)[1] for f in files_path]
self.logger.debug('Files after filtering: %r', files)
# Remove sub-directories that have been ignored in a previous
# iteration (as topdown=False, sub-directories are processed before
# their parent
for d in dirs[:]:
path = join(relpath, d) if relpath != '.' else d
if path not in albums.keys():
dirs.remove(d)
album = Album(relpath, settings, dirs, files, self)
if not album.medias and not album.albums:
self.logger.info('Skip empty album: %r', album)
else:
album.create_output_directories()
albums[relpath] = album
with progressbar(albums.values(), label="%16s" % "Sorting albums",
file=self.progressbar_target) as progress_albums:
for album in progress_albums:
album.sort_subdirs(settings['albums_sort_attr'])
with progressbar(albums.values(), label="%16s" % "Sorting media",
file=self.progressbar_target) as progress_albums:
for album in progress_albums:
album.sort_medias(settings['medias_sort_attr'])
self.logger.debug('Albums:\n%r', albums.values())
signals.gallery_initialized.send(self)
def iter_filenames(paths, exclude):
exclude = filter(None, (exclude or '').split(','))
for path in walk_paths(paths):
if all(not fnmatch.fnmatch(path, pattern) for pattern in exclude):
yield path
def _matches(cls, hit):
# pylint: disable=protected-access
if cls._index._name is None:
return True
return fnmatch(hit.get('_index', ''), '{}*'.format(cls._index._name))
def __init__(self, inventory, play, play_context, variable_manager, all_vars, start_at_done=False):
self._play = play
self._blocks = []
self._variable_manager = variable_manager
self._task_uuid_cache = dict()
# Default options to gather
gather_subset = play_context.gather_subset
gather_timeout = play_context.gather_timeout
fact_path = play_context.fact_path
# Retrieve subset to gather
if self._play.gather_subset is not None:
gather_subset = self._play.gather_subset
# Retrieve timeout for gather
if self._play.gather_timeout is not None:
gather_timeout = self._play.gather_timeout
# Retrieve fact_path
if self._play.fact_path is not None:
fact_path = self._play.fact_path
setup_block = Block(play=self._play)
setup_task = Task(block=setup_block)
setup_task.action = 'setup'
setup_task.name = 'Gathering Facts'
setup_task.tags = ['always']
setup_task.args = {
'gather_subset': gather_subset,
}
if gather_timeout:
setup_task.args['gather_timeout'] = gather_timeout
if fact_path:
setup_task.args['fact_path'] = fact_path
setup_task.set_loader(self._play._loader)
# short circuit fact gathering if the entire playbook is conditional
if self._play._included_conditional is not None:
setup_task.when = self._play._included_conditional[:]
setup_block.block = [setup_task]
setup_block = setup_block.filter_tagged_tasks(play_context, all_vars)
self._blocks.append(setup_block)
self.cache_block_tasks(setup_block)
for block in self._play.compile():
new_block = block.filter_tagged_tasks(play_context, all_vars)
if new_block.has_tasks():
self.cache_block_tasks(new_block)
self._blocks.append(new_block)
for handler_block in self._play.handlers:
self.cache_block_tasks(handler_block)
self._host_states = {}
start_at_matched = False
for host in inventory.get_hosts(self._play.hosts):
self._host_states[host.name] = HostState(blocks=self._blocks)
# if we're looking to start at a specific task, iterate through
# the tasks for this host until we find the specified task
if play_context.start_at_task is not None and not start_at_done:
while True:
(s, task) = self.get_next_task_for_host(host, peek=True)
if s.run_state == self.ITERATING_COMPLETE:
break
if task.name == play_context.start_at_task or fnmatch.fnmatch(task.name, play_context.start_at_task) or \
task.get_name() == play_context.start_at_task or fnmatch.fnmatch(task.get_name(), play_context.start_at_task):
start_at_matched = True
break
else:
self.get_next_task_for_host(host)
# finally, reset the host's state to ITERATING_SETUP
if start_at_matched:
self._host_states[host.name].did_start_at_task = True
self._host_states[host.name].run_state = self.ITERATING_SETUP
if start_at_matched:
# we have our match, so clear the start_at_task field on the
# play context to flag that we've started at a task (and future
# plays won't try to advance)
play_context.start_at_task = None
def _diff(self):
if self.ignore_blank_cards:
cardsa = [c for c in self.a.cards if str(c) != BLANK_CARD]
cardsb = [c for c in self.b.cards if str(c) != BLANK_CARD]
else:
cardsa = list(self.a.cards)
cardsb = list(self.b.cards)
# build dictionaries of keyword values and comments
def get_header_values_comments(cards):
values = {}
comments = {}
for card in cards:
value = card.value
if self.ignore_blanks and isinstance(value, string_types):
value = value.rstrip()
values.setdefault(card.keyword, []).append(value)
comments.setdefault(card.keyword, []).append(card.comment)
return values, comments
valuesa, commentsa = get_header_values_comments(cardsa)
valuesb, commentsb = get_header_values_comments(cardsb)
# Normalize all keyword to upper-case for comparison's sake;
# TODO: HIERARCH keywords should be handled case-sensitively I think
keywordsa = set(k.upper() for k in valuesa)
keywordsb = set(k.upper() for k in valuesb)
self.common_keywords = sorted(keywordsa.intersection(keywordsb))
if len(cardsa) != len(cardsb):
self.diff_keyword_count = (len(cardsa), len(cardsb))
# Any other diff attributes should exclude ignored keywords
keywordsa = keywordsa.difference(self.ignore_keywords)
keywordsb = keywordsb.difference(self.ignore_keywords)
if self.ignore_keyword_patterns:
for pattern in self.ignore_keyword_patterns:
keywordsa = keywordsa.difference(fnmatch.filter(keywordsa,
pattern))
keywordsb = keywordsb.difference(fnmatch.filter(keywordsb,
pattern))
if '*' in self.ignore_keywords:
# Any other differences between keywords are to be ignored
return
left_only_keywords = sorted(keywordsa.difference(keywordsb))
right_only_keywords = sorted(keywordsb.difference(keywordsa))
if left_only_keywords or right_only_keywords:
self.diff_keywords = (left_only_keywords, right_only_keywords)
# Compare count of each common keyword
for keyword in self.common_keywords:
if keyword in self.ignore_keywords:
continue
if self.ignore_keyword_patterns:
skip = False
for pattern in self.ignore_keyword_patterns:
if fnmatch.fnmatch(keyword, pattern):
skip = True
break
if skip:
continue
counta = len(valuesa[keyword])
countb = len(valuesb[keyword])
if counta != countb:
self.diff_duplicate_keywords[keyword] = (counta, countb)
# Compare keywords' values and comments
for a, b in zip(valuesa[keyword], valuesb[keyword]):
if diff_values(a, b, tolerance=self.tolerance):
self.diff_keyword_values[keyword].append((a, b))
else:
# If there are duplicate keywords we need to be able to
# index each duplicate; if the values of a duplicate
# are identical use None here
self.diff_keyword_values[keyword].append(None)
if not any(self.diff_keyword_values[keyword]):
# No differences found; delete the array of Nones
del self.diff_keyword_values[keyword]
if '*' in self.ignore_comments or keyword in self.ignore_comments:
continue
if self.ignore_comment_patterns:
skip = False
for pattern in self.ignore_comment_patterns:
if fnmatch.fnmatch(keyword, pattern):
skip = True
break
if skip:
continue
for a, b in zip(commentsa[keyword], commentsb[keyword]):
if diff_values(a, b):
self.diff_keyword_comments[keyword].append((a, b))
else:
self.diff_keyword_comments[keyword].append(None)
if not any(self.diff_keyword_comments[keyword]):
del self.diff_keyword_comments[keyword]
def find_FEfiles(self):
'''this reads the list created above containing filename pattern and replaces this with the full file name'''
# for each row of the text file split into fields
for i in self.files_to_find:
file1_pattern = i[0]
file1_dye = i[1]
file2_pattern = i[2]
file2_dye = i[3]
out_file_prefix=i[4]
# set filename as none to help identify when no match has been found below
file1_filename = None
file2_filename = None
# search for a FE file which matches the filename pattern
for afile in os.listdir(self.chosenfolder):
# file 1
if fnmatch.fnmatch(afile, file1_pattern):
file1_filename = afile
else:
pass
# file 2
if fnmatch.fnmatch(afile, file2_pattern):
file2_filename = afile
else:
pass
# if both files have been identified add this to a new list else report.
if file1_filename is not None and file2_filename is not None:
self.list_of_files.append((file1_filename, file1_dye, file2_filename, file2_dye,out_file_prefix))
else:
raise ValueError("no match for " + file1_pattern + " and " + file2_pattern)
