def reduce_python(self):
print("Reduce python")
oldpwd = os.getcwd()
try:
print("Remove files unlikely to be used")
os.chdir(join(self.ctx.dist_dir, "root", "python"))
sh.rm("-rf", "share")
sh.rm("-rf", "bin")
os.chdir(join(self.ctx.dist_dir, "root", "python", "lib"))
sh.rm("-rf", "pkgconfig")
sh.rm("libpython2.7.a")
os.chdir(
join(self.ctx.dist_dir, "root", "python", "lib", "python2.7"))
sh.find(".", "-iname", "*.pyc", "-exec", "rm", "{}", ";")
sh.find(".", "-iname", "*.py", "-exec", "rm", "{}", ";")
#sh.find(".", "-iname", "test*", "-exec", "rm", "-rf", "{}", ";")
sh.rm("-rf", "wsgiref", "bsddb", "curses", "idlelib", "hotshot")
sh.rm("-rf", sh.glob("lib*"))
# now create the zip.
print("Create a python27.zip")
sh.rm("config/libpython2.7.a")
sh.rm("config/python.o")
sh.rm("config/config.c.in")
sh.rm("config/makesetup")
sh.rm("config/install-sh")
sh.mv("config", "..")
sh.mv("site-packages", "..")
sh.zip("-r", "../python27.zip", sh.glob("*"))
sh.rm("-rf", sh.glob("*"))
sh.mv("../config", ".")
sh.mv("../site-packages", ".")
finally:
os.chdir(oldpwd)
def compile_contig_matrices(names = None):
if names == None:
fs = sh.find(raw_ass, "-name", "*-smds.coverage.percontig").stdout.split("\n")[:-1]
fs = [f for f in fs if "_2ref" in f]
else:
fs = sh.find(merged_ass + names, "-name", "*-smds.coverage.percontig").stdout.split("\n")[:-1]
df = DataFrame.from_csv(fs[0],sep="\t")
glob = df.ix[:,0:2]
for f in fs:
if names == None:
id = [c for c in f.split("/") if "IH" in c][0]
else:
id = [c.replace("map_","") for c in f.split("/") if "map_" in c][0]
values = DataFrame.from_csv(f,sep="\t")["cov_mean_sample_0"]
assert sum([a!=b for a,b in zip(values.index, glob.index)]) == 0
if sum([a!=b for a,b in zip(values.index, glob.index)]) == 0:
glob[id] = values
else:
print f, "is weird"
if names == None:
glob.to_csv(stats_out + "all_contig_coverages.csv",sep="\t")
else:
glob.to_csv(stats_out + names + "_contig_coverages.csv",sep="\t")
glob[samples].to_csv(stats_out + self.name + "_contig_coverages_for_concoct.csv",sep="\t")
def reduce_python(self):
print("Reduce python")
oldpwd = os.getcwd()
try:
print("Remove files unlikely to be used")
os.chdir(join(self.ctx.dist_dir, "root", "python"))
sh.rm("-rf", "share")
sh.rm("-rf", "bin")
os.chdir(join(self.ctx.dist_dir, "root", "python", "lib"))
sh.rm("-rf", "pkgconfig")
sh.rm("libpython2.7.a")
os.chdir(join(self.ctx.dist_dir, "root", "python", "lib", "python2.7"))
sh.find(".", "-iname", "*.pyc", "-exec", "rm", "{}", ";")
sh.find(".", "-iname", "*.py", "-exec", "rm", "{}", ";")
#sh.find(".", "-iname", "test*", "-exec", "rm", "-rf", "{}", ";")
sh.rm("-rf", "wsgiref", "bsddb", "curses", "idlelib", "hotshot")
sh.rm("-rf", sh.glob("lib*"))
# now create the zip.
print("Create a python27.zip")
sh.rm("config/libpython2.7.a")
sh.rm("config/python.o")
sh.rm("config/config.c.in")
sh.rm("config/makesetup")
sh.rm("config/install-sh")
sh.mv("config", "..")
sh.mv("site-packages", "..")
sh.zip("-r", "../python27.zip", sh.glob("*"))
sh.rm("-rf", sh.glob("*"))
sh.mv("../config", ".")
sh.mv("../site-packages", ".")
finally:
os.chdir(oldpwd)
def run(self):
self.log("Removing .pyc files...")
files = find(self._basedir, "-name", "*.pyc").strip().splitlines()
for f in files:
rm(f)
files = find(self._basedir, "-name", "*\\.so*").strip().splitlines()
for f in files:
self.log("Stripping {0}".format(f))
try:
strip(f)
except:
pass
self.log("Done")
def init(template, project_name):
templates = get_templates()
if template not in templates:
click.echo('%s template not found' % template, err=True)
return
project_dir = f'./{project_name}'
sh.mkdir('-p', project_dir)
sh.cp('-rf', os.path.join(HERE, f'templates/{template}/'), project_dir)
for f in sh.find(project_dir, '-name', '*.sh'):
sh.sed('-i', '', '-e', f's/proj/{project_name}/g', f.strip())
for f in sh.find(project_dir, '-name', '*.yaml'):
sh.sed('-i', '', '-e', f's/proj/{project_name}/g', f.strip())
for f in sh.find(project_dir, '-name', 'Dockerfile*'):
sh.sed('-i', '', '-e', f's/proj/{project_name}/g', f.strip())
def fetch_git_repo(self, repo):
cp = os.path.join(self.checkouts_path, os.path.basename(repo))
if not os.path.exists(cp):
logger.debug('clone %s' % repo)
pygit.Repo.clone_from(repo, cp)
# find galaxy.yml file(s)
res = find(cp, '-name', 'galaxy.yml')
galaxy_files = res.stdout.decode('utf-8').split('\n')
galaxy_files = [x.strip() for x in galaxy_files if x.strip()]
for gf in galaxy_files:
logger.debug(gf)
with open(gf, 'r') as f:
gdata = yaml.load(f.read())
ckey = (gdata['namespace'], gdata['name'])
logger.debug('found fqn: %s.%s' % ckey)
if ckey not in self.collections:
self.collections[ckey] = {
'name': ckey[1],
'namespace': {
'name': ckey[0]
},
'href': repo,
'plugins': {},
'filepath': os.path.dirname(gf)
}
self.index_plugins_in_collection(os.path.dirname(gf), ckey)
res = find(cp, '-name', 'MANIFEST.json')
manifest_files = res.stdout.decode('utf-8').split('\n')
manifest_files = [x.strip() for x in manifest_files if x.strip()]
for mf in manifest_files:
logger.debug(mf)
with open(mf, 'r') as f:
mdata = json.loads(f.read())
ckey = (mdata['collection_info']['namespace'],
mdata['collection_info']['name'])
logger.debug('found fqn: %s.%s' % ckey)
if ckey not in self.collections:
self.collections[ckey] = {
'name': ckey[1],
'namespace': {
'name': ckey[0]
},
'href': repo,
'plugins': {},
'filepath': os.path.dirname(mf)
}
self.index_plugins_in_collection(os.path.dirname(mf), ckey)
def build_level_database(input_path):
print "Generating or regenerating level database."
print "This may take a long time if a lot of files need to be scanned."
paths = [p.strip() for p in find(input_path)]
levels = []
pool = Pool(processes=4)
ratio = 100.0 / len(paths)
processed = 0
last_percent = 0
for data in pool.imap_unordered(process_level, paths):
processed += 1
percent = int(processed * ratio)
if percent > last_percent:
last_percent = percent
print "... {}%".format(percent)
if not data:
continue
levels.append(data)
db = {
"levels": levels,
"version": DATABASE_VERSION,
}
with open("level_db.pickle", "w") as outfile:
pickle.dump(db, outfile)
def build_level_database(input_path):
print "Generating or regenerating level database."
print "This may take a long time if a lot of files need to be scanned."
paths = [p.strip() for p in find(input_path)]
levels = []
pool = Pool(processes=4)
ratio = 100.0 / len(paths)
processed = 0
last_percent = 0
for data in pool.imap_unordered(process_level, paths):
processed += 1
percent = int(processed * ratio)
if percent > last_percent:
last_percent = percent
print "... {}%".format(percent)
if not data:
continue
levels.append(data)
db = {
"levels" : levels,
"version" : DATABASE_VERSION,
}
with open("level_db.pickle", "w") as outfile:
pickle.dump(db, outfile)
def get(self):
query = unquote(self.get_argument('q'))
try:
results = str(
grep('-R', '--exclude-dir', '.git', query, self.settings.repo))
except ErrorReturnCode_1 as e:
results = ''
# TODO filter out duplicates if the filename is already in the search results
# TODO this doesn't exclude the .git folder
try:
results += str(
find(self.settings.repo, '-type', 'f', '-name',
'*' + query + '*'))
except ErrorReturnCode_1 as e:
pass
results = results.replace(self.settings.repo, '').split('\n')[:-1]
formatted_results = []
for result in results:
if 'Binary file' in result:
continue
# TODO this doesn't play well with colons in filenames
stuff = result.split(':')
filename = stuff[0]
if path.basename(filename).startswith('.'):
filename = path.join(path.dirname(filename),
path.basename(filename)[1:])
string = ''.join(stuff[1:])
string = self._highlight(string, query)
formatted_results.append({'filename': filename, 'string': string})
self.render('search.html', query=query, results=formatted_results)
def generate_image_hash(self):
"""Generate hash of Docker context
- Recursively list all files
- Get sha1sum of each file in list
- Sort list (ensures consistency from run to run)
- Get final sha1sum of list
"""
for context_dir in self.docker_context_cm():
sha1sum = sh.sha1sum(
sh.xargs(
sh.sort(
sh.find(
'.', '-type', 'f', '-print0',
_err=sys.stderr.buffer,
_cwd=context_dir),
'-z',
# Locale differences can affect sort order
_env = {**os.environ, 'LC_ALL': 'C.UTF-8'},
_err=sys.stderr.buffer),
'-0', 'sha1sum',
_err=sys.stderr.buffer,
_cwd=context_dir),
_err=sys.stderr.buffer).split()[0]
return sha1sum
def get(self):
query = unquote(self.get_argument('q'))
try:
results = str(grep('-R', '--exclude-dir', '.git', query,
self.settings.repo))
except ErrorReturnCode_1 as e:
results = ''
try:
results += str(find(self.settings.repo, '-type', 'f', '-name',
'*' + query + '*', '-not', '(', '-path',
'%s/%s/*' % (self.settings.repo, '.git') ))
except ErrorReturnCode_1 as e:
pass
results = results.replace(self.settings.repo, '').split('\n')[:-1]
formatted_results = []
for result in results:
if 'Binary file' in result or result == '':
continue
# TODO this doesn't play well with colons in filenames
stuff = result.split(':')
filename = stuff[0]
if path.basename(filename).startswith('.'):
filename = path.join(path.dirname(filename),
path.basename(filename)[1:])
string = ''.join(stuff[1:])
string = self._highlight(string, query)
formatted_results.append({'filename': filename, 'string': string})
self.render('search.html', query=query, results=formatted_results)
def get_test_data():
docs = []
for f in find("examples"):
f = f.strip()
if not f.endswith(".txt"):
continue
elif "animasi_2d" in f:
category = "animasi_2d"
elif "animasi_3d" in f:
category = "animasi_3d"
elif "desain_multimedia" in f:
category = "desain_multimedia"
elif "basis_data" in f:
category = "basis_data"
elif "pemodelan_perangkat_lunak" in f:
category = "pemodelan_perangkat_lunak"
elif "pemrograman_dasar" in f:
category = "pemrograman_dasar"
elif "komputer_terapan_jaringan" in f:
category = "komputer_terapan_jaringan"
elif "komunikasi_data" in f:
category = "komunikasi_data"
else:
category = "sistem_operasi_jaringan"
docs.append((category, f))
return render_template('index.html', data_tests = docs)
def get(self):
query = unquote(self.get_argument('q', ''))
if query == '':
self.redirect('/')
try:
results = str(
grep('-R', '--exclude-dir', '.git', query, self.settings.repo))
except ErrorReturnCode_1 as e:
results = ''
try:
results += str(
find(self.settings.repo, '-type', 'f', '-name',
'*' + query + '*', '-not', '(', '-path',
'%s/%s/*' % (self.settings.repo, '.git')))
except ErrorReturnCode_1 as e:
pass
results = results.replace(self.settings.repo, '').split('\n')[:-1]
formatted_results = []
for result in results:
if 'Binary file' in result or result == '':
continue
# TODO this doesn't play well with colons in filenames
stuff = result.split(':')
filename = stuff[0]
if path.basename(filename).startswith('.'):
filename = path.join(path.dirname(filename),
path.basename(filename)[1:])
string = ''.join(stuff[1:])
string = self.highlight(string, query)
formatted_results.append({'filename': filename, 'string': string})
self.render('search.html', query=query, results=formatted_results)
def _traverse_corpus(path):
for fname in sh.find(path, '-type', 'f'):
with open(fname.strip(), 'r') as fin:
corpus_name = '%s.%s' % (os.path.basename(
os.path.dirname(fname)), os.path.basename(fname))
sentence_no = 1
word_no = 1
sentence = []
metadata = {'META': corpus_name, 'sentence': sentence_no}
for line in tqdm(fin):
if line.strip() == '':
yield Sentence(metadata, sentence, 'idx', 'token', 'lemma',
'tag')
sentence_no += 1
word_no = 1
sentence = []
metadata = {'META': corpus_name, 'sentence': sentence_no}
else:
try:
token, lemma, tag, _ = unicodedata.normalize(
'NFC', line).strip().split()
if tag.startswith('F'):
token = '<PUNCTUATION>'
elif tag.startswith('W'):
token = '<DATE>'
elif tag.startswith('Z'):
token = '<NUMBER>'
sentence.append('%d\t%s\t%s\t%s' %
(word_no, token, lemma, tag))
word_no += 1
except ValueError:
continue
def create(template, project_name):
templates = get_templates()
if template not in templates:
click.echo('%s template not found' % template, err=True)
return
project_dir = './' + project_name
sh.mkdir('-p', project_dir)
sh.cp('-rf', os.path.join(HERE, f'templates/{template}/'), project_dir)
for f in sh.find(project_dir, '-name', '*.py'):
sh.sed('-i', '', '-e', 's/%s/%s/g' % ('proj', project_name), f.strip())
for f in sh.find(project_dir, '-name', 'Dockerfile*'):
sh.sed('-i', '', '-e', 's/%s/%s/g' % ('proj', project_name), f.strip())
# hack: custom model import
if template == 'simple':
sh.sed('-i', '', '-e', 's/%s/%s/g' % ('proj', project_name),
'./%s/migrations/script.py.mako' % project_name)
sh.mv(os.path.join(project_dir, 'proj'),
os.path.join(project_dir, project_name))
def find_files(in_dir):
"""
returns a list of the sequence files in a directory recursively
"""
FASTQ_EXTENSIONS = [".fq", ".fastq"]
files = [sh.find(in_dir, "-name", "*" + x) for x in FASTQ_EXTENSIONS]
return files
def _lookup_snapshots(tag):
try:
dirs = sh.find(CASSANDRA_DATA_DIR, '-name', tag)
except:
logger.warn('Unable to execute find, nodetool did not create ' +
'snapshot?')
dirs = ''
return dirs.splitlines()
def get_children(self):
if not self.output_path:
path = self.path
else:
path = self.output_path
#log.debug("Getting children from path {0}".format(path))
return str(
sh.find(path, "-maxdepth", "1", "!", "-name", "lost+found").stdout,
'utf8').split("\n")[1:-1]
def find_files(in_dir):
"""
returns a list of the sequence files in a directory recursively
"""
FASTQ_EXTENSIONS = [".fq", ".fastq"]
files = [sh.find(in_dir, "-name", "*" + x) for x in FASTQ_EXTENSIONS]
return files
def _lookup_snapshots(self, tag):
try:
dirs = sh.find(self.cassandra_data_dir, '-name', tag)
except:
logger.warning(
'Unable to execute find, nodetool did not create snapshot?')
dirs = ''
return dirs.splitlines()
def enqueue(hash_or_path):
level = find(hash_or_path)
if level:
level_hash = level["level_hash"]
try:
unprocessed.remove(level_hash)
except:
pass
if level_hash not in queue and level_hash not in visited:
queue.append(level_hash)
def run(self):
self.log("Removing unused python code...")
test_dirs = find(self._basedir, "-name", "*test*").strip().splitlines()
for td in test_dirs:
rm("-rf", os.path.join(self._basedir, td))
# twisted_used = ["aplication", "conch", "cred",
# "version", "internet", "mail"]
# twisted_files = find(self._basedir, "-name", "t
self.log("Done")
def enqueue(hash_or_path):
level = find(hash_or_path)
if level:
level_hash = level["level_hash"]
try:
unprocessed.remove(level_hash)
except:
pass
if level_hash not in queue and level_hash not in visited:
queue.append(level_hash)
def setupConfigDirs(dirname, fullPrimaryEmail, primaryEmail, name, uid):
basedir = settings.SCRIPT_DIR + "/kontact"
print(basedir);
try:
sh.rm("-R", )
except:
print("nothing to remove")
sh.cp("-R", "{}/config".format(basedir), "{}/{}".format(basedir, dirname))
sh.find("{}/{}".format(basedir, dirname), "-type", "f", "-exec", "sed", "-i", "s/{fullPrimaryEmail}/" + fullPrimaryEmail + "/g", "{}", "+")
sh.find("{}/{}".format(basedir, dirname), "-type", "f", "-exec", "sed", "-i", "s/{primaryEmail}/" + primaryEmail + "/g", "{}", "+")
sh.find("{}/{}".format(basedir, dirname), "-type", "f", "-exec", "sed", "-i", "s/{name}/" + name + "/g", "{}", "+")
sh.find("{}/{}".format(basedir, dirname), "-type", "f", "-exec", "sed", "-i", "s/{uid}/" + uid + "/g", "{}", "+")
def sync_lychee(complete_sync=False):
log.info("Starting Lychee synchronization")
if complete_sync:
# Delete broken links
log.info("Removing broken symlinks for Lychee")
sh.find(LYCHEE_DATA_PATH, "-xtype", "l", "-delete")
exclusive_mode = 'replace'
else:
exclusive_mode = 'normal'
try:
perform_sync(False, exclusive_mode, True, False, True, False,
BACKUP_PATH, LYCHEE_DATA_PATH, LYCHEESYNC_CONF_FILE)
except Exception as e:
log.exception("Unable to perform Lychee synchronization")
# Flush disk buffers
sh.sync()
log.info("Finished Lychee synchronization")
def update_algorithm(algo, original_name, algorithm_pairs, remote_client,
workspace_path, artifact_path):
api_key = remote_client.apiKey
api_address = remote_client.apiAddress
destination_algorithm_name = algo.algoname
destination_username = algo.username
templatable_username = "******"
repo_path = f"{workspace_path}/{destination_algorithm_name}"
git_path = f"https://{algo.username}:{api_key}@git.{api_address.split('https://api.')[-1]}/git/{destination_username}/{destination_algorithm_name}.git"
os.makedirs(artifact_path, exist_ok=True)
os.makedirs(repo_path, exist_ok=True)
clone_bake = sh.git.bake(C=workspace_path)
publish_bake = sh.git.bake(C=repo_path)
clone_bake.clone(git_path)
sh.rm("-r", f"{repo_path}/src")
sh.cp("-R", f"{artifact_path}/src", f"{repo_path}/src")
sh.cp("-R", f"{artifact_path}/requirements.txt",
f"{repo_path}/requirements.txt")
sh.xargs.sed(sh.find(repo_path, "-not", "-path", "*/\.*", "-type", "f"),
i=f"s/{templatable_username}/{destination_username}/g")
for template_name, new_name in algorithm_pairs:
sh.xargs.sed(sh.find(f"{repo_path}/src", "-not", "-path", "*/\.*",
"-type", "f"),
i=f"s/{template_name}/{new_name}/g")
sh.mv(f"{repo_path}/src/{original_name}.py",
f"{repo_path}/src/{destination_algorithm_name}.py")
sh.mv(f"{repo_path}/src/{original_name}_test.py",
f"{repo_path}/src/{destination_algorithm_name}_test.py")
try:
publish_bake.add(".")
publish_bake.commit(m="automatic initialization commit")
publish_bake.push()
return algo
except Exception as e:
if "Your branch is up to date with" in str(e):
print(
f"algorithm {destination_username}/{destination_algorithm_name} is already up to date, skipping update..."
)
pass
else:
raise e
def create(template, project_name):
templates = get_templates()
if template not in templates:
click.echo('%s template not found' % template, err=True)
return
project_dir = './' + project_name
sh.mkdir('-p', project_dir)
sh.cp('-rf', os.path.join(HERE, 'templates/simple/'), project_dir)
for f in sh.find(project_dir, '-name', '*.py'):
sh.sed('-i', '', '-e', 's/%s/%s/g' % ('proj', project_name), f.strip())
sh.mv(os.path.join(project_dir, 'proj'),
os.path.join(project_dir, project_name))
def _test(self, media_dir, mtimes, expected_structure):
self._set_mtimes(media_dir, mtimes)
# run organize.py on input dir
shutil.rmtree(ORG_PATH, ignore_errors=True)
os.system("python3 organize.py " + media_dir + " " + ORG_PATH)
# test output dir
self.assertTrue(os.path.isdir(ORG_PATH))
self.assertListEqual(sorted(sh.find('.', _cwd=ORG_PATH).splitlines()),
expected_structure)
def edge_search(level):
# this attempts to find near by levels to group together
if not level["edges"]:
return None
right_of = find(level["edges"][1])
left_of = find(level["edges"][3])
east, west = None, None
if right_of:
east = right_of
west = level
elif left_of:
east = level
west = left_of
else:
return None
ne_level, nw_level, se_level, sw_level = None, None, None, None
if east["edges"][2] and west["edges"][2]:
ne_level, nw_level = east, west
se_level = find(east["edges"][2])
sw_level = find(west["edges"][2])
elif east["edges"][0] and west["edges"][0]:
se_level, sw_level = east, west
ne_level = find(east["edges"][0])
nw_level = find(west["edges"][0])
if ne_level and nw_level and se_level and sw_level:
return (nw_level, ne_level, sw_level, se_level)
else:
return (west, east)
def edge_search(level):
# this attempts to find near by levels to group together
if not level["edges"]:
return None
right_of = find(level["edges"][1])
left_of = find(level["edges"][3])
east, west = None, None
if right_of:
east = right_of
west = level
elif left_of:
east = level
west = left_of
else:
return None
ne_level, nw_level, se_level, sw_level = None, None, None, None
if east["edges"][2] and west["edges"][2]:
ne_level, nw_level = east, west
se_level = find(east["edges"][2])
sw_level = find(west["edges"][2])
elif east["edges"][0] and west["edges"][0]:
se_level, sw_level = east, west
ne_level = find(east["edges"][0])
nw_level = find(west["edges"][0])
if ne_level and nw_level and se_level and sw_level:
return (nw_level, ne_level, sw_level, se_level)
else:
return (west, east)
def disable_etc(chroot=None):
'''Disable bluetooth etc. directories in /etc.'''
for module in BadModules:
etc_paths_text = sh.find(fullpath('/etc', chroot))
etc_paths = etc_paths_text.strip().split('\n')
for path in etc_paths:
if path.endswith('/{}'.format(module)):
if os.path.exists(path):
try:
sh.mv('--force', path, path + '-unused')
except e:
print(e)
def compile_contig_matrices():
fs = sh.find(raw_ass, "-name", "*-smds.coverage.percontig").stdout.split("\n")[:-1]
fs = [f for f in fs if "_2ref" in f]
df = DataFrame.from_csv(fs[0],sep="\t")
glob = df.ix[:,0:2]
for f in fs:
id = [c for c in f.split("/") if "IH" in c][0]
values = DataFrame.from_csv(f,sep="\t")["cov_mean_sample_0"]
assert sum([a!=b for a,b in zip(values.index, glob.index)]) == 0
if sum([a!=b for a,b in zip(values.index, glob.index)]) == 0:
glob[id] = values
else:
print f, "is weird"
glob.to_csv(stats_out + "all_contig_coverages.csv",sep="\t")
def _lookup_snapshots(self, tag):
logger.debug('Searching for snapshots with tag: {} '.format(tag))
try:
logger.debug('Executing: %s -name %s', self.cassandra_data_dir, tag)
dirs = sh.find(self.cassandra_data_dir,
'-name',
tag)
logger.debug('Found directory: {}'.format(dirs))
except:
logger.exception('Unable to execute find, nodetool did not create snapshot?')
sys.exit(1)
dirs = ''
return dirs.splitlines()
def get_args(argv):
find_args = '-name *.docx -o -name *.odt'.split()
files = sh.find(test_root('data'), *find_args).strip().split('\n')
parser = argparse.ArgumentParser(argv)
parser.add_argument("--pdf", action='store_true', help='also generate pdfs')
parser.add_argument("-j", "--jobs", action='store', type=int,
help='Parallelization level (default numcores)',
default=multiprocessing.cpu_count())
parser.add_argument("--cache", action='store_true', default=True,
help='also generate pdfs')
parser.add_argument("--no-cache", action='store_false', dest='cache')
parser.add_argument("test_files", nargs='*', default=files,
help='the files to test (default: all)')
return parser.parse_args()
def _get_python_path(self):
_python_paths = [
sh.which('python'),
sh.which('python3'),
sh.which('python2')
]
python_paths = [str(path) for path in _python_paths if path]
if os.path.isfile('/usr/local/python-3.6.5/bin/python'):
python_paths.append('/usr/local/python-3.6.5/bin/python')
if os.path.isdir('/usr/local/Cellar/python'):
out = sh.find('/usr/local/Cellar/python',
'-regex',
'.*/bin/python3[0-9.]*$',
'-type',
'f',
_piped=True)
out = sh.sort(out, _piped=True)
python_paths.append(sh.head(out, '-n1').strip())
useable_pythons = []
python_paths_set = set()
for python_path in python_paths:
if python_path in python_paths_set:
continue
python_paths_set.add(python_path)
if os.path.realpath(python_path) in python_paths_set:
continue
python_paths_set.add(os.path.realpath(python_path))
useable_pythons.append(
(python_path, self._get_python_version(python_path)))
if len(useable_pythons) == 0:
print('Not found python!!')
sys.exit(1)
error = ''
while True:
message = '{}\n{}select python path [{}]: '.format(
'\n'.join([
'{}. {} (v{})'.format(i, *e)
for i, e in enumerate(useable_pythons)
]), error,
','.join([str(i) for i in range(len(useable_pythons))]))
num = int(input(message))
if num < 0 or num >= len(useable_pythons):
error = 'error: invalid input, try again!! '
continue
return useable_pythons[num]
def find_asan_rt_library(abi, asan_rt_path=''):
if not asan_rt_path:
find_path = os.environ['ANDROID_NDK_HOME']
candidates = split_stdout(
sh.find(find_path, "-name", asan_rt_library_names(abi)))
if len(candidates) == 0:
common.MaceLogger.error(
"Toolchain",
"Can't find AddressSanitizer runtime library in %s" %
find_path)
elif len(candidates) > 1:
common.MaceLogger.info(
"More than one AddressSanitizer runtime library, use the 1st")
return candidates[0]
return "%s/%s" % (asan_rt_path, asan_rt_library_names(abi))
def find_asan_rt_library(abi, asan_rt_path=''):
if not asan_rt_path:
find_path = os.environ['ANDROID_NDK_HOME']
candidates = split_stdout(sh.find(find_path, "-name",
asan_rt_library_names(abi)))
if len(candidates) == 0:
common.MaceLogger.error(
"Toolchain",
"Can't find AddressSanitizer runtime library in % s" %
find_path)
elif len(candidates) > 1:
common.MaceLogger.info(
"More than one AddressSanitizer runtime library, use the 1st")
return candidates[0]
return "%s/%s" % (asan_rt_path, asan_rt_library_names(abi))
def get_wc(content_dir):
"""
"""
filetype = "*.markdown"
cd(content_dir)
files_list = find(".", "-name", "*.markdown")
files_arr = files_list.split('\n')
word_count = 0
for f in files_arr:
if f:
try:
file_word_count = int(wc(cat(content_dir + f), "-w"))
word_count += file_word_count
except:
pass
return word_count
def get_wc(content_dir):
"""
"""
filetype = "*.markdown"
cd(content_dir)
files_list = find(".", "-name", "*.markdown")
files_arr = files_list.split('\n')
word_count = 0
for f in files_arr:
if f:
try:
file_word_count = int(wc(cat(content_dir + f), "-w"))
word_count += file_word_count
except:
pass
return word_count
def _locate(self, *args, **kwargs):
if not self._dont_update_db:
try:
print(
'If you do not want to run sudo, use the following command to configure Phigaro:'
)
print(' phigaro-setup --no-updatedb')
with sh.contrib.sudo:
sh.updatedb()
self._dont_update_db = True
except sh.ErrorReturnCode_1:
print('Invalid password')
exit(1)
try:
return [s.rstrip() for s in sh.find(*args, **kwargs)]
except sh.ErrorReturnCode_1:
return []
def _remove_unneeded(self, lib_dir):
self.log("removing unneeded files...")
files = find(lib_dir).strip().splitlines()
keep = [
"QtCore.so", "QtGui.so", "__init__.py", "_utils.py", "PySide", ""
] # empty means the whole pyside dir
if IS_WIN:
keep = [
"QtCore4.dll", "QtGui4.dll", "__init__.py", "_utils.py",
"PySide", "QtGui.pyd", "QtCore.pyd", ""
] # empty means the whole pyside dir
for f in files:
if f.find("PySide") > 0:
if os.path.split(f)[1] not in keep:
rm("-rf", f)
pass
self.log("done.")
def find_image_list():
def generate_type_list(a_list):
b_list = [['-name', name, '-o'] for name in a_list]
# Flatten the list and take out the last '-o'
return list(chain.from_iterable(b_list))[:-1]
black_list = ['rootfs', 'bootfs', 'linux*', 'build*', '*.fs']
white_list = ['*.ext[1-5]', '*.cpio', '*.dd', '*.image', '*.img']
opt_b = [*'-type d'.split(), '(', generate_type_list(black_list), ')']
opt_w = [
*'( -type l -o -type f )'.split(), '(',
generate_type_list(white_list), ')'
]
logger.debug(opt_b)
logger.debug(opt_w)
sh.cd(IMAGE_DIR)
img_list = []
exec_opt = ['-exec', 'file', '-L', '{}', ';']
img_list = sh.find(*opt_b,
'-prune',
'-o',
*opt_w,
*exec_opt,
_tty_out=False)
# Remove empty string and strip the first two characters "./"
img_list = [
remove_control_chars(file_name)[2:] for file_name in img_list
if file_name
]
# Filter out images with broken symbolic link
img_list = [text for text in img_list if 'cannot open' not in text]
# Cut the string into two folds. [NAME: TYPE]
img_list = [[text.split(':')[0], ''.join(text.split(':')[1:])]
for text in img_list]
# Transform into dictionary and parse type string
img_list = [{
'name': kv[0],
'type': imageParser.get_type(kv[1])
} for kv in img_list]
logger.debug(img_list)
return img_list
def fetch(self, output):
clone_dir = self._make_temp_directory("rootfetch-android-git")
sh.git.clone(self.GIT_URL, clone_dir)
tags = set()
for tag in sh.git.tag(_cwd=clone_dir, _iter=True):
tags.add(tag.strip())
sh.git.checkout(self._latest_stable_tag(tags), _cwd=clone_dir)
root_dir = os.path.join(clone_dir, "files")
for cert_path in sh.find(root_dir, "-type", "f", _iter=True):
cert_path = cert_path.strip()
with open(cert_path) as fd:
pem_parts = list()
for line in fd:
pem_parts.append(line)
if line.startswith("-----END CERTIFICATE-----"):
break
output.write(''.join(pem_parts))
output.flush()
def _copy_results(lconf, output_dir):
"copy interesting files from tmp dir to output dir"
# copy the csv parses
parsed_results = fp.join(output_dir, "parsed")
if fp.exists(parsed_results):
shutil.rmtree(parsed_results)
shutil.copytree(lconf.tmp("parsed"), parsed_results)
# copy the svg graphs into single flat dir
graphs_dir = fp.join(output_dir, "graphs")
makedirs(graphs_dir)
svg_files = sh.find(minicorpus_path(lconf, result=True),
"-name", "*.svg", _iter=True)
for svg in (f.strip() for f in svg_files):
svg2 = fp.join(graphs_dir,
fp.basename(fp.dirname(svg)) + ".svg")
shutil.copyfile(svg, svg2)
def find_entries():
"""Returns a dictionary of keys corresponding to entries with a list of
comments. For example:
'nafus-gender.html': {
'filename' : 'nafus-gender.html',
'mtime' : '2012-06-29',
'path' : 'social/gender/',
'comments' : ['nafus-gender-1341219620.15.cmt',
'nafus-gender-1341239352.82.cmt',
'nafus-gender-1341241456.95.cmt'
]},
}
"""
entries = {}
for root, dirnames, filenames in walk(CONTENT_ROOT):
for filename in filenames:
#info(root, filename)
entry = {'filename': filename}
stats = stat(root+'/'+filename)
entry['mtime'] = strftime("%Y-%m-%d", localtime(stats[8])).strip()
fn_base, fn_ext = splitext(basename(filename))
path = root[len(CONTENT_ROOT):]
entry['path'] = path+'/'
info("fn_base = '%s'" % fn_base)
if fn_base.startswith('.'):
continue
info(path, fn_base, fn_ext)
comments = find(COMMENT_ROOT, "-name", fn_base+'*')
if comments:
info(" comments = %s" %comments)
comments_list = []
for comment in comments:
info(split(comment.strip()))
comments_list.append(split(comment.strip())[1])
info(comments_list)
entry['comments'] = comments_list
entries[filename] = entry
return entries
def main(args, remainder):
if args.verbose > 1:
print("Main arguments: %s\nRemaining args: %s" % (args, remainder))
cmd_list = build_cmd_list(args, remainder)
cmd_list_print = build_cmd_list(args, remainder, quote_pattern=True)
cmd_str = "find " + " ".join(cmd_list_print)
if args.verbose or args.print:
print("Command: '%s'" % (cmd_str))
if args.print:
sys.exit(0)
try:
exec_cmd = find(*cmd_list, _iter=True)
for l in exec_cmd:
print(l, end='')
except sh.ErrorReturnCode_1 as err:
cprint("Error when executing command: '%s': %s" % (cmd_str, err), color='red')
def update_data_repos(repo_sel=None):
"""Update repositories."""
repos = sh.find(folder_data, '-name', '.git')
repos = repos.stdout.decode().splitlines()
if repo_sel is not None:
repos = [x for x in repos if [y for y in repo_sel if x.find(y) >= 0]]
msg_error = ''
_log.warning('\nUpdating Working Tree and Syncing repository:')
for repo in repos:
rep = repo.rpartition('/')[0]
sh.cd(rep)
_log.warning('\n' + rep + ': ')
status_repo = sh.git.status(porcelain=True).stdout.decode()
if not status_repo:
_log.warning('\tNothing to commit.')
# If there are unmerged files from previous attempts:
if status_repo and _add_and_commit_files(status_repo):
_log.warning('\tStill have unmerged files. Solve it!')
msg_error += '\n' + rep + ': Still have unmerged files. Solve it!'
continue
# Try to pull the changes if there are no conflicts
# yet and identify new conflicts
try:
sh.git.pull()
_log.warning('\tRepository fetched and merged.')
except sh.ErrorReturnCode as error:
_log.warning('\tThere are unmerged files.')
msg_error += '\n' + rep + ': There are unmerged files.'
continue
# If no new conflicts were identified, try to push
try:
sh.git.push()
_log.warning('\tRepository pushed.')
except sh.ErrorReturnCode as error:
_log.warning('\tProblem pushing repository.')
msg_error += '\n' + rep + ': Problem pushing repository.'
return msg_error
def __init__(self, name, k=31, name_coass = ""):
if isinstance(name, list):
self.name = "coass"+name_coass
self.dir = raw_path
self.reads = [raw_path + n + ".fastq.gz" for n in name]
gzs = [g[:-1] for g in sh.find( raw_path , "-name", "*.fastq.gz") if any([n in g for n in name])]
self.pairs = zip([ d for d in gzs if "R1" in d], [d for d in gzs if "R2" in d])
self.cleans = [[ f.replace("fastq.gz","clean.fastq").replace(raw_path,raw_ass).replace("/PRI_EUSM", "/cleans/PRI_EUSM") for f in p ] for p in self.pairs]
self.singles = [p[0].replace("_R1_001.fastq.gz","_single.clean.fastq") for p in self.pairs]
self.coass = True
else :
self.dir = raw_path +name +"/"
self.coass = False
self.name = name
self.pairs = [[ d for d in os.listdir(self.dir) if ".fastq.gz" in d and "L001" in d], [d for d in os.listdir(self.dir) if ".fastq.gz" in d and "L002" in d]]
self.cleans = [[ f.replace("fastq.gz","clean.fastq") for f in p ] for p in self.pairs]
self.singles = [p[0].replace("_R1_001.fastq.gz","_single.clean.fastq") for p in self.pairs]
self.k = k
self.out_dir = raw_ass + self.name + "/"
self.clean_dir = raw_ass + self.name + "/cleans/"
self.merge_dir = raw_ass + self.name + "/merged/"
if not os.path.exists(raw_ass):
os.makedirs(raw_ass)
if not os.path.exists(stats_out):
os.makedirs(stats_out)
if not os.path.exists(raw_ass + self.name):
os.makedirs(raw_ass + self.name)
self.success_ks = [int(f.split("_")[1]) for f in os.listdir(raw_ass + self.name) if "ray" in f]
if not os.path.exists(self.out_dir):
os.makedirs(self.out_dir)
if not os.path.exists(self.clean_dir):
os.makedirs(self.clean_dir)
if os.path.exists(self.out_dir + self.name + "_stats.json"):
print "Loading stats for", self.name
with open(self.out_dir + self.name + "_stats.json") as handle:
self.stats = json.load(handle)
def __init__(self):
template = Template(templates[args.layout])
files = find(args.dir, "-type", "f", "-name", "*.json").strip().split("\n")
for file in files:
self.generate(file)
if len(self.methods):
print(
template.render(
title=args.title,
description=args.description,
navigation=self.navigation,
methods=self.methods,
header_inc=file_get_contents(args.header),
footer_inc=file_get_contents(args.footer),
)
)
else:
parser.print_help()
def disable_lib_modules(chroot=None):
""" This doesn't help when bluetooth etc. are in the kernel.
Bluetooth and wi-fi are automatically installed and enabled by debian.
None of the standard disabling methods like rmmod have any apparent
effect. So rename
/lib/modules/.../bluetooth
/lib/modules/.../wireless
to *-unused.
"""
for module in BadModules:
lib_module_paths_text = sh.find(fullpath('/lib/modules', chroot))
lib_module_paths = lib_module_paths_text.strip().split('\n')
for path in lib_module_paths:
if path.endswith('/{}'.format(module)):
if os.path.exists(path):
try:
sh.mv('--force', path, path + '-unused')
except e:
print(e)
def get(self):
query = unquote(self.get_argument('q', ''))
if query == '':
self.redirect(self.settings.prefix)
try:
results = str(grep('-Rli', '--exclude-dir', '.git', "--", query,
self.settings.repo))
except ErrorReturnCode_1 as e:
results = ''
try:
results += str(find(self.settings.repo, '-type', 'f', '-iname',
'*' + query + '*', '-not', '(', '-path',
'%s/%s/*' % (self.settings.repo, '.git'), ')'))
except ErrorReturnCode_1 as e:
pass
results = results.replace(self.settings.repo, '').split('\n')[:-1]
formatted_results = []
for filename in results:
if 'Binary file' in filename or filename == '':
continue
if path.basename(filename).startswith('.'):
filename = path.join(path.dirname(filename),
path.basename(filename)[1:])
resultpath = path.join(self.settings.repo, filename[1:])
if path.exists(resultpath):
try:
string = str(grep('-i', "--", query, resultpath ))
except ErrorReturnCode_1 as e:
string = ''
string = self.highlight(string, query)
formatted_results.append({'filename': filename, 'string': string})
self.render('search.html', query=query, results=formatted_results)
def compile_contig_matrices(self):
fs = sh.find(self.merge_dir, "-name", "*-smds.coverage.percontig").stdout.split("\n")[:-1]
df = DataFrame.from_csv(fs[0],sep="\t")
glob = df.ix[:,0:2]
for f in fs:
id = [c.replace("map_","") for c in f.split("/") if "map_" in c][0]
values = DataFrame.from_csv(f,sep="\t")["cov_mean_sample_0"]
assert sum([a!=b for a,b in zip(values.index, glob.index)]) == 0
if sum([a!=b for a,b in zip(values.index, glob.index)]) == 0:
glob[id] = values
else:
print f, "is weird"
samples = list(set([c.split("_")[2] for c in mat.columns if "EUSM" in c]))
processed = {}
processed['length'] = glob['length']
processed['GC'] = glob['GC']
for s in tqdm(samples):
processed[s]= glob[[c for c in glob.columns if s in c]].apply(sum, axis=1)
glob = DataFrame.from_dict(processed)
glob.to_csv(self.stats_out + self.name + "_contig_coverages.csv",sep="\t")
glob[samples].to_csv(self.stats_out + self.name + "_contig_coverages_for_concoct.csv",sep="\t")
return glob
def main(args=None):
parser = gitboots.argparse_setup(globals())
parser.add_argument("-e", metavar="COMMAND", default="rm -f", help="execute command on each found lock")
opts = parser.parse_args(args)
path = git_dir()
for lock in sh.find(path, "-type", "f", "-name", "index.lock"):
lock = lock.strip()
if opts.e != "rm -f":
if "{}" in opts.e:
cmd = opts.e.replace("{}", lock)
else:
cmd = "{cmd} {lock}".format(cmd=opts.e, lock=lock)
logger.info(cmd)
if sh.which("sh"):
sh.sh("-c", cmd)
elif sh.which("cmd.exe"):
sh.cmd("\c", cmd)
else:
raise ValueError("Do not know how to run this command: %s" % cmd)
else:
logger.info("rm -f %s", lock)
os.remove(lock)
self.stream.flush()
def __getattr__(self, attr):
return getattr(self.stream, attr)
if __name__ == '__main__':
sys.stdout = Unbuffered(sys.stdout)
arguments = docopt(__doc__)
input_folder = arguments['-i']
output_head = arguments['-o']
bps_fwd = int(arguments['-n']) if arguments['-n'] else 0
bps_rev = bps_fwd + 1
file_opener = gzip.open if arguments['-g'] else open
lib_list_fwd = [f[:-1] for f in sh.find(input_folder) if "1P_fastq.gz" in f ]
lib_list_rev = [f.replace("1P","2P") for f in lib_list_rev]
#print lib_list
handles = [open(l) for l in lib_list]
out_file = output_head + "_%s.fasta"
entry = ""
ll = len(handles)-1
i=0
handle = None
fileID = 0
min_seq_len = 40
g_counter = 0
trash_count = 0
while ll > -1:
#!/usr/bin/env python
import codecs
import sh
import re
note_path = '/home/zhongsiz/subler.github.io/note'
md_set = sh.find(note_path, '-name', '*.md')
print md_set
def mon_convert(mon_str):
if mon_str == 'Dec':
return '12'
elif mon_str == 'Nov':
return '11'
elif mon_str == 'Otc' or mon_str == 'Oct':
return '10'
elif mon_str == 'Sep':
return '09'
elif mon_str == 'Aug':
return '08'
elif mon_str == 'Jul':
return '07'
elif mon_str == 'Jun':
return '06'
elif mon_str == 'May':
return '05'
elif mon_str == 'Apr' or mon_str == 'Apri':
return '04'
elif mon_str == 'Mar':
return '03'
def grade_q1(uniqname, repo):
repo, path = clone('wk12-dotfiles', uniqname, repo)
with sh.pushd(path):
if os.path.exists(os.path.join(path, 'configs')):
# Handle case of person who put stuff in folder..
files = set(sh.ls('-a', 'configs').split())
elif os.path.exists(os.path.join(path, 'bash')):
# Handle case of person who put stuff in folder..
files = set(sh.ls('-a', 'bash').split())
elif os.path.exists(os.path.join(path, 'dotfiles')):
# Handle case of person who put stuff in folder..
files = set(sh.ls('-a', 'dotfiles').split())
elif os.path.exists(os.path.join(path, 'shell')):
# Handle case of person who put stuff in folder..
files = set(sh.ls('-a', 'shell').split())
else:
files = set(sh.ls('-a').split())
files.discard('.')
files.discard('..')
files.discard('.git')
text = ''
grade = 0
# Viable .bashrc targets we've seen or are in the spirit of the Q
bashrc = set((
'.bashrc',
'bashrc',
'bash.bashrc',
'.bash_profile',
'bash_profile',
'.zshrc',
'zshrc',
'mybashrc',
'bilraj.dotfiles.bashrc',
))
if len(files.intersection(bashrc)):
# Bashrc file
text += '<dd><strong>.bashrc [1.0/1.0]</strong></dd>\n'
grade += 1.0
else:
text += '<dd><strong>.bashrc [0.0/1.0]</strong></dd>\n'
text += '<dd>Output of <tt>find .</tt></dd>\n'
text += '<dd><pre>{}</pre></dd>\n'.format(sh.find('.'))
with grades_lock:
no_bash.add(uniqname)
grade += 0.0
others = files - bashrc
nonfiles = set((
'README',
'README.md',
'readme.md',
'README.txt',
'LICENSE',
'LICENSE.txt',
'setupDotfiles.sh',
'3',
'update.sh',
'system_spec',
'init.sh',
'main.py',
'hello.sh',
'setupDotfiles.sh',
'status_repos.py',
'link.sh',
'test',
'test1.cpp',
'test2.cpp',
'test3.cpp',
'backup.sh',
'.bashrc-miniconda3.bak',
'prog.cpp',
'file1.txt',
'file2.txt',
'file3.txt',
'dotfiles_old',
'setup_dotfiles.sh',
'source.sh',
'q2.py',
'games',
'duncanProf',
'trial.py',
'clean.sh',
'bootstrap.sh',
'q3.py',
'a.cpp',
'make_symlink.sh',
'update_repos.py',
'__pycache__',
'test1',
'shell',
'a.h',
'copy.sh',
'scripts',
'makesymlinks.sh',
'os',
'set_up.sh',
'a',
'pro.py',
'link_files.sh',
'install.sh',
'main.pyc',
))
with grades_lock:
for o in others:
other_dotfiles.add(o)
if len(others):
# Any other dotfile
text += '<dd><strong>Any other dotfile ({}) [1.0/1.0]</strong></dd>\n'.format(others.pop())
grade += 1.0
else:
text += '<dd><strong>Any other dotfile [0.0/1.0]</strong></dd>\n'
text += '<dd>Output of <tt>find .</tt></dd>\n'
text += '<dd><pre>{}</pre></dd>\n'.format(sh.find('.'))
grade += 0.0
q1_entry = '''
<dt>Week 12</dt>
<dd>{:1.1f}/2.0</dd>
<dl>
{}</dl>
'''.format(grade, text)
return grade, text
def generate_setlist(output_path):
print "Generating level playlist..."
boring, duplicates, by_hash, by_path = load_corpus()
visited = []
paths_written = []
unprocessed = by_hash.keys()
queue = []
quad_count = 0
pair_count = 0
ungrouped_count = 0
def find(key):
# accepts either the level name or hash
if key is None:
return None
assert(type(key) == str)
found = None
if by_path.has_key(key):
found = by_path[key]
elif by_hash.has_key(key):
found = by_hash[key]
if found and found["level_hash"] not in visited:
return found
return None
def take_level(setlist, level):
level_hash = level["level_hash"]
visited.append(level_hash)
try:
queue.remove(level_hash)
except:
pass
try:
unprocessed.remove(level_hash)
except:
pass
assert(level["path"] not in paths_written)
paths_written.append(level["path"])
setlist.write(level["path"] + "\n")
def enqueue(hash_or_path):
level = find(hash_or_path)
if level:
level_hash = level["level_hash"]
try:
unprocessed.remove(level_hash)
except:
pass
if level_hash not in queue and level_hash not in visited:
queue.append(level_hash)
def edge_search(level):
# this attempts to find near by levels to group together
if not level["edges"]:
return None
right_of = find(level["edges"][1])
left_of = find(level["edges"][3])
east, west = None, None
if right_of:
east = right_of
west = level
elif left_of:
east = level
west = left_of
else:
return None
ne_level, nw_level, se_level, sw_level = None, None, None, None
if east["edges"][2] and west["edges"][2]:
ne_level, nw_level = east, west
se_level = find(east["edges"][2])
sw_level = find(west["edges"][2])
elif east["edges"][0] and west["edges"][0]:
se_level, sw_level = east, west
ne_level = find(east["edges"][0])
nw_level = find(west["edges"][0])
if ne_level and nw_level and se_level and sw_level:
return (nw_level, ne_level, sw_level, se_level)
else:
return (west, east)
if by_path.has_key('onlinestartlocal.graal'):
enqueue('onlinestartlocal.graal')
elif by_path.has_key('level1.graal'):
enqueue('level1.graal')
with open(output_path, "w") as setlist:
while len(unprocessed) + len(queue) > 0:
pick = None
if len(queue):
pick = queue.pop(0)
else:
pick = unprocessed.pop(0)
level = find(pick)
assert(level is not None)
edges = []
edge_odds = 0.30
adjacent_group = edge_search(level)
if adjacent_group:
if len(adjacent_group) == 2:
pair_count += 1
setlist.write("*** pair ***\n")
edge_odds = 0.20
elif len(adjacent_group) == 4:
quad_count += 1
setlist.write("*** quad ***\n")
edge_odds = 0.10
for tile in adjacent_group:
take_level(setlist, tile)
edges += tile["edges"] or []
else:
ungrouped_count += 1
edges = level["edges"] or []
take_level(setlist, level)
if level["doors"]:
# add all levels linked by doors into the queue
for target in level["doors"]:
enqueue(target)
# small chance of adding neighbors into the queue
for edge_path in set(edges):
if random.random() <= edge_odds:
enqueue(edge_path)
if duplicates:
print "Duplicate levels which have been merged together:"
def path_for_key(key):
return os.path.split(by_hash[key]["path"])[-1]
for path in sorted(map(path_for_key, duplicates.keys())):
level_hash = by_path[path]["level_hash"]
dupe_group = duplicates[level_hash]
count = len(dupe_group) + 1
print " + {} ({}x, hash = {})".format(path, count, level_hash)
for dupe in dupe_group:
print " -", os.path.split(dupe["path"])[-1]
print "quads found:", quad_count
print "pairs found:", pair_count
print "individuals:", ungrouped_count
print "total tweets:", quad_count + pair_count + ungrouped_count
print "total levels:", quad_count * 4 + pair_count * 2 + ungrouped_count
import sys
import os
import os.path
assert os.path.exists ("./synth.c"), "tools must be invoked from the root of the repo"
path = "./docs/docco/"
if not os.path.exists (path):
os.mkdir (path)
docco = ["ext", "node_modules", "docco", "bin", "docco", "-o", path]
import sh
import subprocess
for file in sh.find (".", "-name", "*.c"):
subprocess.check_call (docco + [file])
def fixSampleName(x):
if x.startswith("s_"):
x=x[2:]
return x
def getTumorSamplesFromPairs(pairingFile):
samples=set()
with open(pairingFile) as fp:
for line in fp:
sample=line.strip().split()[1]
samples.add(sample)
return list(samples)
samples=getTumorSamplesFromPairs(getPairingFile())
rdataFiles=sh.find("facets","-name","*_hisens.Rdata").strip().split("\n")
projectNo=getProjectNo()
rdataSampleMap=dict()
for si in samples:
tag="Proj_%s__%s" % (projectNo,fixSampleName(si))
for ri in rdataFiles:
pos = ri.find(tag)
if pos > -1:
rdataSampleMap[si]=ri
facetsMappingFile="mapping.txt"
with open(facetsMappingFile,"w") as fp:
HEADER="Tumor_Sample_Barcode Rdata_filename".split()
print >>fp, "\t".join(HEADER)
for si in sorted(rdataSampleMap):
print >>fp, "\t".join([si,rdataSampleMap[si]])
