def reduce_python(self): print("Reduce python") oldpwd = os.getcwd() try: print("Remove files unlikely to be used") os.chdir(join(self.ctx.dist_dir, "root", "python")) sh.rm("-rf", "share") sh.rm("-rf", "bin") os.chdir(join(self.ctx.dist_dir, "root", "python", "lib")) sh.rm("-rf", "pkgconfig") sh.rm("libpython2.7.a") os.chdir( join(self.ctx.dist_dir, "root", "python", "lib", "python2.7")) sh.find(".", "-iname", "*.pyc", "-exec", "rm", "{}", ";") sh.find(".", "-iname", "*.py", "-exec", "rm", "{}", ";") #sh.find(".", "-iname", "test*", "-exec", "rm", "-rf", "{}", ";") sh.rm("-rf", "wsgiref", "bsddb", "curses", "idlelib", "hotshot") sh.rm("-rf", sh.glob("lib*")) # now create the zip. print("Create a python27.zip") sh.rm("config/libpython2.7.a") sh.rm("config/python.o") sh.rm("config/config.c.in") sh.rm("config/makesetup") sh.rm("config/install-sh") sh.mv("config", "..") sh.mv("site-packages", "..") sh.zip("-r", "../python27.zip", sh.glob("*")) sh.rm("-rf", sh.glob("*")) sh.mv("../config", ".") sh.mv("../site-packages", ".") finally: os.chdir(oldpwd)
def compile_contig_matrices(names = None): if names == None: fs = sh.find(raw_ass, "-name", "*-smds.coverage.percontig").stdout.split("\n")[:-1] fs = [f for f in fs if "_2ref" in f] else: fs = sh.find(merged_ass + names, "-name", "*-smds.coverage.percontig").stdout.split("\n")[:-1] df = DataFrame.from_csv(fs[0],sep="\t") glob = df.ix[:,0:2] for f in fs: if names == None: id = [c for c in f.split("/") if "IH" in c][0] else: id = [c.replace("map_","") for c in f.split("/") if "map_" in c][0] values = DataFrame.from_csv(f,sep="\t")["cov_mean_sample_0"] assert sum([a!=b for a,b in zip(values.index, glob.index)]) == 0 if sum([a!=b for a,b in zip(values.index, glob.index)]) == 0: glob[id] = values else: print f, "is weird" if names == None: glob.to_csv(stats_out + "all_contig_coverages.csv",sep="\t") else: glob.to_csv(stats_out + names + "_contig_coverages.csv",sep="\t") glob[samples].to_csv(stats_out + self.name + "_contig_coverages_for_concoct.csv",sep="\t")
def reduce_python(self): print("Reduce python") oldpwd = os.getcwd() try: print("Remove files unlikely to be used") os.chdir(join(self.ctx.dist_dir, "root", "python")) sh.rm("-rf", "share") sh.rm("-rf", "bin") os.chdir(join(self.ctx.dist_dir, "root", "python", "lib")) sh.rm("-rf", "pkgconfig") sh.rm("libpython2.7.a") os.chdir(join(self.ctx.dist_dir, "root", "python", "lib", "python2.7")) sh.find(".", "-iname", "*.pyc", "-exec", "rm", "{}", ";") sh.find(".", "-iname", "*.py", "-exec", "rm", "{}", ";") #sh.find(".", "-iname", "test*", "-exec", "rm", "-rf", "{}", ";") sh.rm("-rf", "wsgiref", "bsddb", "curses", "idlelib", "hotshot") sh.rm("-rf", sh.glob("lib*")) # now create the zip. print("Create a python27.zip") sh.rm("config/libpython2.7.a") sh.rm("config/python.o") sh.rm("config/config.c.in") sh.rm("config/makesetup") sh.rm("config/install-sh") sh.mv("config", "..") sh.mv("site-packages", "..") sh.zip("-r", "../python27.zip", sh.glob("*")) sh.rm("-rf", sh.glob("*")) sh.mv("../config", ".") sh.mv("../site-packages", ".") finally: os.chdir(oldpwd)
def run(self): self.log("Removing .pyc files...") files = find(self._basedir, "-name", "*.pyc").strip().splitlines() for f in files: rm(f) files = find(self._basedir, "-name", "*\\.so*").strip().splitlines() for f in files: self.log("Stripping {0}".format(f)) try: strip(f) except: pass self.log("Done")
def init(template, project_name): templates = get_templates() if template not in templates: click.echo('%s template not found' % template, err=True) return project_dir = f'./{project_name}' sh.mkdir('-p', project_dir) sh.cp('-rf', os.path.join(HERE, f'templates/{template}/'), project_dir) for f in sh.find(project_dir, '-name', '*.sh'): sh.sed('-i', '', '-e', f's/proj/{project_name}/g', f.strip()) for f in sh.find(project_dir, '-name', '*.yaml'): sh.sed('-i', '', '-e', f's/proj/{project_name}/g', f.strip()) for f in sh.find(project_dir, '-name', 'Dockerfile*'): sh.sed('-i', '', '-e', f's/proj/{project_name}/g', f.strip())
def fetch_git_repo(self, repo): cp = os.path.join(self.checkouts_path, os.path.basename(repo)) if not os.path.exists(cp): logger.debug('clone %s' % repo) pygit.Repo.clone_from(repo, cp) # find galaxy.yml file(s) res = find(cp, '-name', 'galaxy.yml') galaxy_files = res.stdout.decode('utf-8').split('\n') galaxy_files = [x.strip() for x in galaxy_files if x.strip()] for gf in galaxy_files: logger.debug(gf) with open(gf, 'r') as f: gdata = yaml.load(f.read()) ckey = (gdata['namespace'], gdata['name']) logger.debug('found fqn: %s.%s' % ckey) if ckey not in self.collections: self.collections[ckey] = { 'name': ckey[1], 'namespace': { 'name': ckey[0] }, 'href': repo, 'plugins': {}, 'filepath': os.path.dirname(gf) } self.index_plugins_in_collection(os.path.dirname(gf), ckey) res = find(cp, '-name', 'MANIFEST.json') manifest_files = res.stdout.decode('utf-8').split('\n') manifest_files = [x.strip() for x in manifest_files if x.strip()] for mf in manifest_files: logger.debug(mf) with open(mf, 'r') as f: mdata = json.loads(f.read()) ckey = (mdata['collection_info']['namespace'], mdata['collection_info']['name']) logger.debug('found fqn: %s.%s' % ckey) if ckey not in self.collections: self.collections[ckey] = { 'name': ckey[1], 'namespace': { 'name': ckey[0] }, 'href': repo, 'plugins': {}, 'filepath': os.path.dirname(mf) } self.index_plugins_in_collection(os.path.dirname(mf), ckey)
def build_level_database(input_path): print "Generating or regenerating level database." print "This may take a long time if a lot of files need to be scanned." paths = [p.strip() for p in find(input_path)] levels = [] pool = Pool(processes=4) ratio = 100.0 / len(paths) processed = 0 last_percent = 0 for data in pool.imap_unordered(process_level, paths): processed += 1 percent = int(processed * ratio) if percent > last_percent: last_percent = percent print "... {}%".format(percent) if not data: continue levels.append(data) db = { "levels": levels, "version": DATABASE_VERSION, } with open("level_db.pickle", "w") as outfile: pickle.dump(db, outfile)
def build_level_database(input_path): print "Generating or regenerating level database." print "This may take a long time if a lot of files need to be scanned." paths = [p.strip() for p in find(input_path)] levels = [] pool = Pool(processes=4) ratio = 100.0 / len(paths) processed = 0 last_percent = 0 for data in pool.imap_unordered(process_level, paths): processed += 1 percent = int(processed * ratio) if percent > last_percent: last_percent = percent print "... {}%".format(percent) if not data: continue levels.append(data) db = { "levels" : levels, "version" : DATABASE_VERSION, } with open("level_db.pickle", "w") as outfile: pickle.dump(db, outfile)
def get(self): query = unquote(self.get_argument('q')) try: results = str( grep('-R', '--exclude-dir', '.git', query, self.settings.repo)) except ErrorReturnCode_1 as e: results = '' # TODO filter out duplicates if the filename is already in the search results # TODO this doesn't exclude the .git folder try: results += str( find(self.settings.repo, '-type', 'f', '-name', '*' + query + '*')) except ErrorReturnCode_1 as e: pass results = results.replace(self.settings.repo, '').split('\n')[:-1] formatted_results = [] for result in results: if 'Binary file' in result: continue # TODO this doesn't play well with colons in filenames stuff = result.split(':') filename = stuff[0] if path.basename(filename).startswith('.'): filename = path.join(path.dirname(filename), path.basename(filename)[1:]) string = ''.join(stuff[1:]) string = self._highlight(string, query) formatted_results.append({'filename': filename, 'string': string}) self.render('search.html', query=query, results=formatted_results)
def generate_image_hash(self): """Generate hash of Docker context - Recursively list all files - Get sha1sum of each file in list - Sort list (ensures consistency from run to run) - Get final sha1sum of list """ for context_dir in self.docker_context_cm(): sha1sum = sh.sha1sum( sh.xargs( sh.sort( sh.find( '.', '-type', 'f', '-print0', _err=sys.stderr.buffer, _cwd=context_dir), '-z', # Locale differences can affect sort order _env = {**os.environ, 'LC_ALL': 'C.UTF-8'}, _err=sys.stderr.buffer), '-0', 'sha1sum', _err=sys.stderr.buffer, _cwd=context_dir), _err=sys.stderr.buffer).split()[0] return sha1sum
def get(self): query = unquote(self.get_argument('q')) try: results = str(grep('-R', '--exclude-dir', '.git', query, self.settings.repo)) except ErrorReturnCode_1 as e: results = '' try: results += str(find(self.settings.repo, '-type', 'f', '-name', '*' + query + '*', '-not', '(', '-path', '%s/%s/*' % (self.settings.repo, '.git') )) except ErrorReturnCode_1 as e: pass results = results.replace(self.settings.repo, '').split('\n')[:-1] formatted_results = [] for result in results: if 'Binary file' in result or result == '': continue # TODO this doesn't play well with colons in filenames stuff = result.split(':') filename = stuff[0] if path.basename(filename).startswith('.'): filename = path.join(path.dirname(filename), path.basename(filename)[1:]) string = ''.join(stuff[1:]) string = self._highlight(string, query) formatted_results.append({'filename': filename, 'string': string}) self.render('search.html', query=query, results=formatted_results)
def get_test_data(): docs = [] for f in find("examples"): f = f.strip() if not f.endswith(".txt"): continue elif "animasi_2d" in f: category = "animasi_2d" elif "animasi_3d" in f: category = "animasi_3d" elif "desain_multimedia" in f: category = "desain_multimedia" elif "basis_data" in f: category = "basis_data" elif "pemodelan_perangkat_lunak" in f: category = "pemodelan_perangkat_lunak" elif "pemrograman_dasar" in f: category = "pemrograman_dasar" elif "komputer_terapan_jaringan" in f: category = "komputer_terapan_jaringan" elif "komunikasi_data" in f: category = "komunikasi_data" else: category = "sistem_operasi_jaringan" docs.append((category, f)) return render_template('index.html', data_tests = docs)
def get(self): query = unquote(self.get_argument('q', '')) if query == '': self.redirect('/') try: results = str( grep('-R', '--exclude-dir', '.git', query, self.settings.repo)) except ErrorReturnCode_1 as e: results = '' try: results += str( find(self.settings.repo, '-type', 'f', '-name', '*' + query + '*', '-not', '(', '-path', '%s/%s/*' % (self.settings.repo, '.git'))) except ErrorReturnCode_1 as e: pass results = results.replace(self.settings.repo, '').split('\n')[:-1] formatted_results = [] for result in results: if 'Binary file' in result or result == '': continue # TODO this doesn't play well with colons in filenames stuff = result.split(':') filename = stuff[0] if path.basename(filename).startswith('.'): filename = path.join(path.dirname(filename), path.basename(filename)[1:]) string = ''.join(stuff[1:]) string = self.highlight(string, query) formatted_results.append({'filename': filename, 'string': string}) self.render('search.html', query=query, results=formatted_results)
def _traverse_corpus(path): for fname in sh.find(path, '-type', 'f'): with open(fname.strip(), 'r') as fin: corpus_name = '%s.%s' % (os.path.basename( os.path.dirname(fname)), os.path.basename(fname)) sentence_no = 1 word_no = 1 sentence = [] metadata = {'META': corpus_name, 'sentence': sentence_no} for line in tqdm(fin): if line.strip() == '': yield Sentence(metadata, sentence, 'idx', 'token', 'lemma', 'tag') sentence_no += 1 word_no = 1 sentence = [] metadata = {'META': corpus_name, 'sentence': sentence_no} else: try: token, lemma, tag, _ = unicodedata.normalize( 'NFC', line).strip().split() if tag.startswith('F'): token = '<PUNCTUATION>' elif tag.startswith('W'): token = '<DATE>' elif tag.startswith('Z'): token = '<NUMBER>' sentence.append('%d\t%s\t%s\t%s' % (word_no, token, lemma, tag)) word_no += 1 except ValueError: continue
def create(template, project_name): templates = get_templates() if template not in templates: click.echo('%s template not found' % template, err=True) return project_dir = './' + project_name sh.mkdir('-p', project_dir) sh.cp('-rf', os.path.join(HERE, f'templates/{template}/'), project_dir) for f in sh.find(project_dir, '-name', '*.py'): sh.sed('-i', '', '-e', 's/%s/%s/g' % ('proj', project_name), f.strip()) for f in sh.find(project_dir, '-name', 'Dockerfile*'): sh.sed('-i', '', '-e', 's/%s/%s/g' % ('proj', project_name), f.strip()) # hack: custom model import if template == 'simple': sh.sed('-i', '', '-e', 's/%s/%s/g' % ('proj', project_name), './%s/migrations/script.py.mako' % project_name) sh.mv(os.path.join(project_dir, 'proj'), os.path.join(project_dir, project_name))
def find_files(in_dir): """ returns a list of the sequence files in a directory recursively """ FASTQ_EXTENSIONS = [".fq", ".fastq"] files = [sh.find(in_dir, "-name", "*" + x) for x in FASTQ_EXTENSIONS] return files
def _lookup_snapshots(tag): try: dirs = sh.find(CASSANDRA_DATA_DIR, '-name', tag) except: logger.warn('Unable to execute find, nodetool did not create ' + 'snapshot?') dirs = '' return dirs.splitlines()
def get_children(self): if not self.output_path: path = self.path else: path = self.output_path #log.debug("Getting children from path {0}".format(path)) return str( sh.find(path, "-maxdepth", "1", "!", "-name", "lost+found").stdout, 'utf8').split("\n")[1:-1]
def _lookup_snapshots(self, tag): try: dirs = sh.find(self.cassandra_data_dir, '-name', tag) except: logger.warning( 'Unable to execute find, nodetool did not create snapshot?') dirs = '' return dirs.splitlines()
def enqueue(hash_or_path): level = find(hash_or_path) if level: level_hash = level["level_hash"] try: unprocessed.remove(level_hash) except: pass if level_hash not in queue and level_hash not in visited: queue.append(level_hash)
def run(self): self.log("Removing unused python code...") test_dirs = find(self._basedir, "-name", "*test*").strip().splitlines() for td in test_dirs: rm("-rf", os.path.join(self._basedir, td)) # twisted_used = ["aplication", "conch", "cred", # "version", "internet", "mail"] # twisted_files = find(self._basedir, "-name", "t self.log("Done")
def setupConfigDirs(dirname, fullPrimaryEmail, primaryEmail, name, uid): basedir = settings.SCRIPT_DIR + "/kontact" print(basedir); try: sh.rm("-R", ) except: print("nothing to remove") sh.cp("-R", "{}/config".format(basedir), "{}/{}".format(basedir, dirname)) sh.find("{}/{}".format(basedir, dirname), "-type", "f", "-exec", "sed", "-i", "s/{fullPrimaryEmail}/" + fullPrimaryEmail + "/g", "{}", "+") sh.find("{}/{}".format(basedir, dirname), "-type", "f", "-exec", "sed", "-i", "s/{primaryEmail}/" + primaryEmail + "/g", "{}", "+") sh.find("{}/{}".format(basedir, dirname), "-type", "f", "-exec", "sed", "-i", "s/{name}/" + name + "/g", "{}", "+") sh.find("{}/{}".format(basedir, dirname), "-type", "f", "-exec", "sed", "-i", "s/{uid}/" + uid + "/g", "{}", "+")
def sync_lychee(complete_sync=False): log.info("Starting Lychee synchronization") if complete_sync: # Delete broken links log.info("Removing broken symlinks for Lychee") sh.find(LYCHEE_DATA_PATH, "-xtype", "l", "-delete") exclusive_mode = 'replace' else: exclusive_mode = 'normal' try: perform_sync(False, exclusive_mode, True, False, True, False, BACKUP_PATH, LYCHEE_DATA_PATH, LYCHEESYNC_CONF_FILE) except Exception as e: log.exception("Unable to perform Lychee synchronization") # Flush disk buffers sh.sync() log.info("Finished Lychee synchronization")
def update_algorithm(algo, original_name, algorithm_pairs, remote_client, workspace_path, artifact_path): api_key = remote_client.apiKey api_address = remote_client.apiAddress destination_algorithm_name = algo.algoname destination_username = algo.username templatable_username = "******" repo_path = f"{workspace_path}/{destination_algorithm_name}" git_path = f"https://{algo.username}:{api_key}@git.{api_address.split('https://api.')[-1]}/git/{destination_username}/{destination_algorithm_name}.git" os.makedirs(artifact_path, exist_ok=True) os.makedirs(repo_path, exist_ok=True) clone_bake = sh.git.bake(C=workspace_path) publish_bake = sh.git.bake(C=repo_path) clone_bake.clone(git_path) sh.rm("-r", f"{repo_path}/src") sh.cp("-R", f"{artifact_path}/src", f"{repo_path}/src") sh.cp("-R", f"{artifact_path}/requirements.txt", f"{repo_path}/requirements.txt") sh.xargs.sed(sh.find(repo_path, "-not", "-path", "*/\.*", "-type", "f"), i=f"s/{templatable_username}/{destination_username}/g") for template_name, new_name in algorithm_pairs: sh.xargs.sed(sh.find(f"{repo_path}/src", "-not", "-path", "*/\.*", "-type", "f"), i=f"s/{template_name}/{new_name}/g") sh.mv(f"{repo_path}/src/{original_name}.py", f"{repo_path}/src/{destination_algorithm_name}.py") sh.mv(f"{repo_path}/src/{original_name}_test.py", f"{repo_path}/src/{destination_algorithm_name}_test.py") try: publish_bake.add(".") publish_bake.commit(m="automatic initialization commit") publish_bake.push() return algo except Exception as e: if "Your branch is up to date with" in str(e): print( f"algorithm {destination_username}/{destination_algorithm_name} is already up to date, skipping update..." ) pass else: raise e
def create(template, project_name): templates = get_templates() if template not in templates: click.echo('%s template not found' % template, err=True) return project_dir = './' + project_name sh.mkdir('-p', project_dir) sh.cp('-rf', os.path.join(HERE, 'templates/simple/'), project_dir) for f in sh.find(project_dir, '-name', '*.py'): sh.sed('-i', '', '-e', 's/%s/%s/g' % ('proj', project_name), f.strip()) sh.mv(os.path.join(project_dir, 'proj'), os.path.join(project_dir, project_name))
def _test(self, media_dir, mtimes, expected_structure): self._set_mtimes(media_dir, mtimes) # run organize.py on input dir shutil.rmtree(ORG_PATH, ignore_errors=True) os.system("python3 organize.py " + media_dir + " " + ORG_PATH) # test output dir self.assertTrue(os.path.isdir(ORG_PATH)) self.assertListEqual(sorted(sh.find('.', _cwd=ORG_PATH).splitlines()), expected_structure)
def edge_search(level): # this attempts to find near by levels to group together if not level["edges"]: return None right_of = find(level["edges"][1]) left_of = find(level["edges"][3]) east, west = None, None if right_of: east = right_of west = level elif left_of: east = level west = left_of else: return None ne_level, nw_level, se_level, sw_level = None, None, None, None if east["edges"][2] and west["edges"][2]: ne_level, nw_level = east, west se_level = find(east["edges"][2]) sw_level = find(west["edges"][2]) elif east["edges"][0] and west["edges"][0]: se_level, sw_level = east, west ne_level = find(east["edges"][0]) nw_level = find(west["edges"][0]) if ne_level and nw_level and se_level and sw_level: return (nw_level, ne_level, sw_level, se_level) else: return (west, east)
def disable_etc(chroot=None): '''Disable bluetooth etc. directories in /etc.''' for module in BadModules: etc_paths_text = sh.find(fullpath('/etc', chroot)) etc_paths = etc_paths_text.strip().split('\n') for path in etc_paths: if path.endswith('/{}'.format(module)): if os.path.exists(path): try: sh.mv('--force', path, path + '-unused') except e: print(e)
def compile_contig_matrices(): fs = sh.find(raw_ass, "-name", "*-smds.coverage.percontig").stdout.split("\n")[:-1] fs = [f for f in fs if "_2ref" in f] df = DataFrame.from_csv(fs[0],sep="\t") glob = df.ix[:,0:2] for f in fs: id = [c for c in f.split("/") if "IH" in c][0] values = DataFrame.from_csv(f,sep="\t")["cov_mean_sample_0"] assert sum([a!=b for a,b in zip(values.index, glob.index)]) == 0 if sum([a!=b for a,b in zip(values.index, glob.index)]) == 0: glob[id] = values else: print f, "is weird" glob.to_csv(stats_out + "all_contig_coverages.csv",sep="\t")
def _lookup_snapshots(self, tag): logger.debug('Searching for snapshots with tag: {} '.format(tag)) try: logger.debug('Executing: %s -name %s', self.cassandra_data_dir, tag) dirs = sh.find(self.cassandra_data_dir, '-name', tag) logger.debug('Found directory: {}'.format(dirs)) except: logger.exception('Unable to execute find, nodetool did not create snapshot?') sys.exit(1) dirs = '' return dirs.splitlines()
def get_args(argv): find_args = '-name *.docx -o -name *.odt'.split() files = sh.find(test_root('data'), *find_args).strip().split('\n') parser = argparse.ArgumentParser(argv) parser.add_argument("--pdf", action='store_true', help='also generate pdfs') parser.add_argument("-j", "--jobs", action='store', type=int, help='Parallelization level (default numcores)', default=multiprocessing.cpu_count()) parser.add_argument("--cache", action='store_true', default=True, help='also generate pdfs') parser.add_argument("--no-cache", action='store_false', dest='cache') parser.add_argument("test_files", nargs='*', default=files, help='the files to test (default: all)') return parser.parse_args()
def _get_python_path(self): _python_paths = [ sh.which('python'), sh.which('python3'), sh.which('python2') ] python_paths = [str(path) for path in _python_paths if path] if os.path.isfile('/usr/local/python-3.6.5/bin/python'): python_paths.append('/usr/local/python-3.6.5/bin/python') if os.path.isdir('/usr/local/Cellar/python'): out = sh.find('/usr/local/Cellar/python', '-regex', '.*/bin/python3[0-9.]*$', '-type', 'f', _piped=True) out = sh.sort(out, _piped=True) python_paths.append(sh.head(out, '-n1').strip()) useable_pythons = [] python_paths_set = set() for python_path in python_paths: if python_path in python_paths_set: continue python_paths_set.add(python_path) if os.path.realpath(python_path) in python_paths_set: continue python_paths_set.add(os.path.realpath(python_path)) useable_pythons.append( (python_path, self._get_python_version(python_path))) if len(useable_pythons) == 0: print('Not found python!!') sys.exit(1) error = '' while True: message = '{}\n{}select python path [{}]: '.format( '\n'.join([ '{}. {} (v{})'.format(i, *e) for i, e in enumerate(useable_pythons) ]), error, ','.join([str(i) for i in range(len(useable_pythons))])) num = int(input(message)) if num < 0 or num >= len(useable_pythons): error = 'error: invalid input, try again!! ' continue return useable_pythons[num]
def find_asan_rt_library(abi, asan_rt_path=''): if not asan_rt_path: find_path = os.environ['ANDROID_NDK_HOME'] candidates = split_stdout( sh.find(find_path, "-name", asan_rt_library_names(abi))) if len(candidates) == 0: common.MaceLogger.error( "Toolchain", "Can't find AddressSanitizer runtime library in %s" % find_path) elif len(candidates) > 1: common.MaceLogger.info( "More than one AddressSanitizer runtime library, use the 1st") return candidates[0] return "%s/%s" % (asan_rt_path, asan_rt_library_names(abi))
def find_asan_rt_library(abi, asan_rt_path=''): if not asan_rt_path: find_path = os.environ['ANDROID_NDK_HOME'] candidates = split_stdout(sh.find(find_path, "-name", asan_rt_library_names(abi))) if len(candidates) == 0: common.MaceLogger.error( "Toolchain", "Can't find AddressSanitizer runtime library in % s" % find_path) elif len(candidates) > 1: common.MaceLogger.info( "More than one AddressSanitizer runtime library, use the 1st") return candidates[0] return "%s/%s" % (asan_rt_path, asan_rt_library_names(abi))
def get_wc(content_dir): """ """ filetype = "*.markdown" cd(content_dir) files_list = find(".", "-name", "*.markdown") files_arr = files_list.split('\n') word_count = 0 for f in files_arr: if f: try: file_word_count = int(wc(cat(content_dir + f), "-w")) word_count += file_word_count except: pass return word_count
def _locate(self, *args, **kwargs): if not self._dont_update_db: try: print( 'If you do not want to run sudo, use the following command to configure Phigaro:' ) print(' phigaro-setup --no-updatedb') with sh.contrib.sudo: sh.updatedb() self._dont_update_db = True except sh.ErrorReturnCode_1: print('Invalid password') exit(1) try: return [s.rstrip() for s in sh.find(*args, **kwargs)] except sh.ErrorReturnCode_1: return []
def _remove_unneeded(self, lib_dir): self.log("removing unneeded files...") files = find(lib_dir).strip().splitlines() keep = [ "QtCore.so", "QtGui.so", "__init__.py", "_utils.py", "PySide", "" ] # empty means the whole pyside dir if IS_WIN: keep = [ "QtCore4.dll", "QtGui4.dll", "__init__.py", "_utils.py", "PySide", "QtGui.pyd", "QtCore.pyd", "" ] # empty means the whole pyside dir for f in files: if f.find("PySide") > 0: if os.path.split(f)[1] not in keep: rm("-rf", f) pass self.log("done.")
def find_image_list(): def generate_type_list(a_list): b_list = [['-name', name, '-o'] for name in a_list] # Flatten the list and take out the last '-o' return list(chain.from_iterable(b_list))[:-1] black_list = ['rootfs', 'bootfs', 'linux*', 'build*', '*.fs'] white_list = ['*.ext[1-5]', '*.cpio', '*.dd', '*.image', '*.img'] opt_b = [*'-type d'.split(), '(', generate_type_list(black_list), ')'] opt_w = [ *'( -type l -o -type f )'.split(), '(', generate_type_list(white_list), ')' ] logger.debug(opt_b) logger.debug(opt_w) sh.cd(IMAGE_DIR) img_list = [] exec_opt = ['-exec', 'file', '-L', '{}', ';'] img_list = sh.find(*opt_b, '-prune', '-o', *opt_w, *exec_opt, _tty_out=False) # Remove empty string and strip the first two characters "./" img_list = [ remove_control_chars(file_name)[2:] for file_name in img_list if file_name ] # Filter out images with broken symbolic link img_list = [text for text in img_list if 'cannot open' not in text] # Cut the string into two folds. [NAME: TYPE] img_list = [[text.split(':')[0], ''.join(text.split(':')[1:])] for text in img_list] # Transform into dictionary and parse type string img_list = [{ 'name': kv[0], 'type': imageParser.get_type(kv[1]) } for kv in img_list] logger.debug(img_list) return img_list
def fetch(self, output): clone_dir = self._make_temp_directory("rootfetch-android-git") sh.git.clone(self.GIT_URL, clone_dir) tags = set() for tag in sh.git.tag(_cwd=clone_dir, _iter=True): tags.add(tag.strip()) sh.git.checkout(self._latest_stable_tag(tags), _cwd=clone_dir) root_dir = os.path.join(clone_dir, "files") for cert_path in sh.find(root_dir, "-type", "f", _iter=True): cert_path = cert_path.strip() with open(cert_path) as fd: pem_parts = list() for line in fd: pem_parts.append(line) if line.startswith("-----END CERTIFICATE-----"): break output.write(''.join(pem_parts)) output.flush()
def _copy_results(lconf, output_dir): "copy interesting files from tmp dir to output dir" # copy the csv parses parsed_results = fp.join(output_dir, "parsed") if fp.exists(parsed_results): shutil.rmtree(parsed_results) shutil.copytree(lconf.tmp("parsed"), parsed_results) # copy the svg graphs into single flat dir graphs_dir = fp.join(output_dir, "graphs") makedirs(graphs_dir) svg_files = sh.find(minicorpus_path(lconf, result=True), "-name", "*.svg", _iter=True) for svg in (f.strip() for f in svg_files): svg2 = fp.join(graphs_dir, fp.basename(fp.dirname(svg)) + ".svg") shutil.copyfile(svg, svg2)
def find_entries(): """Returns a dictionary of keys corresponding to entries with a list of comments. For example: 'nafus-gender.html': { 'filename' : 'nafus-gender.html', 'mtime' : '2012-06-29', 'path' : 'social/gender/', 'comments' : ['nafus-gender-1341219620.15.cmt', 'nafus-gender-1341239352.82.cmt', 'nafus-gender-1341241456.95.cmt' ]}, } """ entries = {} for root, dirnames, filenames in walk(CONTENT_ROOT): for filename in filenames: #info(root, filename) entry = {'filename': filename} stats = stat(root+'/'+filename) entry['mtime'] = strftime("%Y-%m-%d", localtime(stats[8])).strip() fn_base, fn_ext = splitext(basename(filename)) path = root[len(CONTENT_ROOT):] entry['path'] = path+'/' info("fn_base = '%s'" % fn_base) if fn_base.startswith('.'): continue info(path, fn_base, fn_ext) comments = find(COMMENT_ROOT, "-name", fn_base+'*') if comments: info(" comments = %s" %comments) comments_list = [] for comment in comments: info(split(comment.strip())) comments_list.append(split(comment.strip())[1]) info(comments_list) entry['comments'] = comments_list entries[filename] = entry return entries
def main(args, remainder): if args.verbose > 1: print("Main arguments: %s\nRemaining args: %s" % (args, remainder)) cmd_list = build_cmd_list(args, remainder) cmd_list_print = build_cmd_list(args, remainder, quote_pattern=True) cmd_str = "find " + " ".join(cmd_list_print) if args.verbose or args.print: print("Command: '%s'" % (cmd_str)) if args.print: sys.exit(0) try: exec_cmd = find(*cmd_list, _iter=True) for l in exec_cmd: print(l, end='') except sh.ErrorReturnCode_1 as err: cprint("Error when executing command: '%s': %s" % (cmd_str, err), color='red')
def update_data_repos(repo_sel=None): """Update repositories.""" repos = sh.find(folder_data, '-name', '.git') repos = repos.stdout.decode().splitlines() if repo_sel is not None: repos = [x for x in repos if [y for y in repo_sel if x.find(y) >= 0]] msg_error = '' _log.warning('\nUpdating Working Tree and Syncing repository:') for repo in repos: rep = repo.rpartition('/')[0] sh.cd(rep) _log.warning('\n' + rep + ': ') status_repo = sh.git.status(porcelain=True).stdout.decode() if not status_repo: _log.warning('\tNothing to commit.') # If there are unmerged files from previous attempts: if status_repo and _add_and_commit_files(status_repo): _log.warning('\tStill have unmerged files. Solve it!') msg_error += '\n' + rep + ': Still have unmerged files. Solve it!' continue # Try to pull the changes if there are no conflicts # yet and identify new conflicts try: sh.git.pull() _log.warning('\tRepository fetched and merged.') except sh.ErrorReturnCode as error: _log.warning('\tThere are unmerged files.') msg_error += '\n' + rep + ': There are unmerged files.' continue # If no new conflicts were identified, try to push try: sh.git.push() _log.warning('\tRepository pushed.') except sh.ErrorReturnCode as error: _log.warning('\tProblem pushing repository.') msg_error += '\n' + rep + ': Problem pushing repository.' return msg_error
def __init__(self, name, k=31, name_coass = ""): if isinstance(name, list): self.name = "coass"+name_coass self.dir = raw_path self.reads = [raw_path + n + ".fastq.gz" for n in name] gzs = [g[:-1] for g in sh.find( raw_path , "-name", "*.fastq.gz") if any([n in g for n in name])] self.pairs = zip([ d for d in gzs if "R1" in d], [d for d in gzs if "R2" in d]) self.cleans = [[ f.replace("fastq.gz","clean.fastq").replace(raw_path,raw_ass).replace("/PRI_EUSM", "/cleans/PRI_EUSM") for f in p ] for p in self.pairs] self.singles = [p[0].replace("_R1_001.fastq.gz","_single.clean.fastq") for p in self.pairs] self.coass = True else : self.dir = raw_path +name +"/" self.coass = False self.name = name self.pairs = [[ d for d in os.listdir(self.dir) if ".fastq.gz" in d and "L001" in d], [d for d in os.listdir(self.dir) if ".fastq.gz" in d and "L002" in d]] self.cleans = [[ f.replace("fastq.gz","clean.fastq") for f in p ] for p in self.pairs] self.singles = [p[0].replace("_R1_001.fastq.gz","_single.clean.fastq") for p in self.pairs] self.k = k self.out_dir = raw_ass + self.name + "/" self.clean_dir = raw_ass + self.name + "/cleans/" self.merge_dir = raw_ass + self.name + "/merged/" if not os.path.exists(raw_ass): os.makedirs(raw_ass) if not os.path.exists(stats_out): os.makedirs(stats_out) if not os.path.exists(raw_ass + self.name): os.makedirs(raw_ass + self.name) self.success_ks = [int(f.split("_")[1]) for f in os.listdir(raw_ass + self.name) if "ray" in f] if not os.path.exists(self.out_dir): os.makedirs(self.out_dir) if not os.path.exists(self.clean_dir): os.makedirs(self.clean_dir) if os.path.exists(self.out_dir + self.name + "_stats.json"): print "Loading stats for", self.name with open(self.out_dir + self.name + "_stats.json") as handle: self.stats = json.load(handle)
def __init__(self): template = Template(templates[args.layout]) files = find(args.dir, "-type", "f", "-name", "*.json").strip().split("\n") for file in files: self.generate(file) if len(self.methods): print( template.render( title=args.title, description=args.description, navigation=self.navigation, methods=self.methods, header_inc=file_get_contents(args.header), footer_inc=file_get_contents(args.footer), ) ) else: parser.print_help()
def disable_lib_modules(chroot=None): """ This doesn't help when bluetooth etc. are in the kernel. Bluetooth and wi-fi are automatically installed and enabled by debian. None of the standard disabling methods like rmmod have any apparent effect. So rename /lib/modules/.../bluetooth /lib/modules/.../wireless to *-unused. """ for module in BadModules: lib_module_paths_text = sh.find(fullpath('/lib/modules', chroot)) lib_module_paths = lib_module_paths_text.strip().split('\n') for path in lib_module_paths: if path.endswith('/{}'.format(module)): if os.path.exists(path): try: sh.mv('--force', path, path + '-unused') except e: print(e)
def get(self): query = unquote(self.get_argument('q', '')) if query == '': self.redirect(self.settings.prefix) try: results = str(grep('-Rli', '--exclude-dir', '.git', "--", query, self.settings.repo)) except ErrorReturnCode_1 as e: results = '' try: results += str(find(self.settings.repo, '-type', 'f', '-iname', '*' + query + '*', '-not', '(', '-path', '%s/%s/*' % (self.settings.repo, '.git'), ')')) except ErrorReturnCode_1 as e: pass results = results.replace(self.settings.repo, '').split('\n')[:-1] formatted_results = [] for filename in results: if 'Binary file' in filename or filename == '': continue if path.basename(filename).startswith('.'): filename = path.join(path.dirname(filename), path.basename(filename)[1:]) resultpath = path.join(self.settings.repo, filename[1:]) if path.exists(resultpath): try: string = str(grep('-i', "--", query, resultpath )) except ErrorReturnCode_1 as e: string = '' string = self.highlight(string, query) formatted_results.append({'filename': filename, 'string': string}) self.render('search.html', query=query, results=formatted_results)
def compile_contig_matrices(self): fs = sh.find(self.merge_dir, "-name", "*-smds.coverage.percontig").stdout.split("\n")[:-1] df = DataFrame.from_csv(fs[0],sep="\t") glob = df.ix[:,0:2] for f in fs: id = [c.replace("map_","") for c in f.split("/") if "map_" in c][0] values = DataFrame.from_csv(f,sep="\t")["cov_mean_sample_0"] assert sum([a!=b for a,b in zip(values.index, glob.index)]) == 0 if sum([a!=b for a,b in zip(values.index, glob.index)]) == 0: glob[id] = values else: print f, "is weird" samples = list(set([c.split("_")[2] for c in mat.columns if "EUSM" in c])) processed = {} processed['length'] = glob['length'] processed['GC'] = glob['GC'] for s in tqdm(samples): processed[s]= glob[[c for c in glob.columns if s in c]].apply(sum, axis=1) glob = DataFrame.from_dict(processed) glob.to_csv(self.stats_out + self.name + "_contig_coverages.csv",sep="\t") glob[samples].to_csv(self.stats_out + self.name + "_contig_coverages_for_concoct.csv",sep="\t") return glob
def main(args=None): parser = gitboots.argparse_setup(globals()) parser.add_argument("-e", metavar="COMMAND", default="rm -f", help="execute command on each found lock") opts = parser.parse_args(args) path = git_dir() for lock in sh.find(path, "-type", "f", "-name", "index.lock"): lock = lock.strip() if opts.e != "rm -f": if "{}" in opts.e: cmd = opts.e.replace("{}", lock) else: cmd = "{cmd} {lock}".format(cmd=opts.e, lock=lock) logger.info(cmd) if sh.which("sh"): sh.sh("-c", cmd) elif sh.which("cmd.exe"): sh.cmd("\c", cmd) else: raise ValueError("Do not know how to run this command: %s" % cmd) else: logger.info("rm -f %s", lock) os.remove(lock)
self.stream.flush() def __getattr__(self, attr): return getattr(self.stream, attr) if __name__ == '__main__': sys.stdout = Unbuffered(sys.stdout) arguments = docopt(__doc__) input_folder = arguments['-i'] output_head = arguments['-o'] bps_fwd = int(arguments['-n']) if arguments['-n'] else 0 bps_rev = bps_fwd + 1 file_opener = gzip.open if arguments['-g'] else open lib_list_fwd = [f[:-1] for f in sh.find(input_folder) if "1P_fastq.gz" in f ] lib_list_rev = [f.replace("1P","2P") for f in lib_list_rev] #print lib_list handles = [open(l) for l in lib_list] out_file = output_head + "_%s.fasta" entry = "" ll = len(handles)-1 i=0 handle = None fileID = 0 min_seq_len = 40 g_counter = 0 trash_count = 0 while ll > -1:
#!/usr/bin/env python import codecs import sh import re note_path = '/home/zhongsiz/subler.github.io/note' md_set = sh.find(note_path, '-name', '*.md') print md_set def mon_convert(mon_str): if mon_str == 'Dec': return '12' elif mon_str == 'Nov': return '11' elif mon_str == 'Otc' or mon_str == 'Oct': return '10' elif mon_str == 'Sep': return '09' elif mon_str == 'Aug': return '08' elif mon_str == 'Jul': return '07' elif mon_str == 'Jun': return '06' elif mon_str == 'May': return '05' elif mon_str == 'Apr' or mon_str == 'Apri': return '04' elif mon_str == 'Mar': return '03'
def grade_q1(uniqname, repo): repo, path = clone('wk12-dotfiles', uniqname, repo) with sh.pushd(path): if os.path.exists(os.path.join(path, 'configs')): # Handle case of person who put stuff in folder.. files = set(sh.ls('-a', 'configs').split()) elif os.path.exists(os.path.join(path, 'bash')): # Handle case of person who put stuff in folder.. files = set(sh.ls('-a', 'bash').split()) elif os.path.exists(os.path.join(path, 'dotfiles')): # Handle case of person who put stuff in folder.. files = set(sh.ls('-a', 'dotfiles').split()) elif os.path.exists(os.path.join(path, 'shell')): # Handle case of person who put stuff in folder.. files = set(sh.ls('-a', 'shell').split()) else: files = set(sh.ls('-a').split()) files.discard('.') files.discard('..') files.discard('.git') text = '' grade = 0 # Viable .bashrc targets we've seen or are in the spirit of the Q bashrc = set(( '.bashrc', 'bashrc', 'bash.bashrc', '.bash_profile', 'bash_profile', '.zshrc', 'zshrc', 'mybashrc', 'bilraj.dotfiles.bashrc', )) if len(files.intersection(bashrc)): # Bashrc file text += '<dd><strong>.bashrc [1.0/1.0]</strong></dd>\n' grade += 1.0 else: text += '<dd><strong>.bashrc [0.0/1.0]</strong></dd>\n' text += '<dd>Output of <tt>find .</tt></dd>\n' text += '<dd><pre>{}</pre></dd>\n'.format(sh.find('.')) with grades_lock: no_bash.add(uniqname) grade += 0.0 others = files - bashrc nonfiles = set(( 'README', 'README.md', 'readme.md', 'README.txt', 'LICENSE', 'LICENSE.txt', 'setupDotfiles.sh', '3', 'update.sh', 'system_spec', 'init.sh', 'main.py', 'hello.sh', 'setupDotfiles.sh', 'status_repos.py', 'link.sh', 'test', 'test1.cpp', 'test2.cpp', 'test3.cpp', 'backup.sh', '.bashrc-miniconda3.bak', 'prog.cpp', 'file1.txt', 'file2.txt', 'file3.txt', 'dotfiles_old', 'setup_dotfiles.sh', 'source.sh', 'q2.py', 'games', 'duncanProf', 'trial.py', 'clean.sh', 'bootstrap.sh', 'q3.py', 'a.cpp', 'make_symlink.sh', 'update_repos.py', '__pycache__', 'test1', 'shell', 'a.h', 'copy.sh', 'scripts', 'makesymlinks.sh', 'os', 'set_up.sh', 'a', 'pro.py', 'link_files.sh', 'install.sh', 'main.pyc', )) with grades_lock: for o in others: other_dotfiles.add(o) if len(others): # Any other dotfile text += '<dd><strong>Any other dotfile ({}) [1.0/1.0]</strong></dd>\n'.format(others.pop()) grade += 1.0 else: text += '<dd><strong>Any other dotfile [0.0/1.0]</strong></dd>\n' text += '<dd>Output of <tt>find .</tt></dd>\n' text += '<dd><pre>{}</pre></dd>\n'.format(sh.find('.')) grade += 0.0 q1_entry = ''' <dt>Week 12</dt> <dd>{:1.1f}/2.0</dd> <dl> {}</dl> '''.format(grade, text) return grade, text
def generate_setlist(output_path): print "Generating level playlist..." boring, duplicates, by_hash, by_path = load_corpus() visited = [] paths_written = [] unprocessed = by_hash.keys() queue = [] quad_count = 0 pair_count = 0 ungrouped_count = 0 def find(key): # accepts either the level name or hash if key is None: return None assert(type(key) == str) found = None if by_path.has_key(key): found = by_path[key] elif by_hash.has_key(key): found = by_hash[key] if found and found["level_hash"] not in visited: return found return None def take_level(setlist, level): level_hash = level["level_hash"] visited.append(level_hash) try: queue.remove(level_hash) except: pass try: unprocessed.remove(level_hash) except: pass assert(level["path"] not in paths_written) paths_written.append(level["path"]) setlist.write(level["path"] + "\n") def enqueue(hash_or_path): level = find(hash_or_path) if level: level_hash = level["level_hash"] try: unprocessed.remove(level_hash) except: pass if level_hash not in queue and level_hash not in visited: queue.append(level_hash) def edge_search(level): # this attempts to find near by levels to group together if not level["edges"]: return None right_of = find(level["edges"][1]) left_of = find(level["edges"][3]) east, west = None, None if right_of: east = right_of west = level elif left_of: east = level west = left_of else: return None ne_level, nw_level, se_level, sw_level = None, None, None, None if east["edges"][2] and west["edges"][2]: ne_level, nw_level = east, west se_level = find(east["edges"][2]) sw_level = find(west["edges"][2]) elif east["edges"][0] and west["edges"][0]: se_level, sw_level = east, west ne_level = find(east["edges"][0]) nw_level = find(west["edges"][0]) if ne_level and nw_level and se_level and sw_level: return (nw_level, ne_level, sw_level, se_level) else: return (west, east) if by_path.has_key('onlinestartlocal.graal'): enqueue('onlinestartlocal.graal') elif by_path.has_key('level1.graal'): enqueue('level1.graal') with open(output_path, "w") as setlist: while len(unprocessed) + len(queue) > 0: pick = None if len(queue): pick = queue.pop(0) else: pick = unprocessed.pop(0) level = find(pick) assert(level is not None) edges = [] edge_odds = 0.30 adjacent_group = edge_search(level) if adjacent_group: if len(adjacent_group) == 2: pair_count += 1 setlist.write("*** pair ***\n") edge_odds = 0.20 elif len(adjacent_group) == 4: quad_count += 1 setlist.write("*** quad ***\n") edge_odds = 0.10 for tile in adjacent_group: take_level(setlist, tile) edges += tile["edges"] or [] else: ungrouped_count += 1 edges = level["edges"] or [] take_level(setlist, level) if level["doors"]: # add all levels linked by doors into the queue for target in level["doors"]: enqueue(target) # small chance of adding neighbors into the queue for edge_path in set(edges): if random.random() <= edge_odds: enqueue(edge_path) if duplicates: print "Duplicate levels which have been merged together:" def path_for_key(key): return os.path.split(by_hash[key]["path"])[-1] for path in sorted(map(path_for_key, duplicates.keys())): level_hash = by_path[path]["level_hash"] dupe_group = duplicates[level_hash] count = len(dupe_group) + 1 print " + {} ({}x, hash = {})".format(path, count, level_hash) for dupe in dupe_group: print " -", os.path.split(dupe["path"])[-1] print "quads found:", quad_count print "pairs found:", pair_count print "individuals:", ungrouped_count print "total tweets:", quad_count + pair_count + ungrouped_count print "total levels:", quad_count * 4 + pair_count * 2 + ungrouped_count
import sys import os import os.path assert os.path.exists ("./synth.c"), "tools must be invoked from the root of the repo" path = "./docs/docco/" if not os.path.exists (path): os.mkdir (path) docco = ["ext", "node_modules", "docco", "bin", "docco", "-o", path] import sh import subprocess for file in sh.find (".", "-name", "*.c"): subprocess.check_call (docco + [file])
def fixSampleName(x): if x.startswith("s_"): x=x[2:] return x def getTumorSamplesFromPairs(pairingFile): samples=set() with open(pairingFile) as fp: for line in fp: sample=line.strip().split()[1] samples.add(sample) return list(samples) samples=getTumorSamplesFromPairs(getPairingFile()) rdataFiles=sh.find("facets","-name","*_hisens.Rdata").strip().split("\n") projectNo=getProjectNo() rdataSampleMap=dict() for si in samples: tag="Proj_%s__%s" % (projectNo,fixSampleName(si)) for ri in rdataFiles: pos = ri.find(tag) if pos > -1: rdataSampleMap[si]=ri facetsMappingFile="mapping.txt" with open(facetsMappingFile,"w") as fp: HEADER="Tumor_Sample_Barcode Rdata_filename".split() print >>fp, "\t".join(HEADER) for si in sorted(rdataSampleMap): print >>fp, "\t".join([si,rdataSampleMap[si]])