def _get_packages(self): rprint("[yellow]get packages...") excluded_dirs = [ ".git", ".hg", "dist", "front", "node_modules", "shiny", "temp", "templates", "venv-*", ] example_folder = _wildcard_folder("example") if example_folder: excluded_dirs.append(example_folder) walk = filtered_walk(".", included_files=["__init__.py"], excluded_dirs=excluded_dirs) result = [] for path, subdirs, files in walk: if len(files): path = path.replace(os.sep, ".").strip(".") if path: result.append("{0}".format(path)) app_names = ["app"] if example_folder: app_names.append(example_folder) for name in app_names: if name in result: result.remove(name) result.insert(0, name) return result
def index_content(root_dir, file_types, content_type): """ Scan the media directory, creating an index of file properties for display and serving """ logger.debug('indexing') hasher = sha1() content_dir = os.path.join(root_dir, app.config['CONTENT_DIR']) files = file_paths(filtered_walk(content_dir, included_files=file_types, )) for contentfile in files: rel_path = os.path.relpath(contentfile, root_dir) filepath = os.path.join(root_dir, rel_path) filename = os.path.split(contentfile)[1] local_path = os.path.relpath(filepath, root_dir) if os.path.exists(os.path.join(filepath, 'folder.jpg')): img = os.path.join(filepath, 'folder.jpg') else: img = '' hasher.update(local_path) file_key = hasher.hexdigest() tags = _get_tags(filepath) media = Media() media.type = content_type media.path = filepath media.filename = filename media.file_id = file_key media.tags = tags media.img = img media.type = content_type media.save()
def index_podcasts(): files = file_paths( filtered_walk( 'static/podcasts', included_files=['*.mp3'], )) return files
def get_packages(): print(yellow("get packages...")) excluded_dirs=['.git', '.hg', 'dist', 'templates', 'venv-*'] example_folder = _wildcard_folder('example') if example_folder: excluded_dirs.append(example_folder) walk = filtered_walk( '.', included_files=['__init__.py'], excluded_dirs=excluded_dirs, ) result = [] for path, subdirs, files in walk: if len(files): path = path.replace(os.sep, '.').strip('.') if path: result.append('{0}'.format(path)) app_names = ['app'] if example_folder: app_names.append(example_folder) for name in app_names: if name in result: result.remove(name) result.insert(0, name) return result
def file(path: str, exts: list) -> list: ''' 在{path}中搜索所有扩展名符合{exts}的文件,并转换成绝对路径。 Args: path: 文件或目录 exts: 扩展名列表 Returns: list: 文件绝对路径列表 ''' # 将path转换为绝对路径 if not os.path.isabs(path): abs_path = os.path.join(os.getcwd(), path) else: abs_path = path # 遍历path,并返回指定扩展名的文件 if os.path.isfile(path) and os.path.splitext(path)[1].lower() in exts: return [abs_path] if os.path.isdir(abs_path): return filter( None, list( file_paths( filtered_walk(abs_path, included_files=['*' + e for e in exts])))) return []
def index_content(root_dir, file_types, content_type): """ Scan the media directory, creating an index of file properties for display and serving """ logger.debug('indexing') hasher = sha1() content_dir = os.path.join(root_dir, app.config['CONTENT_DIR']) files = file_paths(filtered_walk( content_dir, included_files=file_types, )) for contentfile in files: rel_path = os.path.relpath(contentfile, root_dir) filepath = os.path.join(root_dir, rel_path) filename = os.path.split(contentfile)[1] local_path = os.path.relpath(filepath, root_dir) if os.path.exists(os.path.join(filepath, 'folder.jpg')): img = os.path.join(filepath, 'folder.jpg') else: img = '' hasher.update(local_path) file_key = hasher.hexdigest() tags = _get_tags(filepath) media = Media() media.type = content_type media.path = filepath media.filename = filename media.file_id = file_key media.tags = tags media.img = img media.type = content_type media.save()
def main(srcdir, destpath, img_size, thread_number): #Iterate csv container csvFiles = file_paths( filtered_walk(srcdir, depth=1, included_files='*.csv')) for csvfile in list(csvFiles): #Download files down(csvfile, destpath, img_size, thread_number)
def directory_to_csv(dicom_path, csv_file_path, tags_in_files, tags_to_exclude): """ directory_to_csv iterates over a directory, finds dicom files with a .dcm extension and then creates a spreadsheet containing all of the tag values for the tags in the csv for every dicom file Args: dicom_path (str): Path to scan for dicom files. csv_file_path (str): Path and file name for the output csv file. tags_in_files (dict): Dictionary containing tags to include in the csv tags_to_exclude (dict): Dictionary containing tags to exclude in the csv Returns: None """ tags_in_files = tags_in_files.copy() # copy because we're going to modify for tag_to_exclude in tags_to_exclude: if tag_to_exclude in tags_in_files: del tags_in_files[tag_to_exclude] # sort by group and then element number tags_in_files = OrderedDict( sorted(tags_in_files.items(), key=(lambda k: (k[1][0], k[1][1])))) dicom_file_paths = file_paths( filtered_walk(dicom_path, included_files=["*.dcm"])) with open(csv_file_path, "w") as f: writer = csv.writer(f) # write the headers header_row = list(tags_in_files.keys()) header_row.append("FilePath") writer.writerow(header_row) # write the rows for dicom_file_path in dicom_file_paths: dicom_file = pydicom.read_file(dicom_file_path) row_vals = [] for keyword in tags_in_files: tag_val = dicom_file.get(keyword) if tag_val is None: tag_val = "" else: if isinstance(tag_val, Sequence) and not isinstance( tag_val, (str, bytes, bytearray)): tag_val = "^".join([str(x) for x in tag_val]) elif not isinstance(tag_val, str): tag_val = str(tag_val) tag_val = (tag_val.replace(",", "^").replace("\n", "").replace( "\r", "")) row_vals.append(tag_val) row_vals.append(dicom_file_path) writer.writerow(row_vals)
def walk(self): walk_iter = walkdir.filtered_walk(self.path) for dirpath, subdirs, files in walk_iter: for fname in files: real_path = os.path.abspath(os.path.join(dirpath, fname)) if os.path.islink(real_path): target = os.readlink(real_path)
def main(): arguments = docopt(__doc__) source = arguments['<source>'] dest = arguments['<dest>'] dirs = walkdir.dir_paths(walkdir.filtered_walk(source)) dirs = [d for d in dirs if d != '.' if list(Path(d).glob('*.mp3'))] random.shuffle(dirs) for d in dirs: print 'Copying {0} ...'.format(d) copy_dir_non_recursively(Path(d), Path(dest))
def main(): args = parseArgs() it = walkdir.filtered_walk( args.input_dir, included_files=['*.s', '*.cpp', '*.c', '*.cxx', '*.h', '*.hpp']) cmwalk = CmWalk() dir_depth = 0 for working_path, subdirs, files in it: print("Generating CMakeLists.txt in %s..." % working_path) # read configuration data of the working path form the json file, 'cfg' will be None if it does not exist. cfg = cmwalk.readCfgJson(working_path) addToCompilerIncludeDirectories = True if cfg: if 'sourceDirectories' in cfg.keys(): includedSourceDirs = [] for subdir in subdirs: if subdir in cfg['sourceDirectories']: includedSourceDirs.append(subdir) subdirs[:] = includedSourceDirs[:] elif 'ignoredDirectories' in cfg.keys(): for ignoredDir in cfg['ignoredDirectories']: try: subdirs.remove(ignoredDir) except: pass if 'ignoredFiles' in cfg.keys(): for ignoredFile in cfg['ignoredFiles']: try: files.remove(ignoredFile) except: pass if 'addToCompilerIncludeDirectories' in cfg.keys(): addToCompilerIncludeDirectories = cfg[ 'addToCompilerIncludeDirectories'] if dir_depth == 0: # generate top level CMakeLists.txt. cmwalk.genTopLevelDirCMakeListsFile(working_path, subdirs, files, cfg) else: # generate CMakeLists.txt in subdirectories. cmwalk.genSubDirCMakeListsFile(working_path, addToCompilerIncludeDirectories, subdirs, files) dir_depth += 1 print("\nFinished the generation of CMakeLists.txt files!")
def _display_files_not_restored(self, restore_to): print count = 0 files = file_paths(filtered_walk(restore_to)) for item in files: count = count + 1 print('{}. {}'.format(count, item)) if count: print(yellow( "The {} files listed above were not restored " "(just so you know).".format(count) )) print
def get_package_data(packages): print(yellow("get package data...")) result = {} for package in packages: if os.path.exists(package) and os.path.isdir(package): for folder_name in os.listdir(package): if folder_name in ('data', 'static', 'templates'): folder = os.path.join(package, folder_name) if os.path.isdir(folder): walk = filtered_walk(folder) for path, subdirs, files in walk: if package not in result: result[package] = [] # remove package folder name result[package].append(os.path.join(*path.split(os.sep)[1:])) return result
def write_manifest_in(is_project, packages): print(yellow("write MANIFEST.in...")) folders = [ 'doc_src', 'docs', ] for p in packages: if not '.' in p: folders = folders + [ os.path.join('{0}'.format(p), 'static'), os.path.join('{0}'.format(p), 'templates'), ] content = [] for f in folders: folder = os.path.join(os.getcwd(), f) if os.path.exists(folder) and os.path.isdir(folder): content.append('recursive-include {0} *'.format(f)) content = content + [ '', 'include LICENSE', ] if is_project: content.append('include manage.py') content = content + [ 'include README', 'include requirements/*.txt', 'include *.txt', '', ] # .yapsy-plugin walk = filtered_walk( '.', included_files=['*.{}'.format(YAPSY_PLUGIN_EXT)], ) for path, subdirs, files in walk: if files: content = content + [ 'include {}/*.{}'.format(path[2:], YAPSY_PLUGIN_EXT), ] # example folder example_folder = _wildcard_folder('example') if example_folder: content = content + [ 'prune {}/'.format(example_folder), ] with open('MANIFEST.in', 'w') as f: f.write('\n'.join(content))
def _get_package_data(self, packages): rprint("[yellow]get package data...") result = {} for package in packages: if os.path.exists(package) and os.path.isdir(package): for folder_name in os.listdir(package): if folder_name in ("data", "static", "templates"): folder = os.path.join(package, folder_name) if os.path.isdir(folder): walk = filtered_walk(folder) for path, subdirs, files in walk: if package not in result: result[package] = [] # remove package folder name result[package].append( os.path.join(*path.split(os.sep)[1:])) return result
def get_packages(): print(yellow("get packages...")) walk = filtered_walk( '.', included_files=['__init__.py'], excluded_dirs=['.hg', 'dist', 'example', 'templates'] ) result = [] for path, subdirs, files in walk: if len(files): path = path.replace(os.sep, '.').strip('.') if path: result.append('{0}'.format(path)) for app_name in ("app", "example"): if app_name in result: result.remove(app_name) result.insert(0, app_name) return result
def sequence(self, formats): '''return final list''' total = [] print 'iQuence Searching Path: %s' % self.topdir fileset = file_paths(filtered_walk(self.topdir, depth=self.depth ,included_files=formats)) #~ print files #for i in fileset: print i #~ for pattern in formats: #~ fileset = self.find_files(pattern) seqDict = self.extractDict(fileset) print seqDict seqlist = self.parseDict(seqDict) #~ print seqlist #~ total+=seqlist cmds = self.genViewerCmds(seqlist) #~ print cmds return cmds
def get_tags_in_files(dicom_path, tag_file_path): """ get_tags_in_files iterates over a directory, finds dicom files with a .dcm extension, and finds all unique dicom tag instances. it then writes the tags out as a csv file. Args: dicom_path (str): Path to scan for dicom files. tag_file_path (str): Path and file name for the output csv file. Returns: dict: A dictionary containing the tags loaded. """ # create the output directory if not os.path.exists(os.path.dirname(tag_file_path)): os.makedirs(os.path.dirname(tag_file_path)) # get the tags tags_in_files = {} dicom_file_paths = file_paths( filtered_walk(dicom_path, included_files=["*.dcm"])) for dicom_file_path in dicom_file_paths: dicom_file = pydicom.read_file(dicom_file_path) for item in dicom_file: if item.keyword not in tags_in_files: group = "0x%04x" % item.tag.group element = "0x%04x" % item.tag.element tags_in_files[ item.keyword] = group, element, item.keyword, item.name # sort the tags tags_in_files = OrderedDict( sorted(tags_in_files.items(), key=(lambda k: (k[1][0], k[1][1])))) # write out the file with open(tag_file_path, "w") as f: writer = csv.writer(f) writer.writerow(["group", "element", "keyword", "name"]) for item in tags_in_files: writer.writerow(tags_in_files[item]) return tags_in_files
def search(path): """ This method takes path to a directory as an argument and returns list of files under it in the format: filename: absolute path """ EXCLUDED_DIRS = ['env', 'venv', '.git' ] EXCLUDED_FILES = ['.gitignore', '*.pyc', '*.out' ] files = {} for i in file_paths(filtered_walk(path, excluded_dirs=EXCLUDED_DIRS, excluded_files=EXCLUDED_FILES)): files[os.path.basename(i)] = os.path.realpath(i) return files
def sequence(self, formats): '''return final list''' total = [] print 'iQuence Searching Path: %s' % self.topdir fileset = file_paths( filtered_walk(self.topdir, depth=self.depth, included_files=formats)) #~ print files #for i in fileset: print i #~ for pattern in formats: #~ fileset = self.find_files(pattern) seqDict = self.extractDict(fileset) print seqDict seqlist = self.parseDict(seqDict) #~ print seqlist #~ total+=seqlist cmds = self.genViewerCmds(seqlist) #~ print cmds return cmds
def main(args): root = args.path if not check(root): raise ValueError("Warning! Root directory isn't valid") max_dep = args.depth max_file = args.file assert max_dep > 0 and max_file > 0 print("{Visual Tree}") dir_count = 0 file_count = 0 for path, _, files in sorted(filtered_walk(root, depth=max_dep)): level = offset(root, path) base = os.path.basename(path) if level == 1: print(path) else: print(INDENT * level + SYMBOL + base) dir_count += 1 # display files level += 1 local_count = 0 display = True for file in sorted(files): file_count += 1 local_count += 1 if display: print(INDENT * level + SYMBOL + file) if local_count == max_file + 1: display = False print(INDENT * level + SYMBOL + "...") summary(root, dir_count, file_count)
def _write_manifest_in(self, is_project, packages): rprint("[yellow]write MANIFEST.in...") folders = ["doc_src", "docs"] for p in packages: if not "." in p: folders = folders + [ os.path.join("{0}".format(p), "static"), os.path.join("{0}".format(p), "templates"), ] content = [] for f in folders: folder = os.path.join(os.getcwd(), f) if os.path.exists(folder) and os.path.isdir(folder): content.append("recursive-include {0} *".format(f)) content = content + ["", "include LICENSE"] if is_project: content.append("include manage.py") content = content + [ "include README", "include requirements/*.txt", "include *.txt", "", ] # .yapsy-plugin walk = filtered_walk( ".", included_files=["*.{}".format(self.YAPSY_PLUGIN_EXT)]) for path, subdirs, files in walk: if files: content = content + [ "include {}/*.{}".format(path[2:], self.YAPSY_PLUGIN_EXT) ] # example folder example_folder = _wildcard_folder("example") if example_folder: content = content + ["prune {}/".format(example_folder)] with open("MANIFEST.in", "w") as f: f.write("\n".join(content))
def _grow(self): '''finds all available branches for tree''' for dir in walkdir.filtered_walk( self._root, excluded_dirs=[settings.BUILD_DIR]): # add the branch branch = Branch(dir[0], self.root) # add the nodes nodes = dir[-1] for filename in nodes: ext = os.path.splitext(filename)[-1] # ignores if filename in settings.EXCLUDED: continue # now build the nodes if filename in settings.UTILITY_NODES: node = UtilityNode(branch, filename) elif ext in settings.EXTENTIONS: node = DocNode(branch, filename) else: node = StaticNode(branch, filename) branch.add_node(node) # now add the branch self.add_branch(branch)
def dir_walk(self, walkpath, followlinks, depth): """ Make a list of directories under the top level The depth determines how many folders will be used in order to limit the memory used. """ if not os.path.isdir(walkpath): raise IOError(walkpath) level = 0 # if 0 it starts in and includes the root directory directoriesInLevels = [] directoriesInLevels.append((walkpath, level)) level += 1 #scan each level and make a list of base folders down to the depth requested, include links while level <= depth: for dirname in dir_paths( filtered_walk(str(walkpath), depth=level, min_depth=level, followlinks=followlinks)): directoriesInLevels.append((dirname, level)) level += 1 #for dirs in dir_paths(filtered_walk(walkpath,depth=depth,min_depth=depth,followlinks=followlinks)): #x.append((dirs,depth)) #x=[dirs[0:len(dirs)] for dirs in dir_paths(filtered_walk(walkpath,depth =1,min_depth=1,followlinks=followlinks))] return directoriesInLevels
if os.path.isdir(rootDir) and os.path.exists(rootDir): indentChar = " " # Depth of the root (i.e. number of "/") levelOffset = rootDir.count(os.path.sep) - 1 # Create filter excluded_filter = [] if not includeHidden: excluded_filter.append(".*") if not includeSystem: excluded_filter += system_file_names print ("\dirtree{%") for dirName, subdirList, fileList in sorted(filtered_walk(rootDir, depth=maxDepth, excluded_dirs=excluded_filter, excluded_files=excluded_filter)): level = get_relative_depth(dirName, levelOffset) baseName = os.path.basename(dirName) if level == 1: # for the first level only print the whole path print(indentChar + "." + str(level) + " {" + escape_illegal(dirName) + "} .") else: print(indentChar * level + "." + str(level) + " {" + escape_illegal((os.path.basename(dirName))) + "} .") level += 1 for fileName in sorted(fileList): print(indentChar * level + "." + str(level) + " {" + escape_illegal(fileName) + "} .") print ("}") else:
rootDir = delete_trailing_slash(pars.parse_args().path) includeHidden = pars.parse_args().includeHidden includeSystem = pars.parse_args().includeSystem maxDepth = pars.parse_args().maxDepth # if the directory exists if os.path.isdir(rootDir) and os.path.exists(rootDir): indentChar = " " # Depth of the root (i.e. number of "/") levelOffset = rootDir.count(os.path.sep) - 1 # Create filter excluded_filter = [] if not includeHidden: excluded_filter.append(".*") if not includeSystem: excluded_filter += system_file_names print ("\dirtree{%") for dirName, subdirList, fileList in sorted(filtered_walk(rootDir, depth=maxDepth, excluded_dirs=excluded_filter, excluded_files=excluded_filter)): level = get_relative_depth(dirName, levelOffset) baseName = os.path.basename(dirName) if level == 1: # for the first level only print the whole path print(indentChar + "." + str(level) + " {" + escape_illegal(dirName) + "} .") else: print(indentChar * level + "." + str(level) + " {" + escape_illegal((os.path.basename(dirName))) + "} .") level += 1 for fileName in sorted(fileList): print(indentChar * level + "." + str(level) + " {" + escape_illegal(fileName) + "} .") print ("}") else: print ("Error: root directory not found")
runpy.run_module('requests') # In[ ]: # In[29]: nbog = raw_input('Name of notebook to tag: ') # In[30]: files = file_paths(filtered_walk('/home/wcmckee/github/', depth=100, included_files=[nbog + '.ipynb'])) # In[35]: #Easier to access ipynb and access the code import cell. #Parse the notebook for import tags #IPython module wrapper for returning list of imported modules #from a ipynb file. #Get the list of modules for tags from this. # In[32]: for fil in files: #print fil
def index_podcasts(): files = file_paths(filtered_walk('static/podcasts', included_files=['*.mp3'], )) return files
def do_tree(self, args): """ Print the contents of the current directory in a format suitable for inclusion in some documentation. """ def colorize(path, fullpath): colors = ["blue", "cyan", "green", ""] color_tags = ["<" + color + ">>" if color else "" for color in colors] tag_index = -1 if os.path.islink(fullpath): tag_index = 1 elif os.path.isdir(fullpath): tag_index = 0 elif os.stat(fullpath).st_mode & stat.S_IXUSR: tag_index = 2 return color_tags[tag_index] + path + "<reset>>" files = all_paths(filtered_walk('.', depth=10, excluded_files=['*.pyc', '.gitignore'], excluded_dirs=[".*", ] ) ) l = list() for f in files: l.append(f) l.sort() previous = list() prev_depth = 0 for position, f in enumerate(l): current = f.split('/') if previous: i = 0 try: n = min(len(current), len(previous)) while i < n and current[i] == previous[i]: i += 1 except IndexError: print_exc() print("\nCurrent:\n") pprint(current) print("\nPrevious:\n") pprint(previous) TWIG = '── ' BRANCH = '├' + TWIG LEAF = '└' + TWIG TRUNK = '│' + ' ' l[position] = colorize(current[-1].strip(), f) if os.path.islink(f): target = os.readlink(f) l[position] += " -> " + colorize(target, target) if i: l[position] = BRANCH + l[position] if i > 1: l[position] = TRUNK * (i-1) + l[position] if i < prev_depth: l[position-1] = re.sub(BRANCH, LEAF, l[position-1]) prev_depth = i previous = current l[-1] = re.sub(BRANCH, LEAF, l[-1]) i = -1 while re.match(TRUNK, l[i]): l[i] = re.sub(TRUNK, INDENT, l[i]) i -= 1 l[i] = re.sub(BRANCH, LEAF, l[i]) printc ('\n'.join(l), right=">>")
def get_file_paths(root_paths, extension='flac'): pattern = '*.%s' % extension for root in root_paths: paths = file_paths(filtered_walk(root, included_files=[pattern])) for file_path in paths: yield file_path
nbog + etnam # In[21]: #Write the blog post #Ask to write content or publish writecont = input('Write content? y/N ') if 'y' in writecont: contenmak = input('content: ') else: #search or manually locate fil. pear = input('path to search: y/N') if 'y' in pear: files = file_paths(filtered_walk(pear, depth=100, included_files=[nbog + etnam])) for fil in files: print (fil) jsve.update({'filefound' : fil}) else: patlim = input('path of file: ') jsve.update({'filefound' : patlim + nbog + etnam}) # In[22]: #fil # In[23]:
def filtered_walk(self, *args, **kwds): return filtered_walk(self.walk(), *args, **kwds)
maxDepth = parser.parse_args().maxDepth # if the directory exists if os.path.isdir(rootDir) and os.path.exists(rootDir): indentChar = " " # Depth of the root levelOffset = rootDir.count(os.path.sep) - 1 # Create filter excluded_filter = [] if not includeHidden: excluded_filter.append(".*") if not includeSystem: excluded_filter += system_file_names print("\dirtree{%") for dirName, subdirList, fileList in sorted( filtered_walk(rootDir, depth=maxDepth, excluded_dirs=excluded_filter)): level = get_relative_depth(dirName, levelOffset) baseName = os.path.basename(dirName) if level == 1: #for the first level only print the wole path print('|' + "." + str(level) + "{" + escape_illegal(dirName) + "}.") else: print('|' + indentChar * (level - 1) + "." + str(level) + "{" + escape_illegal(dirName) + "}.") level += 1 for fileName in sorted(fileList): print('|' + indentChar * (level - 1) + "." + str(level) + " {" + escape_illegal(fileName) + "} .") print("}") else:
def filtered_walk(self, *args, **kwds): return filtered_walk(os.fwalk(self.root_folder), *args, **kwds)
import os.path from walkdir import filtered_walk, dir_paths, all_paths, file_paths files = file_paths(filtered_walk('.', included_files=['*.go'])) packages = {} for f in files: dirn = os.path.dirname(f) if dirn not in packages: packages[dirn] = {} packages[dirn][f] = 1 for n in packages: print "{0}.o: {1}".format(n, " ".join(packages[n].keys())) print "\t$(GCCGO) -L . -I . -c -g -o $@ $^" print "all_pkgs:" + " ".join(["{}.o".format(n) for n in packages]) print "\teho"