def get_files(self, file_name_filters=None): input_zip = ZipFile(self.xlsm_doc_path) result = {} if not file_name_filters: file_name_filters = ['*'] for i in input_zip.namelist(): for filter in file_name_filters: if i == filter or fnmatch.fnmatch(i, filter): result[i] = input_zip.read(i) #Excel Crack is Wack... Excel converts \x5c to \x2f in zip file names and will happily eat it for you. Sample 51762ea84ac51f9e40b1902ebe22c306a732d77a5aa8f03650279d8b21271516 if not result: for i in input_zip.namelist(): for filter in file_name_filters: if i == filter.replace('\x2f','\x5c') or fnmatch.fnmatch(i, filter.replace('\x2f','\x5c')): result[i.replace('\x5c','\x2f')] = input_zip.read(i) return result
def _filter(dir, ls): incs = [opt.split('=').pop() for opt in opts if 'inc=' in opt] filtered = [] for f in ls: _f = os.path.join(dir, f) if not os.path.isdir(_f) and not _f.endswith('.py') and incs: if True not in [fnmatch.fnmatch(_f, inc) for inc in incs]: logging.debug('Not syncing %s, does not match include ' 'filters (%s)' % (_f, incs)) filtered.append(f) else: logging.debug('Including file, which matches include ' 'filters (%s): %s' % (incs, _f)) elif (os.path.isfile(_f) and not _f.endswith('.py')): logging.debug('Not syncing file: %s' % f) filtered.append(f) elif (os.path.isdir(_f) and ('test' in f)): logging.debug('Not syncing directory: %s' % f) filtered.append(f) elif (os.path.isdir(_f) and not os.path.isfile(os.path.join(_f, '__init__.py'))): logging.debug('Not syncing directory: %s' % f) filtered.append(f) return filtered
def do_search(self, arg): matches = [] for cmdname in self._custom_complete: if fnmatch.fnmatch(cmdname, arg): matches.append(cmdname) cprint("Found %s matches:" % len(matches)) print "\n".join(matches)
def _match(all_packages, key_glob_dict): """ Parameters ---------- all_packages : iterable Iterable of package metadata dicts from repodata.json key_glob_dict : iterable of kv pairs Iterable of (key, glob_value) dicts Returns ------- matched : dict Iterable of package metadata dicts which match the `target_packages` (key, glob_value) tuples """ matched = dict() key_glob_dict = { key.lower(): glob.lower() for key, glob in key_glob_dict.items() } for pkg_name, pkg_info in all_packages.items(): matched_all = [] # normalize the strings so that comparisons are easier for key, pattern in key_glob_dict.items(): name = str(pkg_info.get(key, '')).lower() if fnmatch.fnmatch(name, pattern): matched_all.append(True) else: matched_all.append(False) if all(matched_all): matched.update({pkg_name: pkg_info}) return matched
def _archive_get_files_from_glob(arc, glob): arr = [] for cffolder in arc.get_folders(): for cffile in cffolder.get_files(): filename = cffile.get_name().replace('\\', '/') if fnmatch.fnmatch(filename, glob): arr.append(cffile) return arr
def is_filename_allowed(self, name): if not self.file_masks: #print "masks not specified for component" return True for mask in self.file_masks.split(): if fnmatch.fnmatch(name, mask): #print "{} matches to {}".format(name, mask) return True return False
def get_files(self, file_name_filters=None): result = {} if not file_name_filters: file_name_filters = ['*'] for i in self._zf.namelist(): for filter in file_name_filters: if fnmatch.fnmatch(i, filter): result[i] = self._zf.read(i) return result
def types_by_filename(self, filename): """Returns a list of DocTypes matching for the given filename""" if not filename: #FIXME: return default type return [] rv = [] for test in self._globs.keys(): if fnmatch.fnmatch(filename, test): rv += self._globs[test] return rv
def types_by_filename(self, filename): """Returns a list of DocTypes matching for the given filename""" if not filename: # FIXME: return default type return [] rv = [] for test in self._globs.keys(): if fnmatch.fnmatch(filename, test): rv += self._globs[test] return rv
def get_files(self, file_name_filters=None): input_zip = ZipFile(self.xlsm_doc_path) result = {} if not file_name_filters: file_name_filters = ['*'] for i in input_zip.namelist(): for filter in file_name_filters: if i == filter or fnmatch.fnmatch(i, filter): result[i] = input_zip.read(i) return result
def matches(self, filename, filename_matches): for filename_match in filename_matches: if fnmatch.fnmatch(filename, filename_match): return True # perhaps 'filename' is part of a query string, so # try a regex match for filename_match in filename_matches: regex = re.compile(r'\\%s\?' % filename_match) if regex.search(filename): return True return False
def matches(self, filename, filename_matches): for filename_match in filename_matches: if fnmatch.fnmatch(filename, filename_match): return True # perhaps 'filename' is part of a query string, so # try a regex match for filename_match in filename_matches: regex = re.compile(r"\\%s\?" % filename_match) if regex.search(filename): return True return False
def _build_extra_yum_config(self, branch): """Build extra yum configuration for a branch's mock environment according to values in the releasers configuration file """ debug("looking for extra yum configuration") yum_config = [] if self.releaser_config.has_option(self.target, EXTRA_YUM_REPOS): debug("Appending extra yum configuration") yum_config.append(self.releaser_config.get(self.target, EXTRA_YUM_REPOS)) if self.releaser_config.has_option(self.target, EXTRA_YUM_REPOS_FOR): debug("Adding branch-specific extra yum configuraition") for pattern, yum_conf in json.loads(self.releaser_config.get(self.target, EXTRA_YUM_REPOS_FOR)): debug(" matching extra repos pattern '{0}' against '{1}'".format(pattern, branch)) if fnmatch.fnmatch(branch, pattern): debug(" found extra yum configuration for '{0}'".format(branch)) yum_config.append(yum_conf) return "\n".join(yum_config)
def list(self, filter_by=None, incremental=False, fetch_since_days=7): server = xmlrpclib.Server(self._pypi_xmlrpc_url) packages = server.list_packages() if not filter_by: return packages filtered_packages = [] for package in packages: if not True in [fnmatch.fnmatch(package, f) for f in filter_by]: continue filtered_packages.append(package) if incremental: changelog = server.changelog(int(time.time() - fetch_since_days * 24 * 3600)) changed_packages = [tp[0] for tp in changelog if "file" in tp[3]] changed_packages = [package for package in changed_packages if package in filtered_packages] return changed_packages else: return filtered_packages
def type_by_filename(self, filename): """Tries to find only one, the best guess for the type.""" if not filename: return None best = None best_glob = "" best_list = [] for test in self._globs.keys(): if fnmatch.fnmatch(filename, test): if len(test) > len(best_glob): best_glob = test best_list += self._globs[test] if best_glob == '': return None return self._globs[best_glob][0] if len(best_list) > 1: # erks. got more then one result. try different approach # guess the mimetype through the python mimetype lib #import mimetypes gtest = None import subprocess try: gtest = subprocess.Popen( ['file', '-bzki', filename], stdout=subprocess.PIPE).communicate()[0].strip() if gtest.find(';') != -1: gtest = gtest[:gtest.find(';')] except OSError: pass if gtest: for dt in best_list: if gtest in dt.mimes: best = dt else: # use the first one as total fallback :( best = best_list[0] elif len(best_list): best = best_list[0] return best
def list(self, filter_by=None, incremental=False, fetch_since_days=7): server = xmlrpclib.Server(self._pypi_xmlrpc_url) packages = server.list_packages() if not filter_by: return packages filtered_packages = [] for package in packages: if not True in [fnmatch.fnmatch(package, f) for f in filter_by]: continue filtered_packages.append(package) if incremental: changelog = server.changelog(int(time.time() - fetch_since_days*24*3600)) changed_packages = [tp[0] for tp in changelog if 'file' in tp[3]] changed_packages = [package for package in changed_packages if package in filtered_packages] return changed_packages else: return filtered_packages
def type_by_filename(self, filename): """Tries to find only one, the best guess for the type.""" if not filename: return None best = None best_glob = "" best_list = [] for test in self._globs.keys(): if fnmatch.fnmatch(filename, test): if len(test) > len(best_glob): best_glob = test best_list += self._globs[test] if best_glob == '': return None return self._globs[best_glob][0] if len(best_list) > 1: # erks. got more then one result. try different approach # guess the mimetype through the python mimetype lib #import mimetypes gtest = None import subprocess try: gtest = subprocess.Popen(['file', '-bzki', filename], stdout=subprocess.PIPE).communicate()[0].strip() if gtest.find(';') != -1: gtest = gtest[:gtest.find(';')] except OSError: pass if gtest: for dt in best_list: if gtest in dt.mimes: best = dt else: # use the first one as total fallback :( best = best_list[0] elif len(best_list): best = best_list[0] return best
def _build_extra_yum_config(self, branch): """Build extra yum configuration for a branch's mock environment according to values in the releasers configuration file """ debug('looking for extra yum configuration') yum_config = [] if self.releaser_config.has_option(self.target, EXTRA_YUM_REPOS): debug('Appending extra yum configuration') yum_config.append( self.releaser_config.get(self.target, EXTRA_YUM_REPOS)) if self.releaser_config.has_option(self.target, EXTRA_YUM_REPOS_FOR): debug('Adding branch-specific extra yum configuraition') for pattern, yum_conf in json.loads( self.releaser_config.get(self.target, EXTRA_YUM_REPOS_FOR)): debug(" matching extra repos pattern '{0}' against '{1}'". format(pattern, branch)) if fnmatch.fnmatch(branch, pattern): debug(" found extra yum configuration for '{0}'".format( branch)) yum_config.append(yum_conf) return '\n'.join(yum_config)
def _validator(k, v): # Check for exact match if k in type_table: try: return cast_value(v, type_table[k]) except ValueError: pass raise TypeError(k + ": invalid type") # Check wildcards (regexps in the future?) for (candidate_key, candidate_type) in type_table.items(): if fnmatch.fnmatch(k, candidate_key): try: return cast_value(v, candidate_type) except ValueError(): pass raise TypeError(k + ": invalid type") # No matches if relaxed: return v # Finally raise a typeerror raise TypeError(k + ": can't validate")
def _verify_username_vendor_glob(username, username_glob): for tmp in username_glob.split(','): if fnmatch.fnmatch(username, tmp): return True return False
def parse_deps(self, filename, ext_module, verify=True): """ Open a Cython file and extract all of its dependencies. INPUT: filename -- the file to parse verify -- only return existing files (default True) OUTPUT: list of dependency files """ is_cython_file = lambda f:\ fnmatch.fnmatch(f,'*.pyx') or \ fnmatch.fnmatch(f,'*.pxd') or \ fnmatch.fnmatch(f,'*.pxi') # only parse cython files if not is_cython_file(filename): return [] dirname = os.path.split(filename)[0] deps = set() if filename.endswith('.pyx'): pxd_file = filename[:-4] + '.pxd' if os.path.exists(pxd_file): deps.add(pxd_file) raw_deps = [] f = open(filename) for m in dep_regex.finditer(open(filename).read()): groups = m.groups() modules = groups[0] or groups[1] # cimport or from ... cimport if modules is not None: for module in modules.split(','): module = module.strip().split(' ')[0] # get rid of 'as' clause if '.' in module: path = module.replace('.', '/') + '.pxd' base_dependency_name = path else: path = "%s/%s.pxd" % (dirname, module) base_dependency_name = "%s.pxd"%module raw_deps.append((path, base_dependency_name)) else: # include or extern from extern_file = groups[2] or groups[3] path = '%s/%s'%(dirname, extern_file) if not os.path.exists(path): path = extern_file raw_deps.append((path, extern_file)) for path, base_dependency_name in raw_deps: # if we can find the file, add it to the dependencies. path = os.path.normpath(path) if os.path.exists(path): deps.add(path) # we didn't find the file locally, so check the # Cython include path. else: found_include = False for idir in ext_module.include_dirs + CYTHON_INCLUDE_DIRS + include_dirs + extra_include_dirs: new_path = os.path.normpath(idir + '/' + base_dependency_name) if os.path.exists(new_path): deps.add(new_path) found_include = True break new_path = os.path.normpath(idir + base_dependency_name[:-4] + "/__init__.pxd") if os.path.exists(new_path): deps.add(new_path) found_include = True break # so we really couldn't find the dependency -- raise # an exception. if not found_include: msg = 'could not find dependency %s included in %s.'%(path, filename) if is_cython_file(path): raise IOError, msg else: warnings.warn(msg+' I will assume it is a system C/C++ header.') f.close() return list(deps)
def _find(where, what): for dirpath, dirnames, filenames in os.walk(where): for item in dirnames + filenames: if fnmatch.fnmatch(item, what): return os.path.join(dirpath, item) return ''
def parse_deps(self, filename, ext_module, verify=True): """ Open a Cython file and extract all of its dependencies. INPUT: filename -- the file to parse verify -- only return existing files (default True) OUTPUT: list of dependency files """ is_cython_file = lambda f:\ fnmatch.fnmatch(f,'*.pyx') or \ fnmatch.fnmatch(f,'*.pxd') or \ fnmatch.fnmatch(f,'*.pxi') # only parse cython files if not is_cython_file(filename): return [] dirname = os.path.split(filename)[0] deps = set() if filename.endswith('.pyx'): pxd_file = filename[:-4] + '.pxd' if os.path.exists(pxd_file): deps.add(pxd_file) raw_deps = [] f = open(filename) for m in dep_regex.finditer(open(filename).read()): groups = m.groups() modules = groups[0] or groups[1] # cimport or from ... cimport if modules is not None: for module in modules.split(','): module = module.strip().split(' ')[ 0] # get rid of 'as' clause if '.' in module: path = module.replace('.', '/') + '.pxd' base_dependency_name = path else: path = "%s/%s.pxd" % (dirname, module) base_dependency_name = "%s.pxd" % module raw_deps.append((path, base_dependency_name)) else: # include or extern from extern_file = groups[2] or groups[3] path = '%s/%s' % (dirname, extern_file) if not os.path.exists(path): path = extern_file raw_deps.append((path, extern_file)) for path, base_dependency_name in raw_deps: # if we can find the file, add it to the dependencies. path = os.path.normpath(path) if os.path.exists(path): deps.add(path) # we didn't find the file locally, so check the # Cython include path. else: found_include = False for idir in ext_module.include_dirs + CYTHON_INCLUDE_DIRS + include_dirs + extra_include_dirs: new_path = os.path.normpath(idir + '/' + base_dependency_name) if os.path.exists(new_path): deps.add(new_path) found_include = True break new_path = os.path.normpath(idir + base_dependency_name[:-4] + "/__init__.pxd") if os.path.exists(new_path): deps.add(new_path) found_include = True break # so we really couldn't find the dependency -- raise # an exception. if not found_include: msg = 'could not find dependency %s included in %s.' % ( path, filename) if is_cython_file(path): raise IOError, msg else: warnings.warn( msg + ' I will assume it is a system C/C++ header.') f.close() return list(deps)
return dst else: try: urlretrieve(url, dst) info(" Downloaded: %s => %s", url, dst) return dst except Exception as e: error("Can't download %s: %s", url, e) return url def process_rst(path): info("Processing: %s", path) output = "" with open(path, 'r') as rst: for line in rst: urls = URL_RE.findall(line) for url in urls: dst = download_image(url) line = line.replace(url, relpath(dst, dirname(path))) output = output + line with open(path, 'w') as rst: rst.write(output) for directory, subdirs, fnames in walk("."): for fname in fnames: if fnmatch.fnmatch(fname, "*.rst"): process_rst(join(directory, fname))
def list_datasets(pattern="*"): from glob import fnmatch return [PwaDataset.from_file(x) for x in resource_listdir("pwa", "datasets") if x.endswith(".yaml") and fnmatch.fnmatch(x, pattern)]