def download_sample_packages(repository_root, samples_dir): ''' Reads all the files named 'list' present in the samples directory and downloads the packages in each 'list' file. :param repository_root: url of the repository used. :param samples_dir: directory that will be used to store the examples for the repository. .. versionadded:: 0.1 ''' files_list = find_files(samples_dir, 'list') for f in files_list: download_path = os.path.dirname(f) with open(f) as list_file: # for line in list_file: for line in list_file.readlines(): l = line.replace('\n', '').split('/') try: if not os.path.isfile(os.path.join(download_path, l[-1])): logger.info("Downloading -> %s" % l[-1]) urllib.urlretrieve(os.path.join(repository_root, line), os.path.join(download_path, l[-1])) except urllib2.URLError, e: logger.warning('Could not get %s, error code #%s' % (l[-1], e.code))
def get_package_data(path=None, packages=None, data_files=None, exclude_packages=None, exclude_files=None): """ For a list of packages, find the package_data This function scans the subdirectories of a package and considers all non-submodule subdirectories as resources, including them in the package_data Returns a dictionary suitable for setup(package_data=<result>) """ assert path is not None assert packages is not None assert data_files is not None path = os.path.normpath(path) package_data = {} for package in packages: package_data[package] = [] for f in find_files(path=get_path([path, package_to_path(package)]), pattern='*.*'): package_data[package].append(f) for e in exclude_packages + ['ez_setup', 'distribute_setup']: if fnmatch.fnmatch(f, get_path([path, package_to_path(e), '*'])) \ and f in package_data[package]: package_data[package].remove(f) for x in exclude_files + ['*.py']: if fnmatch.fnmatch(f, get_path([path, x])) \ and f in package_data[package]: package_data[package].remove(f) package_data[package] = list( set(package_data[package]) - set(flatten_list(list(zip(*data_files)[1])))) for i, j in enumerate(package_data[package]): package_data[package][i] = package_data[package][i].replace(path + os.sep + package_to_path(package) + os.sep, '') return package_data
def run(self): log.debug("[%s.%s] Compressing JS." % (__name__, self.__class__.__name__)) JSFULL_DIR = get_path( [BASEDIR, 'tribus', 'data', 'static', 'js', 'full']) JSMIN_DIR = get_path( [BASEDIR, 'tribus', 'data', 'static', 'js', 'min']) try: os.makedirs(JSMIN_DIR) except Exception as e: print e for JS_FILE in find_files(path=JSFULL_DIR, pattern='*.js'): JSMIN_FILE = get_path([JSMIN_DIR, os.path.basename(JS_FILE)]) try: with open(JSMIN_FILE, 'w') as _file: _file.write(slimit.minify(open(JS_FILE).read())) _file.close() except Exception as e: print e log.debug("[%s.%s] %s > %s." % (__name__, self.__class__.__name__, JS_FILE, JSMIN_FILE))
def run(self): log.debug("[%s.%s] Compressing CSS." % (__name__, self.__class__.__name__)) CSSFULL_DIR = get_path( [BASEDIR, 'tribus', 'data', 'static', 'css', 'full']) CSSMIN_DIR = get_path( [BASEDIR, 'tribus', 'data', 'static', 'css', 'min']) try: os.makedirs(CSSMIN_DIR) except Exception as e: print e for CSS_FILE in find_files(path=CSSFULL_DIR, pattern='*.css'): CSSMIN_FILE = get_path([CSSMIN_DIR, os.path.basename(CSS_FILE)]) try: with open(CSSMIN_FILE, 'w') as _file: _file.write(cssmin.cssmin(open(CSS_FILE).read())) _file.close() except Exception as e: print e log.debug( "[%s.%s] %s > %s." % (__name__, self.__class__.__name__, CSS_FILE, CSSMIN_FILE))
def get_packages(path=None, exclude_packages=[]): """ Returns a list of all python packages found within directory ``path``, with ``exclude_packages`` packages excluded. :param path: the path where the packages will be searched. It should be supplied as a "cross-platform" (i.e. URL-style) path; it will be converted to the appropriate local path syntax. :param exclude_packages: is a sequence of package names to exclude; ``*`` can be used as a wildcard in the names, such that ``foo.*`` will exclude all subpackages of ``foo`` (but not ``foo`` itself). :return: a list of packages. :rtype: ``list`` .. versionadded:: 0.1 """ assert path assert exclude_packages assert type(path) == str assert type(exclude_packages) == list pkgs = [] path = os.path.normpath(path) for init in find_files(path=path, pattern='__init__.py'): include = True pkg = path_to_package(os.path.dirname(init).replace(path + os.sep, '')) for exclude in exclude_packages: if fnmatch.fnmatch(pkg, exclude + '*'): include = False if include: pkgs.append(pkg) return filter(None, pkgs)
def index_sample_packages(): ''' Indexa los paquetes descargados en el repositorio. ''' from tribus.common.utils import list_items, find_files from tribus.common.reprepro import include_deb dirs = [ os.path.dirname(f) for f in find_files(env.sample_packages_dir, 'list') ] dists = filter(None, list_items(env.sample_packages_dir, dirs=True, files=False)) with lcd('%(reprepro_dir)s' % env): for directory in dirs: # No se me ocurre una mejor forma (dinamica) de hacer esto dist = [ dist_name for dist_name in dists if dist_name in directory ][0] results = [ each for each in os.listdir(directory) if each.endswith('.deb') ] if results: include_deb(env.reprepro_dir, dist, directory) else: logger.info('There are no packages in %s' % directory)
def test_find_files(self): from tribus.common.utils import find_files self.assertEqual( sorted(find_files(path=self.tmpdir, pattern='*.txt')), sorted([ '/tmp/test_io/1/1.txt', '/tmp/test_io/2/2.txt', '/tmp/test_io/2/4.txt' ]))
def run(self): for pyc_file in find_files(path=BASEDIR, pattern='*.pyc'): if os.path.isfile(pyc_file): try: os.unlink(pyc_file) log.debug("[%s.%s] Removing \"%s\"." % (__name__, self.__class__.__name__, pyc_file)) except Exception as e: print e
def handle(self, *args, **options): for dist in list_items(SAMPLES_DIR, True, False): for comp in list_items(os.path.join(SAMPLES_DIR, dist), True, False): for sample in find_files(os.path.join(SAMPLES_DIR, dist, comp)): try: include_deb(LOCAL_ROOT, dist, comp, sample) except: logger.info('There are no packages here!')
def flake8_report(self): """ Outputs flake8 report. """ log.info("\n\nFlake8 Report:") base = get_path([BASEDIR, 'tribus']) pys = find_files(path=base, pattern='*.py') flake8_style = get_style_guide() report = flake8_style.check_files(pys) exit_code = print_report(report, flake8_style)
def run(self): for mo_file in find_files(path=BASEDIR, pattern='*.mo'): if os.path.isfile(mo_file): try: shutil.rmtree(mo_file) log.debug( "[%s.%s] Removing \"%s\"." % (__name__, self.__class__.__name__, mo_file)) except Exception as e: print e
def index_selected(): from tribus.common.utils import list_items, find_files from tribus.common.reprepro import include_deb from tribus.config.pkgrecorder import LOCAL_ROOT, SAMPLES_DIR for dist in list_items(SAMPLES_DIR, True, False): for comp in list_items(os.path.join(SAMPLES_DIR, dist), True, False): for sample in find_files(os.path.join(SAMPLES_DIR, dist, comp)): with lcd('%(reprepro_dir)s' % env): try: include_deb(LOCAL_ROOT, dist, comp, sample) except: logger.info('There are no packages here!')
def fill_db_from_cache(cache_dir_path): ''' Records the data from each control file in the cache folder into the database. :param cache_dir_path: path where the package cache is stored. .. versionadded:: 0.1 ''' local_branches = filter( None, list_items(path=cache_dir_path, dirs=True, files=False)) for branch in local_branches: dist_sub_paths = [ os.path.dirname(f) for f in find_files( os.path.join(cache_dir_path, branch), 'Packages.gz') ] for path in dist_sub_paths: for p in find_files(path, "Packages.gz"): for paragraph in deb822.Packages.iter_paragraphs( gzip.open(p, 'r')): record_paragraph(paragraph, branch)
def pep257_report(self): """ Outputs flake8 report. """ log.info("\n\nPEP257 Report:") base = get_path([BASEDIR, 'tribus']) pys = find_files(path=base, pattern='*.py') report = pep257.check_files(pys) if len(report) > 0: for r in report: log.info(r) else: log.info("\nNo errors found!")
def handle(self, *args, **options): dirs = [ os.path.dirname(f) for f in find_files(sample_packages_dir, 'list') ] dists = filter(None, list_items(sample_packages_dir, dirs=True, files=False)) for directory in dirs: # No se me ocurre una mejor forma (dinamica) de hacer esto dist = [ dist_name for dist_name in dists if dist_name in directory ][0] results = [ each for each in os.listdir(directory) if each.endswith('.deb') ] if results: include_deb(reprepro_dir, dist, directory) else: logger.info('There are no packages in %s' % directory)
def index_sample_packages(): from tribus.common.utils import list_items, find_files dirs = [ os.path.dirname(f) for f in find_files(env.sample_packages_dir, 'list') ] dists = filter(None, list_items(env.sample_packages_dir, dirs=True, files=False)) with lcd('%(reprepro_dir)s' % env): for directory in dirs: # No se me ocurre una mejor forma (dinamica) de hacer esto dist = [ dist_name for dist_name in dists if dist_name in directory ][0] results = [ each for each in os.listdir(directory) if each.endswith('.deb') ] if results: with settings(command='reprepro includedeb %s %s/*.deb' % (dist, directory)): local('%(command)s' % env, capture=False) else: logger.info('There are no packages in %s' % directory)
def test_find_files_is_symlink(self): from tribus.common.utils import find_files self.assertTrue( os.path.islink( sorted(find_files(path=self.tmpdir, pattern='*.txt'))[2]))