def download_sample_packages(repository_root, samples_dir):
'''
Reads all the files named 'list' present in the samples directory
and downloads the packages in each 'list' file.
:param repository_root: url of the repository used.
:param samples_dir: directory that will be used to store the examples for the repository.
.. versionadded:: 0.1
'''
files_list = find_files(samples_dir, 'list')
for f in files_list:
download_path = os.path.dirname(f)
with open(f) as list_file:
# for line in list_file:
for line in list_file.readlines():
l = line.replace('\n', '').split('/')
try:
if not os.path.isfile(os.path.join(download_path, l[-1])):
logger.info("Downloading -> %s" % l[-1])
urllib.urlretrieve(os.path.join(repository_root, line),
os.path.join(download_path, l[-1]))
except urllib2.URLError, e:
logger.warning('Could not get %s, error code #%s' %
(l[-1], e.code))
def get_package_data(path=None, packages=None, data_files=None,
exclude_packages=None, exclude_files=None):
"""
For a list of packages, find the package_data
This function scans the subdirectories of a package and considers all
non-submodule subdirectories as resources, including them in
the package_data
Returns a dictionary suitable for setup(package_data=<result>)
"""
assert path is not None
assert packages is not None
assert data_files is not None
path = os.path.normpath(path)
package_data = {}
for package in packages:
package_data[package] = []
for f in find_files(path=get_path([path, package_to_path(package)]), pattern='*.*'):
package_data[package].append(f)
for e in exclude_packages + ['ez_setup', 'distribute_setup']:
if fnmatch.fnmatch(f, get_path([path, package_to_path(e), '*'])) \
and f in package_data[package]:
package_data[package].remove(f)
for x in exclude_files + ['*.py']:
if fnmatch.fnmatch(f, get_path([path, x])) \
and f in package_data[package]:
package_data[package].remove(f)
package_data[package] = list(
set(package_data[package]) - set(flatten_list(list(zip(*data_files)[1]))))
for i, j in enumerate(package_data[package]):
package_data[package][i] = package_data[package][i].replace(path + os.sep + package_to_path(package) + os.sep, '')
return package_data
def run(self):
log.debug("[%s.%s] Compressing JS." %
(__name__, self.__class__.__name__))
JSFULL_DIR = get_path(
[BASEDIR, 'tribus', 'data', 'static', 'js', 'full'])
JSMIN_DIR = get_path(
[BASEDIR, 'tribus', 'data', 'static', 'js', 'min'])
try:
os.makedirs(JSMIN_DIR)
except Exception as e:
print e
for JS_FILE in find_files(path=JSFULL_DIR, pattern='*.js'):
JSMIN_FILE = get_path([JSMIN_DIR, os.path.basename(JS_FILE)])
try:
with open(JSMIN_FILE, 'w') as _file:
_file.write(slimit.minify(open(JS_FILE).read()))
_file.close()
except Exception as e:
print e
log.debug("[%s.%s] %s > %s." %
(__name__, self.__class__.__name__, JS_FILE, JSMIN_FILE))
def run(self):
log.debug("[%s.%s] Compressing CSS." %
(__name__, self.__class__.__name__))
CSSFULL_DIR = get_path(
[BASEDIR, 'tribus', 'data', 'static', 'css', 'full'])
CSSMIN_DIR = get_path(
[BASEDIR, 'tribus', 'data', 'static', 'css', 'min'])
try:
os.makedirs(CSSMIN_DIR)
except Exception as e:
print e
for CSS_FILE in find_files(path=CSSFULL_DIR, pattern='*.css'):
CSSMIN_FILE = get_path([CSSMIN_DIR, os.path.basename(CSS_FILE)])
try:
with open(CSSMIN_FILE, 'w') as _file:
_file.write(cssmin.cssmin(open(CSS_FILE).read()))
_file.close()
except Exception as e:
print e
log.debug(
"[%s.%s] %s > %s." %
(__name__, self.__class__.__name__, CSS_FILE, CSSMIN_FILE))
def get_packages(path=None, exclude_packages=[]):
"""
Returns a list of all python packages found within directory ``path``, with
``exclude_packages`` packages excluded.
:param path: the path where the packages will be searched. It should
be supplied as a "cross-platform" (i.e. URL-style) path; it
will be converted to the appropriate local path syntax.
:param exclude_packages: is a sequence of package names to exclude;
``*`` can be used as a wildcard in the names,
such that ``foo.*`` will exclude all subpackages
of ``foo`` (but not ``foo`` itself).
:return: a list of packages.
:rtype: ``list``
.. versionadded:: 0.1
"""
assert path
assert exclude_packages
assert type(path) == str
assert type(exclude_packages) == list
pkgs = []
path = os.path.normpath(path)
for init in find_files(path=path, pattern='__init__.py'):
include = True
pkg = path_to_package(os.path.dirname(init).replace(path + os.sep, ''))
for exclude in exclude_packages:
if fnmatch.fnmatch(pkg, exclude + '*'):
include = False
if include:
pkgs.append(pkg)
return filter(None, pkgs)
def index_sample_packages():
'''
Indexa los paquetes descargados en el repositorio.
'''
from tribus.common.utils import list_items, find_files
from tribus.common.reprepro import include_deb
dirs = [
os.path.dirname(f) for f in find_files(env.sample_packages_dir, 'list')
]
dists = filter(None,
list_items(env.sample_packages_dir, dirs=True, files=False))
with lcd('%(reprepro_dir)s' % env):
for directory in dirs:
# No se me ocurre una mejor forma (dinamica) de hacer esto
dist = [
dist_name for dist_name in dists if dist_name in directory
][0]
results = [
each for each in os.listdir(directory) if each.endswith('.deb')
]
if results:
include_deb(env.reprepro_dir, dist, directory)
else:
logger.info('There are no packages in %s' % directory)
def test_find_files(self):
from tribus.common.utils import find_files
self.assertEqual(
sorted(find_files(path=self.tmpdir, pattern='*.txt')),
sorted([
'/tmp/test_io/1/1.txt', '/tmp/test_io/2/2.txt',
'/tmp/test_io/2/4.txt'
]))
def run(self):
for pyc_file in find_files(path=BASEDIR, pattern='*.pyc'):
if os.path.isfile(pyc_file):
try:
os.unlink(pyc_file)
log.debug("[%s.%s] Removing \"%s\"." %
(__name__, self.__class__.__name__, pyc_file))
except Exception as e:
print e
def handle(self, *args, **options):
for dist in list_items(SAMPLES_DIR, True, False):
for comp in list_items(os.path.join(SAMPLES_DIR, dist), True, False):
for sample in find_files(os.path.join(SAMPLES_DIR, dist, comp)):
try:
include_deb(LOCAL_ROOT, dist, comp, sample)
except:
logger.info('There are no packages here!')
def flake8_report(self):
"""
Outputs flake8 report.
"""
log.info("\n\nFlake8 Report:")
base = get_path([BASEDIR, 'tribus'])
pys = find_files(path=base, pattern='*.py')
flake8_style = get_style_guide()
report = flake8_style.check_files(pys)
exit_code = print_report(report, flake8_style)
def run(self):
for mo_file in find_files(path=BASEDIR, pattern='*.mo'):
if os.path.isfile(mo_file):
try:
shutil.rmtree(mo_file)
log.debug(
"[%s.%s] Removing \"%s\"." %
(__name__, self.__class__.__name__, mo_file))
except Exception as e:
print e
def index_selected():
from tribus.common.utils import list_items, find_files
from tribus.common.reprepro import include_deb
from tribus.config.pkgrecorder import LOCAL_ROOT, SAMPLES_DIR
for dist in list_items(SAMPLES_DIR, True, False):
for comp in list_items(os.path.join(SAMPLES_DIR, dist), True, False):
for sample in find_files(os.path.join(SAMPLES_DIR, dist, comp)):
with lcd('%(reprepro_dir)s' % env):
try:
include_deb(LOCAL_ROOT, dist, comp, sample)
except:
logger.info('There are no packages here!')
def fill_db_from_cache(cache_dir_path):
'''
Records the data from each control file in the cache folder into the database.
:param cache_dir_path: path where the package cache is stored.
.. versionadded:: 0.1
'''
local_branches = filter(
None, list_items(path=cache_dir_path, dirs=True, files=False))
for branch in local_branches:
dist_sub_paths = [
os.path.dirname(f) for f in find_files(
os.path.join(cache_dir_path, branch), 'Packages.gz')
]
for path in dist_sub_paths:
for p in find_files(path, "Packages.gz"):
for paragraph in deb822.Packages.iter_paragraphs(
gzip.open(p, 'r')):
record_paragraph(paragraph, branch)
def pep257_report(self):
"""
Outputs flake8 report.
"""
log.info("\n\nPEP257 Report:")
base = get_path([BASEDIR, 'tribus'])
pys = find_files(path=base, pattern='*.py')
report = pep257.check_files(pys)
if len(report) > 0:
for r in report:
log.info(r)
else:
log.info("\nNo errors found!")
def handle(self, *args, **options):
dirs = [
os.path.dirname(f) for f in find_files(sample_packages_dir, 'list')
]
dists = filter(None,
list_items(sample_packages_dir, dirs=True, files=False))
for directory in dirs:
# No se me ocurre una mejor forma (dinamica) de hacer esto
dist = [
dist_name for dist_name in dists if dist_name in directory
][0]
results = [
each for each in os.listdir(directory) if each.endswith('.deb')
]
if results:
include_deb(reprepro_dir, dist, directory)
else:
logger.info('There are no packages in %s' % directory)
def index_sample_packages():
from tribus.common.utils import list_items, find_files
dirs = [
os.path.dirname(f) for f in find_files(env.sample_packages_dir, 'list')
]
dists = filter(None,
list_items(env.sample_packages_dir, dirs=True, files=False))
with lcd('%(reprepro_dir)s' % env):
for directory in dirs:
# No se me ocurre una mejor forma (dinamica) de hacer esto
dist = [
dist_name for dist_name in dists if dist_name in directory
][0]
results = [
each for each in os.listdir(directory) if each.endswith('.deb')
]
if results:
with settings(command='reprepro includedeb %s %s/*.deb' %
(dist, directory)):
local('%(command)s' % env, capture=False)
else:
logger.info('There are no packages in %s' % directory)
def test_find_files_is_symlink(self):
from tribus.common.utils import find_files
self.assertTrue(
os.path.islink(
sorted(find_files(path=self.tmpdir, pattern='*.txt'))[2]))
