def clean_migrations(env):
"""
Merge uncommitted migrations
"""
migration_re = re.compile('^\d{4}_[a-z0-9_]+\.py$')
git_status = subprocess.Popen([
which.git, 'ls-files', '--others', '--exclude-standard',
],
stdout=subprocess.PIPE,
)
encoding = sys.getfilesystemencoding()
matches = collections.defaultdict(list)
for line in git_status.stdout:
file_path = path(line.strip().decode(encoding))
if '/migrations/' in file_path and migration_re.match(file_path.basename()):
matches[file_path.splitall()[1]].append(file_path)
git_status.wait()
for app_label, scripts in matches.items():
if len(scripts) <= 1:
scripts.clear()
if not any(matches.values()):
debug('No uncommited migration to merge')
for scripts in matches.values():
for script in scripts:
script.remove()
if not env.dry_run:
args = ['makemigrations']
args.extend(m for m in matches if matches[m])
call_task('django', args=args)
def _create_bootstrap(script_name, packages_to_install, paver_command_line,
install_paver=True, more_text="", dest_dir='.',
no_site_packages=None, system_site_packages=None,
unzip_setuptools=False, distribute=None, index_url=None,
find_links=None):
# configure easy install template
easy_install_options = []
if index_url:
easy_install_options.extend(["--index-url", index_url])
if find_links:
easy_install_options.extend(
["--find-links", " ".join(find_links)])
easy_install_options = (
easy_install_options
and "'%s', " % "', '".join(easy_install_options) or '')
confd_easy_install_tmpl = (_easy_install_tmpl %
('bin_dir', easy_install_options))
if install_paver:
paver_install = (confd_easy_install_tmpl %
('paver==%s' % setup_meta['version']))
else:
paver_install = ""
options = ""
# if deprecated 'no_site_packages' was specified and 'system_site_packages'
# wasn't, set it from that value
if system_site_packages is None and no_site_packages is not None:
system_site_packages = not no_site_packages
if system_site_packages is not None:
options += (" options.system_site_packages = %s\n" %
bool(system_site_packages))
if unzip_setuptools:
options += " options.unzip_setuptools = True\n"
if distribute is not None:
options += " options.use_distribute = %s\n" % bool(distribute)
options += "\n"
extra_text = """def adjust_options(options, args):
args[:] = ['%s']
%s
def after_install(options, home_dir):
if sys.platform == 'win32':
bin_dir = join(home_dir, 'Scripts')
else:
bin_dir = join(home_dir, 'bin')
%s""" % (dest_dir, options, paver_install)
for package in packages_to_install:
extra_text += confd_easy_install_tmpl % package
if paver_command_line:
command_list = list(paver_command_line.split())
extra_text += " subprocess.call([join(bin_dir, 'paver'),%s)" % repr(command_list)[1:]
extra_text += more_text
bootstrap_contents = venv.create_bootstrap_script(extra_text)
fn = script_name
debug("Bootstrap script extra text: " + extra_text)
def write_script():
open(fn, "w").write(bootstrap_contents)
dry("Write bootstrap script %s" % fn, write_script)
def _create_bootstrap(script_name, packages_to_install, paver_command_line,
install_paver=True, more_text="", dest_dir='.',
no_site_packages=None, system_site_packages=None,
unzip_setuptools=False, distribute=None, index_url=None,
no_index=False, find_links=None, prefer_easy_install=False):
# configure package installation template
install_cmd_options = []
if index_url:
install_cmd_options.extend(['--index-url', index_url])
if no_index:
install_cmd_options.extend(['--no-index'])
if find_links:
for link in find_links:
install_cmd_options.extend(
['--find-links', link])
install_cmd_tmpl = (_easy_install_tmpl if prefer_easy_install
else _pip_then_easy_install_tmpl)
confd_install_cmd_tmpl = (install_cmd_tmpl %
{'bin_dir_var': 'bin_dir', 'cmd_options': install_cmd_options})
# make copy to local scope to add paver to packages to install
packages_to_install = packages_to_install[:]
if install_paver:
packages_to_install.insert(0, 'paver==%s' % setup_meta['version'])
install_cmd = confd_install_cmd_tmpl % {'packages': packages_to_install}
options = ""
# if deprecated 'no_site_packages' was specified and 'system_site_packages'
# wasn't, set it from that value
if system_site_packages is None and no_site_packages is not None:
system_site_packages = not no_site_packages
if system_site_packages is not None:
options += (" options.system_site_packages = %s\n" %
bool(system_site_packages))
if unzip_setuptools:
options += " options.unzip_setuptools = True\n"
if distribute is not None:
options += " options.use_distribute = %s\n" % bool(distribute)
options += "\n"
extra_text = """def adjust_options(options, args):
args[:] = ['%s']
%s
def after_install(options, home_dir):
if sys.platform == 'win32':
bin_dir = join(home_dir, 'Scripts')
else:
bin_dir = join(home_dir, 'bin')
%s""" % (dest_dir, options, install_cmd)
if paver_command_line:
command_list = list(paver_command_line.split())
extra_text += " subprocess.call([join(bin_dir, 'paver'),%s)" % repr(command_list)[1:]
extra_text += more_text
bootstrap_contents = venv.create_bootstrap_script(extra_text)
fn = script_name
debug("Bootstrap script extra text: " + extra_text)
def write_script():
open(fn, "w").write(bootstrap_contents)
dry("Write bootstrap script %s" % fn, write_script)
def build(self):
"""
Invoke the r.js command. It checks the output of the r.js process to
write the list of files in ``cache_file``.
If the r.js process returns with anything else than 0, it raises a
``RuntimeError``.
"""
info('Writing %s', self.out)
command = self.get_command(
baseUrl=self.source,
mainConfigFile=self.config_js,
out=self.out,
**self.params
)
debug('Running: %s', ' '.join(command))
rjs_process = subprocess.Popen(
command,
stdout=subprocess.PIPE,
)
try:
files = self.parse_output(rjs_process.stdout)
finally:
rc = rjs_process.wait()
if rc != 0:
raise RuntimeError('r.js returned with {}'.format(rc))
# The config file is not added by requirejs as the list of files
files.append(self.config_js)
self.cache.write(files)
def _dep_is_ready(self, dep):
debug('>%s.%s is ready', dep.__module__, dep.__name__)
self._present.add(dep)
for dependencies_set in self._index[dep]:
if self._is_ready(dependencies_set):
for action in self._waiting_room.pop(dependencies_set, []):
self._run(action)
def quality(options):
"""Enforces PEP8"""
qc = QualityChecker(WARNING_CODES, ERROR_CODES)
report = qc()
debug('Report is %s', report)
out = getattr(options, 'output', '-')
if out == '-':
outfile = sys.stdout
else:
outfile = open(out, 'w')
try:
report.write(outfile)
finally:
if outfile is not sys.stdout:
outfile.close()
if not report:
return False
if report.has_errors:
error('Critical errors in quality check')
elif report.has_failures:
error('Errors in quality check')
elif report.has_warnings:
error('Warnings in quality check')
strictness = int(getattr(options, 'strictness', 2))
if report.level >= strictness:
raise SystemExit(1)
def gem_check(args):
"""Install a gem if it's not installed"""
to_install = [gem for gem in args if gem not in installed_gems]
if to_install:
call_task('gem_install', args=to_install)
else:
debug('Gems are synchonized')
def __call__(self):
self.start_observer()
debug('Processing loop')
try:
self.run()
finally:
self.stop_observer()
def write(self, files_list):
"""
Write the cache file
"""
debug('Writing cache in %s', self.cache_file)
with open(self.cache_file, 'w') as out_file:
for file in files_list:
out_file.write(text_type(path(file).realpath()))
out_file.write(u'\n')
def watch(self):
"""Watches the sass source directories and launches compilation as needed"""
debug('Building/watching with %s', self._sass)
try:
watcher = Watcher(self._sass)
info('Starting watch %s', self._sass)
watcher()
except KeyboardInterrupt:
pass
def npm_check(args):
"""Install a npm package if it's not installed"""
packages = [package for package in args
if package not in installed_packages and
package not in global_installed_packages]
if packages:
call_task('npm_install', args=packages)
else:
debug('NPM packages are installed')
def wheel(env):
call_task('bdist_wheel')
dist = _get_distribution()
for cmd, x, file in dist.dist_files:
if cmd == 'bdist_wheel':
env.wheel_file = path(file).abspath()
debug('Invented the wheel in %s', env.wheel_file)
break
else:
info('Cannot invent the wheel')
def _guess_settings():
if os.environ.get('DJANGO_SETTINGS_MODULE'):
import importlib
settings = importlib.import_module(os.environ['DJANGO_SETTINGS_MODULE'])
candidate = settings.__file__
if candidate.endswith('.pyc'):
candidate = candidate.rstrip('c')
debug('Guessed %s as settings file', candidate)
return candidate
raise RuntimeError('No settings file found, set DJANGO_SETTINGS_MODULE')
def __init__(self, builder):
self.observer = observers.Observer()
self.queue = moves.queue.Queue()
self._thread = None
self._stop = object()
self.watches = {}
for filepath in builder.paths:
watch = self.observer.schedule(EventDispatcher(self.queue), filepath, recursive=True)
debug('Watching %s', filepath)
self.watches[filepath] = watch
self.builder = builder
def autodiscover(self, tempdir, appdir):
"""
Tries to find under *appdir* in the *tempdir* all the ``js`` files and
returns the matching requirejs module names.
"""
args = []
info('Auto discovering JS %s', appdir)
appdir = path(appdir)
for dir in path(tempdir).walkdirs(appdir):
args.extend(text_type(appdir.joinpath(x.namebase)) for x in dir.files('*.js'))
debug('Autodicovered: %s', args)
return args
def _load(self):
for enabled_lib in self.enabled_libs:
if enabled_lib in self.disabled_libs:
continue
try:
module = importlib.import_module('sett.' + enabled_lib)
for var in vars(module).values():
if isinstance(var, Task):
yield var
except ImportError as ie:
debug('Error loading %s: %s', module, ie)
pass
def start(self):
info('Starting %s', self)
pid_dir = path(self.pid_file).dirname()
if not pid_dir.exists():
pid_dir.makedirs()
process = self._run(self.get_daemon_command())
if self.daemonize:
debug('Waiting for process')
process.wait()
elif self.daemonize is None:
debug('Writing pid in %s', self.pid_file)
self._set_pid(process.pid)
def _create_bootstrap(script_name,
packages_to_install,
paver_command_line,
install_paver=True,
more_text="",
dest_dir='.',
no_site_packages=False,
unzip_setuptools=False):
if install_paver:
paver_install = (_easy_install_template %
('bin_dir', 'paver==%s' % setup_meta['version']))
else:
paver_install = ""
options = ""
if no_site_packages:
options = " options.no_site_packages = True"
if unzip_setuptools:
if options:
options += "\n"
options += " options.unzip_setuptools = True"
if options:
options += "\n"
extra_text = """def adjust_options(options, args):
args[:] = ['%s']
%s
def after_install(options, home_dir):
if sys.platform == 'win32':
bin_dir = join(home_dir, 'Scripts')
else:
bin_dir = join(home_dir, 'bin')
%s""" % (dest_dir, options, paver_install)
for package in packages_to_install:
extra_text += _easy_install_template % ('bin_dir', package)
if paver_command_line:
command_list = []
command_list.extend(paver_command_line.split(" "))
extra_text += " subprocess.call([join(bin_dir, 'paver'),%s)" % repr(
command_list)[1:]
extra_text += more_text
bootstrap_contents = virtualenv.create_bootstrap_script(extra_text)
fn = script_name
debug("Bootstrap script extra text: " + extra_text)
def write_script():
open(fn, "w").write(bootstrap_contents)
dry("Write bootstrap script %s" % (fn), write_script)
def get_root():
root = ROOT
while True:
if root == '/':
break
node_modules = root.joinpath('node_modules')
if node_modules.isdir():
break
root = root.parent
else:
node_modules = ROOT.joinpath('node_modules')
debug('Node modules is %s', node_modules)
return node_modules
def __call__(self, tempdir, args):
if not args:
# Auto discover
args = self.autodiscover(tempdir, self.params.get('appdir', 'app'))
else:
debug('Filtered args: %s', args)
if not args:
info('No file to optimize')
return
build_list = list(self.get_builds(tempdir, args))
builder = parallel(self._build, n=min(4, len(build_list)))
builder.for_each(build_list)
def write_config(path, sections):
debug('Writting %s', path)
with open(path, 'w') as supervisord_conf_file:
for header, instructions in sections.items():
supervisord_conf_file.write('[{}]\n'.format(header))
for instruction, value in instructions.items():
if isinstance(value, bool):
value = str(value).lower()
elif not value:
debug('Skipping empty %s', instruction)
continue
supervisord_conf_file.write('{}={}\n'.format(
instruction, value))
supervisord_conf_file.write('\n')
def get_uwsgi_output():
out = defaults.UWSGI_OUTPUT_FORMAT
if out is None:
# Automatic mode
if yaml:
out = 'yml'
else:
out = 'xml'
debug('Guessing %s output', out)
if out == 'yml':
return YMLOutput(CONFIG_ROOT.joinpath('uwsgi.yml'))
elif out == 'xml':
return XMLOutput(CONFIG_ROOT.joinpath('uwsgi.xml'))
raise NotImplementedError('No format named {}'.format(out))
def _create_bootstrap(script_name, packages_to_install, paver_command_line,
install_paver=True, more_text="", dest_dir='.',
no_site_packages=False, unzip_setuptools=False):
if install_paver:
paver_install = (_easy_install_tmpl %
('bin_dir', 'paver==%s' % setup_meta['version']))
else:
paver_install = ""
options = """
options.no_site_packages = %s
if hasattr(options,"system_site_packages"):
options.system_site_packages = %s
"""%(bool(no_site_packages),not bool(no_site_packages))
if unzip_setuptools:
if options:
options += "\n"
options += " options.unzip_setuptools = True"
if options:
options += "\n"
extra_text = """def adjust_options(options, args):
args[:] = ['%s']
%s
def after_install(options, home_dir):
if sys.platform == 'win32':
bin_dir = join(home_dir, 'Scripts')
else:
bin_dir = join(home_dir, 'bin')
%s""" % (dest_dir, options, paver_install)
for package in packages_to_install:
extra_text += _easy_install_tmpl % ('bin_dir', package)
if paver_command_line:
command_list = list(paver_command_line.split())
extra_text += " subprocess.call([join(bin_dir, 'paver'),%s)" % repr(command_list)[1:]
extra_text += more_text
bootstrap_contents = venv.create_bootstrap_script(extra_text)
fn = script_name
debug("Bootstrap script extra text: " + extra_text)
def write_script():
open(fn, "w").write(bootstrap_contents)
dry("Write bootstrap script %s" % fn, write_script)
def test_runner(options):
debug(options)
from django.core.management import call_command
verbosity = int(options.test_runner.verbosity) if 'verbosity' in options.test_runner else 1
del sys.argv[1:]
for key, values in options.test_runner.items():
if not isinstance(values, list):
values = [values]
for value in values:
sys.argv.append(u'--' + key)
sys.argv.append(text_type(value))
debug('Calling django test with %s', sys.argv)
call_command('test', verbosity=verbosity)
def optional_import(module_name, package_name=None):
"""
Tries to import a module and returns either the module or a proxy class
that raises when accessing an attribute.
>>> models = optional_import('django.db.models')
>>> models.Model
RuntimeError('module django is not installed')
"""
try:
module = __import__(module_name)
if '.' in module_name:
for segment in module_name.split('.')[1:]:
module = getattr(module, segment)
return module
except ImportError as ie:
debug('Cannot import %s: %s', module_name, ie)
return FakeModule(module_name, package_name)
def init(environment):
"""
Initialize the paver
"""
environment.sett = sys.modules['sett']
environment.ROOT = ROOT
environment.defaults = defaults
environment.deployment = DeployContext
environment.which = which
environment.on_init = on_init
environment.task_finders.extend([
TaskAlternativeTaskFinder(loader, task_alternative),
SettTaskFinder(loader),
])
init = getattr(environment.pavement, 'init', None)
debug('Initializing %s', init)
if init and callable(init):
init(environment)
def rjs(options):
"""Usage: rjs [-f|--force] [-B|--builder builder_class_path] [-p APP] [-p APP...]
Build a requirejs app.
Rjs will call the virtual_static class (the default implementation is
sett.requirejs.virtual_static). This task should copy or link in the temp dir
given as the first arg all the static files required for the compilation as if
it was the static root.
Rjs requires npm apps requirejs. It will instanciate a builder class from the
class loaded from builder_class_path (sett.requirejs.RJSBuilder by default) and
launch a build either only of the apps given by the -p flag in the command line
or all autodiscovered apps when no -p is given.
The built files will be created in default.RJS_BUILD_DIR (build/static/js).
Default behavior of rjs is to build autonomous files that contains almond and all
their namespace. This behavior can be configured in a new builder class.
"""
outdir = ROOT.joinpath(defaults.RJS_BUILD_DIR)
outdir.makedirs()
rjs_params = dict(defaults.RJS_PARAMS)
command_params = getattr(options, 'rjs_params', {})
if not isinstance(command_params, dict):
command_params = [x.split('=', 1) for x in command_params]
rjs_params.update(command_params)
Builder = _cls(options, 'builder', RJSBuilder)
BuildClass = _cls(options, 'build_class', AlmondRJSBuild)
debug('Launching %s (%s)', Builder.__name__, BuildClass.__name__)
buidler = Builder(
force=options.get('force', False),
outdir=outdir,
build_class=BuildClass,
params=rjs_params,
)
with Tempdir() as tempdir:
call_task('virtual_static', args=[tempdir])
buidler(tempdir.joinpath('js'), options.get('paths', []))
def make_zip_archive(archive_name, basedir, files=None, file_pattern="*"):
import zipfile
archive_name = path(archive_name).abspath()
curdir = os.getcwd()
os.chdir(basedir)
files_count = 0
try:
# -- STEP: Collect files.
files2 = []
if files is None:
dirs = path(".").listdir(pattern="*")
files2.extend(dirs)
else:
for file_ in files:
if "*" in file_:
parts = path(".").glob(pattern=file_)
parts = [ part.normpath() for part in parts ]
files2.extend(parts)
else:
file_ = path(".")/file_
files2.append(file_.normpath())
files3 = []
for file_ in files2:
if file_.isdir():
files3.extend(file_.walkfiles(file_pattern))
else:
files3.append(file_)
files = files3
# -- STEP: Store files in archive.
archive = zipfile.ZipFile(archive_name, "w", zipfile.ZIP_DEFLATED)
for filename in files:
debug("ZIP: Store %s ..." % filename)
archive.write(filename)
files_count = len(archive.namelist())
archive.close()
finally:
os.chdir(curdir)
return files_count
def is_up_to_date(self):
"""
Determines if the module should be built. It should be built if:
* There is no out file.
* There is no cache.
* There is a file in the cache that has been modified since the out
file have been built.
"""
try:
out_build_time = self.out.stat().st_mtime
except OSError:
return True
debug('Read cache from %s', self.cache_file)
try:
line = self.cache_file
dep_write_time = 0
with open(self.cache_file, 'r') as file:
for line in file:
line = line.strip()
if '!' in line:
continue
dep_write_time = max(dep_write_time, os.stat(line).st_mtime)
except OSError as e:
debug('Choked on %s: %s', line, e)
return True
debug('out written at %s, max(deps) at %s', out_build_time, dep_write_time)
# Return should_write = True when the last dependency was written after
return dep_write_time > out_build_time
def make_zip_archive(archive_name, basedir, files=None, file_pattern="*"):
import zipfile
archive_name = path(archive_name).abspath()
curdir = os.getcwd()
os.chdir(basedir)
files_count = 0
try:
# -- STEP: Collect files.
files2 = []
if files is None:
dirs = path(".").listdir(pattern="*")
files2.extend(dirs)
else:
for file_ in files:
if "*" in file_:
parts = path(".").glob(pattern=file_)
parts = [part.normpath() for part in parts]
files2.extend(parts)
else:
file_ = path(".") / file_
files2.append(file_.normpath())
files3 = []
for file_ in files2:
if file_.isdir():
files3.extend(file_.walkfiles(file_pattern))
else:
files3.append(file_)
files = files3
# -- STEP: Store files in archive.
archive = zipfile.ZipFile(archive_name, "w", zipfile.ZIP_DEFLATED)
for filename in files:
debug("ZIP: Store %s ..." % filename)
archive.write(filename)
files_count = len(archive.namelist())
archive.close()
finally:
os.chdir(curdir)
return files_count
def wait(self):
if self.status == Threaded.INITIAL:
return
assert self.status == Threaded.STARTED
self.status = Threaded.ENDING
self._queue.join()
for t in self._threads:
self._queue.put((None, None))
debug('Waiting threads')
for t in self._threads:
t.join()
self.status = Threaded.ENDED
if self.failed_tasks:
raise RuntimeError('Those tasks failed: {}'.format(
'\n--\n'.join('{}{!r}'.format(self._fn, ft) for ft in self.failed_tasks)
))
return True
def add_installed_app(args):
name, = args
from django.conf import settings
if name in settings.INSTALLED_APPS:
info('App %s is already in INSTALLED_APPS', name)
return
settings_path = _guess_settings()
debug('Install app in %s', settings_path)
with open(settings_path, 'r+') as settings:
settings_lines = settings.readlines()
lines_iterator = enumerate(settings_lines, start=1)
for line_no, line in lines_iterator:
if line.startswith('INSTALLED_APPS'):
debug('Found INSTALLED_APPS at %s', line_no)
break
else:
debug('Did not found INSTALLED_APPS')
return
for line_no, line in lines_iterator:
code = line.split('#', 1)[0]
if ']' in code or ')' in code:
debug('Found last segment at %s', line_no)
break
else:
debug('Did not found list ending')
return
# Copy the indent of the last line
indent = ''.join(itertools.takewhile(str.isspace, settings_lines[line_no - 2]))
settings_lines.insert(line_no - 1, '{}\'{}\',\n'.format(indent, name))
settings.seek(0)
settings.writelines(settings_lines)
def _create_bootstrap(script_name,
packages_to_install,
paver_command_line,
install_paver=True,
more_text="",
dest_dir='.',
no_site_packages=None,
system_site_packages=None,
unzip_setuptools=False,
distribute=None,
index_url=None,
trusted_host=None,
no_index=False,
find_links=None,
prefer_easy_install=False):
# configure package installation template
install_cmd_options = []
if index_url:
install_cmd_options.extend(['--index-url', index_url])
if trusted_host:
install_cmd_options.extend(['--trusted-host', trusted_host])
if no_index:
install_cmd_options.extend(['--no-index'])
if find_links:
for link in find_links:
install_cmd_options.extend(['--find-links', link])
install_cmd_tmpl = (_easy_install_tmpl if prefer_easy_install else
_pip_then_easy_install_tmpl)
confd_install_cmd_tmpl = (install_cmd_tmpl % {
'bin_dir_var': 'bin_dir',
'cmd_options': install_cmd_options
})
# make copy to local scope to add paver to packages to install
packages_to_install = packages_to_install[:]
if install_paver:
packages_to_install.insert(0, 'paver==%s' % setup_meta['version'])
install_cmd = confd_install_cmd_tmpl % {'packages': packages_to_install}
options = ""
# if deprecated 'no_site_packages' was specified and 'system_site_packages'
# wasn't, set it from that value
if system_site_packages is None and no_site_packages is not None:
system_site_packages = not no_site_packages
if system_site_packages is not None:
options += (" options.system_site_packages = %s\n" %
bool(system_site_packages))
if unzip_setuptools:
options += " options.unzip_setuptools = True\n"
if distribute is not None:
options += " options.use_distribute = %s\n" % bool(distribute)
options += "\n"
extra_text = """def adjust_options(options, args):
args[:] = ['%s']
%s
def after_install(options, home_dir):
if sys.platform == 'win32':
bin_dir = join(home_dir, 'Scripts')
else:
bin_dir = join(home_dir, 'bin')
%s""" % (dest_dir, options, install_cmd)
if paver_command_line:
command_list = list(paver_command_line.split())
extra_text += " subprocess.call([join(bin_dir, 'paver'),%s)" % repr(
command_list)[1:]
extra_text += more_text
if has_virtualenv:
bootstrap_contents = venv.create_bootstrap_script(extra_text)
else:
raise BuildFailure(VIRTUALENV_MISSING_ERROR)
fn = script_name
debug("Bootstrap script extra text: " + extra_text)
def write_script():
open(fn, "w").write(bootstrap_contents)
dry("Write bootstrap script %s" % fn, write_script)
