def clean_migrations(env): """ Merge uncommitted migrations """ migration_re = re.compile('^\d{4}_[a-z0-9_]+\.py$') git_status = subprocess.Popen([ which.git, 'ls-files', '--others', '--exclude-standard', ], stdout=subprocess.PIPE, ) encoding = sys.getfilesystemencoding() matches = collections.defaultdict(list) for line in git_status.stdout: file_path = path(line.strip().decode(encoding)) if '/migrations/' in file_path and migration_re.match(file_path.basename()): matches[file_path.splitall()[1]].append(file_path) git_status.wait() for app_label, scripts in matches.items(): if len(scripts) <= 1: scripts.clear() if not any(matches.values()): debug('No uncommited migration to merge') for scripts in matches.values(): for script in scripts: script.remove() if not env.dry_run: args = ['makemigrations'] args.extend(m for m in matches if matches[m]) call_task('django', args=args)
def _create_bootstrap(script_name, packages_to_install, paver_command_line, install_paver=True, more_text="", dest_dir='.', no_site_packages=None, system_site_packages=None, unzip_setuptools=False, distribute=None, index_url=None, find_links=None): # configure easy install template easy_install_options = [] if index_url: easy_install_options.extend(["--index-url", index_url]) if find_links: easy_install_options.extend( ["--find-links", " ".join(find_links)]) easy_install_options = ( easy_install_options and "'%s', " % "', '".join(easy_install_options) or '') confd_easy_install_tmpl = (_easy_install_tmpl % ('bin_dir', easy_install_options)) if install_paver: paver_install = (confd_easy_install_tmpl % ('paver==%s' % setup_meta['version'])) else: paver_install = "" options = "" # if deprecated 'no_site_packages' was specified and 'system_site_packages' # wasn't, set it from that value if system_site_packages is None and no_site_packages is not None: system_site_packages = not no_site_packages if system_site_packages is not None: options += (" options.system_site_packages = %s\n" % bool(system_site_packages)) if unzip_setuptools: options += " options.unzip_setuptools = True\n" if distribute is not None: options += " options.use_distribute = %s\n" % bool(distribute) options += "\n" extra_text = """def adjust_options(options, args): args[:] = ['%s'] %s def after_install(options, home_dir): if sys.platform == 'win32': bin_dir = join(home_dir, 'Scripts') else: bin_dir = join(home_dir, 'bin') %s""" % (dest_dir, options, paver_install) for package in packages_to_install: extra_text += confd_easy_install_tmpl % package if paver_command_line: command_list = list(paver_command_line.split()) extra_text += " subprocess.call([join(bin_dir, 'paver'),%s)" % repr(command_list)[1:] extra_text += more_text bootstrap_contents = venv.create_bootstrap_script(extra_text) fn = script_name debug("Bootstrap script extra text: " + extra_text) def write_script(): open(fn, "w").write(bootstrap_contents) dry("Write bootstrap script %s" % fn, write_script)
def _create_bootstrap(script_name, packages_to_install, paver_command_line, install_paver=True, more_text="", dest_dir='.', no_site_packages=None, system_site_packages=None, unzip_setuptools=False, distribute=None, index_url=None, no_index=False, find_links=None, prefer_easy_install=False): # configure package installation template install_cmd_options = [] if index_url: install_cmd_options.extend(['--index-url', index_url]) if no_index: install_cmd_options.extend(['--no-index']) if find_links: for link in find_links: install_cmd_options.extend( ['--find-links', link]) install_cmd_tmpl = (_easy_install_tmpl if prefer_easy_install else _pip_then_easy_install_tmpl) confd_install_cmd_tmpl = (install_cmd_tmpl % {'bin_dir_var': 'bin_dir', 'cmd_options': install_cmd_options}) # make copy to local scope to add paver to packages to install packages_to_install = packages_to_install[:] if install_paver: packages_to_install.insert(0, 'paver==%s' % setup_meta['version']) install_cmd = confd_install_cmd_tmpl % {'packages': packages_to_install} options = "" # if deprecated 'no_site_packages' was specified and 'system_site_packages' # wasn't, set it from that value if system_site_packages is None and no_site_packages is not None: system_site_packages = not no_site_packages if system_site_packages is not None: options += (" options.system_site_packages = %s\n" % bool(system_site_packages)) if unzip_setuptools: options += " options.unzip_setuptools = True\n" if distribute is not None: options += " options.use_distribute = %s\n" % bool(distribute) options += "\n" extra_text = """def adjust_options(options, args): args[:] = ['%s'] %s def after_install(options, home_dir): if sys.platform == 'win32': bin_dir = join(home_dir, 'Scripts') else: bin_dir = join(home_dir, 'bin') %s""" % (dest_dir, options, install_cmd) if paver_command_line: command_list = list(paver_command_line.split()) extra_text += " subprocess.call([join(bin_dir, 'paver'),%s)" % repr(command_list)[1:] extra_text += more_text bootstrap_contents = venv.create_bootstrap_script(extra_text) fn = script_name debug("Bootstrap script extra text: " + extra_text) def write_script(): open(fn, "w").write(bootstrap_contents) dry("Write bootstrap script %s" % fn, write_script)
def build(self): """ Invoke the r.js command. It checks the output of the r.js process to write the list of files in ``cache_file``. If the r.js process returns with anything else than 0, it raises a ``RuntimeError``. """ info('Writing %s', self.out) command = self.get_command( baseUrl=self.source, mainConfigFile=self.config_js, out=self.out, **self.params ) debug('Running: %s', ' '.join(command)) rjs_process = subprocess.Popen( command, stdout=subprocess.PIPE, ) try: files = self.parse_output(rjs_process.stdout) finally: rc = rjs_process.wait() if rc != 0: raise RuntimeError('r.js returned with {}'.format(rc)) # The config file is not added by requirejs as the list of files files.append(self.config_js) self.cache.write(files)
def _dep_is_ready(self, dep): debug('>%s.%s is ready', dep.__module__, dep.__name__) self._present.add(dep) for dependencies_set in self._index[dep]: if self._is_ready(dependencies_set): for action in self._waiting_room.pop(dependencies_set, []): self._run(action)
def quality(options): """Enforces PEP8""" qc = QualityChecker(WARNING_CODES, ERROR_CODES) report = qc() debug('Report is %s', report) out = getattr(options, 'output', '-') if out == '-': outfile = sys.stdout else: outfile = open(out, 'w') try: report.write(outfile) finally: if outfile is not sys.stdout: outfile.close() if not report: return False if report.has_errors: error('Critical errors in quality check') elif report.has_failures: error('Errors in quality check') elif report.has_warnings: error('Warnings in quality check') strictness = int(getattr(options, 'strictness', 2)) if report.level >= strictness: raise SystemExit(1)
def gem_check(args): """Install a gem if it's not installed""" to_install = [gem for gem in args if gem not in installed_gems] if to_install: call_task('gem_install', args=to_install) else: debug('Gems are synchonized')
def __call__(self): self.start_observer() debug('Processing loop') try: self.run() finally: self.stop_observer()
def write(self, files_list): """ Write the cache file """ debug('Writing cache in %s', self.cache_file) with open(self.cache_file, 'w') as out_file: for file in files_list: out_file.write(text_type(path(file).realpath())) out_file.write(u'\n')
def watch(self): """Watches the sass source directories and launches compilation as needed""" debug('Building/watching with %s', self._sass) try: watcher = Watcher(self._sass) info('Starting watch %s', self._sass) watcher() except KeyboardInterrupt: pass
def npm_check(args): """Install a npm package if it's not installed""" packages = [package for package in args if package not in installed_packages and package not in global_installed_packages] if packages: call_task('npm_install', args=packages) else: debug('NPM packages are installed')
def wheel(env): call_task('bdist_wheel') dist = _get_distribution() for cmd, x, file in dist.dist_files: if cmd == 'bdist_wheel': env.wheel_file = path(file).abspath() debug('Invented the wheel in %s', env.wheel_file) break else: info('Cannot invent the wheel')
def _guess_settings(): if os.environ.get('DJANGO_SETTINGS_MODULE'): import importlib settings = importlib.import_module(os.environ['DJANGO_SETTINGS_MODULE']) candidate = settings.__file__ if candidate.endswith('.pyc'): candidate = candidate.rstrip('c') debug('Guessed %s as settings file', candidate) return candidate raise RuntimeError('No settings file found, set DJANGO_SETTINGS_MODULE')
def __init__(self, builder): self.observer = observers.Observer() self.queue = moves.queue.Queue() self._thread = None self._stop = object() self.watches = {} for filepath in builder.paths: watch = self.observer.schedule(EventDispatcher(self.queue), filepath, recursive=True) debug('Watching %s', filepath) self.watches[filepath] = watch self.builder = builder
def autodiscover(self, tempdir, appdir): """ Tries to find under *appdir* in the *tempdir* all the ``js`` files and returns the matching requirejs module names. """ args = [] info('Auto discovering JS %s', appdir) appdir = path(appdir) for dir in path(tempdir).walkdirs(appdir): args.extend(text_type(appdir.joinpath(x.namebase)) for x in dir.files('*.js')) debug('Autodicovered: %s', args) return args
def _load(self): for enabled_lib in self.enabled_libs: if enabled_lib in self.disabled_libs: continue try: module = importlib.import_module('sett.' + enabled_lib) for var in vars(module).values(): if isinstance(var, Task): yield var except ImportError as ie: debug('Error loading %s: %s', module, ie) pass
def start(self): info('Starting %s', self) pid_dir = path(self.pid_file).dirname() if not pid_dir.exists(): pid_dir.makedirs() process = self._run(self.get_daemon_command()) if self.daemonize: debug('Waiting for process') process.wait() elif self.daemonize is None: debug('Writing pid in %s', self.pid_file) self._set_pid(process.pid)
def _create_bootstrap(script_name, packages_to_install, paver_command_line, install_paver=True, more_text="", dest_dir='.', no_site_packages=False, unzip_setuptools=False): if install_paver: paver_install = (_easy_install_template % ('bin_dir', 'paver==%s' % setup_meta['version'])) else: paver_install = "" options = "" if no_site_packages: options = " options.no_site_packages = True" if unzip_setuptools: if options: options += "\n" options += " options.unzip_setuptools = True" if options: options += "\n" extra_text = """def adjust_options(options, args): args[:] = ['%s'] %s def after_install(options, home_dir): if sys.platform == 'win32': bin_dir = join(home_dir, 'Scripts') else: bin_dir = join(home_dir, 'bin') %s""" % (dest_dir, options, paver_install) for package in packages_to_install: extra_text += _easy_install_template % ('bin_dir', package) if paver_command_line: command_list = [] command_list.extend(paver_command_line.split(" ")) extra_text += " subprocess.call([join(bin_dir, 'paver'),%s)" % repr( command_list)[1:] extra_text += more_text bootstrap_contents = virtualenv.create_bootstrap_script(extra_text) fn = script_name debug("Bootstrap script extra text: " + extra_text) def write_script(): open(fn, "w").write(bootstrap_contents) dry("Write bootstrap script %s" % (fn), write_script)
def get_root(): root = ROOT while True: if root == '/': break node_modules = root.joinpath('node_modules') if node_modules.isdir(): break root = root.parent else: node_modules = ROOT.joinpath('node_modules') debug('Node modules is %s', node_modules) return node_modules
def __call__(self, tempdir, args): if not args: # Auto discover args = self.autodiscover(tempdir, self.params.get('appdir', 'app')) else: debug('Filtered args: %s', args) if not args: info('No file to optimize') return build_list = list(self.get_builds(tempdir, args)) builder = parallel(self._build, n=min(4, len(build_list))) builder.for_each(build_list)
def write_config(path, sections): debug('Writting %s', path) with open(path, 'w') as supervisord_conf_file: for header, instructions in sections.items(): supervisord_conf_file.write('[{}]\n'.format(header)) for instruction, value in instructions.items(): if isinstance(value, bool): value = str(value).lower() elif not value: debug('Skipping empty %s', instruction) continue supervisord_conf_file.write('{}={}\n'.format( instruction, value)) supervisord_conf_file.write('\n')
def get_uwsgi_output(): out = defaults.UWSGI_OUTPUT_FORMAT if out is None: # Automatic mode if yaml: out = 'yml' else: out = 'xml' debug('Guessing %s output', out) if out == 'yml': return YMLOutput(CONFIG_ROOT.joinpath('uwsgi.yml')) elif out == 'xml': return XMLOutput(CONFIG_ROOT.joinpath('uwsgi.xml')) raise NotImplementedError('No format named {}'.format(out))
def _create_bootstrap(script_name, packages_to_install, paver_command_line, install_paver=True, more_text="", dest_dir='.', no_site_packages=False, unzip_setuptools=False): if install_paver: paver_install = (_easy_install_tmpl % ('bin_dir', 'paver==%s' % setup_meta['version'])) else: paver_install = "" options = """ options.no_site_packages = %s if hasattr(options,"system_site_packages"): options.system_site_packages = %s """%(bool(no_site_packages),not bool(no_site_packages)) if unzip_setuptools: if options: options += "\n" options += " options.unzip_setuptools = True" if options: options += "\n" extra_text = """def adjust_options(options, args): args[:] = ['%s'] %s def after_install(options, home_dir): if sys.platform == 'win32': bin_dir = join(home_dir, 'Scripts') else: bin_dir = join(home_dir, 'bin') %s""" % (dest_dir, options, paver_install) for package in packages_to_install: extra_text += _easy_install_tmpl % ('bin_dir', package) if paver_command_line: command_list = list(paver_command_line.split()) extra_text += " subprocess.call([join(bin_dir, 'paver'),%s)" % repr(command_list)[1:] extra_text += more_text bootstrap_contents = venv.create_bootstrap_script(extra_text) fn = script_name debug("Bootstrap script extra text: " + extra_text) def write_script(): open(fn, "w").write(bootstrap_contents) dry("Write bootstrap script %s" % fn, write_script)
def test_runner(options): debug(options) from django.core.management import call_command verbosity = int(options.test_runner.verbosity) if 'verbosity' in options.test_runner else 1 del sys.argv[1:] for key, values in options.test_runner.items(): if not isinstance(values, list): values = [values] for value in values: sys.argv.append(u'--' + key) sys.argv.append(text_type(value)) debug('Calling django test with %s', sys.argv) call_command('test', verbosity=verbosity)
def optional_import(module_name, package_name=None): """ Tries to import a module and returns either the module or a proxy class that raises when accessing an attribute. >>> models = optional_import('django.db.models') >>> models.Model RuntimeError('module django is not installed') """ try: module = __import__(module_name) if '.' in module_name: for segment in module_name.split('.')[1:]: module = getattr(module, segment) return module except ImportError as ie: debug('Cannot import %s: %s', module_name, ie) return FakeModule(module_name, package_name)
def init(environment): """ Initialize the paver """ environment.sett = sys.modules['sett'] environment.ROOT = ROOT environment.defaults = defaults environment.deployment = DeployContext environment.which = which environment.on_init = on_init environment.task_finders.extend([ TaskAlternativeTaskFinder(loader, task_alternative), SettTaskFinder(loader), ]) init = getattr(environment.pavement, 'init', None) debug('Initializing %s', init) if init and callable(init): init(environment)
def rjs(options): """Usage: rjs [-f|--force] [-B|--builder builder_class_path] [-p APP] [-p APP...] Build a requirejs app. Rjs will call the virtual_static class (the default implementation is sett.requirejs.virtual_static). This task should copy or link in the temp dir given as the first arg all the static files required for the compilation as if it was the static root. Rjs requires npm apps requirejs. It will instanciate a builder class from the class loaded from builder_class_path (sett.requirejs.RJSBuilder by default) and launch a build either only of the apps given by the -p flag in the command line or all autodiscovered apps when no -p is given. The built files will be created in default.RJS_BUILD_DIR (build/static/js). Default behavior of rjs is to build autonomous files that contains almond and all their namespace. This behavior can be configured in a new builder class. """ outdir = ROOT.joinpath(defaults.RJS_BUILD_DIR) outdir.makedirs() rjs_params = dict(defaults.RJS_PARAMS) command_params = getattr(options, 'rjs_params', {}) if not isinstance(command_params, dict): command_params = [x.split('=', 1) for x in command_params] rjs_params.update(command_params) Builder = _cls(options, 'builder', RJSBuilder) BuildClass = _cls(options, 'build_class', AlmondRJSBuild) debug('Launching %s (%s)', Builder.__name__, BuildClass.__name__) buidler = Builder( force=options.get('force', False), outdir=outdir, build_class=BuildClass, params=rjs_params, ) with Tempdir() as tempdir: call_task('virtual_static', args=[tempdir]) buidler(tempdir.joinpath('js'), options.get('paths', []))
def make_zip_archive(archive_name, basedir, files=None, file_pattern="*"): import zipfile archive_name = path(archive_name).abspath() curdir = os.getcwd() os.chdir(basedir) files_count = 0 try: # -- STEP: Collect files. files2 = [] if files is None: dirs = path(".").listdir(pattern="*") files2.extend(dirs) else: for file_ in files: if "*" in file_: parts = path(".").glob(pattern=file_) parts = [ part.normpath() for part in parts ] files2.extend(parts) else: file_ = path(".")/file_ files2.append(file_.normpath()) files3 = [] for file_ in files2: if file_.isdir(): files3.extend(file_.walkfiles(file_pattern)) else: files3.append(file_) files = files3 # -- STEP: Store files in archive. archive = zipfile.ZipFile(archive_name, "w", zipfile.ZIP_DEFLATED) for filename in files: debug("ZIP: Store %s ..." % filename) archive.write(filename) files_count = len(archive.namelist()) archive.close() finally: os.chdir(curdir) return files_count
def is_up_to_date(self): """ Determines if the module should be built. It should be built if: * There is no out file. * There is no cache. * There is a file in the cache that has been modified since the out file have been built. """ try: out_build_time = self.out.stat().st_mtime except OSError: return True debug('Read cache from %s', self.cache_file) try: line = self.cache_file dep_write_time = 0 with open(self.cache_file, 'r') as file: for line in file: line = line.strip() if '!' in line: continue dep_write_time = max(dep_write_time, os.stat(line).st_mtime) except OSError as e: debug('Choked on %s: %s', line, e) return True debug('out written at %s, max(deps) at %s', out_build_time, dep_write_time) # Return should_write = True when the last dependency was written after return dep_write_time > out_build_time
def make_zip_archive(archive_name, basedir, files=None, file_pattern="*"): import zipfile archive_name = path(archive_name).abspath() curdir = os.getcwd() os.chdir(basedir) files_count = 0 try: # -- STEP: Collect files. files2 = [] if files is None: dirs = path(".").listdir(pattern="*") files2.extend(dirs) else: for file_ in files: if "*" in file_: parts = path(".").glob(pattern=file_) parts = [part.normpath() for part in parts] files2.extend(parts) else: file_ = path(".") / file_ files2.append(file_.normpath()) files3 = [] for file_ in files2: if file_.isdir(): files3.extend(file_.walkfiles(file_pattern)) else: files3.append(file_) files = files3 # -- STEP: Store files in archive. archive = zipfile.ZipFile(archive_name, "w", zipfile.ZIP_DEFLATED) for filename in files: debug("ZIP: Store %s ..." % filename) archive.write(filename) files_count = len(archive.namelist()) archive.close() finally: os.chdir(curdir) return files_count
def wait(self): if self.status == Threaded.INITIAL: return assert self.status == Threaded.STARTED self.status = Threaded.ENDING self._queue.join() for t in self._threads: self._queue.put((None, None)) debug('Waiting threads') for t in self._threads: t.join() self.status = Threaded.ENDED if self.failed_tasks: raise RuntimeError('Those tasks failed: {}'.format( '\n--\n'.join('{}{!r}'.format(self._fn, ft) for ft in self.failed_tasks) )) return True
def add_installed_app(args): name, = args from django.conf import settings if name in settings.INSTALLED_APPS: info('App %s is already in INSTALLED_APPS', name) return settings_path = _guess_settings() debug('Install app in %s', settings_path) with open(settings_path, 'r+') as settings: settings_lines = settings.readlines() lines_iterator = enumerate(settings_lines, start=1) for line_no, line in lines_iterator: if line.startswith('INSTALLED_APPS'): debug('Found INSTALLED_APPS at %s', line_no) break else: debug('Did not found INSTALLED_APPS') return for line_no, line in lines_iterator: code = line.split('#', 1)[0] if ']' in code or ')' in code: debug('Found last segment at %s', line_no) break else: debug('Did not found list ending') return # Copy the indent of the last line indent = ''.join(itertools.takewhile(str.isspace, settings_lines[line_no - 2])) settings_lines.insert(line_no - 1, '{}\'{}\',\n'.format(indent, name)) settings.seek(0) settings.writelines(settings_lines)
def _create_bootstrap(script_name, packages_to_install, paver_command_line, install_paver=True, more_text="", dest_dir='.', no_site_packages=None, system_site_packages=None, unzip_setuptools=False, distribute=None, index_url=None, trusted_host=None, no_index=False, find_links=None, prefer_easy_install=False): # configure package installation template install_cmd_options = [] if index_url: install_cmd_options.extend(['--index-url', index_url]) if trusted_host: install_cmd_options.extend(['--trusted-host', trusted_host]) if no_index: install_cmd_options.extend(['--no-index']) if find_links: for link in find_links: install_cmd_options.extend(['--find-links', link]) install_cmd_tmpl = (_easy_install_tmpl if prefer_easy_install else _pip_then_easy_install_tmpl) confd_install_cmd_tmpl = (install_cmd_tmpl % { 'bin_dir_var': 'bin_dir', 'cmd_options': install_cmd_options }) # make copy to local scope to add paver to packages to install packages_to_install = packages_to_install[:] if install_paver: packages_to_install.insert(0, 'paver==%s' % setup_meta['version']) install_cmd = confd_install_cmd_tmpl % {'packages': packages_to_install} options = "" # if deprecated 'no_site_packages' was specified and 'system_site_packages' # wasn't, set it from that value if system_site_packages is None and no_site_packages is not None: system_site_packages = not no_site_packages if system_site_packages is not None: options += (" options.system_site_packages = %s\n" % bool(system_site_packages)) if unzip_setuptools: options += " options.unzip_setuptools = True\n" if distribute is not None: options += " options.use_distribute = %s\n" % bool(distribute) options += "\n" extra_text = """def adjust_options(options, args): args[:] = ['%s'] %s def after_install(options, home_dir): if sys.platform == 'win32': bin_dir = join(home_dir, 'Scripts') else: bin_dir = join(home_dir, 'bin') %s""" % (dest_dir, options, install_cmd) if paver_command_line: command_list = list(paver_command_line.split()) extra_text += " subprocess.call([join(bin_dir, 'paver'),%s)" % repr( command_list)[1:] extra_text += more_text if has_virtualenv: bootstrap_contents = venv.create_bootstrap_script(extra_text) else: raise BuildFailure(VIRTUALENV_MISSING_ERROR) fn = script_name debug("Bootstrap script extra text: " + extra_text) def write_script(): open(fn, "w").write(bootstrap_contents) dry("Write bootstrap script %s" % fn, write_script)