def run(self, env): try: self.task.started(TYPE) for extension in self.task.extensions: extension.started(TYPE) with hooks.task_run([self.task] + self.task.extensions): self._run(env) for extension in self.task.extensions: extension.finished(TYPE) self.task.finished(TYPE) except (ConnectionError, AMQPConnectionError): log.exception() for extension in self.task.extensions: extension.failed(TYPE) self.task.failed(TYPE) raise_error("Lost connection to AMQP server") except Exception as e: log.exception() for extension in self.task.extensions: extension.failed(TYPE) self.task.failed(TYPE) raise e finally: if self.connection is not None: utils.call_and_catch(self.connection.close) return self.task
def _get_ftp(self): try: username, password = self._get_auth() if self._tls: ftp = FTP_TLS(self._uri, timeout=CONNECT_TIMEOUT) else: ftp = FTP(self._uri, timeout=CONNECT_TIMEOUT) ftp.login(username, password) if self._tls: ftp.prot_d() if not catch(ftp.cwd, self._path): if self._path.startswith("/"): ftp.cwd("/") components = self._path.split("/") for component in components: if not catch(ftp.cwd, component): ftp.mkd(component) ftp.cwd(component) return ftp except Exception: log.exception() log.warning( "[FTP] failed to establish server connection, disabled") self._disabled = True return None
def catch(func, *args, **kwargs): try: val = func(*args, **kwargs) return val if val is not None else True except Exception: log.exception() return False
def upload(self, node, force=False): if not self._upload and not force: return True with self._cache.get_artifact(node) as artifact: path = self._get_path(node, artifact) temp = self._get_temp(node, artifact) try: log.verbose("[VOLUME] Copying {}", path) fs.copy(artifact.get_archive_path(), temp) # To avoid race-condition, make sure that the artifact still is missing before moving it into place. if not fs.exists(path): fs.rename(temp, path) else: fs.unlink(temp) return True except OSError as e: if e.errno != errno.EEXIST: log.verbose("[VOLUME] Failed to copy artifact, errno={}", os.strerror(e.errno)) return e.errno == errno.EEXIST except Exception: log.exception() finally: fs.unlink(temp, ignore_errors=True) return False
def get_influence(self, task): self.tools = Tools(task, task.joltdir) try: manifest_path = fs.path.join(task.joltdir, task.expand(self.path)) manifest = RepoManifest(task, manifest_path) manifest.parse(fs.path.join(manifest_path, ".repo", "manifest.xml")) result = [] for project in sorted(manifest.projects, key=lambda p: p.name): if self.include is not None and project.path_or_name not in self.include: continue if self.exclude is not None and project.path_or_name in self.exclude: continue with _git_repo_lock: gip = _git_repos.get(project.path_or_name) if gip is None: gip = git.GitInfluenceProvider(project.path_or_name) _git_repos[project.path_or_name] = gip result.append(gip.get_influence(task)) return "\n".join(result) except KeyError: log.exception() assert False, "failed to calculate hash influence for repo manifest at {0}".format( self.path)
def publish(self, artifact, tools): with tools.cwd(tools.builddir()): try: pinned_reqs = self.dependencies if pinned_reqs: tools.write_file("requirements.txt", "\n".join(pinned_reqs)) artifact.collect("requirements.txt") except Exception: log.exception() with tools.cwd(_path): artifact.collect('README.rst') artifact.collect('setup.py') artifact.collect('jolt/*.py') artifact.collect('jolt/*.sh') artifact.collect('jolt/*/*.py') artifact.collect('jolt/*/*/*.py') artifact.collect('jolt/*/*.xslt') artifact.collect('jolt/*/*.template') artifact.collect('jolt/plugins/selfdeploy/README.rst', flatten=True) artifact.collect('jolt/plugins/selfdeploy/setup.py', flatten=True) for e in self.extra_files: with tools.cwd(fs.path.dirname(e)): artifact.collect(fs.path.basename(e))
def call_and_catch_and_log(f, *args, **kwargs): try: return f(*args, **kwargs) except KeyboardInterrupt as e: raise e except Exception: from jolt import log log.exception() return None
def wait(self): for future in as_completed(self.futures): task = self.futures[future] try: future.result() except Exception as error: log.exception() return task, error finally: self.duration_acc += task.duration_running or 0 del self.futures[future] return task, None return None, None
def run(self, env): if self.is_aborted(): return try: self.task.started() with hooks.task_run(self.task): self.task.run(env.cache, force_build=self.force_build, force_upload=self.force_upload) except Exception as e: log.exception() self.task.failed() raise e else: self.task.finished() return self.task
def download(self, node, force=False): if not self._download and not force: return False with self._cache.get_artifact(node) as artifact: path = self._get_path(node, artifact) try: log.verbose("[VOLUME] Copying {}", path) fs.copy(path, artifact.get_archive_path()) return True except OSError as e: if e.errno == errno.ESTALE: log.verbose("[VOLUME] got stale file handle, retrying...") raise StaleFileHandleError(e) else: log.exception() except Exception: log.exception() return False
def selfdeploy(self): """ Installs the correct version of Jolt as specified in execution request. """ tools = Tools() manifest = JoltManifest() try: manifest.parse() ident = manifest.get_parameter("jolt_identity") url = manifest.get_parameter("jolt_url") if not ident or not url: return "jolt" requires = manifest.get_parameter("jolt_requires") log.info("Jolt version: {}", ident) src = "build/selfdeploy/{}/src".format(ident) env = "build/selfdeploy/{}/env".format(ident) if not fs.path.exists(env): try: fs.makedirs(src) tools.run("curl {} | tar zx -C {}", url, src) tools.run("virtualenv {}", env) tools.run(". {}/bin/activate && pip install -e {}", env, src) if requires: tools.run( ". {}/bin/activate && pip install {}", env, requires) except Exception as e: tools.rmtree("build/selfdeploy/{}", ident, ignore_errors=True) raise e return ". {}/bin/activate && jolt".format(env) except Exception as e: log.exception() raise e
def inspect(ctx, task, influence=False, artifact=False, salt=None): """ View information about a task. This command displays information about a task, such as its class documentation, parameters and their accepted values, requirements, task class origin (file/line), influence attributes, artifact identity, cache status, and more. Default parameter values, if any, are highlighted. """ task_name = task task_cls_name, task_params = utils.parse_task_name(task_name) task_registry = TaskRegistry.get() task = task_registry.get_task_class(task_cls_name) raise_task_error_if(not task, task_name, "no such task") from jolt import inspection print() print(" {0}".format(task.name)) print() if task.__doc__: print(" {0}".format(task.__doc__.strip())) print() print(" Parameters") has_param = False params = { key: getattr(task, key) for key in dir(task) if isinstance(utils.getattr_safe(task, key), Parameter) } for item, param in params.items(): has_param = True print(" {0:<15} {1}".format(item, param.help or "")) if not has_param: print(" None") print() print(" Definition") print(" {0:<15} {1} ({2})".format( "File", fs.path.relpath(inspection.getfile(task), JoltLoader.get().joltdir), inspection.getlineno(task))) print() print(" Requirements") manifest = ctx.obj["manifest"] try: task = task_registry.get_task(task_name, manifest=manifest) for req in sorted( utils.as_list(utils.call_or_return(task, task.requires))): print(" {0}".format(task.tools.expand(req))) if not task.requires: print(" None") print() except Exception as e: log.exception() if "has not been set" in str(e): print(" Unavailable (parameters must be set)") print() return print(" Unavailable (exception during evaluation)") print() return if salt: task.taint = salt if artifact: acache = cache.ArtifactCache.get() builder = graph.GraphBuilder(task_registry, manifest) dag = builder.build([task.qualified_name]) tasks = dag.select(lambda graph, node: node.task is task) assert len(tasks) == 1, "graph produced multiple tasks, one expected" proxy = tasks[0] task = proxy.task print(" Cache") print(" Identity {0}".format(proxy.identity)) if acache.is_available_locally(proxy): with acache.get_artifact(proxy) as artifact: print(" Location {0}".format(artifact.path)) print(" Local True ({0})".format( utils.as_human_size(acache.get_artifact(proxy).get_size()))) else: print(" Local False") print(" Remote {0}".format( acache.is_available_remotely(proxy))) print() if influence: print(" Influence") for string in HashInfluenceRegistry.get().get_strings(task): string = string.split(":", 1) print(" {:<18}{}".format(string[0][10:], string[1].strip()))
def _config(ctx, list, delete, global_, user, key, value): """ Configure Jolt. You can query/set/replace/unset configuration keys with this command. Key strings are constructed from the configuration section and the option separated by a dot. There are tree different configuration sources: - A global configuration file - A user configuration file - Temporary configuration passed on the command line. When reading, the values are read from all configuration sources. The options --global and --user can be used to tell the command to read from only one of the sources. If a configuration key is available from multiple sources, temporary CLI configuration has priority followed by the user configuration file and lastly the global configuration file. When writing, the new values are written to the user configuration by default. The options --global and --user can be used to tell the command to write to only one of the sources. When removing keys, the values are removed from all sources. The options --global and --user can be used to restrict removal to one of the sources. To assign a value to a key: $ jolt config jolt.default all # Change name of the default task To list existing keys: $ jolt config -l # List all existing keys $ jolt config -l -g # List keys in the global config file $ jolt config jolt.colors # Display the value of a key. To delete an existing key: $ jolt config -d jolt.colors To pass temporary configuration: $ jolt -c jolt.colors=true config -l """ if delete and not key: raise click.UsageError("--delete requires KEY") if not key and not list and not key: print(ctx.get_help()) sys.exit(1) if global_ and user: raise click.UsageError("--global and --user are mutually exclusive") alias = None if global_: alias = "global" if user: alias = "user" def _print_key(section, opt): value = config.get(section, opt, alias=alias) raise_error_if(value is None, "no such key: {}".format(key)) print("{} = {}".format(key, value)) def _print_section(section): print("{}".format(section)) if list: for section, option, value in config.items(alias): if option: print("{}.{} = {}".format(section, option, value)) else: print(section) elif delete: raise_error_if(config.delete(key, alias) <= 0, "no such key: {}", key) config.save() elif key: section, opt = config.split(key) if value: raise_error_if(opt is None, "invalid configuration key: {}".format(key)) config.set(section, opt, value, alias) try: config.save() except Exception as e: log.exception() raise_error("failed to write configuration file: {}".format(e)) else: if opt: _print_key(section, opt) else: _print_section(section)
def run(self): with open("default.joltxmanifest", "wb") as f: f.write(self.body) log.info("Manifest written") tools = Tools() for recipe in tools.glob("*.jolt"): tools.unlink(recipe) try: jolt = self.selfdeploy() config_file = config.get("amqp", "config", "") if config_file: config_file = "-c " + config_file log.info("Running jolt") tools.run( "{} -vv {} build --worker --result result.joltxmanifest", jolt, config_file, output_stdio=True) except JoltCommandError as e: self.response = "" try: manifest = JoltManifest() try: manifest.parse("result.joltxmanifest") except Exception: manifest.duration = "0" manifest.result = "FAILED" manifest.stdout = "\n".join(e.stdout) manifest.stderr = "\n".join(e.stderr) self.response = manifest.format() except Exception: log.exception() log.error("Task failed") except Exception: log.exception() self.response = "" try: manifest = JoltManifest() try: manifest.parse("result.joltxmanifest") except Exception: manifest.duration = "0" manifest.result = "FAILED" self.response = manifest.format() except Exception: log.exception() log.error("Task failed") else: self.response = "" try: manifest = JoltManifest() try: manifest.parse("result.joltxmanifest") except Exception: manifest.duration = "0" manifest.result = "SUCCESS" self.response = manifest.format() except Exception: log.exception() log.info("Task succeeded") utils.call_and_catch(tools.unlink, "result.joltxmanifest") self.consumer.add_on_job_completed_callback(self)