def dispatch_update_heads(self, sock, cmd): parser = minion.cmd.update_heads(MinionThrowingArgumentParser()) args = parser.parse_args(cmd) with self._heads_mtx: parsed, sources = self.parsed_sources(args.sources) ptrs = {} for src in sources: ptr = None if isinstance(src, minion.parser.FetchSource): ptr = self.get_source_fetch(src) elif isinstance(src, minion.parser.GitSource): ptr = self.get_source_git(src) else: assert False assert ptr is not None logger.debug('head of %s is %s' % (src.name, ptr)) ptrs[src.name.normal] = ptr if os.path.exists(self.HEADS): logger.debug('HEADS file exists; parsing it') old_ptrs = self.sources_load(self.HEADS) else: logger.debug('HEADS file does not exist; will create it') old_ptrs = {} new_ptrs = old_ptrs.copy() new_ptrs.update(ptrs) A = set(new_ptrs.keys()) B = set([p.name.normal for p in parsed]) parsed_names = [p.name.normal for p in parsed] if A != B: missing = B - A missing = sorted(missing, key=parsed_names.index) logger.warning('missing head for %s' % (', '.join(missing),)) self.sources_save(self.HEADS, new_ptrs, parsed_names) return {'status': 'success'}
def dispatch_add_blob(self, sock, cmd): parser = minion.cmd.add_blob(MinionThrowingArgumentParser()) args = parser.parse_args(cmd) for blob in args.blobs: sha256 = self.blobs.add(blob) logger.info('manually added %s as %s...' % (blob, sha256[:8])) return {'status': 'success'}
def dispatch_del_target(self, sock, cmd): parser = minion.cmd.del_target(MinionThrowingArgumentParser()) args = parser.parse_args(cmd) with self._heads_mtx: path = self.TARGET(args.target) if not os.path.exists(path): raise MinionException("target %r doesn't exist" % args.target) logger.info('deleting target %r' % (args.target,)) shutil.rmtree(path) return {'status': 'success'}
def dispatch_forget_build_failure(self, sock, cmd): parser = minion.cmd.forget_build_failure(MinionThrowingArgumentParser()) args = parser.parse_args(cmd) buildname = self.get_build(args.target, args.name) build = open(os.path.join(self.BUILDS, buildname)).read() report = json.loads(build) for x in report['reports']: if x['name'] == args.process: self.processes.forget(x['inputs']) logger.info('removed failed build %s from build %s' % (args.process, buildname))
def main_daemon(argv): parser = argparse.ArgumentParser(prog='minion-daemon') parser.add_argument('--workdir', default=os.environ.get('MINION_WORKDIR', '.'), help='minion working directory') args = parser.parse_args(argv) try: d = MinionDaemon(args.workdir) d.run() except KeyboardInterrupt: sys.exit(0) except MinionException as e: print(e) sys.exit(1)
def dispatch_new_target(self, sock, cmd): parser = minion.cmd.new_target(MinionThrowingArgumentParser()) args = parser.parse_args(cmd) with self._heads_mtx: path = self.TARGET(args.target) if os.path.exists(path): raise MinionException('target %r already exists' % args.target) if not os.path.exists(self.HEADS): raise MinionException('heads missing; run update-heads and retry') os.makedirs(path) logger.info('creating target %r' % (args.target,)) shutil.copyfile(self.HEADS, os.path.join(path, 'AUTO')) shutil.copyfile(self.HEADS, os.path.join(path, 'HEADS')) return {'status': 'success'}
def dispatch_status(self, sock, cmd): parser = minion.cmd.status(MinionThrowingArgumentParser()) args = parser.parse_args(cmd) if args.name is not None: display_name = '%s:%s' % (args.target, args.name) else: display_name = '%s' % args.target logger.info('checking build status of %s' % display_name) build = self.get_build(args.target, args.name) reporter = args.report.replace('-', '_') reporter = getattr(self, 'report_' + reporter, None) if reporter is None: return {'status': 'failure', 'output': 'no such reporter'} if build is None: return {'status': 'failure', 'output': 'no such build as %s' % display_name} logger.info('generating %s report of %s' % (args.report, display_name)) build = open(os.path.join(self.BUILDS, build)).read() report = json.loads(build) return {'status': 'success', 'output': reporter(report)}
def dispatch_build_status(self, sock, cmd): parser = minion.cmd.build_status(MinionThrowingArgumentParser()) args = parser.parse_args(cmd) if args.name is not None: logger.info('checking build status of %s:%s' % (args.target, args.name)) else: logger.info('checking build status of %s' % (args.target,)) build = self.get_build(args.target, args.name) reporter = getattr(self, 'report_' + args.report, None) if reporter is None: raise MinionException('unimplemented report type %r' % args.report) if build == 'in-progress': return {'status': 'success', 'output': '%s is still running' % args.target} if build == 'not found': return {'status': 'success', 'output': 'no such build as %s' % args.target} logger.info('generating %s report of %s' % (args.report, build)) build = open(os.path.join(self.BUILDS, build)).read() report = json.loads(build) return {'status': 'success', 'output': reporter(report)}
def main_tool(argv): sock_default = os.environ.get('MINION_SOCKET', './minion.sock') parser = argparse.ArgumentParser(prog='minion') parser.add_argument('--socket', type=str, default=sock_default, help='socket to talk to minion daemon (default: %s)' % sock_default) subparsers = parser.add_subparsers(help='command-line tools') minion.cmd.update_heads(subparsers.add_parser('update-heads', help='fetch the latest sources')) minion.cmd.new_target(subparsers.add_parser('new-target', help='create a new build target')) minion.cmd.del_target(subparsers.add_parser('del-target', help='remove an existing build target')) minion.cmd.sync_target(subparsers.add_parser('sync-target', help="synchronize a target's sources with the heads")) minion.cmd.set_target_refspec(subparsers.add_parser('set-target-refspec', help='manually set the HEAD for a target/source pair')) minion.cmd.run_build_process(subparsers.add_parser('run-build-process', help='run the build process for a target (async)')) minion.cmd.build_status(subparsers.add_parser('build-status', help='check the status of a build')) minion.cmd.forget_build_failure(subparsers.add_parser('forget-build-failure', help='remove a cached process result for a given build target')) minion.cmd.add_blob(subparsers.add_parser('add-blob', help='manually add a blob')) # run it if not argv: parser.print_help() else: args = parser.parse_args(argv) s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_PASSCRED, 1) s.connect(args.socket) cmd = ' '.join([shlex.quote(arg) for arg in argv]) + '\n' s.sendall(cmd.encode('utf8')) s.shutdown(socket.SHUT_WR) data = b'' while True: x = s.recv(512) if not x: break data += x output = json.loads(data.decode('utf8')) if 'output' in output: stdout = output['output'] stdout = stdout.strip() stdout += '\n' sys.stdout.write(stdout) if output['status'] == 'success': sys.exit(0) else: sys.exit(1)
def dispatch_set_refspec(self, sock, cmd): parser = minion.cmd.set_refspec(MinionThrowingArgumentParser()) args = parser.parse_args(cmd) with self._heads_mtx: path = self.TARGET(args.target) if not os.path.exists(path): raise MinionException("target %r doesn't exist" % args.target) parsed, sources = self.parsed_sources([args.source]) assert len(sources) == 1 source = sources[0] path = self.TARGET(args.target) heads = self.sources_load(os.path.join(path, 'HEADS')) if not isgit(source): raise MinionException('cannot set refspec for non-git source %s' % source) heads[source.name.normal] = self.get_source_git(source, args.refspec) logger.info('updating head %r in target %r to %r' % (source.name.normal, args.target, args.refspec)) parsed_names = [p.name.normal for p in parsed] self.sources_save(os.path.join(path, 'HEADS'), heads, parsed_names) return {'status': 'success'}
def dispatch_build(self, sock, cmd): parser = minion.cmd.build(MinionThrowingArgumentParser()) args = parser.parse_args(cmd) chosen_procs = None if args.processes: args.processes = tuple(args.processes.split(',')) chosen_procs = self.parse_subset(args.processes) report_name = args.name or datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S') report_name = args.target + ':' + report_name logger.info('running build process for %s; results will be saved to %s' % (args.target, report_name)) path = self.TARGET(args.target) if not os.path.exists(path): raise MinionException("target %r doesn't exist" % args.target) with self._heads_mtx: mblob = self.blobs.add(self.MINIONFILE) sblob = self.blobs.add(os.path.join(path, 'HEADS')) minionfile = self.blobs.path(mblob) logger.debug('using minionfile %s' % minionfile) sources = self.blobs.path(sblob) logger.debug('using sources %s' % sources) jc = JobController(self, self.sources_load(sources), report_name) jc.retry_failures = args.retry_failures for proc in self.parse(minionfile): if not isprocess(proc): continue if not args.processes or proc.name in chosen_procs: logger.debug('adding %s' % (proc,)) jc.add(proc) output = {} output['name'] = report_name output['minionfile'] = mblob output['sources'] = sblob output['reports'] = [] with self._builds_mtx: path = os.path.join(self.BUILDS, report_name) if os.path.exists(path) or report_name in self._builds_set: raise MinionException('build %r already exists' % report_name) self._builds_set.add(report_name) self._builds_queue.put((output, jc)) return {'status': 'success'}
def dispatch_sync_target(self, sock, cmd): parser = minion.cmd.sync_target(MinionThrowingArgumentParser()) args = parser.parse_args(cmd) with self._heads_mtx: parsed, sources = self.parsed_sources(args.sources) if not os.path.exists(self.HEADS): raise MinionException('heads missing; run update-heads and retry') path = self.TARGET(args.target) if not os.path.exists(path): raise MinionException("target %r doesn't exist" % args.target) old_auto = self.sources_load(os.path.join(path, 'AUTO')) new_auto = self.sources_load(self.HEADS) heads = self.sources_load(os.path.join(path, 'HEADS')) for src in sources: name = src.name.normal if name not in new_auto: raise MinionException('head %s missing; run update-heads and retry' % src.name) if name not in heads or name not in old_auto or old_auto[name] != new_auto[name]: heads[name] = new_auto[name] logger.info('updating head %r in target %r' % (name, args.target)) parsed_names = [p.name.normal for p in parsed] self.sources_save(os.path.join(path, 'AUTO'), new_auto, parsed_names) self.sources_save(os.path.join(path, 'HEADS'), heads, parsed_names) return {'status': 'success'}