Ejemplo n.º 1
0
 def dispatch_update_heads(self, sock, cmd):
     parser = minion.cmd.update_heads(MinionThrowingArgumentParser())
     args = parser.parse_args(cmd)
     with self._heads_mtx:
         parsed, sources = self.parsed_sources(args.sources)
         ptrs = {}
         for src in sources:
             ptr = None
             if isinstance(src, minion.parser.FetchSource):
                 ptr = self.get_source_fetch(src)
             elif isinstance(src, minion.parser.GitSource):
                 ptr = self.get_source_git(src)
             else:
                 assert False
             assert ptr is not None
             logger.debug('head of %s is %s' % (src.name, ptr))
             ptrs[src.name.normal] = ptr
         if os.path.exists(self.HEADS):
             logger.debug('HEADS file exists; parsing it')
             old_ptrs = self.sources_load(self.HEADS)
         else:
             logger.debug('HEADS file does not exist; will create it')
             old_ptrs = {}
         new_ptrs = old_ptrs.copy()
         new_ptrs.update(ptrs)
         A = set(new_ptrs.keys())
         B = set([p.name.normal for p in parsed])
         parsed_names = [p.name.normal for p in parsed]
         if A != B:
             missing = B - A
             missing = sorted(missing, key=parsed_names.index)
             logger.warning('missing head for %s' % (', '.join(missing),))
         self.sources_save(self.HEADS, new_ptrs, parsed_names)
     return {'status': 'success'}
Ejemplo n.º 2
0
 def dispatch_add_blob(self, sock, cmd):
     parser = minion.cmd.add_blob(MinionThrowingArgumentParser())
     args = parser.parse_args(cmd)
     for blob in args.blobs:
         sha256 = self.blobs.add(blob)
         logger.info('manually added %s as %s...' % (blob, sha256[:8]))
     return {'status': 'success'}
Ejemplo n.º 3
0
 def dispatch_del_target(self, sock, cmd):
     parser = minion.cmd.del_target(MinionThrowingArgumentParser())
     args = parser.parse_args(cmd)
     with self._heads_mtx:
         path = self.TARGET(args.target)
         if not os.path.exists(path):
             raise MinionException("target %r doesn't exist" % args.target)
         logger.info('deleting target %r' % (args.target,))
         shutil.rmtree(path)
     return {'status': 'success'}
Ejemplo n.º 4
0
 def dispatch_forget_build_failure(self, sock, cmd):
     parser = minion.cmd.forget_build_failure(MinionThrowingArgumentParser())
     args = parser.parse_args(cmd)
     buildname = self.get_build(args.target, args.name)
     build = open(os.path.join(self.BUILDS, buildname)).read()
     report = json.loads(build)
     for x in report['reports']:
         if x['name'] == args.process:
             self.processes.forget(x['inputs'])
             logger.info('removed failed build %s from build %s' % (args.process, buildname))
Ejemplo n.º 5
0
def main_daemon(argv):
    parser = argparse.ArgumentParser(prog='minion-daemon')
    parser.add_argument('--workdir', default=os.environ.get('MINION_WORKDIR', '.'),
                        help='minion working directory')
    args = parser.parse_args(argv)
    try:
        d = MinionDaemon(args.workdir)
        d.run()
    except KeyboardInterrupt:
        sys.exit(0)
    except MinionException as e:
        print(e)
        sys.exit(1)
Ejemplo n.º 6
0
 def dispatch_new_target(self, sock, cmd):
     parser = minion.cmd.new_target(MinionThrowingArgumentParser())
     args = parser.parse_args(cmd)
     with self._heads_mtx:
         path = self.TARGET(args.target)
         if os.path.exists(path):
             raise MinionException('target %r already exists' % args.target)
         if not os.path.exists(self.HEADS):
             raise MinionException('heads missing; run update-heads and retry')
         os.makedirs(path)
         logger.info('creating target %r' % (args.target,))
         shutil.copyfile(self.HEADS, os.path.join(path, 'AUTO'))
         shutil.copyfile(self.HEADS, os.path.join(path, 'HEADS'))
     return {'status': 'success'}
Ejemplo n.º 7
0
 def dispatch_status(self, sock, cmd):
     parser = minion.cmd.status(MinionThrowingArgumentParser())
     args = parser.parse_args(cmd)
     if args.name is not None:
         display_name = '%s:%s' % (args.target, args.name)
     else:
         display_name = '%s' % args.target
     logger.info('checking build status of %s' % display_name)
     build = self.get_build(args.target, args.name)
     reporter = args.report.replace('-', '_')
     reporter = getattr(self, 'report_' + reporter, None)
     if reporter is None:
         return {'status': 'failure', 'output': 'no such reporter'}
     if build is None:
         return {'status': 'failure', 'output': 'no such build as %s' % display_name}
     logger.info('generating %s report of %s' % (args.report, display_name))
     build = open(os.path.join(self.BUILDS, build)).read()
     report = json.loads(build)
     return {'status': 'success', 'output': reporter(report)}
Ejemplo n.º 8
0
 def dispatch_build_status(self, sock, cmd):
     parser = minion.cmd.build_status(MinionThrowingArgumentParser())
     args = parser.parse_args(cmd)
     if args.name is not None:
         logger.info('checking build status of %s:%s' % (args.target, args.name))
     else:
         logger.info('checking build status of %s' % (args.target,))
     build = self.get_build(args.target, args.name)
     reporter = getattr(self, 'report_' + args.report, None)
     if reporter is None:
         raise MinionException('unimplemented report type %r' % args.report)
     if build == 'in-progress':
         return {'status': 'success', 'output': '%s is still running' % args.target}
     if build == 'not found':
         return {'status': 'success', 'output': 'no such build as %s' % args.target}
     logger.info('generating %s report of %s' % (args.report, build))
     build = open(os.path.join(self.BUILDS, build)).read()
     report = json.loads(build)
     return {'status': 'success', 'output': reporter(report)}
Ejemplo n.º 9
0
def main_tool(argv):
    sock_default = os.environ.get('MINION_SOCKET', './minion.sock')
    parser = argparse.ArgumentParser(prog='minion')
    parser.add_argument('--socket', type=str, default=sock_default,
                        help='socket to talk to minion daemon (default: %s)' % sock_default)
    subparsers = parser.add_subparsers(help='command-line tools')
    minion.cmd.update_heads(subparsers.add_parser('update-heads', help='fetch the latest sources'))
    minion.cmd.new_target(subparsers.add_parser('new-target', help='create a new build target'))
    minion.cmd.del_target(subparsers.add_parser('del-target', help='remove an existing build target'))
    minion.cmd.sync_target(subparsers.add_parser('sync-target', help="synchronize a target's sources with the heads"))
    minion.cmd.set_target_refspec(subparsers.add_parser('set-target-refspec', help='manually set the HEAD for a target/source pair'))
    minion.cmd.run_build_process(subparsers.add_parser('run-build-process', help='run the build process for a target (async)'))
    minion.cmd.build_status(subparsers.add_parser('build-status', help='check the status of a build'))
    minion.cmd.forget_build_failure(subparsers.add_parser('forget-build-failure', help='remove a cached process result for a given build target'))
    minion.cmd.add_blob(subparsers.add_parser('add-blob', help='manually add a blob'))

    # run it
    if not argv:
        parser.print_help()
    else:
        args = parser.parse_args(argv)
        s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
        s.setsockopt(socket.SOL_SOCKET, socket.SO_PASSCRED, 1)
        s.connect(args.socket)
        cmd = ' '.join([shlex.quote(arg) for arg in argv]) + '\n'
        s.sendall(cmd.encode('utf8'))
        s.shutdown(socket.SHUT_WR)
        data = b''
        while True:
            x = s.recv(512)
            if not x:
                break
            data += x
        output = json.loads(data.decode('utf8'))
        if 'output' in output:
            stdout = output['output']
            stdout = stdout.strip()
            stdout += '\n'
            sys.stdout.write(stdout)
        if output['status'] == 'success':
            sys.exit(0)
        else:
            sys.exit(1)
Ejemplo n.º 10
0
 def dispatch_set_refspec(self, sock, cmd):
     parser = minion.cmd.set_refspec(MinionThrowingArgumentParser())
     args = parser.parse_args(cmd)
     with self._heads_mtx:
         path = self.TARGET(args.target)
         if not os.path.exists(path):
             raise MinionException("target %r doesn't exist" % args.target)
         parsed, sources = self.parsed_sources([args.source])
         assert len(sources) == 1
         source = sources[0]
         path = self.TARGET(args.target)
         heads = self.sources_load(os.path.join(path, 'HEADS'))
         if not isgit(source):
             raise MinionException('cannot set refspec for non-git source %s' % source)
         heads[source.name.normal] = self.get_source_git(source, args.refspec)
         logger.info('updating head %r in target %r to %r' %
                      (source.name.normal, args.target, args.refspec))
         parsed_names = [p.name.normal for p in parsed]
         self.sources_save(os.path.join(path, 'HEADS'), heads, parsed_names)
     return {'status': 'success'}
Ejemplo n.º 11
0
 def dispatch_build(self, sock, cmd):
     parser = minion.cmd.build(MinionThrowingArgumentParser())
     args = parser.parse_args(cmd)
     chosen_procs = None
     if args.processes:
         args.processes = tuple(args.processes.split(','))
         chosen_procs = self.parse_subset(args.processes)
     report_name = args.name or datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S')
     report_name = args.target + ':' + report_name
     logger.info('running build process for %s; results will be saved to %s' % (args.target, report_name))
     path = self.TARGET(args.target)
     if not os.path.exists(path):
         raise MinionException("target %r doesn't exist" % args.target)
     with self._heads_mtx:
         mblob = self.blobs.add(self.MINIONFILE)
         sblob = self.blobs.add(os.path.join(path, 'HEADS'))
     minionfile = self.blobs.path(mblob)
     logger.debug('using minionfile %s' % minionfile)
     sources = self.blobs.path(sblob)
     logger.debug('using sources %s' % sources)
     jc = JobController(self, self.sources_load(sources), report_name)
     jc.retry_failures = args.retry_failures
     for proc in self.parse(minionfile):
         if not isprocess(proc):
             continue
         if not args.processes or proc.name in chosen_procs:
             logger.debug('adding %s' % (proc,))
             jc.add(proc)
     output = {}
     output['name'] = report_name
     output['minionfile'] = mblob
     output['sources'] = sblob
     output['reports'] = []
     with self._builds_mtx:
         path = os.path.join(self.BUILDS, report_name)
         if os.path.exists(path) or report_name in self._builds_set:
             raise MinionException('build %r already exists' % report_name)
         self._builds_set.add(report_name)
         self._builds_queue.put((output, jc))
     return {'status': 'success'}
Ejemplo n.º 12
0
 def dispatch_sync_target(self, sock, cmd):
     parser = minion.cmd.sync_target(MinionThrowingArgumentParser())
     args = parser.parse_args(cmd)
     with self._heads_mtx:
         parsed, sources = self.parsed_sources(args.sources)
         if not os.path.exists(self.HEADS):
             raise MinionException('heads missing; run update-heads and retry')
         path = self.TARGET(args.target)
         if not os.path.exists(path):
             raise MinionException("target %r doesn't exist" % args.target)
         old_auto = self.sources_load(os.path.join(path, 'AUTO'))
         new_auto = self.sources_load(self.HEADS)
         heads = self.sources_load(os.path.join(path, 'HEADS'))
         for src in sources:
             name = src.name.normal
             if name not in new_auto:
                 raise MinionException('head %s missing; run update-heads and retry' % src.name)
             if name not in heads or name not in old_auto or old_auto[name] != new_auto[name]:
                 heads[name] = new_auto[name]
                 logger.info('updating head %r in target %r' % (name, args.target))
         parsed_names = [p.name.normal for p in parsed]
         self.sources_save(os.path.join(path, 'AUTO'), new_auto, parsed_names)
         self.sources_save(os.path.join(path, 'HEADS'), heads, parsed_names)
     return {'status': 'success'}