def dispatch_update_heads(self, sock, cmd):
parser = minion.cmd.update_heads(MinionThrowingArgumentParser())
args = parser.parse_args(cmd)
with self._heads_mtx:
parsed, sources = self.parsed_sources(args.sources)
ptrs = {}
for src in sources:
ptr = None
if isinstance(src, minion.parser.FetchSource):
ptr = self.get_source_fetch(src)
elif isinstance(src, minion.parser.GitSource):
ptr = self.get_source_git(src)
else:
assert False
assert ptr is not None
logger.debug('head of %s is %s' % (src.name, ptr))
ptrs[src.name.normal] = ptr
if os.path.exists(self.HEADS):
logger.debug('HEADS file exists; parsing it')
old_ptrs = self.sources_load(self.HEADS)
else:
logger.debug('HEADS file does not exist; will create it')
old_ptrs = {}
new_ptrs = old_ptrs.copy()
new_ptrs.update(ptrs)
A = set(new_ptrs.keys())
B = set([p.name.normal for p in parsed])
parsed_names = [p.name.normal for p in parsed]
if A != B:
missing = B - A
missing = sorted(missing, key=parsed_names.index)
logger.warning('missing head for %s' % (', '.join(missing),))
self.sources_save(self.HEADS, new_ptrs, parsed_names)
return {'status': 'success'}
def dispatch_add_blob(self, sock, cmd):
parser = minion.cmd.add_blob(MinionThrowingArgumentParser())
args = parser.parse_args(cmd)
for blob in args.blobs:
sha256 = self.blobs.add(blob)
logger.info('manually added %s as %s...' % (blob, sha256[:8]))
return {'status': 'success'}
def dispatch_del_target(self, sock, cmd):
parser = minion.cmd.del_target(MinionThrowingArgumentParser())
args = parser.parse_args(cmd)
with self._heads_mtx:
path = self.TARGET(args.target)
if not os.path.exists(path):
raise MinionException("target %r doesn't exist" % args.target)
logger.info('deleting target %r' % (args.target,))
shutil.rmtree(path)
return {'status': 'success'}
def dispatch_forget_build_failure(self, sock, cmd):
parser = minion.cmd.forget_build_failure(MinionThrowingArgumentParser())
args = parser.parse_args(cmd)
buildname = self.get_build(args.target, args.name)
build = open(os.path.join(self.BUILDS, buildname)).read()
report = json.loads(build)
for x in report['reports']:
if x['name'] == args.process:
self.processes.forget(x['inputs'])
logger.info('removed failed build %s from build %s' % (args.process, buildname))
def main_daemon(argv):
parser = argparse.ArgumentParser(prog='minion-daemon')
parser.add_argument('--workdir', default=os.environ.get('MINION_WORKDIR', '.'),
help='minion working directory')
args = parser.parse_args(argv)
try:
d = MinionDaemon(args.workdir)
d.run()
except KeyboardInterrupt:
sys.exit(0)
except MinionException as e:
print(e)
sys.exit(1)
def dispatch_new_target(self, sock, cmd):
parser = minion.cmd.new_target(MinionThrowingArgumentParser())
args = parser.parse_args(cmd)
with self._heads_mtx:
path = self.TARGET(args.target)
if os.path.exists(path):
raise MinionException('target %r already exists' % args.target)
if not os.path.exists(self.HEADS):
raise MinionException('heads missing; run update-heads and retry')
os.makedirs(path)
logger.info('creating target %r' % (args.target,))
shutil.copyfile(self.HEADS, os.path.join(path, 'AUTO'))
shutil.copyfile(self.HEADS, os.path.join(path, 'HEADS'))
return {'status': 'success'}
def dispatch_status(self, sock, cmd):
parser = minion.cmd.status(MinionThrowingArgumentParser())
args = parser.parse_args(cmd)
if args.name is not None:
display_name = '%s:%s' % (args.target, args.name)
else:
display_name = '%s' % args.target
logger.info('checking build status of %s' % display_name)
build = self.get_build(args.target, args.name)
reporter = args.report.replace('-', '_')
reporter = getattr(self, 'report_' + reporter, None)
if reporter is None:
return {'status': 'failure', 'output': 'no such reporter'}
if build is None:
return {'status': 'failure', 'output': 'no such build as %s' % display_name}
logger.info('generating %s report of %s' % (args.report, display_name))
build = open(os.path.join(self.BUILDS, build)).read()
report = json.loads(build)
return {'status': 'success', 'output': reporter(report)}
def dispatch_build_status(self, sock, cmd):
parser = minion.cmd.build_status(MinionThrowingArgumentParser())
args = parser.parse_args(cmd)
if args.name is not None:
logger.info('checking build status of %s:%s' % (args.target, args.name))
else:
logger.info('checking build status of %s' % (args.target,))
build = self.get_build(args.target, args.name)
reporter = getattr(self, 'report_' + args.report, None)
if reporter is None:
raise MinionException('unimplemented report type %r' % args.report)
if build == 'in-progress':
return {'status': 'success', 'output': '%s is still running' % args.target}
if build == 'not found':
return {'status': 'success', 'output': 'no such build as %s' % args.target}
logger.info('generating %s report of %s' % (args.report, build))
build = open(os.path.join(self.BUILDS, build)).read()
report = json.loads(build)
return {'status': 'success', 'output': reporter(report)}
def main_tool(argv):
sock_default = os.environ.get('MINION_SOCKET', './minion.sock')
parser = argparse.ArgumentParser(prog='minion')
parser.add_argument('--socket', type=str, default=sock_default,
help='socket to talk to minion daemon (default: %s)' % sock_default)
subparsers = parser.add_subparsers(help='command-line tools')
minion.cmd.update_heads(subparsers.add_parser('update-heads', help='fetch the latest sources'))
minion.cmd.new_target(subparsers.add_parser('new-target', help='create a new build target'))
minion.cmd.del_target(subparsers.add_parser('del-target', help='remove an existing build target'))
minion.cmd.sync_target(subparsers.add_parser('sync-target', help="synchronize a target's sources with the heads"))
minion.cmd.set_target_refspec(subparsers.add_parser('set-target-refspec', help='manually set the HEAD for a target/source pair'))
minion.cmd.run_build_process(subparsers.add_parser('run-build-process', help='run the build process for a target (async)'))
minion.cmd.build_status(subparsers.add_parser('build-status', help='check the status of a build'))
minion.cmd.forget_build_failure(subparsers.add_parser('forget-build-failure', help='remove a cached process result for a given build target'))
minion.cmd.add_blob(subparsers.add_parser('add-blob', help='manually add a blob'))
# run it
if not argv:
parser.print_help()
else:
args = parser.parse_args(argv)
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_PASSCRED, 1)
s.connect(args.socket)
cmd = ' '.join([shlex.quote(arg) for arg in argv]) + '\n'
s.sendall(cmd.encode('utf8'))
s.shutdown(socket.SHUT_WR)
data = b''
while True:
x = s.recv(512)
if not x:
break
data += x
output = json.loads(data.decode('utf8'))
if 'output' in output:
stdout = output['output']
stdout = stdout.strip()
stdout += '\n'
sys.stdout.write(stdout)
if output['status'] == 'success':
sys.exit(0)
else:
sys.exit(1)
def dispatch_set_refspec(self, sock, cmd):
parser = minion.cmd.set_refspec(MinionThrowingArgumentParser())
args = parser.parse_args(cmd)
with self._heads_mtx:
path = self.TARGET(args.target)
if not os.path.exists(path):
raise MinionException("target %r doesn't exist" % args.target)
parsed, sources = self.parsed_sources([args.source])
assert len(sources) == 1
source = sources[0]
path = self.TARGET(args.target)
heads = self.sources_load(os.path.join(path, 'HEADS'))
if not isgit(source):
raise MinionException('cannot set refspec for non-git source %s' % source)
heads[source.name.normal] = self.get_source_git(source, args.refspec)
logger.info('updating head %r in target %r to %r' %
(source.name.normal, args.target, args.refspec))
parsed_names = [p.name.normal for p in parsed]
self.sources_save(os.path.join(path, 'HEADS'), heads, parsed_names)
return {'status': 'success'}
def dispatch_build(self, sock, cmd):
parser = minion.cmd.build(MinionThrowingArgumentParser())
args = parser.parse_args(cmd)
chosen_procs = None
if args.processes:
args.processes = tuple(args.processes.split(','))
chosen_procs = self.parse_subset(args.processes)
report_name = args.name or datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S')
report_name = args.target + ':' + report_name
logger.info('running build process for %s; results will be saved to %s' % (args.target, report_name))
path = self.TARGET(args.target)
if not os.path.exists(path):
raise MinionException("target %r doesn't exist" % args.target)
with self._heads_mtx:
mblob = self.blobs.add(self.MINIONFILE)
sblob = self.blobs.add(os.path.join(path, 'HEADS'))
minionfile = self.blobs.path(mblob)
logger.debug('using minionfile %s' % minionfile)
sources = self.blobs.path(sblob)
logger.debug('using sources %s' % sources)
jc = JobController(self, self.sources_load(sources), report_name)
jc.retry_failures = args.retry_failures
for proc in self.parse(minionfile):
if not isprocess(proc):
continue
if not args.processes or proc.name in chosen_procs:
logger.debug('adding %s' % (proc,))
jc.add(proc)
output = {}
output['name'] = report_name
output['minionfile'] = mblob
output['sources'] = sblob
output['reports'] = []
with self._builds_mtx:
path = os.path.join(self.BUILDS, report_name)
if os.path.exists(path) or report_name in self._builds_set:
raise MinionException('build %r already exists' % report_name)
self._builds_set.add(report_name)
self._builds_queue.put((output, jc))
return {'status': 'success'}
def dispatch_sync_target(self, sock, cmd):
parser = minion.cmd.sync_target(MinionThrowingArgumentParser())
args = parser.parse_args(cmd)
with self._heads_mtx:
parsed, sources = self.parsed_sources(args.sources)
if not os.path.exists(self.HEADS):
raise MinionException('heads missing; run update-heads and retry')
path = self.TARGET(args.target)
if not os.path.exists(path):
raise MinionException("target %r doesn't exist" % args.target)
old_auto = self.sources_load(os.path.join(path, 'AUTO'))
new_auto = self.sources_load(self.HEADS)
heads = self.sources_load(os.path.join(path, 'HEADS'))
for src in sources:
name = src.name.normal
if name not in new_auto:
raise MinionException('head %s missing; run update-heads and retry' % src.name)
if name not in heads or name not in old_auto or old_auto[name] != new_auto[name]:
heads[name] = new_auto[name]
logger.info('updating head %r in target %r' % (name, args.target))
parsed_names = [p.name.normal for p in parsed]
self.sources_save(os.path.join(path, 'AUTO'), new_auto, parsed_names)
self.sources_save(os.path.join(path, 'HEADS'), heads, parsed_names)
return {'status': 'success'}