def __init__(self, worker, data): self.start_time = now() self.hook = {} self.env = {} self.publish = [] self.worker = worker self.logger = worker.logger self.json = data.body data = json.loads(data.body) for key, value in data.items(): if keyword.iskeyword(key): key += '_' setattr(self, key, value) self.env.update(os.environ) self.repo_path = os.path.join(worker.repo_dir, self.repo, self.repo + '.git') self.work_path = os.path.join(worker.repo_dir, self.repo, 'work', self.action, '%s@%s' % (self.ref, self.sha1)) self.artefact_path = os.path.join(worker.repo_dir, self.repo, 'artefacts', self.action, '%s@%s' % (self.ref, self.sha1)) if not worker.repo_checkout: self.env.update({'GIT_DIR': self.repo_path, 'GIT_WORK_TREE': self.work_path}) if self.worker.repo_sync and not os.path.exists(self.repo_path): os.makedirs(self.repo_path) if os.path.exists(self.work_path): shutil.rmtree(self.work_path) os.makedirs(self.work_path) if os.path.exists(self.artefact_path): shutil.rmtree(self.artefact_path) os.makedirs(self.artefact_path) self.shell = whelk.Shell(output_callback=OutputLogger(self.logger), run_callback=RunLogger(self.logger), env=self.env, cwd=self.work_path, exit_callback=check_sp) self.pipe = whelk.Pipe(output_callback=OutputLogger(self.logger), run_callback=RunLogger(self.logger), env=self.env, cwd=self.work_path, exit_callback=check_sp) # Add a loghandler for our logfile so everything is properly logged per job self.log = os.path.join(self.artefact_path, 'log') handler = logging.FileHandler(self.log) handler.is_job_handler = True handler.setLevel(logging.DEBUG) handler.setFormatter(logging.Formatter(fmt='[%(asctime)s] %(message)s', datefmt='%H:%M:%S')) self.logger.addHandler(handler)
def publish_results(self): for glb in getattr(self, 'publish', []): for file in glob.glob(os.path.join(self.work_path, glb)): self.logger.info("Adding artefact %s" % file.replace(self.work_path, '')) os.rename(file, os.path.join(self.artefact_path, os.path.basename(file))) local = self.artefact_path + os.sep remote = '/'.join([self.worker.rsync_root, self.repo, 'artefacts', self.action, '%s@%s' % (self.ref, self.sha1)]) + os.sep self.logger.info("Publishing %s => %s" % (local, remote)) args = ['-av', local, remote] if self.worker.rsync_password: args += ['--password-file', self.worker.rsync_password] self.shell.rsync(*args) self.end_time = now() to_submit = {'repo': self.repo, 'ref': self.ref, 'prev_sha1': self.prev_sha1, 'sha1': self.sha1, 'why': 'action-done', 'action': self.action, 'result': self.result, 'start_time': toutctimestamp(self.start_time), 'end_time': toutctimestamp(self.end_time), 'duration': (self.end_time-self.start_time).total_seconds(), 'host': socket.gethostname()} self.worker.bs.use(self.worker.submit_queue) self.worker.bs.put(json.dumps(to_submit), ttr=600)
def schedule(self, job, db): ref = job.get('ref', None) why = job['why'] repo = job['repo'] _c = golem.db.commit _a = golem.db.action _r = golem.db.repository _f = golem.db.artefact refs = {} tags = [] if why == 'reschedule': if job['sha1']: c = db.execute(_c.select().where(sql.and_(_c.c.ref==job['ref'], _c.c.sha1==job['sha1']))).fetchone() if not c: self.logger.error("Commit %s for ref %s cannot be rescheduled, it does not exist" % (job['sha1'], job['ref'])) return else: c = db.execute(_c.select().where(_c.c.ref==job['ref']).order_by(sql.desc(_c.c.submit_time)).limit(1)).fetchone() if not c: self.logger.error("Cannot reschedule actions for ref %s, no commits were processed yet" % job['ref']) return job['prev_sha1'], job['sha1'] = c.prev_sha1, c.sha1 if ref and ref.startswith(('refs/heads', 'refs/tags')): refs[ref] = [(job['prev_sha1'], job['sha1'])] if ref and ref.startswith('refs/tags'): tags = [(ref, 0)] if why == 'action-started': res = db.execute(_a.join(_c).join(_r).select(use_labels=True).where( sql.and_(_r.c.name == job['repo'], _c.c.ref==job['ref'], _c.c.sha1==job['sha1'], _a.c.name==job['action']))).fetchone() aid, cid = res.action_id, res.commit_id db.execute(_a.update().values(status='started', start_time=datetime.datetime.utcfromtimestamp(job['start_time']), host=job['host']).where(_a.c.id==aid)) if why == 'action-done': res = db.execute(_a.join(_c).join(_r).select(use_labels=True).where( sql.and_(_r.c.name == job['repo'], _c.c.ref==job['ref'], _c.c.sha1==job['sha1'], _a.c.name==job['action']))).fetchone() aid, cid = res.action_id, res.commit_id db.execute(_a.update().values(status=job['result'], start_time=datetime.datetime.utcfromtimestamp(job['start_time']), end_time=datetime.datetime.utcfromtimestamp(job['end_time']), duration=job['duration'], host=job['host']).where(_a.c.id==aid)) if job['result'] == 'fail': db.execute(_c.update().values(status=job['result']).where(_c.c.id==cid)) elif db.execute(sql.select([sql.func.count(_a.c.id)]).where(sql.and_(_a.c.status!='success', _a.c.commit==cid))).scalar() == 0: db.execute(_c.update().values(status='success').where(_c.c.id==cid)) artefact_path = os.path.join(self.actions[job['action']].artefact_path, '%s@%s' % (job['ref'], job['sha1'])) for path, _, files in os.walk(artefact_path): for file in files: if file != 'log': file = os.path.join(path, file) rfile = file[len(artefact_path)+len(os.sep):] fsha1 = sha1_file(file) try: db.execute(_f.insert().values(action=aid, filename=rfile, sha1=fsha1)) except: # XXX should reschedule not simply delete things? db.execute(_f.update().where(sql.and_(_f.c.action==aid, _f.c.filename==rfile)).values(sha1=fsha1)) for nf in self.notifiers.values(): for what in nf.process: if fnmatch.fnmatch('action:' + job['action'], what): nf.schedule(job) if why == 'post-receive' and not refs: for head in self.git('for-each-ref', '--format', '%(refname)', 'refs/heads').stdout.splitlines(): lf = os.path.join(self.repo_path, 'logs', 'refs', 'heads', head[11:]) if not os.path.exists(lf): refs[head] = [] else: with open(lf) as fd: log = fd.readlines() refs[head] = [x.split(None, 2)[:2] for x in log] null = '0' * 40 for tag in self.git('for-each-ref', '--format', '%(refname) %(*objectname) %(objectname) %(taggerdate:raw) %(committerdate:raw)', 'refs/tags').stdout.splitlines(): data = tag.split() if not (data[-2].isdigit() and data[-1][1:].isdigit()): # Severely broken tag continue tag, sha = data[:2] ts = data[-2:] ts = int(ts[0]) + (-1 if ts[1][0] == '-' else 1) * (3600 * int(ts[1][1:3]) + 60 * int(ts[1][3:])) refs[tag] = [(null, sha)] tags.append((tag, ts)) if why == 'reschedule': # Set actions back to 'new' # Set dependent actions back to 'new' # Delete files from artefacts if job['action']: actions = [job['action']] else: actions = [x.name for x in db.execute(_a.select().where(sql.and_(_a.c.commit==c.id, _a.c.status=='retry'))).fetchall()] added = True while added: added = False for aname, action in self.actions.items(): if aname in actions: continue for act in actions: if 'action:' + act in action.requires: added = True actions.append(aname) break for action in actions: self.actions[action].clean(job['ref'], job['sha1']) db.execute(_a.update().values(status='new',host=None, start_time=None, end_time=None, duration=None).where(sql.and_(_a.c.commit==c.id, _a.c.name.in_(actions)))) tags.sort(key=lambda x: x[1], reverse=True) for aname, action in self.actions.items(): if job['why'] == 'action-done' and 'action:' + job['action'] not in action.requires: continue my_tags = [] for tag in tags: tag = tag[0][10:] for tag_ in action.tags: if tag_ == tag or (hasattr(tag_, 'match') and tag_.match(tag)): my_tags.append(tag) my_tags = my_tags[:action.backlog+1] for ref in refs: # Do we want to handle this thing? handle = False if ref.startswith('refs/heads'): head = ref[11:] for head_ in action.branches: if head_ == head or (hasattr(head_, 'match') and head_.match(head)): handle = True break elif ref.startswith('refs/tags'): tag = ref[10:] if tag in my_tags: handle = True if not handle: continue for prev_sha1, sha1 in refs[ref][-(action.backlog+1):]: cid = db.execute(_c.select().where(_c.c.repository==self.id).where(_c.c.ref==ref).where(_c.c.sha1==sha1)).fetchone() cid = cid.id if cid else db.execute(_c.insert().values(repository=self.id, ref=ref, sha1=sha1, prev_sha1=prev_sha1, submit_time=now(), status='new')).inserted_primary_key[0] act = db.execute(_a.select().where(_a.c.commit==cid).where(_a.c.name==action.name)).fetchone() if not act: db.execute(_a.insert().values(commit=cid, name=action.name, status='new')) act = db.execute(_a.select().where(_a.c.commit==cid).where(_a.c.name==action.name)).fetchone() # Check if all dependencies have been met if action.requires: requires = [x.replace('action:','') for x in action.requires] actions = db.execute(_a.select().where(_a.c.commit==cid).where(_a.c.name.in_(requires)).where(_a.c.status=='success')).fetchall() if len(actions) != len(action.requires): continue if act.status == 'new': db.execute(_a.update().where(_a.c.id==act.id).values(status='scheduled')) db.execute(_c.update().where(sql.and_(_c.c.id==cid, _c.c.status!='fail')).values(status='in-progress')) action.schedule(ref, prev_sha1, sha1)