def actions_for(self, ref, sha1, db): _c = golem.db.commit _a = golem.db.action _f = golem.db.artefact cid = db.execute(_c.select('id').where(sql.and_(_c.c.ref==ref, _c.c.sha1==sha1))).fetchone()['id'] data = db.execute(_a.select().where(_a.c.commit==cid).order_by(sql.asc(_a.c.start_time))).fetchall() data = [{'name': x.name, 'status': x.status, 'start_time': x.start_time, 'end_time': x.end_time, 'host': x.host, 'duration': x.duration, 'config': self.actions[x.name].config, 'files':[{'filename': y.filename, 'sha1': y.sha1} for y in db.execute(_f.select().where(_f.c.action==x.id)).fetchall()]} for x in data if x.name in self.actions] return data
def __init__(self, daemon, config, db): self.configfile = config self.mtime = os.path.getmtime(config) config = ConfigParser(config) IniConfig.__init__(self, config, 'repo') self.logger = logging.getLogger('golem.repo.' + self.name) self.logger.info("Parsing configuration for %s" % self.name) self.path = os.path.join(daemon.repo_dir, self.name) self.repo_path = os.path.join(self.path, self.name + '.git') self.artefact_path = os.path.join(self.path, 'artefacts') self.shell = whelk.Shell(output_callback=OutputLogger(self.logger), run_callback=RunLogger(self.logger), cwd=self.repo_path) if hasattr(self, 'reflog_url'): self.reflogtype = 'http' self.reflogurl = self.reflog_url elif re.match('^https?://', self.upstream): self.reflogtype = 'http' self.reflogurl = self.upstream + '/logs/%REF%' elif self.upstream.startswith('file://'): self.reflogtype = 'file' self.upstream_path = self.upstream[7:] if self.git('config', 'core.bare', cwd=self.upstream_path).stdout.strip() != 'false': self.upstream_path = os.path.join(self.upstream_path, '.git') elif ':' in self.upstream: self.reflogtype = 'ssh' else: self.reflogtype = 'file' self.upstream_path = self.upstream if self.shell.git('config', 'core.bare', cwd=self.upstream).stdout.strip() != 'false': self.upstream_path = os.path.join(self.upstream_path, '.git') if re.match('^([a-z]+://|git@)github.com', self.upstream): self.reflogtype = 'github' if not self.reflogtype: raise GolemError("Don't know how to fetch reflogs for %s" % self.name) for section in config.sections(): if section.startswith('action:'): action = section[7:] self.logger.info(" Adding action %s" % action) self.actions[action] = Action(config, section) self.actions[action].artefact_path = os.path.join(self.artefact_path, action) self.actions[action].daemon = daemon self.actions[action].repo_name = self.name elif section.startswith('notify:'): nf = section[7:] self.logger.info(" Adding notifier %s" % nf) self.notifiers[nf] = Notifier(config, section) self.notifiers[nf].daemon = daemon self.notifiers[nf].repo_name = self.name changed = True while changed: changed = False for action in self.actions: for req in self.actions[action].requires: req = req[7:] # Backlog is "inherited" from the dependencies if self.actions[req].backlog < self.actions[action].backlog: changed = True self.actions[action].backlog = self.actions[req].backlog # Same for branches and tags. Intersection of all parents if self.actions[req].branches and not self.actions[action].branches: changed = True self.actions[action].branches = copy(self.actions[req].branches) elif self.actions[req].branches: for branch in self.actions[action].branches[:]: if branch not in self.actions[req].branches: changed = True self.actions[action].branches.remove(branch) if self.actions[req].tags and not self.actions[action].tags: changed = True self.actions[action].tags = copy(self.actions[req].tags) elif self.actions[req].tags: for tag in self.actions[action].tags[:]: if tag not in self.actions[req].tags: changed = True self.actions[action].tags.remove(tag) if not daemon.dummy: self.create_dirs() _r = golem.db.repository self.id = db.execute(sql.select([_r.c.id]).where(_r.c.name==self.name)).fetchone() self.id = self.id.id if self.id else db.execute(_r.insert().values(name=self.name)).inserted_primary_key[0]
def schedule(self, job, db): ref = job.get('ref', None) why = job['why'] repo = job['repo'] _c = golem.db.commit _a = golem.db.action _r = golem.db.repository _f = golem.db.artefact refs = {} tags = [] if why == 'reschedule': if job['sha1']: c = db.execute(_c.select().where(sql.and_(_c.c.ref==job['ref'], _c.c.sha1==job['sha1']))).fetchone() if not c: self.logger.error("Commit %s for ref %s cannot be rescheduled, it does not exist" % (job['sha1'], job['ref'])) return else: c = db.execute(_c.select().where(_c.c.ref==job['ref']).order_by(sql.desc(_c.c.submit_time)).limit(1)).fetchone() if not c: self.logger.error("Cannot reschedule actions for ref %s, no commits were processed yet" % job['ref']) return job['prev_sha1'], job['sha1'] = c.prev_sha1, c.sha1 if ref and ref.startswith(('refs/heads', 'refs/tags')): refs[ref] = [(job['prev_sha1'], job['sha1'])] if ref and ref.startswith('refs/tags'): tags = [(ref, 0)] if why == 'action-started': res = db.execute(_a.join(_c).join(_r).select(use_labels=True).where( sql.and_(_r.c.name == job['repo'], _c.c.ref==job['ref'], _c.c.sha1==job['sha1'], _a.c.name==job['action']))).fetchone() aid, cid = res.action_id, res.commit_id db.execute(_a.update().values(status='started', start_time=datetime.datetime.utcfromtimestamp(job['start_time']), host=job['host']).where(_a.c.id==aid)) if why == 'action-done': res = db.execute(_a.join(_c).join(_r).select(use_labels=True).where( sql.and_(_r.c.name == job['repo'], _c.c.ref==job['ref'], _c.c.sha1==job['sha1'], _a.c.name==job['action']))).fetchone() aid, cid = res.action_id, res.commit_id db.execute(_a.update().values(status=job['result'], start_time=datetime.datetime.utcfromtimestamp(job['start_time']), end_time=datetime.datetime.utcfromtimestamp(job['end_time']), duration=job['duration'], host=job['host']).where(_a.c.id==aid)) if job['result'] == 'fail': db.execute(_c.update().values(status=job['result']).where(_c.c.id==cid)) elif db.execute(sql.select([sql.func.count(_a.c.id)]).where(sql.and_(_a.c.status!='success', _a.c.commit==cid))).scalar() == 0: db.execute(_c.update().values(status='success').where(_c.c.id==cid)) artefact_path = os.path.join(self.actions[job['action']].artefact_path, '%s@%s' % (job['ref'], job['sha1'])) for path, _, files in os.walk(artefact_path): for file in files: if file != 'log': file = os.path.join(path, file) rfile = file[len(artefact_path)+len(os.sep):] fsha1 = sha1_file(file) try: db.execute(_f.insert().values(action=aid, filename=rfile, sha1=fsha1)) except: # XXX should reschedule not simply delete things? db.execute(_f.update().where(sql.and_(_f.c.action==aid, _f.c.filename==rfile)).values(sha1=fsha1)) for nf in self.notifiers.values(): for what in nf.process: if fnmatch.fnmatch('action:' + job['action'], what): nf.schedule(job) if why == 'post-receive' and not refs: for head in self.git('for-each-ref', '--format', '%(refname)', 'refs/heads').stdout.splitlines(): lf = os.path.join(self.repo_path, 'logs', 'refs', 'heads', head[11:]) if not os.path.exists(lf): refs[head] = [] else: with open(lf) as fd: log = fd.readlines() refs[head] = [x.split(None, 2)[:2] for x in log] null = '0' * 40 for tag in self.git('for-each-ref', '--format', '%(refname) %(*objectname) %(objectname) %(taggerdate:raw) %(committerdate:raw)', 'refs/tags').stdout.splitlines(): data = tag.split() if not (data[-2].isdigit() and data[-1][1:].isdigit()): # Severely broken tag continue tag, sha = data[:2] ts = data[-2:] ts = int(ts[0]) + (-1 if ts[1][0] == '-' else 1) * (3600 * int(ts[1][1:3]) + 60 * int(ts[1][3:])) refs[tag] = [(null, sha)] tags.append((tag, ts)) if why == 'reschedule': # Set actions back to 'new' # Set dependent actions back to 'new' # Delete files from artefacts if job['action']: actions = [job['action']] else: actions = [x.name for x in db.execute(_a.select().where(sql.and_(_a.c.commit==c.id, _a.c.status=='retry'))).fetchall()] added = True while added: added = False for aname, action in self.actions.items(): if aname in actions: continue for act in actions: if 'action:' + act in action.requires: added = True actions.append(aname) break for action in actions: self.actions[action].clean(job['ref'], job['sha1']) db.execute(_a.update().values(status='new',host=None, start_time=None, end_time=None, duration=None).where(sql.and_(_a.c.commit==c.id, _a.c.name.in_(actions)))) tags.sort(key=lambda x: x[1], reverse=True) for aname, action in self.actions.items(): if job['why'] == 'action-done' and 'action:' + job['action'] not in action.requires: continue my_tags = [] for tag in tags: tag = tag[0][10:] for tag_ in action.tags: if tag_ == tag or (hasattr(tag_, 'match') and tag_.match(tag)): my_tags.append(tag) my_tags = my_tags[:action.backlog+1] for ref in refs: # Do we want to handle this thing? handle = False if ref.startswith('refs/heads'): head = ref[11:] for head_ in action.branches: if head_ == head or (hasattr(head_, 'match') and head_.match(head)): handle = True break elif ref.startswith('refs/tags'): tag = ref[10:] if tag in my_tags: handle = True if not handle: continue for prev_sha1, sha1 in refs[ref][-(action.backlog+1):]: cid = db.execute(_c.select().where(_c.c.repository==self.id).where(_c.c.ref==ref).where(_c.c.sha1==sha1)).fetchone() cid = cid.id if cid else db.execute(_c.insert().values(repository=self.id, ref=ref, sha1=sha1, prev_sha1=prev_sha1, submit_time=now(), status='new')).inserted_primary_key[0] act = db.execute(_a.select().where(_a.c.commit==cid).where(_a.c.name==action.name)).fetchone() if not act: db.execute(_a.insert().values(commit=cid, name=action.name, status='new')) act = db.execute(_a.select().where(_a.c.commit==cid).where(_a.c.name==action.name)).fetchone() # Check if all dependencies have been met if action.requires: requires = [x.replace('action:','') for x in action.requires] actions = db.execute(_a.select().where(_a.c.commit==cid).where(_a.c.name.in_(requires)).where(_a.c.status=='success')).fetchall() if len(actions) != len(action.requires): continue if act.status == 'new': db.execute(_a.update().where(_a.c.id==act.id).values(status='scheduled')) db.execute(_c.update().where(sql.and_(_c.c.id==cid, _c.c.status!='fail')).values(status='in-progress')) action.schedule(ref, prev_sha1, sha1)
def commit(self, ref, sha1, db): _c = golem.db.commit return db.execute(_c.select().where(sql.and_(_c.c.repository==self.id, _c.c.ref==ref, _c.c.sha1==sha1))).fetchone()
def last_commits(self, count, db): _c = golem.db.commit return db.execute(_c.select().where(_c.c.repository==self.id).order_by(sql.desc(_c.c.submit_time)).limit(count)).fetchall()