def actions_for(self, ref, sha1, db):
_c = golem.db.commit
_a = golem.db.action
_f = golem.db.artefact
cid = db.execute(_c.select('id').where(sql.and_(_c.c.ref==ref, _c.c.sha1==sha1))).fetchone()['id']
data = db.execute(_a.select().where(_a.c.commit==cid).order_by(sql.asc(_a.c.start_time))).fetchall()
data = [{'name': x.name, 'status': x.status, 'start_time': x.start_time, 'end_time': x.end_time, 'host': x.host,
'duration': x.duration, 'config': self.actions[x.name].config,
'files':[{'filename': y.filename, 'sha1': y.sha1} for y in db.execute(_f.select().where(_f.c.action==x.id)).fetchall()]}
for x in data if x.name in self.actions]
return data
def __init__(self, daemon, config, db):
self.configfile = config
self.mtime = os.path.getmtime(config)
config = ConfigParser(config)
IniConfig.__init__(self, config, 'repo')
self.logger = logging.getLogger('golem.repo.' + self.name)
self.logger.info("Parsing configuration for %s" % self.name)
self.path = os.path.join(daemon.repo_dir, self.name)
self.repo_path = os.path.join(self.path, self.name + '.git')
self.artefact_path = os.path.join(self.path, 'artefacts')
self.shell = whelk.Shell(output_callback=OutputLogger(self.logger), run_callback=RunLogger(self.logger), cwd=self.repo_path)
if hasattr(self, 'reflog_url'):
self.reflogtype = 'http'
self.reflogurl = self.reflog_url
elif re.match('^https?://', self.upstream):
self.reflogtype = 'http'
self.reflogurl = self.upstream + '/logs/%REF%'
elif self.upstream.startswith('file://'):
self.reflogtype = 'file'
self.upstream_path = self.upstream[7:]
if self.git('config', 'core.bare', cwd=self.upstream_path).stdout.strip() != 'false':
self.upstream_path = os.path.join(self.upstream_path, '.git')
elif ':' in self.upstream:
self.reflogtype = 'ssh'
else:
self.reflogtype = 'file'
self.upstream_path = self.upstream
if self.shell.git('config', 'core.bare', cwd=self.upstream).stdout.strip() != 'false':
self.upstream_path = os.path.join(self.upstream_path, '.git')
if re.match('^([a-z]+://|git@)github.com', self.upstream):
self.reflogtype = 'github'
if not self.reflogtype:
raise GolemError("Don't know how to fetch reflogs for %s" % self.name)
for section in config.sections():
if section.startswith('action:'):
action = section[7:]
self.logger.info(" Adding action %s" % action)
self.actions[action] = Action(config, section)
self.actions[action].artefact_path = os.path.join(self.artefact_path, action)
self.actions[action].daemon = daemon
self.actions[action].repo_name = self.name
elif section.startswith('notify:'):
nf = section[7:]
self.logger.info(" Adding notifier %s" % nf)
self.notifiers[nf] = Notifier(config, section)
self.notifiers[nf].daemon = daemon
self.notifiers[nf].repo_name = self.name
changed = True
while changed:
changed = False
for action in self.actions:
for req in self.actions[action].requires:
req = req[7:]
# Backlog is "inherited" from the dependencies
if self.actions[req].backlog < self.actions[action].backlog:
changed = True
self.actions[action].backlog = self.actions[req].backlog
# Same for branches and tags. Intersection of all parents
if self.actions[req].branches and not self.actions[action].branches:
changed = True
self.actions[action].branches = copy(self.actions[req].branches)
elif self.actions[req].branches:
for branch in self.actions[action].branches[:]:
if branch not in self.actions[req].branches:
changed = True
self.actions[action].branches.remove(branch)
if self.actions[req].tags and not self.actions[action].tags:
changed = True
self.actions[action].tags = copy(self.actions[req].tags)
elif self.actions[req].tags:
for tag in self.actions[action].tags[:]:
if tag not in self.actions[req].tags:
changed = True
self.actions[action].tags.remove(tag)
if not daemon.dummy:
self.create_dirs()
_r = golem.db.repository
self.id = db.execute(sql.select([_r.c.id]).where(_r.c.name==self.name)).fetchone()
self.id = self.id.id if self.id else db.execute(_r.insert().values(name=self.name)).inserted_primary_key[0]
def schedule(self, job, db):
ref = job.get('ref', None)
why = job['why']
repo = job['repo']
_c = golem.db.commit
_a = golem.db.action
_r = golem.db.repository
_f = golem.db.artefact
refs = {}
tags = []
if why == 'reschedule':
if job['sha1']:
c = db.execute(_c.select().where(sql.and_(_c.c.ref==job['ref'], _c.c.sha1==job['sha1']))).fetchone()
if not c:
self.logger.error("Commit %s for ref %s cannot be rescheduled, it does not exist" % (job['sha1'], job['ref']))
return
else:
c = db.execute(_c.select().where(_c.c.ref==job['ref']).order_by(sql.desc(_c.c.submit_time)).limit(1)).fetchone()
if not c:
self.logger.error("Cannot reschedule actions for ref %s, no commits were processed yet" % job['ref'])
return
job['prev_sha1'], job['sha1'] = c.prev_sha1, c.sha1
if ref and ref.startswith(('refs/heads', 'refs/tags')):
refs[ref] = [(job['prev_sha1'], job['sha1'])]
if ref and ref.startswith('refs/tags'):
tags = [(ref, 0)]
if why == 'action-started':
res = db.execute(_a.join(_c).join(_r).select(use_labels=True).where(
sql.and_(_r.c.name == job['repo'], _c.c.ref==job['ref'], _c.c.sha1==job['sha1'], _a.c.name==job['action']))).fetchone()
aid, cid = res.action_id, res.commit_id
db.execute(_a.update().values(status='started', start_time=datetime.datetime.utcfromtimestamp(job['start_time']),
host=job['host']).where(_a.c.id==aid))
if why == 'action-done':
res = db.execute(_a.join(_c).join(_r).select(use_labels=True).where(
sql.and_(_r.c.name == job['repo'], _c.c.ref==job['ref'], _c.c.sha1==job['sha1'], _a.c.name==job['action']))).fetchone()
aid, cid = res.action_id, res.commit_id
db.execute(_a.update().values(status=job['result'], start_time=datetime.datetime.utcfromtimestamp(job['start_time']),
end_time=datetime.datetime.utcfromtimestamp(job['end_time']), duration=job['duration'],
host=job['host']).where(_a.c.id==aid))
if job['result'] == 'fail':
db.execute(_c.update().values(status=job['result']).where(_c.c.id==cid))
elif db.execute(sql.select([sql.func.count(_a.c.id)]).where(sql.and_(_a.c.status!='success', _a.c.commit==cid))).scalar() == 0:
db.execute(_c.update().values(status='success').where(_c.c.id==cid))
artefact_path = os.path.join(self.actions[job['action']].artefact_path, '%s@%s' % (job['ref'], job['sha1']))
for path, _, files in os.walk(artefact_path):
for file in files:
if file != 'log':
file = os.path.join(path, file)
rfile = file[len(artefact_path)+len(os.sep):]
fsha1 = sha1_file(file)
try:
db.execute(_f.insert().values(action=aid, filename=rfile, sha1=fsha1))
except:
# XXX should reschedule not simply delete things?
db.execute(_f.update().where(sql.and_(_f.c.action==aid, _f.c.filename==rfile)).values(sha1=fsha1))
for nf in self.notifiers.values():
for what in nf.process:
if fnmatch.fnmatch('action:' + job['action'], what):
nf.schedule(job)
if why == 'post-receive' and not refs:
for head in self.git('for-each-ref', '--format', '%(refname)', 'refs/heads').stdout.splitlines():
lf = os.path.join(self.repo_path, 'logs', 'refs', 'heads', head[11:])
if not os.path.exists(lf):
refs[head] = []
else:
with open(lf) as fd:
log = fd.readlines()
refs[head] = [x.split(None, 2)[:2] for x in log]
null = '0' * 40
for tag in self.git('for-each-ref', '--format', '%(refname) %(*objectname) %(objectname) %(taggerdate:raw) %(committerdate:raw)', 'refs/tags').stdout.splitlines():
data = tag.split()
if not (data[-2].isdigit() and data[-1][1:].isdigit()):
# Severely broken tag
continue
tag, sha = data[:2]
ts = data[-2:]
ts = int(ts[0]) + (-1 if ts[1][0] == '-' else 1) * (3600 * int(ts[1][1:3]) + 60 * int(ts[1][3:]))
refs[tag] = [(null, sha)]
tags.append((tag, ts))
if why == 'reschedule':
# Set actions back to 'new'
# Set dependent actions back to 'new'
# Delete files from artefacts
if job['action']:
actions = [job['action']]
else:
actions = [x.name for x in db.execute(_a.select().where(sql.and_(_a.c.commit==c.id, _a.c.status=='retry'))).fetchall()]
added = True
while added:
added = False
for aname, action in self.actions.items():
if aname in actions:
continue
for act in actions:
if 'action:' + act in action.requires:
added = True
actions.append(aname)
break
for action in actions:
self.actions[action].clean(job['ref'], job['sha1'])
db.execute(_a.update().values(status='new',host=None, start_time=None, end_time=None,
duration=None).where(sql.and_(_a.c.commit==c.id, _a.c.name.in_(actions))))
tags.sort(key=lambda x: x[1], reverse=True)
for aname, action in self.actions.items():
if job['why'] == 'action-done' and 'action:' + job['action'] not in action.requires:
continue
my_tags = []
for tag in tags:
tag = tag[0][10:]
for tag_ in action.tags:
if tag_ == tag or (hasattr(tag_, 'match') and tag_.match(tag)):
my_tags.append(tag)
my_tags = my_tags[:action.backlog+1]
for ref in refs:
# Do we want to handle this thing?
handle = False
if ref.startswith('refs/heads'):
head = ref[11:]
for head_ in action.branches:
if head_ == head or (hasattr(head_, 'match') and head_.match(head)):
handle = True
break
elif ref.startswith('refs/tags'):
tag = ref[10:]
if tag in my_tags:
handle = True
if not handle:
continue
for prev_sha1, sha1 in refs[ref][-(action.backlog+1):]:
cid = db.execute(_c.select().where(_c.c.repository==self.id).where(_c.c.ref==ref).where(_c.c.sha1==sha1)).fetchone()
cid = cid.id if cid else db.execute(_c.insert().values(repository=self.id, ref=ref, sha1=sha1, prev_sha1=prev_sha1,
submit_time=now(), status='new')).inserted_primary_key[0]
act = db.execute(_a.select().where(_a.c.commit==cid).where(_a.c.name==action.name)).fetchone()
if not act:
db.execute(_a.insert().values(commit=cid, name=action.name, status='new'))
act = db.execute(_a.select().where(_a.c.commit==cid).where(_a.c.name==action.name)).fetchone()
# Check if all dependencies have been met
if action.requires:
requires = [x.replace('action:','') for x in action.requires]
actions = db.execute(_a.select().where(_a.c.commit==cid).where(_a.c.name.in_(requires)).where(_a.c.status=='success')).fetchall()
if len(actions) != len(action.requires):
continue
if act.status == 'new':
db.execute(_a.update().where(_a.c.id==act.id).values(status='scheduled'))
db.execute(_c.update().where(sql.and_(_c.c.id==cid, _c.c.status!='fail')).values(status='in-progress'))
action.schedule(ref, prev_sha1, sha1)
def commit(self, ref, sha1, db):
_c = golem.db.commit
return db.execute(_c.select().where(sql.and_(_c.c.repository==self.id, _c.c.ref==ref, _c.c.sha1==sha1))).fetchone()
def last_commits(self, count, db):
_c = golem.db.commit
return db.execute(_c.select().where(_c.c.repository==self.id).order_by(sql.desc(_c.c.submit_time)).limit(count)).fetchall()