Ejemplo n.º 1
0
    def __init__(self, worker, data):
        self.start_time = now()
        self.hook = {}
        self.env = {}
        self.publish = []
        self.worker = worker
        self.logger = worker.logger
        self.json = data.body

        data = json.loads(data.body)
        for key, value in data.items():
            if keyword.iskeyword(key):
                key += '_'
            setattr(self, key, value)

        self.env.update(os.environ)

        self.repo_path = os.path.join(worker.repo_dir, self.repo, self.repo + '.git')
        self.work_path = os.path.join(worker.repo_dir, self.repo, 'work', self.action, '%s@%s' % (self.ref, self.sha1))
        self.artefact_path = os.path.join(worker.repo_dir, self.repo, 'artefacts', self.action, '%s@%s' % (self.ref, self.sha1))

        if not worker.repo_checkout:
            self.env.update({'GIT_DIR': self.repo_path, 'GIT_WORK_TREE': self.work_path})

        if self.worker.repo_sync and not os.path.exists(self.repo_path):
            os.makedirs(self.repo_path)
        if os.path.exists(self.work_path):
            shutil.rmtree(self.work_path)
        os.makedirs(self.work_path)
        if os.path.exists(self.artefact_path):
            shutil.rmtree(self.artefact_path)
        os.makedirs(self.artefact_path)

        self.shell = whelk.Shell(output_callback=OutputLogger(self.logger), run_callback=RunLogger(self.logger), env=self.env, cwd=self.work_path, exit_callback=check_sp)
        self.pipe = whelk.Pipe(output_callback=OutputLogger(self.logger), run_callback=RunLogger(self.logger), env=self.env, cwd=self.work_path, exit_callback=check_sp)

        # Add a loghandler for our logfile so everything is properly logged per job
        self.log = os.path.join(self.artefact_path, 'log')
        handler = logging.FileHandler(self.log)
        handler.is_job_handler = True
        handler.setLevel(logging.DEBUG)
        handler.setFormatter(logging.Formatter(fmt='[%(asctime)s] %(message)s', datefmt='%H:%M:%S'))
        self.logger.addHandler(handler)
Ejemplo n.º 2
0
 def publish_results(self):
     for glb in getattr(self, 'publish', []):
         for file in glob.glob(os.path.join(self.work_path, glb)):
             self.logger.info("Adding artefact %s" % file.replace(self.work_path, ''))
             os.rename(file, os.path.join(self.artefact_path, os.path.basename(file)))
     local = self.artefact_path + os.sep
     remote = '/'.join([self.worker.rsync_root, self.repo, 'artefacts', self.action, '%s@%s' % (self.ref, self.sha1)]) + os.sep
     self.logger.info("Publishing %s => %s" % (local, remote))
     args = ['-av', local, remote]
     if self.worker.rsync_password:
         args += ['--password-file', self.worker.rsync_password]
     self.shell.rsync(*args)
     self.end_time = now()
     to_submit = {'repo': self.repo, 'ref': self.ref, 'prev_sha1': self.prev_sha1, 'sha1': self.sha1,
                  'why': 'action-done', 'action': self.action, 'result': self.result,
                  'start_time': toutctimestamp(self.start_time), 'end_time': toutctimestamp(self.end_time),
                  'duration': (self.end_time-self.start_time).total_seconds(),
                  'host': socket.gethostname()}
     self.worker.bs.use(self.worker.submit_queue)
     self.worker.bs.put(json.dumps(to_submit), ttr=600)
Ejemplo n.º 3
0
    def schedule(self, job, db):
        ref = job.get('ref', None)
        why = job['why']
        repo = job['repo']

        _c = golem.db.commit
        _a = golem.db.action
        _r = golem.db.repository
        _f = golem.db.artefact

        refs = {}
        tags = []
        if why == 'reschedule':
            if job['sha1']:
                c = db.execute(_c.select().where(sql.and_(_c.c.ref==job['ref'], _c.c.sha1==job['sha1']))).fetchone()
                if not c:
                    self.logger.error("Commit %s for ref %s cannot be rescheduled, it does not exist" % (job['sha1'], job['ref']))
                    return
            else:
                c = db.execute(_c.select().where(_c.c.ref==job['ref']).order_by(sql.desc(_c.c.submit_time)).limit(1)).fetchone()
                if not c:
                    self.logger.error("Cannot reschedule actions for ref %s, no commits were processed yet" % job['ref'])
                    return
            job['prev_sha1'], job['sha1'] = c.prev_sha1, c.sha1

        if ref and ref.startswith(('refs/heads', 'refs/tags')):
            refs[ref] = [(job['prev_sha1'], job['sha1'])]
        if ref and ref.startswith('refs/tags'):
            tags = [(ref, 0)]

        if why == 'action-started':
            res = db.execute(_a.join(_c).join(_r).select(use_labels=True).where(
                    sql.and_(_r.c.name == job['repo'], _c.c.ref==job['ref'], _c.c.sha1==job['sha1'], _a.c.name==job['action']))).fetchone()
            aid, cid = res.action_id, res.commit_id
            db.execute(_a.update().values(status='started', start_time=datetime.datetime.utcfromtimestamp(job['start_time']),
                host=job['host']).where(_a.c.id==aid))

        if why == 'action-done':
            res = db.execute(_a.join(_c).join(_r).select(use_labels=True).where(
                    sql.and_(_r.c.name == job['repo'], _c.c.ref==job['ref'], _c.c.sha1==job['sha1'], _a.c.name==job['action']))).fetchone()
            aid, cid = res.action_id, res.commit_id
            db.execute(_a.update().values(status=job['result'], start_time=datetime.datetime.utcfromtimestamp(job['start_time']), 
                                          end_time=datetime.datetime.utcfromtimestamp(job['end_time']), duration=job['duration'],
                                          host=job['host']).where(_a.c.id==aid))
            if job['result'] == 'fail':
                db.execute(_c.update().values(status=job['result']).where(_c.c.id==cid))
            elif db.execute(sql.select([sql.func.count(_a.c.id)]).where(sql.and_(_a.c.status!='success', _a.c.commit==cid))).scalar() == 0:
                db.execute(_c.update().values(status='success').where(_c.c.id==cid))
            artefact_path = os.path.join(self.actions[job['action']].artefact_path, '%s@%s' % (job['ref'], job['sha1']))
            for path, _, files in os.walk(artefact_path):
                for file in files:
                    if file != 'log':
                        file = os.path.join(path, file)
                        rfile = file[len(artefact_path)+len(os.sep):]
                        fsha1 = sha1_file(file)
                        try:
                            db.execute(_f.insert().values(action=aid, filename=rfile, sha1=fsha1))
                        except:
                            # XXX should reschedule not simply delete things?
                            db.execute(_f.update().where(sql.and_(_f.c.action==aid, _f.c.filename==rfile)).values(sha1=fsha1))
            for nf in self.notifiers.values():
                for what in nf.process:
                    if fnmatch.fnmatch('action:' + job['action'], what):
                        nf.schedule(job)

        if why == 'post-receive' and not refs:
            for head in self.git('for-each-ref', '--format', '%(refname)', 'refs/heads').stdout.splitlines():
                lf = os.path.join(self.repo_path, 'logs', 'refs', 'heads', head[11:])
                if not os.path.exists(lf):
                    refs[head] = []
                else:
                    with open(lf) as fd:
                        log = fd.readlines()
                        refs[head] = [x.split(None, 2)[:2] for x in log]
            null = '0' * 40
            for tag in self.git('for-each-ref', '--format', '%(refname) %(*objectname) %(objectname) %(taggerdate:raw) %(committerdate:raw)', 'refs/tags').stdout.splitlines():
                data = tag.split()
                if not (data[-2].isdigit() and data[-1][1:].isdigit()):
                    # Severely broken tag 
                    continue
                tag, sha = data[:2]
                ts = data[-2:]
                ts = int(ts[0]) + (-1 if ts[1][0] == '-' else 1) * (3600 * int(ts[1][1:3]) + 60 * int(ts[1][3:]))
                refs[tag] = [(null, sha)]
                tags.append((tag, ts))

        if why == 'reschedule':
            # Set actions back to 'new'
            # Set dependent actions back to 'new'
            # Delete files from artefacts
            if job['action']:
                actions = [job['action']]
            else:
                actions = [x.name for x in db.execute(_a.select().where(sql.and_(_a.c.commit==c.id, _a.c.status=='retry'))).fetchall()]
            added = True
            while added:
                added = False
                for aname, action in self.actions.items():
                    if aname in actions:
                        continue
                    for act in actions:
                        if 'action:' + act in action.requires:
                            added = True
                            actions.append(aname)
                            break
            for action in actions:
                self.actions[action].clean(job['ref'], job['sha1'])
            db.execute(_a.update().values(status='new',host=None, start_time=None, end_time=None,
                       duration=None).where(sql.and_(_a.c.commit==c.id, _a.c.name.in_(actions))))

        tags.sort(key=lambda x: x[1], reverse=True)

        for aname, action in self.actions.items():
            if job['why'] == 'action-done' and 'action:' + job['action'] not in action.requires:
                continue
            my_tags = []
            for tag in tags:
                tag = tag[0][10:]
                for tag_ in action.tags:
                    if tag_ == tag or (hasattr(tag_, 'match') and tag_.match(tag)):
                        my_tags.append(tag)
            my_tags = my_tags[:action.backlog+1]
            for ref in refs:
                # Do we want to handle this thing?
                handle = False
                if ref.startswith('refs/heads'):
                    head = ref[11:]
                    for head_ in action.branches:
                        if head_ == head or (hasattr(head_, 'match') and head_.match(head)):
                            handle = True
                            break
                elif ref.startswith('refs/tags'):
                    tag = ref[10:]
                    if tag in my_tags:
                        handle = True

                if not handle:
                    continue

                for prev_sha1, sha1 in refs[ref][-(action.backlog+1):]:
                    cid = db.execute(_c.select().where(_c.c.repository==self.id).where(_c.c.ref==ref).where(_c.c.sha1==sha1)).fetchone()
                    cid = cid.id if cid else db.execute(_c.insert().values(repository=self.id, ref=ref, sha1=sha1, prev_sha1=prev_sha1,
                                                        submit_time=now(), status='new')).inserted_primary_key[0]
                    act = db.execute(_a.select().where(_a.c.commit==cid).where(_a.c.name==action.name)).fetchone()
                    if not act:
                        db.execute(_a.insert().values(commit=cid, name=action.name, status='new'))
                        act = db.execute(_a.select().where(_a.c.commit==cid).where(_a.c.name==action.name)).fetchone()

                    # Check if all dependencies have been met
                    if action.requires:
                        requires = [x.replace('action:','') for x in action.requires]
                        actions = db.execute(_a.select().where(_a.c.commit==cid).where(_a.c.name.in_(requires)).where(_a.c.status=='success')).fetchall()
                        if len(actions) != len(action.requires):
                            continue

                    if act.status == 'new':
                        db.execute(_a.update().where(_a.c.id==act.id).values(status='scheduled'))
                        db.execute(_c.update().where(sql.and_(_c.c.id==cid, _c.c.status!='fail')).values(status='in-progress'))
                        action.schedule(ref, prev_sha1, sha1)