def test_handlePushes_space_files(self): repo = Repository.objects.create(name='mozilla-central', url='file:///' + self.repo) self.assertEqual(handlePushes(repo.pk, []), None) ui = mock_ui() hgcommands.init(ui, self.repo) hgrepo = repository(ui, self.repo) (open(hgrepo.pathto('file.dtd '), 'w') # deliberate trailing space .write(''' <!ENTITY key1 "Hello"> <!ENTITY key2 "Cruel"> ''')) hgcommands.addremove(ui, hgrepo) hgcommands.commit(ui, hgrepo, user="******", message="initial commit") rev0 = hgrepo[0].hex() timestamp = int(time.time()) pushjs0 = PushJS(100, { 'date': timestamp, 'changesets': [rev0], 'user': '******', }) handlePushes(repo.pk, [pushjs0]) file_, = File.objects.all() self.assertEqual(file_.path, 'file.dtd ')
def test_handlePushes_repeated(self): repo = Repository.objects.create(name="mozilla-central", url="file:///" + self.repo) self.assertEqual(handlePushes(repo.pk, []), None) ui = mock_ui() hgcommands.init(ui, self.repo) hgrepo = repository(ui, self.repo) ( open(hgrepo.pathto("file.dtd"), "w").write( """ <!ENTITY key1 "Hello"> <!ENTITY key2 "Cruel"> """ ) ) hgcommands.addremove(ui, hgrepo) hgcommands.commit(ui, hgrepo, user="******", message="initial commit") rev0 = hgrepo[0].hex() timestamp = int(time.time()) pushjs0 = PushJS(100, {"date": timestamp, "changesets": [rev0], "user": "******"}) # first time pushes_initial = Push.objects.all().count() result = handlePushes(repo.pk, [pushjs0]) self.assertEqual(result, 1) pushes_after = Push.objects.all().count() self.assertEqual(pushes_initial, pushes_after - 1) # a second time should be harmless result = handlePushes(repo.pk, [pushjs0]) self.assertEqual(result, 1) pushes_after_after = Push.objects.all().count() self.assertEqual(pushes_after, pushes_after_after)
def process_pushes(self, body, message): if body.get('type') != 'hg-push': return logger.info('got hg-push message %r', body) try: handlePushes(body['repository_id'], [PushJS(p['id'], p) for p in body['pushes']], do_update=False) self.retries = 0 except KeyboardInterrupt: raise except Exception: self.retries += 1 logger.error('handlePushes failed: %d' % self.retries, exc_info=True) if self.sentry: self.sentry.captureException() if self.retries > self.max_retries: # this problem might be real, let's just die # and have a human figure it out raise time.sleep(self.retries) message.requeue() return message.ack()
def test_handlePushes_repeated(self): repo = Repository.objects.create(name='mozilla-central', url='file:///' + self.repo) with hglib.init(self.repo).open() as hgrepo: with open(hgrepo.pathto('file.dtd'), 'w') as fh: fh.write(''' <!ENTITY key1 "Hello"> <!ENTITY key2 "Cruel"> ''') hgrepo.commit(user="******", message="initial commit", addremove=True) rev0 = hgrepo[0].node().decode('ascii') timestamp = int(time.time()) pushjs0 = PushJS(100, { 'date': timestamp, 'changesets': [rev0], 'user': '******', }) # first time pushes_initial = Push.objects.all().count() result = handlePushes(repo.pk, [pushjs0]) self.assertEqual(result, 1) pushes_after = Push.objects.all().count() self.assertEqual(pushes_initial, pushes_after - 1) # a second time should be harmless result = handlePushes(repo.pk, [pushjs0]) self.assertEqual(result, 1) pushes_after_after = Push.objects.all().count() self.assertEqual(pushes_after, pushes_after_after)
def test_handlePushes_space_files(self): repo = Repository.objects.create( name='mozilla-central', url='file:///' + self.repo ) self.assertEqual(handlePushes(repo.pk, []), None) ui = mock_ui() hgcommands.init(ui, self.repo) hgrepo = repository(ui, self.repo) (open(hgrepo.pathto('file.dtd '), 'w') # deliberate trailing space .write(''' <!ENTITY key1 "Hello"> <!ENTITY key2 "Cruel"> ''')) hgcommands.addremove(ui, hgrepo) hgcommands.commit(ui, hgrepo, user="******", message="initial commit") rev0 = hgrepo[0].hex() timestamp = int(time.time()) pushjs0 = PushJS(100, { 'date': timestamp, 'changesets': [rev0], 'user': '******', }) handlePushes(repo.pk, [pushjs0]) file_, = File.objects.all() self.assertEqual(file_.path, 'file.dtd ')
def test_handlePushes_space_files(self): repo = Repository.objects.create(name="mozilla-central", url="file:///" + self.repo) self.assertEqual(handlePushes(repo.pk, []), None) ui = mock_ui() hgcommands.init(ui, self.repo) hgrepo = repository(ui, self.repo) ( open(hgrepo.pathto("file.dtd "), "w").write( # deliberate trailing space """ <!ENTITY key1 "Hello"> <!ENTITY key2 "Cruel"> """ ) ) hgcommands.addremove(ui, hgrepo) hgcommands.commit(ui, hgrepo, user="******", message="initial commit") rev0 = hgrepo[0].hex() timestamp = int(time.time()) pushjs0 = PushJS(100, {"date": timestamp, "changesets": [rev0], "user": "******"}) handlePushes(repo.pk, [pushjs0]) file_, = File.objects.all() self.assertEqual(file_.path, "file.dtd ")
def test_handlePushes_space_files(self): repo = Repository.objects.create(name='mozilla-central', url='file:///' + self.repo) with hglib.init(self.repo).open() as hgrepo: # deliberate trailing space in file name with open(hgrepo.pathto('file.dtd '), 'w') as fh: fh.write(''' <!ENTITY key1 "Hello"> <!ENTITY key2 "Cruel"> ''') hgrepo.commit(user="******", message="initial commit", addremove=True) rev0 = hgrepo[0].node().decode('ascii') timestamp = int(time.time()) pushjs0 = PushJS(100, { 'date': timestamp, 'changesets': [rev0], 'user': '******', }) handlePushes(repo.pk, [pushjs0]) file_, = File.objects.all() self.assertEqual(file_.path, 'file.dtd ')
def test_handlePushes_twice(self): repo = Repository.objects.create( name='mozilla-central', url='file://' + self.repo ) ui = mock_ui() hgcommands.init(ui, self.repo) hgrepo = repository(ui, self.repo) (open(hgrepo.pathto('file.dtd'), 'w') .write(''' <!ENTITY key1 "Hello"> <!ENTITY key2 "Cruel"> ''')) hgcommands.addremove(ui, hgrepo) hgcommands.commit(ui, hgrepo, user="******", message="initial commit") rev0 = hgrepo[0].hex() timestamp = int(time.time()) pushjs0 = PushJS(100, { 'date': timestamp, 'changesets': [rev0], 'user': '******', }) result = handlePushes(repo.pk, [pushjs0]) (open(hgrepo.pathto('file.dtd'), 'w') .write(''' <!ENTITY key1 "Hello"> <!ENTITY key2 "Cruel"> <!ENTITY key3 "World"> ''')) hgcommands.commit(ui, hgrepo, user="******", message="Second commit") rev1 = hgrepo[1].hex() # a second time timestamp = int(time.time()) pushjs0 = PushJS(101, { 'date': timestamp, 'changesets': [rev1], 'user': '******', }) # re-fetch repo = Repository.objects.get(pk=repo.pk) self.assertEqual(repo.changesets.all().count(), 2) result = handlePushes(repo.pk, [pushjs0]) self.assertEqual(result, 1) # re-fetch repo = Repository.objects.get(pk=repo.pk) self.assertEqual(repo.changesets.all().count(), 3)
def test_handlePushes_twice(self): repo = Repository.objects.create(name='mozilla-central', url='file://' + self.repo) ui = mock_ui() hgcommands.init(ui, self.repo) hgrepo = repository(ui, self.repo) (open(hgrepo.pathto('file.dtd'), 'w').write(''' <!ENTITY key1 "Hello"> <!ENTITY key2 "Cruel"> ''')) hgcommands.addremove(ui, hgrepo) hgcommands.commit(ui, hgrepo, user="******", message="initial commit") rev0 = hgrepo[0].hex() timestamp = int(time.time()) pushjs0 = PushJS(100, { 'date': timestamp, 'changesets': [rev0], 'user': '******', }) result = handlePushes(repo.pk, [pushjs0]) (open(hgrepo.pathto('file.dtd'), 'w').write(''' <!ENTITY key1 "Hello"> <!ENTITY key2 "Cruel"> <!ENTITY key3 "World"> ''')) hgcommands.commit(ui, hgrepo, user="******", message="Second commit") rev1 = hgrepo[1].hex() # a second time timestamp = int(time.time()) pushjs0 = PushJS(101, { 'date': timestamp, 'changesets': [rev1], 'user': '******', }) # re-fetch repo = Repository.objects.get(pk=repo.pk) self.assertEqual(repo.changesets.all().count(), 2) result = handlePushes(repo.pk, [pushjs0]) self.assertEqual(result, 1) # re-fetch repo = Repository.objects.get(pk=repo.pk) self.assertEqual(repo.changesets.all().count(), 3)
def test_handlePushes_twice(self): repo = Repository.objects.create( name='mozilla-central', url='file://' + self.repo ) with hglib.init(self.repo).open() as hgrepo: with open(hgrepo.pathto('file.dtd'), 'w') as fh: fh.write(''' <!ENTITY key1 "Hello"> <!ENTITY key2 "Cruel"> ''') hgrepo.commit(user="******", message="initial commit", addremove=True) rev0 = hgrepo[0].node().decode('ascii') timestamp = int(time.time()) pushjs0 = PushJS(100, { 'date': timestamp, 'changesets': [rev0], 'user': '******', }) result = handlePushes(repo.pk, [pushjs0]) with open(hgrepo.pathto('file.dtd'), 'w') as fh: fh.write(''' <!ENTITY key1 "Hello"> <!ENTITY key2 "Cruel"> <!ENTITY key3 "World"> ''') hgrepo.commit(user="******", message="Second commit") rev1 = hgrepo[1].node().decode('ascii') # a second time timestamp = int(time.time()) pushjs0 = PushJS(101, { 'date': timestamp, 'changesets': [rev1], 'user': '******', }) # re-fetch repo = Repository.objects.get(pk=repo.pk) self.assertEqual(repo.changesets.all().count(), 2) result = handlePushes(repo.pk, [pushjs0]) self.assertEqual(result, 1) # re-fetch repo = Repository.objects.get(pk=repo.pk) self.assertEqual(repo.changesets.all().count(), 3)
def test_handlePushes(self): repo = Repository.objects.create( name='mozilla-central', url='file:///' + self.repo ) self.assertEqual(handlePushes(repo.pk, []), None) ui = mock_ui() hgcommands.init(ui, self.repo) hgrepo = repository(ui, self.repo) (open(hgrepo.pathto('file.dtd'), 'w') .write(''' <!ENTITY key1 "Hello"> <!ENTITY key2 "Cruel"> ''')) hgcommands.addremove(ui, hgrepo) hgcommands.commit(ui, hgrepo, user="******", message="initial commit") rev0 = hgrepo[0].hex() timestamp = int(time.time()) push_id = 100 username = '******' pushjs0 = PushJS(push_id, { 'date': timestamp, 'changesets': [rev0], 'user': username, }) result = handlePushes(repo.pk, [pushjs0]) self.assertEqual(result, 1) # expect all of these to have been created push, = Push.objects.all() branch, = Branch.objects.all() changeset, = push.changesets.all() self.assertEqual(push.repository, repo) self.assertEqual(push.push_id, push_id) self.assertEqual(push.user, username) self.assertEqual(push.push_date.strftime('%Y%m%d%H%M'), datetime.datetime.utcnow().strftime('%Y%m%d%H%M')) self.assertEqual(changeset.description, 'initial commit') self.assertEqual(changeset.user, 'Jane Doe <*****@*****.**>') self.assertEqual(changeset.revision, rev0) self.assertEqual(changeset.branch, branch) self.assertEqual(branch.name, 'default')
def test_handlePushes_twice(self): repo = Repository.objects.create(name="mozilla-central", url="file://" + self.repo) ui = mock_ui() hgcommands.init(ui, self.repo) hgrepo = repository(ui, self.repo) ( open(hgrepo.pathto("file.dtd"), "w").write( """ <!ENTITY key1 "Hello"> <!ENTITY key2 "Cruel"> """ ) ) hgcommands.addremove(ui, hgrepo) hgcommands.commit(ui, hgrepo, user="******", message="initial commit") rev0 = hgrepo[0].hex() timestamp = int(time.time()) pushjs0 = PushJS(100, {"date": timestamp, "changesets": [rev0], "user": "******"}) result = handlePushes(repo.pk, [pushjs0]) ( open(hgrepo.pathto("file.dtd"), "w").write( """ <!ENTITY key1 "Hello"> <!ENTITY key2 "Cruel"> <!ENTITY key3 "World"> """ ) ) hgcommands.commit(ui, hgrepo, user="******", message="Second commit") rev1 = hgrepo[1].hex() # a second time timestamp = int(time.time()) pushjs0 = PushJS(101, {"date": timestamp, "changesets": [rev1], "user": "******"}) # re-fetch repo = Repository.objects.get(pk=repo.pk) self.assertEqual(repo.changesets.all().count(), 2) result = handlePushes(repo.pk, [pushjs0]) self.assertEqual(result, 1) # re-fetch repo = Repository.objects.get(pk=repo.pk) self.assertEqual(repo.changesets.all().count(), 3)
def on_hg_changegroup(self, repo_name, payload): try: repo = Repository.objects.get(name=repo_name) except Repository.DoesNotExist: logging.info(f"push:skipping {repo_name}") return new_pushid = max( p['pushid'] for p in payload['data']['pushlog_pushes'] ) logging.info( f"push:handle {repo.url} {repo.last_known_push()}-{new_pushid}" ) pushes = utils.PushJS.pushes_for(repo, new_pushid) logging.info(f"push: found {len(pushes)} pushes for {repo_name}") utils.handlePushes(repo.id, pushes)
def test_handlePushes(self): repo = Repository.objects.create(name='mozilla-central', url='file:///' + self.repo) self.assertEqual(handlePushes(repo.pk, []), None) ui = mock_ui() hgcommands.init(ui, self.repo) hgrepo = repository(ui, self.repo) (open(hgrepo.pathto('file.dtd'), 'w').write(''' <!ENTITY key1 "Hello"> <!ENTITY key2 "Cruel"> ''')) hgcommands.addremove(ui, hgrepo) hgcommands.commit(ui, hgrepo, user="******", message="initial commit") rev0 = hgrepo[0].hex() timestamp = int(time.time()) push_id = 100 username = '******' pushjs0 = PushJS(push_id, { 'date': timestamp, 'changesets': [rev0], 'user': username, }) result = handlePushes(repo.pk, [pushjs0]) self.assertEqual(result, 1) # expect all of these to have been created push, = Push.objects.all() branch, = Branch.objects.all() changeset, = push.changesets.all() self.assertEqual(push.repository, repo) self.assertEqual(push.push_id, push_id) self.assertEqual(push.user, username) self.assertEqual(push.push_date.strftime('%Y%m%d%H%M'), datetime.datetime.utcnow().strftime('%Y%m%d%H%M')) self.assertEqual(changeset.description, 'initial commit') self.assertEqual(changeset.user, 'Jane Doe <*****@*****.**>') self.assertEqual(changeset.revision, rev0) self.assertEqual(changeset.branch, branch) self.assertEqual(branch.name, 'default')
def test_handlePushes_cause_repoerror(self): repo = Repository.objects.create( name='mozilla-central', url='file:///does/not/exist' ) self.assertEqual(handlePushes(repo.pk, []), None) ui = mock_ui() hgcommands.init(ui, self.repo) hgrepo = repository(ui, self.repo) (open(hgrepo.pathto('file.dtd'), 'w') .write(''' <!ENTITY key1 "Hello"> <!ENTITY key2 "Cruel"> ''')) hgcommands.addremove(ui, hgrepo) hgcommands.commit(ui, hgrepo, user="******", message="initial commit") rev0 = hgrepo[0].hex() timestamp = int(time.time()) pushjs0 = PushJS(100, { 'date': timestamp, 'changesets': [rev0], 'user': '******', }) self.assertRaises(RepoError, handlePushes, repo.pk, [pushjs0])
def test_handlePushes_cause_repoerror(self): repo = Repository.objects.create(name='mozilla-central', url='file:///does/not/exist') self.assertEqual(handlePushes(repo.pk, []), None) ui = mock_ui() hgcommands.init(ui, self.repo) hgrepo = repository(ui, self.repo) (open(hgrepo.pathto('file.dtd'), 'w').write(''' <!ENTITY key1 "Hello"> <!ENTITY key2 "Cruel"> ''')) hgcommands.addremove(ui, hgrepo) hgcommands.commit(ui, hgrepo, user="******", message="initial commit") rev0 = hgrepo[0].hex() timestamp = int(time.time()) pushjs0 = PushJS(100, { 'date': timestamp, 'changesets': [rev0], 'user': '******', }) self.assertRaises(RepoError, handlePushes, repo.pk, [pushjs0])
def test_handlePushes(self): repo = Repository.objects.create(name="mozilla-central", url="file:///" + self.repo) self.assertEqual(handlePushes(repo.pk, []), None) ui = mock_ui() hgcommands.init(ui, self.repo) hgrepo = repository(ui, self.repo) ( open(hgrepo.pathto("file.dtd"), "w").write( """ <!ENTITY key1 "Hello"> <!ENTITY key2 "Cruel"> """ ) ) hgcommands.addremove(ui, hgrepo) hgcommands.commit(ui, hgrepo, user="******", message="initial commit") rev0 = hgrepo[0].hex() timestamp = int(time.time()) push_id = 100 username = "******" pushjs0 = PushJS(push_id, {"date": timestamp, "changesets": [rev0], "user": username}) result = handlePushes(repo.pk, [pushjs0]) self.assertEqual(result, 1) # expect all of these to have been created push, = Push.objects.all() branch, = Branch.objects.all() changeset, = push.changesets.all() self.assertEqual(push.repository, repo) self.assertEqual(push.push_id, push_id) self.assertEqual(push.user, username) self.assertEqual(push.push_date.strftime("%Y%m%d%H%M"), datetime.datetime.utcnow().strftime("%Y%m%d%H%M")) self.assertEqual(changeset.description, "initial commit") self.assertEqual(changeset.user, "Jane Doe <*****@*****.**>") self.assertEqual(changeset.revision, rev0) self.assertEqual(changeset.branch, branch) self.assertEqual(branch.name, "default")
def test_handlePushes_repeated(self): repo = Repository.objects.create(name='mozilla-central', url='file:///' + self.repo) self.assertEqual(handlePushes(repo.pk, []), None) ui = mock_ui() hgcommands.init(ui, self.repo) hgrepo = repository(ui, self.repo) (open(hgrepo.pathto('file.dtd'), 'w').write(''' <!ENTITY key1 "Hello"> <!ENTITY key2 "Cruel"> ''')) hgcommands.addremove(ui, hgrepo) hgcommands.commit(ui, hgrepo, user="******", message="initial commit") rev0 = hgrepo[0].hex() timestamp = int(time.time()) pushjs0 = PushJS(100, { 'date': timestamp, 'changesets': [rev0], 'user': '******', }) # first time pushes_initial = Push.objects.all().count() result = handlePushes(repo.pk, [pushjs0]) self.assertEqual(result, 1) pushes_after = Push.objects.all().count() self.assertEqual(pushes_initial, pushes_after - 1) # a second time should be harmless result = handlePushes(repo.pk, [pushjs0]) self.assertEqual(result, 1) pushes_after_after = Push.objects.all().count() self.assertEqual(pushes_after, pushes_after_after)
def test_handlePushes_repeated(self): repo = Repository.objects.create( name='mozilla-central', url='file:///' + self.repo ) hgrepo = hglib.init(self.repo).open() (open(hgrepo.pathto('file.dtd'), 'w') .write(''' <!ENTITY key1 "Hello"> <!ENTITY key2 "Cruel"> ''')) hgrepo.commit(user="******", message="initial commit", addremove=True) rev0 = hgrepo[0].node() timestamp = int(time.time()) pushjs0 = PushJS(100, { 'date': timestamp, 'changesets': [rev0], 'user': '******', }) # first time pushes_initial = Push.objects.all().count() result = handlePushes(repo.pk, [pushjs0]) self.assertEqual(result, 1) pushes_after = Push.objects.all().count() self.assertEqual(pushes_initial, pushes_after - 1) # a second time should be harmless result = handlePushes(repo.pk, [pushjs0]) self.assertEqual(result, 1) pushes_after_after = Push.objects.all().count() self.assertEqual(pushes_after, pushes_after_after)
def test_handlePushes(self): repo = Repository.objects.create( name='mozilla-central', url='file:///' + self.repo ) hgrepo = hglib.init(self.repo).open() (open(hgrepo.pathto('file.dtd'), 'w') .write(''' <!ENTITY key1 "Hello"> <!ENTITY key2 "Cruel"> ''')) hgrepo.commit(user="******", message="initial commit", addremove=True) rev0 = hgrepo[0].node() timestamp = int(time.time()) push_id = 100 username = '******' pushjs0 = PushJS(push_id, { 'date': timestamp, 'changesets': [rev0], 'user': username, }) result = handlePushes(repo.pk, [pushjs0]) self.assertEqual(result, 1) # expect all of these to have been created push, = Push.objects.all() branch, = Branch.objects.all() changeset, = push.changesets.all() self.assertEqual(push.repository, repo) self.assertEqual(push.push_id, push_id) self.assertEqual(push.user, username) self.assertEqual( push.push_date, datetime.datetime.utcfromtimestamp(timestamp) ) self.assertEqual(changeset.description, 'initial commit') self.assertEqual(changeset.user, 'Jane Doe <*****@*****.**>') self.assertEqual(hglib.util.b(changeset.revision), rev0) self.assertEqual(changeset.branch, branch) self.assertEqual(branch.name, 'default')
def test_handlePushes(self): repo = Repository.objects.create( name='mozilla-central', url='file:///' + self.repo ) with hglib.init(self.repo).open() as hgrepo: with open(hgrepo.pathto('file.dtd'), 'w') as fh: fh.write(''' <!ENTITY key1 "Hello"> <!ENTITY key2 "Cruel"> ''') hgrepo.commit(user="******", message="initial commit", addremove=True) rev0 = hgrepo[0].node().decode('ascii') timestamp = int(time.time()) push_id = 100 username = '******' pushjs0 = PushJS(push_id, { 'date': timestamp, 'changesets': [rev0], 'user': username, }) result = handlePushes(repo.pk, [pushjs0]) self.assertEqual(result, 1) # expect all of these to have been created push, = Push.objects.all() branch, = Branch.objects.all() changeset, = push.changesets.all() self.assertEqual(push.repository, repo) self.assertEqual(push.push_id, push_id) self.assertEqual(push.user, username) self.assertEqual( push.push_date, datetime.datetime.utcfromtimestamp(timestamp) ) self.assertEqual(changeset.description, 'initial commit') self.assertEqual(changeset.user, 'Jane Doe <*****@*****.**>') self.assertEqual(changeset.revision, rev0) self.assertEqual(changeset.branch, branch) self.assertEqual(branch.name, 'default')
def test_handlePushes_messedup_revisions(self): repo = Repository.objects.create(name="mozilla-central", url="file:///" + self.repo) self.assertEqual(handlePushes(repo.pk, []), None) ui = mock_ui() hgcommands.init(ui, self.repo) hgrepo = repository(ui, self.repo) ( open(hgrepo.pathto("file.dtd"), "w").write( """ <!ENTITY key1 "Hello"> <!ENTITY key2 "Cruel"> """ ) ) hgcommands.addremove(ui, hgrepo) hgcommands.commit(ui, hgrepo, user="******", message="initial commit") rev0 = hgrepo[0].hex() timestamp = int(time.time()) pushjs0 = PushJS(100, {"date": timestamp, "changesets": [rev0[::-1]], "user": "******"}) self.assertRaises(RepoLookupError, handlePushes, repo.pk, [pushjs0])
def processPushes(self, pushes, repo): '''process the pushes for the given repository. This code also adds all pushes that are older than the newest push on this repo, in order. If the amount of pushes exceeds the limit, pushback the current repo to get more changes. If we're emptying another's repo push cache, re-poll???. ''' if len(pushes) == self.limit: self.repos.pushback(repo) if self.debug: log.msg("submitting %s to %s" % (', '.join(map(str, pushes)), repo.name)) def get_date(p, op): return op(map(lambda d: d['date'], p.itervalues())) maxdate = pushes[-1].date tips = sorted( ((id, p[0].date) for id, p in self.cache.iteritems() if p), key=lambda t: t[1]) while pushes: if tips and pushes[0].date > tips[0][1]: # other repos come first, get them done other = self.cache[tips[0][0]] if len(tips) > 1: stopdate = min(pushes[0].date, self.cache[tips[1][0]][0].date) else: stopdate = pushes[0].date i = 0 while i < len(other) and \ other[i].date <= stopdate: i += 1 submits = other[:i] if self.debug: log.msg("pushing %s to %d" % (", ".join(map(str, submits)), tips[0][0])) handlePushes(tips[0][0], submits, self.do_update) del other[:i] if not other: # other repo is empty, let's see if we need to load more if tips[0][0] in self.moredata: self.moredata.pop(tips[0][0]) other_repo = Repository.objects.get(id=tips[0][0]) self.repos.pushback(other_repo) return tips = sorted(((id, p[0].date) for id, p in self.cache.iteritems() if p), key=lambda t: t[1]) else: i = 0 if tips: stopdate = self.cache[tips[0][0]][0].date while i < len(pushes) and \ pushes[i].date <= stopdate: i += 1 else: i = len(pushes) submits = pushes[:i] if self.debug: log.msg("pushing %s to %d" % (", ".join(map(str, submits)), repo.id)) handlePushes(repo.id, submits, self.do_update) del pushes[:i]
def processPushes(self, pushes, repo): '''process the pushes for the given repository. This code also adds all pushes that are older than the newest push on this repo, in order. If the amount of pushes exceeds the limit, pushback the current repo to get more changes. If we're emptying another's repo push cache, re-poll???. ''' if len(pushes) == self.limit: self.repos.pushback(repo) if self.debug: log.msg("submitting %s to %s" % (', '.join(map(str, pushes)), repo.name)) def get_date(p, op): return op(map(lambda d: d['date'], p.itervalues())) maxdate = pushes[-1].date tips = sorted(((id, p[0].date) for id, p in self.cache.iteritems() if p), key = lambda t: t[1]) while pushes: if tips and pushes[0].date > tips[0][1]: # other repos come first, get them done other = self.cache[tips[0][0]] if len(tips) > 1: stopdate = min(pushes[0].date, self.cache[tips[1][0]][0].date) else: stopdate = pushes[0].date i = 0 while i < len(other) and \ other[i].date <= stopdate: i += 1 submits = other[:i] if self.debug: log.msg("pushing %s to %d" % (", ".join(map(str, submits)), tips[0][0])) handlePushes(tips[0][0], submits, self.do_update) del other[:i] if not other: # other repo is empty, let's see if we need to load more if tips[0][0] in self.moredata: self.moredata.pop(tips[0][0]) other_repo = Repository.objects.get(id=tips[0][0]) self.repos.pushback(other_repo) return tips = sorted(((id, p[0].date) for id, p in self.cache.iteritems() if p), key = lambda t: t[1]) else: i = 0 if tips: stopdate = self.cache[tips[0][0]][0].date while i < len(pushes) and \ pushes[i].date <= stopdate: i += 1 else: i = len(pushes) submits = pushes[:i] if self.debug: log.msg("pushing %s to %d" % (", ".join(map(str, submits)), repo.id)) handlePushes(repo.id, submits, self.do_update) del pushes[:i]