def run(self, revision_filter=None, project_filter=None, job_group_filter=None): """ Returns True if new completed jobs were loaded, False otherwise. """ builds_4hr = common.fetch_json(settings.BUILDAPI_BUILDS4H_URL) job_collections, job_ids_seen = self.transform(builds_4hr, revision_filter=revision_filter, project_filter=project_filter, job_group_filter=job_group_filter) if job_collections: th_publisher.post_treeherder_collections(job_collections, chunk_size=settings.BUILDAPI_BUILDS4H_CHUNK_SIZE) cache.set(CACHE_KEYS['complete'], job_ids_seen) return bool(job_collections)
def run(self, revision_filter=None, project_filter=None, job_group_filter=None): """ Returns True if new completed jobs were loaded, False otherwise. """ builds_4hr = common.fetch_json(settings.BUILDAPI_BUILDS4H_URL) job_collections, job_ids_seen = self.transform( builds_4hr, revision_filter=revision_filter, project_filter=project_filter, job_group_filter=job_group_filter) if job_collections: th_publisher.post_treeherder_collections( job_collections, chunk_size=settings.BUILDAPI_BUILDS4H_CHUNK_SIZE) cache.set(CACHE_KEYS['complete'], job_ids_seen) return bool(job_collections)
def load(self, th_collections, chunk_size=1): th_publisher.post_treeherder_collections(th_collections, chunk_size)
def run(self, source_url, repository, changeset=None): # get the last object seen from cache. this will # reduce the number of pushes processed every time last_push_id = cache.get("{0}:last_push_id".format(repository)) if not changeset and last_push_id: startid_url = "{}&startID={}".format(source_url, last_push_id) logger.info("Extracted last push for '%s', '%s', from cache, " "attempting to get changes only from that point at: %s" % (repository, last_push_id, startid_url)) # Use the cached ``last_push_id`` value (saved from the last time # this API was called) for this repo. Use that value as the # ``startID`` to get all new pushes from that point forward. extracted_content = self.extract(startid_url) if extracted_content['lastpushid'] < last_push_id: # Push IDs from Mercurial are incremental. If we cached a value # from one call to this API, and a subsequent call told us that # the ``lastpushid`` is LOWER than the one we have cached, then # the Mercurial IDs were reset. # In this circumstance, we can't rely on the cached id, so must # throw it out and get the latest 10 pushes. logger.warning(("Got a ``lastpushid`` value of {} lower than " "the cached value of {} due to Mercurial repo reset. " "Getting latest changes for '{}' instead").format( extracted_content['lastpushid'], last_push_id, repository ) ) cache.delete("{0}:last_push_id".format(repository)) extracted_content = self.extract(source_url) else: if changeset: logger.info("Getting all pushes for '%s' corresponding to " "changeset '%s'" % (repository, changeset)) extracted_content = self.extract(source_url + "&changeset=" + changeset) else: logger.warning("Unable to get last push from cache for '%s', " "getting all pushes" % repository) extracted_content = self.extract(source_url) # ``pushes`` could be empty if there are no new ones since we last # fetched pushes = extracted_content['pushes'] if not pushes: return None last_push_id = max(map(lambda x: int(x), pushes.keys())) last_push = pushes[str(last_push_id)] top_revision = last_push["changesets"][-1]["node"] transformed = self.transform(pushes, repository) th_publisher.post_treeherder_collections(transformed) if not changeset: # only cache the last push if we're not fetching a specific # changeset cache.set("{0}:last_push_id".format(repository), last_push_id) return top_revision
def run(self, source_url, repository, changeset=None): # get the last object seen from cache. this will # reduce the number of pushes processed every time last_push_id = cache.get("{0}:last_push_id".format(repository)) if not changeset and last_push_id: startid_url = "{}&startID={}".format(source_url, last_push_id) logger.info( "Extracted last push for '%s', '%s', from cache, " "attempting to get changes only from that point at: %s" % (repository, last_push_id, startid_url)) # Use the cached ``last_push_id`` value (saved from the last time # this API was called) for this repo. Use that value as the # ``startID`` to get all new pushes from that point forward. extracted_content = self.extract(startid_url) if extracted_content['lastpushid'] < last_push_id: # Push IDs from Mercurial are incremental. If we cached a value # from one call to this API, and a subsequent call told us that # the ``lastpushid`` is LOWER than the one we have cached, then # the Mercurial IDs were reset. # In this circumstance, we can't rely on the cached id, so must # throw it out and get the latest 10 pushes. logger.warning( ("Got a ``lastpushid`` value of {} lower than " "the cached value of {} due to Mercurial repo reset. " "Getting latest changes for '{}' instead").format( extracted_content['lastpushid'], last_push_id, repository)) cache.delete("{0}:last_push_id".format(repository)) extracted_content = self.extract(source_url) else: if changeset: logger.info("Getting all pushes for '%s' corresponding to " "changeset '%s'" % (repository, changeset)) extracted_content = self.extract(source_url + "&changeset=" + changeset) else: logger.warning("Unable to get last push from cache for '%s', " "getting all pushes" % repository) extracted_content = self.extract(source_url) # ``pushes`` could be empty if there are no new ones since we last # fetched pushes = extracted_content['pushes'] if not pushes: return None last_push_id = max(map(lambda x: int(x), pushes.keys())) last_push = pushes[str(last_push_id)] top_revision = last_push["changesets"][-1]["node"] transformed = self.transform(pushes, repository) th_publisher.post_treeherder_collections(transformed) if not changeset: # only cache the last push if we're not fetching a specific # changeset cache.set("{0}:last_push_id".format(repository), last_push_id) return top_revision
def load(self, th_collections): th_publisher.post_treeherder_collections(th_collections)