def run(self, revision_filter=None, project_filter=None, job_group_filter=None):
""" Returns True if new completed jobs were loaded, False otherwise. """
builds_4hr = common.fetch_json(settings.BUILDAPI_BUILDS4H_URL)
job_collections, job_ids_seen = self.transform(builds_4hr,
revision_filter=revision_filter,
project_filter=project_filter,
job_group_filter=job_group_filter)
if job_collections:
th_publisher.post_treeherder_collections(job_collections,
chunk_size=settings.BUILDAPI_BUILDS4H_CHUNK_SIZE)
cache.set(CACHE_KEYS['complete'], job_ids_seen)
return bool(job_collections)
def run(self,
revision_filter=None,
project_filter=None,
job_group_filter=None):
""" Returns True if new completed jobs were loaded, False otherwise. """
builds_4hr = common.fetch_json(settings.BUILDAPI_BUILDS4H_URL)
job_collections, job_ids_seen = self.transform(
builds_4hr,
revision_filter=revision_filter,
project_filter=project_filter,
job_group_filter=job_group_filter)
if job_collections:
th_publisher.post_treeherder_collections(
job_collections,
chunk_size=settings.BUILDAPI_BUILDS4H_CHUNK_SIZE)
cache.set(CACHE_KEYS['complete'], job_ids_seen)
return bool(job_collections)
def load(self, th_collections, chunk_size=1):
th_publisher.post_treeherder_collections(th_collections, chunk_size)
def run(self, source_url, repository, changeset=None):
# get the last object seen from cache. this will
# reduce the number of pushes processed every time
last_push_id = cache.get("{0}:last_push_id".format(repository))
if not changeset and last_push_id:
startid_url = "{}&startID={}".format(source_url, last_push_id)
logger.info("Extracted last push for '%s', '%s', from cache, "
"attempting to get changes only from that point at: %s" %
(repository, last_push_id, startid_url))
# Use the cached ``last_push_id`` value (saved from the last time
# this API was called) for this repo. Use that value as the
# ``startID`` to get all new pushes from that point forward.
extracted_content = self.extract(startid_url)
if extracted_content['lastpushid'] < last_push_id:
# Push IDs from Mercurial are incremental. If we cached a value
# from one call to this API, and a subsequent call told us that
# the ``lastpushid`` is LOWER than the one we have cached, then
# the Mercurial IDs were reset.
# In this circumstance, we can't rely on the cached id, so must
# throw it out and get the latest 10 pushes.
logger.warning(("Got a ``lastpushid`` value of {} lower than "
"the cached value of {} due to Mercurial repo reset. "
"Getting latest changes for '{}' instead").format(
extracted_content['lastpushid'],
last_push_id,
repository
)
)
cache.delete("{0}:last_push_id".format(repository))
extracted_content = self.extract(source_url)
else:
if changeset:
logger.info("Getting all pushes for '%s' corresponding to "
"changeset '%s'" % (repository, changeset))
extracted_content = self.extract(source_url + "&changeset=" +
changeset)
else:
logger.warning("Unable to get last push from cache for '%s', "
"getting all pushes" % repository)
extracted_content = self.extract(source_url)
# ``pushes`` could be empty if there are no new ones since we last
# fetched
pushes = extracted_content['pushes']
if not pushes:
return None
last_push_id = max(map(lambda x: int(x), pushes.keys()))
last_push = pushes[str(last_push_id)]
top_revision = last_push["changesets"][-1]["node"]
transformed = self.transform(pushes, repository)
th_publisher.post_treeherder_collections(transformed)
if not changeset:
# only cache the last push if we're not fetching a specific
# changeset
cache.set("{0}:last_push_id".format(repository), last_push_id)
return top_revision
def load(self, th_collections, chunk_size=1):
th_publisher.post_treeherder_collections(th_collections, chunk_size)
def run(self, source_url, repository, changeset=None):
# get the last object seen from cache. this will
# reduce the number of pushes processed every time
last_push_id = cache.get("{0}:last_push_id".format(repository))
if not changeset and last_push_id:
startid_url = "{}&startID={}".format(source_url, last_push_id)
logger.info(
"Extracted last push for '%s', '%s', from cache, "
"attempting to get changes only from that point at: %s" %
(repository, last_push_id, startid_url))
# Use the cached ``last_push_id`` value (saved from the last time
# this API was called) for this repo. Use that value as the
# ``startID`` to get all new pushes from that point forward.
extracted_content = self.extract(startid_url)
if extracted_content['lastpushid'] < last_push_id:
# Push IDs from Mercurial are incremental. If we cached a value
# from one call to this API, and a subsequent call told us that
# the ``lastpushid`` is LOWER than the one we have cached, then
# the Mercurial IDs were reset.
# In this circumstance, we can't rely on the cached id, so must
# throw it out and get the latest 10 pushes.
logger.warning(
("Got a ``lastpushid`` value of {} lower than "
"the cached value of {} due to Mercurial repo reset. "
"Getting latest changes for '{}' instead").format(
extracted_content['lastpushid'], last_push_id,
repository))
cache.delete("{0}:last_push_id".format(repository))
extracted_content = self.extract(source_url)
else:
if changeset:
logger.info("Getting all pushes for '%s' corresponding to "
"changeset '%s'" % (repository, changeset))
extracted_content = self.extract(source_url + "&changeset=" +
changeset)
else:
logger.warning("Unable to get last push from cache for '%s', "
"getting all pushes" % repository)
extracted_content = self.extract(source_url)
# ``pushes`` could be empty if there are no new ones since we last
# fetched
pushes = extracted_content['pushes']
if not pushes:
return None
last_push_id = max(map(lambda x: int(x), pushes.keys()))
last_push = pushes[str(last_push_id)]
top_revision = last_push["changesets"][-1]["node"]
transformed = self.transform(pushes, repository)
th_publisher.post_treeherder_collections(transformed)
if not changeset:
# only cache the last push if we're not fetching a specific
# changeset
cache.set("{0}:last_push_id".format(repository), last_push_id)
return top_revision
def load(self, th_collections):
th_publisher.post_treeherder_collections(th_collections)
