def test_reschedule_campaign_with_valid_data(
self, token_first, campaign_in_db, talent_pool, candidate_first,
smartlist_first, schedule_a_campaign, candidate_device_first):
"""
Reschedule a campaign with valid data and it should return 200 response.
"""
sleep(15)
data = generate_campaign_schedule_data(frequency_id=Frequency.DAILY)
response = send_request(
'put', PushCampaignApiUrl.SCHEDULE % campaign_in_db['id'],
token_first, data)
assert response.status_code == codes.OK
response = response.json()
assert 'task_id' in response
assert 'message' in response
task_id = response['task_id']
assert task_id
# campaign = get_campaign(campaign_in_db['id'], token_first)['campaign']
# match_schedule_data(data, campaign)
retry(get_blasts,
attempts=30,
sleepscale=1,
sleeptime=3,
retry_exceptions=(AssertionError, ),
args=(campaign_in_db['id'], token_first),
kwargs={'count': 2})
def test_schedule_a_campaign_with_user_from_same_domain(
self, smartlist_first, campaign_in_db, talent_pool, token_first,
token_same_domain, candidate_device_first):
"""
In this test, we will schedule a campaign using different user's auth token, but user is from same domain ,
as the actual owner of the campaign. So we are expecting that , response will be OK and campaign will be
scheduled.
"""
campaign_id = campaign_in_db['id']
data = generate_campaign_schedule_data(frequency_id=Frequency.DAILY)
response = schedule_campaign(campaign_id,
data,
token_same_domain,
expected_status=(codes.OK, ))
assert 'task_id' in response
assert 'message' in response
task_id = response['task_id']
assert task_id
# campaign = get_campaign(campaign_in_db['id'], token_first)['campaign']
# match_schedule_data(data, campaign)
# There should be a campaign schedule activity
assert_activity(Activity.MessageIds.CAMPAIGN_SCHEDULE,
campaign_in_db['id'], 'push_campaign',
token_same_domain)
retry(get_blasts,
sleeptime=3,
attempts=30,
sleepscale=1,
retry_exceptions=(AssertionError, ),
args=(campaign_id, token_first),
kwargs={'count': 1})
def test_reschedule_a_campaign_with_user_from_same_domain(
self, token_first, token_same_domain, campaign_in_db,
schedule_a_campaign):
"""
In this test, we will reschedule a campaign using different user's auth token, but user is from same domain ,
as the actual owner of the campaign. So we are expecting that , response will be OK and campaign will be
rescheduled.
"""
# schedule_a_campaign fixture schedules a job. So, wait for next 10 seconds for the job to run and increase
# blast count
sleep(10)
data = generate_campaign_schedule_data(frequency_id=Frequency.DAILY)
response = reschedule_campaign(campaign_in_db['id'],
data,
token_same_domain,
expected_status=(codes.OK, ))
assert 'task_id' in response
assert 'message' in response
task_id = response['task_id']
assert task_id
# campaign = get_campaign(campaign_in_db['id'], token_first)['campaign']
# match_schedule_data(data, campaign)
retry(get_blasts,
attempts=30,
sleepscale=1,
sleeptime=3,
retry_exceptions=(AssertionError, ),
args=(campaign_in_db['id'], token_first),
kwargs={'count': 2})
def test_schedule_a_campaign_with_valid_data(self, smartlist_first,
campaign_in_db, talent_pool,
token_first,
candidate_device_first):
"""
In this test, we will schedule a campaign with all valid data and it should return an OK response and campaign
should be scheduled.
"""
data = generate_campaign_schedule_data()
response = schedule_campaign(campaign_in_db['id'],
data,
token_first,
expected_status=(codes.OK, ))
assert 'task_id' in response
assert 'message' in response
task_id = response['task_id']
assert task_id
# There should be a campaign schedule activity
assert_activity(Activity.MessageIds.CAMPAIGN_SCHEDULE,
campaign_in_db['id'], 'push_campaign', token_first)
# campaign = get_campaign(campaign_in_db['id'], token_first)['campaign']
# match_schedule_data(data, campaign)
retry(get_blasts,
sleeptime=3,
attempts=20,
sleepscale=1,
retry_exceptions=(AssertionError, ),
args=(campaign_in_db['id'], token_first),
kwargs={'count': 1})
def smartlist_with_two_candidates_with_no_device_associated(
request, token_first, user_first, candidate_first,
candidate_same_domain, talent_pipeline):
"""
This fixture creates a smartlist that contains two candidates from domain_first.
Both candidates do not have any push device associated with them.
:param request: request object
:param candidate_first: candidate object
:param candidate_same_domain: candidate object
:param token_first: access token for user_first
:return: smartlist object (dict)
"""
candidate_ids = [candidate_first['id'], candidate_same_domain['id']]
time.sleep(10)
smartlist = create_smartlist(candidate_ids, talent_pipeline['id'],
token_first)['smartlist']
smartlist_id = smartlist['id']
retry(get_smartlist_candidates,
sleeptime=3,
attempts=50,
sleepscale=1,
retry_exceptions=(AssertionError, ),
args=(smartlist_id, token_first),
kwargs={'count': 2})
return smartlist
def fetch_history(self, context, data):
try:
retry(
self._fetch_history,
sleeptime=5,
retry_exceptions=(ccxt_errors.RequestTimeout),
args=(context, data),
cleanup=lambda: self.log.warning("CCXT request timed out, retrying..."),
)
return True
except ccxt_errors.ExchangeNotAvailable:
self.log.error(
f"{self.exchange} API is currently unavailable, skipping trading step"
)
return False
except ccxt_errors.DDoSProtection:
self.log.error("Hit Rate limit, skipping trade step")
return False
except SystemExit:
self.log.warning("Not retrying history due to algo exit")
return False
except Exception:
self.log.error("Could not fetch latest history", exec_info=True)
return False
def add_chain(self):
from redo import retry
import requests
import pem
def get_chain():
r = requests.get(self.chain_url)
r.raise_for_status()
return r.text
chain = retry(get_chain)
req = {"chain": []}
chain = pem.parse(chain)
for i in range(len(chain)):
cert = crypto.load_certificate(crypto.FILETYPE_PEM, str(chain[i]))
der = crypto.dump_certificate(crypto.FILETYPE_ASN1, cert)
req["chain"].append(base64.b64encode(der))
def post_chain():
r = requests.post(self.log_url + '/ct/v1/add-chain', json=req)
r.raise_for_status()
return r.json()
resp = retry(post_chain)
sct = SignedCertificateTimestamp(resp)
self.write_to_file(self.sct_filename, sct.to_rfc6962())
def download_file_from_url(url, local_dest, extract=False):
"""Receive a file in a URL and download it, i.e. for the hostutils tooltool manifest
the url received would be formatted like this:
config/tooltool-manifests/linux64/hostutils.manifest"""
if os.path.exists(local_dest):
LOG.info("file already exists at: %s" % local_dest)
if not extract:
return True
else:
LOG.info("downloading: %s to %s" % (url, local_dest))
try:
retry(urlretrieve, args=(url, local_dest), attempts=3, sleeptime=5)
except Exception:
LOG.error("Failed to download file: %s" % local_dest,
exc_info=True)
if os.path.exists(local_dest):
# delete partial downloaded file
os.remove(local_dest)
return False
if not extract:
return os.path.exists(local_dest)
typ = archive_type(local_dest)
if typ is None:
LOG.info("Not able to determine archive type for: %s" % local_dest)
return False
extract_archive(local_dest, os.path.dirname(local_dest), typ)
return True
def consume(self):
""" Start the consumer and read messages from Kafka sending self.batch_size messages
to splunk in a perpetual loop. Attempt incremental backoff self.retry_attempts
times if any batch of messages gets a non 200 status code otherwise give up and exit.
"""
self.consumer = self.getConsumer(self.client.topics[self.topic])
# create splunk hec instance
splunk_hec = hec(self.splunk_server, self.splunk_hec_port,
self.splunk_hec_channel, self.splunk_hec_token,
self.splunk_sourcetype, self.splunk_source,
self.use_https, self.verify_ssl, self.use_compression,
self.compresslevel)
while (True):
m = self.consumer.consume()
# Append messages to list until we've hit self.batch_size
if (len(self.messages) <= self.batch_size):
self.messages.append(m.value)
# Send messages to Splunk HEC
if (len(self.messages) == self.batch_size):
retry(self.sendToSplunk,
attempts=self.retry_attempts,
sleeptime=self.sleeptime,
max_sleeptime=self.max_sleeptime,
sleepscale=self.sleepscale,
jitter=self.jitter,
retry_exceptions=(Exception, ),
args=(splunk_hec, ))
def cancel_order(self, order_param, symbol=None, params={}):
"""Cancel an open order.
Parameters
----------
order_param : str or Order
The order_id or order object to cancel.
symbol: str
The tradingPair symbol
params: dict, optional
Extra parameters to pass to the exchange
"""
log.info("canceling an order")
order_id = order_param
if isinstance(order_param, zp.Order) or \
isinstance(order_param, Order):
order_id = order_param.id
if not self.simulate_orders:
exchange_name = self.blotter.orders[order_id].asset.exchange
exchange = self.exchanges[exchange_name]
retry(action=exchange.cancel_order,
attempts=self.attempts['cancel_order_attempts'],
sleeptime=self.attempts['retry_sleeptime'],
retry_exceptions=(ExchangeRequestError, ),
cleanup=lambda: log.warn(
'attempting to cancel the order again'),
args=(order_id, symbol, params))
self.blotter.cancel(order_id)
else:
self.blotter.cancel(order_id)
def migration(args):
limit = int(args.limit)
offset = 0
url = URL % (limit, offset)
response = retry(requests.get, args=(url, )).json()
datasets = response['result']
while len(datasets) > 0:
for data in datasets:
Testjob = Testjobs(data['slave'], data['result'],
data['build_system_type'], data['duration'],
data['platform'], data['buildtype'],
data['testtype'], data['bugid'], data['branch'],
data['revision'], data['date'],
data['failure_classification'],
data['failures'])
try:
session.add(Testjob)
session.commit()
except Exception as error:
logging.warning(error)
session.rollback()
finally:
session.close()
# The process will move forward by set offset
offset += limit
url = URL % (limit, offset)
response = retry(requests.get, args=(url, )).json()
datasets = response['result']
def get_iam_last_access_details(iam, arn):
'''Retrieves IAM last accessed details for the given user/group/role ARN'''
job = redo.retry(iam.generate_service_last_accessed_details,
attempts=5,
sleeptime=5,
kwargs={'Arn': arn})
job_id = job['JobId']
marker = None
service_results = []
tries = 0
while True:
result = redo.retry(iam.get_service_last_accessed_details,
attempts=5,
sleeptime=5,
kwargs={'JobId': job_id})
if result['JobStatus'] == 'IN_PROGRESS':
print("Awaiting job")
elif result['JobStatus'] == 'FAILED':
raise Exception(f"Could not get access information for {arn}")
else:
service_results.extend(paginate_access_details(job_id, result))
break
time.sleep(10)
return service_results
def add_chain(self):
from redo import retry
import requests
import pem
def get_chain():
r = requests.get(self.chain_url)
r.raise_for_status()
return r.text
chain = retry(get_chain)
req = {"chain": []}
chain = pem.parse(chain)
for i in range(len(chain)):
cert = crypto.load_certificate(crypto.FILETYPE_PEM, str(chain[i]))
der = crypto.dump_certificate(crypto.FILETYPE_ASN1, cert)
req["chain"].append(base64.b64encode(der))
def post_chain():
r = requests.post(self.log_url + '/ct/v1/add-chain', json=req)
r.raise_for_status()
return r.json()
resp = retry(post_chain)
sct = SignedCertificateTimestamp(resp)
self.write_to_file(self.sct_filename, sct.to_rfc6962())
def refresh_antivirus(self):
self.info("Refreshing clamav db...")
try:
redo.retry(lambda: sh.freshclam("--stdout", "--verbose", _timeout=300, _err_to_out=True))
self.info("Done.")
except sh.ErrorReturnCode:
self.warning("Freshclam failed, skipping DB update")
def cancel_order(self, order_param, exchange_name, symbol=None, params={}):
"""Cancel an open order.
Parameters
----------
order_param : str or Order
The order_id or order object to cancel.
exchange_name: str
The name of exchange to cancel the order in
symbol: str
The tradingPair symbol
params: dict, optional
Extra parameters to pass to the exchange
"""
exchange = self.exchanges[exchange_name]
order_id = order_param
if isinstance(order_param, zp.Order):
order_id = order_param.id
retry(action=exchange.cancel_order,
attempts=self.attempts['cancel_order_attempts'],
sleeptime=self.attempts['retry_sleeptime'],
retry_exceptions=(ExchangeRequestError, ),
cleanup=lambda: log.warn('cancelling order again.'),
args=(order_id, symbol, params))
def test_campaign_send_to_smartlist_with_two_candidates_with_and_without_push_device(self, token_first,
campaign_with_two_candidates_with_and_without_push_device):
"""
- This tests the endpoint /v1/push-campaigns/:id/send
In this test I want to test the scenario that if a push campaign is being sent to multiple candidates and
there is one or more but not all candidates that do not have a push device associated with them,
then it should not raise an InvalidUsage error but sends should be equal to number of candidates
that have devices associated with them.
"""
campaign_id = campaign_with_two_candidates_with_and_without_push_device['id']
send_campaign(campaign_id, token_first, expected_status=(codes.OK,))
# Assert campaign send activity
assert_activity(Activity.MessageIds.CAMPAIGN_SEND, campaign_id, 'push_campaign', token_first)
response = retry(get_blasts, sleeptime=SLEEP_INTERVAL, attempts=RETRY_ATTEMPTS * 2, sleepscale=1,
retry_exceptions=(AssertionError,), args=(campaign_id, token_first), kwargs={'count': 1})
blasts = response['blasts']
blast_id = blasts[0]['id']
# There should be only one send because second candidate in smartlist does not have any push device associated
# with him.
response = retry(get_blast_sends, sleeptime=SLEEP_INTERVAL, attempts=RETRY_ATTEMPTS * 2, sleepscale=1,
retry_exceptions=(AssertionError,), args=(blast_id, campaign_id, token_first),
kwargs={'count': 1})
assert len(response['sends']) == 1
def check_url(self, session, url):
from redo import retry
try:
from urllib.parse import urlparse
except ImportError:
# Python 2
from urlparse import urlparse
mozilla_locations = [
'download-installer.cdn.mozilla.net',
'download.cdn.mozilla.net',
'download.mozilla.org',
'archive.mozilla.org',
]
def do_check_url():
self.log("Checking {}".format(url))
r = session.head(url, verify=True, timeout=10, allow_redirects=True)
try:
r.raise_for_status()
except Exception:
self.error("FAIL: {}, status: {}".format(url, r.status_code))
raise
final_url = urlparse(r.url)
if final_url.scheme != 'https':
self.error('FAIL: URL scheme is not https: {}'.format(r.url))
if final_url.netloc not in mozilla_locations:
self.error('FAIL: host not in allowed locations: {}'.format(r.url))
retry(do_check_url, sleeptime=3, max_sleeptime=10, attempts=3)
def submit_locale(task, config, auth0_secrets, backend_version):
"""Submit a release blob to balrog."""
upstream_artifacts = get_upstream_artifacts(task)
# Read the manifest from disk
manifest = get_manifest(config, upstream_artifacts)
suffixes = task["payload"].get("suffixes", [""])
for e in manifest:
for suffix in suffixes:
# Get release metadata from manifest
submitter, release = create_locale_submitter(
e,
suffix,
auth0_secrets,
config,
backend_version=backend_version)
# Connect to balrog and submit the metadata
# Going back to the original number of attempts so that we avoid sleeping too much in between
# retries to get Out-of-memory in the GCP workers. Until we figure out what's bumping the spike
# in memory usage from 130 -> ~400 Mb, let's keep this as it was, historically
retry(lambda: submitter.run(**release),
jitter=5,
sleeptime=10,
max_sleeptime=30,
attempts=10)
def updateWorkloadStateTable(self,action):
self.currentTime = str(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
if (action == Orchestrator.ACTION_START) and (self.scalingProfile is not None):
UpdateExpressionAttr='SET Profile= :profile, LastActionTime= :currentTime, LastActionType= :actionType'
ExpressionAttributeValuesAttr={
':profile': self.scalingProfile,
':currentTime': self.currentTime,
':actionType': action
}
else:
UpdateExpressionAttr='SET LastActionTime= :currentTime, LastActionType= :actionType'
ExpressionAttributeValuesAttr={
':currentTime': self.currentTime,
':actionType': action,
}
try:
retry(self.WorkloadStateTable.update_item, attempts=5, sleeptime=0,jitter=0, kwargs= {
"Key":{
'Workload': self.partitionTargetValue,
},
"UpdateExpression":UpdateExpressionAttr,
"ExpressionAttributeValues":ExpressionAttributeValuesAttr
})
except Exception as e:
msg = 'Orchestrator::updateWorkloadStateTable() Exception encountered during DDB update %s -->' % e
logger.error(msg + str(e))
def refresh_antivirus(self):
self.info("Refreshing clamav db...")
try:
redo.retry(lambda: sh.freshclam(
"--stdout", "--verbose", _timeout=300, _err_to_out=True))
self.info("Done.")
except sh.ErrorReturnCode:
self.warning("Freshclam failed, skipping DB update")
def testRetrySelectiveExceptionSucceed(self):
retry(
_raiseCustomException,
attempts=2,
sleeptime=0,
jitter=0,
retry_exceptions=(NewError,),
)
def set_readonly(task, config, auth0_secrets):
state_updater = create_state_updater(api_root=config["api_root"],
auth0_secrets=auth0_secrets)
args = [
task["payload"]["product"].capitalize(), task["payload"]["version"],
task["payload"]["build_number"]
]
retry(lambda: state_updater.run(*args))
def testRetrySelectiveExceptionSucceed(self):
retry(
_raiseCustomException,
attempts=2,
sleeptime=0,
jitter=0,
retry_exceptions=(NewError, ),
)
def slavealloc_disable(self, machine, loanid):
try:
url = furl(current_app.config.get("SLAVEAPI_URL", None))
url.path.add(machine).add("actions").add("disable")
loan_bug = Loans.query.get(loanid).bug_id
postdata = dict(reason="Being loaned on slaveloan bug %s" % loan_bug)
retry(requests.post, args=(str(url),), kwargs=dict(data=postdata)).json()
return machine
except Exception as exc: # pylint: disable=W0703
logger.exception(exc)
self.retry(exc=exc)
def test_campaign_send_to_candidate_with_no_device(self, token_first, campaign_in_db):
"""
In this test, we will send a campaign to a valid candidate (in same domain), but candidate
has no device associated with him. So no campaign will be sent which will result in
one blasts and no sends.
"""
campaign_id = campaign_in_db['id']
send_campaign(campaign_id, token_first, expected_status=(codes.OK,))
retry(get_blasts, sleeptime=SLEEP_INTERVAL, attempts=RETRY_ATTEMPTS * 2, sleepscale=1,
retry_exceptions=(AssertionError,), args=(campaign_id, token_first), kwargs={'count': 1})
get_and_assert_zero(PushCampaignApiUrl.SENDS % campaign_id, 'sends', token_first)
def slavealloc_disable(self, machine, loanid):
try:
url = furl(current_app.config.get("SLAVEAPI_URL", None))
url.path.add(machine).add("actions").add("disable")
loan_bug = Loans.query.get(loanid).bug_id
postdata = dict(reason="Being loaned on slaveloan bug %s" % loan_bug)
retry(requests.post, args=(str(url), ),
kwargs=dict(data=postdata)).json()
return machine
except Exception as exc: # pylint: disable=W0703
logger.exception(exc)
self.retry(exc=exc)
def _scan_files(self):
"""Scan the files we've collected. We do the download and scan concurrently to make
it easier to have a coherent log afterwards. Uses the venv python."""
self.info("Refreshing clamav db...")
redo.retry(lambda:
sh.freshclam("--stdout", "--verbose", _timeout=300, _err_to_out=True))
self.info("Done.")
external_tools_path = os.path.join(
os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))), 'external_tools')
self.run_command([self.query_python_path(), os.path.join(external_tools_path,'extract_and_run_command.py'),
'-j{}'.format(self.config['scan_parallelization']),
'clamscan', '--no-summary', '--', self.dest_dir])
def submit_toplevel(task, config, balrog_auth):
"""Push a top-level release blob to balrog."""
auth = balrog_auth
partials = {}
if task['payload'].get('partial_versions'):
for v in task['payload']['partial_versions'].split(','):
v = v.strip() # we have whitespace after the comma
version, build_number = v.split("build")
partials[version] = {"buildNumber": build_number}
suffixes = task['payload'].get('update_line', {}).keys() or ['']
for suffix in suffixes:
creator = create_creator(
api_root=config['api_root'],
auth=auth,
dummy=config['dummy'],
suffix=task['payload'].get('blob_suffix', '') + suffix,
complete_mar_filename_pattern=task['payload'].get(
'complete_mar_filename_pattern'),
complete_mar_bouncer_product_pattern=task['payload'].get(
'complete_mar_bouncer_product_pattern'),
)
retry(lambda: creator.run(
appVersion=task['payload']['app_version'],
productName=task['payload']['product'].capitalize(),
version=task['payload']['version'],
buildNumber=task['payload']['build_number'],
updateChannels=task['payload']['channel_names'],
ftpServer=task['payload']['archive_domain'],
bouncerServer=task['payload']['download_domain'],
enUSPlatforms=task['payload']['platforms'],
hashFunction='sha512',
partialUpdates=partials,
requiresMirrors=task['payload']['require_mirrors'],
updateLine=task['payload'].get('update_line', {}).get(suffix),
))
pusher = create_pusher(
api_root=config['api_root'],
auth=auth,
dummy=config['dummy'],
suffix=task['payload'].get('blob_suffix', ''),
)
retry(lambda: pusher.run(
productName=task['payload']['product'].capitalize(),
version=task['payload']['version'],
build_number=task['payload']['build_number'],
rule_ids=task['payload']['rules_to_update'],
))
def test_campaign_send_to_smartlist_with_two_candidates_with_no_push_device(self, token_first,
campaign_with_two_candidates_with_no_push_device_associated):
"""
- This tests the endpoint /v1/push-campaigns/:id/send
In this test I want to test the scenario that if a push campaign is being sent to multiple candidates
and there is no push device associated with any candidate, then API will return OK response but no campaign
will be sent. i.e. blasts = 1, sends = 0
"""
campaign_id = campaign_with_two_candidates_with_no_push_device_associated['id']
send_campaign(campaign_id, token_first, expected_status=(codes.OK,))
retry(get_blasts, sleeptime=SLEEP_INTERVAL, attempts=RETRY_ATTEMPTS * 2, sleepscale=1,
retry_exceptions=(AssertionError,), args=(campaign_id, token_first), kwargs={'count': 1})
get_and_assert_zero(PushCampaignApiUrl.SENDS % campaign_id, 'sends', token_first)
def wait_for_started(self):
# late imports because we need the jenkins virtualenv to be activated
# (this is done in the constructor)
import redo
import requests
session = requests.Session()
def wait_for_jenkins():
if not session.get('http://localhost:8080').status_code == 200:
raise Exception('Jenkins did not start successfully.')
redo.retry(wait_for_jenkins, sleeptime=0.5, jitter=0, sleepscale=1,
attempts=120)
def test_send_campaign_to_invalid_email_address(self, access_token_first,
user_first,
talent_pipeline):
"""
In this test, we will send an email campaign to one candidate with invalid email address.
After bounce, this email will be marked as bounced and when we will try to send this campaign
through API, no email campaign will be sent, because only one candidate is associated with this campaign but
his email address has been marked as Bounced.
"""
with app.app_context():
campaign = create_email_campaign_in_db(user_first.id)
# create candidate
email_campaign_blast, smartlist_id, candidate_ids = create_campaign_data(
access_token_first,
campaign.id,
talent_pipeline,
candidate_count=1)
invalid_email = 'invalid_' + fake.uuid4() + '@gmail.com'
email = CandidateEmail.get_email_by_candidate_id(candidate_ids[0])
email.update(address=invalid_email)
send_campaign_email_to_candidate(campaign,
email,
candidate_ids[0],
blast_id=email_campaign_blast.id)
retry(assert_is_bounced,
sleeptime=3,
attempts=100,
sleepscale=1,
args=(email, ),
retry_exceptions=(AssertionError, ))
blast_url = EmailCampaignApiUrl.BLASTS % campaign.id
campaign_blasts = CampaignsTestsHelpers.get_blasts_with_polling(
campaign,
timeout=300,
blasts_url=blast_url,
access_token=access_token_first)
campaign_blast = campaign_blasts[0]
assert campaign_blast['bounces'] == 1
# Since there is no candidate associated with campaign with valid email, so no more blasts would be created
response = requests.post(EmailCampaignApiUrl.SEND % campaign.id,
headers=dict(Authorization='Bearer %s' %
access_token_first))
assert response.status_code == requests.codes.OK
CampaignsTestsHelpers.assert_campaign_blasts(
campaign, 1, access_token=access_token_first, timeout=300)
# without sleep, finalizer deletes campaign object and code fails to find this object when bounce occurs
time.sleep(10)
def test_search_for_candidate_in_pipeline(self, user_first,
access_token_first, talent_pool):
"""
Test: Use Pipeline search params to search for a candidate
"""
user_first.role_id = Role.get_by_name('DOMAIN_ADMIN').id
db.session.commit()
skill_name = fake.word()
# Create multiple candidates
data = FakeCandidatesData.create(talent_pool=talent_pool,
count=16,
skill_name=skill_name)
candidate_ids = create_candidates_from_candidate_api(
access_token_first, data, return_candidate_ids_only=True)
number_of_talent_pipelines = 15
candidate_id = candidate_ids[-1]
# Add Pipelines
pipeline_data = {
"talent_pipelines": [{
"talent_pool_id":
talent_pool.id,
"name":
'testing_{}'.format(str(fake.uuid4())[:5]),
"date_needed":
"2017-11-30",
"search_params": {
"skills": [skill_name]
}
} for _ in range(number_of_talent_pipelines)]
}
create_resp = send_request('post', PIPELINE_URL, access_token_first,
pipeline_data)
print response_info(create_resp)
# Ensure talent pipelines were created
get_resp = send_request('get', PIPELINE_URL, access_token_first)
assert set([tp['id'] for tp in get_resp.json()['talent_pipelines']
]).issubset(create_resp.json()['talent_pipelines'])
# Assert on results
retry(assert_result,
sleeptime=3,
attempts=10,
sleepscale=1,
args=(candidate_id, access_token_first),
kwargs=dict(expected_count=number_of_talent_pipelines),
retry_exceptions=(AssertionError, ))
def submit_toplevel(task, config, auth0_secrets, backend_version):
"""Push a top-level release blob to balrog."""
partials = {}
if task["payload"].get("partial_versions"):
for v in task["payload"]["partial_versions"].split(","):
v = v.strip() # we have whitespace after the comma
version, build_number = v.split("build")
partials[version] = {"buildNumber": build_number}
suffixes = list(task["payload"].get("update_line", {}).keys()) or [""]
for suffix in suffixes:
creator = create_creator(
api_root=config["api_root"],
auth0_secrets=auth0_secrets,
dummy=config["dummy"],
suffix=task["payload"].get("blob_suffix", "") + suffix,
complete_mar_filename_pattern=task["payload"].get(
"complete_mar_filename_pattern"),
complete_mar_bouncer_product_pattern=task["payload"].get(
"complete_mar_bouncer_product_pattern"),
backend_version=backend_version,
)
retry(lambda: creator.run(
appVersion=task["payload"]["app_version"],
productName=task["payload"]["product"].capitalize(),
version=task["payload"]["version"],
buildNumber=task["payload"]["build_number"],
updateChannels=task["payload"]["channel_names"],
ftpServer=task["payload"]["archive_domain"],
bouncerServer=task["payload"]["download_domain"],
enUSPlatforms=task["payload"]["platforms"],
hashFunction="sha512",
partialUpdates=partials,
requiresMirrors=task["payload"]["require_mirrors"],
updateLine=task["payload"].get("update_line", {}).get(suffix),
))
pusher = create_pusher(api_root=config["api_root"],
auth0_secrets=auth0_secrets,
dummy=config["dummy"],
suffix=task["payload"].get("blob_suffix", ""))
retry(lambda: pusher.run(
productName=task["payload"]["product"].capitalize(),
version=task["payload"]["version"],
build_number=task["payload"]["build_number"],
rule_ids=task["payload"]["rules_to_update"],
))
def submit_locale(task, config, auth0_secrets):
"""Submit a release blob to balrog."""
upstream_artifacts = get_upstream_artifacts(task)
# Read the manifest from disk
manifest = get_manifest(config, upstream_artifacts)
suffixes = task['payload'].get('suffixes', [''])
for e in manifest:
for suffix in suffixes:
# Get release metadata from manifest
submitter, release = create_locale_submitter(e, suffix, auth0_secrets, config)
# Connect to balrog and submit the metadata
retry(lambda: submitter.run(**release), jitter=5, sleeptime=10, max_sleeptime=30, attempts=20)
def worker(item):
source, destination = item
def copy_key():
source_key = bucket.get_key(source)
dest_key = bucket.get_key(destination)
# According to http://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonResponseHeaders.html
# S3 key MD5 is represented as ETag, except when objects are
# uploaded using multipart method. In this case objects's ETag
# is constructed using its MD5, minus symbol, and number of
# part. See http://stackoverflow.com/questions/12186993/what-is-the-algorithm-to-compute-the-amazon-s3-etag-for-a-file-larger-than-5gb#answer-19896823
source_md5 = source_key.etag.split("-")[0]
if dest_key:
dest_md5 = dest_key.etag.split("-")[0]
else:
dest_md5 = None
if not dest_key:
self.info("Copying {} to {}".format(source, destination))
bucket.copy_key(destination, self.config["bucket_name"],
source)
elif source_md5 == dest_md5:
self.warning(
"{} already exists with the same content ({}), skipping copy".format(
destination, dest_md5))
else:
self.fatal(
"{} already exists with the different content (src ETag: {}, dest ETag: {}), aborting".format(
destination, source_key.etag, dest_key.etag))
return retry(copy_key, sleeptime=5, max_sleeptime=60,
retry_exceptions=(S3CopyError, S3ResponseError))
def query_pushes_by_pushid_range(repo_url, start_id, end_id, version=VERSION, return_revision_list=False):
"""
Return an ordered list of pushes (oldest first).
repo_url - represents the URL to clone a repo
start_id - from which pushid to start with (oldest)
end_id - from which pushid to end with (most recent)
version - version of json-pushes to use (see docs)
return_revision_list - return a list of revisions if it's True
"""
push_list = []
url = "%s?startID=%s&endID=%s&version=%s&tipsonly=1" % (
JSON_PUSHES % {"repo_url": repo_url},
start_id - 1, # off by one to compensate for pushlog as it skips start_id
end_id,
version,
)
LOG.debug("About to fetch %s" % url)
req = retry(requests.get, args=(url,))
pushes = req.json()["pushes"]
for push_id in sorted(pushes.keys()):
# Querying by push ID is preferred because date ordering is
# not guaranteed (due to system clock skew)
# We can interact with self-serve with the 12 char representation
push_list.append(Push(push_id=push_id, push_info=pushes[push_id]))
if return_revision_list:
return _pushes_to_list(push_list)
return push_list
def update_runnableapi():
"""
Use it to update runnablejobs.json file.
"""
url = "https://index.taskcluster.net/v1/task/gecko.v2.%s.latest.firefox.decision/"
latest_task = retry(requests.get, args=(url % "mozilla-inbound", ),
kwargs={'headers': {'accept-encoding': 'json'}, 'verify': True}).json()
task_id = latest_task['taskId']
# The format of expires is like 2017-07-04T22:13:23.248Z and we only want 2017-07-04 part
expires = latest_task['expires'].split('T')[0]
time_tuple = datetime.datetime.strptime(expires, "%Y-%m-%d").timetuple()
new_timestamp = time.mktime(time_tuple)
path = ROOT_DIR + '/runablejobs.json'
# we do nothing if the timestamp of runablejobs.json is equal with the latest task
# otherwise we download and update it
if os.path.isfile(path):
with open(path, 'r+') as data:
# read the timesstamp of this task from json file
oldtime = json.loads(data.read())['meta']['timetamp']
if oldtime == new_timestamp:
print "The runnable json file is latest already."
return
else:
print "It's going to update your runnable jobs data."
download_runnable_jobs(new_timestamp, task_id)
else:
print "It's going to help you download the runnable jobs file."
download_runnable_jobs(new_timestamp, task_id)
def get_changed_files(repository, revision):
"""
Get the set of files changed in the push headed by the given revision.
Responses are cached, so multiple calls with the same arguments are OK.
"""
key = repository, revision
if key not in _cache:
url = '%s/json-automationrelevance/%s' % (repository.rstrip('/'), revision)
logger.debug("Querying version control for metadata: %s", url)
def get_automationrelevance():
response = requests.get(url, timeout=5)
return response.json()
contents = retry(get_automationrelevance, attempts=2, sleeptime=10)
logger.debug('{} commits influencing task scheduling:'
.format(len(contents['changesets'])))
changed_files = set()
for c in contents['changesets']:
logger.debug(" {cset} {desc}".format(
cset=c['node'][0:12],
desc=c['desc'].splitlines()[0].encode('ascii', 'ignore')))
changed_files |= set(c['files'])
_cache[key] = changed_files
return _cache[key]
def query_push_by_revision(repo_url, revision, full=False, return_revision_list=False):
"""
Return a dictionary with meta-data about a push including:
* changesets
* date
* user
repo_url - represents the URL to clone a rep
revision - the revision used to set the query range
full - query whole information of a push if it's True
return_revision_list - return a list of revisions if it's True
"""
url = "%s?changeset=%s&tipsonly=1" % (JSON_PUSHES % {"repo_url": repo_url}, revision)
if full:
url += "&full=1"
LOG.debug("About to fetch %s" % url)
req = retry(requests.get, args=(url,))
data = req.json()
assert len(data) == 1, "We should only have information about one push"
if not full:
LOG.debug("Push info: %s" % str(data))
push_id, push_info = data.popitem()
push = Push(push_id=push_id, push_info=push_info)
else:
LOG.debug("Requesting the info with full=1 can yield too much unnecessary output " "to debug anything properly")
if return_revision_list:
return push.changesets[0].node
return push
def worker(item):
source, destination = item
def copy_key():
if destination in to_keys_checksums:
# compare md5
if from_keys_checksums[source] != to_keys_checksums[
destination]:
raise ScriptWorkerTaskException(
"{} already exists with different content "
"(src etag: {}, dest etag: {}), aborting".format(
destination, from_keys_checksums[source],
to_keys_checksums[destination]))
else:
log.warning(
"{} already exists with the same content ({}), "
"skipping copy".format(destination,
to_keys_checksums[destination]))
else:
log.info("Copying {} to {}".format(source, destination))
boto_client.copy_object(Bucket=context.bucket_name,
CopySource={
"Bucket": context.bucket_name,
"Key": source
},
Key=destination)
return retry(copy_key,
sleeptime=5,
max_sleeptime=60,
retry_exceptions=(ClientError, ))
def download_file(url, target, retry_sleep=RETRY_SLEEP, attempts=3):
"""Downloads a file, given an URL in the target path.
The function will attempt several times on failures.
"""
def _download_file(url, target):
req = requests.get(url, stream=True, timeout=30)
target_dir = target.parent.resolve()
if str(target_dir) != "":
target_dir.mkdir(exist_ok=True)
with target.open("wb") as f:
for chunk in req.iter_content(chunk_size=1024):
if not chunk:
continue
f.write(chunk)
f.flush()
return target
return retry(
_download_file,
args=(url, target),
attempts=attempts,
sleeptime=retry_sleep,
jitter=0,
)
def order(self, asset, amount, style, order_id=None):
log.debug('ordering {} {}'.format(amount, asset.symbol))
if amount == 0:
log.warn('skipping 0 amount orders')
return None
if self.simulate_orders:
return super(ExchangeBlotter, self).order(
asset, amount, style, order_id
)
else:
order = retry(
action=self.exchange_order,
attempts=self.attempts['order_attempts'],
sleeptime=self.attempts['retry_sleeptime'],
retry_exceptions=(ExchangeRequestError,),
cleanup=lambda: log.warn('Ordering again.'),
args=(asset, amount, style),
)
self.open_orders[order.asset].append(order)
self.orders[order.id] = order
self.new_orders.append(order)
return order.id
def _get_product_uptake(self, tuxedo_server_url, auth,
related_product, os):
from redo import retry
import requests
url = get_tuxedo_uptake_url(tuxedo_server_url, related_product, os)
self.info("Requesting {} from tuxedo".format(url))
def get_tuxedo_page():
r = requests.get(url, auth=auth,
verify=False, timeout=60)
r.raise_for_status()
return r.content
def calculateUptake(page):
doc = xml.dom.minidom.parseString(page)
uptake_values = []
for element in doc.getElementsByTagName('available'):
for node in element.childNodes:
if node.nodeType == xml.dom.minidom.Node.TEXT_NODE and \
node.data.isdigit():
uptake_values.append(int(node.data))
if not uptake_values:
uptake_values = [0]
return min(uptake_values)
page = retry(get_tuxedo_page)
uptake = calculateUptake(page)
self.info("Current uptake for {} is {}".format(related_product, uptake))
return uptake
def download_runnable_jobs(new_timestamp, task_id=None):
if task_id:
url = TREEHERDER_HOST.format('mozilla-inbound', task_id)
data = retry(requests.get, args=(url, ), kwargs={'headers': headers}).json()
if len(data['results']) > 0:
data['meta'].update({'timetamp': new_timestamp})
with open(ROOT_DIR + '/runnablejobs.json', 'w') as f:
json.dump(data, f)
def _update_bouncer_alias(self, tuxedo_server_url, auth,
related_product, alias):
from redo import retry
import requests
url = "%s/create_update_alias" % tuxedo_server_url
data = {"alias": alias, "related_product": related_product}
self.log("Updating {} to point to {} using {}".format(alias,
related_product,
url))
# Wrap the real call to hide credentials from retry's logging
def do_update_bouncer_alias():
r = requests.post(url, data=data, auth=auth,
verify=False, timeout=60)
r.raise_for_status()
retry(do_update_bouncer_alias)
def worker(item):
source, destination = item
self.info("Downloading {} to {}".format(source, destination))
key = bucket.get_key(source)
return retry(key.get_contents_to_filename,
args=(destination, ),
sleeptime=5, max_sleeptime=60,
retry_exceptions=(S3CopyError, S3ResponseError))
def start_disable_slave(self, machine, loanid):
try:
url = furl(current_app.config.get("SLAVEAPI_URL", None))
url.path.add(machine).add("actions").add("shutdown_buildslave")
ret = retry(requests.post, args=(str(url),), ).json()
return (ret["requestid"], machine)
except Exception as exc:
logger.exception(exc)
self.retry(exc=exc)
def query_low_value_tasks(self, project, bbb=False):
# Request the set of low value tasks from the SETA service. Low value tasks will be
# optimized out of the task graph.
low_value_tasks = []
if not bbb:
# we want to get low priority taskcluster jobs
url = SETA_ENDPOINT % (project, 'taskcluster')
else:
# we want low priority buildbot jobs
url = SETA_ENDPOINT % (project, 'buildbot&priority=5')
# Try to fetch the SETA data twice, falling back to an empty list of low value tasks.
# There are 10 seconds between each try.
try:
logger.debug("Retrieving low-value jobs list from SETA")
response = retry(requests.get, attempts=2, sleeptime=10,
args=(url, ),
kwargs={'timeout': 60, 'headers': ''})
task_list = json.loads(response.content).get('jobtypes', '')
if type(task_list) == dict and len(task_list) > 0:
if type(task_list.values()[0]) == list and len(task_list.values()[0]) > 0:
low_value_tasks = task_list.values()[0]
# bb job types return a list instead of a single string,
# convert to a single string to match tc tasks format
if type(low_value_tasks[0]) == list:
low_value_tasks = [self._get_task_string(x) for x in low_value_tasks]
# ensure no build tasks slipped in, we never want to optimize out those
low_value_tasks = [x for x in low_value_tasks if 'build' not in x.lower()]
# In the event of request times out, requests will raise a TimeoutError.
except exceptions.Timeout:
logger.warning("SETA timeout, we will treat all test tasks as high value.")
# In the event of a network problem (e.g. DNS failure, refused connection, etc),
# requests will raise a ConnectionError.
except exceptions.ConnectionError:
logger.warning("SETA connection error, we will treat all test tasks as high value.")
# In the event of the rare invalid HTTP response(e.g 404, 401),
# requests will raise an HTTPError exception
except exceptions.HTTPError:
logger.warning("We got bad Http response from ouija,"
" we will treat all test tasks as high value.")
# We just print the error out as a debug message if we failed to catch the exception above
except exceptions.RequestException as error:
logger.warning(error)
# When we get invalid JSON (i.e. 500 error), it results in a ValueError (bug 1313426)
except ValueError as error:
logger.warning("Invalid JSON, possible server error: {}".format(error))
return low_value_tasks
def format_in_table(active_jobs, master):
results = {}
sum_removed = 0
sum_remaining = 0
data = retry(requests.get, args=('http://alertmanager.allizom.org/data/jobnames/', ),
kwargs={'headers': headers,
'verify': True}).json()
running_jobs = data['results']
for jobtype in active_jobs:
key = "%s_%s" % (jobtype[0], jobtype[1])
if key not in results:
results[key] = []
for item in master:
if item[0] == jobtype[0] and item[1] == jobtype[1]:
results[key].append(item[2])
keys = results.keys()
keys.sort()
missing_jobs = []
for key in keys:
data = results[key]
data.sort()
output = ""
for platform, buildtype, test in active_jobs:
if "%s_%s" % (platform, buildtype) != key:
continue
output += '\t'
if test in data or '' in data:
found = False
for job in running_jobs:
if job['name'] == test:
output += job['job_type_symbol']
found = True
break
if not found:
output += '**'
missing_jobs.append(test)
sum_removed += 1
else:
output += "--"
sum_remaining += 1
print "%s%s" % (key, output)
if missing_jobs:
print "** new jobs which need a code: %s" % ','.join(missing_jobs)
print "Total removed %s" % (sum_removed)
print "Total remaining %s" % (sum_remaining)
print "Total jobs %s" % (sum_removed + sum_remaining)
def worker(item):
source, destination = item
self.info("Copying {} to {}".format(source, destination))
return retry(bucket.copy_key,
args=(destination,
self.config["bucket_name"],
source),
sleeptime=5, max_sleeptime=60,
retry_exceptions=(S3CopyError, S3ResponseError))
def retry_get(url, **karwgs):
"""
More robust `requests.get` equivalent function.
This is equivalent to the requests.get function, except that
it will retry the requests call three times in case of HTTPError or
ConnectionError.
"""
return redo.retry(get_http_session().get, attempts=3, sleeptime=1,
retry_exceptions=(requests.exceptions.HTTPError,
requests.exceptions.ConnectionError),
args=(url,), kwargs=karwgs)
def waitfor_disable_slave(self, data, loanid):
requestid, machine = data
try:
url = furl(current_app.config.get("SLAVEAPI_URL", None))
url.path.add(machine).add("actions").add("shutdown_buildslave")
url.args["requestid"] = requestid
ret = retry(requests.get, args=(str(url),), kwargs=dict()).json()
if ret["state"] in (0, 1):
# 0 = PENDING, 1 = RUNNING (3=Failed and 2=Success)
raise Exception("Continue waiting for disabled slave")
except Exception as exc:
self.retry(exc=exc)
def query_the_runnablejobs(task_id, repo_name='mozilla-inbound'):
url = RUNNABLE_API.format(repo_name, task_id)
try:
data = retry(requests.get, args=(url, ), kwargs={'headers': HEADERS}).json()
if data:
# A lot of code components still rely on the file being on disk
with open(get_runnable_jobs_path(), 'w') as f:
json.dump(data, f, indent=2, sort_keys=True)
return data
except:
LOG.warning("We failed to get runnablejobs via %s" % url)
return None
def query_low_value_tasks(self, project):
# Request the set of low value tasks from the SETA service. Low value tasks will be
# optimized out of the task graph.
if project not in SETA_PROJECTS:
logger.debug("SETA is not enabled for project `{}`".format(project))
return []
logger.debug("Querying SETA service for low-value tasks on {}".format(project))
low_value_tasks = []
url = SETA_ENDPOINT % project
# Try to fetch the SETA data twice, falling back to an empty list of low value tasks.
# There are 10 seconds between each try.
try:
logger.debug("Retrieving low-value jobs list from SETA")
response = retry(requests.get, attempts=2, sleeptime=10,
args=(url, ),
kwargs={'timeout': 5, 'headers': headers})
task_list = json.loads(response.content).get('jobtypes', '')
if len(task_list) > 0:
low_value_tasks = task_list.values()[0]
# Bug 1315145, disable SETA for tier-1 platforms until backfill is implemented.
low_value_tasks = [x for x in low_value_tasks if x.find('debug') == -1]
low_value_tasks = [x for x in low_value_tasks if x.find('asan') == -1]
# In the event of request times out, requests will raise a TimeoutError.
except exceptions.Timeout:
logger.warning("SETA server is timeout, we will treat all test tasks as high value.")
# In the event of a network problem (e.g. DNS failure, refused connection, etc),
# requests will raise a ConnectionError.
except exceptions.ConnectionError:
logger.warning("SETA server is timeout, we will treat all test tasks as high value.")
# In the event of the rare invalid HTTP response(e.g 404, 401),
# requests will raise an HTTPError exception
except exceptions.HTTPError:
logger.warning("We got bad Http response from ouija,"
" we will treat all test tasks as high value.")
# We just print the error out as a debug message if we failed to catch the exception above
except exceptions.RequestException as error:
logger.warning(error)
# When we get invalid JSON (i.e. 500 error), it results in a ValueError (bug 1313426)
except ValueError as error:
logger.warning("Invalid JSON, possible server error: {}".format(error))
return low_value_tasks
def get_transactions(self, bar_data):
if self.simulate_orders:
return super(ExchangeBlotter, self).get_transactions(bar_data)
else:
return retry(
action=self.get_exchange_transactions,
attempts=self.attempts['get_transactions_attempts'],
sleeptime=self.attempts['retry_sleeptime'],
retry_exceptions=(ExchangeRequestError,),
cleanup=lambda: log.warn(
'Fetching exchange transactions again.'
)
)
def get_raw_data(start_date, end_date):
if not end_date:
end_date = datetime.datetime.now()
if not start_date:
start_date = end_date - datetime.timedelta(days=SETA_WINDOW)
url = "http://alertmanager.allizom.org/data/seta/?startDate=%s&endDate=%s" % \
(start_date.strftime("%Y-%m-%d"), end_date.strftime("%Y-%m-%d"))
response = retry(requests.get, args=(url, ),
kwargs={'headers': headers, 'verify': True})
data = json.loads(response.content)
return data['failures']
def valid_revision(repo_url, revision):
"""Verify that a revision exists in the given repository URL."""
global VALID_CACHE
if (repo_url, revision) in VALID_CACHE:
return VALID_CACHE[(repo_url, revision)]
LOG.debug("Determine if the revision is valid.")
url = "%s?changeset=%s&tipsonly=1" % (JSON_PUSHES % {"repo_url": repo_url}, revision)
data = retry(requests.get, args=(url,)).json()
ret = True
# A valid revision will return a dictionary with information about exactly one revision
if len(data) != 1:
LOG.warning("Revision %s not found on branch %s" % (revision, repo_url))
ret = False
VALID_CACHE[(repo_url, revision)] = ret
return ret
def worker(item):
source, destination = item
def copy_key():
dest_key = bucket.get_key(destination)
source_key = bucket.get_key(source)
if not dest_key:
self.info("Copying {} to {}".format(source, destination))
bucket.copy_key(destination, self.config["bucket_name"],
source)
elif source_key.etag == dest_key.etag:
self.warning(
"{} already exists with the same content ({}), skipping copy".format(
destination, dest_key.etag))
else:
self.fatal(
"{} already exists with the different content (src: {}, dest: {}), aborting".format(
destination, source_key.etag, dest_key.etag))
return retry(copy_key, sleeptime=5, max_sleeptime=60,
retry_exceptions=(S3CopyError, S3ResponseError))
def query_pushes_by_revision_range(
repo_url, from_revision, to_revision, version=VERSION, tipsonly=True, return_revision_list=False
):
"""
Return an ordered list of pushes (by date - oldest (starting) first).
repo_url - represents the URL to clone a repo
from_revision - from which revision to start with (oldest)
to_revision - from which revision to end with (newest)
version - version of json-pushes to use (see docs)
tipsonly - only return the tip most push been returned if it's True
return_revision_list - return a list of revisions if it's True
"""
push_list = []
url = "%s?fromchange=%s&tochange=%s&version=%d" % (
JSON_PUSHES % {"repo_url": repo_url},
from_revision,
to_revision,
version,
)
if tipsonly:
url += "&tipsonly=1"
LOG.debug("About to fetch %s" % url)
req = retry(requests.get, args=(url,))
pushes = req.json()["pushes"]
# json-pushes does not include the starting revision
push_list.append(query_push_by_revision(repo_url, from_revision))
for push_id in sorted(pushes.keys()):
# Querying by push ID is perferred because date ordering is
# not guaranteed (due to system clock skew)
# We can interact with self-serve with the full char representation
push_list.append(Push(push_id=push_id, push_info=pushes[push_id]))
if return_revision_list:
return _pushes_to_list(push_list)
return push_list
