def test_payload_tracker(self):
topic = "platform.payload-status"
good_msg = {"service": "advisor-pup",
"request_id": "dootdoot",
"payload_id": "dootdoot",
"status": "test",
"status_msg": "unittest",
"account": "123456",
"date": tracker.get_time()}
self.assertEqual({"topic": topic, "msg": good_msg}, tracker.payload_tracker("dootdoot", "123456", "test", "unittest"))
def fail_upload(data, response, extra):
mnm.invalid.inc()
logger.info("request_id [%s] validation failed with error: %s",
data['request_id'],
response.get('error'),
extra=extra)
data_to_produce = {
'topic': 'platform.upload.validation',
'msg': {
'account': data['account'],
'request_id': data['request_id'],
'validation': 'failure'
}
}
produce_queue.append(
tracker.payload_tracker(data["request_id"], data["account"], "failure",
"upload failed pup validation"))
return data_to_produce
def succeed_upload(data, response, extra):
mnm.valid.inc()
logger.info("request_id [%s] validation successful",
data['request_id'],
extra=extra)
data_to_produce = {
'topic': 'platform.upload.validation',
'msg': {
'id': response.get('id') if response else None,
'service': data['service'],
'request_id': data['request_id'],
'account': data['account'],
'principal': data['principal'],
'b64_identity': data.get('b64_identity'),
'satellite_managed': data.get('satellite_managed'),
'validation': 'success'
}
}
produce_queue.append(
tracker.payload_tracker(data["request_id"], data["account"], "success",
"Upload passed pup validation"))
return data_to_produce
async def post_to_inventory(facts, msg):
extra = get_extra()
produce_queue.append(
tracker.payload_tracker(extra["request_id"], extra["account"],
"posting",
"posting upload details to inventory"))
post = {**facts, **msg}
post['account'] = post.pop('account')
post['facts'] = []
if post.get('metadata'):
post['facts'].append({
'facts': post.pop('metadata'),
'namespace': 'insights-client'
})
headers = {
'x-rh-identity': post['b64_identity'],
'Content-Type': 'application/json',
'x-rh-insights-request-id': post['request_id']
}
extra["account"] = post["account"]
extra["request_id"] = post["request_id"]
try:
timeout = aiohttp.ClientTimeout(total=60)
async with aiohttp.ClientSession() as session:
async with session.post(configuration.INVENTORY_URL,
data=json.dumps([post]),
headers=headers,
timeout=timeout) as response:
response_json = await response.json()
if response.status != 207:
error = response_json.get('detail')
logger.error('Failed to post to inventory: %s',
error,
extra=extra)
return {"error": "Failed to post to inventory."}
elif response_json['data'][0][
'status'] != 200 and response_json['data'][0][
'status'] != 201:
mnm.inventory_post_failure.inc()
logger.error(
'request_id [%s] failed to post to inventory.',
msg['request_id'],
extra=extra)
logger.error('inventory error response: %s',
await response.text(),
extra=extra)
return {"error": "Failed to post to inventory."}
else:
mnm.inventory_post_success.inc()
logger.info("request_id [%s] posted to inventory: ID [%s]",
msg['request_id'],
response_json['data'][0]['host']['id'],
extra={
"request_id": post['request_id'],
"account": post["account"]
})
return response_json['data'][0]['host']
await session.close()
except ClientConnectionError as e:
logger.error(
"request_id [%s] failed to post to inventory, unable to connect: %s",
msg['request_id'],
e,
extra=extra)
return {"error": "Unable to update inventory. Service unavailable"}
async def handle_file(msgs):
extra = get_extra()
for msg in msgs:
try:
data = json.loads(msg.value)
extra["account"] = data["account"]
extra["request_id"] = data["request_id"]
except Exception:
logger.exception("handle_file(): unable to decode msg as json: %s",
msg.value)
continue
produce_queue.append(
tracker.payload_tracker(extra["request_id"], extra["account"],
"processing", "processing upload in pup"))
mnm.total.inc()
try:
result = await validate(data['url'], data["request_id"],
data["account"])
except Exception as e:
logger.exception("Validation encountered error: %s",
e,
extra=extra)
continue
if result is None:
logger.info("Validation resulted in a None...ignoring msg",
extra=extra)
continue
# we do not want to POST the system profile to inventory
# until after we get an id
system_profile = result.pop("system_profile")
data["satellite_managed"] = system_profile.get("satellite_managed")
data["satellite_id"] = system_profile.get("satellite_id")
if len(result) > 0 and 'error' not in result:
if not data.get('id'):
logger.info(
"Inventory ID not included in message from upload-service [%s]",
data["request_id"],
extra=extra)
response = await post_to_inventory(result, data)
else:
logger.info(
"Not posting to inventory, using ID from upload-service (%s)",
data.get("id"),
extra=extra)
response = {"id": data.get('id')}
if response.get('error'):
data_to_produce = fail_upload(data, response, extra)
else:
# As long as we get an id back from inventory
# we can send the system profile
system_profile_queue.append({
"id": response["id"],
"account": data["account"],
"request_id": data["request_id"],
"system_profile": system_profile
})
data_to_produce = succeed_upload(data, response, extra)
else:
data_to_produce = fail_upload(data, result, extra)
produce_queue.append(
tracker.payload_tracker(data["request_id"], data["account"],
"responding",
"sending validation response to ingress"))
produce_queue.append(data_to_produce)
current_archives.append(data["request_id"])
logger.info(
"data for topic [%s], request_id [%s] put on produce queue (qsize now: %d): %s",
data_to_produce['topic'],
data_to_produce['msg']['request_id'],
len(produce_queue),
data_to_produce,
extra=extra)