def main(msg: func.QueueMessage) -> None:
try:
message = OATQueueMessage.parse_obj(msg.get_json())
clp_id = message.clp_id
detections = message.detections
post_data = message.post_data
token = utils.find_token_by_clp(clp_id, API_TOKENS)
if not token:
raise GeneralException(f'Token not found for clp: {clp_id}')
# get workbench detail
raw_logs = get_search_data(token, post_data)
# transform data
transfromed_logs = _transfrom_logs(clp_id, detections, raw_logs)
# send to log analytics
log_analytics = LogAnalytics(WORKSPACE_ID, WORKSPACE_KEY, OAT_LOG_TYPE)
log_analytics.post_data(transfromed_logs)
logging.info(f'Send oat data successfully. count: {len(transfromed_logs)}.')
except HTTPError as e:
logging.exception(f'Fail to get search data! Exception: {e}')
raise
except:
logging.exception('Internal error.')
raise
def main(msg: func.QueueMessage) -> None:
body = msg.get_json()
date = body['date']
year, month, _ = date.split("T")[0].split("-")
city = body['city']
country = body['country']
temperature = body['temperature']
# Obtain HOST and MASTER_KEY from: https://portal.azure.com/#@[user_email]/resource/subscriptions/[subscription_id]/resourceGroups/[resource_group_name]/providers/Microsoft.DocumentDb/databaseAccounts/[db_account_name]/keys
HOST = "//TODO"
MASTER_KEY = "//TODO"
# construct your own doc_link and document as needed. This is a sample.
# Please see the readme for details regarding the code below.
client = cosmos_client.CosmosClient(HOST, {'masterKey': MASTER_KEY})
database_link = 'dbs/' + 'weather-data'
collection_link = database_link + '/colls/' + 'weather-data'
doc_id = year + "-" + month + "___" + city + "-" + country
doc_link = collection_link + '/docs/' + doc_id
try:
document = {
'id': doc_id,
'city': city,
'country': country,
'Temperature_List': [str(temperature)],
'Month': month,
'Year': year
}
client.CreateItem(collection_link, document)
except:
response = client.ReadItem(doc_link)
response["Temperature_List"].append(str(temperature))
client.UpsertItem(collection_link, response)
logging.info('Python queue trigger function processed a queue item:')
def main(msgIn: func.QueueMessage):
try:
args = json.loads(msgIn.get_body())
except:
args = msgIn.get_json()
os.environ['__PW_ACTIVATION_ID'] = str(msgIn.id)
if 'remote_invoker' in args:
logger.info("Pywren v{} - Starting invoker".format(__version__))
function_invoker(args)
else:
logger.info("Pywren v{} - Starting execution".format(__version__))
function_handler(args)
return {"Execution": "Finished"}
def main(rcaMsg: func.QueueMessage) -> None:
try:
payload = rcaMsg.get_json()
clp_id = payload['clp_id']
workbench_id = payload['workbench_id']
task_id = payload['task_id']
task_name = payload['task_name']
target_guid = payload['target_guid']
target_info = payload['target_info']
token = utils.find_token_by_clp(clp_id, API_TOKENS)
if not token:
raise GeneralException(f'Token not found for clp: {clp_id}')
rca_task_detail = get_rca_task_detail(token, task_id, target_guid)
target_info = {
'xdrCustomerID': clp_id,
'taskId': task_id,
'taskName': task_name,
'agentEntity': target_info,
'workbenchId': workbench_id
}
rca_task_result_log = transform_utils.transform_rca_result(target_info, rca_task_detail)
if len(rca_task_result_log) > 0:
log_type = configurations.get_rca_log_type()
log_analytics = LogAnalytics(WORKSPACE_ID, WORKSPACE_KEY, log_type)
log_analytics.post_data(
rca_task_result_log
)
logging.info(
f'Send rca data successfully. Task id: {task_id}, Task name: {task_name},Target guid: {target_guid}'
)
except HTTPError as e:
logging.exception(
f'Fail to get rca detail! Exception: {e}',
)
raise
except:
logging.exception('Internal error.')
raise
def main(msgIn: func.QueueMessage, msgOut: func.Out[func.QueueMessage]):
try:
args = json.loads(msgIn.get_body())
except Exception:
args = msgIn.get_json()
os.environ['__LITHOPS_ACTIVATION_ID'] = str(msgIn.id)
setup_lithops_logger(args['log_level'])
if 'get_preinstalls' in args:
logger.info("Lithops v{} - Generating metadata".format(__version__))
runtime_meta = get_runtime_preinstalls()
msgOut.set(json.dumps(runtime_meta))
elif 'remote_invoker' in args:
logger.info("Lithops v{} - Starting invoker".format(__version__))
function_invoker(args)
else:
logger.info("Lithops v{} - Starting execution".format(__version__))
function_handler(args)
def main_queue(msgIn: func.QueueMessage, msgOut: func.Out[func.QueueMessage]):
try:
payload = json.loads(msgIn.get_body())
except Exception:
payload = msgIn.get_json()
setup_lithops_logger(payload['log_level'])
os.environ['__LITHOPS_ACTIVATION_ID'] = str(msgIn.id)
os.environ['__LITHOPS_BACKEND'] = 'Azure Functions (event)'
if 'get_preinstalls' in payload:
logger.info("Lithops v{} - Generating metadata".format(__version__))
runtime_meta = get_runtime_preinstalls()
msgOut.set(json.dumps(runtime_meta))
elif 'remote_invoker' in payload:
logger.info("Lithops v{} - Starting Azure Functions (event) invoker".format(__version__))
function_invoker(payload)
else:
logger.info("Lithops v{} - Starting Azure Functions (event) execution".format(__version__))
function_handler(payload)
def main(msg: func.QueueMessage) -> None:
jsonDict = msg.get_json() # Parse QueueMessage to Python object.
module_name = jsonDict['title'].split('_')[0] # Get module name.
# Build a Markdown formate link that link to the product page.
caption = '[' + jsonDict['title'] + ']' + '(' +\
'https://www.freitag.ch/en/' + module_name +\
'?productID=' + jsonDict['RowKey'] + ')'
# Build up data that will be sent out.
data = {
'chat_id':
CHANNEL_ID,
'media': [
getInputMediaPhoto(src, caption)
for src in jsonDict['product_cover_photo']
]
}
url = 'https://api.telegram.org/bot' + BOT_TOKEN + '/sendMediaGroup'
# Post data as JSON to Telegram API.
requests.post(url, json=data)
logging.info('Python queue trigger function processed a queue item:')
def main(req: func.QueueMessage) -> func.HttpResponse:
try:
logging.info("SendGrid email triggered.")
logging.debug("Parsing message data from request body")
body = req.get_json()
from_email = Email(body["from"])
to_email = To(body["to"])
subject = Subject(body["subject"])
template_id = TemplateId(body["template_id"])
logging.debug("Getting template value substitutions")
substitutions = []
for substitution_key in body["substitutions"].keys():
message_substitution = Substitution(
key=substitution_key,
value=body["substitutions"][substitution_key])
substitutions.append(message_substitution)
logging.info("Message contents parsed from request input.")
sg = sendgrid.SendGridAPIClient(
api_key=environ.get('SENDGRID_API_KEY'))
logging.info("SendGrid client initialized")
mail = Mail(from_email=from_email,
to_email=to_email,
subject=subject,
global_substitutions=substitutions)
mail.template_id = template_id
logging.info("Message initialized")
response = sg.client.mail.send.post(request_body=mail.get())
logging.info("Message sent!")
return func.HttpResponse(body=json.dumps(response.body),
status_code=response.status_code,
mimetype="application/json")
except Exception as email_exception:
logging.error("Error sending email!")
logging.error(email_exception)
def main(msg: func.QueueMessage):
message_json = msg.get_json()
runner = DBTRunner()
runner.go(**message_json)
def main(msg: func.QueueMessage):
result = json.dumps({
'id':
msg.id,
'body':
msg.get_json(),
'expiration_time':
(msg.expiration_time.isoformat() if msg.expiration_time else None),
'insertion_time':
(msg.insertion_time.isoformat() if msg.insertion_time else None),
'time_next_visible':
(msg.time_next_visible.isoformat() if msg.time_next_visible else None),
'pop_receipt':
msg.pop_receipt,
'dequeue_count':
msg.dequeue_count
})
# get the json fron the queue
res = json.loads(result)
# create empty list for bulk update
payload_file = []
glossaryGuid = os.environ.get('glossaryGuid')
#calculate if it is a single term to be updated or a bulk update
singleTerm = False
if len(res['body']) == 1:
singleTerm = True
# the purviewcli is very verbose in the logs, therefore we need to suppress the output and use logging.warning
# the createGlossaryItems function is configured to log only warnings in the 'hosts.json' file
logging.warning("singleTerm: %s" % (singleTerm))
# parse the json
for objBody in res['body']:
termValue = parse_json_recursively(objBody, "termName")
# Terms name with uppper / lower case combinations can cause the import to fail - set the first character to lower case
termValue = termValue[0].lower() + termValue[1:]
if termValue != None:
importTerm = True
# A term name can't have a period (".") in the Azure purview catalog, so we need to remove it
termValue = termValue.replace(".", " ")
termName = "--termName=" + unicodedata.normalize('NFKD', termValue)
#check if term already exists in catalog
sys.argv = [
"pv", "search", "query",
"--keywords=\"" + re.escape(termValue) + "\""
]
f = io.StringIO()
with redirect_stdout(f):
try:
pv.main()
except:
logging.error("failed to search Azure Purview")
logging.warning("sys.argv = %s" % (sys.argv))
logging.error(sys.exc_info()[0])
logging.exception("The exception is:")
# if the search value is 0 it doesn't exits. If it is greater than Zero we need to parse the JSON for an extact match
if int(json.loads(f.getvalue())['@search.count']) > 0:
for element in json.loads(f.getvalue())['value']:
qualifiedName = parse_json_recursively(element, 'name')
# try all combinations of escaping
if qualifiedName != None and (qualifiedName == termValue \
or qualifiedName == re.escape(termValue) \
or re.escape(qualifiedName) == re.escape(termValue) \
or re.escape(qualifiedName) == termValue):
importTerm = False
logging.warning(
"term %s already exists and will not be imported" %
(termValue))
# The term will be imported if it doesn't already exist in the catalog
if importTerm == True:
# get the description
definitionValue = parse_json_recursively(
objBody, "longDescription")
if definitionValue != None:
definitionName = "--longDescription=" + unicodedata.normalize(
'NFKD', definitionValue)
if singleTerm == False:
#check termName is not already in the list
if termName.replace(
'"', "") not in json.dumps(payload_file):
logging.warning(
"Term Name %s will be bulk-imported" %
(termValue))
# append termName and defintionName to the payload file that will be bulk imported
anchor = {"glossaryGuid": glossaryGuid}
json_file = {}
json_file['anchor'] = anchor
json_file[
'longDescription'] = definitionName.replace(
'"', "")
json_file['name'] = termName.replace('"', "")
payload_file.append(json_file)
elif singleTerm == True:
# import a single term straight away
sys.argv = [
"pv", "glossary", "createTerm", glossaryGuid,
termName.replace('"', ""), "--longDescription=" +
definitionValue.replace('"', "")
]
logging.warning(
"Term Name %s will be imported as a single item" %
(termValue))
anchor = {"glossaryGuid": glossaryGuid}
json_file = {}
json_file['anchor'] = anchor
json_file['longDescription'] = definitionName.replace(
'"', "")
json_file['name'] = termName.replace('"', "")
singlePayload = []
singlePayload.append(json_file)
fileName = str(randrange(6000)) + 'payload.json'
tempFilePath = os.path.join(tempfile.gettempdir(),
fileName)
f = open(tempFilePath, "w+")
f.write(json.dumps(singlePayload))
f.flush()
f.close
sys.argv = [
"pv", "glossary", "createTerm",
"--payload-file=" + tempFilePath
]
logging.warning(
"Term will be imported: The Arguments are: %s " %
(sys.argv))
pv.main()
else:
logging.error("no description found for the term %s" %
(termValue))
logging.warning("Number of items parsed is: %s" % (len(res['body'])))
# bulk update
if (len(payload_file) > 0):
fileName = str(randrange(6000)) + 'payload.json'
tempFilePath = os.path.join(tempfile.gettempdir(), fileName)
f = open(tempFilePath, "w+")
f.write(json.dumps(payload_file))
f.flush()
f.close
sys.argv = [
"pv", "glossary", "createTerms", "--payload-file=" + tempFilePath
]
logging.warning("Term will be imported: The Arguments are: %s " %
(sys.argv))
pv.main()
#update purview
logging.warning(
"Terms will be imported into the catalog. The arguments are:")
logging.warning(sys.argv)
try:
f = io.StringIO()
with redirect_stdout(f):
pv.main()
logging.warning("Results of the glossary update are:")
logging.warning(f.getvalue())
#add some logging for debugging purposes
logging.warning("payload length is : %s" % (len(payload_file)))
except AttributeError as error:
# Output expected AttributeErrors.
logging.error(
"failed to update Azure Purview because of an attribute error")
logging.exception(error)
logging.error(sys.exc_info()[0])
except:
logging.error("failed to update Azure Purview")
logging.error(sys.exc_info()[0])
logging.error("Results of the glossary update are:")
logging.error(f.getvalue())
#add some logging for debugging purposes
logging.warning("payload length is : %s" % (len(payload_file)))
logging.exception("The exception is:")
def main(wbMsg: func.QueueMessage, rcaMsg: func.Out[typing.List[str]]) -> None:
try:
payload = wbMsg.get_json()
clp_id = payload['clp_id']
workbench_record = payload['workbench_record']
workbench_id = workbench_record['workbenchId']
token = utils.find_token_by_clp(clp_id, API_TOKENS)
if not token:
raise GeneralException(f'Token not found for clp: {clp_id}')
# get workbench detail
workbench_detail = get_workbench_detail(token, workbench_id)
# transform data
customized_workbench_json = customize_json(
clp_id, workbench_detail, workbench_record
)
# send to log analytics
log_analytics = LogAnalytics(WORKSPACE_ID, WORKSPACE_KEY, WB_LOG_TYPE)
log_analytics.post_data(customized_workbench_json)
logging.info(f'Send workbench data successfully. Workbench id: {workbench_id}.')
rca_tasks = []
rac_task_log = []
# get rca task
rca_raw_tasks = get_rca_task(token, workbench_id,)
for task in rca_raw_tasks:
task_status = task['status']
if task_status != 'PROCESS_COMPLETE':
logging.warning(
f'Get rca task with status: {task_status}, Workbench id: {workbench_id}. No need to get rca detail.'
)
continue
# process prca task info
rac_task_log.append(transform_utils.transform_rca_task(clp_id, workbench_id ,task))
for target in task['targets']:
target_status = target['targetStatus']
if target_status != 'PROCESS_COMPLETE':
logging.warning(
f'Get rca target with status: {target_status}, Workbench id: {workbench_id}. No need to get rca detail.'
)
continue
target_info = target.copy()
target_info.pop('targetStatus')
rca_tasks.append(
build_queue_message(
clp_id, workbench_id, task['id'], task['name'], target['guid'], target_info
)
)
if len(rac_task_log) > 0:
log_analytics = LogAnalytics(WORKSPACE_ID, WORKSPACE_KEY, RCA_TASK_LOG_TYPE)
log_analytics.post_data(rac_task_log)
logging.info(f'Send prca task data successfully. Workbench id: {workbench_id}, Count: {len(rac_task_log)}.')
if rca_tasks:
rcaMsg.set(rca_tasks)
except HTTPError as e:
logging.exception(
f'Fail to get workbench detail! Exception: {e}',
)
raise
except:
logging.exception('Internal error.')
raise
