def main(req: func.HttpRequest,
doc: func.Out[func.Document]) -> func.HttpResponse:
logging.info('Python HTTP trigger function processed a request.')
name = req.route_params.get('name')
if not name:
try:
req_body = req.get_json()
except ValueError:
pass
else:
name = req_body.get('name')
if name:
newdocs = func.DocumentList()
newproduct_dict = {"id": str(uuid.uuid4()), "name": name}
newdocs.append(func.Document.from_dict(newproduct_dict))
doc.set(newdocs)
return func.HttpResponse(
f"Hello, {name}. This HTTP triggered function executed successfully."
)
else:
return func.HttpResponse(
"This HTTP triggered function executed successfully. Pass a name in the query string or in the request body for a personalized response.",
status_code=200)
def test_main_handler_should_generate_events_if_hidden_attrib_is_found():
out = OutImpl()
documents = func.DocumentList()
for i in range(10):
documents.append(func.Document.from_dict(generate_sample_document()))
for i in range(5):
documents.append(func.Document.from_dict(generate_sample_document(False)))
main_handler(documents, out)
assert len(out.get()) == 10
def main(mytimer: func.TimerRequest, azServicesOut: func.Out[func.Document],
azCapabilitiesOut: func.Out[func.Document]) -> str:
logging.basicConfig()
logging.getLogger().setLevel(logging.DEBUG)
utc_timestamp = datetime.datetime.utcnow().replace(
tzinfo=datetime.timezone.utc).isoformat()
timestamp = datetime.datetime.now()
if mytimer.past_due:
logging.warn('The timer is past due!')
logging.info('Python timer trigger function ran at %s', timestamp)
sl = AzGovProductAvailabilty()
# sl.initialize()
try:
svcList = sl.getServicesList()
svcDocs = func.DocumentList()
for service in svcList:
logging.info(service)
svcDocs.append(func.Document.from_dict(service))
azServicesOut.set(svcDocs)
capList = sl.getCapabilitiesList()
capDocs = func.DocumentList()
for cap in capList:
logging.info(cap)
capDocs.append(func.Document.from_dict(cap))
azCapabilitiesOut.set(capDocs)
except Exception as e:
logging.error('Error:' + e)
return 0
def main(myblob: func.InputStream, outputDoc: func.Out[func.DocumentList]):
logging.info(f"Python blob trigger function processed blob \n"
f"Name: {myblob.name}\n"
f"Blob Size: {myblob.length} bytes")
outdata = json.loads(myblob.read().decode('utf-8'))
download_time = outdata["download_time"]
gameweek = transformations.get_game_week(outdata["events"])
transformations.add_gw_and_download_time(outdata["elements"],
download_time, gameweek)
transformations.add_unique_id(outdata["elements"])
outputDoc.set(func.DocumentList(outdata["elements"]))
def main(documents: func.DocumentList) -> func.DocumentList:
logging.info(f"CosmosDB trigger executed!")
for doc in documents:
logging.info(f"Document: {doc.to_json()}")
returnDocs = []
for x in range(0, 3):
newDoc = func.Document.from_dict({"text": str(x), "foo": "bar"})
returnDocs.append(newDoc)
return func.DocumentList(returnDocs)
def main(documentsin: func.DocumentList,
documentsout: func.Out[func.Document]) -> str:
# Set DocumentList to update second collection with
returnDocs = func.DocumentList()
# Loop around documents in incoming DocumentList
for doc in documentsin:
logging.info(f"Document: {doc.to_json()}")
newDoc = func.Document.from_json(doc.to_json())
logging.info("newDoc: %s", newDoc)
returnDocs.append(newDoc)
documentsout.set(returnDocs)
def main(hourdocin: func.DocumentList,
hourdocout: func.Out[func.Document]) -> str:
# Set DocumentList to update second collection with
returnDocs = func.DocumentList()
# Loop around documents in incoming DocumentList, and update a value
for doc in hourdocin:
logging.info(f"Document: {doc.to_json()}")
newDoc = func.Document.from_json(doc.to_json())
newDoc["value"] = newDoc["value"] + 10
logging.info("newDoc: %s", newDoc.to_json())
returnDocs.append(newDoc)
hourdocout.set(returnDocs)
def main(inputblob: func.InputStream, doc: func.Out[func.Document],
outputblob: func.Out[str]):
request_json = json.load(inputblob)
text = request_json["Text"]
logging.info(f"Processing request from blob {inputblob.name}")
response_doc = aze.extract_entities([text], inputblob.name)
outputblob.set(json.dumps(response_doc))
doclist = func.DocumentList()
doclist.append(func.Document.from_dict(response_doc))
doc.set(doclist)
logging.info(f"extractEntities was executed successfully.")
def main(docs: func.DocumentList, outdoc: func.Out[func.Document]) -> str:
newdocs = func.DocumentList()
for doc in docs:
logging.info(doc.to_json())
## Process Something
clear_text = doc["text"]
encrypted_text = process_rot13(clear_text)
## Create a new doc (type:Dict)
newdoc_dict = {"name": doc["name"], "text": encrypted_text}
## Append the new doc to DocumentList for output
newdocs.append(func.Document.from_dict(newdoc_dict))
## Set the DocumentList to outdoc to store into CosmosDB using CosmosDB output binding
outdoc.set(newdocs)
def main(mytimer: func.TimerRequest,
azCosmosOut: func.Out[func.Document]) -> str:
logging.basicConfig()
logging.getLogger().setLevel(logging.DEBUG)
utc_timestamp = datetime.datetime.utcnow().replace(
tzinfo=datetime.timezone.utc).isoformat()
timestamp = datetime.datetime.now()
if mytimer.past_due:
logging.warn('The timer is past due!')
logging.info('Python timer trigger function ran at %s', timestamp)
sl = AuditScopeList()
#sl.initialize()
try:
#outdata = {
# "id": "Audit Scopes",
# "clouds": [
# sl.getAzureGovernmentJson(),
# sl.getAzurePublicJson()
# ]
#}
#outdoc.set(func.Document.from_json(json.dumps(outdata)))
# azPubOut.set(func.Document.from_json(json.dumps(sl.getAzurePublicJson())))
# azGovOut.set(func.Document.from_json(json.dumps(sl.getAzureGovernmentJson())))
cosmosArray = sl.getCosmosArray()
newdocs = func.DocumentList()
for service in cosmosArray:
logging.info(service)
newdocs.append(func.Document.from_dict(service))
azCosmosOut.set(newdocs)
except Exception as e:
logging.error('Error:' + e)
return 0
def main(req: func.HttpRequest,
cosmosDB: func.Out[func.Document]) -> func.HttpResponse:
logging.basicConfig()
logging.getLogger().setLevel(logging.DEBUG)
logging.info('Python HTTP trigger function processed a request.')
name = req.params.get('name')
if not name:
try:
req_body = req.get_json()
except ValueError:
pass
else:
name = req_body.get('name')
# ***************************************************************
sc = AuditScopeList()
av = AzGovProductAvailabilty()
dj = DataJoiner(sc, av)
m = dj.getJoinedData()
# logging.debug(json.dumps(m))
l = func.DocumentList()
for i in m.values():
logging.info(i)
l.append(func.Document.from_dict(i))
cosmosDB.set(l)
# ***************************************************************
if name:
return func.HttpResponse(
f"Hello, {name}. This HTTP triggered function executed successfully."
)
else:
return func.HttpResponse(
"This HTTP triggered function executed successfully. Pass a name in the query string or in the request body for a personalized response.",
status_code=200)
def main(mytimer: func.TimerRequest,
outputDocument: func.Out[func.Document]) -> None:
getNRTData = GetNRTDataFromDefinedSat()
logging.info("Delete container")
deleteResponse = getNRTData.removeContainer()
logging.info(deleteResponse)
getNRTData.getListofFiles()
jsonFilePath = getNRTData.getDataFromArchives()
if os.path.isfile(jsonFilePath):
logging.info("Database update STARTED!")
newDocs = func.DocumentList()
modifyFile = ModifyFile()
with open(jsonFilePath, "r") as f:
datastore = json.load(f)
for data in datastore:
newdoc_dict = modifyFile.forestFireDataStructure(data)
newDocs.append(func.Document.from_dict(newdoc_dict))
outputDocument.set(newDocs)
os.remove(jsonFilePath)
logging.info("Database update DONE!")
def main(reservationsBlob: func.InputStream, locationsBlob: func.InputStream,
document: func.Out[func.DocumentList]):
logging.info(f"Python blob trigger function processed blob \n"
f"Name: {reservationsBlob.name}\n"
f"Blob Size: {reservationsBlob.length} bytes")
reservations = parse_csv_blob(reservationsBlob, reservation_schema)
locations = parse_csv_blob(locationsBlob, location_schema)
for r in reservations:
# Denormalize each reservation by adding location information
location = next((l for l in locations if l["id"] == r["location_id"]),
None)
r["location_title"] = location["title"] if location else None
r["location_wgs84_polygon"] = location[
"wgs84_polygon"] if location else None
# Cosmos DB requires string id's
r["id"] = str(r["id"])
# Store all reservations in CosmosDB
document.set(func.DocumentList(reservations))
def main(documents: func.DocumentList, events: func.Out[func.Document]):
if documents:
new_events = func.DocumentList()
for doc in documents:
logging.info(doc.to_json())
event_context = doc.get("_last_event_ctx")
if event_context is not None:
new_events.append(
func.Document.from_dict({
"id":
str(uuid.uuid4()),
"date":
datetime.utcnow().isoformat(),
"user_id":
event_context.get("user_id"),
"action":
event_context.get("action"),
"description":
event_context.get("description"),
"item_id":
doc.get("id"),
"container_id":
event_context.get("container_id"),
"session_id":
event_context.get("session_id"),
"tenant_id":
event_context.get("tenant_id"),
}))
else:
logging.warning("- Not saved!")
if len(new_events):
events.set(new_events)
else:
logging.warning("No valid events were found!")
def main(mytimer: func.TimerRequest,
document: func.Out[func.Document]) -> None:
utc_timestamp = (datetime.datetime.utcnow().replace(
tzinfo=datetime.timezone.utc).isoformat())
get_vars()
# Read the secrets from the environment
client_id = os.environ.get("SWISSCOM_CLIENT_ID")
client_secret = os.environ.get("SWISSCOM_CLIENT_SECRET")
# See https://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html#backend-application-flow.
client = BackendApplicationClient(client_id=client_id)
oauth = OAuth2Session(client=client)
# Fetch an access token.
oauth.fetch_token(token_url=TOKEN_URL,
client_id=client_id,
client_secret=client_secret)
# Use the access token to query an endpoint.
resp = oauth.get(
"https://api.swisscom.com/layer/heatmaps/demo/grids/municipalities/261",
headers={"scs-version": "2"},
)
if not resp.ok:
logging.error("Failed to reach Swisscom API")
return
tiles = resp.json()["tiles"]
logging.info("Loaded %d tiles from Swisscom", len(tiles))
tile_density = {}
tile_ids = (tile["tileId"] for tile in tiles)
for chunk in partition(100, tile_ids):
resp = oauth.get(
"https://api.swisscom.com/layer/heatmaps/demo/heatmaps/dwell-density/daily/2020-03-28", # TODO this should load data for the previous day instead
params={"tiles": chunk},
headers={"scs-version": "2"},
)
if not resp.ok:
logging.error("Failed to reach Swisscom API: %s", resp.json())
continue
tile_density.update(
(tile["tileId"], tile["score"]) for tile in resp.json()["tiles"])
logging.info("Loaded densitiy for %d tiles from Swisscom",
len(tile_density))
documents = func.DocumentList()
for tile in tiles:
tile_id = tile["tileId"]
if tile_id not in tile_density:
continue
density = tile_density[tile_id]
location = {
"type": "Point",
"coordinates": [tile["ll"]["x"], tile["ll"]["y"]]
}
documents.append(
func.Document.from_dict({
"id": str(tile_id),
"tileId": tile_id,
"density": density,
"location": location
}))
document.set(documents)
logging.info("Finished outputting data")
def main(documents: func.DocumentList, outdoc: func.Out[func.Document]):
if documents:
logging.info('Total Documents: %s', str(len(documents)))
# Grab Env Variables
# Local Development = local.settings.json
# In Production = application settings in Azure
Cognitive_Endpoint = os.environ['Cognitive_Endpoint']
Cognitive_Key = os.environ['Cognitive_Key']
# SDK Auth Flow
credentials = CognitiveServicesCredentials(Cognitive_Key)
text_analytics = TextAnalyticsClient(endpoint=Cognitive_Endpoint,
credentials=credentials)
# Create Doc List to append each doc to
eventlist = func.DocumentList()
# Set Batch Variables
batch_size = 100
batch_job = []
computed_batches = []
# look through every issue, filter to issue creation and set variables
#check NER to avoid recursive loop
for documents_items in documents:
if len(documents_items["NER"]) == 0:
eventId = documents_items["id"]
IssueTitle = documents_items["issue_title"]
#Create dict to pass to cognitive service issue titles
doc_phrase_dict = {
"id": eventId,
"language": "en",
"text": IssueTitle
}
# Append to batch_job for batch call
batch_job.append(doc_phrase_dict)
# Call API every 100 docs or we reach the end of the document list
if len(batch_job) >= batch_size or eventId == documents[-1]["id"]:
# Pass Batch to SDK
response = text_analytics.key_phrases(documents=batch_job)
# Loop through each item and update the original document
for response_items in response.documents:
# Apply NER findings back to each document
computed_batches.append(response_items)
# Now we loop through the original list and join the NER cells back
for documents_items in documents:
if len(documents_items["NER"]) == 0:
for x in computed_batches:
if x.id == documents_items["id"]:
# Update NER and append to list
documents_items["NER"] = x.key_phrases
eventlist.append(func.Document.from_dict(documents_items))
# Delete item from computed batches
# HELP! Can't figure out!
# want to do something like: del computed_batches[x.id]
## Set the DocumentList to outdoc to store into CosmosDB using CosmosDB output binding
logging.info("Item Count: %s" % (len(eventlist)))
outdoc.set(eventlist)
def main(inputblob: func.InputStream, outputdb: func.Out[func.Document]):
logging.info(
'Data uploaded to raw Blob Storage, function processing a new request.'
)
t.sleep(45)
# Store output in variable Data as JSON
with gzip.open(inputblob) as lines:
data = [json.loads(i) for i in lines]
# Create Doc List for batch upload
NewEventList = func.DocumentList()
# Empty variables
NER = []
count = 0
#Log number of records
records_count = str(len(data))
logging.info(f'All Records: {records_count}')
# Format records for CosmosDB Insertion
for event in data:
# filter to only issue-related events
if "Issue" in event["type"]:
# Filter to 'New Issues' only
if event["payload"]["issue"]["comments"] == 0:
#Count # of new records in loop
count += 1
# Reshape Data + Store Record
new_eventrecord = {
"id": event["id"],
"issue_id": event["payload"]["issue"]["id"],
"issue_title": event["payload"]["issue"]["title"],
"issue_num": event["payload"]["issue"]["number"],
"repo_id": event["repo"]["id"],
"repo_name": event["repo"]["name"].rsplit('/', 1)[1],
"created_datetime": event["created_at"],
"lastupdated_datetime": event["created_at"],
"issue_url": event["payload"]["issue"]["html_url"],
"last_state": event["payload"]["action"],
"NER": NER,
"json": event
}
#Append Record to list of CosmosDB events for batch
NewEventList.append(func.Document.from_dict(new_eventrecord))
# If Issue != New, do nothing
else:
pass
#if event type != Issue, do nothing
else:
pass
t.sleep(5)
outputdb.set(NewEventList)
logging.info(
f'Total Record Count | New Issue Creation Records: {str(count)}')
