def ok(
data: Union[BaseResponse, Sequence[BaseResponse], ModelMixin,
Sequence[ModelMixin]]
) -> HttpResponse:
if isinstance(data, BaseResponse):
return HttpResponse(data.json(exclude_none=True),
mimetype="application/json")
if isinstance(data, list) and len(data) > 0 and isinstance(
data[0], BaseResponse):
decoded = [json.loads(x.json(exclude_none=True)) for x in data]
return HttpResponse(json.dumps(decoded), mimetype="application/json")
if isinstance(data, ModelMixin):
return HttpResponse(
data.json(exclude_none=True, exclude=data.export_exclude()),
mimetype="application/json",
)
decoded = [
x.raw(exclude_none=True, exclude=x.export_exclude()) if isinstance(
x, ModelMixin) else x for x in data
]
return HttpResponse(
json.dumps(decoded),
mimetype="application/json",
)
async def process_modals(req: HttpRequest) -> HttpResponse:
"""
Modal processor (main entry point).
Processes markdown files and substitutes the placeholders.
Parameters
----------
req: HttpRequest
HTTP trigger request (web hook).
Returns
-------
HttpResponse
Response to the HTTP request (200 if successful, otherwise 500).
"""
logging.info(
f"--- Web hook has triggered the function. Starting the process...")
event_loop = get_event_loop()
try:
task = process_github_files(repository_url=REPOSITORY_ARCHIVE_URL,
callback=process_and_upload_data,
excluded=EXCLUDED_ITEMS,
base_path=REPOSITORY_BASE_PATH,
container=STORAGE_CONTAINER)
event_loop.create_task(task)
except Exception as err:
logging.exception(err)
return HttpResponse(status_code=int(HTTPStatus.INTERNAL_SERVER_ERROR))
return HttpResponse(status_code=int(HTTPStatus.OK))
def main(req: HttpRequest) -> HttpResponse:
logging.info(
'Python HTTP trigger function processed a request to parse text for relevent information.'
)
body_to_return = {}
text_from_image = None
try:
req_body = req.get_body()
if req_body is None or req_body == b'':
raise ValueError()
except ValueError:
return HttpResponse(json.dumps({"Error": "No body"}), status_code=404)
else:
try:
tweet, image_url = parse_tweet_url_from_body(
req_body.decode('utf-8'))
cities_referenced = fetch_city_info(tweet)
resources_referenced = fetch_resources_info(tweet)
contact_details = fetch_contact_info(tweet)
if image_url is not None and _is_valid_url(image_url):
text_from_image = process_image_url_to_get_text(
image_url=image_url)
cities_referenced_in_image = fetch_city_info(
' '.join(text_from_image))
resources_referenced_in_image = fetch_resources_info(
' '.join(text_from_image))
contact_details_in_image = fetch_contact_info(
' '.join(text_from_image))
except Exception as e:
logging.error(e)
return HttpResponse(json.dumps({"Error": "No tweet"}),
status_code=404)
body_to_return = {
"cities":
str(cities_referenced),
"resources_referenced":
str(resources_referenced),
"contact_details":
str(contact_details),
"text_from_image":
text_from_image if text_from_image else "",
"cities_from_image":
cities_referenced_in_image if text_from_image else "",
"resources_referenced_from_image":
resources_referenced_in_image if text_from_image else "",
"contact_details_from_image":
contact_details_in_image if text_from_image else ""
}
return HttpResponse(json.dumps(body_to_return), status_code=200)
def main(req: HttpRequest) -> HttpResponse:
ds_secret_key = environ.get(ENV_VAR_KEY_SECRET)
if not ds_secret_key:
return HttpResponse('Could not load Dark Sky API secret', status_code=500)
ds_location = environ.get(ENV_VAR_KEY_LOCATION)
if not ds_location:
return HttpResponse('Could not load location key', status_code=500)
r = httpx.get(f'https://api.darksky.net/forecast/{ds_secret_key}/{ds_location}?exclude=minutely')
if not r.status_code == 200:
return HttpResponse(f'Got status code {r.status_code} from Dark Sky API', status_code=500)
return HttpResponse(json.dumps(build_response_json(r.json())), mimetype='application/json')
def inner(req: HttpRequest, lastUpdateTimestamp: str) -> HttpResponse:
code, response, raw_query, formatter = func(req, lastUpdateTimestamp)
if len(lastUpdateTimestamp.split('.')[-1].strip("Z")) == 7:
lastUpdateTimestamp = lastUpdateTimestamp.replace("0Z", "").replace("5Z", "") + "Z"
data = response
if isinstance(response, (list, dict)):
data = dumps(response, separators=(",", ":"))
gzipped_data = compress(data.encode())
timestamp = str()
if lastUpdateTimestamp:
timestamp = datetime.strptime(lastUpdateTimestamp, "%Y-%m-%dT%H:%M:%S.%fZ")
headers = {
"Content-Type": instance_settings.RESPONSE_TYPE[formatter],
**UNIVERSAL_HEADERS
}
if 'csv' in instance_settings.RESPONSE_TYPE[formatter] and len(data):
headers['Content-Disposition'] = (
f"attachment;filename=data_{timestamp.strftime('%Y-%b-%d')}.csv"
)
headers['X-Content-Type-Options'] = 'nosniff'
del headers["Content-Type"]
if code < 400:
headers.update({
"Cache-Control": "public, max-age=60",
"Content-Location": f"/v1/data?{raw_query}"
})
if timestamp:
headers.update({
"Last-Modified": timestamp.strftime("%a, %d %b %Y %H:%M:%S GMT")
})
if req.method == "HEAD":
return HttpResponse(
status_code=204 if code < 400 else int(code),
headers=headers
)
return HttpResponse(
body=gzipped_data,
status_code=int(code),
headers=headers
)
async def process_layouts(req: HttpRequest) -> HttpResponse:
logging.info(f"--- Web hook has triggered the function. Starting the process...")
event_loop = get_event_loop()
try:
task = process_github_files(
repository_url=REPOSITORY_ARCHIVE_URL,
callback=process_and_upload_data,
excluded=EXCLUDED_ITEMS,
base_path=REPOSITORY_BASE_PATH,
container=STORAGE_CONTAINER
)
event_loop.create_task(task)
except Exception as err:
logging.exception(err)
return HttpResponse(status_code=int(HTTPStatus.INTERNAL_SERVER_ERROR))
return HttpResponse(status_code=int(HTTPStatus.OK))
async def main(req: HttpRequest) -> HttpResponse:
body = req.get_body()
data = loads(body)
file_name = data.get("fileName")
if file_name is None:
return HttpResponse(None, status_code=400)
hashable_filename = str.join("", file_pattern.search(file_name).groups())
instance_id = blake2b(hashable_filename.encode()).hexdigest()
try:
register_file(filepath=file_name,
timestamp=datetime.utcnow(),
instance_id=instance_id)
except IntegrityError:
return HttpResponse(None, status_code=403)
response = dumps({"instance_id": instance_id})
return HttpResponse(response,
status_code=200,
headers={"content-type": "application/json"})
def not_ok(error: Error,
*,
status_code: int = 400,
context: Union[str, UUID]) -> HttpResponse:
if 400 <= status_code and status_code <= 599:
logging.error("request error - %s: %s" % (str(context), error.json()))
return HttpResponse(error.json(),
status_code=status_code,
mimetype="application/json")
else:
raise Exception(
"status code %s is not int the expected range [400; 599]" %
status_code)
def main(req: HttpRequest) -> HttpResponse:
logging.info("Running health checks")
logging.info("Testing database connection and response")
session = Session()
conn = session.connection()
try:
q = conn.execute(text("SELECT 1 AS result;"))
_ = q.fetchone()
logging.info("Database is alive")
except Exception as err:
session.rollback()
logging.warning("Database is dead")
raise err
finally:
session.close()
storage_kws = dict(
container="pipeline",
path="healthchecks/etl",
content_type="text/plain",
compressed=False
)
logging.info("Testing storage connection")
try:
with StorageClient(**storage_kws) as storage:
storage.upload("test")
storage.download()
storage.delete()
logging.info("Storage is alive")
except Exception as err:
logging.warning("Storage is dead")
raise err
logging.info("All tests passed.")
return HttpResponse(b"ALIVE", status_code=200)
def response(self) -> HttpResponse:
return HttpResponse(
body=self.body,
status_code=self.status_code,
headers=dict(self.headers),
)
def get_json(response: func.HttpResponse) -> Dict:
"""Get JSON from an HttpResponse."""
return json.loads(response.get_body().decode("utf-8"))
def main(req: HttpRequest) -> HttpResponse:
logging.info('Python HTTP trigger twittercode-daily started')
code = get_daily_code()
logging.info(f"Today's code was {code} !")
return HttpResponse(code, status_code=200, mimetype="application/json")
def response_to_body(self, response: func.HttpResponse):
return [response.get_body()]
def redirect(location: str) -> HttpResponse:
return HttpResponse(status_code=302, headers={"Location": location})
