def reader(url):
#DIRECTLY PULL DATA FROM GITHUB WITH REQUESTS
resp = requests.get(url).text
#WRITE TO CSV FILE
with open("github.csv", "w") as f:
f.write(resp)
#READ DATA DIRECTLY INTO DATAFRAME
df = pd.read_csv("github.csv")
print(f"\nThe length of the dataframe is {len(df)} rows")
return df
def get_data():
"""Helper for getting COVID cases map data"""
headers = {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/80.0.3987.163 Safari/537.36',
'origin': 'https://www.arcgis.com', 'referer': 'https://www.arcgis.com/apps/opsdashboard/index.html'}
a = requests.get('https://services9.arcgis.com/N9p5hsImWXAccRNI/arcgis/rest/services/Nc2JKvYFoAEOFCG5JSI6/'
'FeatureServer/1/query?f=json&where=1%3D1&returnGeometry=true&'
'spatialRel=esriSpatialRelIntersects&maxAllowableOffset=39135&'
'geometryType=esriGeometryEnvelope&inSR=102100&outFields=*&outSR=102100&resultType=tile',
headers=headers)
return a.status_code, a.json()
def get_data(url):
try:
res = requests.get(url)
res.raise_for_status()
return json.loads(res.text)
except requests.exceptions.HTTPError as eh:
print("HTTPError:", eh)
except requests.exceptions.ConnectionError as ec:
print("ConnectionError:", ec)
except requests.exceptions.Timeout as et:
print("Timeout:", et)
except requests.exceptions.RequestException as er:
print("RequestException:", er)
def lambda_handler(event, context):
print(event)
object_get_context = event["getObjectContext"]
request_route = object_get_context["outputRoute"]
request_token = object_get_context["outputToken"]
s3_url = object_get_context["inputS3Url"]
# Get object from S3
response = requests.get(s3_url)
original_object = response.content.decode('utf-8')
# Transform object
transformed_object = original_object.upper()
# Write object back to S3 Object Lambda
s3 = boto3.client('s3')
s3.write_get_object_response(Body=transformed_object,
RequestRoute=request_route,
RequestToken=request_token)
return {'status_code': 200}
import csv
import sys
from datetime import datetime
from shutil import copyfile
sys.path.append('/home/a4iv1kv14b88/covid_map/modules')
from modules import requests
NOW = datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
response = requests.get('https://api.findcovidtesting.com/api/v1/location')
if not response.status_code == 200:
exit(1)
fields = max([list(i.keys()) for i in response.json()])
with open('/home/a4iv1kv14b88/covid_map/outputs/venues_{}.csv'.format(NOW),
'w') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fields)
writer.writeheader()
writer.writerows(response.json())
copyfile(
"/home/a4iv1kv14b88/covid_map/outputs/venues_{}.csv".format(NOW),
"/home/a4iv1kv14b88/genasys.com/public_html/wp-content/uploads/venues.csv")