def handle(req):
"""handle a request to the function
Args:
req (str): request body
"""
with open("/var/openfaas/secrets/ftp-prefix", "r") as secret:
ftp_prefix = (secret.read())
with open("/var/openfaas/secrets/recipe-engine", "r") as secret:
recipe_engine = (secret.read())
config = json.loads(req)
config['version_name'] = datetime.datetime.today().strftime("%Y/%m/%d")\
if config['version_name'] == '' \
else config['version_name']
archiver = Archiver(engine=recipe_engine, ftp_prefix=ftp_prefix)
try:
archiver.archive_table(config)
log_config(config, recipe_engine)
return {"status": "success"}
except:
return {
"status": "failure",
"error": f"{sys.exc_info()}",
}
def test_ftp_ziped_shp():
from cook import Archiver
import os
engine = os.environ.get('RECIPE_ENGINE', '')
ftp_prefix = os.environ.get('FTP_PREFIX', '')
archiver = Archiver(engine=engine, ftp_prefix=ftp_prefix)
archiver.archive_table(
config={
'schema_name': 'test',
'version_name': 'ziped_shp',
"path": "ftp://agencySourceData/dcp/DCP_Quartersection_Index.zip",
'geometryType': 'MULTIPOLYGON',
'srcSRS': 'EPSG:2263',
'dstSRS': 'EPSG:4326',
'layerCreationOptions': ['OVERWRITE=YES', 'PRECISION=NO'],
'srcOpenOptions': []
})
def test_geojson():
from cook import Archiver
import os
engine = os.environ.get('RECIPE_ENGINE', '')
ftp_prefix = os.environ.get('FTP_PREFIX', '')
archiver = Archiver(engine=engine, ftp_prefix=ftp_prefix)
archiver.archive_table(
config={
'schema_name': 'test',
'version_name': 'opendata_geojson',
'path':
'https://data.cityofnewyork.us/api/geospatial/k2ya-ucmv?method=export&format=GeoJSON',
'geometryType': 'MULTIPOLYGON',
'srcSRS': 'EPSG:4326',
'dstSRS': 'EPSG:4326',
'layerCreationOptions': ['OVERWRITE=YES'],
})
def test_csv():
from cook import Archiver
import os
engine = os.environ.get('RECIPE_ENGINE', '')
ftp_prefix = os.environ.get('FTP_PREFIX', '')
archiver = Archiver(engine=engine, ftp_prefix=ftp_prefix)
archiver.archive_table(
config={
'schema_name':
'test',
'version_name':
'opendata_csv',
'path':
'https://data.cityofnewyork.us/api/views/n5mv-nfpy/rows.csv',
'geometryType':
'POINT',
'srcSRS':
'EPSG:4326',
'dstSRS':
'EPSG:4326',
'layerCreationOptions': ['OVERWRITE=YES'],
'srcOpenOptions': [
'AUTODETECT_TYPE=NO', 'EMPTY_STRING_AS_NULL=YES',
'GEOM_POSSIBLE_NAMES=*geom*', 'X_POSSIBLE_NAMES=longitude,x',
'Y_POSSIBLE_NAMES=latitude,y'
],
'newFieldNames': [
'BOROUGH', 'BLOCK', 'LOT', 'PARCEL_NAME', 'PARCEL_ADDRESS',
'JURIS', 'JurisDescription', 'RPAD', 'RPAD_DESCRIPTION',
'PROP_FRONT', 'PROP_DEPTH', 'PROP_SQFT', 'IRREG', 'BLD_FRONT',
'BLD_DEPTH', 'BLD_SQFT', 'NUM_BLD', 'FLOORS', 'CommunityBoard',
'COUNCILDISTRICT', 'COUNCILMEMBER_NAME', 'PR_ZONE', 'OV_ZONE',
'SD_ZONE', 'BBL', 'WATERFRONT', 'URBANRENEWALSITE', 'Agency',
'Owned_Leased', 'PrimaryUse', 'FinalCommitment',
'Agreement_Lease_Out', 'Postcode', 'Latitude', 'Longitude',
'CensusTract', 'BIN', 'NTA'
],
'SQLStatement':
'SELECT * FROM dcas_ipis LIMIT 5'
})
def test_ziped_csv():
from cook import Archiver
import os
engine = os.environ.get('RECIPE_ENGINE', '')
ftp_prefix = os.environ.get('FTP_PREFIX', '')
archiver = Archiver(engine=engine, ftp_prefix=ftp_prefix)
archiver.archive_table(
config={
'schema_name': 'test',
'version_name': 'ziped_csv',
"path":
"https://www1.nyc.gov/assets/planning/download/zip/data-maps/open-data/nyczoningtaxlotdb_20190731.zip/NY_ZoningTaxLotDB_20190731.csv",
'geometryType': 'POINT',
'srcSRS': 'EPSG:2263',
'dstSRS': 'EPSG:4326',
'layerCreationOptions': ['OVERWRITE=YES', 'PRECISION=NO'],
'srcOpenOptions': []
})
def run_recipes(recipe):
try:
if recipe in sh_recipes:
os.system(f"bash {Path(__file__).parent}/recipe_info/{recipe}.sh")
elif recipe in py_recipes:
os.system(f"python3 {Path(__file__).parent}/recipe_info/{recipe}.py")
else:
convert_recipe(recipe)
recipe_config = load_recipe_json(recipe)
archiver = Archiver(
engine=os.environ["RECIPE_ENGINE"],
ftp_prefix=os.environ["FTP_PREFIX"],
s3_endpoint=os.environ.get("AWS_S3_ENDPOINT", "").replace(
"https://", ""
),
s3_secret_access_key=os.environ.get("AWS_SECRET_ACCESS_KEY", ""),
s3_access_key_id=os.environ.get("AWS_ACCESS_KEY_ID", ""),
)
archiver.archive_table(recipe_config)
# update_metatable()
except KeyError:
click.secho("\n Did you set your RECIPE_ENGINE and FTP_PREFIX? \n", fg="red")
def test_s3_csv():
from cook import Archiver
import os
engine = os.environ.get('RECIPE_ENGINE', '')
ftp_prefix = os.environ.get('FTP_PREFIX', '')
s3_endpoint = os.environ.get('AWS_S3_ENDPOINT', '')
s3_secret_access_key = os.environ.get('AWS_SECRET_ACCESS_KEY', '')
s3_access_key_id = os.environ.get('AWS_ACCESS_KEY_ID', '')
archiver = Archiver(engine=engine,
ftp_prefix=ftp_prefix,
s3_endpoint=s3_endpoint,
s3_secret_access_key=s3_secret_access_key,
s3_access_key_id=s3_access_key_id)
archiver.archive_table(
config={
'schema_name': 'test',
'version_name': 's3_csv',
'path': 's3://edm-recipes/2019-11-14/recipes.csv',
'geometryType': 'NONE'
})
def test_s3_zip():
from cook import Archiver
import os
engine = os.environ.get('RECIPE_ENGINE', '')
ftp_prefix = os.environ.get('FTP_PREFIX', '')
s3_endpoint = os.environ.get('AWS_S3_ENDPOINT', '')
s3_secret_access_key = os.environ.get('AWS_SECRET_ACCESS_KEY', '')
s3_access_key_id = os.environ.get('AWS_ACCESS_KEY_ID', '')
archiver = Archiver(engine=engine,
ftp_prefix=ftp_prefix,
s3_endpoint=s3_endpoint,
s3_secret_access_key=s3_secret_access_key,
s3_access_key_id=s3_access_key_id)
archiver.archive_table(
config={
'schema_name': 'test',
'version_name': 's3_zip',
'path':
's3://edm-storage/ZONING_FEATURES/commercial_overlays/20190830/commercial_overlays.zip',
'geometryType': 'NONE'
})
if key in ["Zip Code", "Zipcode"]:
key = "zipcode"
if "hour" in key.lower():
key = "hour"
result[key] = value
data.append(result)
results += data
df = pd.DataFrame(results).drop_duplicates().reset_index()
df = df.drop(columns=["index"])
output_path = f"recipes/facdb/{table_name}.csv"
df.to_csv(output_path)
recipe_config = {
"dstSRS": "EPSG:4326",
"srcSRS": "EPSG:4326",
"schema_name": table_name,
"version_name": "",
"geometryType": "NONE",
"layerCreationOptions": ["OVERWRITE=YES", "PRECISION=NO"],
"metaInfo": "https://www1.nyc.gov/site/hra/locations/locations.page",
"path": output_path,
"srcOpenOptions": ["AUTODETECT_TYPE=NO", "EMPTY_STRING_AS_NULL=YES"],
"newFieldNames": [],
}
archiver = Archiver(
engine=os.environ["RECIPE_ENGINE"], ftp_prefix=os.environ["FTP_PREFIX"]
)
archiver.archive_table(recipe_config)
from sqlalchemy import create_engine
from datetime import datetime
from ast import literal_eval
from cook import Archiver
import streamlit as st
import json
import boto3
import os
conn = create_engine(os.environ.get("RECIPE_ENGINE"))
archiver = Archiver(
engine=os.environ["RECIPE_ENGINE"],
ftp_prefix=os.environ["FTP_PREFIX"],
s3_endpoint=os.environ.get("AWS_S3_ENDPOINT", "").replace("https://", ""),
s3_secret_access_key=os.environ.get("AWS_SECRET_ACCESS_KEY", ""),
s3_access_key_id=os.environ.get("AWS_ACCESS_KEY_ID", ""),
)
session = boto3.session.Session()
client = session.client(
"s3",
region_name="nyc3",
endpoint_url=os.environ.get("AWS_S3_ENDPOINT", ""),
aws_access_key_id=os.environ.get("AWS_ACCESS_KEY_ID", ""),
aws_secret_access_key=os.environ.get("AWS_SECRET_ACCESS_KEY", ""),
)
def get_schema():
schemas = conn.execute(
"select distinct schema_name from meta.metadata").fetchall()
return [dict(row)["schema_name"] for row in schemas]