def handle(req): """handle a request to the function Args: req (str): request body """ with open("/var/openfaas/secrets/ftp-prefix", "r") as secret: ftp_prefix = (secret.read()) with open("/var/openfaas/secrets/recipe-engine", "r") as secret: recipe_engine = (secret.read()) config = json.loads(req) config['version_name'] = datetime.datetime.today().strftime("%Y/%m/%d")\ if config['version_name'] == '' \ else config['version_name'] archiver = Archiver(engine=recipe_engine, ftp_prefix=ftp_prefix) try: archiver.archive_table(config) log_config(config, recipe_engine) return {"status": "success"} except: return { "status": "failure", "error": f"{sys.exc_info()}", }
def test_ftp_ziped_shp(): from cook import Archiver import os engine = os.environ.get('RECIPE_ENGINE', '') ftp_prefix = os.environ.get('FTP_PREFIX', '') archiver = Archiver(engine=engine, ftp_prefix=ftp_prefix) archiver.archive_table( config={ 'schema_name': 'test', 'version_name': 'ziped_shp', "path": "ftp://agencySourceData/dcp/DCP_Quartersection_Index.zip", 'geometryType': 'MULTIPOLYGON', 'srcSRS': 'EPSG:2263', 'dstSRS': 'EPSG:4326', 'layerCreationOptions': ['OVERWRITE=YES', 'PRECISION=NO'], 'srcOpenOptions': [] })
def test_geojson(): from cook import Archiver import os engine = os.environ.get('RECIPE_ENGINE', '') ftp_prefix = os.environ.get('FTP_PREFIX', '') archiver = Archiver(engine=engine, ftp_prefix=ftp_prefix) archiver.archive_table( config={ 'schema_name': 'test', 'version_name': 'opendata_geojson', 'path': 'https://data.cityofnewyork.us/api/geospatial/k2ya-ucmv?method=export&format=GeoJSON', 'geometryType': 'MULTIPOLYGON', 'srcSRS': 'EPSG:4326', 'dstSRS': 'EPSG:4326', 'layerCreationOptions': ['OVERWRITE=YES'], })
def test_csv(): from cook import Archiver import os engine = os.environ.get('RECIPE_ENGINE', '') ftp_prefix = os.environ.get('FTP_PREFIX', '') archiver = Archiver(engine=engine, ftp_prefix=ftp_prefix) archiver.archive_table( config={ 'schema_name': 'test', 'version_name': 'opendata_csv', 'path': 'https://data.cityofnewyork.us/api/views/n5mv-nfpy/rows.csv', 'geometryType': 'POINT', 'srcSRS': 'EPSG:4326', 'dstSRS': 'EPSG:4326', 'layerCreationOptions': ['OVERWRITE=YES'], 'srcOpenOptions': [ 'AUTODETECT_TYPE=NO', 'EMPTY_STRING_AS_NULL=YES', 'GEOM_POSSIBLE_NAMES=*geom*', 'X_POSSIBLE_NAMES=longitude,x', 'Y_POSSIBLE_NAMES=latitude,y' ], 'newFieldNames': [ 'BOROUGH', 'BLOCK', 'LOT', 'PARCEL_NAME', 'PARCEL_ADDRESS', 'JURIS', 'JurisDescription', 'RPAD', 'RPAD_DESCRIPTION', 'PROP_FRONT', 'PROP_DEPTH', 'PROP_SQFT', 'IRREG', 'BLD_FRONT', 'BLD_DEPTH', 'BLD_SQFT', 'NUM_BLD', 'FLOORS', 'CommunityBoard', 'COUNCILDISTRICT', 'COUNCILMEMBER_NAME', 'PR_ZONE', 'OV_ZONE', 'SD_ZONE', 'BBL', 'WATERFRONT', 'URBANRENEWALSITE', 'Agency', 'Owned_Leased', 'PrimaryUse', 'FinalCommitment', 'Agreement_Lease_Out', 'Postcode', 'Latitude', 'Longitude', 'CensusTract', 'BIN', 'NTA' ], 'SQLStatement': 'SELECT * FROM dcas_ipis LIMIT 5' })
def test_ziped_csv(): from cook import Archiver import os engine = os.environ.get('RECIPE_ENGINE', '') ftp_prefix = os.environ.get('FTP_PREFIX', '') archiver = Archiver(engine=engine, ftp_prefix=ftp_prefix) archiver.archive_table( config={ 'schema_name': 'test', 'version_name': 'ziped_csv', "path": "https://www1.nyc.gov/assets/planning/download/zip/data-maps/open-data/nyczoningtaxlotdb_20190731.zip/NY_ZoningTaxLotDB_20190731.csv", 'geometryType': 'POINT', 'srcSRS': 'EPSG:2263', 'dstSRS': 'EPSG:4326', 'layerCreationOptions': ['OVERWRITE=YES', 'PRECISION=NO'], 'srcOpenOptions': [] })
def run_recipes(recipe): try: if recipe in sh_recipes: os.system(f"bash {Path(__file__).parent}/recipe_info/{recipe}.sh") elif recipe in py_recipes: os.system(f"python3 {Path(__file__).parent}/recipe_info/{recipe}.py") else: convert_recipe(recipe) recipe_config = load_recipe_json(recipe) archiver = Archiver( engine=os.environ["RECIPE_ENGINE"], ftp_prefix=os.environ["FTP_PREFIX"], s3_endpoint=os.environ.get("AWS_S3_ENDPOINT", "").replace( "https://", "" ), s3_secret_access_key=os.environ.get("AWS_SECRET_ACCESS_KEY", ""), s3_access_key_id=os.environ.get("AWS_ACCESS_KEY_ID", ""), ) archiver.archive_table(recipe_config) # update_metatable() except KeyError: click.secho("\n Did you set your RECIPE_ENGINE and FTP_PREFIX? \n", fg="red")
def test_s3_csv(): from cook import Archiver import os engine = os.environ.get('RECIPE_ENGINE', '') ftp_prefix = os.environ.get('FTP_PREFIX', '') s3_endpoint = os.environ.get('AWS_S3_ENDPOINT', '') s3_secret_access_key = os.environ.get('AWS_SECRET_ACCESS_KEY', '') s3_access_key_id = os.environ.get('AWS_ACCESS_KEY_ID', '') archiver = Archiver(engine=engine, ftp_prefix=ftp_prefix, s3_endpoint=s3_endpoint, s3_secret_access_key=s3_secret_access_key, s3_access_key_id=s3_access_key_id) archiver.archive_table( config={ 'schema_name': 'test', 'version_name': 's3_csv', 'path': 's3://edm-recipes/2019-11-14/recipes.csv', 'geometryType': 'NONE' })
def test_s3_zip(): from cook import Archiver import os engine = os.environ.get('RECIPE_ENGINE', '') ftp_prefix = os.environ.get('FTP_PREFIX', '') s3_endpoint = os.environ.get('AWS_S3_ENDPOINT', '') s3_secret_access_key = os.environ.get('AWS_SECRET_ACCESS_KEY', '') s3_access_key_id = os.environ.get('AWS_ACCESS_KEY_ID', '') archiver = Archiver(engine=engine, ftp_prefix=ftp_prefix, s3_endpoint=s3_endpoint, s3_secret_access_key=s3_secret_access_key, s3_access_key_id=s3_access_key_id) archiver.archive_table( config={ 'schema_name': 'test', 'version_name': 's3_zip', 'path': 's3://edm-storage/ZONING_FEATURES/commercial_overlays/20190830/commercial_overlays.zip', 'geometryType': 'NONE' })
if key in ["Zip Code", "Zipcode"]: key = "zipcode" if "hour" in key.lower(): key = "hour" result[key] = value data.append(result) results += data df = pd.DataFrame(results).drop_duplicates().reset_index() df = df.drop(columns=["index"]) output_path = f"recipes/facdb/{table_name}.csv" df.to_csv(output_path) recipe_config = { "dstSRS": "EPSG:4326", "srcSRS": "EPSG:4326", "schema_name": table_name, "version_name": "", "geometryType": "NONE", "layerCreationOptions": ["OVERWRITE=YES", "PRECISION=NO"], "metaInfo": "https://www1.nyc.gov/site/hra/locations/locations.page", "path": output_path, "srcOpenOptions": ["AUTODETECT_TYPE=NO", "EMPTY_STRING_AS_NULL=YES"], "newFieldNames": [], } archiver = Archiver( engine=os.environ["RECIPE_ENGINE"], ftp_prefix=os.environ["FTP_PREFIX"] ) archiver.archive_table(recipe_config)
from sqlalchemy import create_engine from datetime import datetime from ast import literal_eval from cook import Archiver import streamlit as st import json import boto3 import os conn = create_engine(os.environ.get("RECIPE_ENGINE")) archiver = Archiver( engine=os.environ["RECIPE_ENGINE"], ftp_prefix=os.environ["FTP_PREFIX"], s3_endpoint=os.environ.get("AWS_S3_ENDPOINT", "").replace("https://", ""), s3_secret_access_key=os.environ.get("AWS_SECRET_ACCESS_KEY", ""), s3_access_key_id=os.environ.get("AWS_ACCESS_KEY_ID", ""), ) session = boto3.session.Session() client = session.client( "s3", region_name="nyc3", endpoint_url=os.environ.get("AWS_S3_ENDPOINT", ""), aws_access_key_id=os.environ.get("AWS_ACCESS_KEY_ID", ""), aws_secret_access_key=os.environ.get("AWS_SECRET_ACCESS_KEY", ""), ) def get_schema(): schemas = conn.execute( "select distinct schema_name from meta.metadata").fetchall() return [dict(row)["schema_name"] for row in schemas]