def get_detail(self, search_item, location_type, csv=False, output_dir=None, extra_param=None): """Retrieves location detail product data from the First Street Foundation API given a list of search_items and returns a list of Location Detail objects. Args: search_item (list/file): A First Street Foundation IDs, lat/lng pair, address, or a file of First Street Foundation IDs location_type (str): The location lookup type csv (bool): To output extracted data to a csv or not output_dir (str): The output directory to save the generated csvs extra_param (str): Extra parameter to be added to the url Returns: A list of Location Detail Raises: InvalidArgument: The location provided is empty TypeError: The location provided is not a string """ if not location_type: raise InvalidArgument("No loocation type provided: {}".format(location_type)) elif not isinstance(location_type, str): raise TypeError("location is not a string") # Get data from api and create objects api_datas = self.call_api(search_item, "location", "detail", location_type, extra_param=extra_param) if location_type == 'property': product = [LocationDetailProperty(api_data) for api_data in api_datas] elif location_type == 'neighborhood': product = [LocationDetailNeighborhood(api_data) for api_data in api_datas] elif location_type == 'city': product = [LocationDetailCity(api_data) for api_data in api_datas] elif location_type == 'zcta': product = [LocationDetailZcta(api_data) for api_data in api_datas] elif location_type == 'tract': product = [LocationDetailTract(api_data) for api_data in api_datas] elif location_type == 'county': product = [LocationDetailCounty(api_data) for api_data in api_datas] elif location_type == 'cd': product = [LocationDetailCd(api_data) for api_data in api_datas] elif location_type == 'state': product = [LocationDetailState(api_data) for api_data in api_datas] else: raise NotImplementedError if csv: csv_format.to_csv(product, "location", "detail", location_type, output_dir=output_dir) logging.info("Location Detail Data Ready.") return product
def get_summary(self, search_item, location_type, csv=False, output_dir=None, extra_param=None): """Retrieves historic summary product data from the First Street Foundation API given a list of search_items and returns a list of Historic Summary objects. Args: search_item (list/file): A First Street Foundation IDs, lat/lng pair, address, or a file of First Street Foundation IDs location_type (str): The location lookup type csv (bool): To output extracted data to a csv or not output_dir (str): The output directory to save the generated csvs extra_param (str): Extra parameter to be added to the url Returns: A list of Historic Summary Raises: InvalidArgument: The location provided is empty TypeError: The location provided is not a string """ if not location_type: raise InvalidArgument(location_type) elif not isinstance(location_type, str): raise TypeError("location is not a string") # Get data from api and create objects api_datas = self.call_api(search_item, "historic", "summary", location_type, extra_param=extra_param) product = [HistoricSummary(api_data) for api_data in api_datas] if csv: csv_format.to_csv(product, "historic", "summary", location_type, output_dir=output_dir) logging.info("Historic Summary Data Ready.") return product
def get_summary(self, search_items, location_type, csv=False, output_dir=None, extra_param=None): """Retrieves AAL summary product data from the First Street Foundation API given a list of search_items and returns a list of AAL Summary objects. Args: search_items (list/file): A First Street Foundation IDs, lat/lng pair, address, or a file of First Street Foundation IDs location_type (str): The location lookup type csv (bool): To output extracted data to a csv or not output_dir (str): The output directory to save the generated csvs extra_param (dict): Extra parameter to be added to the url Returns: A list of AAL Summary Raises: InvalidArgument: The location provided is empty TypeError: The location provided is not a string """ if not location_type: raise InvalidArgument(location_type) elif not isinstance(location_type, str): raise TypeError("location is not a string") # Get data from api and create objects if extra_param and "depths" in extra_param: extra_param["depths"] = ','.join(map(str, extra_param["depths"])) api_datas = self.call_api(search_items, "economic/aal", "summary", location_type, extra_param=extra_param) product = [] for api_data, fsid in api_datas: api_data["fsid"] = fsid if location_type == "property": product.append(AALSummaryProperty(api_data)) else: product.append(AALSummaryOther(api_data)) if csv: csv_format.to_csv(product, "economic_aal", "summary", location_type, output_dir=output_dir) logging.info("AAL Summary Data Ready.") return product
def get_events_by_location(self, search_item, location_type, csv=False, connection_limit=100, output_dir=None, extra_param=None): """Retrieves historic summary product data from the First Street Foundation API given a list of location search_items and returns a list of Historic Summary objects. Args: search_item (list/file): A First Street Foundation IDs, lat/lng pair, address, or a file of First Street Foundation IDs location_type (str): The location lookup type csv (bool): To output extracted data to a csv or not connection_limit (int): max number of connections to make output_dir (str): The output directory to save the generated csvs extra_param (str): Extra parameter to be added to the url Returns: A list of Historic Event Raises: InvalidArgument: The location provided is empty TypeError: The location provided is not a string """ if not location_type: raise InvalidArgument(location_type) elif not isinstance(location_type, str): raise TypeError("location is not a string") # Get data from api and create objects api_datas = self.call_api(search_item, "historic", "summary", location_type, connection_limit=connection_limit) summary = [HistoricSummary(api_data) for api_data in api_datas] search_item = list(set([event.get("eventId") for sum_hist in summary if sum_hist.historic for event in sum_hist.historic])) if search_item: api_datas_event = self.call_api(search_item, "historic", "event", None, connection_limit=connection_limit, extra_param=extra_param) else: api_datas_event = [{"eventId": None}] event = [HistoricEvent(api_data) for api_data in api_datas_event] if csv: csv_format.to_csv([summary, event], "historic", "summary_event", location_type, output_dir=output_dir) logging.info("Historic Summary Event Data Ready.") return [summary, event]
def get_details_by_location(self, search_item, location_type, csv=False, output_dir=None, extra_param=None): """Retrieves adaptation detail product data from the First Street Foundation API given a list of location search_items and returns a list of Adaptation Detail objects. Args: search_item (list/file): A First Street Foundation IDs, lat/lng pair, address, or a file of First Street Foundation IDs location_type (str): The location lookup type csv (bool): To output extracted data to a csv or not output_dir (str): The output directory to save the generated csvs extra_param (str): Extra parameter to be added to the url Returns: A list of list of Adaptation Summary and Adaptation Detail Raises: InvalidArgument: The location provided is empty TypeError: The location provided is not a string """ if not location_type: raise InvalidArgument(location_type) elif not isinstance(location_type, str): raise TypeError("location is not a string") # Get data from api and create objects api_datas_summary = self.call_api(search_item, "adaptation", "summary", location_type, extra_param=extra_param) summary = [AdaptationSummary(api_data) for api_data in api_datas_summary] search_items = list(set([adaptation for sum_adap in summary if sum_adap.adaptation for adaptation in sum_adap.adaptation])) if search_items: api_datas_detail = self.call_api(search_items, "adaptation", "detail", None, extra_param=extra_param) else: api_datas_detail = [{"adaptationId": None, "valid_id": False}] detail = [AdaptationDetail(api_data) for api_data in api_datas_detail] if csv: csv_format.to_csv([summary, detail], "adaptation", "summary_detail", location_type, output_dir=output_dir) logging.info("Adaptation Summary Detail Data Ready.") return [summary, detail]
def get_count(self, search_item, location_type, csv=False, connection_limit=100, output_dir=None, extra_param=None): """Retrieves probability count product data from the First Street Foundation API given a list of search_items and returns a list of Probability Count objects. Args: search_item (list/file): A First Street Foundation IDs, lat/lng pair, address, or a file of First Street Foundation IDs location_type (str): The location lookup type csv (bool): To output extracted data to a csv or not connection_limit (int): max number of connections to make output_dir (str): The output directory to save the generated csvs extra_param (str): Extra parameter to be added to the url Returns: A list of Probability Count Raises: InvalidArgument: The location provided is empty TypeError: The location provided is not a string """ if not location_type: raise InvalidArgument(location_type) elif not isinstance(location_type, str): raise TypeError("location is not a string") # Get data from api and create objects api_datas = self.call_api(search_item, "probability", "count", location_type, connection_limit=connection_limit, extra_param=extra_param) product = [ProbabilityCount(api_data) for api_data in api_datas] if csv: csv_format.to_csv(product, "probability", "count", location_type, output_dir=output_dir) logging.info("Probability Count Data Ready.") return product
def get_probability_depth(self, year, return_period, coordinate, image=False, output_dir=None, extra_param=None): """Retrieves probability depth tile data from the First Street Foundation API given a list of search_items and returns a list of Probability Depth Tile objects. Args: year (int): The year to get the tile return_period (int): The return period to get the tile coordinate (list of tuple): A list of coordinates in the form of [(x_1, y_1, z_1), (x_2, y_2, z_2), ...] image (bool): To output extracted image to a png or not output_dir (str): The output directory to save the generated tile extra_param (str): Extra parameter to be added to the url Returns: A list of Probability Depth tiles """ if not year: raise InvalidArgument(year) elif not isinstance(year, int): raise TypeError("year is not an int") elif year not in [2020, 2035, 2050]: logging.error("Year provided is not one of: 2020, 2035, 2050") raise InvalidArgument(year) if not return_period: raise InvalidArgument(return_period) elif not isinstance(return_period, int): raise TypeError("return period is not an int") elif return_period not in [500, 100, 20, 5, 2]: logging.error( "Year provided is not one of: 500, 100, 20, 5, 2. " "(2 year return period is only available for coastal areas.)") raise InvalidArgument(year) # Get data from api and create objects api_datas = self.call_api(coordinate, "tile", "probability", tile_product="depth", year=year, return_period=return_period, extra_param=extra_param) if image: for data in api_datas: if data: date = datetime.datetime.today().strftime( '%Y_%m_%d_%H_%M_%S') # Set file name to the current date, time, and product file_name = "_".join([ date, "probability_depth", str(year), str(return_period), str(data.get("coordinate")) ]) + ".png" if not output_dir: output_dir = os.getcwd() + "/output_data" if not os.path.exists(output_dir): os.makedirs(output_dir) with open(output_dir + '/' + file_name, "wb") as f: f.write(data['image']) logging.info("Image(s) generated to '{}'.".format(output_dir)) product = [ ProbabilityDepthTile(api_data, year, return_period) for api_data in api_datas ] logging.info("Probability Depth Tile Ready.") return product
def get_historic_event(self, event_id, coordinate, image=False, output_dir=None, extra_param=None): """Retrieves historic event tile data from the First Street Foundation API given a list of search_items and returns a list of Historic Event Tile objects. Args: event_id (int): A First Street Foundation eventId coordinate (list of tuple): A list of coordinates in the form of [(x_1, y_1, z_1), (x_2, y_2, z_2), ...] image (bool): To output extracted image to a png or not output_dir (str): The output directory to save the generated tile extra_param (str): Extra parameter to be added to the url Returns: A list of Probability Count Raises: InvalidArgument: The location provided is empty TypeError: The location provided is not a string """ if not event_id: raise InvalidArgument(event_id) elif not isinstance(event_id, int): raise TypeError("event id is not an int") # Get data from api and create objects api_datas = self.call_api(coordinate, "tile", "historic", tile_product="event", event_id=event_id, extra_param=extra_param) if image: for data in api_datas: if data: date = datetime.datetime.today().strftime( '%Y_%m_%d_%H_%M_%S') # Set file name to the current date, time, and product file_name = "_".join([ date, "historic_event", str(event_id), str(data.get("coordinate")) ]) + ".png" if not output_dir: output_dir = os.getcwd() + "/output_data" if not os.path.exists(output_dir): os.makedirs(output_dir) with open(output_dir + '/' + file_name, "wb") as f: f.write(data.get("image")) logging.info("Image(s) generated to '{}'.".format(output_dir)) product = [ HistoricEventTile(api_data, event_id) for api_data in api_datas ] logging.info("Historic Event Tile Ready.") return product
elif argument.product == 'fema.get_nfip': fs.fema.get_nfip(search_items, argument.location, csv=True, extra_param=argument.extra_param) elif argument.product == 'environmental.get_precipitation': fs.environmental.get_precipitation(search_items, csv=True, extra_param=argument.extra_param) elif argument.product == 'tile.get_probability_depth': fs.tile.get_probability_depth(year=int(argument.year), return_period=int(argument.return_period), coordinate=search_items, image=True) elif argument.product == 'tile.get_historic_event': fs.tile.get_historic_event(event_id=int(argument.event_id), coordinate=search_items, image=True) # AND FILES else: logging.error("Product not found. Please check that the argument" " provided is correct: {}".format(argument.product)) else: raise InvalidArgument("No search items were provided from either a search item list or a file. " "List: '{}', File Name: '{}'".format(argument.search_items, argument.file))
def call_api(self, search_item, product, product_subtype, location=None, tile_product=None, year=None, return_period=None, event_id=None, extra_param=None): """Receives an item, a product, a product subtype, and a location to create and call an endpoint to the First Street Foundation API. Args: search_item (list/file): A First Street Foundation IDs, lat/lng pair, address, or a file of First Street Foundation IDs product (str): The overall product to call product_subtype (str): The product subtype (if suitable) location (str/None): The location type (if suitable) tile_product (str/None): The tile product (if suitable) year (int/None): The year for probability depth tiles (if suitable) return_period (int/None): The return period for probability depth tiles (if suitable) event_id (int/None): The event_id for historic tiles (if suitable) extra_param (dict): Extra parameter to be added to the url Returns: A list of JSON responses """ # Not a list. This means it's should be a file if not isinstance(search_item, list): # Check if it's a file if isinstance(search_item, str) and os.path.isfile(search_item): # Get search items from file search_item = read_search_items_from_file(search_item) else: raise InvalidArgument( "File provided is not a list or a valid file. " "Please check the file name and path. '{}'".format( str(search_item))) else: # Check tile product if tile_product: if not all(isinstance(t, tuple) for t in search_item): raise TypeError( "Input must be a list of coordinates in a tuple of (z, x, y). " "Provided Arg: {}".format(search_item)) if not all( isinstance(coord, int) for t in search_item for coord in t): raise TypeError( "Each coordinate in the tuple must be an integer. Provided Arg: {}" .format(search_item)) if not all(0 < t[0] <= 18 for t in search_item): raise TypeError( "Max zoom is 18. Provided Arg: {}".format(search_item)) # else: # Ensure for historic and adaptation the search items are EventIDs or AdaptationIDs if ((product == "adaptation" and product_subtype == "detail") or (product == "historic" and product_subtype == "event") or (product == "economic/avm" and product_subtype == "provider")) and \ not all(isinstance(t, int) for t in search_item): raise TypeError("Input must be an integer for this product. " "Provided Arg: {}".format(search_item)) # No items found if not search_item: raise InvalidArgument(search_item) base_url = self._http.options.get('url') version = self._http.version # Create the endpoint endpoints = [] for item in search_item: if location: endpoint = "/".join( [base_url, version, product, product_subtype, location]) elif tile_product: if event_id: endpoint = "/".join([ base_url, version, product, product_subtype, tile_product, str(event_id), "/".join(map(str, item)) ]) else: endpoint = "/".join([ base_url, version, product, product_subtype, tile_product, str(year), str(return_period), "/".join(map(str, item)) ]) else: endpoint = "/".join( [base_url, version, product, product_subtype]) if not tile_product: if not extra_param: formatted_params = "" else: formatted_params = urllib.parse.urlencode(extra_param) # fsid if isinstance(item, int): endpoint = endpoint + "/{}".format(item) + "?{}".format( formatted_params) # lat/lng elif isinstance(item, tuple): endpoint = endpoint + "?lat={}&lng={}&{}".format( item[0], item[1], formatted_params) # address elif isinstance(item, str): endpoint = endpoint + "?address={}&{}".format( item, formatted_params) endpoints.append((endpoint, item, product, product_subtype)) # Asynchronously call the API for each endpoint loop = asyncio.get_event_loop() response = loop.run_until_complete( self._http.endpoint_execute(endpoints)) if product == "economic/aal": return zip(response, search_item) return response
def call_api(self, search_item, product, product_subtype, location=None, tile_product=None, year=None, return_period=None, event_id=None, connection_limit=100, extra_param=None): """Receives an item, a product, a product subtype, and a location to create and call an endpoint to the First Street Foundation API. Args: search_item (list/file): A First Street Foundation IDs, lat/lng pair, address, or a file of First Street Foundation IDs product (str): The overall product to call product_subtype (str): The product subtype (if suitable) location (str/None): The location type (if suitable) tile_product (str/None): The tile product (if suitable) year (int/None): The year for probability depth tiles (if suitable) return_period (int/None): The return period for probability depth tiles (if suitable) event_id (int/None): The event_id for historic tiles (if suitable) connection_limit (int): max number of connections to make extra_param (str): Extra parameter to be added to the url Returns: A list of JSON responses """ # Not a list if not isinstance(search_item, list): # Check if it's a file if isinstance(search_item, str): if os.path.isfile(os.getcwd() + "/" + search_item): # Get search items from file search_item = read_search_items_from_file(os.getcwd() + "/" + search_item) else: raise InvalidArgument( "File provided is not a valid file. " "Please check the file name. '{}'".format( os.path.curdir + str(search_item))) else: raise InvalidArgument( "File provided is not a list or a valid file. " "Please check the file name. '{}'".format( os.path.curdir + str(search_item))) if tile_product: if not all(isinstance(t, tuple) for t in search_item): raise TypeError( "Input must be a list of coordinates in a tuple of (z, x, y). Provided Arg: {}" .format(search_item)) if not all( isinstance(coord, int) for t in search_item for coord in t): raise TypeError( "Each coordinate in the tuple must be an integer. Provided Arg: {}" .format(search_item)) if not all(0 < t[0] <= 18 for t in search_item): raise TypeError( "Max zoom is 18. Provided Arg: {}".format(search_item)) # No items found if not search_item: raise InvalidArgument(search_item) base_url = self._http.options.get('url') version = self._http.version # Create the endpoint endpoints = [] for item in search_item: if location: endpoint = "/".join( [base_url, version, product, product_subtype, location]) elif tile_product: if event_id: endpoint = "/".join([ base_url, version, product, product_subtype, tile_product, str(event_id), "/".join(map(str, item)) ]) else: endpoint = "/".join([ base_url, version, product, product_subtype, tile_product, str(year), str(return_period), "/".join(map(str, item)) ]) else: endpoint = "/".join( [base_url, version, product, product_subtype]) if not tile_product: # fsid if isinstance(item, int): endpoint = endpoint + "/{}".format(item) + "?{}".format( extra_param) # lat/lng elif isinstance(item, tuple): endpoint = endpoint + "?lat={}&lng={}&{}".format( item[0], item[1], extra_param) # address elif isinstance(item, str): endpoint = endpoint + "?address={}&{}".format( item, extra_param) endpoints.append((endpoint, item, product, product_subtype)) # Asynchronously call the API for each endpoint loop = asyncio.get_event_loop() response = loop.run_until_complete( self._http.endpoint_execute(endpoints, connection_limit)) return response