def main(): # Parse commandline arguments parser = argparse.ArgumentParser(description='Get layer name, environment and verbosity for gfw-sync.') parser.add_argument('--environment', '-e', default='DEV', choices=('DEV', 'PROD'), help='the environment/config files to use for this run') parser.add_argument('--layer', '-l', required=True, help='the data layer to process; must match a value for tech_title in the config') parser.add_argument('--verbose', '-v', default='debug', choices=('debug', 'info', 'warning', 'error'), help='set verbosity level to print and write to file') args = parser.parse_args() # Instantiate logger; write to {dir}\logs logging = logger.build_logger(args.verbose) logging.info("\n{0}\n{1} v{2}\n{0}\n".format('*' * 50, settings.get_settings(args.environment)['tool_info']['name'], settings.get_settings(args.environment)['tool_info']['version'])) logging.critical('Starting | {0}'.format(args.layer)) # Open the correct sheet of the config table (PROD | DEV) and get the layerdef # Config table: https://docs.google.com/spreadsheets/d/1pkJCLNe9HWAHqxQh__s-tYQr9wJzGCb6rmRBPj8yRWI/edit#gid=0 layerdef = gs.get_layerdef(args.layer, args.environment) # Pass the layerdef to the build_layer function layer = layer_decision_tree.build_layer(layerdef, args.environment) # Update the layer in the output data sources layer.update() # Update the last-updated timestamp in the config table gs.update_gs_timestamp(args.layer, args.environment) # Delete scratch workspace layer.cleanup() logging.critical('Finished | {0}'.format(args.layer))
def update(self): """ Carry out the standard VectorLayer update for the country-specific data (self._update()) Then grab the global_layerdef and use that to update the global layer of which this is a part :return: """ # Update the country-specific layer-- same as for a standard vector layer self._update() # Grab the info about the global layer that we need to update global_layerdef = gs.get_layerdef(self.global_layer, self.gfw_env) # Update the global layer using it's own layerdef self.update_global_layer(global_layerdef)
def build_layer(layerdef, gfw_env): """ Used to get a layerdef for our layer of interest, then build a layer object based on the type of layer input :param layerdef: a layerdef that defines a layers inputs/outputs from the Google Sheet :param gfw_env: the environment in which we're running the script (PROD | DEV) :return: a layer object to gfw-sync so that it can call the update method """ if layerdef["type"] == "simple_vector": layer = VectorLayer(layerdef) elif layerdef["type"] == "raster": layer = RasterLayer(layerdef) elif layerdef["type"] == "imazon_vector": datasource = ImazonDataSource(layerdef) layer = VectorLayer(datasource.get_layer()) elif layerdef["type"] == "gran_chaco_vector": datasource = GranChacoDataSource(layerdef) layer = VectorLayer(datasource.get_layer()) elif layerdef["type"] == "hot_osm_export": datasource = HotOsmExportDataSource(layerdef) layer = VectorLayer(datasource.get_layer()) elif layerdef["type"] == "wdpa_vector": datasource = WDPADatasource(layerdef) layer = VectorLayer(datasource.get_layer()) elif layerdef["type"] == "forest_atlas_vector": datasource = ForestAtlasDataSource(layerdef) layer = CountryVectorLayer(datasource.get_layer()) elif layerdef["type"] == "global_forest_change": datasource = GlobalForestChange(layerdef) layer = GlobalForestChangeLayer(datasource.get_layer()) elif layerdef["type"] == "country_vector": if layerdef['global_layer']: # Get the associated layerdef for the global_layer specified by our original dataset of interest # This is important because if we updated gab_logging, we also need to updated gfw_logging logging.debug('Found a value for global_layer. Validating this input using the VectorLayer schema') global_layerdef = gs.get_layerdef(layerdef['global_layer'], gfw_env) # Use the output value as the source so that all the tests (validating that the "source" exists etc pass # This allows us to leave the source field in the google spreadsheet blank for this dataset, which makes # sense. The source for a global layer is made up of a bunch of smaller country layers global_layerdef['source'] = global_layerdef['esri_service_output'] VectorLayer(global_layerdef) logging.debug('Global layer validation complete') layer = CountryVectorLayer(layerdef) else: logging.error('Expecting to find global_layer associated with country_vector but did not.' 'If no global_layer associated, this should be classified as simple_vector. Exiting.') sys.exit(1) elif layerdef["type"] == "global_vector": logging.error('Please update global vector data by updating a country_vector dataset and specifying ' 'the global layer in the global_layer column \n Exiting now.') sys.exit(1) else: logging.error("Layer type {0} unknown".format(layerdef["type"])) sys.exit(1) return layer