def construct_export_filename(entity, extension): export_file_name = "{db_entity}_{timestamp}.{extension}".format( db_entity=entity.name.replace(" ", "_"), timestamp=timestamp(), extension=extension) return export_file_name
def _export_query_results(job, class_name, unique_id, filename, query, event='queryResultExportCompleted'): try: job.status = "Exporting" job.save() # add timestamp to filename to prevent conflicts filename = filename + "_" + timestamp() result = OGRETL().export(filename, query, 'CSV', geom=None) job.data = '/' + filename + ".csv" job.save() send_message_to_client(job.user.id, dict(event=event, class_name=class_name, unique_id=unique_id, job_id=str(job.hashid))) job.status = 'Complete' except Exception, e: logger.error(e) job.status = "Failed" exc_type, exc_value, exc_traceback = sys.exc_info() readable_exception = traceback.format_exception(exc_type, exc_value, exc_traceback) job.data = readable_exception job.save() send_message_to_client(job.user.id, dict(event=job.type + " failed", trace=readable_exception))
def dump_built_forms(): """ Exports all of the content of all of the tables that represent BuiltForm to a SQL file that can be most easily distributed to other machines and loaded more quickly for tests. Avoids redundantly running the costly calculation of the built form relationships. :return: """ built_form_tables = [ "building", "buildingattributes", "buildingpercent", "buildingtype", "buildingtypecategory", "buildingusedefinition", "buildingusepercent", "builtform", "builform_tags", "builtformset", "builtformset_built_forms", "flatbuiltform", "infrastructure", "infrastructureattributeset", "infrastructurepercent", "infrastructuretype", "placetype", "placetypecomponent", "placetypecomponentpercent", "medium" ] formatted_list_of_tables = "" for table_name in built_form_tables: formatted_list_of_tables += '-t footprint_{table_name} '.format( table_name=table_name) dump_args = dict( database=settings.DATABASES['default']['NAME'], formatted_list_of_tables=formatted_list_of_tables, output_file="{sql_path}/built_form_export_{timestamp}.sql".format( sql_path=settings.SQL_PATH, timestamp=timestamp()), tmp_db_name="urbanfootprint_builtform_dump") print dump_args dump_command = "pg_dump {database} {formatted_list_of_tables} > {output_file}" create_tmp_db = "createdb {tmp_db_name} --template template_postgis" restore_dump_to_tmp_db = "psql {tmp_db_name} -f {output_file}" delete_unrelated_media = """psql {tmp_db_name} -c "DELETE from footprint_medium where \"key\"::text NOT LIKE 'built_form%'" """ delete_dumpfile = "rm {output_file}" dump_isolated_built_form_relations = "pg_dump {tmp_db_name} {formatted_list_of_tables} > {output_file}" drop_tmp_db = "psql {database} -c \"drop database {tmp_db_name}\"" for command in [ dump_command, create_tmp_db, restore_dump_to_tmp_db, delete_unrelated_media, delete_dumpfile, dump_isolated_built_form_relations, drop_tmp_db ]: try: subprocess.call(command.format(**dump_args), shell=True) except Exception, E: print E
def dump_built_forms(): """ Exports all of the content of all of the tables that represent BuiltForm to a SQL file that can be most easily distributed to other machines and loaded more quickly for tests. Avoids redundantly running the costly calculation of the built form relationships. :return: """ built_form_tables = [ "building", "buildingattributes", "buildingpercent", "buildingtype", "buildingtypecategory", "buildingusedefinition", "buildingusepercent", "builtform", "builform_tags", "builtformset", "builtformset_built_forms", "flatbuiltform", "infrastructure", "infrastructureattributeset", "infrastructurepercent", "infrastructuretype", "placetype", "placetypecomponent", "placetypecomponentpercent", "medium" ] formatted_list_of_tables = "" for table_name in built_form_tables: formatted_list_of_tables += '-t footprint_{table_name} '.format(table_name=table_name) dump_args = dict( database=settings.DATABASES['default']['NAME'], formatted_list_of_tables=formatted_list_of_tables, output_file="{sql_path}/built_form_export_{timestamp}.sql".format( sql_path=settings.SQL_PATH, timestamp=timestamp() ), tmp_db_name="urbanfootprint_builtform_dump" ) print dump_args dump_command = "pg_dump {database} {formatted_list_of_tables} > {output_file}" create_tmp_db = "createdb {tmp_db_name} --template template_postgis" restore_dump_to_tmp_db = "psql {tmp_db_name} -f {output_file}" delete_unrelated_media = """psql {tmp_db_name} -c "DELETE from footprint_medium where \"key\"::text NOT LIKE 'built_form%'" """ delete_dumpfile = "rm {output_file}" dump_isolated_built_form_relations = "pg_dump {tmp_db_name} {formatted_list_of_tables} > {output_file}" drop_tmp_db = "psql {database} -c \"drop database {tmp_db_name}\"" for command in [dump_command, create_tmp_db, restore_dump_to_tmp_db, delete_unrelated_media, delete_dumpfile, dump_isolated_built_form_relations, drop_tmp_db]: try: subprocess.call(command.format(**dump_args), shell=True) except Exception, E: print E
def update(self, **kwargs): scenario = self.config_entity.subclassed logger.debug('{0}:Starting Agriculture Core Analysis for {1}'.format(timestamp(), self.config_entity)) if isinstance(scenario, BaseScenario): agriculture_db_entity_key = DbEntityKey.BASE_AGRICULTURE_CANVAS elif isinstance(scenario, FutureScenario): agriculture_db_entity_key = DbEntityKey.FUTURE_AGRICULTURE_CANVAS else: raise Exception("Config Entity is not a Future or Base Scenario, cannot run AgricultureCore.") ids = kwargs.get('ids', None) agriculture_feature_class = self.config_entity.db_entity_feature_class(agriculture_db_entity_key) if ids: features = agriculture_feature_class.objects.filter(id__in=ids) else: features = agriculture_feature_class.objects.filter(built_form__isnull=False) feature_count = features.count() if not feature_count: logger.info("No features to process!") return logger.debug("Processing {0} features...".format(feature_count)) iterator_start = datetime.datetime.utcnow().replace(tzinfo=utc) self.progress(0.05, **kwargs) if feature_count <= 36: increment_portion = (.9 / feature_count) + .001 equal_portion = 1 else: increment_portion = .05 equal_portion = int((feature_count - 1) / 18) i = 1 for feature in features.iterator(): if i % equal_portion == 0: self.progress(increment_portion, **kwargs) if not feature.built_form: feature.built_form_key = None feature.crop_yield = 0 feature.market_value = 0 feature.production_cost = 0 feature.water_consumption = 0 feature.labor_force = 0 feature.truck_trips = 0 else: applied_acres = feature.acres_gross * feature.density_pct * feature.dev_pct agriculture_attribute_set = feature.built_form.resolve_built_form(feature.built_form).agriculture_attribute_set feature.built_form_key = feature.built_form.key feature.crop_yield = agriculture_attribute_set.crop_yield * applied_acres feature.market_value = agriculture_attribute_set.unit_price * feature.crop_yield feature.production_cost = agriculture_attribute_set.cost * applied_acres feature.water_consumption = agriculture_attribute_set.water_consumption * applied_acres feature.labor_force = agriculture_attribute_set.labor_input * applied_acres feature.truck_trips = agriculture_attribute_set.truck_trips * applied_acres feature.save(update_fields=self.ANALYSIS_FIELDS) i += 1 total_time = datetime.datetime.utcnow().replace(tzinfo=utc) - iterator_start logger.debug("Processed {0} features in {1}: {2} per feature".format( feature_count, total_time, total_time/feature_count )) self.progress(.9, **kwargs) logger.debug('{0}:Finished Agriculture Core Analysis for {1} '.format(timestamp(), self.config_entity)) self.update_dependent_scenarios(features, scenario)
def update(self, **kwargs): scenario = self.config_entity.subclassed logger.debug('{0}:Starting Agriculture Core Analysis for {1}'.format( timestamp(), self.config_entity)) if isinstance(scenario, BaseScenario): agriculture_db_entity_key = DbEntityKey.BASE_AGRICULTURE_CANVAS elif isinstance(scenario, FutureScenario): agriculture_db_entity_key = DbEntityKey.FUTURE_AGRICULTURE_CANVAS else: raise Exception( "Config Entity is not a Future or Base Scenario, cannot run AgricultureCore." ) ids = kwargs.get('ids', None) agriculture_feature_class = self.config_entity.db_entity_feature_class( agriculture_db_entity_key) if ids: features = agriculture_feature_class.objects.filter(id__in=ids) else: features = agriculture_feature_class.objects.filter( built_form__isnull=False) feature_count = features.count() if not feature_count: logger.info("No features to process!") return logger.debug("Processing {0} features...".format(feature_count)) iterator_start = datetime.datetime.utcnow().replace(tzinfo=utc) self.progress(0.05, **kwargs) if feature_count <= 36: increment_portion = (.9 / feature_count) + .001 equal_portion = 1 else: increment_portion = .05 equal_portion = int((feature_count - 1) / 18) i = 1 for feature in features.iterator(): if i % equal_portion == 0: self.progress(increment_portion, **kwargs) if not feature.built_form: feature.built_form_key = None feature.crop_yield = 0 feature.market_value = 0 feature.production_cost = 0 feature.water_consumption = 0 feature.labor_force = 0 feature.truck_trips = 0 else: applied_acres = feature.acres_gross * feature.density_pct * feature.dev_pct agriculture_attribute_set = feature.built_form.resolve_built_form( feature.built_form).agriculture_attribute_set feature.built_form_key = feature.built_form.key feature.crop_yield = agriculture_attribute_set.crop_yield * applied_acres feature.market_value = agriculture_attribute_set.unit_price * feature.crop_yield feature.production_cost = agriculture_attribute_set.cost * applied_acres feature.water_consumption = agriculture_attribute_set.water_consumption * applied_acres feature.labor_force = agriculture_attribute_set.labor_input * applied_acres feature.truck_trips = agriculture_attribute_set.truck_trips * applied_acres feature.save(update_fields=self.ANALYSIS_FIELDS) i += 1 total_time = datetime.datetime.utcnow().replace( tzinfo=utc) - iterator_start logger.debug("Processed {0} features in {1}: {2} per feature".format( feature_count, total_time, total_time / feature_count)) self.progress(.9, **kwargs) logger.debug('{0}:Finished Agriculture Core Analysis for {1} '.format( timestamp(), self.config_entity)) self.update_dependent_scenarios(features, scenario)