def ingest_resource(context, resource_dict): if toolkit.asbool(resource_dict.get('spatial_parent', 'False')): try: task = toolkit.get_action('task_status_show')( { 'ignore_auth': True }, { 'entity_id': resource_dict['id'], 'task_type': 'spatial_ingest', 'key': 'spatialingestor' }) if task.get('state') in ['pending']: # There already is a pending Spatialingestor submission, # skip this one ... log.debug( 'Skipping Spatial Ingestor submission for resource {0}'. format(resource_dict['id'])) return except toolkit.ObjectNotFound: pass try: log.debug('Submitting resource {0} to Spatial Ingestor'.format( resource_dict['id'])) toolkit.get_action('spatialingestor_job_submit')( context, { 'resource_id': resource_dict['id'], 'job_type': 'spatial_ingest' }) except toolkit.ValidationError, e: log.error(e)
def get_db_cursor(): try: connection = psycopg2.connect(dbname=postgist_info['db_name'], user=postgist_info['db_user'], password=postgist_info['db_pass'], host=postgist_info['db_host'], port=postgist_info.get('db_port', None)) connection.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT) return connection.cursor(), connection except Exception, e: log.error("Failed to connect with PostGIS with error {0}".format(str(e))) return None
def _reingest_all(self): context = {'user': toolkit.get_action('user_show')({'ignore_auth': True}, { 'id': config.get('ckan.spatialingestor.ckan_user', 'default') })} pkg_ids = [r[0] for r in model.Session.query(model.Package.id).filter(model.Package.state != 'deleted').all()] log.info("Re-ingesting spatial resources for all packages...") total_packages = len(pkg_ids) for counter, pkg_id in enumerate(pkg_ids): sys.stdout.write("\rRe-ingesting spatial resources for dataset {0}/{1}".format(counter + 1, total_packages)) sys.stdout.flush() pkg_dict = model.Package.get(pkg_id).as_dict() try: for res in pkg_dict['resources']: ingest_resource(context, res['id'], False) except Exception, e: log.error("Processing {0} failed with error {1}, continuing...".format(pkg_dict['name'], str(e)))
if "http://data.gov.au/geoserver/" in res_dict.get('url', ''): toolkit.get_action('resource_delete')({'ignore_auth': True}, res_dict) res = requests.delete(geoserver_wsurl + '/' + input_pkg['name'] + '?recurse=true&quietOnNotFound', auth=geoserver_credentials) log.info("Geoserver recursive workspace deletion returned {0}".format(res)) db_cursor.execute("DROP TABLE IF EXISTS {tab_name}".format(tab_name=table_name)) log.ingo("Dropped SQL table {0}".format(table_name)) db_res = get_db_cursor() if db_res is None: log.error("Failed to open SQL connection to PostGIS DB") return None cursor, connection = db_res pkg_ids = [r[0] for r in model.Session.query(model.Package.id).all()] log.info("Migrating legacy spatial ingestion on all packages...") total_packages = len(pkg_ids) for counter, pkg_id in enumerate(pkg_ids): sys.stdout.write("\rProcessing dataset {0}/{1}".format(counter + 1, total_packages)) sys.stdout.flush() pkg_dict = model.Package.get(pkg_id).as_dict() try: process_pkg(cursor, pkg_dict) except Exception, e:
toolkit.get_action('spatialingestor_job_submit')( context, { 'resource_id': resource_dict['id'], 'job_type': 'spatial_ingest' }) except toolkit.ValidationError, e: log.error(e) elif is_spatially_ingestible_resource(resource_dict): try: dataset = toolkit.get_action('package_show')( context, { 'id': resource_dict['package_id'], }) except Exception, e: log.error( "Failed to retrieve package ID: {0} with error {1}".format( resource_dict['package_id'], str(e))) return log.info("Loaded dataset {0}.".format(dataset['name'])) # We auto_process spatial file by updating the resource, which will re-trigger this method resource_dict['spatial_parent'] = 'True' try: toolkit.get_action('resource_update')(context, resource_dict) except toolkit.ValidationError, e: log.error(e) def purge_resource_datastores(context, resource_dict): # Have to be careful about how to delete child resources here in a sense that