def ingest_resource(context, resource_dict):
if toolkit.asbool(resource_dict.get('spatial_parent', 'False')):
try:
task = toolkit.get_action('task_status_show')(
{
'ignore_auth': True
}, {
'entity_id': resource_dict['id'],
'task_type': 'spatial_ingest',
'key': 'spatialingestor'
})
if task.get('state') in ['pending']:
# There already is a pending Spatialingestor submission,
# skip this one ...
log.debug(
'Skipping Spatial Ingestor submission for resource {0}'.
format(resource_dict['id']))
return
except toolkit.ObjectNotFound:
pass
try:
log.debug('Submitting resource {0} to Spatial Ingestor'.format(
resource_dict['id']))
toolkit.get_action('spatialingestor_job_submit')(
context, {
'resource_id': resource_dict['id'],
'job_type': 'spatial_ingest'
})
except toolkit.ValidationError, e:
log.error(e)
def get_db_cursor():
try:
connection = psycopg2.connect(dbname=postgist_info['db_name'],
user=postgist_info['db_user'],
password=postgist_info['db_pass'],
host=postgist_info['db_host'],
port=postgist_info.get('db_port', None))
connection.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
return connection.cursor(), connection
except Exception, e:
log.error("Failed to connect with PostGIS with error {0}".format(str(e)))
return None
def _reingest_all(self):
context = {'user': toolkit.get_action('user_show')({'ignore_auth': True}, {
'id': config.get('ckan.spatialingestor.ckan_user', 'default')
})}
pkg_ids = [r[0] for r in model.Session.query(model.Package.id).filter(model.Package.state != 'deleted').all()]
log.info("Re-ingesting spatial resources for all packages...")
total_packages = len(pkg_ids)
for counter, pkg_id in enumerate(pkg_ids):
sys.stdout.write("\rRe-ingesting spatial resources for dataset {0}/{1}".format(counter + 1, total_packages))
sys.stdout.flush()
pkg_dict = model.Package.get(pkg_id).as_dict()
try:
for res in pkg_dict['resources']:
ingest_resource(context, res['id'], False)
except Exception, e:
log.error("Processing {0} failed with error {1}, continuing...".format(pkg_dict['name'], str(e)))
if "http://data.gov.au/geoserver/" in res_dict.get('url', ''):
toolkit.get_action('resource_delete')({'ignore_auth': True}, res_dict)
res = requests.delete(geoserver_wsurl + '/' + input_pkg['name'] + '?recurse=true&quietOnNotFound',
auth=geoserver_credentials)
log.info("Geoserver recursive workspace deletion returned {0}".format(res))
db_cursor.execute("DROP TABLE IF EXISTS {tab_name}".format(tab_name=table_name))
log.ingo("Dropped SQL table {0}".format(table_name))
db_res = get_db_cursor()
if db_res is None:
log.error("Failed to open SQL connection to PostGIS DB")
return None
cursor, connection = db_res
pkg_ids = [r[0] for r in model.Session.query(model.Package.id).all()]
log.info("Migrating legacy spatial ingestion on all packages...")
total_packages = len(pkg_ids)
for counter, pkg_id in enumerate(pkg_ids):
sys.stdout.write("\rProcessing dataset {0}/{1}".format(counter + 1, total_packages))
sys.stdout.flush()
pkg_dict = model.Package.get(pkg_id).as_dict()
try:
process_pkg(cursor, pkg_dict)
except Exception, e:
toolkit.get_action('spatialingestor_job_submit')(
context, {
'resource_id': resource_dict['id'],
'job_type': 'spatial_ingest'
})
except toolkit.ValidationError, e:
log.error(e)
elif is_spatially_ingestible_resource(resource_dict):
try:
dataset = toolkit.get_action('package_show')(
context, {
'id': resource_dict['package_id'],
})
except Exception, e:
log.error(
"Failed to retrieve package ID: {0} with error {1}".format(
resource_dict['package_id'], str(e)))
return
log.info("Loaded dataset {0}.".format(dataset['name']))
# We auto_process spatial file by updating the resource, which will re-trigger this method
resource_dict['spatial_parent'] = 'True'
try:
toolkit.get_action('resource_update')(context, resource_dict)
except toolkit.ValidationError, e:
log.error(e)
def purge_resource_datastores(context, resource_dict):
# Have to be careful about how to delete child resources here in a sense that