rabbit.send_message(self.queue_name, body)
counter += 1
rabbit.close_connection()
logger.info('DONE bulk update')
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='SDSClient')
parser.add_argument('--debug', '-d', action='store_true', help='creates debug files for datasets queries' )
args = parser.parse_args()
sds = SDSClient(services_config['sds'], other_config['timeout'], 'odp_queue', ODPClient())
if args.debug:
#query dataset
dataset_url = 'http://www.eea.europa.eu/themes/biodiversity/document-library/natura-2000/natura-2000-network-statistics/natura-2000-barometer-statistics/statistics/barometer-statistics'
dataset_identifier = dataset_url.split('/')[-1]
result_rdf, result_json, msg = sds.query_dataset(dataset_url, dataset_identifier)
if not msg:
dump_rdf('.debug.1.sds.%s.rdf.xml' % dataset_identifier, result_rdf)
dump_json('.debug.2.sds.%s.json.txt' % dataset_identifier, result_json)
#query all datasets - UNCOMMENT IF YOU NEED THIS
#result_json, msg = sds.query_all_datasets()
#if not msg:
# dump_json('.debug.3.sds.all_datasets.json.txt', result_json)
# dump_rdf('.debug.4.sds.all_datasets.csv.txt', '\n'.join(('\t'.join(x) for x in sds.parse_datasets_json(result_json))))
else:
#initiate a bulk update operation
sds.bulk_update()
""" Call ckan action
"""
try:
name, datapackage = self.transformJSON2DataPackage(dataset_json,
dataset_data_rdf)
if action in ['update', 'create']:
if action == 'create':
datapackage[u'name'] = name
return self.package_create(datapackage)
else:
return self.package_update(datapackage)
if action == 'delete':
return self.package_delete(datapackage)
except Exception, error:
return ["error", "%s: %s" %(type(error).__name__, error)]
if __name__ == '__main__':
odp = ODPClient()
#queries by ODP name
#package = odp.package_show(u'XsJfLAZ4guXeAL4bjHNA')
#package = odp.package_search(prop='name', value=u'FPi519FhZ8UHVCNmdjhqPg')
#query by dataset's SDS/ODP identifier
dataset_identifier = 'european-union-emissions-trading-scheme-eu-ets-data-from-citl-8'
package = odp.package_search(prop='identifier', value=dataset_identifier)
dump_json('.debug.1.odp.package.%s.json.txt' % dataset_identifier, package)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='CKANClient')
parser.add_argument('--debug', '-d', action='store_true', help='create debug file for dataset data from SDS and the builded package for ODP' )
args = parser.parse_args()
cc = CKANClient('odp_queue')
if args.debug:
dataset_url = 'http://www.eea.europa.eu/data-and-maps/data/european-union-emissions-trading-scheme-eu-ets-data-from-citl-8'
dataset_identifier = dataset_url.split('/')[-1]
#query dataset data from SDS
dataset_rdf, dataset_json, msg = cc.get_dataset_data(dataset_url, dataset_identifier)
if not msg:
dump_rdf('.debug.1.sds.%s.rdf.xml' % dataset_identifier, dataset_rdf)
dump_json('.debug.2.sds.%s.json.txt' % dataset_identifier, dataset_json)
#build the package structure with data from SDS
package_name, package_data = cc.odp.transformJSON2DataPackage(dataset_json, dataset_rdf)
dump_json('.debug.3.cc.%s.json.txt' % dataset_identifier, package_data)
#query and retreive the ODP package data
package = cc.odp.package_search(prop='identifier', value=dataset_identifier)[0]
dump_json('.debug.4.odp.package.%s.json.txt' % dataset_identifier, package)
#merge the ODP package with the package build from SDS data
package.update(package_data)
dump_json('.debug.5.cc.package.%s.json.txt' % dataset_identifier, package)
#update ODP - CAREFULLY WHEN UNCOMMENT THE FOLLOWING LINES - THE ODP DATASET GETS UPDATED!
package_response, msg = cc.odp.package_update(package_data)
