def test_create_package_with_tags(self):
djss = DataJSONSchema1_1(original_dataset=self.test_datajson_dataset)
djss.ckan_owner_org_id = CKAN_ORG_ID
package = djss.transform_to_ckan_dataset()
cpa = CKANPortalAPI(base_url=CKAN_BASE_URL, api_key=CKAN_API_KEY)
res = cpa.create_package(ckan_package=package, on_duplicated='DELETE')
assert res['success'] == True
result = res['result']
# read it
res = cpa.show_package(ckan_package_id_or_name=result['id'])
assert res['success'] == True
ckan_dataset = res['result']
assert 'extras' in ckan_dataset
assert [['005:45']] == [
extra['value'] for extra in ckan_dataset['extras']
if extra['key'] == 'bureauCode'
]
assert [['005:047']] == [
extra['value'] for extra in ckan_dataset['extras']
if extra['key'] == 'programCode'
]
def test_create_package(self):
cpa = CKANPortalAPI(base_url=CKAN_BASE_URL, api_key=CKAN_API_KEY)
# error if duplicated
dataset_title = 'Dataset number {}'.format(random.randint(1, 999999))
dataset_name = slugify(dataset_title)
package = {
'name': dataset_name,
'title': dataset_title,
'owner_org': CKAN_ORG_ID
}
res = cpa.create_package(ckan_package=package)
print(res)
self.assertTrue(res['success'])
def test_create_harvest_source(self):
logger.info('Creating harvest source')
cpa = CKANPortalAPI(base_url=CKAN_BASE_URL, api_key=CKAN_API_KEY)
cpa.delete_all_harvest_sources(harvest_type='harvest',
source_type='datajson')
title = 'Energy JSON test {}'.format(random.randint(1, 999999))
url = 'http://www.energy.gov/data-{}.json'.format(
random.randint(1, 999999))
res = cpa.create_harvest_source(
title=title,
url=url,
owner_org_id=CKAN_ORG_ID,
source_type='datajson',
notes='Some tests about local harvesting sources creation',
frequency='WEEKLY')
self.assertTrue(res['success'])
harvest_source = res['result']
logger.info('Created: {}'.format(res['success']))
# read it
res = cpa.show_package(ckan_package_id_or_name=harvest_source['id'])
self.assertTrue(res['success'])
self.assertEqual(harvest_source['url'], url)
self.assertEqual(harvest_source['title'], title)
self.assertEqual(harvest_source['type'], 'harvest')
self.assertEqual(harvest_source['source_type'], 'datajson')
# search for it
results = cpa.search_harvest_packages(rows=1000,
harvest_type='harvest',
source_type='datajson')
created_ok = False
for datasets in results:
for dataset in datasets:
# print('FOUND: {}'.format(dataset['name']))
if dataset['name'] == harvest_source['name']:
created_ok = True
logger.info('Found!')
else:
logger.info('Other harvest source: {}'.format(
dataset['name']))
assert created_ok == True
# create a dataset with this harvest_soure_id
dataset_title = 'Dataset number {}'.format(random.randint(1, 999999))
dataset_name = slugify(dataset_title)
tags = [{'name': 'tag81'}, {'name': 'tag82'}]
randval = random.randint(1, 999)
extras = [
{
'key': 'harvest_source_id',
'value': harvest_source['id']
},
{
'key': 'harvest_source_title',
'value': harvest_source['title']
},
# {'key': 'harvest_object_id', 'value': harvest_source['id']}, # ? not sure
{
'key': 'harvest_ng_source_id',
'value': harvest_source['id']
},
{
'key': 'harvest_ng_source_title',
'value': harvest_source['title']
},
{
'key': 'try_a_extra',
'value': randval
}
]
package = {
'name': dataset_name,
'title': dataset_title,
'owner_org': CKAN_ORG_ID,
'tags': tags,
'extras': extras
}
res2 = cpa.create_package(ckan_package=package)
self.assertTrue(res2['success'])
logger.info('Package with harvest source: {}'.format(res2['success']))
# read full dataset
res3 = cpa.show_package(ckan_package_id_or_name=dataset_name)
self.assertTrue(res3['success'])
ckan_dataset = res3['result']
logger.info(
'Package with harvest source readed: {}'.format(ckan_dataset))
assert 'extras' in ckan_dataset
assert [str(randval)] == [
extra['value'] for extra in ckan_dataset['extras']
if extra['key'] == 'try_a_extra'
]
# my custom ID (not connected to a real harvest ID)
assert [harvest_source['id']] == [
extra['value'] for extra in ckan_dataset['extras']
if extra['key'] == 'harvest_ng_source_id'
]
# check if this package is related to harvest source
total_datasets_in_source = 0
datasets_from_source = cpa.search_harvest_packages(
harvest_source_id=harvest_source['id'])
connected_ok = False
for datasets in datasets_from_source:
for dataset in datasets:
total_datasets_in_source += 1
if dataset['name'] == dataset_name:
connected_ok = True
logger.info('Found!')
else:
# we just expect one dataset
error = '{} != {} ------ {}'.format(
dataset['name'], dataset_name, dataset)
logger.error(error)
assert error == False
assert connected_ok == True
assert total_datasets_in_source == 1
logger.info(
f' +++++++++++++ total_datasets_in_source={total_datasets_in_source}'
)
# this fails, harvest process is more complex that just add an extra
# assert [harvest_source['id']] == [extra['value'] for extra in ckan_dataset['extras'] if extra['key'] == 'harvest_source_id']
# delete both
logger.info('Delete CKAN package: {}'.format(ckan_dataset['id']))
res4 = cpa.delete_package(ckan_package_id_or_name=ckan_dataset['id'])
self.assertTrue(res4['success'])
logger.info('Delete Harvest source: {}'.format(harvest_source['id']))
res5 = cpa.delete_package(ckan_package_id_or_name=harvest_source['id'])
self.assertTrue(res5['success'])
def test_assing_collection_pkg_id(self):
config.CKAN_API_KEY = CKAN_API_KEY
config.CKAN_CATALOG_URL = CKAN_BASE_URL
config.CKAN_OWNER_ORG = CKAN_ORG_ID
config.SOURCE_ID = HARVEST_SOURCE_ID
config.SOURCE_NAME = 'Some harvest source'
r1 = {
'comparison_results': {
'action': 'create',
'new_data': {
'identifier': 'USDA-9000', # data.json id
'isPartOf': 'USDA-8000',
'title': 'R1 the first datajson',
'headers': {
"schema_version": "1.1",
"@context": "https://project-open-data.cio.gov/v1.1/schema/catalog.jsonld",
"@id": "https://www2.ed.gov/data.json",
"@type": "dcat:Catalog",
"conformsTo": "https://project-open-data.cio.gov/v1.1/schema",
"describedBy": "https://project-open-data.cio.gov/v1.1/schema/catalog.json",
}
}
},
}
r2 = {
'name': 'r2-second',
'title': 'R2 the second',
'owner_org': CKAN_ORG_ID,
'resources': [],
'comparison_results': {
'action': 'update',
'new_data': {
'identifier': 'USDA-8000', # data.json id
'title': 'R2-second',
'headers': {
"schema_version": "1.1",
"@context": "https://project-open-data.cio.gov/v1.1/schema/catalog.jsonld",
"@id": "https://www2.ed.gov/data.json",
"@type": "dcat:Catalog",
"conformsTo": "https://project-open-data.cio.gov/v1.1/schema",
"describedBy": "https://project-open-data.cio.gov/v1.1/schema/catalog.json",
}
}
},
}
r3 = {
'owner_org': CKAN_ORG_ID,
'comparison_results': {
'action': 'create',
'new_data': {
'identifier': 'USDA-7000', # data.json id
'isPartOf': 'USDA-1000', # not exists
'title': 'R3 the third',
'headers': {
"schema_version": "1.1",
"@context": "https://project-open-data.cio.gov/v1.1/schema/catalog.jsonld",
"@id": "https://www2.ed.gov/data.json",
"@type": "dcat:Catalog",
"conformsTo": "https://project-open-data.cio.gov/v1.1/schema",
"describedBy": "https://project-open-data.cio.gov/v1.1/schema/catalog.json",
}
}
},
}
r4 = {
'name': 'r4-fourth',
'title': 'R4 the fourth',
'owner_org': CKAN_ORG_ID,
'resources': [],
'comparison_results': {
'action': 'update',
'new_data': {
'identifier': 'USDA-6000', # data.json id
'isPartOf': 'USDA-8000',
'title': 'R4-fourth',
'headers': {
"schema_version": "1.1",
"@context": "https://project-open-data.cio.gov/v1.1/schema/catalog.jsonld",
"@id": "https://www2.ed.gov/data.json",
"@type": "dcat:Catalog",
"conformsTo": "https://project-open-data.cio.gov/v1.1/schema",
"describedBy": "https://project-open-data.cio.gov/v1.1/schema/catalog.json",
}
}
},
}
# create the required datasets
cpa = CKANPortalAPI(base_url=CKAN_BASE_URL, api_key=CKAN_API_KEY)
res = cpa.create_package(ckan_package=r2, on_duplicated='DELETE')
r2['id'] = res['result']['id']
res = cpa.create_package(ckan_package=r4, on_duplicated='DELETE')
r4['id'] = res['result']['id']
# delete r1 and r3
params = [{'fq': f'+identifier:"USDA-7000"'},
{'fq': f'+identifier:"USDA-9000"'}]
for param in params:
for pkgs in cpa.search_packages(search_params=param):
for pkg in pkgs:
cpa.delete_package(ckan_package_id_or_name=pkg['id'])
rowss = [
[r1, r2, r3, r4],
[r4, r3, r2, r1] # same with a different order
]
for rows in rowss:
rows_processed = []
for row in write_results_to_ckan(rows):
rows_processed.append(row)
# read the package
package_show = cpa.show_package(ckan_package_id_or_name=row['id'])
package = package_show['result']
extras = package.get('extras', None)
assert type(extras) == list
logger.info(f'writed package: {package}')
identifier = [extra['value'] for extra in extras if extra['key'] == 'identifier'][0]
if identifier == 'USDA-9000': # is R1
r1['id'] = row['id']
r1['new_package'] = package
elif identifier == 'USDA-8000': # is R2
assert r2['id'] == row['id']
r2['new_package'] = package
elif identifier == 'USDA-7000': # is R3
r3['id'] = row['id']
r3['new_package'] = package
elif identifier == 'USDA-6000': # is R4
assert r4['id'] == row['id']
r4['new_package'] = package
else:
assert "You never get here {}".format(row['id']) == False
for row in assing_collection_pkg_id(rows_processed):
datajson_dataset = row['comparison_results']['new_data']
package_show = cpa.show_package(ckan_package_id_or_name=row['id'])
package = package_show['result']
logger.info(f'Assigned package: {package}')
extras = package.get('extras', None)
assert type(extras) == list
if row['id'] == r1['id']: # this is part of r2-0002 dataset
ckan_collection_package_id = [extra['value'] for extra in extras if extra['key'] == 'collection_package_id'][0]
assert ckan_collection_package_id == r2['id']
elif row['id'] == r4['id']: # this is part of r2-0002 dataset
ckan_collection_package_id = [extra['value'] for extra in extras if extra['key'] == 'collection_package_id'][0]
assert ckan_collection_package_id == r2['id']
elif row['id'] == r3['id']: # this has a unknown father
ckan_collection_package_id = [extra['value'] for extra in extras if extra['key'] == 'collection_package_id']
assert [] == ckan_collection_package_id
elif row['id'] == r2['id']: # this has no father
ckan_collection_package_id = [extra['value'] for extra in extras if extra['key'] == 'collection_package_id']
assert [] == ckan_collection_package_id
else:
assert "You never get here {}".format(row['id']) == False
def write_results_to_ckan(rows):
""" each row it's a dataset to delete/update/create """
actions = {}
c = 0
for row in rows:
c += 1
if 'is_duplicate' in row:
continue
comparison_results = row['comparison_results']
action = comparison_results['action']
if action not in actions.keys():
actions[action] = {'total': 0, 'success': 0, 'fails': 0}
actions[action]['total'] += 1
dump_comp_res = json.dumps(comparison_results, indent=4)
# logger.info(f'Previous results {dump_comp_res}')
""" comparison_results is something like this
row['comparison_results'] {
"action": "update" | "delete" | "create",
"ckan_id": "1bfc8520-17b0-46b9-9940-a6646615436c",
"new_data": {data json dataset format},
"reason": "Some reason for the action"
}
"""
results = {'success': False, 'warnings': [], 'errors': []}
comparison_results['action_results'] = results
if action == 'error':
results['errors'].append(comparison_results['reason'])
yield row
continue
# if it's an update we need to merge internal resources
if action == 'update':
existing_resources = row['resources']
elif action == 'create':
existing_resources = None
if action in ['update', 'create']:
datajson_dataset = comparison_results['new_data']
# add required extras
# set catalog extras
for key, value in datajson_dataset['headers'].items():
if key in ['@context', '@id', 'conformsTo', 'describedBy']:
datajson_dataset[f'catalog_{key}'] = value
schema_version = datajson_dataset['headers'][
'schema_version'] # 1.1 or 1.0
assert schema_version in ['1.0', '1.1'] # main error
datajson_dataset['source_schema_version'] = schema_version
datajson_dataset['source_hash'] = hash_dataset(
datasetdict=datajson_dataset)
# harvest extras
# check if a local harvest source is required
# https://github.com/ckan/ckanext-harvest/blob/master/ckanext/harvest/logic/action/create.py#L27
datajson_dataset['harvest_ng_source_title'] = config.SOURCE_NAME
datajson_dataset['harvest_ng_source_id'] = config.SOURCE_ID
# CKAN hides this extras if we not define as harvest type
# if https://github.com/ckan/ckanext-harvest/blob/3a72337f1e619bf9ea3221037ca86615ec22ae2f/ckanext/harvest/plugin.py#L125
datajson_dataset['harvest_source_title'] = config.SOURCE_NAME
datajson_dataset['harvest_source_id'] = config.SOURCE_ID
if schema_version == '1.1':
djss = DataJSONSchema1_1(original_dataset=datajson_dataset)
else:
results['errors'].append(
'We are not ready to harvest 1.0 schema datasets. Add it to harvester'
)
yield row
continue
# raise Exception('We are not ready to harvest 1.0 schema datasets. Check if this kind of dataset still exists')
# ORG is required!
djss.ckan_owner_org_id = config.CKAN_OWNER_ORG
ckan_dataset = djss.transform_to_ckan_dataset(
existing_resources=existing_resources)
if action == 'create':
cpa = CKANPortalAPI(base_url=config.CKAN_CATALOG_URL,
api_key=config.CKAN_API_KEY)
try:
ckan_response = cpa.create_package(ckan_package=ckan_dataset)
except Exception as e:
ckan_response = {'success': False, 'error': str(e)}
results['success'] = ckan_response['success']
results['ckan_response'] = ckan_response
if ckan_response['success']:
actions[action]['success'] += 1
# add this new CKAN ID in the case we need as collection_pkg_id
row['id'] = ckan_response['result']['id']
comparison_results['ckan_id'] = ckan_response['result']['id']
else:
actions[action]['fails'] += 1
error = 'Error creating dataset: {}'.format(
ckan_response['error'])
results['errors'].append(error)
elif action == 'update':
cpa = CKANPortalAPI(base_url=config.CKAN_CATALOG_URL,
api_key=config.CKAN_API_KEY)
try:
ckan_response = cpa.update_package(ckan_package=ckan_dataset)
except Exception as e:
ckan_response = {'success': False, 'error': str(e)}
results['success'] = ckan_response['success']
results['ckan_response'] = ckan_response
# row['id'] = comparison_results['ckan_id']
if ckan_response['success']:
actions[action]['success'] += 1
else:
actions[action]['fails'] += 1
error = 'Error updating dataset: {}'.format(
ckan_response['error'])
results['errors'].append(error)
elif action == 'delete':
ckan_id = row['comparison_results']['ckan_id']
cpa = CKANPortalAPI(base_url=config.CKAN_CATALOG_URL,
api_key=config.CKAN_API_KEY)
try:
ckan_response = cpa.delete_package(
ckan_package_id_or_name=ckan_id)
except Exception as e:
ckan_response = {'success': False, 'error': str(e)}
results['success'] = ckan_response['success']
results['ckan_response'] = ckan_response
error = 'Error updating dataset: {}'.format(ckan_response['error'])
results['errors'].append(error)
if ckan_response['success']:
actions[action]['success'] += 1
else:
actions[action]['fails'] += 1
elif action == 'ignore':
continue
results = {'success': True}
else:
error = 'Unexpected action for this dataset: {}'.format(action)
results = {'success': False, 'error': error}
results['timestamp'] = datetime.now(
pytz.utc).isoformat() # iso format move as string to save to disk
yield row
logger.info(f'Actions detected {actions}')