def test_create_package_with_tags(self):
djss = DataJSONSchema1_1(original_dataset=self.test_datajson_dataset)
djss.ckan_owner_org_id = CKAN_ORG_ID
package = djss.transform_to_ckan_dataset()
cpa = CKANPortalAPI(base_url=CKAN_BASE_URL, api_key=CKAN_API_KEY)
res = cpa.create_package(ckan_package=package, on_duplicated='DELETE')
assert res['success'] == True
result = res['result']
# read it
res = cpa.show_package(ckan_package_id_or_name=result['id'])
assert res['success'] == True
ckan_dataset = res['result']
assert 'extras' in ckan_dataset
assert [['005:45']] == [
extra['value'] for extra in ckan_dataset['extras']
if extra['key'] == 'bureauCode'
]
assert [['005:047']] == [
extra['value'] for extra in ckan_dataset['extras']
if extra['key'] == 'programCode'
]
def test_create_harvest_source(self):
logger.info('Creating harvest source')
cpa = CKANPortalAPI(base_url=CKAN_BASE_URL, api_key=CKAN_API_KEY)
cpa.delete_all_harvest_sources(harvest_type='harvest',
source_type='datajson')
title = 'Energy JSON test {}'.format(random.randint(1, 999999))
url = 'http://www.energy.gov/data-{}.json'.format(
random.randint(1, 999999))
res = cpa.create_harvest_source(
title=title,
url=url,
owner_org_id=CKAN_ORG_ID,
source_type='datajson',
notes='Some tests about local harvesting sources creation',
frequency='WEEKLY')
self.assertTrue(res['success'])
harvest_source = res['result']
logger.info('Created: {}'.format(res['success']))
# read it
res = cpa.show_package(ckan_package_id_or_name=harvest_source['id'])
self.assertTrue(res['success'])
self.assertEqual(harvest_source['url'], url)
self.assertEqual(harvest_source['title'], title)
self.assertEqual(harvest_source['type'], 'harvest')
self.assertEqual(harvest_source['source_type'], 'datajson')
# search for it
results = cpa.search_harvest_packages(rows=1000,
harvest_type='harvest',
source_type='datajson')
created_ok = False
for datasets in results:
for dataset in datasets:
# print('FOUND: {}'.format(dataset['name']))
if dataset['name'] == harvest_source['name']:
created_ok = True
logger.info('Found!')
else:
logger.info('Other harvest source: {}'.format(
dataset['name']))
assert created_ok == True
# create a dataset with this harvest_soure_id
dataset_title = 'Dataset number {}'.format(random.randint(1, 999999))
dataset_name = slugify(dataset_title)
tags = [{'name': 'tag81'}, {'name': 'tag82'}]
randval = random.randint(1, 999)
extras = [
{
'key': 'harvest_source_id',
'value': harvest_source['id']
},
{
'key': 'harvest_source_title',
'value': harvest_source['title']
},
# {'key': 'harvest_object_id', 'value': harvest_source['id']}, # ? not sure
{
'key': 'harvest_ng_source_id',
'value': harvest_source['id']
},
{
'key': 'harvest_ng_source_title',
'value': harvest_source['title']
},
{
'key': 'try_a_extra',
'value': randval
}
]
package = {
'name': dataset_name,
'title': dataset_title,
'owner_org': CKAN_ORG_ID,
'tags': tags,
'extras': extras
}
res2 = cpa.create_package(ckan_package=package)
self.assertTrue(res2['success'])
logger.info('Package with harvest source: {}'.format(res2['success']))
# read full dataset
res3 = cpa.show_package(ckan_package_id_or_name=dataset_name)
self.assertTrue(res3['success'])
ckan_dataset = res3['result']
logger.info(
'Package with harvest source readed: {}'.format(ckan_dataset))
assert 'extras' in ckan_dataset
assert [str(randval)] == [
extra['value'] for extra in ckan_dataset['extras']
if extra['key'] == 'try_a_extra'
]
# my custom ID (not connected to a real harvest ID)
assert [harvest_source['id']] == [
extra['value'] for extra in ckan_dataset['extras']
if extra['key'] == 'harvest_ng_source_id'
]
# check if this package is related to harvest source
total_datasets_in_source = 0
datasets_from_source = cpa.search_harvest_packages(
harvest_source_id=harvest_source['id'])
connected_ok = False
for datasets in datasets_from_source:
for dataset in datasets:
total_datasets_in_source += 1
if dataset['name'] == dataset_name:
connected_ok = True
logger.info('Found!')
else:
# we just expect one dataset
error = '{} != {} ------ {}'.format(
dataset['name'], dataset_name, dataset)
logger.error(error)
assert error == False
assert connected_ok == True
assert total_datasets_in_source == 1
logger.info(
f' +++++++++++++ total_datasets_in_source={total_datasets_in_source}'
)
# this fails, harvest process is more complex that just add an extra
# assert [harvest_source['id']] == [extra['value'] for extra in ckan_dataset['extras'] if extra['key'] == 'harvest_source_id']
# delete both
logger.info('Delete CKAN package: {}'.format(ckan_dataset['id']))
res4 = cpa.delete_package(ckan_package_id_or_name=ckan_dataset['id'])
self.assertTrue(res4['success'])
logger.info('Delete Harvest source: {}'.format(harvest_source['id']))
res5 = cpa.delete_package(ckan_package_id_or_name=harvest_source['id'])
self.assertTrue(res5['success'])
def test_assing_collection_pkg_id(self):
config.CKAN_API_KEY = CKAN_API_KEY
config.CKAN_CATALOG_URL = CKAN_BASE_URL
config.CKAN_OWNER_ORG = CKAN_ORG_ID
config.SOURCE_ID = HARVEST_SOURCE_ID
config.SOURCE_NAME = 'Some harvest source'
r1 = {
'comparison_results': {
'action': 'create',
'new_data': {
'identifier': 'USDA-9000', # data.json id
'isPartOf': 'USDA-8000',
'title': 'R1 the first datajson',
'headers': {
"schema_version": "1.1",
"@context": "https://project-open-data.cio.gov/v1.1/schema/catalog.jsonld",
"@id": "https://www2.ed.gov/data.json",
"@type": "dcat:Catalog",
"conformsTo": "https://project-open-data.cio.gov/v1.1/schema",
"describedBy": "https://project-open-data.cio.gov/v1.1/schema/catalog.json",
}
}
},
}
r2 = {
'name': 'r2-second',
'title': 'R2 the second',
'owner_org': CKAN_ORG_ID,
'resources': [],
'comparison_results': {
'action': 'update',
'new_data': {
'identifier': 'USDA-8000', # data.json id
'title': 'R2-second',
'headers': {
"schema_version": "1.1",
"@context": "https://project-open-data.cio.gov/v1.1/schema/catalog.jsonld",
"@id": "https://www2.ed.gov/data.json",
"@type": "dcat:Catalog",
"conformsTo": "https://project-open-data.cio.gov/v1.1/schema",
"describedBy": "https://project-open-data.cio.gov/v1.1/schema/catalog.json",
}
}
},
}
r3 = {
'owner_org': CKAN_ORG_ID,
'comparison_results': {
'action': 'create',
'new_data': {
'identifier': 'USDA-7000', # data.json id
'isPartOf': 'USDA-1000', # not exists
'title': 'R3 the third',
'headers': {
"schema_version": "1.1",
"@context": "https://project-open-data.cio.gov/v1.1/schema/catalog.jsonld",
"@id": "https://www2.ed.gov/data.json",
"@type": "dcat:Catalog",
"conformsTo": "https://project-open-data.cio.gov/v1.1/schema",
"describedBy": "https://project-open-data.cio.gov/v1.1/schema/catalog.json",
}
}
},
}
r4 = {
'name': 'r4-fourth',
'title': 'R4 the fourth',
'owner_org': CKAN_ORG_ID,
'resources': [],
'comparison_results': {
'action': 'update',
'new_data': {
'identifier': 'USDA-6000', # data.json id
'isPartOf': 'USDA-8000',
'title': 'R4-fourth',
'headers': {
"schema_version": "1.1",
"@context": "https://project-open-data.cio.gov/v1.1/schema/catalog.jsonld",
"@id": "https://www2.ed.gov/data.json",
"@type": "dcat:Catalog",
"conformsTo": "https://project-open-data.cio.gov/v1.1/schema",
"describedBy": "https://project-open-data.cio.gov/v1.1/schema/catalog.json",
}
}
},
}
# create the required datasets
cpa = CKANPortalAPI(base_url=CKAN_BASE_URL, api_key=CKAN_API_KEY)
res = cpa.create_package(ckan_package=r2, on_duplicated='DELETE')
r2['id'] = res['result']['id']
res = cpa.create_package(ckan_package=r4, on_duplicated='DELETE')
r4['id'] = res['result']['id']
# delete r1 and r3
params = [{'fq': f'+identifier:"USDA-7000"'},
{'fq': f'+identifier:"USDA-9000"'}]
for param in params:
for pkgs in cpa.search_packages(search_params=param):
for pkg in pkgs:
cpa.delete_package(ckan_package_id_or_name=pkg['id'])
rowss = [
[r1, r2, r3, r4],
[r4, r3, r2, r1] # same with a different order
]
for rows in rowss:
rows_processed = []
for row in write_results_to_ckan(rows):
rows_processed.append(row)
# read the package
package_show = cpa.show_package(ckan_package_id_or_name=row['id'])
package = package_show['result']
extras = package.get('extras', None)
assert type(extras) == list
logger.info(f'writed package: {package}')
identifier = [extra['value'] for extra in extras if extra['key'] == 'identifier'][0]
if identifier == 'USDA-9000': # is R1
r1['id'] = row['id']
r1['new_package'] = package
elif identifier == 'USDA-8000': # is R2
assert r2['id'] == row['id']
r2['new_package'] = package
elif identifier == 'USDA-7000': # is R3
r3['id'] = row['id']
r3['new_package'] = package
elif identifier == 'USDA-6000': # is R4
assert r4['id'] == row['id']
r4['new_package'] = package
else:
assert "You never get here {}".format(row['id']) == False
for row in assing_collection_pkg_id(rows_processed):
datajson_dataset = row['comparison_results']['new_data']
package_show = cpa.show_package(ckan_package_id_or_name=row['id'])
package = package_show['result']
logger.info(f'Assigned package: {package}')
extras = package.get('extras', None)
assert type(extras) == list
if row['id'] == r1['id']: # this is part of r2-0002 dataset
ckan_collection_package_id = [extra['value'] for extra in extras if extra['key'] == 'collection_package_id'][0]
assert ckan_collection_package_id == r2['id']
elif row['id'] == r4['id']: # this is part of r2-0002 dataset
ckan_collection_package_id = [extra['value'] for extra in extras if extra['key'] == 'collection_package_id'][0]
assert ckan_collection_package_id == r2['id']
elif row['id'] == r3['id']: # this has a unknown father
ckan_collection_package_id = [extra['value'] for extra in extras if extra['key'] == 'collection_package_id']
assert [] == ckan_collection_package_id
elif row['id'] == r2['id']: # this has no father
ckan_collection_package_id = [extra['value'] for extra in extras if extra['key'] == 'collection_package_id']
assert [] == ckan_collection_package_id
else:
assert "You never get here {}".format(row['id']) == False
def assing_collection_pkg_id(rows):
""" detect new CKAN ids for collections.
The IDs are at different rows so we need to iterate all rows
"""
# create a list of datajson identifiers -> CKAN indetifiers
# to detect collection IDs
related_ids = {}
need_update_rows = [] # need to save the collection_pkg_id
for row in rows:
comparison_results = row['comparison_results']
action = comparison_results['action']
if action not in ['update', 'create']:
yield row
else:
datajson_dataset = comparison_results['new_data']
old_identifier = datajson_dataset['identifier'] # ID at data.json
# If I'm creating a new resource that not exists at CKAN then I have no ID
new_identifier = row.get('id', None) # ID at CKAN
related_ids[old_identifier] = new_identifier
# if is part of a collection, get the CKAN ID
is_part_of = datajson_dataset.get('isPartOf', None)
if is_part_of is None:
yield row
else:
need_update_rows.append(row)
cpa = CKANPortalAPI(base_url=config.CKAN_CATALOG_URL,
api_key=config.CKAN_API_KEY)
for row in need_update_rows:
comparison_results = row['comparison_results']
datajson_dataset = comparison_results['new_data']
old_identifier = datajson_dataset['isPartOf'] # ID at data.json
new_ckan_identifier = related_ids.get(old_identifier, None)
if new_ckan_identifier is not None:
res3 = cpa.show_package(ckan_package_id_or_name=row['id'])
if res3['success'] != True:
error = 'Unable to read package for update collection_pkg_id'
comparison_results['action_results']['errors'].append(error)
else:
# update ckan package
ckan_dataset = res3['result']
ckan_dataset = set_extra(ckan_dataset=ckan_dataset,
key='collection_package_id',
value=new_ckan_identifier)
try:
ckan_response = cpa.update_package(
ckan_package=ckan_dataset)
except Exception as e:
error = f'Error updating collection_package_id at {ckan_dataset}: {e}'
comparison_results['action_results']['errors'].append(
error)
else:
error = f'Unable to detect the collection_pkg_id at {row}'
comparison_results['action_results']['errors'].append(error)
yield row