def test_package_publish_microdata(self):
context = {'model': model, 'user': self.sysadmin['name']}
# Patch requests
url = 'https://microdata.unhcr.org/index.php/api/datasets/create/survey/DATASET'
responses.add_passthru('http')
responses.add(responses.POST,
url,
status=200,
json={
'status': 'success',
'dataset': {
'id': 1
},
})
# Publish to microdata
survey = toolkit.get_action('package_publish_microdata')(
context, {
'id': self.dataset['id'],
'nation': 'nation',
'repoid': 'repoid',
})
# Check calls
call = responses.calls[0]
assert len(responses.calls) == 1
assert call.request.url == url
assert call.request.headers['X-Api-Key'] == 'API-KEY'
assert call.request.headers['Content-Type'] == 'application/json'
assert (json.loads(
call.request.body) == helpers.convert_dataset_to_microdata_survey(
self.dataset, 'nation', 'repoid'))
assert survey[
'url'] == 'https://microdata.unhcr.org/index.php/catalog/1'
def test_convert_dataset_to_microdata_survey_minimal(self):
dataset = factories.Dataset(
operational_purpose_of_data='cartography',
name='dataset',
unit_of_measurement='individual',
keywords=['3', '4'],
archived='False',
data_collector=['acf'],
data_collection_technique='f2f',
sampling_procedure='nonprobability',
)
survey = helpers.convert_dataset_to_microdata_survey(dataset,
nation='nation',
repoid='repoid')
assert_equals(
survey, {
'repositoryid': 'REPOID',
'access_policy': 'na',
'published': 0,
'overwrite': 'no',
'study_desc': {
'title_statement': {
'idno': u'DATASET',
'title': u'Test Dataset'
},
'authoring_entity':
[{
'affiliation': 'UNHCR',
'name': 'Office of the High Commissioner for Refugees'
}],
'study_info': {
'topics': [{
'topic': 'Health'
}, {
'topic': 'Water Sanitation Hygiene'
}],
'nation': [{
'name': 'nation'
}],
'abstract':
'Just another test dataset.',
'analysis_unit':
'individual',
},
'method': {
'data_collection': {
'data_collectors': [
{
'name': 'Action contre la faim'
},
],
'sampling_procedure': 'Non-probability',
'coll_mode': 'Face-to-face interview',
},
},
},
})
def test_convert_dataset_to_microdata_survey(self):
dataset = factories.Dataset(
operational_purpose_of_data='cartography',
name='dataset',
maintainer='maintainer',
maintainer_email='*****@*****.**',
version='1',
tags=[{
'name': 'Keyword1'
}, {
'name': 'Keyword2'
}],
unit_of_measurement='individual',
keywords=['3', '4'],
date_range_start='2015-01-01',
date_range_end='2016-01-01',
geog_coverage='world',
data_collector='ACF,UNHCR',
data_collection_technique='f2f',
sampling_procedure='nonprobability',
data_collection_notes='Notes about data collection',
weight_notes='Notes about weight',
clean_ops_notes='Notes about cleaning',
response_rate_notes='Notes about response',
)
survey = helpers.convert_dataset_to_microdata_survey(dataset,
nation='nation',
repoid='repoid')
assert_equals(
survey, {
'repositoryid': 'REPOID',
'access_policy': 'na',
'published': 0,
'overwrite': 'no',
'study_desc': {
'title_statement': {
'idno': u'DATASET',
'title': u'Test Dataset'
},
'authoring_entity':
[{
'affiliation': 'UNHCR',
'name': 'Office of the High Commissioner for Refugees'
}],
'distribution_statement': {
'contact': [
{
'name': 'maintainer',
'email': '*****@*****.**'
},
],
},
'version_statement': {
'version': '1',
},
'study_info': {
'keywords': [
{
'keyword': 'Keyword1'
},
{
'keyword': 'Keyword2'
},
],
'topics': [{
'topic': 'Health'
}, {
'topic': 'Water Sanitation Hygiene'
}],
'coll_dates': [
{
'start': '2015-01-01',
'end': '2016-01-01'
},
],
'nation': [{
'name': 'nation'
}],
'abstract':
'Just another test dataset.',
'geog_coverage':
'world',
'analysis_unit':
'individual',
},
'method': {
'data_collection': {
'data_collectors': [
{
'name': 'ACF'
},
{
'name': 'UNHCR'
},
],
'sampling_procedure':
'Non-probability',
'coll_mode':
'Face-to-face interview',
'coll_situation':
'Notes about data collection',
'weight':
'Notes about weight',
'cleaning_operations':
'Notes about cleaning',
},
'analysis_info': {
'response_rate': 'Notes about response',
}
},
},
})
def package_publish_microdata(context, data_dict):
default_error = 'Unknown microdata error'
# Get data
dataset_id = data_dict.get('id')
nation = data_dict.get('nation')
repoid = data_dict.get('repoid')
# Check access
toolkit.check_access('sysadmin', context)
api_key = config.get('ckanext.unhcr.microdata_api_key')
if not api_key:
raise toolkit.NotAuthorized('Microdata API Key is not set')
# Get dataset/survey
headers = {'X-Api-Key': api_key}
dataset = toolkit.get_action('package_show')(context, {'id': dataset_id})
survey = helpers.convert_dataset_to_microdata_survey(
dataset, nation, repoid)
idno = survey['study_desc']['title_statement']['idno']
try:
# Publish dataset
url = 'https://microdata.unhcr.org/index.php/api/datasets/create/survey/%s' % idno
response = requests.post(url, headers=headers, json=survey).json()
if response.get('status') != 'success':
raise RuntimeError(str(response.get('errors', default_error)))
template = 'https://microdata.unhcr.org/index.php/catalog/%s'
survey['url'] = template % response['dataset']['id']
survey['resources'] = []
survey['files'] = []
# Pubish resources/files
file_name_counter = {}
if dataset.get('resources', []):
url = 'https://microdata.unhcr.org/index.php/api/datasets/%s/%s'
for resource in dataset.get('resources', []):
# resource
resouce_url = url % (idno, 'resources')
md_resource = helpers.convert_resource_to_microdata_resource(
resource)
response = requests.post(resouce_url,
headers=headers,
json=md_resource).json()
if response.get('status') != 'success':
raise RuntimeError(
str(response.get('errors', default_error)))
survey['resources'].append(response['resource'])
# file
file_url = url % (idno, 'files')
file_name = resource['url'].split('/')[-1]
file_path = helpers.get_resource_file_path(resource)
file_mime = resource['mimetype']
if not file_name or not file_path:
continue
file_name_counter.setdefault(file_name, 0)
file_name_counter[file_name] += 1
if file_name_counter[file_name] > 1:
file_name = helpers.add_file_name_suffix(
file_name, file_name_counter[file_name] - 1)
with open(file_path, 'rb') as file_obj:
file = (file_name, file_obj, file_mime)
response = requests.post(file_url,
headers=headers,
files={
'file': file
}).json()
# TODO: update
# it's a hack to overcome incorrect Microdata responses
# usopported file types fail this way and we are skipping them
if not isinstance(response, dict):
continue
if response.get('status') != 'success':
raise RuntimeError(
str(response.get('errors', default_error)))
survey['files'].append(response)
except requests.exceptions.HTTPError:
log.exception(exception)
raise RuntimeError('Microdata connection failed')
return survey