def consume_transaction_file(context, data_dict):
# noinspection PyUnresolvedReferences
"""
Triggers a background task to start consuming the transaction file.
:param transactionFile: Daily registration transactions
:type transactionFile: dict
"""
try:
def my_get(a_data_dict, key, expected):
value = a_data_dict.get(key)
if not value:
raise _ValidationError({key: u'Missing value'})
if not isinstance(value, expected):
raise _ValidationError(
{key: u'Invalid format ({value}), '
u'expecting a {type}'.format(
value=value,
type=expected.__name__)})
return value
if u'transactionFile' not in data_dict:
transaction_ssh_host = config.get(
'ckanext.stcndm.transaction_ssh_host')
if not transaction_ssh_host:
raise _NotFound({
u'transactionFile': u'ckanext.stcndm.transaction_ssh_host '
u'missing from CKAN config file'})
transaction_ssh_user = config.get(
'ckanext.stcndm.transaction_ssh_user')
if not transaction_ssh_user:
raise _NotFound({
u'transactionFile': u'ckanext.stcndm.transaction_ssh_user '
u'missing from CKAN config file'})
transaction_ssh_path = config.get(
'ckanext.stcndm.transaction_ssh_path')
if not transaction_ssh_path:
raise _NotFound({
u'transactionFile': u'ckanext.stcndm.transaction_ssh_path '
u'missing from CKAN config file'})
from paramiko import (SSHClient,
AuthenticationException,
SSHException)
from socket import gaierror
client = SSHClient()
client.load_system_host_keys()
try:
client.connect(transaction_ssh_host,
username=transaction_ssh_user)
stdin, stdout, stderr = client.exec_command(
'cat '+transaction_ssh_path)
data_dict = json.loads(stdout.read())
except gaierror as e:
raise _NotFound({
u'transactionFile': 'Invalid host: ' +
transaction_ssh_host + ' : ' + e[1]
})
except ValueError as e:
raise _ValidationError({
u'transactionFile': 'is ' + transaction_ssh_path +
' the correct path to the '
'transaction file?: ' + e.message
})
except AuthenticationException as e:
raise _NotAuthorized({
u'transactionFile': 'ssh ' +
transaction_ssh_user + '@' +
transaction_ssh_host + ' failed: ' +
e.message
})
except SSHException as e:
raise _ValidationError({
u'transactionFile': 'ssh mis-configured: ' +
e.message
})
transaction_dict = my_get(data_dict, u'transactionFile', dict)
daily_dict = my_get(transaction_dict, u'daily', dict)
release_date_text = my_get(daily_dict, u'release_date', basestring)
releases = my_get(daily_dict, u'release', list)
lc = ckanapi.LocalCKAN(context=context)
results = []
for release in releases:
if not isinstance(release, dict):
raise _ValidationError({
u'release': u'Invalid format, '
u'expecting a list of dict'
})
product_id = my_get(release, u'id', basestring)
letter_id = my_get(release, u'letter_id', basestring)
stats_in_brief = my_get(release, u'stats_in_brief', basestring)
title = {
u'en': my_get(release, u'title_en', basestring),
u'fr': my_get(release, u'title_fr', basestring)
}
reference_period = {
u'en': release.get(u'refper_en'),
u'fr': release.get(u'refper_fr')
}
theme_list = my_get(release, u'theme', list)
cube_list = release.get(u'cube', [])
survey_list = release.get(u'survey', [])
product_list = release.get(u'product', [])
url = {
u'en': u'/daily-quotidien/{release_date}/dq{release_date}'
u'{letter_id}-eng.htm'.format(
release_date=release_date_text[:10],
letter_id=letter_id
),
u'fr': u'/daily-quotidien/{release_date}/dq{release_date}'
u'{letter_id}-fra.htm'.format(
release_date=release_date_text[:10],
letter_id=letter_id
)
}
try:
results.append(lc.action.RegisterDaily(
** {
u'releaseDate': to_utc(release_date_text,
def_date=default_release_date),
u'productId': u'00240001' + product_id,
u'statsInBrief': stats_in_brief,
u'productTitle': title,
u'referencePeriod': reference_period,
u'themeList': theme_list,
u'cubeList': cube_list,
u'surveyList': survey_list,
u'productList': product_list,
u'url': url
}
))
except _ValidationError as ve:
results.append({
u'product_id_new': u'00240001'+product_id,
u'error': ve.error_dict})
stcndm_helpers.write_audit_log("consume_transaction_file", results)
except _ValidationError as ve:
stcndm_helpers.write_audit_log("consume_transaction_file", ve, 3)
raise ve
return results
def consume_transaction_file(context, data_dict):
# noinspection PyUnresolvedReferences
"""
Triggers a background task to start consuming the transaction file.
:param transactionFile: Daily registration transactions
:type transactionFile: dict
"""
try:
def my_get(a_data_dict, key, expected):
value = a_data_dict.get(key)
if not value:
raise _ValidationError({key: u'Missing value'})
if not isinstance(value, expected):
raise _ValidationError(
{key:
u'Invalid format ({value}), expecting a {type}'.format(
value=value,
type=expected.__name__)})
return value
if u'transactionFile' not in data_dict:
transaction_file = config.get('ckanext.stcndm.transaction_file')
if not transaction_file:
raise _ValidationError({
u'transactionFile': u'Path to transactionFile missing from'
u'CKAN config file'})
try:
transaction_file = open(transaction_file)
data_dict = json.load(transaction_file)
except (IOError, ValueError) as e:
raise _ValidationError({
u'transactionFile': e.message
})
transaction_dict = my_get(data_dict, u'transactionFile', dict)
daily_dict = my_get(transaction_dict, u'daily', dict)
release_date_text = my_get(daily_dict, u'release_date', basestring)
try:
release_date = datetime.datetime.strptime(
release_date_text,
u'%Y-%m-%dT%H:%M:%S'
)
except ValueError:
raise _ValidationError(
{u'release_date':
u'Invalid format ({date_text}), '
u'expecting YYYY-MM-DDTHH:MM:SS'.format(
date_text=release_date_text)})
releases = my_get(daily_dict, u'release', list)
lc = ckanapi.LocalCKAN(context=context)
results = []
for release in releases:
if not isinstance(release, dict):
raise _ValidationError({
u'release': u'Invalid format, '
u'expecting a list of dict'
})
product_id = my_get(release, u'id', basestring)
letter_id = my_get(release, u'letter_id', basestring)
stats_in_brief = my_get(release, u'stats_in_brief', basestring)
title = {
u'en': my_get(release, u'title_en', basestring),
u'fr': my_get(release, u'title_fr', basestring)
}
reference_period = {
u'en': release.get(u'refper_en'),
u'fr': release.get(u'refper_fr')
}
theme_list = my_get(release, u'theme', list)
cube_list = release.get(u'cube', [])
survey_list = release.get(u'survey', [])
product_list = release.get(u'product', [])
url = {
u'en': u'/daily-quotidien/{release_date}/dq{release_date}'
u'{letter_id}-eng.htm'.format(
release_date=release_date.strftime(u'%y%m%d'),
letter_id=letter_id
),
u'fr': u'/daily-quotidien/{release_date}/dq{release_date}'
u'{letter_id}-fra.htm'.format(
release_date=release_date.strftime(u'%y%m%d'),
letter_id=letter_id
)
}
try:
results.append(lc.action.RegisterDaily(
** {
u'releaseDate': release_date_text,
u'productId': u'00240001' + product_id,
u'statsInBrief': stats_in_brief,
u'productTitle': title,
u'referencePeriod': reference_period,
u'themeList': theme_list,
u'cubeList': cube_list,
u'surveyList': survey_list,
u'productList': product_list,
u'url': url
}
))
except _ValidationError as ve:
results.append({
u'product_id_new': u'00240001'+product_id,
u'error': ve.error_dict})
stcndm_helpers.write_audit_log("consume_transaction_file", results)
except _ValidationError as ve:
stcndm_helpers.write_audit_log("consume_transaction_file", ve, 3)
raise ve
return results
def consume_transaction_file(context, data_dict):
# noinspection PyUnresolvedReferences
"""
Triggers a background task to start consuming the transaction file.
:param transactionFile: Daily registration transactions
:type transactionFile: dict
"""
try:
def my_get(a_data_dict, key, expected):
value = a_data_dict.get(key)
if not value:
raise _ValidationError({key: u'Missing value'})
if not isinstance(value, expected):
raise _ValidationError({
key:
u'Invalid format ({value}), '
u'expecting a {type}'.format(value=value,
type=expected.__name__)
})
return value
if u'transactionFile' not in data_dict:
transaction_ssh_host = config.get(
'ckanext.stcndm.transaction_ssh_host')
if not transaction_ssh_host:
raise _NotFound({
u'transactionFile':
u'ckanext.stcndm.transaction_ssh_host '
u'missing from CKAN config file'
})
transaction_ssh_user = config.get(
'ckanext.stcndm.transaction_ssh_user')
if not transaction_ssh_user:
raise _NotFound({
u'transactionFile':
u'ckanext.stcndm.transaction_ssh_user '
u'missing from CKAN config file'
})
transaction_ssh_path = config.get(
'ckanext.stcndm.transaction_ssh_path')
if not transaction_ssh_path:
raise _NotFound({
u'transactionFile':
u'ckanext.stcndm.transaction_ssh_path '
u'missing from CKAN config file'
})
from paramiko import (SSHClient, AuthenticationException,
SSHException)
from socket import gaierror
client = SSHClient()
client.load_system_host_keys()
try:
client.connect(transaction_ssh_host,
username=transaction_ssh_user)
stdin, stdout, stderr = client.exec_command(
'cat ' + transaction_ssh_path)
data_dict = json.loads(stdout.read())
except gaierror as e:
raise _NotFound({
u'transactionFile':
'Invalid host: ' + transaction_ssh_host + ' : ' + e[1]
})
except ValueError as e:
raise _ValidationError({
u'transactionFile':
'is ' + transaction_ssh_path + ' the correct path to the '
'transaction file?: ' + e.message
})
except AuthenticationException as e:
raise _NotAuthorized({
u'transactionFile':
'ssh ' + transaction_ssh_user + '@' +
transaction_ssh_host + ' failed: ' + e.message
})
except SSHException as e:
raise _ValidationError(
{u'transactionFile': 'ssh mis-configured: ' + e.message})
transaction_dict = my_get(data_dict, u'transactionFile', dict)
daily_dict = my_get(transaction_dict, u'daily', dict)
release_date_text = my_get(daily_dict, u'release_date', basestring)
releases = my_get(daily_dict, u'release', list)
lc = ckanapi.LocalCKAN(context=context)
results = []
for release in releases:
if not isinstance(release, dict):
raise _ValidationError({
u'release':
u'Invalid format, '
u'expecting a list of dict'
})
product_id = my_get(release, u'id', basestring)
letter_id = my_get(release, u'letter_id', basestring)
stats_in_brief = my_get(release, u'stats_in_brief', basestring)
title = {
u'en': my_get(release, u'title_en', basestring),
u'fr': my_get(release, u'title_fr', basestring)
}
reference_period = {
u'en': release.get(u'refper_en'),
u'fr': release.get(u'refper_fr')
}
theme_list = my_get(release, u'theme', list)
cube_list = release.get(u'cube', [])
survey_list = release.get(u'survey', [])
product_list = release.get(u'product', [])
url = {
u'en':
u'/daily-quotidien/{release_date}/dq{release_date}'
u'{letter_id}-eng.htm'.format(
release_date=release_date_text[:10], letter_id=letter_id),
u'fr':
u'/daily-quotidien/{release_date}/dq{release_date}'
u'{letter_id}-fra.htm'.format(
release_date=release_date_text[:10], letter_id=letter_id)
}
try:
results.append(
lc.action.RegisterDaily(
**{
u'releaseDate':
to_utc(release_date_text,
def_date=default_release_date),
u'productId':
u'00240001' + product_id,
u'statsInBrief':
stats_in_brief,
u'productTitle':
title,
u'referencePeriod':
reference_period,
u'themeList':
theme_list,
u'cubeList':
cube_list,
u'surveyList':
survey_list,
u'productList':
product_list,
u'url':
url
}))
except _ValidationError as ve:
results.append({
u'product_id_new': u'00240001' + product_id,
u'error': ve.error_dict
})
stcndm_helpers.write_audit_log("consume_transaction_file", results)
except _ValidationError as ve:
stcndm_helpers.write_audit_log("consume_transaction_file", ve, 3)
raise ve
return results
