def get_upcoming_releases(context, data_dict):
# noinspection PyUnresolvedReferences
"""
Return all records with a publish_status_code of Verified (08) and a
release date between the two parameters.
:param startDate: Beginning of date range
:param endDate: End of date range
:returns: All matching results.
:rtype: list of dicts
"""
# TODO: date validation? anything else?
context['ignore_capacity_check'] = True
lc = ckanapi.LocalCKAN(context=context)
start_date = to_utc(_get_or_bust(data_dict, 'startDate'),
def_date=default_release_date)
end_date = to_utc(_get_or_bust(data_dict, 'endDate'),
def_date=default_release_date)
result = lc.action.package_search(
q=(
'last_release_date:[{start_date}Z TO {end_date}Z] '
'AND publish_status_code:8'
).format(
start_date=start_date,
end_date=end_date
),
rows=500
)
# Per JIRA #5173, return an empty list instead of the standard
# NotFound Exception (404).
results = result.get('results', [])
# Per JIRA #5206, resolve parent product types and URLs.
for release_result in results:
if not release_result.get('parent_id'):
continue
parent_result = lc.action.package_search(
q='product_id_new:{pid}'.format(pid=release_result['parent_id']),
rows=1
)
if parent_result['results']:
parent_result = parent_result['results'][0]
else:
continue
release_result.update({
'title': parent_result.get('title'),
'url': parent_result.get('url'),
'product_type': parent_result.get('type')
})
return {'count': result['count'], 'results': results}
def update_release_date_and_status(context, data_dict):
"""
Update the release date and publishing status code for the parent
record of a given product.
:param productId: product ID
:type productId: str
:param productType: product type code
:type productType: str
:param releaseDate: release date
:type releaseDate: str
:param publishingStatus: publish status code
:type publishingStatus: str
:param status: status code
:type status: str
:return: updated package
:rtype: dict
"""
# the client explicitly asked that we accept product_type as a parameter
# even though it can be grabbed from the existing dataset
product_id = _get_or_bust(data_dict, 'productId')
product_type = _get_or_bust(data_dict, 'productType')
release_date = to_utc(_get_or_bust(data_dict, 'releaseDate'),
def_date=default_release_date)
publishing_status = _get_or_bust(data_dict, 'publishingStatus')
status = _get_or_bust(data_dict, 'status')
business_logic = {'10': {'type': 'cube',
'update_product': True,
'update_children': True},
'11': {'type': 'view',
'update_product': False,
'update_children': False},
'12': {'type': 'indicator',
'update_product': False,
'update_children': False},
'13': {'type': 'chart',
'update_product': False,
'update_children': False},
'14': {'type': 'map',
'update_product': False,
'update_children': False},
'20': {'type': 'publication',
'update_product': True,
'update_children': False},
'21': {'type': 'video',
'update_product': True,
'update_children': False},
'22': {'type': 'conference',
'update_product': True,
'update_children': False},
'23': {'type': 'service',
'update_product': True,
'update_children': False},
'24': {'type': 'daily',
'update_product': True,
'update_children': False},
'25': {'type': 'pumf',
'update_product': True,
'update_children': False},
'26': {'type': 'generic',
'update_product': False,
'update_children': False},
}
updated_products = []
def _update_product(product_id,
product_type,
release_date,
publishing_status):
new_values = {'last_release_date': release_date,
'publishing_status': publishing_status,
'status_code': status
}
lc = ckanapi.LocalCKAN(context=context)
result = lc.action.package_search(
q=(
'type:{product_type} AND '
'product_id_new:{product_id}'
).format(
product_type=business_logic[product_type]['type'],
product_id=product_id),
rows=1
)
if not result['count']:
raise _NotFound('Product not found')
elif result['count'] > 1:
raise _ValidationError(
'More than one product with given productid found'
)
product = result['results'][0]
product.update(new_values)
lc.action.package_update(**product)
updated_products.append(product['product_id_new'])
def _update_children(product_id,
release_date,
publishing_status):
new_values = {'last_release_date': release_date,
'publishing_status': publishing_status,
'status_code': status
}
lc = ckanapi.LocalCKAN(context=context)
response = lc.action.package_search(
q=(
'(type:view OR '
'type:indicator OR '
'type:chart OR '
'type:map) AND '
'top_parent_id:{top_parent_id}'
).format(top_parent_id=product_id),
rows=1000
)
if response['count'] > 0:
for product in response['results']:
product.update(new_values)
lc.action.package_update(**product)
updated_products.append(product['product_id_new'])
if business_logic[product_type]['update_product']:
_update_product(product_id,
product_type,
release_date,
publishing_status)
if business_logic[product_type]['update_children']:
_update_children(product_id,
release_date,
publishing_status)
return {'updated_products': updated_products}
def get_issues_by_pub_status(context, data_dict):
# noinspection PyUnresolvedReferences
"""
Fields (listed below) for all product issues of type "productType" with a
last_publish_status_code equal to that passed in with a release date
between the two input parameters
:param lastPublishStatusCode: Possible values are outlined on
https://confluence.statcan.ca/display/NDMA/Publishing+workflow
:param startReleaseDate: Beginning of date range
:param endReleaseDate: End of date range
:param productTypeCode: Possible values are outlined on
https://confluence.statcan.ca/pages/viewpage.action?pageId=20416770.
If no product type is passed in, assume all product types are
requested.
:rtype: list of dicts
"""
# TODO: date validation? anything else?
last_publish_status_code = _get_or_bust(
data_dict,
'lastPublishStatusCode'
)
start_release_date = to_utc(_get_or_bust(data_dict, 'startReleaseDate'),
def_date=default_release_date)
end_release_date = to_utc(_get_or_bust(data_dict, 'endReleaseDate'),
def_date=default_release_date)
if 'productType' in data_dict and data_dict['productType']:
product_type_code = data_dict['productTypeCode']
else:
product_type_code = '["" TO *]'
q = (
'last_release_date:[{startReleaseDate}Z TO {endReleaseDate}Z] AND '
'last_publish_status_code:{lastPublishStatusCode} AND '
'product_type_code:{productTypeCode}'
).format(
startReleaseDate=start_release_date,
endReleaseDate=end_release_date,
lastPublishStatusCode=last_publish_status_code,
productTypeCode=product_type_code
)
lc = ckanapi.LocalCKAN()
i = 0
n = 1
while i < n:
query_results = lc.action.package_search(
q=q,
rows=1000,
start=i*1000)
i += 1
n = query_results['count'] / 1000.0
count = query_results['count']
if count == 0:
raise _NotFound
desired_extras = [
'product_type_code',
'product_id_new',
'issue_no',
'correction_id',
'last_publish_status_code',
'release_date',
'reference_period',
'url'
]
output = []
for result in query_results['results']:
result_dict = {}
for extra in desired_extras:
if extra in result:
result_dict[extra] = result[extra] or ''
output.append(result_dict)
return {'count': count, 'results': output}
def consume_transaction_file(context, data_dict):
# noinspection PyUnresolvedReferences
"""
Triggers a background task to start consuming the transaction file.
:param transactionFile: Daily registration transactions
:type transactionFile: dict
"""
try:
def my_get(a_data_dict, key, expected):
value = a_data_dict.get(key)
if not value:
raise _ValidationError({key: u'Missing value'})
if not isinstance(value, expected):
raise _ValidationError(
{key: u'Invalid format ({value}), '
u'expecting a {type}'.format(
value=value,
type=expected.__name__)})
return value
if u'transactionFile' not in data_dict:
transaction_ssh_host = config.get(
'ckanext.stcndm.transaction_ssh_host')
if not transaction_ssh_host:
raise _NotFound({
u'transactionFile': u'ckanext.stcndm.transaction_ssh_host '
u'missing from CKAN config file'})
transaction_ssh_user = config.get(
'ckanext.stcndm.transaction_ssh_user')
if not transaction_ssh_user:
raise _NotFound({
u'transactionFile': u'ckanext.stcndm.transaction_ssh_user '
u'missing from CKAN config file'})
transaction_ssh_path = config.get(
'ckanext.stcndm.transaction_ssh_path')
if not transaction_ssh_path:
raise _NotFound({
u'transactionFile': u'ckanext.stcndm.transaction_ssh_path '
u'missing from CKAN config file'})
from paramiko import (SSHClient,
AuthenticationException,
SSHException)
from socket import gaierror
client = SSHClient()
client.load_system_host_keys()
try:
client.connect(transaction_ssh_host,
username=transaction_ssh_user)
stdin, stdout, stderr = client.exec_command(
'cat '+transaction_ssh_path)
data_dict = json.loads(stdout.read())
except gaierror as e:
raise _NotFound({
u'transactionFile': 'Invalid host: ' +
transaction_ssh_host + ' : ' + e[1]
})
except ValueError as e:
raise _ValidationError({
u'transactionFile': 'is ' + transaction_ssh_path +
' the correct path to the '
'transaction file?: ' + e.message
})
except AuthenticationException as e:
raise _NotAuthorized({
u'transactionFile': 'ssh ' +
transaction_ssh_user + '@' +
transaction_ssh_host + ' failed: ' +
e.message
})
except SSHException as e:
raise _ValidationError({
u'transactionFile': 'ssh mis-configured: ' +
e.message
})
transaction_dict = my_get(data_dict, u'transactionFile', dict)
daily_dict = my_get(transaction_dict, u'daily', dict)
release_date_text = my_get(daily_dict, u'release_date', basestring)
releases = my_get(daily_dict, u'release', list)
lc = ckanapi.LocalCKAN(context=context)
results = []
for release in releases:
if not isinstance(release, dict):
raise _ValidationError({
u'release': u'Invalid format, '
u'expecting a list of dict'
})
product_id = my_get(release, u'id', basestring)
letter_id = my_get(release, u'letter_id', basestring)
stats_in_brief = my_get(release, u'stats_in_brief', basestring)
title = {
u'en': my_get(release, u'title_en', basestring),
u'fr': my_get(release, u'title_fr', basestring)
}
reference_period = {
u'en': release.get(u'refper_en'),
u'fr': release.get(u'refper_fr')
}
theme_list = my_get(release, u'theme', list)
cube_list = release.get(u'cube', [])
survey_list = release.get(u'survey', [])
product_list = release.get(u'product', [])
url = {
u'en': u'/daily-quotidien/{release_date}/dq{release_date}'
u'{letter_id}-eng.htm'.format(
release_date=release_date_text[:10],
letter_id=letter_id
),
u'fr': u'/daily-quotidien/{release_date}/dq{release_date}'
u'{letter_id}-fra.htm'.format(
release_date=release_date_text[:10],
letter_id=letter_id
)
}
try:
results.append(lc.action.RegisterDaily(
** {
u'releaseDate': to_utc(release_date_text,
def_date=default_release_date),
u'productId': u'00240001' + product_id,
u'statsInBrief': stats_in_brief,
u'productTitle': title,
u'referencePeriod': reference_period,
u'themeList': theme_list,
u'cubeList': cube_list,
u'surveyList': survey_list,
u'productList': product_list,
u'url': url
}
))
except _ValidationError as ve:
results.append({
u'product_id_new': u'00240001'+product_id,
u'error': ve.error_dict})
stcndm_helpers.write_audit_log("consume_transaction_file", results)
except _ValidationError as ve:
stcndm_helpers.write_audit_log("consume_transaction_file", ve, 3)
raise ve
return results
def consume_transaction_file(context, data_dict):
# noinspection PyUnresolvedReferences
"""
Triggers a background task to start consuming the transaction file.
:param transactionFile: Daily registration transactions
:type transactionFile: dict
"""
try:
def my_get(a_data_dict, key, expected):
value = a_data_dict.get(key)
if not value:
raise _ValidationError({key: u'Missing value'})
if not isinstance(value, expected):
raise _ValidationError({
key:
u'Invalid format ({value}), '
u'expecting a {type}'.format(value=value,
type=expected.__name__)
})
return value
if u'transactionFile' not in data_dict:
transaction_ssh_host = config.get(
'ckanext.stcndm.transaction_ssh_host')
if not transaction_ssh_host:
raise _NotFound({
u'transactionFile':
u'ckanext.stcndm.transaction_ssh_host '
u'missing from CKAN config file'
})
transaction_ssh_user = config.get(
'ckanext.stcndm.transaction_ssh_user')
if not transaction_ssh_user:
raise _NotFound({
u'transactionFile':
u'ckanext.stcndm.transaction_ssh_user '
u'missing from CKAN config file'
})
transaction_ssh_path = config.get(
'ckanext.stcndm.transaction_ssh_path')
if not transaction_ssh_path:
raise _NotFound({
u'transactionFile':
u'ckanext.stcndm.transaction_ssh_path '
u'missing from CKAN config file'
})
from paramiko import (SSHClient, AuthenticationException,
SSHException)
from socket import gaierror
client = SSHClient()
client.load_system_host_keys()
try:
client.connect(transaction_ssh_host,
username=transaction_ssh_user)
stdin, stdout, stderr = client.exec_command(
'cat ' + transaction_ssh_path)
data_dict = json.loads(stdout.read())
except gaierror as e:
raise _NotFound({
u'transactionFile':
'Invalid host: ' + transaction_ssh_host + ' : ' + e[1]
})
except ValueError as e:
raise _ValidationError({
u'transactionFile':
'is ' + transaction_ssh_path + ' the correct path to the '
'transaction file?: ' + e.message
})
except AuthenticationException as e:
raise _NotAuthorized({
u'transactionFile':
'ssh ' + transaction_ssh_user + '@' +
transaction_ssh_host + ' failed: ' + e.message
})
except SSHException as e:
raise _ValidationError(
{u'transactionFile': 'ssh mis-configured: ' + e.message})
transaction_dict = my_get(data_dict, u'transactionFile', dict)
daily_dict = my_get(transaction_dict, u'daily', dict)
release_date_text = my_get(daily_dict, u'release_date', basestring)
releases = my_get(daily_dict, u'release', list)
lc = ckanapi.LocalCKAN(context=context)
results = []
for release in releases:
if not isinstance(release, dict):
raise _ValidationError({
u'release':
u'Invalid format, '
u'expecting a list of dict'
})
product_id = my_get(release, u'id', basestring)
letter_id = my_get(release, u'letter_id', basestring)
stats_in_brief = my_get(release, u'stats_in_brief', basestring)
title = {
u'en': my_get(release, u'title_en', basestring),
u'fr': my_get(release, u'title_fr', basestring)
}
reference_period = {
u'en': release.get(u'refper_en'),
u'fr': release.get(u'refper_fr')
}
theme_list = my_get(release, u'theme', list)
cube_list = release.get(u'cube', [])
survey_list = release.get(u'survey', [])
product_list = release.get(u'product', [])
url = {
u'en':
u'/daily-quotidien/{release_date}/dq{release_date}'
u'{letter_id}-eng.htm'.format(
release_date=release_date_text[:10], letter_id=letter_id),
u'fr':
u'/daily-quotidien/{release_date}/dq{release_date}'
u'{letter_id}-fra.htm'.format(
release_date=release_date_text[:10], letter_id=letter_id)
}
try:
results.append(
lc.action.RegisterDaily(
**{
u'releaseDate':
to_utc(release_date_text,
def_date=default_release_date),
u'productId':
u'00240001' + product_id,
u'statsInBrief':
stats_in_brief,
u'productTitle':
title,
u'referencePeriod':
reference_period,
u'themeList':
theme_list,
u'cubeList':
cube_list,
u'surveyList':
survey_list,
u'productList':
product_list,
u'url':
url
}))
except _ValidationError as ve:
results.append({
u'product_id_new': u'00240001' + product_id,
u'error': ve.error_dict
})
stcndm_helpers.write_audit_log("consume_transaction_file", results)
except _ValidationError as ve:
stcndm_helpers.write_audit_log("consume_transaction_file", ve, 3)
raise ve
return results
def register_daily(context, data_dict):
# noinspection PyUnresolvedReferences
"""
Register a Daily.
Automatically populate fields based on provided parameters.
:param releaseDate: e.g. 2015-06-30T09:45
:type releaseDate: datetime - required
:param productId: 00240001 followed by 3 - 6 digit
sequence id (i.e. 00240001654321)
:type productId: str - required
:param statsInBrief: 0 - do not display in navigation
1 - display in navigation
:type statsInBrief: str - required
:param productTitle: EN/FR title
:type productTitle: dict - required
:param referencePeriod: EN/FR reference period
:type referencePeriod: dict - required
:param themeList: list of theme IDs
:type themeList: list - required
:param cubeList: list of related cube IDs
:type cubeList: list - optional
:param surveyList: list of theme IDs
:type surveyList: list - optional
:param productList: list of IDs of child products
:type productList: list - optional
:param url: EN/FR url
:type url: dict - required
:return: new package
:rtype: dict
:raises: ValidationError
"""
def my_get(a_data_dict, key, expected, required=True):
value = a_data_dict.get(key)
if value:
if not isinstance(value, expected):
raise _ValidationError({
key:
u'Invalid format ({value}), '
u'expecting a {type}'.format(value=value,
type=expected.__name__)
})
elif required:
raise _ValidationError({key: u'Missing value'})
return value
release_date_str = my_get(data_dict, u'releaseDate', basestring)
product_id = my_get(data_dict, u'productId', basestring)
if not re.match('^00240001[0-9]{3,6}$', product_id):
raise _ValidationError({
u'product_id':
u'Invalid ({product_id}), '
u'expected 00240001 followed by '
u'3 - 6 digit sequence number'.format(product_id=product_id)
})
lc = ckanapi.LocalCKAN(context=context)
stats_in_brief = my_get(data_dict, u'statsInBrief', basestring)
if stats_in_brief not in [u'0', u'1']:
raise _ValidationError({
u'stats_in_brief':
u'Invalid ({value}), expecting 0 or 1 '.format(
value=stats_in_brief)
})
if stats_in_brief == u'0':
display_code = u'3' # Hide Product Info page & Navigation Link
content_type_codes = [u'2015'] # Other
else:
display_code = u'1' # Show Product Info Page
content_type_codes = [u'2016'] # Analysis/Stats in brief
product_title = my_get(data_dict, u'productTitle', dict)
reference_period = my_get(data_dict, u'referencePeriod', dict)
theme_list = my_get(data_dict, u'themeList', list)
related = []
cube_list = my_get(data_dict, u'cubeList', list, required=False)
if cube_list:
related.extend(cube_list)
survey_list = my_get(data_dict, u'surveyList', list, required=False)
if survey_list:
related.extend(survey_list)
product_list = my_get(data_dict, u'productList', list, required=False)
if product_list:
related.extend(product_list)
url = my_get(data_dict, u'url', dict)
daily_dict = {
u'name':
u'daily-{0}'.format(product_id),
u'owner_org':
u'statcan',
u'private':
False,
u'product_type_code':
u'24',
u'type':
u'daily',
u'content_type_codes':
content_type_codes,
u'last_release_date':
to_utc(release_date_str, def_date=default_release_date),
u'product_id_new':
product_id,
u'display_code':
display_code,
u'title':
product_title,
u'reference_period':
reference_period,
u'subject_codes':
theme_list,
u'related_products':
related if related else missing,
u'url':
url
}
try:
new_product = lc.action.package_create(**daily_dict)
except _ValidationError as e:
if e.error_dict.get('name', []) == [u'That URL is already in use.']:
new_product = lc.action.package_update(**daily_dict)
else:
raise
new_product = lc.action.GetProduct(productId=new_product['product_id_new'],
fl='product_id_new')
set_related_id(product_id, related)
return new_product
