def get_upcoming_releases(context, data_dict): # noinspection PyUnresolvedReferences """ Return all records with a publish_status_code of Verified (08) and a release date between the two parameters. :param startDate: Beginning of date range :param endDate: End of date range :returns: All matching results. :rtype: list of dicts """ # TODO: date validation? anything else? context['ignore_capacity_check'] = True lc = ckanapi.LocalCKAN(context=context) start_date = to_utc(_get_or_bust(data_dict, 'startDate'), def_date=default_release_date) end_date = to_utc(_get_or_bust(data_dict, 'endDate'), def_date=default_release_date) result = lc.action.package_search( q=( 'last_release_date:[{start_date}Z TO {end_date}Z] ' 'AND publish_status_code:8' ).format( start_date=start_date, end_date=end_date ), rows=500 ) # Per JIRA #5173, return an empty list instead of the standard # NotFound Exception (404). results = result.get('results', []) # Per JIRA #5206, resolve parent product types and URLs. for release_result in results: if not release_result.get('parent_id'): continue parent_result = lc.action.package_search( q='product_id_new:{pid}'.format(pid=release_result['parent_id']), rows=1 ) if parent_result['results']: parent_result = parent_result['results'][0] else: continue release_result.update({ 'title': parent_result.get('title'), 'url': parent_result.get('url'), 'product_type': parent_result.get('type') }) return {'count': result['count'], 'results': results}
def update_release_date_and_status(context, data_dict): """ Update the release date and publishing status code for the parent record of a given product. :param productId: product ID :type productId: str :param productType: product type code :type productType: str :param releaseDate: release date :type releaseDate: str :param publishingStatus: publish status code :type publishingStatus: str :param status: status code :type status: str :return: updated package :rtype: dict """ # the client explicitly asked that we accept product_type as a parameter # even though it can be grabbed from the existing dataset product_id = _get_or_bust(data_dict, 'productId') product_type = _get_or_bust(data_dict, 'productType') release_date = to_utc(_get_or_bust(data_dict, 'releaseDate'), def_date=default_release_date) publishing_status = _get_or_bust(data_dict, 'publishingStatus') status = _get_or_bust(data_dict, 'status') business_logic = {'10': {'type': 'cube', 'update_product': True, 'update_children': True}, '11': {'type': 'view', 'update_product': False, 'update_children': False}, '12': {'type': 'indicator', 'update_product': False, 'update_children': False}, '13': {'type': 'chart', 'update_product': False, 'update_children': False}, '14': {'type': 'map', 'update_product': False, 'update_children': False}, '20': {'type': 'publication', 'update_product': True, 'update_children': False}, '21': {'type': 'video', 'update_product': True, 'update_children': False}, '22': {'type': 'conference', 'update_product': True, 'update_children': False}, '23': {'type': 'service', 'update_product': True, 'update_children': False}, '24': {'type': 'daily', 'update_product': True, 'update_children': False}, '25': {'type': 'pumf', 'update_product': True, 'update_children': False}, '26': {'type': 'generic', 'update_product': False, 'update_children': False}, } updated_products = [] def _update_product(product_id, product_type, release_date, publishing_status): new_values = {'last_release_date': release_date, 'publishing_status': publishing_status, 'status_code': status } lc = ckanapi.LocalCKAN(context=context) result = lc.action.package_search( q=( 'type:{product_type} AND ' 'product_id_new:{product_id}' ).format( product_type=business_logic[product_type]['type'], product_id=product_id), rows=1 ) if not result['count']: raise _NotFound('Product not found') elif result['count'] > 1: raise _ValidationError( 'More than one product with given productid found' ) product = result['results'][0] product.update(new_values) lc.action.package_update(**product) updated_products.append(product['product_id_new']) def _update_children(product_id, release_date, publishing_status): new_values = {'last_release_date': release_date, 'publishing_status': publishing_status, 'status_code': status } lc = ckanapi.LocalCKAN(context=context) response = lc.action.package_search( q=( '(type:view OR ' 'type:indicator OR ' 'type:chart OR ' 'type:map) AND ' 'top_parent_id:{top_parent_id}' ).format(top_parent_id=product_id), rows=1000 ) if response['count'] > 0: for product in response['results']: product.update(new_values) lc.action.package_update(**product) updated_products.append(product['product_id_new']) if business_logic[product_type]['update_product']: _update_product(product_id, product_type, release_date, publishing_status) if business_logic[product_type]['update_children']: _update_children(product_id, release_date, publishing_status) return {'updated_products': updated_products}
def get_issues_by_pub_status(context, data_dict): # noinspection PyUnresolvedReferences """ Fields (listed below) for all product issues of type "productType" with a last_publish_status_code equal to that passed in with a release date between the two input parameters :param lastPublishStatusCode: Possible values are outlined on https://confluence.statcan.ca/display/NDMA/Publishing+workflow :param startReleaseDate: Beginning of date range :param endReleaseDate: End of date range :param productTypeCode: Possible values are outlined on https://confluence.statcan.ca/pages/viewpage.action?pageId=20416770. If no product type is passed in, assume all product types are requested. :rtype: list of dicts """ # TODO: date validation? anything else? last_publish_status_code = _get_or_bust( data_dict, 'lastPublishStatusCode' ) start_release_date = to_utc(_get_or_bust(data_dict, 'startReleaseDate'), def_date=default_release_date) end_release_date = to_utc(_get_or_bust(data_dict, 'endReleaseDate'), def_date=default_release_date) if 'productType' in data_dict and data_dict['productType']: product_type_code = data_dict['productTypeCode'] else: product_type_code = '["" TO *]' q = ( 'last_release_date:[{startReleaseDate}Z TO {endReleaseDate}Z] AND ' 'last_publish_status_code:{lastPublishStatusCode} AND ' 'product_type_code:{productTypeCode}' ).format( startReleaseDate=start_release_date, endReleaseDate=end_release_date, lastPublishStatusCode=last_publish_status_code, productTypeCode=product_type_code ) lc = ckanapi.LocalCKAN() i = 0 n = 1 while i < n: query_results = lc.action.package_search( q=q, rows=1000, start=i*1000) i += 1 n = query_results['count'] / 1000.0 count = query_results['count'] if count == 0: raise _NotFound desired_extras = [ 'product_type_code', 'product_id_new', 'issue_no', 'correction_id', 'last_publish_status_code', 'release_date', 'reference_period', 'url' ] output = [] for result in query_results['results']: result_dict = {} for extra in desired_extras: if extra in result: result_dict[extra] = result[extra] or '' output.append(result_dict) return {'count': count, 'results': output}
def consume_transaction_file(context, data_dict): # noinspection PyUnresolvedReferences """ Triggers a background task to start consuming the transaction file. :param transactionFile: Daily registration transactions :type transactionFile: dict """ try: def my_get(a_data_dict, key, expected): value = a_data_dict.get(key) if not value: raise _ValidationError({key: u'Missing value'}) if not isinstance(value, expected): raise _ValidationError( {key: u'Invalid format ({value}), ' u'expecting a {type}'.format( value=value, type=expected.__name__)}) return value if u'transactionFile' not in data_dict: transaction_ssh_host = config.get( 'ckanext.stcndm.transaction_ssh_host') if not transaction_ssh_host: raise _NotFound({ u'transactionFile': u'ckanext.stcndm.transaction_ssh_host ' u'missing from CKAN config file'}) transaction_ssh_user = config.get( 'ckanext.stcndm.transaction_ssh_user') if not transaction_ssh_user: raise _NotFound({ u'transactionFile': u'ckanext.stcndm.transaction_ssh_user ' u'missing from CKAN config file'}) transaction_ssh_path = config.get( 'ckanext.stcndm.transaction_ssh_path') if not transaction_ssh_path: raise _NotFound({ u'transactionFile': u'ckanext.stcndm.transaction_ssh_path ' u'missing from CKAN config file'}) from paramiko import (SSHClient, AuthenticationException, SSHException) from socket import gaierror client = SSHClient() client.load_system_host_keys() try: client.connect(transaction_ssh_host, username=transaction_ssh_user) stdin, stdout, stderr = client.exec_command( 'cat '+transaction_ssh_path) data_dict = json.loads(stdout.read()) except gaierror as e: raise _NotFound({ u'transactionFile': 'Invalid host: ' + transaction_ssh_host + ' : ' + e[1] }) except ValueError as e: raise _ValidationError({ u'transactionFile': 'is ' + transaction_ssh_path + ' the correct path to the ' 'transaction file?: ' + e.message }) except AuthenticationException as e: raise _NotAuthorized({ u'transactionFile': 'ssh ' + transaction_ssh_user + '@' + transaction_ssh_host + ' failed: ' + e.message }) except SSHException as e: raise _ValidationError({ u'transactionFile': 'ssh mis-configured: ' + e.message }) transaction_dict = my_get(data_dict, u'transactionFile', dict) daily_dict = my_get(transaction_dict, u'daily', dict) release_date_text = my_get(daily_dict, u'release_date', basestring) releases = my_get(daily_dict, u'release', list) lc = ckanapi.LocalCKAN(context=context) results = [] for release in releases: if not isinstance(release, dict): raise _ValidationError({ u'release': u'Invalid format, ' u'expecting a list of dict' }) product_id = my_get(release, u'id', basestring) letter_id = my_get(release, u'letter_id', basestring) stats_in_brief = my_get(release, u'stats_in_brief', basestring) title = { u'en': my_get(release, u'title_en', basestring), u'fr': my_get(release, u'title_fr', basestring) } reference_period = { u'en': release.get(u'refper_en'), u'fr': release.get(u'refper_fr') } theme_list = my_get(release, u'theme', list) cube_list = release.get(u'cube', []) survey_list = release.get(u'survey', []) product_list = release.get(u'product', []) url = { u'en': u'/daily-quotidien/{release_date}/dq{release_date}' u'{letter_id}-eng.htm'.format( release_date=release_date_text[:10], letter_id=letter_id ), u'fr': u'/daily-quotidien/{release_date}/dq{release_date}' u'{letter_id}-fra.htm'.format( release_date=release_date_text[:10], letter_id=letter_id ) } try: results.append(lc.action.RegisterDaily( ** { u'releaseDate': to_utc(release_date_text, def_date=default_release_date), u'productId': u'00240001' + product_id, u'statsInBrief': stats_in_brief, u'productTitle': title, u'referencePeriod': reference_period, u'themeList': theme_list, u'cubeList': cube_list, u'surveyList': survey_list, u'productList': product_list, u'url': url } )) except _ValidationError as ve: results.append({ u'product_id_new': u'00240001'+product_id, u'error': ve.error_dict}) stcndm_helpers.write_audit_log("consume_transaction_file", results) except _ValidationError as ve: stcndm_helpers.write_audit_log("consume_transaction_file", ve, 3) raise ve return results
def consume_transaction_file(context, data_dict): # noinspection PyUnresolvedReferences """ Triggers a background task to start consuming the transaction file. :param transactionFile: Daily registration transactions :type transactionFile: dict """ try: def my_get(a_data_dict, key, expected): value = a_data_dict.get(key) if not value: raise _ValidationError({key: u'Missing value'}) if not isinstance(value, expected): raise _ValidationError({ key: u'Invalid format ({value}), ' u'expecting a {type}'.format(value=value, type=expected.__name__) }) return value if u'transactionFile' not in data_dict: transaction_ssh_host = config.get( 'ckanext.stcndm.transaction_ssh_host') if not transaction_ssh_host: raise _NotFound({ u'transactionFile': u'ckanext.stcndm.transaction_ssh_host ' u'missing from CKAN config file' }) transaction_ssh_user = config.get( 'ckanext.stcndm.transaction_ssh_user') if not transaction_ssh_user: raise _NotFound({ u'transactionFile': u'ckanext.stcndm.transaction_ssh_user ' u'missing from CKAN config file' }) transaction_ssh_path = config.get( 'ckanext.stcndm.transaction_ssh_path') if not transaction_ssh_path: raise _NotFound({ u'transactionFile': u'ckanext.stcndm.transaction_ssh_path ' u'missing from CKAN config file' }) from paramiko import (SSHClient, AuthenticationException, SSHException) from socket import gaierror client = SSHClient() client.load_system_host_keys() try: client.connect(transaction_ssh_host, username=transaction_ssh_user) stdin, stdout, stderr = client.exec_command( 'cat ' + transaction_ssh_path) data_dict = json.loads(stdout.read()) except gaierror as e: raise _NotFound({ u'transactionFile': 'Invalid host: ' + transaction_ssh_host + ' : ' + e[1] }) except ValueError as e: raise _ValidationError({ u'transactionFile': 'is ' + transaction_ssh_path + ' the correct path to the ' 'transaction file?: ' + e.message }) except AuthenticationException as e: raise _NotAuthorized({ u'transactionFile': 'ssh ' + transaction_ssh_user + '@' + transaction_ssh_host + ' failed: ' + e.message }) except SSHException as e: raise _ValidationError( {u'transactionFile': 'ssh mis-configured: ' + e.message}) transaction_dict = my_get(data_dict, u'transactionFile', dict) daily_dict = my_get(transaction_dict, u'daily', dict) release_date_text = my_get(daily_dict, u'release_date', basestring) releases = my_get(daily_dict, u'release', list) lc = ckanapi.LocalCKAN(context=context) results = [] for release in releases: if not isinstance(release, dict): raise _ValidationError({ u'release': u'Invalid format, ' u'expecting a list of dict' }) product_id = my_get(release, u'id', basestring) letter_id = my_get(release, u'letter_id', basestring) stats_in_brief = my_get(release, u'stats_in_brief', basestring) title = { u'en': my_get(release, u'title_en', basestring), u'fr': my_get(release, u'title_fr', basestring) } reference_period = { u'en': release.get(u'refper_en'), u'fr': release.get(u'refper_fr') } theme_list = my_get(release, u'theme', list) cube_list = release.get(u'cube', []) survey_list = release.get(u'survey', []) product_list = release.get(u'product', []) url = { u'en': u'/daily-quotidien/{release_date}/dq{release_date}' u'{letter_id}-eng.htm'.format( release_date=release_date_text[:10], letter_id=letter_id), u'fr': u'/daily-quotidien/{release_date}/dq{release_date}' u'{letter_id}-fra.htm'.format( release_date=release_date_text[:10], letter_id=letter_id) } try: results.append( lc.action.RegisterDaily( **{ u'releaseDate': to_utc(release_date_text, def_date=default_release_date), u'productId': u'00240001' + product_id, u'statsInBrief': stats_in_brief, u'productTitle': title, u'referencePeriod': reference_period, u'themeList': theme_list, u'cubeList': cube_list, u'surveyList': survey_list, u'productList': product_list, u'url': url })) except _ValidationError as ve: results.append({ u'product_id_new': u'00240001' + product_id, u'error': ve.error_dict }) stcndm_helpers.write_audit_log("consume_transaction_file", results) except _ValidationError as ve: stcndm_helpers.write_audit_log("consume_transaction_file", ve, 3) raise ve return results
def register_daily(context, data_dict): # noinspection PyUnresolvedReferences """ Register a Daily. Automatically populate fields based on provided parameters. :param releaseDate: e.g. 2015-06-30T09:45 :type releaseDate: datetime - required :param productId: 00240001 followed by 3 - 6 digit sequence id (i.e. 00240001654321) :type productId: str - required :param statsInBrief: 0 - do not display in navigation 1 - display in navigation :type statsInBrief: str - required :param productTitle: EN/FR title :type productTitle: dict - required :param referencePeriod: EN/FR reference period :type referencePeriod: dict - required :param themeList: list of theme IDs :type themeList: list - required :param cubeList: list of related cube IDs :type cubeList: list - optional :param surveyList: list of theme IDs :type surveyList: list - optional :param productList: list of IDs of child products :type productList: list - optional :param url: EN/FR url :type url: dict - required :return: new package :rtype: dict :raises: ValidationError """ def my_get(a_data_dict, key, expected, required=True): value = a_data_dict.get(key) if value: if not isinstance(value, expected): raise _ValidationError({ key: u'Invalid format ({value}), ' u'expecting a {type}'.format(value=value, type=expected.__name__) }) elif required: raise _ValidationError({key: u'Missing value'}) return value release_date_str = my_get(data_dict, u'releaseDate', basestring) product_id = my_get(data_dict, u'productId', basestring) if not re.match('^00240001[0-9]{3,6}$', product_id): raise _ValidationError({ u'product_id': u'Invalid ({product_id}), ' u'expected 00240001 followed by ' u'3 - 6 digit sequence number'.format(product_id=product_id) }) lc = ckanapi.LocalCKAN(context=context) stats_in_brief = my_get(data_dict, u'statsInBrief', basestring) if stats_in_brief not in [u'0', u'1']: raise _ValidationError({ u'stats_in_brief': u'Invalid ({value}), expecting 0 or 1 '.format( value=stats_in_brief) }) if stats_in_brief == u'0': display_code = u'3' # Hide Product Info page & Navigation Link content_type_codes = [u'2015'] # Other else: display_code = u'1' # Show Product Info Page content_type_codes = [u'2016'] # Analysis/Stats in brief product_title = my_get(data_dict, u'productTitle', dict) reference_period = my_get(data_dict, u'referencePeriod', dict) theme_list = my_get(data_dict, u'themeList', list) related = [] cube_list = my_get(data_dict, u'cubeList', list, required=False) if cube_list: related.extend(cube_list) survey_list = my_get(data_dict, u'surveyList', list, required=False) if survey_list: related.extend(survey_list) product_list = my_get(data_dict, u'productList', list, required=False) if product_list: related.extend(product_list) url = my_get(data_dict, u'url', dict) daily_dict = { u'name': u'daily-{0}'.format(product_id), u'owner_org': u'statcan', u'private': False, u'product_type_code': u'24', u'type': u'daily', u'content_type_codes': content_type_codes, u'last_release_date': to_utc(release_date_str, def_date=default_release_date), u'product_id_new': product_id, u'display_code': display_code, u'title': product_title, u'reference_period': reference_period, u'subject_codes': theme_list, u'related_products': related if related else missing, u'url': url } try: new_product = lc.action.package_create(**daily_dict) except _ValidationError as e: if e.error_dict.get('name', []) == [u'That URL is already in use.']: new_product = lc.action.package_update(**daily_dict) else: raise new_product = lc.action.GetProduct(productId=new_product['product_id_new'], fl='product_id_new') set_related_id(product_id, related) return new_product