def csc_dataset_form_value(text): """ :param text: {lang: text} dict or text string Convert "language-text" to users' language by looking up languag in dict or using gettext if not a dict but. If the text doesn't exist look for an available text """ if not text: return u'' if hasattr(text, 'get'): final_text = u'' try: prefer_lang = lang() except: prefer_lang = config.get('ckan.locale_default', 'es') else: try: final_text = text[prefer_lang] except KeyError: pass if not final_text: locale_order = config.get('ckan.locale_order', '').split() for l in locale_order: if l in text and text[l]: final_text = text[l] break return final_text t = gettext(text) if isinstance(t, str): return t.decode('utf-8') return t
def check_config(self): exit = False for key in ('ckanext.azurefilestore.connect_str', 'ckanext.azurefilestore.container_name', 'ckanext.azurefilestore.storage_account', 'ckanext.azurefilestore.account_key'): if not config.get(key): print 'You must set the "{0}" option in your ini file'.format( key) exit = True if exit: sys.exit(1) print 'All configuration options defined' connect_str = config.get('ckanext.azurefilestore.connect_str') container_name = config.get('ckanext.azurefilestore.container_name') blob_service_client = BlobServiceClient.from_connection_string( connect_str) # Check if container exists try: container_client = blob_service_client.get_container_client( container_name) except Exception as e: print 'An error was found while getting or creating the container:' print str(e) sys.exit(1) print 'Configuration OK!'
def _catalog_graph(self, dataset_ref, dataset_dict): data_catalog = BNode() self.g.add((dataset_ref, SCHEMA.includedInDataCatalog, data_catalog)) self.g.add((data_catalog, RDF.type, SCHEMA.DataCatalog)) self.g.add((data_catalog, SCHEMA.name, Literal(config.get('ckan.site_title')))) self.g.add((data_catalog, SCHEMA.description, Literal(config.get('ckan.site_description')))) self.g.add((data_catalog, SCHEMA.url, Literal(config.get('ckan.site_url'))))
def valid_signature_required(secret_prefix): signature_header_name = config.get( plugin_config_prefix + 'signature_header_name', 'X-Hub-Signature') api_secret = config.get(plugin_config_prefix + secret_prefix + '_secret', 'secret') def decorator(func): def wrapper(context, data): if signature_header_name in request.headers: if request.headers[signature_header_name].startswith('sha1='): algo, received_hmac = request.headers[ signature_header_name].rsplit('=') # since python 3, bytes are not directly str so key must be encoded : https://stackoverflow.com/a/43882903/2862821 # else ERROR [ckan.views.api] key: expected bytes or bytearray, but got 'str' computed_hmac = hmac.new(bytes(api_secret, 'utf-8'), request.get_data(), sha1).hexdigest() if received_hmac.lower() != computed_hmac: log.info('Invalid HMAC') raise toolkit.NotAuthorized(_('Invalid HMAC')) else: log.info('Invalid HMAC algo') raise toolkit.ValidationError(_('Invalid HMAC algo')) else: log.info('No HMAC in the header') raise toolkit.NotAuthorized(_("No HMAC in the header")) return func(context, data) return wrapper return decorator
def is_map_viewer_format(resource): viewer_url = config.get('ckanext.geodatagov.spatial_preview.url') viewer_formats = config.get('ckanext.geodatagov.spatial_preview.formats', 'wms kml kmz').strip().split(' ') return viewer_url and resource.get('url') and resource.get( 'format', '').lower() in viewer_formats
def get_name_from_siret(siret): apiKey = config.get('ckanext.ozwillo_organization_api.verifsiret_apikey', '') secretKey = config.get( 'ckanext.ozwillo_organization_api.verifsiret_secretkey', '') url = "https://www.numero-de-siret.com/api/siret?siret=" + siret result = None if apiKey == '' or secretKey == '': log.error( 'Verif-siret config incomplete, please register your api key and api secret in the config file' ) result = '' return result try: get = requests.get(url, auth=(apiKey, secretKey)) if get.status_code != 200: raise requests.ConnectionError() except requests.exceptions.RequestException as err: log.error('An error occurred: {0}'.format(err)) else: try: result = get.json()['array_return'][0]['LIBCOM'] except (IndexError, TypeError, AttributeError): log.error('No organization found for this siret number') finally: return result
def issues_enabled(dataset): '''Returns whether issues are enabled for the given dataset (dict)''' # config options allow you to only enable issues for particular datasets or # organizations datasets_with_issues_enabled = set(toolkit.aslist( config.get('ckanext.issues.enabled_for_datasets') )) organizations_with_issues_enabled = set(toolkit.aslist( config.get('ckanext.issues.enabled_for_organizations') )) if datasets_with_issues_enabled or organizations_with_issues_enabled: if datasets_with_issues_enabled and \ dataset['name'] in datasets_with_issues_enabled: return True elif organizations_with_issues_enabled and \ (dataset.get('organization') or {}).get('name') in \ organizations_with_issues_enabled: return True return False else: extras = dataset.get('extras', []) for extra in extras: if extra.get('key') == 'issues_enabled': return toolkit.asbool(extra.get('value')) else: return toolkit.asbool( config.get('ckanext.issues.enabled_without_extra', True) )
def csc_dataset_form_lang_and_value(text): """ :param text: {lang: text} dict or text string Convert "language-text" to users' language by looking up languag in dict, if the text doesn't exit look for an available text """ if not text: return {'': u''} if hasattr(text, 'get'): final_text = u'' try: prefer_lang = lang() except: prefer_lang = config.get('ckan.locale_default', 'es') else: try: prefer_lang = str(prefer_lang) final_text = text[prefer_lang] except KeyError: pass if not final_text: locale_order = config.get('ckan.locale_order', '').split() for l in locale_order: if l in text and text[l]: final_text = text[l] prefer_lang = l break return {prefer_lang: final_text} return {'': u''}
def _feedback_mail_recipient(recipient_name, recipient_email, sender_name, sender_url, subject, body, headers): """ Assemble and send a feedback email from the provided parts. """ # Flake8 B006: Don't initialize this in the parameter, because # default parameters are shared between calls, and dict is mutable. headers = headers or {} mail_from = config.get('smtp.mail_from') body = add_msg_niceties(recipient_name, body, sender_name, sender_url) msg = MIMEText(body.encode('utf-8'), 'plain', 'utf-8') for key, value in headers.items(): msg[key] = value subject = Header(subject.encode('utf-8'), 'utf-8') msg['Subject'] = subject msg['From'] = _("%s <%s>") % (sender_name, mail_from) msg['To'] = ", ".join(recipient_email) msg['Date'] = Utils.formatdate(time()) msg['X-Mailer'] = "CKAN %s" % __version__ # Send the email using Python's smtplib. smtp_connection = smtplib.SMTP() smtp_server = config.get('smtp.server', 'localhost') smtp_starttls = asbool(config.get('smtp.starttls')) smtp_user = config.get('smtp.user') smtp_password = config.get('smtp.password') smtp_connection.connect(smtp_server) try: # Identify ourselves and prompt the server for supported features. smtp_connection.ehlo() # If 'smtp.starttls' is on in CKAN config, try to put the SMTP # connection into TLS mode. if smtp_starttls: if smtp_connection.has_extn('STARTTLS'): smtp_connection.starttls() # Re-identify ourselves over TLS connection. smtp_connection.ehlo() else: raise MailerException("SMTP server does not support STARTTLS") # If 'smtp.user' is in CKAN config, try to login to SMTP server. if smtp_user: assert smtp_password, ("If smtp.user is configured then " "smtp.password must be configured as well.") smtp_connection.login(smtp_user, smtp_password) smtp_connection.sendmail(mail_from, recipient_email, msg.as_string()) LOG.info("Sent email to %s", ','.join(recipient_email)) except smtplib.SMTPException as ex: msg = '%r' % ex LOG.exception(msg) raise MailerException(msg) finally: smtp_connection.quit()
def configure(self, config): connection_variable = self._config_template % "connection" self.drupal_connection_url = config.get(connection_variable) if not self.drupal_connection_url: raise Exception('YtpDrupalPlugin: required configuration variable missing: %s' % (connection_variable)) self.cancel_url = config.get(self._config_template % "cancel_url", "/cancel-user.php") self._node_type = config.get(self._config_template % "node_type", self._node_type) self._translations_disabled = asbool(config.get(self._config_template % "translations_disabled", "false")) self.engine = sqlalchemy.create_engine(self.drupal_connection_url)
def matomo_snippet(): data = { "matomo_domain": config.get('ckanext.matomo.domain'), "matomo_script_domain": config.get('ckanext.matomo.script_domain', config.get('ckanext.matomo.domain')), "matomo_site_id": config.get('ckanext.matomo.site_id') } return render_snippet("matomo/snippets/matomo.html", data)
def update_organization_dataset_names(old_org_name, new_org_name, org_id): """ This is used to change all pakcgae names when publisher id is changed. Because IATI uses naming convention for dataset which includes publisher id :param old_org_name: str :param new_org_name: str :param new_org_name: str (uuid) :return: None """ # Run this after org is updated completely time.sleep(30) context = { 'model': model, 'session': model.Session, 'site_url': config.get('ckan.site_url'), 'user': config.get('iati.admin_user.name'), 'apikey': config.get('iati.admin_user.api_key'), 'api_version': 3, 'disable_archiver': True } _fq = 'owner_org:{}'.format(org_id) _org_packages = get_action('package_search')(context, { 'fq': _fq, 'rows': 10000, 'include_private': True }) for pkg in _org_packages.get('results', []): # Replace only 1st occurrence log.info("Updating the package: {} with private state - {}".format( pkg['id'], str(pkg['private']))) log.info("Old org: {}".format(old_org_name)) log.info("New org: {}".format(new_org_name)) new_package_name = pkg.get('name', '').replace(old_org_name, new_org_name, 1) log.info("New package name: {}".format(new_package_name)) if pkg.get('name', '') != new_package_name: context["disable_archiver"] = True try: _res = get_action('package_patch')(context, { 'id': pkg['id'], 'name': new_package_name }) except logic.ValidationError as e: log.error(e) except Exception as e: log.error( "Org name change detected but package update failed for some reason" ) log.error(e) return None
def uploaded_file_redirect(self, upload_to, filename): '''Redirect static file requests to their location on azure.''' storage_account = config.get('ckanext.azurefilestore.storage_account') # Remove last characted if it's a slash # if host_name[-1] == '/': # host_name = host_name[:-1] storage_path = AzureUploader.get_storage_path(upload_to) filepath = os.path.join(storage_path, filename) redirect_url = 'https://{storage_account}.blob.core.windows.net/{container_name}/{filepath}'\ .format( container_name=config.get('ckanext.azurefilestore.container_name'), filepath=filepath, storage_account=storage_account) redirect(redirect_url)
def download(context, resource, url_timeout=URL_TIMEOUT, max_content_length=MAX_CONTENT_LENGTH, data_formats=DATA_FORMATS): res = None resource_changed = False link_context = "{}" link_data = json.dumps({ 'url': resource['url'], 'url_timeout': url_timeout }) user_agent_string = config.get('ckanext.archiver.user_agent_string', 'curl/7.35.0') def _download_resource(resource_url, timeout): _request_headers = {'User-Agent': user_agent_string} # Part of 403 url error - if user agent is missing or # some sites do not accept IATI (CKAN) as user agent. # hence setting default user agent to Mozilla/5.0. try: response = requests.get(resource['url'], timeout=url_timeout, headers=_request_headers, verify=True) except Exception, e: request_headers['User-Agent'] = 'curl/7.35.0' response = requests.get(resource['url'], timeout=url_timeout, headers=_request_headers, verify=False) return response
def index(self, resource_id, cn): try: dns = config.get("ckanext.cfpb_ldap_query.base_dns").split('|') except ValueError: flash_error("At least one valid DN must be configured.") redirect_to("get_access_request", resource_id=resource_id, cn=cn) resource = get_action('resource_show')({}, data_dict={ 'id': resource_id }) package = get_action('package_show')({}, data_dict={ 'id': resource['package_id'] }) role_description = get_role(json.loads(resource['db_roles']), cn)['description'] return render( 'ckanext/cfpb-extrafields/access_index.html', { "resource": resource, "package": package, "description": role_description, "cn": cn, "context": context, "dn": dns[0] })
def extract_redirects(publisher_name=None): """ Extracts the required redirects from the database tables and return the result as tuple TODD: Change the below query using sqlalchemy This is data is used to store in the table iati_redirects :return: tuple """ _db_conn = create_engine(config.get('sqlalchemy.url')).connect() if publisher_name: _query = ''' SELECT DISTINCT public.group.id, CAST(CAST(activity.data AS json) ->> 'group' AS json) ->> 'name' AS old_name, CAST(activity.timestamp AS VARCHAR) as activity_recorded_at FROM public.group, activity WHERE public.group.id = activity.object_id AND public.group.name = '{publisher_name}' AND activity.activity_type = 'changed organization' AND public.group.name != CAST(CAST(activity.data AS json) ->> 'group' AS json) ->> 'name' LIMIT 20; '''.format(publisher_name=publisher_name) else: _query = ''' SELECT DISTINCT public.group.id, public.group.name as current_name, CAST(CAST(activity.data AS json) ->> 'group' AS json) ->> 'name' AS old_name FROM public.group, activity WHERE public.group.id = activity.object_id AND activity.activity_type = 'changed organization' AND public.group.name != CAST(CAST(activity.data AS json) ->> 'group' AS json) ->> 'name'; ''' log.info(_query) res = _db_conn.execute(_query) return tuple(res)
def get_drupal_session_token(self, domain, service, cookie_header=''): '''return text of X-CSRF-Token)''' token_url = 'https://' + domain + '/' + service + '/?q=services/session/token' verify_cert = config.get('ckanext.drupal8.development_cert', '') or True token_request = requests.get(token_url, headers={"Cookie": cookie_header}, verify=verify_cert) token = token_request.text return token
def get_resource_info_by_id(cls, resource_id): resource = model.Session.query( model.Resource).filter(model.Resource.id == resource_id).first() res_name = None res_package_name = None res_package_id = None if resource is not None: res_package_name = resource.package.title or resource.package.name res_package_id = resource.package.name name_translated_json = resource.extras.get('name_translated') if name_translated_json is not None: try: name_translated = json.loads(name_translated_json) languages = aslist(config.get('ckan.locales_offered', [])) res_name = next( (name_translated[lang] for lang in languages if name_translated.get(lang)), None) except json.JSONDecodeError: pass res_name = res_name or resource.description or resource.format return [res_name, res_package_name, res_package_id]
def dataportal_admin(): # items = self._get_config_form_items() if request.method == "POST": data = request.form try: data_dict = logic.clean_dict( dict_fns.unflatten(logic.tuplize_dict(logic.parse_params(request.form, ignore_keys=CACHE_PARAMETERS))) ) del data_dict["save"] data = logic.get_action("config_option_update")({"user": g.user}, data_dict) except logic.ValidationError as e: errors = e.error_dict error_summary = e.error_summary extra_vars = {"data": data, "errors": errors, "error_summary": error_summary} return toolkit.render("admin/dataportal.html", extra_vars=extra_vars) return toolkit.redirect_to("dataportaltheme_blueprint.dataportal_admin") else: admin_schema = schema.update_configuration_schema() data = {} for key in admin_schema: data[key] = config.get(key) extra_vars = {"data": data, "errors": {}} return toolkit.render("admin/dataportal.html", extra_vars=extra_vars)
def generate_datapackage_json(package_id): '''Generates the datapackage - metadata that would be saved as datapackage.json. ''' context = {'model': model, 'session': model.Session} dataset = get_action('package_show')(context, {'id': package_id}) # filter out resources that are not suitable for inclusion in the data # package local_ckan = ckanapi.LocalCKAN() dataset, resources_to_include, existing_zip_resource = \ remove_resources_that_should_not_be_included_in_the_datapackage( dataset) # get the datapackage (metadata) datapackage = ckanapi.datapackage.dataset_to_datapackage(dataset) # populate datapackage with the schema from the Datastore data # dictionary ckan_and_datapackage_resources = zip(resources_to_include, datapackage.get('resources', [])) for res, datapackage_res in ckan_and_datapackage_resources: ckanapi.datapackage.populate_datastore_res_fields(ckan=local_ckan, res=res) ckanapi.datapackage.populate_schema_from_datastore( cres=res, dres=datapackage_res) # add in any other dataset fields, if configured fields_to_include = config.get( u'ckanext.downloadall.dataset_fields_to_add_to_datapackage', u'').split() for key in fields_to_include: datapackage[key] = dataset.get(key) return (datapackage, ckan_and_datapackage_resources, existing_zip_resource)
def download_table_template(validation_schema): """ Downloads a CSV template file for the specified validation schema. """ try: validator_config = config.get('ckanext.validator.schema_config') schemed_table = validator_config.get(validation_schema) template = schemed_table.create_template() csv_content = template.to_csv(header=False, index=False, encoding='utf-8') return Response(csv_content, mimetype="text/csv", headers={ "Content-disposition": "attachment; filename=" + str(validation_schema) + ".csv" }) except AttributeError as e: logging.error(e) abort(404, "404 Not Found Error: No schema exists for " + validation_schema) except Exception as e: logging.error(e) abort( 500, "500 Internal server error: Something went wrong whilst " "generating your template " + validation_schema)
def get_url(): kwargs = {'headers': headers, 'timeout': DOWNLOAD_TIMEOUT, 'verify': SSL_VERIFY, 'stream': True} # just gets the headers for now if 'ckan.download_proxy' in config: proxy = config.get('ckan.download_proxy') kwargs['proxies'] = {'http': proxy, 'https': proxy} return requests.get(url, **kwargs)
def _comment_or_issue_report(issue_or_comment, user_ref, dataset_id, session): user_obj = model.User.get(user_ref) try: issue_or_comment.report_abuse(session, user_obj.id) except IntegrityError: session.rollback() raise ReportAlreadyExists( p.toolkit._('Issue has already been reported by this user')) try: # if you're an org admin/editor (can edit the dataset), it gets marked # as abuse immediately context = { 'user': user_ref, 'session': session, 'model': model, } p.toolkit.check_access('package_update', context, data_dict={'id': dataset_id}) issue_or_comment.change_visibility(session, u'hidden') issue_or_comment.abuse_status = issuemodel.AbuseStatus.abuse.value return { 'visibility': issue_or_comment.visibility, 'abuse_reports': issue_or_comment.abuse_reports, 'abuse_status': issue_or_comment.abuse_status } except p.toolkit.NotAuthorized: max_strikes = config.get('ckanext.issues.max_strikes') if (max_strikes and len(issue_or_comment.abuse_reports) >= p.toolkit.asint(max_strikes)): issue_or_comment.change_visibility(session, u'hidden') finally: # commit the IssueReport and changes to the Issue/Comment session.commit()
def save_redirect_in_filestore_directory(data): """ Saves the redirects data as cs in filestore where all the resources are stored i.e. in path /var/lib/ckan/ :param data: :return: """ _redirects_dir = os.path.join(config.get('ckan.storage_path', ''), "redirects") try: if not os.path.isdir(_redirects_dir): os.mkdir(_redirects_dir) except Exception as e: log.error(e) log.error("Error when creating a new directory in redirects path") return None columns = ('publisher_id', 'current_name', 'old_name') try: with open(os.path.join(_redirects_dir, 'redirects.csv'), 'w') as redirects_file: writer = csv.writer(redirects_file, delimiter=',') writer.writerow(columns) for row in data: writer.writerow(row) log.info("CSV file generated in {}".format(_redirects_dir)) except Exception as e: log.error(e) log.error("Failed to write data to csv..") return None return None
def get_types(): _TYPES = [messytables.StringType, messytables.DecimalType, messytables.IntegerType, messytables.DateUtilType] # TODO make this configurable # TYPES = web.app.config.get('TYPES', _TYPES) TYPE_MAPPING = config.get('TYPE_MAPPING', _TYPE_MAPPING) return _TYPES, TYPE_MAPPING
def get_view_data(group): view_data = [] site_url = config.get("ckan.site_url") if group: f_group_packages = logic.get_action("group_package_show") packages = f_group_packages(data_dict={"id": group, "limit": 8}) else: packages = logic.get_action("current_package_list_with_resources")(data_dict={}) ignored_views = ["text_view", "recline_graph_view", "recline_view"] for package in packages: for resource in package["resources"]: if len(view_data) > 7: break # Get view for resource views = logic.get_action("resource_view_list")(data_dict={"id": resource["id"]}) # print(views) for view in views: if view["view_type"] in ignored_views: continue url = "%s/dataset/%s/resource/%s/view/%s" % (site_url, package["name"], resource["id"], view["id"]) link = "%s/dataset/%s/resource/%s" % (site_url, package["name"], resource["id"]) view_data.append({"url": url, "name": view["title"], "link": link}) return view_data
def _send_activation_notification_email(context, organization_dict): model = context['model'] members = p.toolkit.get_action('member_list')(context, { 'id': organization_dict['id'] }) admins = [m for m in members if m[1] == 'user' and m[2] == 'Admin'] subject = config.get('iati.publisher_activation_email_subject', 'IATI Registry Publisher Activation') group_link = urljoin(site_url, '/publisher/' + organization_dict['name']) for admin in admins: user = model.User.get(admin[0]) if user and user.email: user_name = user.fullname or user.name content = emailer.publisher_activation_body_template.format( user_name=user_name.encode('utf8'), group_title=organization_dict['title'].encode('utf8'), group_link=group_link, user_email=user.email, site_url=site_url) emailer.send_email(content, subject, user.email, content_type='html') log.debug( '[email] Publisher activated notification email sent to user {0}' .format(user.name))
def resource_preview_custom(resource, pkg_id): resource_format = resource.get('format', '').lower() if is_map_viewer_format(resource): viewer_url = config.get('ckanext.geodatagov.spatial_preview.url') url = '{viewer_url}?{params}'.format( viewer_url=viewer_url, params=get_map_viewer_params(resource)) return p.toolkit.render_snippet( "dataviewer/snippets/data_preview.html", data={ 'embed': False, 'resource_url': url, 'raw_resource_url': resource['url'] }) elif resource_format in ('web map application', 'arcgis online map') \ and ('webmap=' in resource.get('url') or 'services=' in resource.get('url')): url = resource['url'].replace('viewer.html', 'embedViewer.html') return p.toolkit.render_snippet( "dataviewer/snippets/data_preview.html", data={ 'embed': False, 'resource_url': url, 'raw_resource_url': resource['url'] }) return h.resource_preview(resource, pkg_id)
def _post_analytics(self, user, request_obj_type, request_function, request_id): if config.get('googleanalytics.id'): data_dict = { "v": 1, "tid": config.get('googleanalytics.id'), "cid": hashlib.md5(user).hexdigest(), # customer id should be obfuscated "t": "event", "dh": c.environ['HTTP_HOST'], "dp": c.environ['PATH_INFO'], "dr": c.environ.get('HTTP_REFERER', ''), "ec": "CKAN API Request", "ea": request_obj_type + request_function, "el": request_id, } plugin.CscGAPlugin.analytics_queue.put(data_dict)
def qdes_get_dataset_review_period(): period = config.get('ckanext.qdes_schema.dataset_review_period', constants.DEFAULT_DATASET_REVIEW_PERIOD) # For some reason, dev database is return empty string. if not period: period = constants.DEFAULT_DATASET_REVIEW_PERIOD return int(period)
def configure(self, config): connection_variable = self._config_template % "connection" self.drupal_connection_url = config.get(connection_variable) if not self.drupal_connection_url: raise Exception( 'YtpDrupalPlugin: required configuration variable missing: %s' % (connection_variable)) self.cancel_url = config.get(self._config_template % "cancel_url", "/cancel-user.php") self._node_type = config.get(self._config_template % "node_type", self._node_type) self._translations_disabled = asbool( config.get(self._config_template % "translations_disabled", "false")) self.engine = sqlalchemy.create_engine(self.drupal_connection_url)
def user_password_has_repeated_chars(value): allow_repeated_chars = asbool( config.get('ckan.fortify.password_policy.allow_repeated_chars', True)) if not allow_repeated_chars: for character in value: if character * 2 in value: return True return False
def migrate_orgs(ctx, config, dryrun): load_config(config or ctx.obj['config']) default_lang = c.get('ckan.locale_default', 'en') languages = ['fi', 'en', 'sv'] translated_fields = ['title', 'description'] org_patches = [] for old_org_dict in org_generator(): flatten_extras(old_org_dict) patch = {} for field in translated_fields: translated_field = '%s_translated' % field if translated_field not in old_org_dict: patch[translated_field] = {default_lang: old_org_dict[field]} for language in languages: value = old_org_dict.get('%s_%s' % (field, language)) if value is not None and value != "": patch[translated_field][language] = value else: patch[field] = old_org_dict[translated_field].get(default_lang) if 'features' not in old_org_dict: # 'adminstration' is used in previous data model if old_org_dict.get('public_adminstration_organization'): patch['features'] = ["public_administration_organization"] if patch: patch['id'] = old_org_dict['id'] org_patches.append(patch) if dryrun: print '\n'.join('%s' % p for p in org_patches) else: apply_org_patches(org_patches)
def migrate(ctx, config, dryrun): load_config(config or ctx.obj['config']) default_lang = c.get('ckan.locale_default', 'en') package_patches = [] resource_patches = [] for old_package_dict in package_generator('*:*', 1000): if 'title_translated' in old_package_dict: continue original_language = old_package_dict.get('original_language', default_lang) langs = old_package_dict.get('translations', []) patch = { 'id': old_package_dict['id'], 'title_translated': { original_language: old_package_dict['title'] }, 'notes_translated': { original_language: old_package_dict['notes'] }, 'copyright_notice_translated': { original_language: old_package_dict.get('copyright_notice', '') }, 'external_urls': old_package_dict.get('extra_information', []) } if old_package_dict.get('tags'): patch['keywords'] = {'fi': [tag['name'] for tag in old_package_dict.get('tags', []) if tag['vocabulary_id'] is None]} if old_package_dict.get('content_type'): patch['content_type'] = {'fi': [s for s in old_package_dict.get('content_type', "").split(',') if s]} if old_package_dict.get('license_id') is None: patch['license_id'] = 'other' for lang in langs: patch['title_translated'][lang] = old_package_dict.get('title_' + lang, '') patch['notes_translated'][lang] = old_package_dict.get('notes_' + lang, '') patch['copyright_notice_translated'][lang] = old_package_dict.get('copyright_notice_' + lang, '') patch['resources'] = old_package_dict.get('resources') for resource in patch.get('resources', []): resource['name_translated'] = { original_language: resource.get('name', '') or '' } resource['description_translated'] = { original_language: resource.get('description', '') or '' } if resource.get('temporal_granularity') and type(resource.get('temporal_granularity')) is not dict: resource['temporal_granularity'] = { original_language: resource.get('temporal_granularity') } else: del resource['temporal_granularity'] if resource.get('update_frequency') and type(resource.get('update_frequency')) is not dict: resource['update_frequency'] = { original_language: resource.get('update_frequency') } else: del resource['update_frequency'] for lang in langs: resource['name_translated'][lang] = resource.get('name_' + lang, '') resource['description_translated'][lang] = resource.get('description_' + lang, '') if resource.get('temporal_granularity_' + lang): resource['temporal_granularity'][lang] = resource.get('temporal_granularity_' + lang) if resource.get('temporal_granularity_' + lang): resource['update_frequency'][lang] = resource.get('update_frequency_' + lang) package_patches.append(patch) if dryrun: print '\n'.join('%s' % p for p in package_patches) print '\n'.join('%s' % p for p in resource_patches) else: apply_patches(package_patches, resource_patches)