def csc_dataset_form_value(text):
"""
:param text: {lang: text} dict or text string
Convert "language-text" to users' language by looking up
languag in dict or using gettext if not a dict but. If the text
doesn't exist look for an available text
"""
if not text:
return u''
if hasattr(text, 'get'):
final_text = u''
try:
prefer_lang = lang()
except:
prefer_lang = config.get('ckan.locale_default', 'es')
else:
try:
final_text = text[prefer_lang]
except KeyError:
pass
if not final_text:
locale_order = config.get('ckan.locale_order', '').split()
for l in locale_order:
if l in text and text[l]:
final_text = text[l]
break
return final_text
t = gettext(text)
if isinstance(t, str):
return t.decode('utf-8')
return t
def check_config(self):
exit = False
for key in ('ckanext.azurefilestore.connect_str',
'ckanext.azurefilestore.container_name',
'ckanext.azurefilestore.storage_account',
'ckanext.azurefilestore.account_key'):
if not config.get(key):
print 'You must set the "{0}" option in your ini file'.format(
key)
exit = True
if exit:
sys.exit(1)
print 'All configuration options defined'
connect_str = config.get('ckanext.azurefilestore.connect_str')
container_name = config.get('ckanext.azurefilestore.container_name')
blob_service_client = BlobServiceClient.from_connection_string(
connect_str)
# Check if container exists
try:
container_client = blob_service_client.get_container_client(
container_name)
except Exception as e:
print 'An error was found while getting or creating the container:'
print str(e)
sys.exit(1)
print 'Configuration OK!'
def _catalog_graph(self, dataset_ref, dataset_dict):
data_catalog = BNode()
self.g.add((dataset_ref, SCHEMA.includedInDataCatalog, data_catalog))
self.g.add((data_catalog, RDF.type, SCHEMA.DataCatalog))
self.g.add((data_catalog, SCHEMA.name, Literal(config.get('ckan.site_title'))))
self.g.add((data_catalog, SCHEMA.description, Literal(config.get('ckan.site_description'))))
self.g.add((data_catalog, SCHEMA.url, Literal(config.get('ckan.site_url'))))
def valid_signature_required(secret_prefix):
signature_header_name = config.get(
plugin_config_prefix + 'signature_header_name', 'X-Hub-Signature')
api_secret = config.get(plugin_config_prefix + secret_prefix + '_secret',
'secret')
def decorator(func):
def wrapper(context, data):
if signature_header_name in request.headers:
if request.headers[signature_header_name].startswith('sha1='):
algo, received_hmac = request.headers[
signature_header_name].rsplit('=')
# since python 3, bytes are not directly str so key must be encoded : https://stackoverflow.com/a/43882903/2862821
# else ERROR [ckan.views.api] key: expected bytes or bytearray, but got 'str'
computed_hmac = hmac.new(bytes(api_secret, 'utf-8'),
request.get_data(),
sha1).hexdigest()
if received_hmac.lower() != computed_hmac:
log.info('Invalid HMAC')
raise toolkit.NotAuthorized(_('Invalid HMAC'))
else:
log.info('Invalid HMAC algo')
raise toolkit.ValidationError(_('Invalid HMAC algo'))
else:
log.info('No HMAC in the header')
raise toolkit.NotAuthorized(_("No HMAC in the header"))
return func(context, data)
return wrapper
return decorator
def is_map_viewer_format(resource):
viewer_url = config.get('ckanext.geodatagov.spatial_preview.url')
viewer_formats = config.get('ckanext.geodatagov.spatial_preview.formats',
'wms kml kmz').strip().split(' ')
return viewer_url and resource.get('url') and resource.get(
'format', '').lower() in viewer_formats
def get_name_from_siret(siret):
apiKey = config.get('ckanext.ozwillo_organization_api.verifsiret_apikey',
'')
secretKey = config.get(
'ckanext.ozwillo_organization_api.verifsiret_secretkey', '')
url = "https://www.numero-de-siret.com/api/siret?siret=" + siret
result = None
if apiKey == '' or secretKey == '':
log.error(
'Verif-siret config incomplete, please register your api key and api secret in the config file'
)
result = ''
return result
try:
get = requests.get(url, auth=(apiKey, secretKey))
if get.status_code != 200:
raise requests.ConnectionError()
except requests.exceptions.RequestException as err:
log.error('An error occurred: {0}'.format(err))
else:
try:
result = get.json()['array_return'][0]['LIBCOM']
except (IndexError, TypeError, AttributeError):
log.error('No organization found for this siret number')
finally:
return result
def issues_enabled(dataset):
'''Returns whether issues are enabled for the given dataset (dict)'''
# config options allow you to only enable issues for particular datasets or
# organizations
datasets_with_issues_enabled = set(toolkit.aslist(
config.get('ckanext.issues.enabled_for_datasets')
))
organizations_with_issues_enabled = set(toolkit.aslist(
config.get('ckanext.issues.enabled_for_organizations')
))
if datasets_with_issues_enabled or organizations_with_issues_enabled:
if datasets_with_issues_enabled and \
dataset['name'] in datasets_with_issues_enabled:
return True
elif organizations_with_issues_enabled and \
(dataset.get('organization') or {}).get('name') in \
organizations_with_issues_enabled:
return True
return False
else:
extras = dataset.get('extras', [])
for extra in extras:
if extra.get('key') == 'issues_enabled':
return toolkit.asbool(extra.get('value'))
else:
return toolkit.asbool(
config.get('ckanext.issues.enabled_without_extra', True)
)
def csc_dataset_form_lang_and_value(text):
"""
:param text: {lang: text} dict or text string
Convert "language-text" to users' language by looking up
languag in dict, if the text
doesn't exit look for an available text
"""
if not text:
return {'': u''}
if hasattr(text, 'get'):
final_text = u''
try:
prefer_lang = lang()
except:
prefer_lang = config.get('ckan.locale_default', 'es')
else:
try:
prefer_lang = str(prefer_lang)
final_text = text[prefer_lang]
except KeyError:
pass
if not final_text:
locale_order = config.get('ckan.locale_order', '').split()
for l in locale_order:
if l in text and text[l]:
final_text = text[l]
prefer_lang = l
break
return {prefer_lang: final_text}
return {'': u''}
def _feedback_mail_recipient(recipient_name, recipient_email, sender_name,
sender_url, subject, body, headers):
""" Assemble and send a feedback email from the provided parts.
"""
# Flake8 B006: Don't initialize this in the parameter, because
# default parameters are shared between calls, and dict is mutable.
headers = headers or {}
mail_from = config.get('smtp.mail_from')
body = add_msg_niceties(recipient_name, body, sender_name, sender_url)
msg = MIMEText(body.encode('utf-8'), 'plain', 'utf-8')
for key, value in headers.items():
msg[key] = value
subject = Header(subject.encode('utf-8'), 'utf-8')
msg['Subject'] = subject
msg['From'] = _("%s <%s>") % (sender_name, mail_from)
msg['To'] = ", ".join(recipient_email)
msg['Date'] = Utils.formatdate(time())
msg['X-Mailer'] = "CKAN %s" % __version__
# Send the email using Python's smtplib.
smtp_connection = smtplib.SMTP()
smtp_server = config.get('smtp.server', 'localhost')
smtp_starttls = asbool(config.get('smtp.starttls'))
smtp_user = config.get('smtp.user')
smtp_password = config.get('smtp.password')
smtp_connection.connect(smtp_server)
try:
# Identify ourselves and prompt the server for supported features.
smtp_connection.ehlo()
# If 'smtp.starttls' is on in CKAN config, try to put the SMTP
# connection into TLS mode.
if smtp_starttls:
if smtp_connection.has_extn('STARTTLS'):
smtp_connection.starttls()
# Re-identify ourselves over TLS connection.
smtp_connection.ehlo()
else:
raise MailerException("SMTP server does not support STARTTLS")
# If 'smtp.user' is in CKAN config, try to login to SMTP server.
if smtp_user:
assert smtp_password, ("If smtp.user is configured then "
"smtp.password must be configured as well.")
smtp_connection.login(smtp_user, smtp_password)
smtp_connection.sendmail(mail_from, recipient_email, msg.as_string())
LOG.info("Sent email to %s", ','.join(recipient_email))
except smtplib.SMTPException as ex:
msg = '%r' % ex
LOG.exception(msg)
raise MailerException(msg)
finally:
smtp_connection.quit()
def configure(self, config):
connection_variable = self._config_template % "connection"
self.drupal_connection_url = config.get(connection_variable)
if not self.drupal_connection_url:
raise Exception('YtpDrupalPlugin: required configuration variable missing: %s' % (connection_variable))
self.cancel_url = config.get(self._config_template % "cancel_url", "/cancel-user.php")
self._node_type = config.get(self._config_template % "node_type", self._node_type)
self._translations_disabled = asbool(config.get(self._config_template % "translations_disabled", "false"))
self.engine = sqlalchemy.create_engine(self.drupal_connection_url)
def matomo_snippet():
data = {
"matomo_domain":
config.get('ckanext.matomo.domain'),
"matomo_script_domain":
config.get('ckanext.matomo.script_domain',
config.get('ckanext.matomo.domain')),
"matomo_site_id":
config.get('ckanext.matomo.site_id')
}
return render_snippet("matomo/snippets/matomo.html", data)
def update_organization_dataset_names(old_org_name, new_org_name, org_id):
"""
This is used to change all pakcgae names when publisher id is changed. Because IATI
uses naming convention for dataset which includes publisher id
:param old_org_name: str
:param new_org_name: str
:param new_org_name: str (uuid)
:return: None
"""
# Run this after org is updated completely
time.sleep(30)
context = {
'model': model,
'session': model.Session,
'site_url': config.get('ckan.site_url'),
'user': config.get('iati.admin_user.name'),
'apikey': config.get('iati.admin_user.api_key'),
'api_version': 3,
'disable_archiver': True
}
_fq = 'owner_org:{}'.format(org_id)
_org_packages = get_action('package_search')(context, {
'fq': _fq,
'rows': 10000,
'include_private': True
})
for pkg in _org_packages.get('results', []):
# Replace only 1st occurrence
log.info("Updating the package: {} with private state - {}".format(
pkg['id'], str(pkg['private'])))
log.info("Old org: {}".format(old_org_name))
log.info("New org: {}".format(new_org_name))
new_package_name = pkg.get('name', '').replace(old_org_name,
new_org_name, 1)
log.info("New package name: {}".format(new_package_name))
if pkg.get('name', '') != new_package_name:
context["disable_archiver"] = True
try:
_res = get_action('package_patch')(context, {
'id': pkg['id'],
'name': new_package_name
})
except logic.ValidationError as e:
log.error(e)
except Exception as e:
log.error(
"Org name change detected but package update failed for some reason"
)
log.error(e)
return None
def uploaded_file_redirect(self, upload_to, filename):
'''Redirect static file requests to their location on azure.'''
storage_account = config.get('ckanext.azurefilestore.storage_account')
# Remove last characted if it's a slash
# if host_name[-1] == '/':
# host_name = host_name[:-1]
storage_path = AzureUploader.get_storage_path(upload_to)
filepath = os.path.join(storage_path, filename)
redirect_url = 'https://{storage_account}.blob.core.windows.net/{container_name}/{filepath}'\
.format(
container_name=config.get('ckanext.azurefilestore.container_name'),
filepath=filepath,
storage_account=storage_account)
redirect(redirect_url)
def download(context,
resource,
url_timeout=URL_TIMEOUT,
max_content_length=MAX_CONTENT_LENGTH,
data_formats=DATA_FORMATS):
res = None
resource_changed = False
link_context = "{}"
link_data = json.dumps({
'url': resource['url'],
'url_timeout': url_timeout
})
user_agent_string = config.get('ckanext.archiver.user_agent_string',
'curl/7.35.0')
def _download_resource(resource_url, timeout):
_request_headers = {'User-Agent': user_agent_string}
# Part of 403 url error - if user agent is missing or
# some sites do not accept IATI (CKAN) as user agent.
# hence setting default user agent to Mozilla/5.0.
try:
response = requests.get(resource['url'],
timeout=url_timeout,
headers=_request_headers,
verify=True)
except Exception, e:
request_headers['User-Agent'] = 'curl/7.35.0'
response = requests.get(resource['url'],
timeout=url_timeout,
headers=_request_headers,
verify=False)
return response
def index(self, resource_id, cn):
try:
dns = config.get("ckanext.cfpb_ldap_query.base_dns").split('|')
except ValueError:
flash_error("At least one valid DN must be configured.")
redirect_to("get_access_request", resource_id=resource_id, cn=cn)
resource = get_action('resource_show')({},
data_dict={
'id': resource_id
})
package = get_action('package_show')({},
data_dict={
'id': resource['package_id']
})
role_description = get_role(json.loads(resource['db_roles']),
cn)['description']
return render(
'ckanext/cfpb-extrafields/access_index.html', {
"resource": resource,
"package": package,
"description": role_description,
"cn": cn,
"context": context,
"dn": dns[0]
})
def extract_redirects(publisher_name=None):
"""
Extracts the required redirects from the database tables and return the result as tuple
TODD: Change the below query using sqlalchemy
This is data is used to store in the table iati_redirects
:return: tuple
"""
_db_conn = create_engine(config.get('sqlalchemy.url')).connect()
if publisher_name:
_query = '''
SELECT DISTINCT public.group.id,
CAST(CAST(activity.data AS json) ->> 'group' AS json) ->> 'name' AS old_name,
CAST(activity.timestamp AS VARCHAR) as activity_recorded_at
FROM public.group, activity
WHERE
public.group.id = activity.object_id AND
public.group.name = '{publisher_name}' AND
activity.activity_type = 'changed organization' AND
public.group.name != CAST(CAST(activity.data AS json) ->> 'group' AS json) ->> 'name' LIMIT 20;
'''.format(publisher_name=publisher_name)
else:
_query = '''
SELECT DISTINCT public.group.id, public.group.name as current_name,
CAST(CAST(activity.data AS json) ->> 'group' AS json) ->> 'name' AS old_name
FROM public.group, activity
WHERE
public.group.id = activity.object_id AND
activity.activity_type = 'changed organization' AND
public.group.name != CAST(CAST(activity.data AS json) ->> 'group' AS json) ->> 'name';
'''
log.info(_query)
res = _db_conn.execute(_query)
return tuple(res)
def get_drupal_session_token(self, domain, service, cookie_header=''):
'''return text of X-CSRF-Token)'''
token_url = 'https://' + domain + '/' + service + '/?q=services/session/token'
verify_cert = config.get('ckanext.drupal8.development_cert', '') or True
token_request = requests.get(token_url, headers={"Cookie": cookie_header}, verify=verify_cert)
token = token_request.text
return token
def get_resource_info_by_id(cls, resource_id):
resource = model.Session.query(
model.Resource).filter(model.Resource.id == resource_id).first()
res_name = None
res_package_name = None
res_package_id = None
if resource is not None:
res_package_name = resource.package.title or resource.package.name
res_package_id = resource.package.name
name_translated_json = resource.extras.get('name_translated')
if name_translated_json is not None:
try:
name_translated = json.loads(name_translated_json)
languages = aslist(config.get('ckan.locales_offered', []))
res_name = next(
(name_translated[lang]
for lang in languages if name_translated.get(lang)),
None)
except json.JSONDecodeError:
pass
res_name = res_name or resource.description or resource.format
return [res_name, res_package_name, res_package_id]
def dataportal_admin():
# items = self._get_config_form_items()
if request.method == "POST":
data = request.form
try:
data_dict = logic.clean_dict(
dict_fns.unflatten(logic.tuplize_dict(logic.parse_params(request.form, ignore_keys=CACHE_PARAMETERS)))
)
del data_dict["save"]
data = logic.get_action("config_option_update")({"user": g.user}, data_dict)
except logic.ValidationError as e:
errors = e.error_dict
error_summary = e.error_summary
extra_vars = {"data": data, "errors": errors, "error_summary": error_summary}
return toolkit.render("admin/dataportal.html", extra_vars=extra_vars)
return toolkit.redirect_to("dataportaltheme_blueprint.dataportal_admin")
else:
admin_schema = schema.update_configuration_schema()
data = {}
for key in admin_schema:
data[key] = config.get(key)
extra_vars = {"data": data, "errors": {}}
return toolkit.render("admin/dataportal.html", extra_vars=extra_vars)
def generate_datapackage_json(package_id):
'''Generates the datapackage - metadata that would be saved as
datapackage.json.
'''
context = {'model': model, 'session': model.Session}
dataset = get_action('package_show')(context, {'id': package_id})
# filter out resources that are not suitable for inclusion in the data
# package
local_ckan = ckanapi.LocalCKAN()
dataset, resources_to_include, existing_zip_resource = \
remove_resources_that_should_not_be_included_in_the_datapackage(
dataset)
# get the datapackage (metadata)
datapackage = ckanapi.datapackage.dataset_to_datapackage(dataset)
# populate datapackage with the schema from the Datastore data
# dictionary
ckan_and_datapackage_resources = zip(resources_to_include,
datapackage.get('resources', []))
for res, datapackage_res in ckan_and_datapackage_resources:
ckanapi.datapackage.populate_datastore_res_fields(ckan=local_ckan,
res=res)
ckanapi.datapackage.populate_schema_from_datastore(
cres=res, dres=datapackage_res)
# add in any other dataset fields, if configured
fields_to_include = config.get(
u'ckanext.downloadall.dataset_fields_to_add_to_datapackage',
u'').split()
for key in fields_to_include:
datapackage[key] = dataset.get(key)
return (datapackage, ckan_and_datapackage_resources, existing_zip_resource)
def download_table_template(validation_schema):
"""
Downloads a CSV template file for the specified validation schema.
"""
try:
validator_config = config.get('ckanext.validator.schema_config')
schemed_table = validator_config.get(validation_schema)
template = schemed_table.create_template()
csv_content = template.to_csv(header=False,
index=False,
encoding='utf-8')
return Response(csv_content,
mimetype="text/csv",
headers={
"Content-disposition":
"attachment; filename=" + str(validation_schema) +
".csv"
})
except AttributeError as e:
logging.error(e)
abort(404,
"404 Not Found Error: No schema exists for " + validation_schema)
except Exception as e:
logging.error(e)
abort(
500, "500 Internal server error: Something went wrong whilst "
"generating your template " + validation_schema)
def get_url():
kwargs = {'headers': headers, 'timeout': DOWNLOAD_TIMEOUT,
'verify': SSL_VERIFY, 'stream': True} # just gets the headers for now
if 'ckan.download_proxy' in config:
proxy = config.get('ckan.download_proxy')
kwargs['proxies'] = {'http': proxy, 'https': proxy}
return requests.get(url, **kwargs)
def _comment_or_issue_report(issue_or_comment, user_ref, dataset_id, session):
user_obj = model.User.get(user_ref)
try:
issue_or_comment.report_abuse(session, user_obj.id)
except IntegrityError:
session.rollback()
raise ReportAlreadyExists(
p.toolkit._('Issue has already been reported by this user'))
try:
# if you're an org admin/editor (can edit the dataset), it gets marked
# as abuse immediately
context = {
'user': user_ref,
'session': session,
'model': model,
}
p.toolkit.check_access('package_update',
context,
data_dict={'id': dataset_id})
issue_or_comment.change_visibility(session, u'hidden')
issue_or_comment.abuse_status = issuemodel.AbuseStatus.abuse.value
return {
'visibility': issue_or_comment.visibility,
'abuse_reports': issue_or_comment.abuse_reports,
'abuse_status': issue_or_comment.abuse_status
}
except p.toolkit.NotAuthorized:
max_strikes = config.get('ckanext.issues.max_strikes')
if (max_strikes and len(issue_or_comment.abuse_reports) >=
p.toolkit.asint(max_strikes)):
issue_or_comment.change_visibility(session, u'hidden')
finally:
# commit the IssueReport and changes to the Issue/Comment
session.commit()
def save_redirect_in_filestore_directory(data):
"""
Saves the redirects data as cs in filestore where all the resources are stored
i.e. in path /var/lib/ckan/
:param data:
:return:
"""
_redirects_dir = os.path.join(config.get('ckan.storage_path', ''),
"redirects")
try:
if not os.path.isdir(_redirects_dir):
os.mkdir(_redirects_dir)
except Exception as e:
log.error(e)
log.error("Error when creating a new directory in redirects path")
return None
columns = ('publisher_id', 'current_name', 'old_name')
try:
with open(os.path.join(_redirects_dir, 'redirects.csv'),
'w') as redirects_file:
writer = csv.writer(redirects_file, delimiter=',')
writer.writerow(columns)
for row in data:
writer.writerow(row)
log.info("CSV file generated in {}".format(_redirects_dir))
except Exception as e:
log.error(e)
log.error("Failed to write data to csv..")
return None
return None
def get_types():
_TYPES = [messytables.StringType, messytables.DecimalType,
messytables.IntegerType, messytables.DateUtilType]
# TODO make this configurable
# TYPES = web.app.config.get('TYPES', _TYPES)
TYPE_MAPPING = config.get('TYPE_MAPPING', _TYPE_MAPPING)
return _TYPES, TYPE_MAPPING
def get_view_data(group):
view_data = []
site_url = config.get("ckan.site_url")
if group:
f_group_packages = logic.get_action("group_package_show")
packages = f_group_packages(data_dict={"id": group, "limit": 8})
else:
packages = logic.get_action("current_package_list_with_resources")(data_dict={})
ignored_views = ["text_view", "recline_graph_view", "recline_view"]
for package in packages:
for resource in package["resources"]:
if len(view_data) > 7:
break
# Get view for resource
views = logic.get_action("resource_view_list")(data_dict={"id": resource["id"]})
# print(views)
for view in views:
if view["view_type"] in ignored_views:
continue
url = "%s/dataset/%s/resource/%s/view/%s" % (site_url, package["name"], resource["id"], view["id"])
link = "%s/dataset/%s/resource/%s" % (site_url, package["name"], resource["id"])
view_data.append({"url": url, "name": view["title"], "link": link})
return view_data
def _send_activation_notification_email(context, organization_dict):
model = context['model']
members = p.toolkit.get_action('member_list')(context, {
'id': organization_dict['id']
})
admins = [m for m in members if m[1] == 'user' and m[2] == 'Admin']
subject = config.get('iati.publisher_activation_email_subject',
'IATI Registry Publisher Activation')
group_link = urljoin(site_url, '/publisher/' + organization_dict['name'])
for admin in admins:
user = model.User.get(admin[0])
if user and user.email:
user_name = user.fullname or user.name
content = emailer.publisher_activation_body_template.format(
user_name=user_name.encode('utf8'),
group_title=organization_dict['title'].encode('utf8'),
group_link=group_link,
user_email=user.email,
site_url=site_url)
emailer.send_email(content,
subject,
user.email,
content_type='html')
log.debug(
'[email] Publisher activated notification email sent to user {0}'
.format(user.name))
def resource_preview_custom(resource, pkg_id):
resource_format = resource.get('format', '').lower()
if is_map_viewer_format(resource):
viewer_url = config.get('ckanext.geodatagov.spatial_preview.url')
url = '{viewer_url}?{params}'.format(
viewer_url=viewer_url, params=get_map_viewer_params(resource))
return p.toolkit.render_snippet(
"dataviewer/snippets/data_preview.html",
data={
'embed': False,
'resource_url': url,
'raw_resource_url': resource['url']
})
elif resource_format in ('web map application', 'arcgis online map') \
and ('webmap=' in resource.get('url') or 'services=' in resource.get('url')):
url = resource['url'].replace('viewer.html', 'embedViewer.html')
return p.toolkit.render_snippet(
"dataviewer/snippets/data_preview.html",
data={
'embed': False,
'resource_url': url,
'raw_resource_url': resource['url']
})
return h.resource_preview(resource, pkg_id)
def _post_analytics(self, user, request_obj_type, request_function,
request_id):
if config.get('googleanalytics.id'):
data_dict = {
"v": 1,
"tid": config.get('googleanalytics.id'),
"cid": hashlib.md5(user).hexdigest(),
# customer id should be obfuscated
"t": "event",
"dh": c.environ['HTTP_HOST'],
"dp": c.environ['PATH_INFO'],
"dr": c.environ.get('HTTP_REFERER', ''),
"ec": "CKAN API Request",
"ea": request_obj_type + request_function,
"el": request_id,
}
plugin.CscGAPlugin.analytics_queue.put(data_dict)
def qdes_get_dataset_review_period():
period = config.get('ckanext.qdes_schema.dataset_review_period',
constants.DEFAULT_DATASET_REVIEW_PERIOD)
# For some reason, dev database is return empty string.
if not period:
period = constants.DEFAULT_DATASET_REVIEW_PERIOD
return int(period)
def configure(self, config):
connection_variable = self._config_template % "connection"
self.drupal_connection_url = config.get(connection_variable)
if not self.drupal_connection_url:
raise Exception(
'YtpDrupalPlugin: required configuration variable missing: %s'
% (connection_variable))
self.cancel_url = config.get(self._config_template % "cancel_url",
"/cancel-user.php")
self._node_type = config.get(self._config_template % "node_type",
self._node_type)
self._translations_disabled = asbool(
config.get(self._config_template % "translations_disabled",
"false"))
self.engine = sqlalchemy.create_engine(self.drupal_connection_url)
def user_password_has_repeated_chars(value):
allow_repeated_chars = asbool(
config.get('ckan.fortify.password_policy.allow_repeated_chars', True))
if not allow_repeated_chars:
for character in value:
if character * 2 in value:
return True
return False
def migrate_orgs(ctx, config, dryrun):
load_config(config or ctx.obj['config'])
default_lang = c.get('ckan.locale_default', 'en')
languages = ['fi', 'en', 'sv']
translated_fields = ['title', 'description']
org_patches = []
for old_org_dict in org_generator():
flatten_extras(old_org_dict)
patch = {}
for field in translated_fields:
translated_field = '%s_translated' % field
if translated_field not in old_org_dict:
patch[translated_field] = {default_lang: old_org_dict[field]}
for language in languages:
value = old_org_dict.get('%s_%s' % (field, language))
if value is not None and value != "":
patch[translated_field][language] = value
else:
patch[field] = old_org_dict[translated_field].get(default_lang)
if 'features' not in old_org_dict:
# 'adminstration' is used in previous data model
if old_org_dict.get('public_adminstration_organization'):
patch['features'] = ["public_administration_organization"]
if patch:
patch['id'] = old_org_dict['id']
org_patches.append(patch)
if dryrun:
print '\n'.join('%s' % p for p in org_patches)
else:
apply_org_patches(org_patches)
def migrate(ctx, config, dryrun):
load_config(config or ctx.obj['config'])
default_lang = c.get('ckan.locale_default', 'en')
package_patches = []
resource_patches = []
for old_package_dict in package_generator('*:*', 1000):
if 'title_translated' in old_package_dict:
continue
original_language = old_package_dict.get('original_language', default_lang)
langs = old_package_dict.get('translations', [])
patch = {
'id': old_package_dict['id'],
'title_translated': {
original_language: old_package_dict['title']
},
'notes_translated': {
original_language: old_package_dict['notes']
},
'copyright_notice_translated': {
original_language: old_package_dict.get('copyright_notice', '')
},
'external_urls': old_package_dict.get('extra_information', [])
}
if old_package_dict.get('tags'):
patch['keywords'] = {'fi': [tag['name']
for tag in old_package_dict.get('tags', []) if tag['vocabulary_id'] is None]}
if old_package_dict.get('content_type'):
patch['content_type'] = {'fi': [s for s in old_package_dict.get('content_type', "").split(',') if s]}
if old_package_dict.get('license_id') is None:
patch['license_id'] = 'other'
for lang in langs:
patch['title_translated'][lang] = old_package_dict.get('title_' + lang, '')
patch['notes_translated'][lang] = old_package_dict.get('notes_' + lang, '')
patch['copyright_notice_translated'][lang] = old_package_dict.get('copyright_notice_' + lang, '')
patch['resources'] = old_package_dict.get('resources')
for resource in patch.get('resources', []):
resource['name_translated'] = {
original_language: resource.get('name', '') or ''
}
resource['description_translated'] = {
original_language: resource.get('description', '') or ''
}
if resource.get('temporal_granularity') and type(resource.get('temporal_granularity')) is not dict:
resource['temporal_granularity'] = {
original_language: resource.get('temporal_granularity')
}
else:
del resource['temporal_granularity']
if resource.get('update_frequency') and type(resource.get('update_frequency')) is not dict:
resource['update_frequency'] = {
original_language: resource.get('update_frequency')
}
else:
del resource['update_frequency']
for lang in langs:
resource['name_translated'][lang] = resource.get('name_' + lang, '')
resource['description_translated'][lang] = resource.get('description_' + lang, '')
if resource.get('temporal_granularity_' + lang):
resource['temporal_granularity'][lang] = resource.get('temporal_granularity_' + lang)
if resource.get('temporal_granularity_' + lang):
resource['update_frequency'][lang] = resource.get('update_frequency_' + lang)
package_patches.append(patch)
if dryrun:
print '\n'.join('%s' % p for p in package_patches)
print '\n'.join('%s' % p for p in resource_patches)
else:
apply_patches(package_patches, resource_patches)