def update_package_deprecation(ctx, config, dryrun):
load_config(config or ctx.obj['config'])
# after loop contains list of ID's of just deprecated packages
deprecated_now = []
package_patches = []
deprecation_offset = package_deprecation_offset()
for old_package_dict in package_generator('*:*', 1000):
valid_till = old_package_dict.get('valid_till')
if valid_till is not None:
current_deprecation = old_package_dict.get('deprecated')
deprecation = check_package_deprecation(valid_till, deprecation_offset)
if current_deprecation != deprecation:
patch = {'id': old_package_dict['id'], 'deprecated': deprecation}
package_patches.append(patch)
# Initially deprecation value will be undefined, so send email only when actually deprecated
if current_deprecation is False and deprecation is True:
deprecated_now.append(old_package_dict['id'])
if dryrun:
print '\n'.join(('%s | %s' % (p, p['id'] in deprecated_now)) for p in package_patches)
else:
# No resources patches so empty parameter is passed
apply_patches(package_patches, [])
send_package_deprecation_emails(deprecated_now)
def update_package_deprecation(ctx, config, dryrun):
load_config(config or ctx.obj['config'])
# deprecation emails will be sent to items inside deprecated_now array
deprecated_now = []
package_patches = []
# Get only packages with a valid_till field and some value in the valid_till field
for old_package_dict in package_generator(
'valid_till:* AND -valid_till:""', 1000):
valid_till = old_package_dict.get('valid_till')
# For packages that have a valid_till date set depracated field to true or false
# deprecation means that a package has been valid but is now old and not valid anymore.
# This does not take into account if the package is currently valid eg. valid_from.
if valid_till is not None:
current_deprecation = old_package_dict.get('deprecated')
deprecation = check_package_deprecation(valid_till)
if current_deprecation != deprecation:
patch = {
'id': old_package_dict['id'],
'deprecated': deprecation
}
package_patches.append(patch)
# Send email only when actually deprecated. Initial deprecation is undefined when adding this feature
if current_deprecation is False and deprecation is True:
deprecated_now.append(old_package_dict['id'])
if dryrun:
print '\n'.join(('%s | %s' % (p, p['id'] in deprecated_now))
for p in package_patches)
else:
# No resources patches so empty parameter is passed
apply_patches(package_patches, [])
send_package_deprecation_emails(deprecated_now)
def dump(ctx, resource_id, output_file, config, format, offset, limit, bom):
load_config(config or ctx.obj['config'])
dump_to(resource_id,
output_file,
fmt=format,
offset=offset,
limit=limit,
options={u'bom': bom})
def dump(ctx, resource_id, output_file, config, format, offset, limit, bom):
load_config(config or ctx.obj['config'])
dump_to(
resource_id,
output_file,
fmt=format,
offset=offset,
limit=limit,
options={u'bom': bom})
def migrate_author_email(ctx, config, dryrun):
load_config(config or ctx.obj['config'])
package_patches = []
for old_package_dict in package_generator('*:*', 1000):
if old_package_dict.get('author_email') is None:
patch = {'id': old_package_dict['id'], 'author_email': ''}
package_patches.append(patch)
if dryrun:
print '\n'.join('%s' % p for p in package_patches)
else:
# No resources patches so empty parameter is passed
apply_patches(package_patches, [])
def migrate_author_email(ctx, config, dryrun):
load_config(config or ctx.obj['config'])
package_patches = []
for old_package_dict in package_generator('*:*', 1000):
if old_package_dict.get('author_email') is None:
patch = {'id': old_package_dict['id'], 'author_email': ''}
package_patches.append(patch)
if dryrun:
print '\n'.join('%s' % p for p in package_patches)
else:
# No resources patches so empty parameter is passed
apply_patches(package_patches, [])
def migrate_high_value_datasets(ctx, config, dryrun):
load_config(config or ctx.obj['config'])
package_patches = []
for old_package_dict in package_generator('*:*', 1000):
if old_package_dict.get('high_value_dataset_category'):
patch = {'id': old_package_dict['id'], 'international_benchmarks': old_package_dict['high_value_dataset_category']}
package_patches.append(patch)
if dryrun:
print '\n'.join('%s' % p for p in package_patches)
else:
# No resources patches so empty parameter is passed
apply_patches(package_patches, [])
def batch_edit(ctx, config, search_string, dryrun, group):
load_config(config or ctx.obj['config'])
group_assigns = {}
if group:
group_assigns[group] = []
for package_dict in package_generator(search_string, 1000):
if group:
group_assigns[group].append(package_dict['name'])
if dryrun:
print '\n'.join('Add %s to group %s' % (p, g) for (g, ps) in group_assigns.items() for p in ps)
else:
if group:
apply_group_assigns(group_assigns)
def batch_edit(ctx, config, search_string, dryrun, group):
load_config(config or ctx.obj['config'])
group_assigns = {}
if group:
group_assigns[group] = []
for package_dict in package_generator(search_string, 1000):
if group:
group_assigns[group].append(package_dict['name'])
if dryrun:
print '\n'.join('Add %s to group %s' % (p, g)
for (g, ps) in group_assigns.items() for p in ps)
else:
if group:
apply_group_assigns(group_assigns)
def add_to_groups(ctx, config, dryrun):
load_config(config or ctx.obj['config'])
context = {'ignore_auth': True}
groups = get_action('group_list')(context, {})
users = get_action('user_list')(context, {})
data_dicts = []
for group in groups:
memberships = get_action('member_list')(context, {'id': group})
for user in users:
if not any(id for (id, type, capacity) in memberships if id == user['id']):
data_dicts.append({'id': group, 'username': user['name'], 'role': 'editor'})
if dryrun:
print '\n'.join('%s' % d for d in data_dicts)
else:
for d in data_dicts:
get_action('group_member_create')(context, d)
def migrate_high_value_datasets(ctx, config, dryrun):
load_config(config or ctx.obj['config'])
package_patches = []
for old_package_dict in package_generator('*:*', 1000):
if old_package_dict.get('high_value_dataset_category'):
patch = {
'id':
old_package_dict['id'],
'international_benchmarks':
old_package_dict['high_value_dataset_category']
}
package_patches.append(patch)
if dryrun:
print '\n'.join('%s' % p for p in package_patches)
else:
# No resources patches so empty parameter is passed
apply_patches(package_patches, [])
def fetch(ctx, base_dir, year, start_from_beginning, package_id, config):
load_config(config or ctx.obj['config'])
if Path(os.path.join(base_dir, 'data', 'json', 'prh_data',
'all_done.txt')).is_file():
print("Nothing to do")
else:
PRHData().get_prh_data(base_directory=base_dir,
year=year,
start_from_beginning=start_from_beginning)
filename = make_csv_of_prh_data(base_directory=base_dir)
from scripts.prh.upload_to_ckan import upload_to_ckan
upload_to_ckan(package_id, filename)
with open(
os.path.join(base_dir, 'data', 'json', 'prh_data',
'all_done.txt'), 'w') as done_file:
done_file.write("done")
done_file.close()
def set_permissions(ctx, config):
load_config(config or ctx.obj['config'])
write_url = _parse_db_config(u'ckan.datastore.write_url')
read_url = _parse_db_config(u'ckan.datastore.read_url')
db_url = _parse_db_config(u'sqlalchemy.url')
# Basic validation that read and write URLs reference the same database.
# This obviously doesn't check they're the same database (the hosts/ports
# could be different), but it's better than nothing, I guess.
if write_url['db_name'] != read_url['db_name']:
exit(u"The datastore write_url and read_url must refer to the same "
u"database!")
sql = permissions_sql(maindb=db_url['db_name'],
datastoredb=write_url['db_name'],
mainuser=db_url['db_user'],
writeuser=write_url['db_user'],
readuser=read_url['db_user'])
print(sql)
def set_permissions(ctx, config):
load_config(config or ctx.obj['config'])
write_url = parse_db_config(u'ckan.datastore.write_url')
read_url = parse_db_config(u'ckan.datastore.read_url')
db_url = parse_db_config(u'sqlalchemy.url')
# Basic validation that read and write URLs reference the same database.
# This obviously doesn't check they're the same database (the hosts/ports
# could be different), but it's better than nothing, I guess.
if write_url['db_name'] != read_url['db_name']:
exit(u"The datastore write_url and read_url must refer to the same "
u"database!")
sql = permissions_sql(
maindb=db_url['db_name'],
datastoredb=write_url['db_name'],
mainuser=db_url['db_user'],
writeuser=write_url['db_user'],
readuser=read_url['db_user'])
print(sql)
def migrate_orgs(ctx, config, dryrun):
load_config(config or ctx.obj['config'])
default_lang = c.get('ckan.locale_default', 'en')
languages = ['fi', 'en', 'sv']
translated_fields = ['title', 'description']
org_patches = []
for old_org_dict in org_generator():
flatten_extras(old_org_dict)
patch = {}
for field in translated_fields:
translated_field = '%s_translated' % field
if translated_field not in old_org_dict:
patch[translated_field] = {default_lang: old_org_dict[field]}
for language in languages:
value = old_org_dict.get('%s_%s' % (field, language))
if value is not None and value != "":
patch[translated_field][language] = value
else:
patch[field] = old_org_dict[translated_field].get(default_lang)
if 'features' not in old_org_dict:
# 'adminstration' is used in previous data model
if old_org_dict.get('public_adminstration_organization'):
patch['features'] = ["public_administration_organization"]
if patch:
patch['id'] = old_org_dict['id']
org_patches.append(patch)
if dryrun:
print '\n'.join('%s' % p for p in org_patches)
else:
apply_org_patches(org_patches)
def migrate_orgs(ctx, config, dryrun):
load_config(config or ctx.obj['config'])
default_lang = c.get('ckan.locale_default', 'en')
languages = ['fi', 'en', 'sv']
translated_fields = ['title', 'description']
org_patches = []
for old_org_dict in org_generator():
flatten_extras(old_org_dict)
patch = {}
for field in translated_fields:
translated_field = '%s_translated' % field
if translated_field not in old_org_dict:
patch[translated_field] = {default_lang: old_org_dict[field]}
for language in languages:
value = old_org_dict.get('%s_%s' % (field, language))
if value is not None and value != "":
patch[translated_field][language] = value
else:
patch[field] = old_org_dict[translated_field].get(default_lang)
if 'features' not in old_org_dict:
# 'adminstration' is used in previous data model
if old_org_dict.get('public_adminstration_organization'):
patch['features'] = ["public_administration_organization"]
if patch:
patch['id'] = old_org_dict['id']
org_patches.append(patch)
if dryrun:
print '\n'.join('%s' % p for p in org_patches)
else:
apply_org_patches(org_patches)
def add_to_groups(ctx, config, dryrun):
load_config(config or ctx.obj['config'])
context = {'ignore_auth': True}
groups = get_action('group_list')(context, {})
users = get_action('user_list')(context, {})
data_dicts = []
for group in groups:
memberships = get_action('member_list')(context, {'id': group})
for user in users:
if not any(id for (id, type, capacity) in memberships
if id == user['id']):
data_dicts.append({
'id': group,
'username': user['name'],
'role': 'editor'
})
if dryrun:
print '\n'.join('%s' % d for d in data_dicts)
else:
for d in data_dicts:
get_action('group_member_create')(context, d)
def migrate(ctx, config, dryrun):
load_config(config or ctx.obj['config'])
default_lang = c.get('ckan.locale_default', 'en')
package_patches = []
resource_patches = []
for old_package_dict in package_generator('*:*', 1000):
if 'title_translated' in old_package_dict:
continue
original_language = old_package_dict.get('original_language', default_lang)
langs = old_package_dict.get('translations', [])
patch = {
'id': old_package_dict['id'],
'title_translated': {
original_language: old_package_dict['title']
},
'notes_translated': {
original_language: old_package_dict['notes']
},
'copyright_notice_translated': {
original_language: old_package_dict.get('copyright_notice', '')
},
'external_urls': old_package_dict.get('extra_information', [])
}
if old_package_dict.get('tags'):
patch['keywords'] = {'fi': [tag['name']
for tag in old_package_dict.get('tags', []) if tag['vocabulary_id'] is None]}
if old_package_dict.get('content_type'):
patch['content_type'] = {'fi': [s for s in old_package_dict.get('content_type', "").split(',') if s]}
if old_package_dict.get('license_id') is None:
patch['license_id'] = 'other'
for lang in langs:
patch['title_translated'][lang] = old_package_dict.get('title_' + lang, '')
patch['notes_translated'][lang] = old_package_dict.get('notes_' + lang, '')
patch['copyright_notice_translated'][lang] = old_package_dict.get('copyright_notice_' + lang, '')
patch['resources'] = old_package_dict.get('resources')
for resource in patch.get('resources', []):
resource['name_translated'] = {
original_language: resource.get('name', '') or ''
}
resource['description_translated'] = {
original_language: resource.get('description', '') or ''
}
if resource.get('temporal_granularity') and type(resource.get('temporal_granularity')) is not dict:
resource['temporal_granularity'] = {
original_language: resource.get('temporal_granularity')
}
else:
del resource['temporal_granularity']
if resource.get('update_frequency') and type(resource.get('update_frequency')) is not dict:
resource['update_frequency'] = {
original_language: resource.get('update_frequency')
}
else:
del resource['update_frequency']
for lang in langs:
resource['name_translated'][lang] = resource.get('name_' + lang, '')
resource['description_translated'][lang] = resource.get('description_' + lang, '')
if resource.get('temporal_granularity_' + lang):
resource['temporal_granularity'][lang] = resource.get('temporal_granularity_' + lang)
if resource.get('temporal_granularity_' + lang):
resource['update_frequency'][lang] = resource.get('update_frequency_' + lang)
package_patches.append(patch)
if dryrun:
print '\n'.join('%s' % p for p in package_patches)
print '\n'.join('%s' % p for p in resource_patches)
else:
apply_patches(package_patches, resource_patches)
def init_db(ctx, config):
load_config((config or ctx.obj['config']))
commands.init_db()
def fetch(ctx, config, dryrun, since, until):
load_config(config or ctx.obj['config'])
commands.fetch(dryrun, since, until)
def migrate(ctx, config, dryrun):
load_config(config or ctx.obj['config'])
default_lang = c.get('ckan.locale_default', 'en')
package_patches = []
resource_patches = []
for old_package_dict in package_generator('*:*', 1000):
if 'title_translated' in old_package_dict:
continue
original_language = old_package_dict.get('original_language',
default_lang)
langs = old_package_dict.get('translations', [])
patch = {
'id': old_package_dict['id'],
'title_translated': {
original_language: old_package_dict['title']
},
'notes_translated': {
original_language: old_package_dict['notes']
},
'copyright_notice_translated': {
original_language:
old_package_dict.get('copyright_notice', '')
},
'external_urls': old_package_dict.get('extra_information', [])
}
if old_package_dict.get('tags'):
patch['keywords'] = {
'fi': [
tag['name'] for tag in old_package_dict.get('tags', [])
if tag['vocabulary_id'] is None
]
}
if old_package_dict.get('content_type'):
patch['content_type'] = {
'fi': [
s for s in old_package_dict.get('content_type', "").split(
',') if s
]
}
if old_package_dict.get('license_id') is None:
patch['license_id'] = 'other'
for lang in langs:
patch['title_translated'][lang] = old_package_dict.get(
'title_' + lang, '')
patch['notes_translated'][lang] = old_package_dict.get(
'notes_' + lang, '')
patch['copyright_notice_translated'][lang] = old_package_dict.get(
'copyright_notice_' + lang, '')
patch['resources'] = old_package_dict.get('resources')
for resource in patch.get('resources', []):
resource['name_translated'] = {
original_language: resource.get('name', '') or ''
}
resource['description_translated'] = {
original_language: resource.get('description', '') or ''
}
if resource.get('temporal_granularity') and type(
resource.get('temporal_granularity')) is not dict:
resource['temporal_granularity'] = {
original_language: resource.get('temporal_granularity')
}
else:
del resource['temporal_granularity']
if resource.get('update_frequency') and type(
resource.get('update_frequency')) is not dict:
resource['update_frequency'] = {
original_language: resource.get('update_frequency')
}
else:
del resource['update_frequency']
for lang in langs:
resource['name_translated'][lang] = resource.get(
'name_' + lang, '')
resource['description_translated'][lang] = resource.get(
'description_' + lang, '')
if resource.get('temporal_granularity_' + lang):
resource['temporal_granularity'][lang] = resource.get(
'temporal_granularity_' + lang)
if resource.get('temporal_granularity_' + lang):
resource['update_frequency'][lang] = resource.get(
'update_frequency_' + lang)
package_patches.append(patch)
if dryrun:
print '\n'.join('%s' % p for p in package_patches)
print '\n'.join('%s' % p for p in resource_patches)
else:
apply_patches(package_patches, resource_patches)
def pytest_sessionstart(session):
"""Initialize CKAN environment.
"""
load_config(session.config.option.ckan_ini)
def send_harvester_status_emails(ctx, config, dryrun, force, all_harvesters):
load_config(config or ctx.obj['config'])
email_notification_recipients = t.aslist(
t.config.get('ckanext.ytp.harvester_status_recipients', ''))
if not email_notification_recipients and not dryrun:
print 'No recipients configured'
return
status_opts = {} if not all_harvesters else {
'include_manual': True,
'include_never_run': True
}
status = get_action('harvester_status')({}, status_opts)
errored_runs = any(item.get('errors') != 0 for item in status.values())
running = (item.get('started') for item in status.values()
if item.get('status') == 'running')
stuck_runs = any(_elapsed_since(started).days > 1 for started in running)
if not (errored_runs or stuck_runs) and not force:
print 'Nothing to report'
return
if len(status) == 0:
print 'No harvesters matching criteria found'
return
site_title = t.config.get('ckan.site_title', '')
today = datetime.now().date().isoformat()
status_templates = {
'running':
'%%(title)-%ds | Running since %%(time)s with %%(errors)d errors',
'finished':
'%%(title)-%ds | Finished %%(time)s with %%(errors)d errors',
'pending': '%%(title)-%ds | Pending since %%(time)s'
}
unknown_status_template = '%%(title)-%ds | Unknown status: %%(status)s'
max_title_length = max(len(title) for title in status)
def status_string(title, values):
template = status_templates.get(values.get('status'),
unknown_status_template)
status = values.get('status')
time_field = 'finished' if status == 'finished' else 'started'
return template % max_title_length % {
'title': title,
'time': _pretty_time(values.get(time_field)),
'status': status,
'errors': values.get('errors')
}
msg = '%(site_title)s - Harvester summary %(today)s\n\n%(status)s' % {
'site_title':
site_title,
'today':
today,
'status':
'\n'.join(
status_string(title, values) for title, values in status.items())
}
for recipient in email_notification_recipients:
email = {
'recipient_name': recipient,
'recipient_email': recipient,
'subject': '%s - Harvester summary %s' % (site_title, today),
'body': msg
}
if dryrun:
print 'to: %s' % recipient
else:
try:
mailer.mail_recipient(**email)
except mailer.MailerException as e:
print 'Sending harvester summary to %s failed: %s' % (
recipient, e)
if dryrun:
print msg
def clear(ctx, base_dir, config):
load_config(config or ctx.obj['config'])
try:
shutil.rmtree(os.path.join(base_dir, 'data', 'json', 'prh_data'))
except OSError:
print("Directory does not exist")
help=u'Delete activity detail')
parser.add_argument(u'--dataset', help=u'just migrate this particular '
u'dataset - specify its name')
args = parser.parse_args()
assert args.config, u'You must supply a --config'
try:
from ckan.lib.cli import load_config
except ImportError:
# for CKAN 2.6 and earlier
def load_config(config):
from ckan.lib.cli import CkanCommand
cmd = CkanCommand(name=None)
class Options(object):
pass
cmd.options = Options()
cmd.options.config = config
cmd._load_config()
return
print(u'Loading config')
load_config(args.config)
if not args.dataset:
migrate_all_datasets()
wipe_activity_detail(delete_activity_detail=args.delete)
else:
errors = defaultdict(int)
with PackageDictizeMonkeyPatch():
migrate_dataset(args.dataset, errors)
print_errors(errors)
parser.add_argument(u'--delete', choices=[u'yes', u'no'],
help=u'Delete activity detail')
parser.add_argument(u'--dataset', help=u'just migrate this particular '
u'dataset - specify its name')
args = parser.parse_args()
assert args.config, u'You must supply a --config'
try:
from ckan.lib.cli import load_config
except ImportError:
# for CKAN 2.6 and earlier
def load_config(config):
from ckan.lib.cli import CkanCommand
cmd = CkanCommand(name=None)
class Options(object):
pass
cmd.options = Options()
cmd.options.config = config
cmd._load_config()
return
print(u'Loading config')
load_config(args.config)
if not args.dataset:
migrate_all_datasets()
wipe_activity_detail(delete_activity_detail=args.delete)
else:
errors = defaultdict(int)
migrate_dataset(args.dataset, errors)
print_errors(errors)
