def update_package_deprecation(ctx, config, dryrun): load_config(config or ctx.obj['config']) # after loop contains list of ID's of just deprecated packages deprecated_now = [] package_patches = [] deprecation_offset = package_deprecation_offset() for old_package_dict in package_generator('*:*', 1000): valid_till = old_package_dict.get('valid_till') if valid_till is not None: current_deprecation = old_package_dict.get('deprecated') deprecation = check_package_deprecation(valid_till, deprecation_offset) if current_deprecation != deprecation: patch = {'id': old_package_dict['id'], 'deprecated': deprecation} package_patches.append(patch) # Initially deprecation value will be undefined, so send email only when actually deprecated if current_deprecation is False and deprecation is True: deprecated_now.append(old_package_dict['id']) if dryrun: print '\n'.join(('%s | %s' % (p, p['id'] in deprecated_now)) for p in package_patches) else: # No resources patches so empty parameter is passed apply_patches(package_patches, []) send_package_deprecation_emails(deprecated_now)
def update_package_deprecation(ctx, config, dryrun): load_config(config or ctx.obj['config']) # deprecation emails will be sent to items inside deprecated_now array deprecated_now = [] package_patches = [] # Get only packages with a valid_till field and some value in the valid_till field for old_package_dict in package_generator( 'valid_till:* AND -valid_till:""', 1000): valid_till = old_package_dict.get('valid_till') # For packages that have a valid_till date set depracated field to true or false # deprecation means that a package has been valid but is now old and not valid anymore. # This does not take into account if the package is currently valid eg. valid_from. if valid_till is not None: current_deprecation = old_package_dict.get('deprecated') deprecation = check_package_deprecation(valid_till) if current_deprecation != deprecation: patch = { 'id': old_package_dict['id'], 'deprecated': deprecation } package_patches.append(patch) # Send email only when actually deprecated. Initial deprecation is undefined when adding this feature if current_deprecation is False and deprecation is True: deprecated_now.append(old_package_dict['id']) if dryrun: print '\n'.join(('%s | %s' % (p, p['id'] in deprecated_now)) for p in package_patches) else: # No resources patches so empty parameter is passed apply_patches(package_patches, []) send_package_deprecation_emails(deprecated_now)
def dump(ctx, resource_id, output_file, config, format, offset, limit, bom): load_config(config or ctx.obj['config']) dump_to(resource_id, output_file, fmt=format, offset=offset, limit=limit, options={u'bom': bom})
def dump(ctx, resource_id, output_file, config, format, offset, limit, bom): load_config(config or ctx.obj['config']) dump_to( resource_id, output_file, fmt=format, offset=offset, limit=limit, options={u'bom': bom})
def migrate_author_email(ctx, config, dryrun): load_config(config or ctx.obj['config']) package_patches = [] for old_package_dict in package_generator('*:*', 1000): if old_package_dict.get('author_email') is None: patch = {'id': old_package_dict['id'], 'author_email': ''} package_patches.append(patch) if dryrun: print '\n'.join('%s' % p for p in package_patches) else: # No resources patches so empty parameter is passed apply_patches(package_patches, [])
def migrate_author_email(ctx, config, dryrun): load_config(config or ctx.obj['config']) package_patches = [] for old_package_dict in package_generator('*:*', 1000): if old_package_dict.get('author_email') is None: patch = {'id': old_package_dict['id'], 'author_email': ''} package_patches.append(patch) if dryrun: print '\n'.join('%s' % p for p in package_patches) else: # No resources patches so empty parameter is passed apply_patches(package_patches, [])
def migrate_high_value_datasets(ctx, config, dryrun): load_config(config or ctx.obj['config']) package_patches = [] for old_package_dict in package_generator('*:*', 1000): if old_package_dict.get('high_value_dataset_category'): patch = {'id': old_package_dict['id'], 'international_benchmarks': old_package_dict['high_value_dataset_category']} package_patches.append(patch) if dryrun: print '\n'.join('%s' % p for p in package_patches) else: # No resources patches so empty parameter is passed apply_patches(package_patches, [])
def batch_edit(ctx, config, search_string, dryrun, group): load_config(config or ctx.obj['config']) group_assigns = {} if group: group_assigns[group] = [] for package_dict in package_generator(search_string, 1000): if group: group_assigns[group].append(package_dict['name']) if dryrun: print '\n'.join('Add %s to group %s' % (p, g) for (g, ps) in group_assigns.items() for p in ps) else: if group: apply_group_assigns(group_assigns)
def batch_edit(ctx, config, search_string, dryrun, group): load_config(config or ctx.obj['config']) group_assigns = {} if group: group_assigns[group] = [] for package_dict in package_generator(search_string, 1000): if group: group_assigns[group].append(package_dict['name']) if dryrun: print '\n'.join('Add %s to group %s' % (p, g) for (g, ps) in group_assigns.items() for p in ps) else: if group: apply_group_assigns(group_assigns)
def add_to_groups(ctx, config, dryrun): load_config(config or ctx.obj['config']) context = {'ignore_auth': True} groups = get_action('group_list')(context, {}) users = get_action('user_list')(context, {}) data_dicts = [] for group in groups: memberships = get_action('member_list')(context, {'id': group}) for user in users: if not any(id for (id, type, capacity) in memberships if id == user['id']): data_dicts.append({'id': group, 'username': user['name'], 'role': 'editor'}) if dryrun: print '\n'.join('%s' % d for d in data_dicts) else: for d in data_dicts: get_action('group_member_create')(context, d)
def migrate_high_value_datasets(ctx, config, dryrun): load_config(config or ctx.obj['config']) package_patches = [] for old_package_dict in package_generator('*:*', 1000): if old_package_dict.get('high_value_dataset_category'): patch = { 'id': old_package_dict['id'], 'international_benchmarks': old_package_dict['high_value_dataset_category'] } package_patches.append(patch) if dryrun: print '\n'.join('%s' % p for p in package_patches) else: # No resources patches so empty parameter is passed apply_patches(package_patches, [])
def fetch(ctx, base_dir, year, start_from_beginning, package_id, config): load_config(config or ctx.obj['config']) if Path(os.path.join(base_dir, 'data', 'json', 'prh_data', 'all_done.txt')).is_file(): print("Nothing to do") else: PRHData().get_prh_data(base_directory=base_dir, year=year, start_from_beginning=start_from_beginning) filename = make_csv_of_prh_data(base_directory=base_dir) from scripts.prh.upload_to_ckan import upload_to_ckan upload_to_ckan(package_id, filename) with open( os.path.join(base_dir, 'data', 'json', 'prh_data', 'all_done.txt'), 'w') as done_file: done_file.write("done") done_file.close()
def set_permissions(ctx, config): load_config(config or ctx.obj['config']) write_url = _parse_db_config(u'ckan.datastore.write_url') read_url = _parse_db_config(u'ckan.datastore.read_url') db_url = _parse_db_config(u'sqlalchemy.url') # Basic validation that read and write URLs reference the same database. # This obviously doesn't check they're the same database (the hosts/ports # could be different), but it's better than nothing, I guess. if write_url['db_name'] != read_url['db_name']: exit(u"The datastore write_url and read_url must refer to the same " u"database!") sql = permissions_sql(maindb=db_url['db_name'], datastoredb=write_url['db_name'], mainuser=db_url['db_user'], writeuser=write_url['db_user'], readuser=read_url['db_user']) print(sql)
def set_permissions(ctx, config): load_config(config or ctx.obj['config']) write_url = parse_db_config(u'ckan.datastore.write_url') read_url = parse_db_config(u'ckan.datastore.read_url') db_url = parse_db_config(u'sqlalchemy.url') # Basic validation that read and write URLs reference the same database. # This obviously doesn't check they're the same database (the hosts/ports # could be different), but it's better than nothing, I guess. if write_url['db_name'] != read_url['db_name']: exit(u"The datastore write_url and read_url must refer to the same " u"database!") sql = permissions_sql( maindb=db_url['db_name'], datastoredb=write_url['db_name'], mainuser=db_url['db_user'], writeuser=write_url['db_user'], readuser=read_url['db_user']) print(sql)
def migrate_orgs(ctx, config, dryrun): load_config(config or ctx.obj['config']) default_lang = c.get('ckan.locale_default', 'en') languages = ['fi', 'en', 'sv'] translated_fields = ['title', 'description'] org_patches = [] for old_org_dict in org_generator(): flatten_extras(old_org_dict) patch = {} for field in translated_fields: translated_field = '%s_translated' % field if translated_field not in old_org_dict: patch[translated_field] = {default_lang: old_org_dict[field]} for language in languages: value = old_org_dict.get('%s_%s' % (field, language)) if value is not None and value != "": patch[translated_field][language] = value else: patch[field] = old_org_dict[translated_field].get(default_lang) if 'features' not in old_org_dict: # 'adminstration' is used in previous data model if old_org_dict.get('public_adminstration_organization'): patch['features'] = ["public_administration_organization"] if patch: patch['id'] = old_org_dict['id'] org_patches.append(patch) if dryrun: print '\n'.join('%s' % p for p in org_patches) else: apply_org_patches(org_patches)
def migrate_orgs(ctx, config, dryrun): load_config(config or ctx.obj['config']) default_lang = c.get('ckan.locale_default', 'en') languages = ['fi', 'en', 'sv'] translated_fields = ['title', 'description'] org_patches = [] for old_org_dict in org_generator(): flatten_extras(old_org_dict) patch = {} for field in translated_fields: translated_field = '%s_translated' % field if translated_field not in old_org_dict: patch[translated_field] = {default_lang: old_org_dict[field]} for language in languages: value = old_org_dict.get('%s_%s' % (field, language)) if value is not None and value != "": patch[translated_field][language] = value else: patch[field] = old_org_dict[translated_field].get(default_lang) if 'features' not in old_org_dict: # 'adminstration' is used in previous data model if old_org_dict.get('public_adminstration_organization'): patch['features'] = ["public_administration_organization"] if patch: patch['id'] = old_org_dict['id'] org_patches.append(patch) if dryrun: print '\n'.join('%s' % p for p in org_patches) else: apply_org_patches(org_patches)
def add_to_groups(ctx, config, dryrun): load_config(config or ctx.obj['config']) context = {'ignore_auth': True} groups = get_action('group_list')(context, {}) users = get_action('user_list')(context, {}) data_dicts = [] for group in groups: memberships = get_action('member_list')(context, {'id': group}) for user in users: if not any(id for (id, type, capacity) in memberships if id == user['id']): data_dicts.append({ 'id': group, 'username': user['name'], 'role': 'editor' }) if dryrun: print '\n'.join('%s' % d for d in data_dicts) else: for d in data_dicts: get_action('group_member_create')(context, d)
def migrate(ctx, config, dryrun): load_config(config or ctx.obj['config']) default_lang = c.get('ckan.locale_default', 'en') package_patches = [] resource_patches = [] for old_package_dict in package_generator('*:*', 1000): if 'title_translated' in old_package_dict: continue original_language = old_package_dict.get('original_language', default_lang) langs = old_package_dict.get('translations', []) patch = { 'id': old_package_dict['id'], 'title_translated': { original_language: old_package_dict['title'] }, 'notes_translated': { original_language: old_package_dict['notes'] }, 'copyright_notice_translated': { original_language: old_package_dict.get('copyright_notice', '') }, 'external_urls': old_package_dict.get('extra_information', []) } if old_package_dict.get('tags'): patch['keywords'] = {'fi': [tag['name'] for tag in old_package_dict.get('tags', []) if tag['vocabulary_id'] is None]} if old_package_dict.get('content_type'): patch['content_type'] = {'fi': [s for s in old_package_dict.get('content_type', "").split(',') if s]} if old_package_dict.get('license_id') is None: patch['license_id'] = 'other' for lang in langs: patch['title_translated'][lang] = old_package_dict.get('title_' + lang, '') patch['notes_translated'][lang] = old_package_dict.get('notes_' + lang, '') patch['copyright_notice_translated'][lang] = old_package_dict.get('copyright_notice_' + lang, '') patch['resources'] = old_package_dict.get('resources') for resource in patch.get('resources', []): resource['name_translated'] = { original_language: resource.get('name', '') or '' } resource['description_translated'] = { original_language: resource.get('description', '') or '' } if resource.get('temporal_granularity') and type(resource.get('temporal_granularity')) is not dict: resource['temporal_granularity'] = { original_language: resource.get('temporal_granularity') } else: del resource['temporal_granularity'] if resource.get('update_frequency') and type(resource.get('update_frequency')) is not dict: resource['update_frequency'] = { original_language: resource.get('update_frequency') } else: del resource['update_frequency'] for lang in langs: resource['name_translated'][lang] = resource.get('name_' + lang, '') resource['description_translated'][lang] = resource.get('description_' + lang, '') if resource.get('temporal_granularity_' + lang): resource['temporal_granularity'][lang] = resource.get('temporal_granularity_' + lang) if resource.get('temporal_granularity_' + lang): resource['update_frequency'][lang] = resource.get('update_frequency_' + lang) package_patches.append(patch) if dryrun: print '\n'.join('%s' % p for p in package_patches) print '\n'.join('%s' % p for p in resource_patches) else: apply_patches(package_patches, resource_patches)
def init_db(ctx, config): load_config((config or ctx.obj['config'])) commands.init_db()
def fetch(ctx, config, dryrun, since, until): load_config(config or ctx.obj['config']) commands.fetch(dryrun, since, until)
def migrate(ctx, config, dryrun): load_config(config or ctx.obj['config']) default_lang = c.get('ckan.locale_default', 'en') package_patches = [] resource_patches = [] for old_package_dict in package_generator('*:*', 1000): if 'title_translated' in old_package_dict: continue original_language = old_package_dict.get('original_language', default_lang) langs = old_package_dict.get('translations', []) patch = { 'id': old_package_dict['id'], 'title_translated': { original_language: old_package_dict['title'] }, 'notes_translated': { original_language: old_package_dict['notes'] }, 'copyright_notice_translated': { original_language: old_package_dict.get('copyright_notice', '') }, 'external_urls': old_package_dict.get('extra_information', []) } if old_package_dict.get('tags'): patch['keywords'] = { 'fi': [ tag['name'] for tag in old_package_dict.get('tags', []) if tag['vocabulary_id'] is None ] } if old_package_dict.get('content_type'): patch['content_type'] = { 'fi': [ s for s in old_package_dict.get('content_type', "").split( ',') if s ] } if old_package_dict.get('license_id') is None: patch['license_id'] = 'other' for lang in langs: patch['title_translated'][lang] = old_package_dict.get( 'title_' + lang, '') patch['notes_translated'][lang] = old_package_dict.get( 'notes_' + lang, '') patch['copyright_notice_translated'][lang] = old_package_dict.get( 'copyright_notice_' + lang, '') patch['resources'] = old_package_dict.get('resources') for resource in patch.get('resources', []): resource['name_translated'] = { original_language: resource.get('name', '') or '' } resource['description_translated'] = { original_language: resource.get('description', '') or '' } if resource.get('temporal_granularity') and type( resource.get('temporal_granularity')) is not dict: resource['temporal_granularity'] = { original_language: resource.get('temporal_granularity') } else: del resource['temporal_granularity'] if resource.get('update_frequency') and type( resource.get('update_frequency')) is not dict: resource['update_frequency'] = { original_language: resource.get('update_frequency') } else: del resource['update_frequency'] for lang in langs: resource['name_translated'][lang] = resource.get( 'name_' + lang, '') resource['description_translated'][lang] = resource.get( 'description_' + lang, '') if resource.get('temporal_granularity_' + lang): resource['temporal_granularity'][lang] = resource.get( 'temporal_granularity_' + lang) if resource.get('temporal_granularity_' + lang): resource['update_frequency'][lang] = resource.get( 'update_frequency_' + lang) package_patches.append(patch) if dryrun: print '\n'.join('%s' % p for p in package_patches) print '\n'.join('%s' % p for p in resource_patches) else: apply_patches(package_patches, resource_patches)
def pytest_sessionstart(session): """Initialize CKAN environment. """ load_config(session.config.option.ckan_ini)
def send_harvester_status_emails(ctx, config, dryrun, force, all_harvesters): load_config(config or ctx.obj['config']) email_notification_recipients = t.aslist( t.config.get('ckanext.ytp.harvester_status_recipients', '')) if not email_notification_recipients and not dryrun: print 'No recipients configured' return status_opts = {} if not all_harvesters else { 'include_manual': True, 'include_never_run': True } status = get_action('harvester_status')({}, status_opts) errored_runs = any(item.get('errors') != 0 for item in status.values()) running = (item.get('started') for item in status.values() if item.get('status') == 'running') stuck_runs = any(_elapsed_since(started).days > 1 for started in running) if not (errored_runs or stuck_runs) and not force: print 'Nothing to report' return if len(status) == 0: print 'No harvesters matching criteria found' return site_title = t.config.get('ckan.site_title', '') today = datetime.now().date().isoformat() status_templates = { 'running': '%%(title)-%ds | Running since %%(time)s with %%(errors)d errors', 'finished': '%%(title)-%ds | Finished %%(time)s with %%(errors)d errors', 'pending': '%%(title)-%ds | Pending since %%(time)s' } unknown_status_template = '%%(title)-%ds | Unknown status: %%(status)s' max_title_length = max(len(title) for title in status) def status_string(title, values): template = status_templates.get(values.get('status'), unknown_status_template) status = values.get('status') time_field = 'finished' if status == 'finished' else 'started' return template % max_title_length % { 'title': title, 'time': _pretty_time(values.get(time_field)), 'status': status, 'errors': values.get('errors') } msg = '%(site_title)s - Harvester summary %(today)s\n\n%(status)s' % { 'site_title': site_title, 'today': today, 'status': '\n'.join( status_string(title, values) for title, values in status.items()) } for recipient in email_notification_recipients: email = { 'recipient_name': recipient, 'recipient_email': recipient, 'subject': '%s - Harvester summary %s' % (site_title, today), 'body': msg } if dryrun: print 'to: %s' % recipient else: try: mailer.mail_recipient(**email) except mailer.MailerException as e: print 'Sending harvester summary to %s failed: %s' % ( recipient, e) if dryrun: print msg
def clear(ctx, base_dir, config): load_config(config or ctx.obj['config']) try: shutil.rmtree(os.path.join(base_dir, 'data', 'json', 'prh_data')) except OSError: print("Directory does not exist")
help=u'Delete activity detail') parser.add_argument(u'--dataset', help=u'just migrate this particular ' u'dataset - specify its name') args = parser.parse_args() assert args.config, u'You must supply a --config' try: from ckan.lib.cli import load_config except ImportError: # for CKAN 2.6 and earlier def load_config(config): from ckan.lib.cli import CkanCommand cmd = CkanCommand(name=None) class Options(object): pass cmd.options = Options() cmd.options.config = config cmd._load_config() return print(u'Loading config') load_config(args.config) if not args.dataset: migrate_all_datasets() wipe_activity_detail(delete_activity_detail=args.delete) else: errors = defaultdict(int) with PackageDictizeMonkeyPatch(): migrate_dataset(args.dataset, errors) print_errors(errors)
parser.add_argument(u'--delete', choices=[u'yes', u'no'], help=u'Delete activity detail') parser.add_argument(u'--dataset', help=u'just migrate this particular ' u'dataset - specify its name') args = parser.parse_args() assert args.config, u'You must supply a --config' try: from ckan.lib.cli import load_config except ImportError: # for CKAN 2.6 and earlier def load_config(config): from ckan.lib.cli import CkanCommand cmd = CkanCommand(name=None) class Options(object): pass cmd.options = Options() cmd.options.config = config cmd._load_config() return print(u'Loading config') load_config(args.config) if not args.dataset: migrate_all_datasets() wipe_activity_detail(delete_activity_detail=args.delete) else: errors = defaultdict(int) migrate_dataset(args.dataset, errors) print_errors(errors)