def create_to_registry(package_id): ''' Create new dataset in registry with the values from OG Called by: * main() Call: * get_data_from_url() * replace_branch_and_data_steward() * replace_regions() * ''' with open("Data//fieldsAdded.json") as json_fp: add_fields = json.load(json_fp) og_data = get_data_from_url(package_id, "open_gov_url") for k,v in add_fields.items(): og_data[k] = add_fields[k] #Post to registry replace_branch_and_data_steward(og_data) replace_regions(og_data) default_resource_date_published(og_data) reg_site = os.getenv("registry_url") registry_key = os.getenv("registry_api_key") rckan = RemoteCKAN(reg_site, apikey=registry_key) try: ret = rckan.call_action("package_create", data_dict=og_data) except Exception as e: print(e) return False return True
def __init__(self, data_transfer_config, logger): super().__init__(data_transfer_config, logger) self.from_infra = data_transfer_config['from_source']['located_at'] self.to_infra = data_transfer_config['to_target']['located_at'] if 'croupier.nodes.CKAN_dataset' in self.from_infra['type_hierarchy']: self.direction = 'download' self.ckan_dataset = self.from_infra self.ckan_resource = data_transfer_config['from_source'][ 'resource'] elif 'croupier.nodes.CKAN_dataset' in self.to_infra['type_hierarchy']: self.direction = 'upload' self.ckan_dataset = self.to_infra self.ckan_resource = data_transfer_config['to_target']['resource'] else: logger.error( 'CKANAPI Data Transfer must have a "CKAN_dataset" as one of its endpoints' ) raise Exception self.dataset_info = self.ckan_dataset['dataset_info'] self.endpoint = self.ckan_dataset['endpoint'] self.apikey = self.ckan_dataset['credentials']['auth-header'] self.api = RemoteCKAN(self.endpoint, apikey=self.apikey) if not self.dataset_info['package_id']: self._find_dataset() try: self.api.action.site_read() except ServerIncompatibleError: self.logger.error( 'Could not connect to CKAN server, non valid endpoint')
def test_post(): package_id = "e328838f-3bfc-4d86-9cc5-23de0b549c91" apicall = "api/3/action/package_show" q_param="?id=" +package_id site = os.getenv("registry_url") registry_key = os.getenv("registry_api_key") ret_as_string = query_with_get(site, apicall, q_param, apikey=registry_key) res_as_dict = json.loads(ret_as_string)['result'] data_as_dict = res_as_dict data_as_dict['title'] = res_as_dict['title'] + str(randint(0,99)) del data_as_dict['id'] del data_as_dict['revision_id'] if "aafc_subject" in data_as_dict: del data_as_dict['aafc_subject'] data_as_d = {} with open("Data//fromAlexis.json") as json_fp: data_as_d = json.load(json_fp) # do a post registry_key = os.getenv("registry_api_key") rckan = RemoteCKAN(site, apikey=registry_key) try: ret = rckan.call_action("package_create", data_dict=data_as_d)#data_as_dict ) except Exception as e: pass #post_to_site(site, "api/3/action/package_create",data_as_dict,registry_key) pass
def __init__(self): self.ua = ( 'waarismijnstemlokaal/1.0 (+https://waarismijnstemlokaal.nl/)') self.ckanapi = RemoteCKAN(app.config['CKAN_URL'], apikey=app.config['CKAN_API_KEY'], user_agent=self.ua).action self.elections = app.config['CKAN_CURRENT_ELECTIONS'] self.resources_metadata = self._get_resources_metadata()
def get_n_post(package_id): """ Called by main() Call get_data_from_reg() Get an data from registry , modify and post it to OG :param package_id: :return: """ og_data = get_data_from_reg(package_id) # replace for k, v in to_replace.items(): og_data[k] = to_replace[k] # Use branch and data steward values to populate metadata_contact field (necessary in sync_with_og script). data_steward = og_data["data_steward_email"].strip() branch = og_data["organization"]["description"].split('|') branch_en = branch[0].strip() branch_fr = branch[1].strip() to_add = { "metadata_contact": { "fr": "Gouvernement du Canada; Agriculture et Agroalimentaire Canada, " + branch_fr + ", " + data_steward, "en": "Government of Canada; Agriculture and Agri-Food Canada, " + branch_en + ", " + data_steward } } for k, v in to_add.items(): og_data[k] = to_add[k] # remove for k in to_remove: del og_data[k] og_site = os.getenv("open_gov_registry_url") og_key = os.getenv("open_gov_registry_api_key") rckan = RemoteCKAN(og_site, apikey=og_key) # First try to create new package. # If package create fails, it's possible that the package already exists # Try to update the package. If both of these actions fail, return false. try: ret = rckan.call_action("package_create", data_dict=og_data) except Exception as e1: print(e1) try: ret = rckan.call_action("package_update", data_dict=og_data) except Exception as e2: print(e2) return False return True return True
def setUp(self): self.ckan_portal = RemoteCKAN(self.portal_url, apikey=self.apikey) for dataset in self.test_datasets: try: self.ckan_portal.call_action('dataset_purge', data_dict={'id': dataset['id']}) except NotFound: continue for dataset in self.test_datasets: self.ckan_portal.call_action('package_create', data_dict=dataset)
def export_to_ckan(arguments): input_dir = arguments['--input'] input_dir = input_dir if path.isabs(input_dir) else path.abspath(input_dir) ckan_host = arguments['--site'] organization = arguments['--organization'] api_key = arguments['--api-key'] public = arguments['--public'] if not ckan_host: ckan_host = prompt({ 'type': 'input', 'name': 'ckan_host', 'message': 'Inform your CKAN API Site: ' })['ckan_host'] if not api_key: api_key = prompt({ 'type': 'password', 'name': 'api_key', 'message': 'Inform your CKAN API Key:' })['api_key'] ckan = RemoteCKAN(ckan_host, apikey=api_key) organizations = ckan.call_action("organization_list", {"all_fields": True}) if not organization: organization = prompt({ 'type': 'list', 'name': 'organization', 'message': 'Select the organization:', 'choices': [o['name'] for o in organizations] })['organization'] packages_cache = {} selected_org = next( filter(lambda o: o['name'] == organization, organizations)) for meta_file in filter(lambda f: f.endswith("meta.json"), tqdm(os.listdir(input_dir))): original_file = meta_file.replace('.meta.json', '') original_file = path.join(input_dir, original_file) meta_file = path.join(input_dir, meta_file) if path.exists(original_file): with open(meta_file, 'r', encoding='latin-1') as mfp: meta_data = json.load(fp=mfp) export_file(ckan, original_file, meta_data, selected_org, public, packages_cache) else: print(f"{original_file} not found. Deleting meta the data file.") os.remove(meta_file)
def search(query, datatype=None, apikey=apikey): """Takes a string input and returns search results matching existing CKAN objects (organisation, dataset, resource, user). Arguments: query -- a single search term For 'resource' datatypes this defaults to searching the resource name. Additional arguments can be added using the following syntax: 'query1 field2:query2' Valid resource fields can be used as field terms for the search. datatype -- organisation, project, dataset, resource, user (default None) apikey -- valid CKAN API key (default None) Private datasets will only be shown to authorised API keys """ # Create empty list object to contain API call results d = [] # Get datatype if not specified by user if datatype == None: datatype = input( 'Are you looking for an organisation, project, dataset, resource or user?\n\n' ).lower().strip() # Check for datatype organisation or project if datatype == 'organisation' or datatype == 'project': d = RemoteCKAN(url, apikey).action.organization_autocomplete(q=query) # Check for dataytpe dataset elif datatype == 'dataset': resources = RemoteCKAN( url, apikey).action.package_search(q=query)['results'] for r in range(len(resources)): d.append({k: resources[r][k] for k in ('name', 'title')}) # Check for datatype resource elif datatype == 'resource': query = ''.join(['name:', query]) resources = RemoteCKAN( url, apikey).action.resource_search(query=query.split(" "))['results'] for r in range(len(resources)): d.append({ k: resources[r][k] for k in ('description', 'format', 'id', 'last_modified', 'name', 'package_id', 'revision_id') }) # Check for datatype user elif datatype == 'user': d = RemoteCKAN(url, apikey).action.user_autocomplete(q=query) else: print( 'Please try a different search query and type a valid data type. This can be an organisation, project, dataset, resource or user.\n' ) # Format results dataframe to be returned if len(d) == 0: print('Cannot find %s for this search term' % datatype) elif len(d[0]) > len(d) > 0: return (pd.DataFrame(d).T) elif len(d) > 0: return (pd.DataFrame(d))
def __call__(self): endpoint = getattr(self.api.action, self.action) metadata = {} for item_id in set(self.entity_ids): try: metadata[item_id] = endpoint(id=item_id) except errors.NotFound: url = f"action={self.action} id={item_id}" logging.error(f"CKAN API NotFound error: {url}") RemoteCKAN.close(self.api) return metadata
def get_data_from_url(package_id, url): #site = os.getenv("registry_url") site = os.getenv(url) rckan = RemoteCKAN(site) data_as_d = {"id":package_id} try: ret = rckan.call_action("package_show", data_dict=data_as_d)#data_as_dict ) except Exception as e: # if no data exists yet, return empty ret = [] return ret
def __init__(self, url, apikey=None): self.apikey = apikey self.remote = RemoteCKAN(url, apikey=self.apikey) try: res = self.call_action('site_read') except Exception: raise CkanReadError() # else: logger.info('Open CKAN connection with api key: {}'.format(apikey)) if not res: self.close() raise CkanApiError()
def scidm_resources_download(token, package_id): scidm = RemoteCKAN('https://scidm.nchc.org.tw', apikey=token) # 遍尋包含的資料 name_list = [] url_list = [] for data in scidm.call_action('package_show', {'id': package_id})['resources']: name_list.append(data['name']) url_list.append(data['url']) download_data = pd.DataFrame({'name': name_list, 'url': url_list}) return download_data
def show(name, datatype=None, apikey=apikey): """Takes a string input and returns an existing CKAN object (organisation, dataset or resource). Arguments: name -- valid name or id of CKAN data object. For 'resource' datatypes this must be the 36 digit 'id'. datatype -- organisation, project, dataset or resource (default None) apikey -- a valid CKAN API key. Private datasets will only be shown to authorised API keys (default None) """ d = {} #create empty dict object to contain API call results if datatype == None: datatype = input( 'Is this an organisation, a project, a dataset or a resource?\n\n' ).lower().strip() if datatype == 'organisation' or datatype == 'project': if name == 'all': d = RemoteCKAN(url, apikey).action.organization_list() else: try: d = RemoteCKAN(url, apikey).action.organization_show( id=name, include_datasets=True, include_groups=False, include_tags=False, include_followers=False, include_users=False) except Exception: print( 'This is neither a valid organisation nor a valid project') elif datatype == 'dataset': try: d = RemoteCKAN(url, apikey).action.package_show(id=name) except Exception: print('This is not a valid dataset') elif datatype == 'resource': try: d = RemoteCKAN(url, apikey).action.resource_show(id=name) except Exception: print( 'This is not a valid resource. Check that you are using the resource id, not the name, and try again.' ) else: print( 'Oops. Use a valid name and type a valid data type. This can be an organisation, project, dataset or resource.' ) if len(d) > 0: return pd.Series(d)
def save(self, *args, **kwargs): from ckanapi import RemoteCKAN configuracao_sistema = ConfiguracaoSistema.objects.first() ckan = RemoteCKAN(configuracao_sistema.url_ckan, apikey=configuracao_sistema.token_ckan) try: if not self.id: # Criando a Organização no CKAN retorno = ckan.action.organization_create( name=self.slug, title=self.nome, description=self.descricao, image_url=self.url_logomarca) self.id_ckan = retorno.get('id') else: # Atualizando a Organização no CKAN retorno = ckan.action.organization_patch( id=self.id_ckan, name=self.slug, title=self.nome, description=self.descricao, image_url=self.url_logomarca) self.id_ckan = retorno.get('id') except: pass super(Organizacao, self).save(*args, **kwargs)
def add_groups(new_hri_docs, ckan_source="https://hri.dataportaali.com/data"): group_map = cats_to_groups_map() c = RemoteCKAN(ckan_source, user_agent='ckanapiexample/1.0') groups = c.action.group_list(all_fields=True) groups_data = dict( ((g['name'], g) for g in groups) ) yes_groups = set() no_groups = set() for doc in new_hri_docs: cats = get_cat(doc) doc_groups = [] for cat in cats: gdata = groups_data.get(group_map.get(cat)) if gdata: doc_groups.append(gdata) if doc_groups: doc['groups'] = doc_groups
def package_search(url, organization): obj = {'api': 'package_search', 'count': 0, 'results': []} try: apiserver = RemoteCKAN(url) doLoop = True pos = 0 count = 100 while doLoop: rs = apiserver.action.package_search( q="organization:{}".format(organization), start=pos, rows=count) if rs: n = len(rs['results']) if n > 0: obj['results'].extend(rs['results']) pos += n else: doLoop = False else: doLoop = False obj['count'] = len(obj['results']) except Exception as e: print('package_search ERROR:', e) return obj
def getMetadata(key, value, id): try: ckan = RemoteCKAN(key, user_agent=ua) show = ckan.action.package_show(id=value) writeMeta(show, id, value) except: print('\n Terjadi kesalahan')
def _handle_messages(self, config, messages): self.log.debug(f'{self.group_name} | reading {len(messages)} messages') ckan_instance = self._job_ckan(config=config) server_url = ckan_instance.definition.get('url') api_key = ckan_instance.definition.get('key') ckan_remote = RemoteCKAN(server_url, apikey=api_key) count = 0 records = [] topic = None for msg in messages: topic = msg.topic schema = msg.schema if schema != self._schemas.get(topic): self.log.info(f'{self._id} Schema change on {topic}') self._schemas[topic] = schema fields, definition_names = extract_fields_from_schema(schema) fields = prepare_fields_for_resource(fields, definition_names) self._topic_fields[topic] = fields else: self.log.debug('Schema unchanged.') records.append(msg.value) resource = self.submit_artefacts(topic, schema, ckan_remote) count += 1 if resource: self._create_resource_in_datastore(resource, ckan_remote) self.send_data_to_datastore(self._topic_fields[topic], records, resource, ckan_remote) self.log.info(f'processed {count} {topic} docs')
def main(): ini_config = ConfigParser() ini_config.read('geogratis.ini') remote_ckan_url = ini_config.get('ckan', 'ckan.url') factory = MetadataDatasetModelGeogratisFactory() # Create CKAN API connector to the portal ckan_portal = RemoteCKAN( remote_ckan_url, user_agent='converter/1.0 http://open.canada.ca/data') # Page through the datasets on session = connect_to_database() last_id = 0 try: while True: ckan_records = ckan_portal.action.package_search( q= 'extras_collection:geogratis AND extras_org_title_at_publication:"Natural Resources Canada"', rows=100, start=last_id) if not ckan_records: break else: for r in ckan_records['results']: rp = session.query(GeogratisRecord).filter( GeogratisRecord.uuid == r['name']).all() if not rp: print r['name'] last_id += 100 except Exception, e: print >> stderr, e.message pass
def test_resource_upload_unicode_param(self): uname = b't\xc3\xab\xc3\x9ft resource'.decode('utf-8') with RemoteCKAN(TEST_CKAN) as ckan: res = ckan.call_action( 'test_upload', {'option': uname}, files={'upload': StringIO(NUMBER_THING_CSV)}) self.assertEqual(res.get('option'), uname)
def ckan_api_staging(self): user_agent = None CKAN_API_KEY = "put_api_key" CKAN_URL = os.environ.get("CKAN_URL", "https://staging.basedosdados.org") return RemoteCKAN(CKAN_URL, user_agent=user_agent, apikey=CKAN_API_KEY)
def get_ckan_record(record_id): """ Retrieve a CKAN dataset record from a remote CKAN portal :param record_id: Unique Identifier for the dataset - For Open Canada, these are always UUID's :return: The CKAN package, or an empty dict if the dataset could not be retrieved """ remote_ckan_url = Config.get('ckan', 'remote_url') user_agent = Config.get('web', 'user_agent') with RemoteCKAN(remote_ckan_url, user_agent=user_agent) as ckan_instance: package_record = {} try: package_record = ckan_instance.action.package_show(id=record_id) except NotFound: # This is a new record! logger.info( 'get_ckan_record(): Cannot find record {0}'.format(record_id)) except requests.exceptions.ConnectionError as ce: logger.error( 'get_ckan_record(): Fatal connection error {0}'.format( ce.message)) exit(code=500) return package_record
def portal_update_worker(self, source): """ a process that accepts package ids on stdin which are passed to the package_show API on the remote CKAN instance and compared to the local version of the same package. The local package is then created, updated, deleted or left unchanged. This process outputs that action as a string 'created', 'updated', 'deleted' or 'unchanged' """ registry = RemoteCKAN(source) portal = LocalCKAN() now = datetime.now() for package_id in iter(sys.stdin.readline, ''): try: data = registry.action.package_show(id=package_id.strip()) source_pkg = data['result'] except NotAuthorized: source_pkg = None _trim_package(source_pkg) if source_pkg: # treat unpublished packages same as deleted packages if not source_pkg['portal_release_date'] or isodate( source_pkg['portal_release_date'], None) > now: source_pkg = None try: # don't pass user in context so deleted packages # raise NotAuthorized target_pkg = portal.call_action('package_show', {'id':package_id.strip()}, {}) except (NotFound, NotAuthorized): target_pkg = None _trim_package(target_pkg) if target_pkg is None and source_pkg is None: result = 'unchanged' elif target_pkg is None: # CREATE portal.action.package_create(**source_pkg) result = 'created' elif source_pkg is None: # DELETE portal.action.package_delete(id=package_id.strip()) result = 'deleted' elif source_pkg == target_pkg: result = 'unchanged' else: # UPDATE portal.action.package_update(**source_pkg) result = 'updated' sys.stdout.write(result + '\n') try: sys.stdout.flush() except IOError: break
def delete_ckan_record(package_id): """ Remove a dataset and its associated resource from CKAN :param package_id: :return: Nothing """ # First, verify and get the resource ID package_record = get_ckan_record(package_id) if len(package_record) == 0: logger.warn("delete_ckan_record(): cannot find record ID {0}".format( package_id)) return # Get rid of the resource remote_ckan_url = Config.get('ckan', 'remote_url') remote_ckan_api = Config.get('ckan', 'remote_api_key') user_agent = Config.get('web', 'user_agent') with RemoteCKAN(remote_ckan_url, user_agent=user_agent, apikey=remote_ckan_api) as ckan_instance: try: delete_blob( ckan_container, 'resources/{0}/{1}'.format( package_record['resources'][0]['id'], package_record['resources'][0]['name'].lower())) ckan_instance.action.package_delete(id=package_record['id']) ckan_instance.action.dataset_purge(id=package_record['id']) logger.info("Deleted expired CKAN record {0}".format( package_record['id'])) except Exception as ex: logger.error("delete_ckan_record(): {0}".format(ex.message))
def __init__(self, datasets_dir=None): """Default constructor that initializes an instance of CKAN. Besides the ckan instance, initializes a dict to register the paths of downloaded data in the following form { "packageX": { "resourceX1": "path to the file", ... } } Args: datasets_dir - the path of the local directory where datasets (packages) should be saved. Defaults to the value of the DATASETS environment variable. """ if datasets_dir: self.datasets_dir = datasets_dir else: try: self.datasets_dir = os.environ['DATASETS'] except KeyError: current_dir = Path().absolute() self.datasets_dir = str(current_dir.parent) + '/datasets' self.ckan = RemoteCKAN(CvmClient.CVM_URL) self.local_files = {}
class Action(ActionRuntime): category = "Data Sourcing" required_config = """ url: # URL for the CKAN endpoint action: # CKAN action to perform """ optional_config = """ ignore_not_found: False # Ignore collection not found errors """ def on_start(self, config): self.ckan = RemoteCKAN(config["url"]) def on_input(self, item): action = self.config["action"] args = self.config.get("arguments", {}) try: data = self.ckan.call_action(action, args) except errors.NotFound: if self.config["ignore_not_found"]: return else: raise if isinstance(data, list): for new_item in data: self.put(new_item) else: self.put(data)
def get_ckan_record(record_id): """ Retrieve a CKAN dataset record from a remote CKAN portal :param record_id: Unique Identifier for the dataset - For Open Canada, these are always UUID's :return: The CKAN package, or an empty dict if the dataset could not be retrieved """ remote_ckan_url = Config.get('ckan', 'remote_url') user_agent = Config.get('web', 'user_agent') with RemoteCKAN(remote_ckan_url, user_agent=user_agent) as ckan_instance: package_record = {} try: package_record = ckan_instance.action.package_show(id=record_id) except NotFound: # This is a new record! cprint('Record {0} does not exist'.format(record_id), 'yellow') except requests.exceptions.ConnectionError as ce: cprint('get_ckan_record(): Fatal connection error {0}'.format( ce.message), 'red', attrs=['blink']) exit(code=500) except CKANAPIError as ne: cprint( 'get_ckan_record(): Unexpected error {0}'.format(ne.message), 'yellow') return package_record
def main(): ini_config = ConfigParser() ini_config.read('geogratis.ini') remote_ckan_url = ini_config.get('ckan', 'ckan.url') args = argparser.parse_args() factory = MetadataDatasetModelGeogratisFactory() now_str = datetime.now().strftime('%Y-%m-%dT%H:%M:%S.000Z') # Create CKAN API connector to the portal ckan_portal = RemoteCKAN( remote_ckan_url, user_agent='converter/1.0 http://open.canada.ca/data') # Potentially doing a VERY large ORM query. If we don't limit the read, then SQLAlchemy will try to pull # everything into memory. Therefore the query must be paged. Paging requires keeping track of the sequential # record ID's session = connect_to_database() last_id = 0 scan_date = None setting = get_setting('last_conversion_run') if args.since != '': try: scan_date = datetime.fromtimestamp( time.mktime(time.strptime(args.since, '%Y-%m-%d'))) except ValueError: logging.error("Incorrect since date format. Use YYYY-MM-DD") session.close() exit() except Exception, e: logging.error(e.message) session.close() exit()
def __init__(self, config): self.c = config self.registry = RemoteCKAN(config.api_url, apikey=config.api_key) with urllib.request.urlopen('https://raw.githubusercontent.com/depositar/' + \ 'ckanext-data-depositario/master/ckanext/' + \ 'data_depositario/schemas/dataset.yaml') as url: self.schema = yaml.load(url.read())
def get_package_parameter(site, package_id, parameter, API_key=None): # Some package parameters you can fetch from the WPRDC with # this function are: # 'geographic_unit', 'owner_org', 'maintainer', 'data_steward_email', # 'relationships_as_object', 'access_level_comment', # 'frequency_publishing', 'maintainer_email', 'num_tags', 'id', # 'metadata_created', 'group', 'metadata_modified', 'author', # 'author_email', 'state', 'version', 'department', 'license_id', # 'type', 'resources', 'num_resources', 'data_steward_name', 'tags', # 'title', 'frequency_data_change', 'private', 'groups', # 'creator_user_id', 'relationships_as_subject', 'data_notes', # 'name', 'isopen', 'url', 'notes', 'license_title', # 'temporal_coverage', 'related_documents', 'license_url', # 'organization', 'revision_id' try: ckan = RemoteCKAN(site, apikey=API_key) metadata = ckan.action.package_show(id=package_id) desired_string = metadata[parameter] #print("The parameter {} for this package is {}".format(parameter,metadata[parameter])) except: raise RuntimeError( "Unable to obtain package parameter '{}' for package with ID {}". format(parameter, package_id)) return desired_string
def copy_datasets(self, remote, package_ids=None): """ a process that accepts package ids on stdin which are passed to the package_show API on the remote CKAN instance and compared to the local version of the same package. The local package is then created, updated, deleted or left unchanged. This process outputs that action as a string 'created', 'updated', 'deleted' or 'unchanged' """ if self.options.push_apikey and not self.options.fetch: registry = LocalCKAN() portal = RemoteCKAN(remote, apikey=self.options.push_apikey) elif self.options.fetch: registry = RemoteCKAN(remote) portal = LocalCKAN() else: print "exactly one of -f or -a options must be specified" return now = datetime.now() if not package_ids: package_ids = iter(sys.stdin.readline, '') for package_id in package_ids: package_id = package_id.strip() reason = None target_deleted = False try: source_pkg = registry.action.package_show(id=package_id) except NotAuthorized: source_pkg = None except (CKANAPIError, urllib2.URLError), e: sys.stdout.write(json.dumps([package_id, 'source error', unicode(e.args)]) + '\n') raise if source_pkg and source_pkg['state'] == 'deleted': source_pkg = None if source_pkg and source_pkg['type'] != 'dataset': # non-default dataset types ignored source_pkg = None _trim_package(source_pkg) if source_pkg and not self.options.mirror: # treat unpublished packages same as deleted packages if not source_pkg['portal_release_date']: source_pkg = None reason = 'release date not set' elif isodate(source_pkg['portal_release_date'], None) > now: source_pkg = None reason = 'release date in future' try: target_pkg = portal.call_action('package_show', {'id':package_id}) except (NotFound, NotAuthorized): target_pkg = None except (CKANAPIError, urllib2.URLError), e: sys.stdout.write(json.dumps([package_id, 'target error', unicode(e.args)]) + '\n') raise
def test_good_oldstyle(self): ckan = RemoteCKAN(TEST_CKAN) self.assertEqual( ckan.action.organization_list(), ['aa', 'bb', 'cc']) ckan.close()
def post(self, request, publisher_id): user = request.user.organisationuser iati_user_id = user.iati_user_id publisher = Publisher.objects.get(pk=publisher_id) source_url = request.data.get('source_url', None) if not source_url: raise exceptions.APIException(detail="no source_url provided") user = request.user organisationuser = user.organisationuser api_key = organisationuser.iati_api_key client = RemoteCKAN(settings.CKAN_URL, apikey=api_key) # TODO: should this be the name? - 2017-02-20 source_name = '{}-organisations'.format(publisher.name) # get all published organisations, except for the ones that are just # modified organisations = Organisation.objects.filter( ready_to_publish=True, publisher=publisher ) try: orgList = client.call_action('organization_list_for_user', {}) except BaseException: raise exceptions.APIException( detail="Can't get organisation list for user".format( iati_user_id )) primary_org_id = orgList[0]['id'] try: # sync main datasets to IATI registry registry_dataset = client.call_action('package_create', { "resources": [ {"url": source_url} ], "name": source_name, "filetype": "organisation", "date_updated": datetime.now().strftime('%Y-%m-%d %H:%M'), "organisation_count": organisations.count(), "title": source_name, "owner_org": primary_org_id, "url": source_url, }) except Exception as e: # try to recover from case when the dataset already exists (just # update it instead) old_package = client.call_action('package_show', { "name_or_id": source_name, }) if not old_package: raise exceptions.APIException( detail="Failed publishing dataset" ) registry_dataset = client.call_action('package_update', { "id": old_package.get('id'), "resources": [ {"url": source_url} ], "name": source_name, "filetype": "organisation", "date_updated": datetime.now().strftime('%Y-%m-%d %H:%M'), "organisation_count": organisations.count(), "title": source_name, "owner_org": primary_org_id, "url": source_url, }) # 0. create_or_update Dataset object dataset = Dataset.objects.get( filetype=2, publisher=publisher, added_manually=True, ) dataset.iati_id = registry_dataset['id'] dataset.name = source_name dataset.title = source_name dataset.source_url = source_url dataset.is_parsed = False dataset.save() # update the affected organisations flags organisations.update( published=True, modified=False, ready_to_publish=True) # remove the old datasets from the registry # TODO: query the registry to remove a dataset - 2017-01-16 # TODO: remove old datasets locally as well - 2017-01-16 # TODO: Or just ask the user to remove the old datasets by hand? # - 2017-02-20 # return Dataset object serializer = DatasetSerializer(dataset, context={'request': request}) return Response(serializer.data)
def post(self, request): # TODO: If verifying for the first time, OrganisationGroup and # OrganisationAdminGroup don't exist yet. - 2016-10-25 # publisher = Publisher.objects.get(pk=publisher_id) # group = OrganisationAdminGroup.objects.get(publisher_id=publisher_id) user = request.user.organisationuser api_key = request.data.get('apiKey') user_id = request.data.get('userId') if not api_key or not user_id: raise exceptions.ParseError( detail="apiKey or userId not specified") client = RemoteCKAN(settings.CKAN_URL, apikey=api_key) try: client.call_action('user_show', { "id": user_id, "include_datasets": True, }) except BaseException: raise exceptions.APIException( detail="user with id {} not found".format(user_id)) # print('got user') # print(result) try: orgList = client.call_action('organization_list_for_user', {}) except BaseException: raise exceptions.APIException( detail="Can't get organisation list for user".format(user_id)) if not len(orgList): raise exceptions.APIException( detail="This user has no organisations yet".format(user_id)) primary_org_id = orgList[0]['id'] try: primary_org = client.call_action( 'organization_show', {"id": primary_org_id}) except BaseException: raise exceptions.APIException( detail="Can't call organization_show for organization with id\ {}".format(primary_org_id)) return Response(status=401) # print('got primary_org') # print(primary_org) if not primary_org: raise exceptions.APIException( detail="Can't call organization_show for organization with id\ {}".format(primary_org_id)) primary_org_iati_id = primary_org.get('publisher_iati_id') publisher_org = get_or_none(Organisation, organisation_identifier=primary_org_iati_id ) if not publisher_org: raise exceptions.APIException( detail="publisher_iati_id of {} not found in Organisation\ standard, correct this in the IATI registry".format( primary_org_iati_id ) ) # TODO: add organisation foreign key - 2016-10-25 publisher = Publisher.objects.update_or_create( iati_id=primary_org_id, publisher_iati_id=primary_org_iati_id, defaults={ "name": primary_org.get('name'), "display_name": primary_org.get('display_name'), "organisation": publisher_org, } ) organisation_group = OrganisationGroup.objects.get_or_create( publisher=publisher[0], defaults={ "name": "{} Organisation Group".format(primary_org.get('name')) } ) organisation_group[0].organisationuser_set.add(user) if publisher[1]: # has been created organisation_admin_group = OrganisationAdminGroup.objects.\ get_or_create( publisher=publisher[0], defaults={ "owner": user, "name": "{} Organisation Admin Group".format( primary_org.get('name') ), } ) else: # already exists organisation_admin_group = OrganisationAdminGroup.objects.\ get_or_create( publisher=publisher[0], defaults={ "name": "{} Organisation Admin Group".format( primary_org.get('name') ), } ) organisation_admin_group[0].organisationuser_set.add(user) user.iati_api_key = api_key user.iati_user_id = user_id user.save() serializer = OrganisationUserSerializer( user, context={ 'request': request, } ) return Response(serializer.data)