def populate_harvest_job(self, harvest_job, set_ids, config, client): # Check if this source has been harvested before previous_job = Session.query(HarvestJob) \ .filter(HarvestJob.source == harvest_job.source) \ .filter(HarvestJob.gather_finished != None) \ .filter(HarvestJob.id != harvest_job.id) \ .order_by(HarvestJob.gather_finished.desc()) \ .limit(1).first() last_time = None if previous_job and previous_job.finished and model.Package.get(harvest_job.source.id).metadata_modified < previous_job.gather_started: last_time = previous_job.gather_started.isoformat() # Collect package ids package_ids = list(self.get_package_ids(set_ids, config, last_time, client)) log.debug('Identifiers: %s', package_ids) if not self._recreate(harvest_job) and package_ids: converted_identifiers = {} for identifier in package_ids: converted_identifiers[datapid_to_name(identifier)] = identifier if identifier.endswith(u'm'): converted_identifiers[datapid_to_name(u"%ss" % identifier[0:-1])] = identifier for package in model.Session.query(model.Package).filter(model.Package.name.in_(converted_identifiers.keys())).all(): converted_name = package.name if converted_identifiers[converted_name] not in package_ids: converted_name = "%sm" % converted_name[0:-1] package_ids.remove(converted_identifiers[converted_name]) if previous_job: for previous_error in [error.guid for error in Session.query(HarvestObject). filter(HarvestObject.harvest_job_id == previous_job.id). filter(HarvestObject.state == 'ERROR').all()]: if previous_error not in package_ids: package_ids.append(previous_error) try: object_ids = [] if len(package_ids): for package_id in islice(package_ids, config['limit']) if 'limit' in config else package_ids: # Create a new HarvestObject for this identifier obj = HarvestObject(guid=package_id, job=harvest_job) obj.save() object_ids.append(obj.id) log.debug('Object ids: {i}'.format(i=object_ids)) return object_ids else: self._save_gather_error('No packages received for URL: {u}'.format( u=harvest_job.source.url), harvest_job) return None except Exception as e: self._save_gather_error('Gather: {e}'.format(e=e), harvest_job) raise
def _to_name(cls, identifier): """ Convert identifier to CKAN package name. :param identifier: identifier string :return: CKAN package name """ return datapid_to_name(cls._to_identifier(identifier))
def default_name_from_id(key, data, errors, context): ''' If name not given, generate name from package.id :param key: key :param data: data :param errors: validation errors :param context: context ''' if not data.get(key): id = data.get(('id',)) data[key] = utils.datapid_to_name(id)
def test_datapid_to_name(self): name = utils.datapid_to_name('http://example.com/some/thing?good=true') assert name assert '/' not in name
def _ddi2ckan(self, original_url, original_xml, harvest_object): '''Extract package values from bs4 object 'ddi_xml' parsed from xml ''' # TODO: Use .extract() and .string.extract() function so handled elements are removed from ddi_xml. doc_citation = "ddi_xml.codeBook.docDscr.citation" stdy_dscr = "ddi_xml.codeBook.stdyDscr" #################################################################### # Read mandatory metadata fields: # #################################################################### # Authors & organizations authors = self.get_authors(self.ddi_xml.stdyDscr.citation, 'AuthEnty') agent = authors[:] agent.extend(self.get_contributors(self.ddi_xml.stdyDscr.citation)) # Availability availability = AVAILABILITY_DEFAULT if _access_request_URL_is_found(): availability = 'direct_download' if _is_fsd(original_url): availability = AVAILABILITY_FSD # Keywords keywords = self.get_keywords(self.ddi_xml.stdyDscr.stdyInfo.subject) # Language # TODO: Where/how to extract multiple languages: 'language': u'eng, fin, swe' ? language = self.convert_language( self._read_value("ddi_xml.codeBook.get('xml:lang')")) # Titles titles = self._read_value(stdy_dscr + ".citation.titlStmt(['titl', 'parTitl'])") or \ self._read_value(doc_citation + ".titlStmt(['titl', 'parTitl'])", mandatory_field=True) # langtitle=[dict(lang=self.convert_language(a.get('xml:lang', '')), value=a.text) for a in titles] # [{"lang":"fin", "value":"otsikko"}, {"lang:"en", "value":"title"}] # convert the titles to a JSON string of type {"fin":"otsikko", "eng","title"} transl_json = {} first_title = "" # default to finnish, since first title has no lang value, which causes the validator to whine # we might want to update the DDI harvester to accept a language configuration parameter, if # we decide to harvest DDI resources from other sources. default_lang = "fi" for title in titles: transl_json[self.convert_language(title.get('xml:lang', default_lang))] = title.text # we want to get save the first title for use lateron if not first_title: first_title = title.text title = json.dumps(transl_json) # License # TODO: Extract prettier output. Should we check that element contains something? # Should this be in optional section if not mandatory_field? license_url = self._read_value(stdy_dscr + ".dataAccs.useStmt.get_text(separator=u' ')", mandatory_field=False) if _is_fsd(original_url): license_id = LICENSE_ID_FSD else: license_id = LICENSE_ID_DEFAULT # Contact (package_extra.key: contact_[k]_name in database, contact in WUI) contact_name = self._read_value(stdy_dscr + ".citation.distStmt('contact')") or \ self._read_value(stdy_dscr + ".citation.distStmt('distrbtr')") or \ self._read_value(doc_citation + ".prodStmt('producer')", mandatory_field=True) # TODO: clean out (or ask FSD to clean) mid text newlines (eg. in FSD2482) if contact_name and contact_name[0].text: contact_name = contact_name[0].text else: contact_name = self._read_value(stdy_dscr + ".citation.prodStmt.producer.get('affiliation')", mandatory_field=True) if _is_fsd(original_url): contact_email = CONTACT_EMAIL_FSD # TODO: Allow trying other email also in FSD metadata else: contact_email = self._read_value(stdy_dscr + ".citation.distStmt.contact.get('email')", mandatory_field=True) # Modified date version = self.get_attr_optional(self.ddi_xml.stdyDscr.citation, 'prodDate', 'date') or \ self.get_attr_mandatory(self.ddi_xml.stdyDscr.citation, 'version', 'date') # Name name_prefix = self._read_value(stdy_dscr + ".citation.titlStmt.IDNo.get('agency')", mandatory_field=False) name_id = self._read_value(stdy_dscr + ".citation.titlStmt.IDNo.text", mandatory_field=False) if not name_prefix: name_prefix = self._read_value(doc_citation + ".titlStmt.IDNo['agency']", mandatory_field=True) if not name_id: name_id = self._read_value(doc_citation + ".titlStmt.IDNo.text", mandatory_field=True) name = utils.datapid_to_name(name_prefix + name_id) pids = list() pids.append({'id': name, 'type': 'data', 'primary': 'True', 'provider': name_prefix}) # Should we generate a version PID? # vpid = utils.generate_pid() # pids.append({'id': vpid, 'type': 'version', 'provider': 'kata'}) # Original web page as resource # For FSD 'URI' leads to summary web page of data, hence format='html' orig_web_page = self._read_value(doc_citation + ".holdings.get('URI', '')") if orig_web_page: orig_web_page_resource = {'description': first_title, 'format': u'html', 'resource_type': 'documentation', 'url': orig_web_page} else: orig_web_page_resource = {} # Owner owner = self._read_value(stdy_dscr + ".citation.prodStmt.producer.text") or \ self._read_value(stdy_dscr + ".citation.rspStmt.AuthEnty.text") or \ self._read_value(doc_citation + ".prodStmt.producer.string", mandatory_field=True) agent.append({'role': 'owner', 'name': owner}) # Owner organisation if harvest_object: hsid = harvest_object.harvest_source_id hsooid = model.Session.query(model.Package).filter(model.Package.id==hsid).one().owner_org owner_org = model.Session.query(model.Group).filter(model.Group.id==hsooid).one().name else: owner_org = u'' # Distributor (Agent: distributor, the same is used as contact) agent.append({ 'role': 'distributor', 'name': contact_name}) #################################################################### # Read optional metadata fields: # #################################################################### # Availability if _is_fsd(original_url): access_request_url = ACCESS_REQUEST_URL_FSD else: access_request_url = u'' # Contact contact_phone = self._read_value(doc_citation + ".holdings.get('callno')") or \ self._read_value(stdy_dscr + ".citation.holdings.get('callno')") contact_URL = self._read_value( stdy_dscr + ".dataAccs.setAvail.accsPlac.get('URI')") or \ self._read_value( stdy_dscr + ".citation.distStmt.contact.get('URI')") or \ self._read_value( stdy_dscr + ".citation.distStmt.distrbtr.get('URI')") or \ CONTACT_URL_FSD if _is_fsd(original_url) else None # convert the descriptions to a JSON string of type {"fin":"aineiston kuvaus", "eng","dataset description"} descriptions = self._read_value(stdy_dscr + ".stdyInfo.abstract('p')") if not descriptions: descriptions = self._read_value(stdy_dscr + ".citation.serStmt.serInfo('p')") translated_notes = {} for des in descriptions: lang = self.convert_language(des.get('xml:lang', 'fi')) if lang in translated_notes: translated_notes[lang] += '\r\n\r\n' + des.text else: translated_notes[lang] = des.text notes = json.dumps(translated_notes) # Discipline discipline = self.get_discipline(self.ddi_xml.stdyDscr.stdyInfo.subject) # Dataset lifetime events events = self._get_events(stdy_dscr, authors) # Geographic coverage geo_cover = self.get_geo_coverage(self.ddi_xml) # Temporal coverage temp_start, temp_end = self.get_temporal_coverage(self.ddi_xml) # Citation citation = self._read_value(stdy_dscr + ".citation.biblCit.text", mandatory_field=False) #################################################################### # Flatten rest to 'XPath/path/to/element': 'value' pairs # #################################################################### etree_xml = etree.fromstring(str(self.ddi_xml)) flattened_ddi = importcore.generic_xml_metadata_reader(etree_xml.find('.//{*}docDscr')) xpath_dict = flattened_ddi.getMap() flattened_ddi = importcore.generic_xml_metadata_reader(etree_xml.find('.//{*}stdyDscr')) xpath_dict.update(flattened_ddi.getMap()) package_dict = dict( access_application_URL=u'', access_request_URL=unicode(access_request_url), agent=agent, algorithm=u'', # To be implemented straight in 'resources' availability=unicode(availability), contact=[{'name': contact_name, 'email': contact_email, 'URL': contact_URL, 'phone': contact_phone}], direct_download_URL=u'', # To be implemented straight in 'resources discipline=discipline, event=events, geographic_coverage=geo_cover, groups=[], id=self._get_id_by_name(name) or generate_pid(), # langtitle=langtitle, langdis=u'True', # HUOMAA! language=language, license_URL=license_url, license_id=license_id, mimetype=u'', # To be implemented straight in 'resources name=name, notes=notes or u'', pids=pids, owner_org=owner_org, resources=[orig_web_page_resource], tag_string=keywords, temporal_coverage_begin=temp_start, temporal_coverage_end=temp_end, # title=langtitle[0].get('value'), # Must exist in package dict title=title, type='dataset', version=version, version_PID='', citation=citation ) package_dict['xpaths'] = xpath_dict # Above line creates: # package_dict = { # 'access_request_url': 'some_url', # # ... # 'xpaths': {'stdyDscr/othrStdyMat.0/relPubl.34': # 'Uskon asia: nuorisobarometri 2006 (2006).'}, # {'stdyD...': 'Some value'}] # } #package_dict['extras'].update(_save_ddi_variables_to_csv(ddi_xml, somepkg)) # Vanhojen koodien järjestys: #_save_original_xml_and_link_as_resources() #_save_ddi_variables_to_csv() #_create_group_based_on_organizations() #_last_statements_to_rewrite() # JuhoL: Set harvest object to some end state and commit if harvest_object is not None: harvest_object.content = None # Should this be flushed? model.Session.flush() #model.repo.commit() return package_dict