class FunderAgent(Parser):
schema = tools.GuessAgentType(tools.OneOf(ctx.funderName,
ctx.contributorName),
default='organization')
name = tools.OneOf(ctx.funderName, ctx.contributorName)
identifiers = tools.Map(
tools.Delegate(AgentIdentifier),
tools.Try(tools.IRI(
tools.OneOf(ctx.funderIdentifier,
tools.RunPython(force_text, ctx.nameIdentifier),
tools.Static(None))),
exceptions=(ValueError, )))
class Extra:
name_identifier = tools.Try(ctx.nameIdentifier)
name_identifier_scheme = tools.Try(
ctx.nameIdentifier['@nameIdentifierScheme'])
name_identifier_scheme_uri = tools.Try(
ctx.nameIdentifier['@schemeURI'])
funder_identifier = tools.Try(ctx.funderIdentifier)
funder_identifier_type = tools.Try(ctx.funderIdentifierType)
contributor_type = tools.Try(ctx.contributorType)
class Person(Parser):
given_name = tools.OneOf(
ctx.embeds.users.data.attributes.given_name,
ctx.embeds.users.errors[0].meta.given_name,
)
family_name = tools.OneOf(
ctx.embeds.users.data.attributes.family_name,
ctx.embeds.users.errors[0].meta.family_name,
)
additional_name = tools.OneOf(
ctx.embeds.users.data.attributes.middle_names,
ctx.embeds.users.errors[0].meta.middle_names,
)
suffix = tools.OneOf(
ctx.embeds.users.data.attributes.suffix,
ctx.embeds.users.errors[0].meta.suffix,
)
identifiers = tools.Map(
tools.Delegate(AgentIdentifier),
tools.RunPython('registered', ctx.embeds.users.data.links.html),
tools.Try(ctx.embeds.users.data.links.profile_image),
)
class Extra:
locale = tools.Try(ctx.embeds.users.data.attributes.locale)
date_registered = tools.Try(
ctx.embeds.users.data.attributes.date_registered)
active = tools.Try(ctx.embeds.users.data.attributes.active)
timezone = tools.Try(ctx.embeds.users.data.attributes.timezone)
def registered(self, context):
if self.context['attributes']['unregistered_contributor']:
return None
return context
class Extra:
name_identifier = tools.Try(ctx.nameIdentifier)
name_identifier_scheme = tools.Try(
ctx.nameIdentifier['@nameIdentifierScheme'])
name_identifier_scheme_uri = tools.Try(
ctx.nameIdentifier['@schemeURI'])
contributor_type = tools.Try(ctx.contributorType)
# v.4 new givenName and familyName properties
given_name = tools.OneOf(ctx.creatorName['@givenName'],
ctx.contributorName['@givenName'],
tools.Static(None))
family_name = tools.OneOf(ctx.creatorName['@familyName'],
ctx.contributorName['@familyName'],
tools.Static(None))
class CreativeWork(Parser):
schema = tools.RunPython('get_type', ctx)
title = tools.RunPython('get_title', ctx)
description = Soup(ctx, 'p', class_='genericfile_description')['#text']
date_published = tools.ParseDate(
Soup(ctx, itemprop='datePublished')['#text'])
date_updated = tools.ParseDate(Soup(ctx, itemprop='dateModified')['#text'])
rights = tools.OneOf(tools.RunPython('get_rights_url', ctx),
tools.RunPython('get_dd', ctx, 'Rights')['#text'],
tools.Static(None))
language = tools.Try(
tools.ParseLanguage(Soup(ctx, itemprop='inLanguage')['#text']))
tags = tools.Map(tools.Delegate(ThroughTags), Soup(ctx,
itemprop='keywords'))
identifiers = tools.Map(
tools.Delegate(WorkIdentifier),
tools.Try(tools.RunPython('get_dd', ctx, 'Permanent Link')),
)
related_agents = tools.Concat(
tools.Map(tools.Delegate(Creator), Soup(ctx, itemprop='creator')),
tools.Map(tools.Delegate(Contributor), Soup(ctx,
itemprop='contributor')),
tools.Map(tools.Delegate(Publisher), Soup(ctx, itemprop='publisher')),
)
class Extra:
gwu_unit = tools.RunPython('get_dd', ctx, 'GW Unit')['#text']
related_url = tools.RunPython('get_dd', ctx, 'Related URL')['#text']
previous_publication_information = tools.RunPython(
'get_dd', ctx, 'Previous Publication Information')['#text']
depositor = tools.RunPython('get_dd', ctx, 'Depositor')['#text']
characterization = tools.RunPython('get_dd', ctx,
'Characterization')['#text']
def get_type(self, obj):
return {
'http://schema.org/CreativeWork': 'CreativeWork',
'http://schema.org/Article': 'Article',
'http://schema.org/Book': 'Book',
}.get(obj.soup.find('div')['itemtype'], 'CreativeWork')
def get_title(self, obj):
title = obj.h1.soup
title.find('span', class_='label').decompose()
return title.get_text()
def get_dd(self, obj, dt):
dt_tag = obj.soup.find('dt', string=dt)
if dt_tag:
return SoupXMLDict(soup=dt_tag.find_next_sibling('dd'))
return None
def get_rights_url(self, obj):
dd = self.get_dd(obj, 'Rights')
return dd.soup.find('i', class_='glyphicon-new-window').parent['href']
class ContributorAgent(Parser):
schema = tools.OneOf(
tools.GuessAgentType(tools.RunPython(get_agent_type, ctx,
person=False),
default='organization'),
tools.GuessAgentType(tools.OneOf(ctx.creatorName,
ctx.contributorName)))
name = tools.OneOf(ctx.creatorName, ctx.contributorName)
identifiers = tools.Map(
tools.Delegate(AgentIdentifier),
tools.Try(tools.IRI(tools.RunPython(force_text, ctx.nameIdentifier)),
exceptions=(ValueError, )))
related_agents = tools.Map(
tools.Delegate(IsAffiliatedWith),
tools.Concat(
tools.Try(
tools.Filter(lambda x: bool(x),
tools.RunPython(force_text, ctx.affiliation)))))
class Extra:
name_identifier = tools.Try(ctx.nameIdentifier)
name_identifier_scheme = tools.Try(
ctx.nameIdentifier['@nameIdentifierScheme'])
name_identifier_scheme_uri = tools.Try(
ctx.nameIdentifier['@schemeURI'])
contributor_type = tools.Try(ctx.contributorType)
# v.4 new givenName and familyName properties
given_name = tools.OneOf(ctx.creatorName['@givenName'],
ctx.contributorName['@givenName'],
tools.Static(None))
family_name = tools.OneOf(ctx.creatorName['@familyName'],
ctx.contributorName['@familyName'],
tools.Static(None))
class OAICreativeWork(Parser):
default_type = None
type_map = None
schema = tools.RunPython(
'get_schema',
tools.OneOf(ctx.record.metadata.dc['dc:type'], tools.Static(None)))
title = tools.Join(
tools.RunPython('force_text',
tools.Try(ctx.record.metadata.dc['dc:title'])))
description = tools.Join(
tools.RunPython('force_text',
tools.Try(ctx.record.metadata.dc['dc:description'])))
identifiers = tools.Map(
tools.Delegate(OAIWorkIdentifier),
tools.Unique(
tools.Map(
tools.Try(tools.IRI(), exceptions=(ValueError, )),
tools.Filter(
not_citation,
tools.RunPython(
'force_text',
tools.Concat(
tools.Try(ctx.record.metadata.dc['dc:identifier']),
tools.Try(ctx.record.header['identifier'])))))))
related_works = tools.Concat(
tools.Map(
tools.Delegate(OAIWorkRelation),
tools.Unique(
tools.Map(tools.Try(tools.IRI(), exceptions=(ValueError, )),
tools.RunPython('get_relation', ctx)))))
related_agents = tools.Concat(
tools.Map(tools.Delegate(OAICreator),
tools.Try(ctx.record.metadata.dc['dc:creator'])),
tools.Map(tools.Delegate(OAIContributor),
tools.Try(ctx.record.metadata.dc['dc:contributor'])),
tools.Map(
tools.Delegate(OAIPublisher),
tools.RunPython('force_text',
tools.Try(
ctx.record.metadata.dc['dc:publisher']))),
)
rights = tools.Join(tools.Try(ctx.record.metadata.dc['dc:rights']))
# Note: this is only taking the first language in the case of multiple languages
language = tools.ParseLanguage(
tools.Try(ctx.record.metadata.dc['dc:language'][0]), )
subjects = tools.Map(
tools.Delegate(OAIThroughSubjects),
tools.Subjects(
tools.Map(
tools.RunPython('tokenize'),
tools.RunPython(
'force_text',
tools.Concat(
tools.Try(ctx.record.header.setSpec),
tools.Try(ctx.record.metadata.dc['dc:type']),
tools.Try(ctx.record.metadata.dc['dc:format']),
tools.Try(ctx.record.metadata.dc['dc:subject']),
)))))
tags = tools.Map(
tools.Delegate(OAIThroughTags),
tools.Concat(tools.Map(
tools.RunPython('tokenize'),
tools.RunPython(
'force_text',
tools.Concat(
tools.Try(ctx.record.header.setSpec),
tools.Try(ctx.record.metadata.dc['dc:type']),
tools.Try(ctx.record.metadata.dc['dc:format']),
tools.Try(ctx.record.metadata.dc['dc:subject']),
))),
deep=True))
date_updated = tools.ParseDate(ctx.record.header.datestamp)
is_deleted = tools.RunPython('check_status',
tools.Try(ctx.record.header['@status']))
class Extra:
"""
Fields that are combined in the base parser are relisted as singular elements that match
their original entry to preserve raw data structure.
"""
# An agent responsible for making contributions to the resource.
contributor = tools.Try(ctx.record.metadata.dc['dc:contributor'])
# The spatial or temporal topic of the resource, the spatial applicability of the resource,
# or the jurisdiction under which the resource is relevant.
coverage = tools.Try(ctx.record.metadata.dc['dc:coverage'])
# An agent primarily responsible for making the resource.
creator = tools.Try(ctx.record.metadata.dc['dc:creator'])
# A point or period of time associated with an event in the lifecycle of the resource.
dates = tools.Try(ctx.record.metadata.dc['dc:date'])
# The file format, physical medium, or dimensions of the resource.
resource_format = tools.Try(ctx.record.metadata.dc['dc:format'])
# An unambiguous reference to the resource within a given context.
identifiers = tools.Concat(
tools.Try(ctx.record.metadata.dc['dc:identifier']),
tools.Try(ctx.record.header['identifier']))
# A related resource.
relation = tools.RunPython('get_relation', ctx)
# A related resource from which the described resource is derived.
source = tools.Try(ctx.record.metadata.dc['dc:source'])
# The nature or genre of the resource.
resource_type = tools.Try(ctx.record.metadata.dc['dc:type'])
set_spec = tools.Try(ctx.record.header.setSpec)
# Language also stored in the Extra class in case the language reported cannot be parsed by ParseLanguage
language = tools.Try(ctx.record.metadata.dc['dc:language'])
# Status in the header, will exist if the resource is deleted
status = tools.Try(ctx.record.header['@status'])
def check_status(self, status):
if status == 'deleted':
return True
return False
def get_schema(self, types):
if not types or not self.type_map:
return self.default_type
if isinstance(types, str):
types = [types]
for t in types:
if isinstance(t, dict):
t = t['#text']
t = t.lower()
if t in self.type_map:
return self.type_map[t]
return self.default_type
def force_text(self, data):
if isinstance(data, dict):
return data['#text']
if isinstance(data, str):
return data
fixed = []
for datum in (data or []):
if datum is None:
continue
if isinstance(datum, dict):
if '#text' not in datum:
logger.warn('Skipping %s, no #text key exists', datum)
continue
fixed.append(datum['#text'])
elif isinstance(datum, str):
fixed.append(datum)
else:
raise Exception(datum)
return fixed
def tokenize(self, data):
if isinstance(data, str):
data = [data]
tokens = []
for item in data:
tokens.extend(
[x.strip() for x in re.split(r'(?: - )|\.|,', item) if x])
return tokens
def get_relation(self, ctx):
if not ctx['record'].get('metadata'):
return []
relation = ctx['record']['metadata']['dc'].get('dc:relation') or []
identifiers = ctx['record']['metadata']['dc'].get(
'dc:identifier') or []
if isinstance(identifiers, dict):
identifiers = (identifiers, )
identifiers = ''.join(i['#text'] if isinstance(i, dict) else i
for i in identifiers if i)
identifiers = re.sub(
'http|:|/', '',
identifiers + ctx['record']['header']['identifier'])
if isinstance(relation, dict):
relation = (relation['#text'], )
return [
r for r in relation
if r and re.sub('http|:|/', '', r) not in identifiers
]
class ContributorRelation(Parser):
schema = 'Contributor'
agent = tools.Delegate(ContributorAgent, ctx)
cited_as = tools.OneOf(ctx.creatorName, ctx.contributorName)
class Contributor(Parser):
agent = tools.Delegate(Person, ctx)
cited_as = tools.OneOf(
ctx.embeds.users.data.attributes.full_name,
ctx.embeds.users.errors[0].meta.full_name,
)