def _send(self, method, response_type, **kwargs):
url = purl.URL(self.base_url)
# debug(resource_ids)
debug("entered send", self.uriparts, self.baseparts)
for p in self.uriparts:
url = url.add_path_segment(p)
request = requests.Request(method, url, **kwargs)
if self.session is None:
self.session = requests.Session()
prepped = self.auth_callable(self.session.prepare_request(request))
debug(prepped, prepped.url)
resp = self.session.send(prepped)
self.sent = True
# remove post endpoint for subsequent requests
if method == 'post':
post_endpoint = self.uriparts.pop()
if resp.status_code == 404:
raise ResourceNotFound(resp.content)
if response_type == ResponseType.JSON:
return resp.json()
elif response_type == ResponseType.RAW:
return resp
else:
return generate_resources(resp, self.client)
def test_with_purl(self):
self.adapter.register_uri('GET',
purl.URL('mock://www.tester.com/a'),
text='resp')
resp = self.session.get('mock://www.tester.com/a')
self.assertEqual('resp', resp.text)
def filter__is_current_nav(nav, main_nav=None):
path = purl.URL(request.url).path()
if 'full_matcher' in nav and nav['full_matcher']:
return True if re.search(nav['full_matcher'], path) else False
elif 'matcher' in nav and nav['matcher']:
matcher = nav['matcher']
if main_nav and 'matcher' in main_nav and main_nav['matcher']:
matcher = main_nav['matcher'] + matcher
return True if re.search(matcher, path) else False
return False
def _send(self, method, response_type, **kwargs):
if response_type == ResponseType.PAGINATED:
return PaginatedList(self, method, response_type, **kwargs)
url = purl.URL(self.base_url)
# debug(resource_ids)
debug("entered send", self.uriparts, self.baseparts)
for p in self.uriparts:
url = url.add_path_segment(p)
for k, v in self.queryparts.items():
url = url.query_param(k, v)
debug(url.as_string())
request = requests.Request(method, url, **kwargs)
if self.session is None:
self.session = requests.Session()
prepped = self.auth_callable(self.session.prepare_request(request))
debug(prepped, prepped.url)
resp = self.session.send(prepped)
self.sent = True
# remove post endpoint for subsequent requests
if method == 'post':
post_endpoint = self.uriparts.pop()
if resp.status_code == 404:
raise ResourceNotFound(resp.content)
# TODO handle errors
# {'code': 'unauthorized_operation', 'kind': 'error', 'error': 'Authorization failure.', 'general_problem': "You aren't authorized to access the requested resource.", 'possible_fix': "Your project permissions are determined on the Project Membership page. If you are receiving this error you may be trying to access the wrong project, or the project API access is disabled, or someone listed as the project's Owner needs to change your membership type."}
if not 200 <= resp.status_code <= 300:
debug(resp.status_code)
raise PivotalError(resp.content)
if response_type == ResponseType.JSON:
return resp.json()
elif response_type == ResponseType.RAW:
return resp
elif response_type == ResponseType.RESOURCE:
return generate_resources(resp, self.client)
else:
raise Exception('unknown response type')
def domain(self):
return purl.URL(self.url).domain()
def load(args):
glottolog = args.repos
fts.index('fts_index', models.Ref.fts, DBSession.bind)
DBSession.execute("CREATE EXTENSION IF NOT EXISTS unaccent WITH SCHEMA public;")
version = assert_release(glottolog.repos)
dataset = common.Dataset(
id='glottolog',
name="{0} {1}".format(glottolog.publication.web.name, version),
publisher_name=glottolog.publication.publisher.name,
publisher_place=glottolog.publication.publisher.place,
publisher_url=glottolog.publication.publisher.url,
license=glottolog.publication.license.url,
domain=purl.URL(glottolog.publication.web.url).domain(),
contact=glottolog.publication.web.contact,
jsondata={'license_icon': 'cc-by.png', 'license_name': glottolog.publication.license.name},
)
data = Data()
for e in glottolog.editors.values():
if e.current:
ed = data.add(common.Contributor, e.id, id=e.id, name=e.name)
common.Editor(dataset=dataset, contributor=ed, ord=int(e.ord))
DBSession.add(dataset)
contrib = data.add(common.Contribution, 'glottolog', id='glottolog', name='Glottolog')
DBSession.add(common.ContributionContributor(
contribution=contrib, contributor=data['Contributor']['hammarstroem']))
#
# Add Parameters:
#
add = functools.partial(add_parameter, data)
add('fc', name='Family classification')
add('sc', name='Subclassification')
add('aes',
args.repos.aes_status.values(),
name=args.repos.aes_status.__defaults__['name'],
pkw=dict(
jsondata=dict(
reference_id=args.repos.aes_status.__defaults__['reference_id'],
sources=[attr.asdict(v) for v in args.repos.aes_sources.values()],
scale=[attr.asdict(v) for v in args.repos.aes_status.values()])),
dekw=lambda de: dict(name=de.name, number=de.ordinal, jsondata=dict(icon=de.icon)),
)
add('med',
args.repos.med_types.values(),
name='Most Extensive Description',
dekw=lambda de: dict(
name=de.name, description=de.description, number=de.rank, jsondata=dict(icon=de.icon)),
)
add('macroarea',
args.repos.macroareas.values(),
pkw=dict(
description=args.repos.macroareas.__defaults__['description'],
jsondata=dict(reference_id=args.repos.macroareas.__defaults__['reference_id'])),
dekw=lambda de: dict(
name=de.name,
description=de.description,
jsondata=dict(geojson=read_macroarea_geojson(args.repos, de.name, de.description)),
),
)
add('ltype',
args.repos.language_types.values(),
name='Language Type',
dekw=lambda de: dict(name=de.category, description=de.description),
delookup='category',
)
add('country',
args.repos.countries,
dekw=lambda de: dict(name=de.id, description=de.name),
)
legacy = jsonlib.load(gc2version(args))
for gc, version in legacy.items():
data.add(models.LegacyCode, gc, id=gc, version=version)
#
# Now load languoid data, keeping track of relations that can only be inserted later.
#
lgsources = defaultdict(list)
# Note: We rely on languoids() yielding languoids in the "right" order, i.e. such that top-level
# nodes will precede nested nodes. This order must be preserved using an `OrderedDict`:
nodemap = OrderedDict([(l.id, l) for l in glottolog.languoids()])
lgcodes = {k: v.id for k, v in args.repos.languoids_by_code(nodemap).items()}
for lang in nodemap.values():
for ref in lang.sources:
lgsources['{0.provider}#{0.bibkey}'.format(ref)].append(lang.id)
load_languoid(glottolog, data, lang, nodemap)
for gc in glottolog.glottocodes:
if gc not in data['Languoid'] and gc not in legacy:
common.Config.add_replacement(gc, None, model=common.Language)
for obj in jsonlib.load(glottolog.references_path('replacements.json')):
common.Config.add_replacement(
'{0}'.format(obj['id']),
'{0}'.format(obj['replacement']) if obj['replacement'] else None,
model=common.Source)
DBSession.flush()
for doctype in glottolog.hhtypes:
data.add(
models.Doctype, doctype.id, id=doctype.id,
name=doctype.name,
description=doctype.description,
abbr=doctype.abbv,
ord=doctype.rank)
for bib in glottolog.bibfiles:
data.add(
models.Provider,
bib.id,
id=bib.id,
name=bib.title,
description=bib.description,
abbr=bib.abbr,
url=bib.url)
DBSession.flush()
s = time()
for i, entry in enumerate(
BibFile(glottolog.build_path('monster-utf8.bib'), api=glottolog).iterentries()):
if i % 10000 == 0:
args.log.info('{0}: {1:.3}'.format(i, time() - s))
s = time()
ref = load_ref(data, entry, lgcodes, lgsources)
if 'macro_area' in entry.fields:
mas = []
for ma in split_text(entry.fields['macro_area'], separators=',;', strip=True):
ma = 'North America' if ma == 'Middle America' else ma
ma = glottolog.macroareas.get('Papunesia' if ma == 'Papua' else ma)
mas.append(ma.name)
ref.macroareas = ', '.join(mas)
def using_server(context, server):
context.serverUrl = purl.URL(server)
