def custom_result_renderer(res, **kwargs):
if res['status'] != 'ok' or \
not res.get('action', None) == 'meta_extract':
# logging complained about this already
return
if 'state' in res and 'extractor' in res:
# extractor report, special treatment
ui.message('{name}({state})'.format(
name=ac.color_word(res['extractor'], ac.BOLD),
state=','.join(
'{}{}{}{}'.format(
# boolean states get a + or - prefix
'+' if v is True else '-' if v is False else '',
k,
'=' if not isinstance(v, bool) else '',
v if not isinstance(v, bool) else '')
for k, v in iteritems(res['state'])
# this is an extractor property, and mostly serves
# internal purposes
if k not in ('unique_exclude', )),
))
return
if kwargs.get('format', None) == 'jsonld':
# special case of a JSON-LD report request
# all reports are consolidated into a single
# graph, dumps just that (no pretty printing, can
# be done outside)
ui.message(
jsondumps(
res['metadata'],
# support utf-8 output
ensure_ascii=False,
# this cannot happen, spare the checks
check_circular=False,
# this will cause the output to not necessarily be
# JSON compliant, but at least contain all info that went
# in, and be usable for javascript consumers
allow_nan=True,
))
return
# list the path, available metadata keys, and tags
path = op.relpath(res['path'], res['refds']) if res.get(
'refds', None) else res['path']
meta = res.get('metadata', {})
ui.message('{path}{type}:{spacer}{meta}{tags}'.format(
path=ac.color_word(path, ac.BOLD),
type=' ({})'.format(ac.color_word(res['type'], ac.MAGENTA))
if 'type' in res else '',
spacer=' ' if len([m for m in meta if m != 'tag']) else '',
meta=','.join(k for k in sorted(meta.keys())
if k not in ('tag', '@context', '@id'))
if meta else ' -' if 'metadata' in res else ' {}'.format(','.join(
e for e in res['extractors']
if e not in ('datalad_core', 'metalad_core',
'metalad_annex'))) if 'extractors' in res else '',
tags='' if 'tag' not in meta else ' [{}]'.format(','.join(
assure_list(meta['tag'])))))
def write_json(self, value, seconds):
if not isinstance(value, (basestring, )):
value = jsondumps(value, indent=4)
self.response.headers['Content-Type'] = 'application/x-javascript'
self.response.headers['Expires'] = (datetime.now() + timedelta(hours=1)).ctime()
self.response.headers['Cache-Control'] = 'max-age=' + str(seconds)
cb = parseqs(self.request.query_string).get('callback', (None, ))[0]
if cb:
value = '%s(%s)' % (cb, value)
self.response.out.write(value)
def __call__(self):
catalog = getToolByName(self.context, 'portal_catalog')
members = catalog.uniqueValuesFor('getMembers')
results = []
for member in members:
if member:
for mem in member:
if mem not in results:
results.append(mem)
return jsondumps(results)
def __call__(self):
self.request.response.setHeader('Content-type', 'application/json')
portal_catalog = getToolByName(self.context, 'portal_catalog')
portal_membership = getToolByName(self.context, 'portal_membership')
#FIXME We probably want to return names as in Full Name <email>
#but have to be careful that it's email header safe for
#unicode values
# Grab email addresses from Person content type from
# collective.contacts
member_address = []
for member in portal_membership.listMembers():
member_email = member.getProperty('email')
member_fullname = member.getProperty('fullname')
member_complete = '%s <%s>' %(member_fullname,member_email)
member_address.append(member_complete)
persons = [ i.getObject() for i in \
portal_catalog.searchResults(portal_type='Person') ]
person_address = []
for person in persons:
if person.work_email != '' :
person_email = person.work_email
person_name = '%s %s' %(person.getFirstName(),person.getLastName())
person_complete = '%s <%s>' %(person_name, person_email)
person_address.append(person_complete)
# Grab emails from portal members
groups = ['Group:%s' % g.id for g in api.group.get_groups()]
return jsondumps(person_address + member_address + groups)
def create_epic_handle(location, checksum=None):
"""Create a new handle for a file.
Parameters:
location: The location (URL) of the file.
checksum: Optional parameter, store the checksum of the file as well.
Returns:
the URI of the new handle, raises a 503 exception if an error occurred.
"""
# httplib2.debuglevel = 4
# Ensure all these are strings
username = str(current_app.config.get('CFG_EPIC_USERNAME'))
password = str(current_app.config.get('CFG_EPIC_PASSWORD'))
baseurl = str(current_app.config.get('CFG_EPIC_BASEURL'))
prefix = str(current_app.config.get('CFG_EPIC_PREFIX'))
# If the proxy and proxy ports are set in the invenio-local.conf file
# read them and set the proxy. If not, do nothing.
try:
proxy = current_app.config.get('CFG_SITE_PROXY')
proxy_port = current_app.config.get('CFG_SITE_PROXYPORT')
except:
proxy = None
proxy_port = 80
if proxy is not None:
import socks
http = httplib2.Http(proxy_info=httplib2.ProxyInfo(
socks.PROXY_TYPE_HTTP, proxy, proxy_port))
else:
http = httplib2.Http()
http.add_credentials(username, password)
if not (prefix.endswith('/')):
prefix += '/'
if baseurl.endswith('/'):
uri = baseurl + prefix
else:
uri = baseurl + '/' + prefix
# for a PUT, add 'If-None-Match': '*' to the header
hdrs = {'Content-Type': 'application/json', 'Accept': 'application/json'}
if checksum:
new_handle_json = jsondumps([{
'type': 'URL',
'parsed_data': location
}, {
'type': 'CHECKSUM',
'parsed_data': checksum
}])
else:
new_handle_json = jsondumps([{'type': 'URL', 'parsed_data': location}])
current_app.logger.debug("EPIC PID json: " + new_handle_json)
if current_app.config.get('TESTING', False) or current_app.config.get(
'FAKE_EPIC_PID', False):
# special case for unit/functional testing: it's useful to get a PID,
# which otherwise will not get allocated due to missing credentials;
# this also speeds up testing just a bit, by avoiding HTTP requests
uuid = location.split('/')[-1] # record id
fake_pid_url = 'http://example.com/epic/handle/0000/{}'.format(uuid)
class FakeResponse(dict):
status = 201
response = FakeResponse(location=fake_pid_url)
else:
try:
response, content = http.request(uri,
method='POST',
headers=hdrs,
body=new_handle_json)
except Exception as e:
raise EpicPIDError("EPIC PID Exception") from e
current_app.logger.debug("EPIC PID Request completed")
if response.status != 201:
msg = "EPIC PID Not Created: Response status: {}".format(
response.status)
current_app.logger.debug(msg)
raise EpicPIDError(msg)
# get the handle as returned by EPIC
hdl = response['location']
pid = '/'.join(urlparse(hdl).path.split('/')[3:])
CFG_HANDLE_SYSTEM_BASEURL = current_app.config.get(
'CFG_HANDLE_SYSTEM_BASEURL')
return urljoin(CFG_HANDLE_SYSTEM_BASEURL, pid)
sample_fmeta2 = {
# same as above
"something": "stupid",
# different complex type
"complextype": {
"entity": {
"some": "few",
"properties": "here",
},
}
}
custom_metadata_tree = {
'.metadata': {
'content': {
'sub': {
'one.json': jsondumps(sample_fmeta1),
'two.json': jsondumps(sample_fmeta2),
'three.json': jsondumps(sample_fmeta1),
},
},
},
'sub': {
'one': '1',
'two': '2',
'three': '3',
},
}
@with_tree(custom_metadata_tree)
def test_unique_values(path):
def createHandle(location,checksum=None,suffix=''):
""" Create a new handle for a file.
Parameters:
location: The location (URL) of the file.
checksum: Optional parameter, store the checksum of the file as well.
suffix: The suffix of the handle. Default: ''.
Returns the URI of the new handle, raises a 503 exception if an error occurred.
"""
httplib2.debuglevel = 4
# Ensure all these are strings
username = str(CFG_EPIC_USERNAME)
password = str(CFG_EPIC_PASSWORD)
baseurl = str(CFG_EPIC_BASEURL)
prefix = str(CFG_EPIC_PREFIX)
# If the proxy and proxy ports are set in the invenio-local.conf file
# read them and set the proxy. If not, do nothing.
try:
from invenio.config import CFG_SITE_PROXY as proxy
from invenio.config import CFG_SITE_PROXYPORT as proxyPort
except ImportError:
proxy = None
proxyPort = 80
if proxy is not None:
import socks
http = httplib2.Http(proxy_info = httplib2.ProxyInfo(socks.PROXY_TYPE_HTTP,
proxy, proxyPort))
http.disable_ssl_certificate_validation = True
else:
http = httplib2.Http()
http.disable_ssl_certificate_validation = True
http.add_credentials(username, password)
if not (prefix.endswith('/')):
prefix += '/'
if baseurl.endswith('/'):
uri = baseurl + prefix
else:
uri = baseurl + '/' + prefix
if suffix != '':
uri += "/" + suffix.lstrip(prefix+"/")
# for a PUT, add 'If-None-Match': '*' to the header
hdrs = {'Content-Type':'application/json', 'Accept': 'application/json'}
if checksum:
new_handle_json = jsondumps([{'type': 'URL',
'parsed_data': location},
{'type':'CHECKSUM','parsed_data': checksum}])
else:
new_handle_json = jsondumps([{'type':'URL',
'parsed_data': location}])
current_app.logger.debug("json: " + new_handle_json)
try:
response, content = http.request(uri, method='POST',
headers=hdrs, body=new_handle_json)
except Exception as e:
current_app.logger.debug(e)
abort(503)
else:
current_app.logger.debug("Request completed")
if response.status != 201:
current_app.logger.debug(
"Not Created: Response status: %s" % response.status)
abort(response.status)
# make sure to only return the handle and strip off the baseuri
# if it is included
hdl = response['location']
if hdl.startswith(uri):
hdl = hdl[len(uri):len(hdl)]
elif hdl.startswith(uri + '/'):
hdl = hdl[len(uri + '/'):len(hdl)]
return hdl
def create_epic_handle(location, checksum=None):
"""Create a new handle for a file.
Parameters:
location: The location (URL) of the file.
checksum: Optional parameter, store the checksum of the file as well.
Returns:
the URI of the new handle, raises a 503 exception if an error occurred.
"""
# httplib2.debuglevel = 4
# Ensure all these are strings
username = str(current_app.config.get('CFG_EPIC_USERNAME'))
password = str(current_app.config.get('CFG_EPIC_PASSWORD'))
baseurl = str(current_app.config.get('CFG_EPIC_BASEURL'))
prefix = str(current_app.config.get('CFG_EPIC_PREFIX'))
# If the proxy and proxy ports are set in the invenio-local.conf file
# read them and set the proxy. If not, do nothing.
try:
proxy = current_app.config.get('CFG_SITE_PROXY')
proxy_port = current_app.config.get('CFG_SITE_PROXYPORT')
except:
proxy = None
proxy_port = 80
if proxy is not None:
import socks
http = httplib2.Http(
proxy_info=httplib2.ProxyInfo(socks.PROXY_TYPE_HTTP,
proxy, proxy_port))
else:
http = httplib2.Http()
http.add_credentials(username, password)
if not (prefix.endswith('/')):
prefix += '/'
if baseurl.endswith('/'):
uri = baseurl + prefix
else:
uri = baseurl + '/' + prefix
# for a PUT, add 'If-None-Match': '*' to the header
hdrs = {'Content-Type': 'application/json', 'Accept': 'application/json'}
if checksum:
new_handle_json = jsondumps([{'type': 'URL',
'parsed_data': location},
{'type': 'CHECKSUM',
'parsed_data': checksum}])
else:
new_handle_json = jsondumps([{'type': 'URL',
'parsed_data': location}])
current_app.logger.debug("EPIC PID json: " + new_handle_json)
if current_app.config.get('TESTING', False) or current_app.config.get('FAKE_EPIC_PID', False):
# special case for unit/functional testing: it's useful to get a PID,
# which otherwise will not get allocated due to missing credentials;
# this also speeds up testing just a bit, by avoiding HTTP requests
uuid = location.split('/')[-1] # record id
fake_pid_url = 'http://example.com/epic/handle/0000/{}'.format(uuid)
class FakeResponse(dict):
status=201
response = FakeResponse(location=fake_pid_url)
else:
try:
response, content = http.request(
uri, method='POST', headers=hdrs, body=new_handle_json)
except Exception as e:
raise EpicPIDError("EPIC PID Exception") from e
current_app.logger.debug("EPIC PID Request completed")
if response.status != 201:
msg = "EPIC PID Not Created: Response status: {}".format(response.status)
current_app.logger.debug(msg)
raise EpicPIDError(msg)
# get the handle as returned by EPIC
hdl = response['location']
pid = '/'.join(urlparse(hdl).path.split('/')[3:])
CFG_HANDLE_SYSTEM_BASEURL = current_app.config.get(
'CFG_HANDLE_SYSTEM_BASEURL')
return urljoin(CFG_HANDLE_SYSTEM_BASEURL, pid)
def createHandle(location, checksum=None, suffix=''):
""" Create a new handle for a file.
Parameters:
location: The location (URL) of the file.
checksum: Optional parameter, store the checksum of the file as well.
suffix: The suffix of the handle. Default: ''.
Returns:
the URI of the new handle, raises a 503 exception if an error occurred.
"""
# httplib2.debuglevel = 4
# Ensure all these are strings
username = str(current_app.config.get('CFG_EPIC_USERNAME'))
password = str(current_app.config.get('CFG_EPIC_PASSWORD'))
baseurl = str(current_app.config.get('CFG_EPIC_BASEURL'))
prefix = str(current_app.config.get('CFG_EPIC_PREFIX'))
# If the proxy and proxy ports are set in the invenio-local.conf file
# read them and set the proxy. If not, do nothing.
try:
proxy = current_app.config.get('CFG_SITE_PROXY')
proxy_port = current_app.config.get('CFG_SITE_PROXYPORT')
except:
proxy = None
proxy_port = 80
if proxy is not None:
import socks
http = httplib2.Http(
proxy_info=httplib2.ProxyInfo(socks.PROXY_TYPE_HTTP,
proxy, proxy_port))
else:
http = httplib2.Http()
http.add_credentials(username, password)
if not (prefix.endswith('/')):
prefix += '/'
if baseurl.endswith('/'):
uri = baseurl + prefix
else:
uri = baseurl + '/' + prefix
if suffix != '':
uri += "/" + suffix.lstrip(prefix + "/")
# for a PUT, add 'If-None-Match': '*' to the header
hdrs = {'Content-Type': 'application/json', 'Accept': 'application/json'}
if checksum:
new_handle_json = jsondumps([{'type': 'URL',
'parsed_data': location},
{'type': 'CHECKSUM',
'parsed_data': checksum}])
else:
new_handle_json = jsondumps([{'type': 'URL',
'parsed_data': location}])
current_app.logger.debug("EPIC PID json: " + new_handle_json)
try:
response, content = http.request(
uri, method='POST', headers=hdrs, body=new_handle_json)
except Exception as e:
current_app.logger.debug(e)
raise EpicPIDError("EPIC PID Exception") from e
current_app.logger.debug("EPIC PID Request completed")
if response.status != 201:
msg = "EPIC PID Not Created: Response status: {}".format(response.status)
current_app.logger.debug(msg)
raise EpicPIDError(msg)
# get the handle as returned by EPIC
hdl = response['location']
pid = '/'.join(urlparse(hdl).path.split('/')[3:])
CFG_HANDLE_SYSTEM_BASEURL = current_app.config.get(
'CFG_HANDLE_SYSTEM_BASEURL')
return urljoin(CFG_HANDLE_SYSTEM_BASEURL, pid)
"@type": "Place",
"geo": {
"@type": "GeoShape",
"box": "18.0 -65.0 72.0 172.0"
}
}
}
testmeta = {"@id": "magic", "name": "silence"}
@known_failure
@with_tree(
tree={
'.metadata': {
'dataset.json': jsondumps(sample_jsonld)
},
'down': {
'customloc': jsondumps(testmeta)
}
})
def test_custom_dsmeta(path):
ds = Dataset(path).create(force=True)
sample_jsonld_ = dict(sample_jsonld)
sample_jsonld_.update({'@id': ds.id})
# enable custom extractor
# use default location
ds.config.add('datalad.metadata.nativetype',
'metalad_custom',
where='dataset')
ds.save()
}
sample_fmeta = {
'@id': 'datalad:SHA1-s1--356a192b7913b04c54574d18c28d46e6395428ab',
"something": "stupid",
"complextype": {
"entity": {
"some": "many",
"properties": "here",
},
"age": "young",
"numbers": [3, 2, 1, 0],
}
}
meta_tree = {
'.metadata': {
'dataset.json': jsondumps(sample_dsmeta),
'content': {
'sub': {
'one.json': jsondumps(sample_fmeta),
'nothing.json': '{}',
},
},
},
'sub': {
'one': '1',
'nothing': '2',
},
}
@with_tempfile(mkdir=True)
storage = History.all().filter('url =', url).get()
if storage:
## this assumes that the schema has already been
## parsed and fixed at least one time.
schema = jsonloads(storage.payload)
self.cache_set(schema, lang)
else:
## 1b. total failure
schema = {}
else:
## 2. store in cache and history table; the size should
## be okay because the schema was parsed successfully.
storage = History.all().filter('url =', url).get()
if storage is None:
storage = History(url=url)
storage.payload = jsondumps(schema, indent=4)
storage.put()
self.cache_set(schema, lang)
return schema
class ItemsApp(ProxyApp):
##
## Proxy for the items of a given Steam ID. Does not support language codes.
##
cache_time = 60 * 5
items_url_fs = ('http://api.steampowered.com/ITFItems_440/GetPlayerItems/v0001/'
'?key=%s&SteamID=%s')
def get(self, id64):
self.write_json(self.get_items(id64), seconds=self.cache_time)
def createHandle(location, checksum=None, suffix=''):
""" Create a new handle for a file.
Parameters:
location: The location (URL) of the file.
checksum: Optional parameter, store the checksum of the file as well.
suffix: The suffix of the handle. Default: ''.
Returns the URI of the new handle, raises a 503 exception if an error occurred.
"""
httplib2.debuglevel = 4
# Ensure all these are strings
username = str(current_app.config.get('CFG_EPIC_USERNAME'))
password = str(current_app.config.get('CFG_EPIC_PASSWORD'))
baseurl = str(current_app.config.get('CFG_EPIC_BASEURL'))
prefix = str(current_app.config.get('CFG_EPIC_PREFIX'))
# If the proxy and proxy ports are set in the invenio-local.conf file
# read them and set the proxy. If not, do nothing.
try:
proxy = current_app.config.get('CFG_SITE_PROXY')
proxy_port = current_app.config.get('CFG_SITE_PROXYPORT')
except:
proxy = None
proxy_port = 80
if proxy is not None:
import socks
http = httplib2.Http(
proxy_info=httplib2.ProxyInfo(socks.PROXY_TYPE_HTTP,
proxy, proxy_port))
else:
http = httplib2.Http()
http.add_credentials(username, password)
if not (prefix.endswith('/')):
prefix += '/'
if baseurl.endswith('/'):
uri = baseurl + prefix
else:
uri = baseurl + '/' + prefix
if suffix != '':
uri += "/" + suffix.lstrip(prefix + "/")
# for a PUT, add 'If-None-Match': '*' to the header
hdrs = {'Content-Type': 'application/json', 'Accept': 'application/json'}
if checksum:
new_handle_json = jsondumps([{'type': 'URL',
'parsed_data': location},
{'type': 'CHECKSUM',
'parsed_data': checksum}])
else:
new_handle_json = jsondumps([{'type': 'URL',
'parsed_data': location}])
current_app.logger.debug("json: " + new_handle_json)
try:
response, content = http.request(uri, method='POST',
headers=hdrs, body=new_handle_json)
except Exception as e:
current_app.logger.debug(e)
abort(503)
else:
current_app.logger.debug("Request completed")
if response.status != 201:
current_app.logger.debug(
"Not Created: Response status: %s" % response.status)
abort(response.status)
# get the handle as returned by EPIC
hdl = response['location']
pid = '/'.join(urlparse(hdl).path.split('/')[3:])
CFG_HANDLE_SYSTEM_BASEURL = current_app.config.get(
'CFG_HANDLE_SYSTEM_BASEURL')
return urljoin(CFG_HANDLE_SYSTEM_BASEURL, pid)