def _render_service(self, path, service, methods):
env = self.state.document.settings.env
service_id = "service-%d" % env.new_serialno('service')
service_node = nodes.section(ids=[service_id])
service_node += nodes.title(text='Service at %s' %
service.route_name)
if service.description is not None:
service_node += rst2node(_dedent(service.description))
for method, info in methods.items():
method_id = '%s-%s' % (service_id, method)
method_node = nodes.section(ids=[method_id])
method_node += nodes.title(text=method)
docstring = info['func'].__doc__ or ""
if 'validator' in info:
validators = to_list(info['validator'])
for validator in validators:
if validator.__doc__ is not None:
if docstring is not None:
docstring += '\n' + validator.__doc__.strip()
if 'accept' in info:
accept = info['accept']
if callable(accept):
if accept.__doc__ is not None:
docstring += accept.__doc__.strip()
else:
accept = to_list(accept)
accept_node = nodes.strong(text='Accepted content types:')
node_accept_list = nodes.bullet_list()
accept_node += node_accept_list
for item in accept:
temp = nodes.list_item()
temp += nodes.inline(text=item)
node_accept_list += temp
method_node += accept_node
node = rst2node(docstring)
if node is not None:
method_node += node
renderer = info['renderer']
if renderer == 'simplejson':
renderer = 'json'
response = nodes.paragraph()
response += nodes.strong(text='Response: %s' % renderer)
method_node += response
service_node += method_node
return service_node
def _result_list_compact(response):
items = []
pointer_results = JsonPointer('/ops:world-patent-data/ops:biblio-search/ops:search-result/exchange-documents')
pointer_application_reference = JsonPointer('/exchange-document/bibliographic-data/application-reference/document-id')
pointer_publication_reference = JsonPointer('/exchange-document/bibliographic-data/publication-reference/document-id')
pointer_invention_title = JsonPointer('/exchange-document/bibliographic-data/invention-title')
pointer_abstract = JsonPointer('/exchange-document/abstract')
pointer_applicant = JsonPointer('/exchange-document/bibliographic-data/parties/applicants/applicant')
pointer_inventor = JsonPointer('/exchange-document/bibliographic-data/parties/inventors/inventor')
results = to_list(pointer_results.resolve(response))
for result in results:
pubref = pointer_publication_reference.resolve(result)
pubref_number, pubref_date = _get_document_number_date(pubref, 'epodoc')
pubref_date = pubref_date and '-'.join([pubref_date[:4], pubref_date[4:6], pubref_date[6:8]])
appref = pointer_application_reference.resolve(result)
appref_number, appref_date = _get_document_number_date(appref, 'epodoc')
appref_date = appref_date and '-'.join([appref_date[:4], appref_date[4:6], appref_date[6:8]])
try:
titles = to_list(pointer_invention_title.resolve(result))
titles = map(_format_title, titles)
except JsonPointerException:
titles = None
try:
abstracts = to_list(pointer_abstract.resolve(result))
abstracts = map(_format_abstract, abstracts)
except JsonPointerException:
abstracts = None
try:
applicants = to_list(pointer_applicant.resolve(result))
applicants = _mogrify_parties(applicants, 'applicant-name')
except JsonPointerException:
applicants = None
try:
inventors = to_list(pointer_inventor.resolve(result))
inventors = _mogrify_parties(inventors, 'inventor-name')
except JsonPointerException:
inventors = None
item = {
'abstract': abstracts,
'appdate': appref_date,
'appnumber': appref_number,
'pubdate': pubref_date,
'pubnumber': pubref_number,
'title': titles,
'applicant': applicants,
'inventor': inventors,
}
items.append(item)
return items
def _render_service(self, path, service, methods):
env = self.state.document.settings.env
service_id = "service-%d" % env.new_serialno('service')
service_node = nodes.section(ids=[service_id])
service_node += nodes.title(text='Service at %s' % service.route_name)
if service.description is not None:
service_node += rst2node(_dedent(service.description))
for method, info in methods.items():
method_id = '%s-%s' % (service_id, method)
method_node = nodes.section(ids=[method_id])
method_node += nodes.title(text=method)
docstring = info['func'].__doc__ or ""
if 'validator' in info:
validators = to_list(info['validator'])
for validator in validators:
if validator.__doc__ is not None:
if docstring is not None:
docstring += '\n' + validator.__doc__.strip()
if 'accept' in info:
accept = info['accept']
if callable(accept):
if accept.__doc__ is not None:
docstring += accept.__doc__.strip()
else:
accept = to_list(accept)
accept_node = nodes.strong(text='Accepted content types:')
node_accept_list = nodes.bullet_list()
accept_node += node_accept_list
for item in accept:
temp = nodes.list_item()
temp += nodes.inline(text=item)
node_accept_list += temp
method_node += accept_node
node = rst2node(docstring)
if node is not None:
method_node += node
renderer = info['renderer']
if renderer == 'simplejson':
renderer = 'json'
response = nodes.paragraph()
response += nodes.strong(text='Response: %s' % renderer)
method_node += response
service_node += method_node
return service_node
def get_arguments(self, conf=None):
"""Return a dictionary of arguments. Takes arguments from the :param
conf: param and merges it with the arguments passed in the constructor.
:param conf: the dictionary to use.
"""
if conf is None:
conf = {}
arguments = {}
for arg in self.mandatory_arguments:
# get the value from the passed conf, then from the instance, then
# from the default class settings.
arguments[arg] = conf.pop(arg, getattr(self, arg, None))
for arg in self.list_arguments:
# rather than overwriting, extend the defined lists if
# any. take care of re-creating the lists before appending
# items to them, to avoid modifications to the already
# existing ones
value = list(getattr(self, arg, []))
if arg in conf:
value.extend(to_list(conf.pop(arg)))
arguments[arg] = value
# schema validation handling
if 'schema' in conf:
arguments['schema'] = (
CorniceSchema.from_colander(conf.pop('schema')))
# Allow custom error handler
arguments['error_handler'] = conf.pop('error_handler',
getattr(self, 'error_handler',
json_error))
# exclude some validators or filters
if 'exclude' in conf:
for item in to_list(conf.pop('exclude')):
for container in arguments['validators'], arguments['filters']:
if item in container:
container.remove(item)
# also include the other key,value pair we don't know anything about
arguments.update(conf)
# if some keys have been defined service-wide, then we need to add
# them to the returned dict.
if hasattr(self, 'arguments'):
for key, value in self.arguments.items():
if key not in arguments:
arguments[key] = value
return arguments
def get_arguments(self, conf=None):
"""Return a dictionary of arguments. Takes arguments from the :param
conf: param and merges it with the arguments passed in the constructor.
:param conf: the dictionary to use.
"""
if conf is None:
conf = {}
arguments = {}
for arg in self.mandatory_arguments:
# get the value from the passed conf, then from the instance, then
# from the default class settings.
arguments[arg] = conf.pop(arg, getattr(self, arg, None))
for arg in self.list_arguments:
# rather than overwriting, extend the defined lists if
# any. take care of re-creating the lists before appending
# items to them, to avoid modifications to the already
# existing ones
value = list(getattr(self, arg, []))
if arg in conf:
value.extend(to_list(conf.pop(arg)))
arguments[arg] = value
# schema validation handling
if 'schema' in conf:
arguments['schema'] = (CorniceSchema.from_colander(
conf.pop('schema')))
# Allow custom error handler
arguments['error_handler'] = conf.pop(
'error_handler', getattr(self, 'error_handler', json_error))
# exclude some validators or filters
if 'exclude' in conf:
for item in to_list(conf.pop('exclude')):
for container in arguments['validators'], arguments['filters']:
if item in container:
container.remove(item)
# also include the other key,value pair we don't know anything about
arguments.update(conf)
# if some keys have been defined service-wide, then we need to add
# them to the returned dict.
if hasattr(self, 'arguments'):
for key, value in self.arguments.items():
if key not in arguments:
arguments[key] = value
return arguments
def __init__(self,
name,
path,
description=None,
cors_policy=None,
depth=1,
**kw):
self.name = name
self.path = path
self.description = description
self.cors_expose_all_headers = True
self._cors_enabled = None
if cors_policy:
for key, value in cors_policy.items():
kw.setdefault('cors_' + key, value)
for key in self.list_arguments:
# default_{validators,filters} and {filters,validators} don't
# have to be mutables, so we need to create a new list from them
extra = to_list(kw.get(key, []))
kw[key] = []
kw[key].extend(getattr(self, 'default_%s' % key, []))
kw[key].extend(extra)
self.arguments = self.get_arguments(kw)
for key, value in self.arguments.items():
# avoid squashing Service.decorator if ``decorator``
# argument is used to specify a default pyramid view
# decorator
if key != 'decorator':
setattr(self, key, value)
if hasattr(self, 'factory') and hasattr(self, 'acl'):
raise KeyError("Cannot specify both 'acl' and 'factory'")
# instantiate some variables we use to keep track of what's defined for
# this service.
self.defined_methods = []
self.definitions = []
# add this service to the list of available services
SERVICES.append(self)
# register aliases for the decorators
for verb in ('GET', 'POST', 'PUT', 'DELETE', 'OPTIONS', 'PATCH'):
setattr(self, verb.lower(),
functools.partial(self.decorator, verb))
if VENUSIAN:
# this callback will be called when config.scan (from pyramid) will
# be triggered.
def callback(context, name, ob):
config = context.config.with_package(info.module)
config.add_cornice_service(self)
info = venusian.attach(self,
callback,
category='pyramid',
depth=depth)
def ops_family_members(document_number):
pointer_results = JsonPointer('/ops:world-patent-data/ops:patent-family/ops:family-member')
pointer_publication_reference = JsonPointer('/publication-reference/document-id')
pointer_application_reference = JsonPointer('/application-reference/document-id')
#pointer_priority_claim_reference = JsonPointer('/priority-claim/document-id')
response = ops_family_inpadoc('publication', document_number, '')
family_members = OPSFamilyMembers()
family_members.raw = to_list(pointer_results.resolve(response))
for result in family_members.raw:
# B.1 get publication and application references
pubref = pointer_publication_reference.resolve(result)
pubref_number, pubref_date = _get_document_number_date(pubref, 'docdb')
pubref_number_epodoc, pubref_date_epodoc = _get_document_number_date(pubref, 'epodoc')
appref = pointer_application_reference.resolve(result)
appref_number, appref_date = _get_document_number_date(appref, 'docdb')
family_members.items.append({
'publication': {'number-docdb': pubref_number, 'date': pubref_date, 'number-epodoc': pubref_number_epodoc, },
'application': {'number-docdb': appref_number, 'date': appref_date},
})
#log.info('Family members for %s:\n%s', document_number, family_members)
return family_members
def __init__(self, header_files, library_file, include_path=None, library_path=None, cache_path=None):
self.header_files = to_list(header_files)
self.library_file = library_file
self.include_path = include_path or os.curdir
self.library_path = library_path or os.curdir
self.cache_path = cache_path or './var'
self.include_path = os.path.abspath(self.include_path)
self.library_path = os.path.abspath(self.library_path)
cache_key = \
self.include_path.replace('/', '_') + \
u'-' + \
u'_'.join(self.header_files) + \
u'.pyclibrary'
self.cache_file = os.path.join(self.cache_path, cache_key)
logger.info('Setting up library "{}" with headers "{}", cache file is "{}"'.format(
self.library_file, ', '.join(self.header_files), self.cache_file))
# holding the library essentials
self.parser = None
self.clib = None
self.annotations = None
self.setup()
self.parse()
self.parse_annotations()
self.load()
def enrich_image_inquiry_info(info):
"""
Enrich image inquiry information.
If DRAWINGS can be properly detected, add information to "meta" dictionary of document and return True.
"""
meta = {}
enriched = False
# Compute page offset to first drawing from "FullDocument" information
entry = info.get('FullDocument')
if entry and 'ops:document-section' in entry:
sections = entry.get('ops:document-section', [])
for section in to_list(sections):
if section['@name'] == 'DRAWINGS':
meta['drawing-start-page'] = int(section['@start-page'])
enriched = True
break
# Compute number of drawing pages
if 'drawing-start-page' in meta:
if 'Drawing' in info:
meta['drawing-total-count'] = int(info['Drawing']['@number-of-pages'])
else:
meta['drawing-total-count'] = int(info['FullDocument']['@number-of-pages']) - meta['drawing-start-page'] + 1
info['META'] = meta
return enriched
def _notfound(request):
match = request.matchdict
if match is not None:
pattern = request.matched_route.pattern
service = request.registry['cornice_services'].get(pattern)
if (service is not None
and isinstance(request.exception, PredicateMismatch)
and request.method in service.defined_methods):
# maybe was it the accept predicate that was not matched
# in this case, returns a HTTP 406 NOT ACCEPTABLE with the
# list of available choices
api_kwargs = service.definitions[request.method]
if 'accept' in api_kwargs:
accept = api_kwargs.get('accept')
acceptable = [a for a in util.to_list(accept) if
not callable(a)]
if 'acceptable' in request.info:
for content_type in request.info['acceptable']:
if content_type not in acceptable:
acceptable.append(content_type)
if not request.accept.best_match(acceptable):
# if not, return the list of accepted headers
resp = request.response
resp.status = 406
resp.content_type = "application/json"
resp.body = json.dumps(acceptable)
return resp
# 404
return request.exception
def register_service_views(config, service):
"""Register the routes of the given service into the pyramid router.
:param config: the pyramid configuration object that will be populated.
:param service: the service object containing the definitions
"""
services = config.registry.setdefault("cornice_services", {})
services[service.path] = service
# keep track of the registered routes
registered_routes = []
# register the fallback view, which takes care of returning good error
# messages to the user-agent
for method, view, args in service.definitions:
args = dict(args) # make a copy of the dict to not modify it
args["request_method"] = method
decorated_view = decorate_view(view, dict(args), method)
for item in ("filters", "validators", "schema", "klass"):
if item in args:
del args[item]
# if acl is present, then convert it to a "factory"
if "acl" in args:
args["factory"] = make_route_factory(args.pop("acl"))
route_args = {}
if "factory" in args:
route_args["factory"] = args.pop("factory")
# register the route name with the path if it's not already done
if service.path not in registered_routes:
config.add_route(service.path, service.path, **route_args)
config.add_view(view=get_fallback_view(service), route_name=service.path)
registered_routes.append(service.path)
# loop on the accept fields: we need to build custom predicate if
# callables were passed
if "accept" in args:
for accept in to_list(args.pop("accept", ())):
predicates = args.get("custom_predicates", [])
if callable(accept):
predicate_checker = functools.partial(match_accept_header, accept)
predicates.append(predicate_checker)
args["custom_predicates"] = predicates
else:
# otherwise it means that it is a "standard" accept,
# so add it as such.
args["accept"] = accept
# We register multiple times the same view with different
# accept / custom_predicates arguments
config.add_view(view=decorated_view, route_name=service.path, **args)
else:
# it is a simple view, we don't need to loop on the definitions
# and just add it one time.
config.add_view(view=decorated_view, route_name=service.path, **args)
def run_acquisition(document_number, doctypes=None):
numbers = to_list(document_number)
doctypes = doctypes or 'xml'
doctypes = to_list(doctypes)
log.info(
'PDF archive acquisition for doctypes={doctypes}, numbers={numbers}'.
format(doctypes=doctypes, numbers=numbers))
# v1: Native xmlrpclib
#with XmlRpcTimeoutServer(archive_service_baseurl + '/RPC2', 15) as server:
# return server.runAcquisition(numbers, doctypes)
# v2: With "requests" transport
url = archive_service_baseurl + '/RPC2'
transport = RequestsTransport(session=get_client(), timeout=(2, 17))
transport.use_https = use_https
server = xmlrpclib.ServerProxy(url, transport=transport)
return server.runAcquisition(numbers, doctypes)
def __init__(self,
name,
path=None,
description=None,
cors_policy=None,
depth=1,
pyramid_route=None,
**kw):
self.name = name
self.path = path
self.pyramid_route = pyramid_route
if not self.path and not self.pyramid_route:
raise TypeError('You need to pass path or pyramid_route arg')
self.description = description
self.cors_expose_all_headers = True
self._cors_enabled = None
if cors_policy:
for key, value in cors_policy.items():
kw.setdefault('cors_' + key, value)
for key in self.list_arguments:
# default_{validators,filters} and {filters,validators} don't
# have to be mutables, so we need to create a new list from them
extra = to_list(kw.get(key, []))
kw[key] = []
kw[key].extend(getattr(self, 'default_%s' % key, []))
kw[key].extend(extra)
self.arguments = self.get_arguments(kw)
for key, value in self.arguments.items():
# avoid squashing Service.decorator if ``decorator``
# argument is used to specify a default pyramid view
# decorator
if key != 'decorator':
setattr(self, key, value)
if hasattr(self, 'acl'):
raise ConfigurationError("'acl' is not supported")
# instantiate some variables we use to keep track of what's defined for
# this service.
self.defined_methods = []
self.definitions = []
# add this service to the list of available services
SERVICES.append(self)
# this callback will be called when config.scan (from pyramid) will
# be triggered.
def callback(context, name, ob):
config = context.config.with_package(info.module)
config.add_cornice_service(self)
info = venusian.attach(self, callback, category='pyramid', depth=depth)
def __init__(self, name, path=None, description=None, cors_policy=None,
depth=1, pyramid_route=None, **kw):
self.name = name
self.path = path
self.pyramid_route = pyramid_route
if not self.path and not self.pyramid_route:
raise TypeError('You need to pass path or pyramid_route arg')
self.description = description
self.cors_expose_all_headers = True
self._cors_enabled = None
if cors_policy:
for key, value in cors_policy.items():
kw.setdefault('cors_' + key, value)
for key in self.list_arguments:
# default_{validators,filters} and {filters,validators} don't
# have to be mutables, so we need to create a new list from them
extra = to_list(kw.get(key, []))
kw[key] = []
kw[key].extend(getattr(self, 'default_%s' % key, []))
kw[key].extend(extra)
self.arguments = self.get_arguments(kw)
for key, value in self.arguments.items():
# avoid squashing Service.decorator if ``decorator``
# argument is used to specify a default pyramid view
# decorator
if key != 'decorator':
setattr(self, key, value)
if hasattr(self, 'acl'):
raise ConfigurationError("'acl' is not supported")
# instantiate some variables we use to keep track of what's defined for
# this service.
self.defined_methods = []
self.definitions = []
# add this service to the list of available services
SERVICES.append(self)
# register aliases for the decorators
for verb in ('GET', 'POST', 'PUT', 'DELETE', 'OPTIONS', 'PATCH'):
setattr(self, verb.lower(),
functools.partial(self.decorator, verb))
# this callback will be called when config.scan (from pyramid) will
# be triggered.
def callback(context, name, ob):
config = context.config.with_package(info.module)
config.add_cornice_service(self)
info = venusian.attach(self, callback, category='pyramid', depth=depth)
def _pop_predicate_definition(args, kind):
"""
Build a dictionary enriched by "kind" of predicate definition list.
This is required for evaluation in ``_mungle_view_args``.
"""
values = to_list(args.pop(kind, ()))
# In much the same way as filter(), the map() function [in Python 3] now
# returns an iterator. (In Python 2, it returned a list.)
# http://getpython3.com/diveintopython3/porting-code-to-python-3-with-2to3.html#map
values = list(map(lambda value: {'kind': kind, 'value': value}, values))
return values
def _pop_predicate_definition(args, kind):
"""
Build a dictionary enriched by "kind" of predicate definition list.
This is required for evaluation in ``_mungle_view_args``.
"""
values = to_list(args.pop(kind, ()))
# In much the same way as filter(), the map() function [in Python 3] now
# returns an iterator. (In Python 2, it returned a list.)
# http://getpython3.com/diveintopython3/porting-code-to-python-3-with-2to3.html#map
values = list(map(lambda value: {'kind': kind, 'value': value}, values))
return values
def _get_document_number_date(docref, id_type):
docref_list = to_list(docref)
for document_id in docref_list:
if document_id['@document-id-type'] == id_type:
if id_type == 'epodoc':
doc_number = document_id['doc-number']['$']
else:
doc_number = document_id['country']['$'] + document_id['doc-number']['$'] + document_id['kind']['$']
date = document_id.get('date', {}).get('$')
return doc_number, date
return None, None
def filter_argumentlist(self, method, argname, filter_callables=False):
"""
Helper method to ``get_acceptable`` and ``get_contenttypes``. DRY.
"""
result = []
for meth, view, args in self.definitions:
if meth.upper() == method.upper():
result_tmp = to_list(args.get(argname))
if filter_callables:
result_tmp = [a for a in result_tmp if not callable(a)]
result.extend(result_tmp)
return result
def filter_argumentlist(self, method, argname, filter_callables=False):
"""
Helper method to ``get_acceptable`` and ``get_contenttypes``. DRY.
"""
result = []
for meth, view, args in self.definitions:
if meth.upper() == method.upper():
result_tmp = to_list(args.get(argname))
if filter_callables:
result_tmp = [a for a in result_tmp if not callable(a)]
result.extend(result_tmp)
return result
def __init__(self, channel=None, graphing=None, strategy=None):
MultiService.__init__(self)
# TODO: Sanity checks/assertions against channel, graphing and strategy
# TODO: Make subsystems dynamic
self.subsystems = ['channel', 'graphing', 'strategy']
self.channel = channel or Bunch(realm=None, subscriptions=[])
self.graphing = to_list(graphing)
self.strategy = strategy
self.name = u'service-mig-' + self.channel.get('realm', unicode(id(self)))
def call_service(func, api_kwargs, context, request):
wrap_request(request)
# Checking that the accept headers sent by the client
# match what can be handled by the function, if specified.
#
# If not, returns a HTTP 406 NOT ACCEPTABLE with the list of available
# choices
if 'accept' in request.headers and 'accept' in api_kwargs:
accept = api_kwargs.get('accept')
if callable(accept):
acceptable = accept(request)
else:
acceptable = accept
acceptable = to_list(acceptable)
# does it comply with the headers sent by the client?
best_match = request.accept.best_match(acceptable)
if best_match:
return func(request)
else:
# if not, return the list of accepted headers
resp = request.response
resp.status = 406
resp.content_type = "application/json"
resp.body = json.dumps(acceptable)
return resp
for validator in to_list(api_kwargs.get('validator', [])):
validator(request)
if len(request.errors) > 0:
return json_error(request.errors)
return func(request)
def call_service(func, api_kwargs, context, request):
wrap_request(request)
# Checking that the accept headers sent by the client
# match what can be handled by the function, if specified.
#
# If not, returns a HTTP 406 NOT ACCEPTABLE with the list of available
# choices
if 'accept' in request.headers and 'accept' in api_kwargs:
accept = api_kwargs.get('accept')
if callable(accept):
acceptable = accept(request)
else:
acceptable = accept
acceptable = to_list(acceptable)
# does it comply with the headers sent by the client?
best_match = request.accept.best_match(acceptable)
if best_match:
return func(request)
else:
# if not, return the list of accepted headers
resp = request.response
resp.status = 406
resp.content_type = "application/json"
resp.body = json.dumps(acceptable)
return resp
for validator in to_list(api_kwargs.get('validator', [])):
validator(request)
if len(request.errors) > 0:
return json_error(request.errors)
return func(request)
def __init__(self, name, path, description=None, cors_policy=None, depth=1,
**kw):
self.name = name
self.path = path
self.description = description
self.cors_expose_all_headers = True
self._schemas = {}
self._cors_enabled = None
if cors_policy:
for key, value in list(cors_policy.items()):
kw.setdefault('cors_' + key, value)
for key in self.list_arguments:
# default_{validators,filters} and {filters,validators} doesn't
# have to be mutables, so we need to create a new list from them
extra = to_list(kw.get(key, []))
kw[key] = []
kw[key].extend(getattr(self, 'default_%s' % key, []))
kw[key].extend(extra)
self.arguments = self.get_arguments(kw)
for key, value in list(self.arguments.items()):
setattr(self, key, value)
if hasattr(self, 'factory') and hasattr(self, 'acl'):
raise KeyError("Cannot specify both 'acl' and 'factory'")
# instanciate some variables we use to keep track of what's defined for
# this service.
self.defined_methods = []
self.definitions = []
# add this service to the list of available services
SERVICES.append(self)
# register aliases for the decorators
for verb in ('GET', 'POST', 'PUT', 'DELETE', 'OPTIONS', 'PATCH'):
setattr(self, verb.lower(),
functools.partial(self.decorator, verb))
if VENUSIAN:
# this callback will be called when config.scan (from pyramid) will
# be triggered.
def callback(context, name, ob):
config = context.config.with_package(info.module)
config.add_cornice_service(self)
info = venusian.attach(self, callback, category='pyramid',
depth=depth)
def pdf_make_metadata(title, producer, pagecount, page_sections=None):
page_sections = page_sections and to_list(page_sections) or []
date = pdf_now()
tpl = """
InfoBegin
InfoKey: Title
InfoValue: {title}
InfoBegin
InfoKey: Producer
InfoValue: {producer}
InfoBegin
InfoKey: Creator
InfoValue:
InfoBegin
InfoKey: ModDate
InfoValue:
InfoBegin
InfoKey: CreationDate
InfoValue: {date}
NumberOfPages: {pagecount}
"""
metadata = tpl.format(**locals())
# https://stackoverflow.com/questions/2969479/merge-pdfs-with-pdftk-with-bookmarks/20333267#20333267
bookmark_tpl = """
BookmarkBegin
BookmarkTitle: {title}
BookmarkLevel: {level}
BookmarkPageNumber: {start_page}
"""
for page_section in page_sections:
name = page_section['@name']
start_page = page_section['@start-page']
if name == 'SEARCH_REPORT':
title = 'Search-report'
else:
title = name.title()
level = 1
metadata += bookmark_tpl.format(**locals())
return metadata
def callback(context, name, ob):
config = context.config.with_package(info.module)
self._define(config, method)
config.add_apidoc((self.route_pattern, method), func, self,
**_api_kw)
view_kw = _api_kw.copy()
for arg in _CORNICE_ARGS:
view_kw.pop(arg, None)
# method decorators
if 'attr' in view_kw:
@functools.wraps(getattr(ob, kw['attr']))
def view(request):
meth = getattr(ob(request), kw['attr'])
return meth()
del view_kw['attr']
view = functools.partial(call_service, view,
self.definitions[method])
else:
view = functools.partial(call_service, ob,
self.definitions[method])
# set the module of the partial function
setattr(view, '__module__', getattr(ob, '__module__'))
# handle accept headers as custom predicates if needed
if 'accept' in view_kw:
for accept in to_list(view_kw.pop('accept', ())):
_view_kw = view_kw.copy()
predicates = view_kw.get('custom_predicates', [])
if callable(accept):
predicates.append(
functools.partial(match_accept_header,
accept))
_view_kw['custom_predicates'] = predicates
else:
_view_kw['accept'] = accept
config.add_view(view=view, route_name=self.route_name,
**_view_kw)
else:
config.add_view(view=view, route_name=self.route_name,
**view_kw)
def callback(context, name, ob):
config = context.config.with_package(info.module)
self._define(config, method)
config.add_apidoc((self.route_pattern, method), func, self,
**_api_kw)
view_kw = _api_kw.copy()
for arg in _CORNICE_ARGS:
view_kw.pop(arg, None)
# method decorators
if 'attr' in view_kw:
@functools.wraps(getattr(ob, kw['attr']))
def view(request):
meth = getattr(ob(request), kw['attr'])
return meth()
del view_kw['attr']
view = functools.partial(call_service, view, _api_kw)
else:
view = functools.partial(call_service, ob, _api_kw)
# set the module of the partial function
setattr(view, '__module__', getattr(ob, '__module__'))
# handle accept headers as custom predicates if needed
if 'accept' in view_kw:
for accept in to_list(view_kw.pop('accept', ())):
_view_kw = view_kw.copy()
predicates = view_kw.get('custom_predicates', [])
if callable(accept):
predicates.append(
functools.partial(match_accept_header, accept))
_view_kw['custom_predicates'] = predicates
else:
_view_kw['accept'] = accept
config.add_view(view=view,
route_name=self.route_name,
**_view_kw)
else:
config.add_view(view=view,
route_name=self.route_name,
**view_kw)
def get_acceptable(self, method, filter_callables=False):
"""return a list of acceptable content-type headers that were defined
for this service.
:param method: the method to get the acceptable content-types for
:param filter_callables: it is possible to give acceptable
content-types dinamycally, with callables.
This filter or not the callables (default:
False)
"""
acceptable = []
for meth, view, args in self.definitions:
if meth.upper() == method.upper():
acc = to_list(args.get('accept'))
if filter_callables:
acc = [a for a in acc if not callable(a)]
acceptable.extend(acc)
return acceptable
def get_acceptable(self, method, filter_callables=False):
"""return a list of acceptable content-type headers that were defined
for this service.
:param method: the method to get the acceptable content-types for
:param filter_callables: it is possible to give acceptable
content-types dinamycally, with callables.
This filter or not the callables (default:
False)
"""
acceptable = []
for meth, view, args in self.definitions:
if meth.upper() == method.upper():
acc = to_list(args.get('accept'))
if filter_callables:
acc = [a for a in acc if not isinstance(a, collections.Callable)]
acceptable.extend(acc)
return acceptable
def patch_files(self, filepatterns, data):
filepatterns = to_list(filepatterns)
directory = os.path.join(self.repo.working_dir, self.workingdir)
# cd into appropriate directory
with local.cwd(directory):
for pattern in filepatterns:
# Find files matching pattern in directory
# This is plumbum's syntax for shell globbing
path = local.path()
lp_files = path // str(pattern)
# Iterate and patch all files
for lp_file in lp_files:
filepath = str(lp_file)
self.patch_file(filepath, data)
def patch_files(self, filepatterns, data):
filepatterns = to_list(filepatterns)
directory = os.path.join(self.repo.working_dir, self.workingdir)
# cd into appropriate directory
with local.cwd(directory):
for pattern in filepatterns:
# Find files matching pattern in directory
# This is plumbum's syntax for shell globbing
path = local.path()
lp_files = path // str(pattern)
# Iterate and patch all files
for lp_file in lp_files:
filepath = str(lp_file)
self.patch_file(filepath, data)
def is_amendment_only(node):
"""
Is FullDocument reference a correction page of amendments like US2010252183A1?
{u'@desc': u'FullDocument',
u'@link': u'published-data/images/US/8052819/X6/fullimage',
u'@number-of-pages': u'1',
u'@system': u'ops.epo.org',
u'ops:document-format-options': {u'ops:document-format': [{u'$': u'application/pdf'},
{u'$': u'application/tiff'}]},
u'ops:document-section': {u'@name': u'AMENDMENT',
u'@start-page': u'1'}}],
"""
if is_fulldocument(node):
sections = to_list(node.get('ops:document-section', []))
if len(sections) == 1 and sections[0]['@name'] == u'AMENDMENT':
return True
return False
def ops_family_inpadoc(reference_type, document_number, constituents, xml=False):
"""
Request family information from OPS in JSON format.
reference_type = publication|application|priority
constituents = biblio|legal
Examples:
- http://ops.epo.org/3.1/rest-services/family/publication/docdb/EP.1491501.A1/biblio,legal
- http://ops.epo.org/3.1/rest-services/family/publication/docdb/EP0666666/biblio
- http://ops.epo.org/3.1/rest-services/family/publication/docdb/EP0666666.A2/biblio
- http://ops.epo.org/3.1/rest-services/family/publication/docdb/EP0666666.B1/biblio
"""
# Compute document identifier.
document_id = split_patent_number(document_number)
ops_id = epo_ops.models.Epodoc(document_id.country + document_id.number, document_id.kind)
# Acquire family information from OPS.
with ops_client(xml=xml) as ops:
response = ops.family(reference_type, ops_id, constituents=to_list(constituents))
return handle_response(response, 'ops-family')
def ops_register(reference_type, document_number, constituents=None, xml=False):
"""
Request register information from OPS in JSON or XML format.
reference_type = publication|application|priority
Examples:
- http://ops.epo.org/3.1/rest-services/register/publication/epodoc/EP2485810/biblio
- http://ops.epo.org/3.1/rest-services/register/publication/epodoc/EP2485810/biblio,legal.json
"""
if constituents is None:
constituents = 'biblio,legal'
# Compute document identifier.
document_id = split_patent_number(document_number)
#ops_id = epo_ops.models.Docdb(document_id.number, document_id.country, document_id.kind)
ops_id = epo_ops.models.Epodoc(document_id.country + document_id.number, document_id.kind)
# Acquire register information from OPS.
with ops_client(xml=xml) as ops:
response = ops.register(reference_type, ops_id, constituents=to_list(constituents))
return handle_response(response, 'ops-register')
def ops_family_publication_docdb_xml(reference_type, document_number, constituents):
"""
Request family information from OPS in XML format.
reference_type = publication|application|priority
constituents = biblio|legal
Examples:
- http://ops.epo.org/3.1/rest-services/family/publication/docdb/EP.1491501.A1/biblio,legal
"""
# Compute document identifier.
document_id = split_patent_number(document_number)
ops_id = epo_ops.models.Docdb(document_id.number, document_id.country, document_id.kind)
# Acquire family information from OPS.
ops = get_ops_client()
# FIXME: Better use "accept_type" on a per-request basis supported by ``python-epo-ops-client``.
ops.accept_type = 'application/xml'
response = ops.family(reference_type, ops_id, constituents=to_list(constituents))
ops.accept_type = 'application/json'
return handle_response(response, 'ops-family')
def _fallback_view(self, request):
"""Fallback view for this service, called when nothing else matches.
This method provides the view logic to be executed when the request
does not match any explicitly-defined view. Its main responsibility
is to produce an accurate error response, such as HTTPMethodNotAllowed
or HTTPNotAcceptable.
"""
# Maybe we failed to match any definitions for the request method?
if request.method not in self.defined_methods:
response = HTTPMethodNotAllowed()
response.allow = self.defined_methods
return response
# Maybe we failed to match an acceptable content-type?
# First search all the definitions to find the acceptable types.
# XXX: precalculate this like the defined_methods list?
acceptable = []
for api_kwargs in self.definitions:
if api_kwargs['request_method'] != request.method:
continue
if 'accept' in api_kwargs:
accept = to_list(api_kwargs.get('accept'))
acceptable.extend(a for a in accept if not callable(a))
if 'acceptable' in request.info:
for content_type in request.info['acceptable']:
if content_type not in acceptable:
acceptable.append(content_type)
# Now check if that was actually the source of the problem.
if not request.accept.best_match(acceptable):
response = HTTPNotAcceptable()
response.content_type = "application/json"
response.body = json.dumps(acceptable)
return response
# In the absence of further information about what went wrong,
# let upstream deal with the mismatch.
raise PredicateMismatch(self.name)
def ops_published_data_search_real(constituents, query, range):
# OPS client object, impersonated for the current user.
ops = get_ops_client()
# Send request to OPS.
range_begin, range_end = map(int, range.split('-'))
response = ops.published_data_search(
query, range_begin=range_begin, range_end=range_end, constituents=to_list(constituents))
# Decode OPS response from JSON
payload = handle_response(response, 'ops-search')
if response.headers['content-type'].startswith('application/json'):
# Decode total number of results.
pointer_total_count = JsonPointer('/ops:world-patent-data/ops:biblio-search/@total-result-count')
count_total = int(pointer_total_count.resolve(payload))
# Raise an exception to skip caching empty results.
if count_total == 0:
raise NoResultsException('No results', data=payload)
return payload
def __init__(self,
header_files,
library_file,
include_path=None,
library_path=None,
cache_path=None):
self.header_files = to_list(header_files)
self.library_file = library_file
self.include_path = include_path or os.curdir
self.library_path = library_path or os.curdir
self.cache_path = cache_path or './var'
self.include_path = os.path.abspath(self.include_path)
self.library_path = os.path.abspath(self.library_path)
cache_key = \
self.include_path.replace('/', '_') + \
u'-' + \
u'_'.join(self.header_files) + \
u'.pyclibrary'
self.cache_file = os.path.join(self.cache_path, cache_key)
logger.info(
'Setting up library "{}" with headers "{}", cache file is "{}"'.
format(self.library_file, ', '.join(self.header_files),
self.cache_file))
# holding the library essentials
self.parser = None
self.clib = None
self.annotations = None
self.setup()
self.parse()
self.parse_annotations()
self.load()
def _fallback_view(self, request):
"""Fallback view for this service, called when nothing else matches.
This method provides the view logic to be executed when the request
does not match any explicitly-defined view. Its main responsibility
is to produce an accurate error response, such as HTTPMethodNotAllowed
or HTTPNotAcceptable.
"""
# Maybe we failed to match any definitions for the request method?
if request.method not in self.defined_methods:
response = HTTPMethodNotAllowed()
response.allow = self.defined_methods
return response
# Maybe we failed to match an acceptable content-type?
# First search all the definitions to find the acceptable types.
# XXX: precalculate this like the defined_methods list?
acceptable = []
for api_kwargs in self.definitions:
if api_kwargs["request_method"] != request.method:
continue
if "accept" in api_kwargs:
accept = to_list(api_kwargs.get("accept"))
acceptable.extend(a for a in accept if not callable(a))
if "acceptable" in request.info:
for content_type in request.info["acceptable"]:
if content_type not in acceptable:
acceptable.append(content_type)
# Now check if that was actually the source of the problem.
if not request.accept.best_match(acceptable):
response = HTTPNotAcceptable()
response.content_type = "application/json"
response.body = json.dumps(acceptable)
return response
# In the absence of further information about what went wrong,
# let upstream deal with the mismatch.
raise PredicateMismatch(self.name)
def _render_service(self, service):
service_id = "service-%d" % self.env.new_serialno("service")
service_node = nodes.section(ids=[service_id])
service_node += nodes.title(text="Service at %s" % service.path)
if service.description is not None:
service_node += rst2node(trim(service.description))
for method, view, args in service.definitions:
method_id = "%s-%s" % (service_id, method)
method_node = nodes.section(ids=[method_id])
method_node += nodes.title(text=method)
docstring = trim(view.__doc__ or "") + "\n"
if "schema" in args:
schema = args["schema"]
attrs_node = nodes.inline()
for location in ("headers", "querystring", "body"):
attributes = schema.get_attributes(location=location)
if attributes:
attrs_node += nodes.inline(text="values in the %s" % location)
location_attrs = nodes.bullet_list()
for attr in attributes:
temp = nodes.list_item()
desc = "%s : " % attr.name
# Get attribute data-type
if hasattr(attr, "type"):
attr_type = attr.type
elif hasattr(attr, "typ"):
attr_type = attr.typ.__class__.__name__
desc += " %s, " % attr_type
if attr.required:
desc += "required "
else:
desc += "optional "
temp += nodes.inline(text=desc)
location_attrs += temp
attrs_node += location_attrs
method_node += attrs_node
for validator in args.get("validators", ()):
if validator.__doc__ is not None:
docstring += trim(validator.__doc__)
if "accept" in args:
accept = to_list(args["accept"])
if callable(accept):
if accept.__doc__ is not None:
docstring += accept.__doc__.strip()
else:
accept_node = nodes.strong(text="Accepted content types:")
node_accept_list = nodes.bullet_list()
accept_node += node_accept_list
for item in accept:
temp = nodes.list_item()
temp += nodes.inline(text=item)
node_accept_list += temp
method_node += accept_node
node = rst2node(docstring)
DocFieldTransformer(self).transform_all(node)
if node is not None:
method_node += node
renderer = args["renderer"]
if renderer == "simplejson":
renderer = "json"
response = nodes.paragraph()
response += nodes.strong(text="Response: %s" % renderer)
method_node += response
service_node += method_node
return service_node
def _filter(attr):
if not hasattr(attr, "location"):
valid_location = 'body' in location
else:
valid_location = attr.location in to_list(location)
return valid_location and attr.required in to_list(required)
def api(self, **kw):
"""Decorates a function to make it a service.
Options can be passed to the decorator. The methods get, post, put and
delete are aliases to this one, specifying the "request_method"
argument for convenience.
:param request_method: the request method. Should be one of GET, POST,
PUT, DELETE, OPTIONS, HEAD, TRACE or CONNECT
:param decorators: A sequence of decorators which should be applied
to the view callable before it's returned. Will be
applied in order received, i.e. the last decorator
in the sequence will be the outermost wrapper.
All the constructor options, minus name and path, can be overwritten in
here.
"""
view_wrapper = self.get_view_wrapper(kw)
method = kw.get("request_method", "GET") # default is GET
api_kw = self.kw.copy()
api_kw.update(kw)
# sanitize the keyword arguments
if "renderer" not in api_kw:
api_kw["renderer"] = self.renderer
if "validator" in api_kw:
msg = "'validator' is deprecated, please use 'validators'"
warnings.warn(msg, DeprecationWarning)
api_kw["validators"] = api_kw.pop("validator")
validators = []
validators.extend(to_list(api_kw.get("validators", [])))
validators.extend(DEFAULT_VALIDATORS)
filters = []
filters.extend(to_list(api_kw.get("filters", [])))
filters.extend(DEFAULT_FILTERS)
# excluded validators/filters
for item in to_list(api_kw.pop("exclude", [])):
for items in validators, filters:
if item in items:
items.remove(item)
if "schema" in api_kw:
schema = CorniceSchema.from_colander(api_kw.pop("schema"))
validators.append(validate_colander_schema(schema))
self.schemas[method] = schema
api_kw["filters"] = filters
api_kw["validators"] = validators
def _api(func):
_api_kw = api_kw.copy()
self.definitions.append(_api_kw)
def callback(context, name, ob):
config = context.config.with_package(info.module)
self._define(config, method)
config.add_apidoc((self.route_pattern, method), func, self, **_api_kw)
view_kw = _api_kw.copy()
for arg in _CORNICE_ARGS:
view_kw.pop(arg, None)
# method decorators
if "attr" in view_kw:
@functools.wraps(getattr(ob, kw["attr"]))
def view(request):
meth = getattr(ob(request), kw["attr"])
return meth()
del view_kw["attr"]
view = functools.partial(call_service, view, _api_kw)
else:
view = functools.partial(call_service, ob, _api_kw)
# set the module of the partial function
setattr(view, "__module__", getattr(ob, "__module__"))
# handle accept headers as custom predicates if needed
if "accept" in view_kw:
for accept in to_list(view_kw.pop("accept", ())):
_view_kw = view_kw.copy()
predicates = view_kw.get("custom_predicates", [])
if callable(accept):
predicates.append(functools.partial(match_accept_header, accept))
_view_kw["custom_predicates"] = predicates
else:
_view_kw["accept"] = accept
config.add_view(view=view, route_name=self.route_name, **_view_kw)
else:
config.add_view(view=view, route_name=self.route_name, **view_kw)
func = view_wrapper(func)
info = venusian.attach(func, callback, category="pyramid")
if info.scope == "class":
# if the decorator was attached to a method in a class, or
# otherwise executed at class scope, we need to set an
# 'attr' into the settings if one isn't already in there
if "attr" not in kw:
kw["attr"] = func.__name__
kw["_info"] = info.codeinfo # fbo "action_method"
return func
return _api
def register_service_views(config, service):
"""Register the routes of the given service into the pyramid router.
:param config: the pyramid configuration object that will be populated.
:param service: the service object containing the definitions
"""
services = config.registry.setdefault('cornice_services', {})
services[service.path] = service
# keep track of the registered routes
registered_routes = []
# register the fallback view, which takes care of returning good error
# messages to the user-agent
for method, view, args in service.definitions:
args = dict(args) # make a copy of the dict to not modify it
args['request_method'] = method
decorated_view = decorate_view(view, dict(args), method)
for item in ('filters', 'validators', 'schema', 'klass',
'error_handler'):
if item in args:
del args[item]
# if acl is present, then convert it to a "factory"
if 'acl' in args:
args["factory"] = make_route_factory(args.pop('acl'))
route_args = {}
if 'factory' in args:
route_args['factory'] = args.pop('factory')
# register the route name with the path if it's not already done
if service.path not in registered_routes:
config.add_route(service.path, service.path, **route_args)
config.add_view(view=get_fallback_view(service),
route_name=service.path)
registered_routes.append(service.path)
# loop on the accept fields: we need to build custom predicate if
# callables were passed
if 'accept' in args:
for accept in to_list(args.pop('accept', ())):
predicates = args.get('custom_predicates', [])
if callable(accept):
predicate_checker = functools.partial(match_accept_header,
accept)
predicates.append(predicate_checker)
args['custom_predicates'] = predicates
else:
# otherwise it means that it is a "standard" accept,
# so add it as such.
args['accept'] = accept
# We register multiple times the same view with different
# accept / custom_predicates arguments
config.add_view(view=decorated_view, route_name=service.path,
**args)
else:
# it is a simple view, we don't need to loop on the definitions
# and just add it one time.
config.add_view(view=decorated_view, route_name=service.path,
**args)
def __init__(self, bus, topic, transform, logger):
self.bus = bus
self.topic = topic
self.transform = to_list(transform)
self.log = logger or Logger()
def ops_biblio_documents(patent):
data = get_ops_biblio_data('publication', patent)
documents = to_list(data['ops:world-patent-data']['exchange-documents']['exchange-document'])
return documents
def _extract_operation_from_view(self, view, args):
"""
Extract swagger operation details from colander view definitions.
:param view:
View to extract information from.
:param args:
Arguments from the view decorator.
:rtype: dict
:returns: Operation definition.
"""
op = {
'responses': {
'default': {
'description': 'UNDOCUMENTED RESPONSE'
}
},
}
# If 'produces' are not defined in the view, try get from renderers
renderer = args.get('renderer', '')
is_json_renderer = (
"json" in renderer # allows for "json" or "simplejson"
or renderer == Service.renderer # default renderer is json.
)
if is_json_renderer:
produces = ['application/json']
elif renderer == 'xml':
produces = ['text/xml']
else:
produces = None
if produces:
op.setdefault('produces', produces)
# Get explicit accepted content-types
consumes = args.get('content_type')
if consumes is not None:
# convert to a list, if it's not yet one
consumes = to_list(consumes)
# It is possible to add callables for content_type, so we have to
# to filter those out, since we cannot evaluate those here.
consumes = [x for x in consumes if not callable(x)]
op['consumes'] = consumes
# Get parameters from view schema
is_colander = self._is_colander_schema(args)
if is_colander:
schema = self._extract_transform_colander_schema(args)
parameters = self.parameters.from_schema(schema)
else:
# Bail out for now
parameters = None
if parameters:
op['parameters'] = parameters
# Get summary from docstring
if isinstance(view, six.string_types):
if 'klass' in args:
ob = args['klass']
view_ = getattr(ob, view.lower())
docstring = trim(view_.__doc__)
else:
docstring = str(trim(view.__doc__))
if docstring and self.summary_docstrings:
op['summary'] = docstring
# Get response definitions
if 'response_schemas' in args:
op['responses'] = self.responses.from_schema_mapping(
args['response_schemas'])
# Get response tags
if 'tags' in args:
op['tags'] = args['tags']
# Get response operationId
if 'operation_id' in args:
op['operationId'] = args['operation_id']
# Get security policies
if 'api_security' in args:
op['security'] = args['api_security']
return op
def read_config(configfiles, kind=None):
configfiles_requested = to_list(configfiles)
config = ConfigParser()
configfiles_used = config.read(configfiles_requested)
settings = convert_config(config, kind=kind)
return settings, configfiles_used
def _filter(attr):
return (attr.location in to_list(location) and
attr.required in to_list(required))
def _render_service(self, path, service, methods):
env = self.state.document.settings.env
service_id = "service-%d" % env.new_serialno('service')
service_node = nodes.section(ids=[service_id])
service_node += nodes.title(text='Service at %s' %
service.route_name)
if service.description is not None:
service_node += rst2node(trim(service.description))
for method, info in methods.items():
method_id = '%s-%s' % (service_id, method)
method_node = nodes.section(ids=[method_id])
method_node += nodes.title(text=method)
if 'attr' in info:
docstring = getattr(info['func'], info['attr']).__doc__ or ""
else:
docstring = info['func'].__doc__ or ""
docstring = trim(docstring) + '\n'
if method in service.schemas:
schema = service.schemas[method]
attrs_node = nodes.inline()
for location in ('headers', 'querystring', 'body'):
attributes = schema.get_attributes(location=location)
if attributes:
attrs_node += nodes.inline(
text='values in the %s' % location)
location_attrs = nodes.bullet_list()
for attr in attributes:
temp = nodes.list_item()
desc = "%s : " % attr.name
if hasattr(attr, 'type'):
desc += " %s, " % attr.type
if attr.required:
desc += "required "
else:
desc += "optional "
temp += nodes.inline(text=desc)
location_attrs += temp
attrs_node += location_attrs
method_node += attrs_node
if 'validators' in info:
for validator in info['validators']:
if validator.__doc__ is not None:
if docstring is not None:
doc = trim(validator.__doc__)
docstring += '\n' + doc
if 'accept' in info:
accept = info['accept']
if callable(accept):
if accept.__doc__ is not None:
docstring += accept.__doc__.strip()
else:
accept = to_list(accept)
accept_node = nodes.strong(text='Accepted content types:')
node_accept_list = nodes.bullet_list()
accept_node += node_accept_list
for item in accept:
temp = nodes.list_item()
temp += nodes.inline(text=item)
node_accept_list += temp
method_node += accept_node
node = rst2node(docstring)
DocFieldTransformer(self).transform_all(node)
if node is not None:
method_node += node
renderer = info['renderer']
if renderer == 'simplejson':
renderer = 'json'
response = nodes.paragraph()
response += nodes.strong(text='Response: %s' % renderer)
method_node += response
service_node += method_node
return service_node
def analytics_family(query):
payload = {}
family_has_statistics = {}
family_has_designated_states = {}
# A. aggregate list of publication numbers
# http://ops.epo.org/3.1/rest-services/published-data/search/full-cycle/?q=pa=%22MAMMUT%20SPORTS%20GROUP%20AG%22
# TODO: step through all pages
response = ops_published_data_search('biblio', query, '1-50')
pointer_results = JsonPointer('/ops:world-patent-data/ops:biblio-search/ops:search-result/exchange-documents')
pointer_family_id = JsonPointer('/exchange-document/@family-id')
pointer_publication_reference = JsonPointer('/exchange-document/bibliographic-data/publication-reference/document-id')
# A.1 compute distinct list with unique families
family_representatives = {}
results = to_list(pointer_results.resolve(response))
for result in results:
family_id = pointer_family_id.resolve(result)
# TODO: currently, use first document as family representative; this could change
if family_id not in family_representatives:
document_id_entries = pointer_publication_reference.resolve(result)
doc_number, date = _get_document_number_date(document_id_entries, 'epodoc')
if doc_number:
family_representatives[family_id] = doc_number
# B. Enrich all family representatives
# http://ops.epo.org/3.1/rest-services/family/application/docdb/US19288494.xml
for family_id, document_number in family_representatives.iteritems():
payload.setdefault(family_id, {})
# B.1 Aggregate all family members
try:
family = ops_family_members(document_number)
family_members = family.items
payload[family_id]['family-members'] = family_members
except Exception as ex:
request = get_current_request()
del request.errors[:]
log.warn('Could not fetch OPS family for {0}'.format(document_number))
continue
# B.2 Use first active priority
for family_member_raw in family.raw:
if 'priority-claim' not in payload[family_id]:
for priority_claim in to_list(family_member_raw['priority-claim']):
try:
if priority_claim['priority-active-indicator']['$'] == 'YES':
prio_number, prio_date = _get_document_number_date(priority_claim['document-id'], 'docdb')
payload[family_id]['priority-claim'] = {'number-docdb': prio_number, 'date': prio_date}
except KeyError:
pass
# B.3 Compute word- and image-counts for EP publication
for statistics_country in ['EP', 'WO', 'AT', 'CA', 'CH', 'GB', 'ES']:
if family_id in family_has_statistics:
break
for family_member in family_members:
pubref_number = family_member['publication']['number-epodoc']
if pubref_number.startswith(statistics_country):
statistics = {}
# B.3.1 get data about claims
try:
claims_response = ops_claims(pubref_number)
pointer_claims = JsonPointer('/ops:world-patent-data/ftxt:fulltext-documents/ftxt:fulltext-document/claims')
claims = pointer_claims.resolve(claims_response)
claim_paragraphs = []
for part in to_list(claims['claim']['claim-text']):
claim_paragraphs.append(part['$'])
claim_text = '\n'.join(claim_paragraphs)
statistics['claims-language'] = claims['@lang']
statistics['claims-words-first'] = len(claim_paragraphs[0].split())
statistics['claims-words-total'] = len(claim_text.split())
statistics['claims-count'] = len(claim_paragraphs)
except Exception as ex:
request = get_current_request()
del request.errors[:]
log.warn('Could not fetch OPS claims for {0}'.format(pubref_number))
# B.3.2 get data about description
try:
description_response = ops_description(pubref_number)
pointer_description = JsonPointer('/ops:world-patent-data/ftxt:fulltext-documents/ftxt:fulltext-document/description')
descriptions = pointer_description.resolve(description_response)
description_paragraphs = []
for part in to_list(descriptions['p']):
description_paragraphs.append(part['$'])
description_text = '\n'.join(description_paragraphs)
statistics['description-words-total'] = len(description_text.split())
except Exception as ex:
request = get_current_request()
del request.errors[:]
log.warn('Could not fetch OPS description for {0}'.format(pubref_number))
if statistics:
# B.3.3 get data about image count
try:
pubref_number_docdb = family_member['publication']['number-docdb']
imginfo = inquire_images(pubref_number_docdb)
statistics['drawings-count'] = imginfo['META']['drawing-total-count']
except Exception as ex:
request = get_current_request()
del request.errors[:]
family_member['statistics'] = statistics
family_has_statistics[family_id] = True
break
# B.4 compute designated states
pointer_designated_states = JsonPointer('/ops:world-patent-data/ops:register-search/reg:register-documents/reg:register-document/reg:bibliographic-data/reg:designation-of-states')
for country in ['EP', 'WO']:
if family_id in family_has_designated_states:
break
for family_member in family_members:
pubref_number = family_member['publication']['number-epodoc']
if pubref_number.startswith(country):
try:
reginfo_payload = ops_register('publication', pubref_number, 'biblio')
except:
request = get_current_request()
del request.errors[:]
log.warn('Could not fetch OPS register information for {0}'.format(pubref_number))
continue
designated_states_list = pointer_designated_states.resolve(reginfo_payload)
designated_states_info = to_list(designated_states_list)[0]
try:
regional_info = designated_states_info['reg:designation-pct']['reg:regional']
family_member.setdefault('register', {})
family_member['register']['designated-states'] = {
'gazette-num': designated_states_info['@change-gazette-num'],
'region': regional_info['reg:region']['reg:country']['$'],
'countries': list(_flatten_ops_json_list(regional_info['reg:country'])),
}
family_has_designated_states[family_id] = True
break
except Exception as ex:
log.error('Retrieving designated states for {0} failed.'.format(pubref_number))
return payload
def api(self, **kw):
"""Decorates a function to make it a service.
Options can be passed to the decorator. The methods get, post, put and
delete are aliases to this one, specifying the "request_method"
argument for convenience.
;param request_method: the request method. Should be one of GET, POST,
PUT, DELETE, OPTIONS, HEAD, TRACE or CONNECT
All the constructor options, minus name and path, can be overwritten in
here.
"""
method = kw.get('request_method', 'GET') # default is GET
api_kw = self.kw.copy()
api_kw.update(kw)
# sanitize the keyword arguments
if 'renderer' not in api_kw:
api_kw['renderer'] = self.renderer
if 'validator' in api_kw:
msg = "'validator' is deprecated, please use 'validators'"
warnings.warn(msg, DeprecationWarning)
api_kw['validators'] = api_kw.pop('validator')
validators = []
validators.extend(to_list(api_kw.get('validators', [])))
validators.extend(DEFAULT_VALIDATORS)
filters = []
filters.extend(to_list(api_kw.get('filters', [])))
filters.extend(DEFAULT_FILTERS)
# excluded validators/filters
for item in to_list(api_kw.pop('exclude', [])):
for items in validators, filters:
if item in items:
items.remove(item)
if 'schema' in api_kw:
schema = CorniceSchema.from_colander(api_kw.pop('schema'))
validators.append(validate_colander_schema(schema))
self.schemas[method] = schema
api_kw['filters'] = filters
api_kw['validators'] = validators
def _api(func):
_api_kw = api_kw.copy()
self.definitions[method] = _api_kw.copy()
def callback(context, name, ob):
config = context.config.with_package(info.module)
self._define(config, method)
config.add_apidoc((self.route_pattern, method), func, self,
**_api_kw)
view_kw = _api_kw.copy()
for arg in _CORNICE_ARGS:
view_kw.pop(arg, None)
# method decorators
if 'attr' in view_kw:
@functools.wraps(getattr(ob, kw['attr']))
def view(request):
meth = getattr(ob(request), kw['attr'])
return meth()
del view_kw['attr']
view = functools.partial(call_service, view,
self.definitions[method])
else:
view = functools.partial(call_service, ob,
self.definitions[method])
# set the module of the partial function
setattr(view, '__module__', getattr(ob, '__module__'))
# handle accept headers as custom predicates if needed
if 'accept' in view_kw:
for accept in to_list(view_kw.pop('accept', ())):
_view_kw = view_kw.copy()
predicates = view_kw.get('custom_predicates', [])
if callable(accept):
predicates.append(
functools.partial(match_accept_header,
accept))
_view_kw['custom_predicates'] = predicates
else:
_view_kw['accept'] = accept
config.add_view(view=view, route_name=self.route_name,
**_view_kw)
else:
config.add_view(view=view, route_name=self.route_name,
**view_kw)
info = venusian.attach(func, callback, depth=self.depth, category='pyramid')
if info.scope == 'class':
# if the decorator was attached to a method in a class, or
# otherwise executed at class scope, we need to set an
# 'attr' into the settings if one isn't already in there
if 'attr' not in kw:
kw['attr'] = func.__name__
kw['_info'] = info.codeinfo # fbo "action_method"
return func
return _api
def _render_service(self, service):
service_id = "service-%d" % self.env.new_serialno('service')
service_node = nodes.section(ids=[service_id])
title = '%s service at %s' % (service.name.title(), service.path)
title = title.replace('Collection_', '')
service_node += nodes.title(text=title)
if service.description is not None:
service_node += rst2node(trim(service.description))
for method, view, args in service.definitions:
if method == 'HEAD':
# Skip head - this is essentially duplicating the get docs.
continue
if method in self.options.get('ignore-methods', []):
continue
method_id = '%s-%s' % (service_id, method)
method_node = nodes.section(ids=[method_id])
method_node += nodes.title(text=method)
if is_string(view):
if 'klass' in args:
ob = args['klass']
view_ = getattr(ob, view.lower())
docstring = trim(view_.__doc__ or "") + '\n'
else:
docstring = trim(view.__doc__ or "") + '\n'
print('Replacing BIMT strings with app strings in API docstrings.')
docstring = docstring.replace(
'BIMT', self.options.get('app-title', 'BIMT'))
docstring = docstring.replace(
'http://localhost:8080', self.options.get('app-url', 'http://localhost:8080')) # noqa
attrs_node = None
if 'schema' in args:
schema = args['schema']
attrs_node = nodes.inline()
for location in ('header', 'querystring', 'body'):
attributes = schema.get_attributes(location=location)
if attributes:
attrs_node += nodes.inline(
text='Values in the %s:' % location)
location_attrs = nodes.bullet_list()
for attr in attributes:
temp = nodes.list_item()
# check for permissions
column_name = attr.name
if column_name == 'version':
column_name = 'v'
elif column_name == 'created':
column_name = 'c'
elif column_name == 'modified':
column_name = 'm'
# find attr in SQLAlchemy table definition and
# check that it has view permission
if not getattr(args['klass'], 'model', None):
continue
columns = args['klass'].model.__table__.columns
if column_name in columns.keys():
info = getattr(columns[column_name], 'info', None)
permission = info.get('colanderalchemy', {}).get('view_permission')
if permission != BimtPermissions.view:
continue
else:
continue
# Get attribute data-type
if hasattr(attr, 'type'):
attr_type = attr.type
elif hasattr(attr, 'typ'):
attr_type = attr.typ.__class__.__name__
else:
attr_type = None
temp += nodes.strong(text=attr.name)
if attr_type is not None:
temp += nodes.inline(text=' (%s)' % attr_type)
if not attr.required or attr.description:
temp += nodes.inline(text=' - ')
if not attr.required:
if attr.missing is not None and \
not isinstance(attr.missing, colander._drop):
default = json.dumps(attr.missing)
temp += nodes.inline(
text='(default: %s) ' % default)
else:
temp += nodes.inline(
text='(optional) ')
if attr.description:
temp += nodes.inline(text=attr.description)
location_attrs += temp
attrs_node += location_attrs
for validator in args.get('validators', ()):
if validator.__doc__ is not None:
docstring += trim(validator.__doc__)
accept_node = False
if 'accept' in args:
accept = to_list(args['accept'])
if callable(accept):
if accept.__doc__ is not None:
docstring += accept.__doc__.strip()
elif len(accept) == 1:
accept_node = nodes.strong(
text='Accepted content type: {}'.format(accept[0]))
else:
accept_node = nodes.strong(text='Accepted content types:')
node_accept_list = nodes.bullet_list()
accept_node += node_accept_list
for item in accept:
temp = nodes.list_item()
temp += nodes.inline(text=item)
node_accept_list += temp
node = rst2node(docstring)
DocFieldTransformer(self).transform_all(node)
if node is not None:
method_node += node
if attrs_node:
method_node += attrs_node
if accept_node:
method_node += accept_node
renderer = args['renderer']
if renderer == 'simplejson':
renderer = 'json'
response = nodes.paragraph()
response += nodes.strong(text='Response: %s' % renderer)
method_node += response
service_node += method_node
return service_node
def register_service_views(config, service):
"""Register the routes of the given service into the pyramid router.
:param config: the pyramid configuration object that will be populated.
:param service: the service object containing the definitions
"""
services = config.registry.setdefault('cornice_services', {})
services[service.path] = service
# keep track of the registered routes
registered_routes = []
# before doing anything else, register a view for the OPTIONS method
# if we need to
if service.cors_enabled and 'OPTIONS' not in service.defined_methods:
service.add_view('options', view=get_cors_preflight_view(service))
# register the fallback view, which takes care of returning good error
# messages to the user-agent
cors_validator = get_cors_validator(service)
cors_filter = get_cors_filter(service)
for method, view, args in service.definitions:
args = copy.deepcopy(args) # make a copy of the dict to not modify it
args['request_method'] = method
if service.cors_enabled:
args['validators'].insert(0, cors_validator)
args['filters'].append(cors_filter)
decorated_view = decorate_view(view, dict(args), method)
for item in ('filters', 'validators', 'schema', 'klass',
'error_handler') + CORS_PARAMETERS:
if item in args:
del args[item]
# if acl is present, then convert it to a "factory"
if 'acl' in args:
args["factory"] = make_route_factory(args.pop('acl'))
route_args = {}
if 'factory' in args:
route_args['factory'] = args.pop('factory')
# register the route name with the path if it's not already done
if service.path not in registered_routes:
config.add_route(service.path, service.path, **route_args)
config.add_view(view=get_fallback_view(service),
route_name=service.path)
registered_routes.append(service.path)
# loop on the accept fields: we need to build custom predicate if
# callables were passed
if 'accept' in args:
for accept in to_list(args.pop('accept', ())):
predicates = args.get('custom_predicates', [])
if callable(accept):
predicate_checker = functools.partial(match_accept_header,
accept)
predicates.append(predicate_checker)
args['custom_predicates'] = predicates
else:
# otherwise it means that it is a "standard" accept,
# so add it as such.
args['accept'] = accept
# We register multiple times the same view with different
# accept / custom_predicates arguments
config.add_view(view=decorated_view, route_name=service.path,
**args)
else:
# it is a simple view, we don't need to loop on the definitions
# and just add it one time.
config.add_view(view=decorated_view, route_name=service.path,
**args)
def _render_service(self, service):
service_id = "service-%d" % self.env.new_serialno('service')
service_node = nodes.section(ids=[service_id])
title = '%s service at %s' % (service.name.title(), service.path)
service_node += nodes.title(text=title)
if service.description is not None:
service_node += rst2node(trim(service.description))
for method, view, args in service.definitions:
if method == 'HEAD':
#Skip head - this is essentially duplicating the get docs.
continue
method_id = '%s-%s' % (service_id, method)
method_node = nodes.section(ids=[method_id])
method_node += nodes.title(text=method)
if is_string(view):
if 'klass' in args:
ob = args['klass']
view_ = getattr(ob, view.lower())
docstring = trim(view_.__doc__ or "") + '\n'
else:
docstring = trim(view.__doc__ or "") + '\n'
if 'schema' in args:
schema = args['schema']
attrs_node = nodes.inline()
for location in ('header', 'querystring', 'body'):
attributes = schema.get_attributes(location=location)
if attributes:
attrs_node += nodes.inline(
text='values in the %s' % location)
location_attrs = nodes.bullet_list()
for attr in attributes:
temp = nodes.list_item()
desc = "%s : " % attr.name
# Get attribute data-type
if hasattr(attr, 'type'):
attr_type = attr.type
elif hasattr(attr, 'typ'):
attr_type = attr.typ.__class__.__name__
desc += " %s, " % attr_type
if attr.required:
desc += "required "
else:
desc += "optional "
temp += nodes.inline(text=desc)
location_attrs += temp
attrs_node += location_attrs
method_node += attrs_node
for validator in args.get('validators', ()):
if validator.__doc__ is not None:
docstring += trim(validator.__doc__)
if 'accept' in args:
accept = to_list(args['accept'])
if callable(accept):
if accept.__doc__ is not None:
docstring += accept.__doc__.strip()
else:
accept_node = nodes.strong(text='Accepted content types:')
node_accept_list = nodes.bullet_list()
accept_node += node_accept_list
for item in accept:
temp = nodes.list_item()
temp += nodes.inline(text=item)
node_accept_list += temp
method_node += accept_node
node = rst2node(docstring)
DocFieldTransformer(self).transform_all(node)
if node is not None:
method_node += node
renderer = args['renderer']
if renderer == 'simplejson':
renderer = 'json'
response = nodes.paragraph()
response += nodes.strong(text='Response: %s' % renderer)
method_node += response
service_node += method_node
return service_node
def _render_service(self, service):
service_id = "service-%d" % self.env.new_serialno('service')
service_node = nodes.section(ids=[service_id])
title = '%s service at %s' % (service.name.title(), service.path)
service_node += nodes.title(text=title)
if service.description is not None:
service_node += rst2node(trim(service.description))
for method, view, args in service.definitions:
if method == 'HEAD':
#Skip head - this is essentially duplicating the get docs.
continue
method_id = '%s-%s' % (service_id, method)
method_node = nodes.section(ids=[method_id])
method_node += nodes.title(text=method)
if is_string(view):
if 'klass' in args:
ob = args['klass']
view_ = getattr(ob, view.lower())
docstring = trim(view_.__doc__ or "") + '\n'
else:
docstring = trim(view.__doc__ or "") + '\n'
if 'schema' in args:
schema = args['schema']
attrs_node = nodes.inline()
for location in ('header', 'querystring', 'body'):
attributes = schema.get_attributes(location=location)
if attributes:
attrs_node += nodes.inline(text='values in the %s' %
location)
location_attrs = nodes.bullet_list()
for attr in attributes:
temp = nodes.list_item()
desc = "%s : " % attr.name
# Get attribute data-type
if hasattr(attr, 'type'):
attr_type = attr.type
elif hasattr(attr, 'typ'):
attr_type = attr.typ.__class__.__name__
desc += " %s, " % attr_type
if attr.required:
desc += "required "
else:
desc += "optional "
temp += nodes.inline(text=desc)
location_attrs += temp
attrs_node += location_attrs
method_node += attrs_node
for validator in args.get('validators', ()):
if validator.__doc__ is not None:
docstring += trim(validator.__doc__)
if 'accept' in args:
accept = to_list(args['accept'])
if callable(accept):
if accept.__doc__ is not None:
docstring += accept.__doc__.strip()
else:
accept_node = nodes.strong(text='Accepted content types:')
node_accept_list = nodes.bullet_list()
accept_node += node_accept_list
for item in accept:
temp = nodes.list_item()
temp += nodes.inline(text=item)
node_accept_list += temp
method_node += accept_node
node = rst2node(docstring)
DocFieldTransformer(self).transform_all(node)
if node is not None:
method_node += node
renderer = args['renderer']
if renderer == 'simplejson':
renderer = 'json'
response = nodes.paragraph()
response += nodes.strong(text='Response: %s' % renderer)
method_node += response
service_node += method_node
return service_node
def results_swap_family_members(response):
#pointer_results = JsonPointer('/ops:world-patent-data/ops:biblio-search/ops:search-result/ops:publication-reference')
#entries = pointer_results.resolve(results)
publication_numbers = []
# DE, EP..B, WO, EP..A2, EP..A3, EP, US
priorities = [
{'filter': lambda patent: patent.country.startswith('DE') and not patent.kind.startswith('D1')},
{'filter': lambda patent: patent.country.startswith('EP') and patent.kind.startswith('B')},
{'filter': 'WO'},
{'filter': lambda patent: patent.country.startswith('EP') and patent.kind.startswith('A')},
{'filter': 'EP'},
{'filter': 'US'},
]
def match_filter(item, filter):
if callable(filter):
patent = split_patent_number(item)
outcome = filter(patent)
else:
outcome = item.startswith(filter)
return outcome
pointer_results = JsonPointer('/ops:world-patent-data/ops:biblio-search/ops:search-result/exchange-documents')
pointer_publication_reference = JsonPointer('/bibliographic-data/publication-reference/document-id')
#pointer_publication_reference = JsonPointer('/exchange-document/bibliographic-data/publication-reference/document-id')
# A.1 compute distinct list with unique families
family_representatives = {}
chunks = to_list(pointer_results.resolve(response))
all_results = []
for chunk in chunks:
#print 'chunk:', chunk
# Prepare list of document cycles
#chunk_results = to_list(pointer_publication_reference.resolve(chunk))
cycles = to_list(chunk['exchange-document'])
# Publication number of first cycle in EPODOC format
representation = cycles[0]
pubref = pointer_publication_reference.resolve(representation)
representation_pubref_epodoc, _ = _get_document_number_date(pubref, 'epodoc')
# All publication numbers in DOCDB format
representation_pubrefs_docdb = []
for cycle in cycles:
pubref = pointer_publication_reference.resolve(cycle)
representation_pubref_docdb, _ = _get_document_number_date(pubref, 'docdb')
representation_pubrefs_docdb.append(representation_pubref_docdb)
# Debugging
#print 'representation_pubref_epodoc:', representation_pubref_epodoc
#print 'representation_pubrefs_docdb:', representation_pubrefs_docdb
# Fetch family members. When failing, use first cycle as representation.
try:
family_info = ops_family_members(representation_pubref_epodoc)
except:
log.warning('Failed to fetch family information for %s', representation_pubref_epodoc)
chunk['exchange-document'] = representation
request = get_current_request()
del request.errors[:]
continue
#members = family_info.publications_by_country()
#pprint(members)
# Find replacement from list of family members controlled by priority list.
for prio in priorities:
filter = prio['filter']
# Debugging
#print 'checking prio:', filter
if match_filter(representation_pubref_epodoc, filter):
break
bibdata = None
found = False
for member in family_info.items:
# Debugging
#print 'member:'; pprint(member)
member_pubnum = member['publication']['number-docdb']
if match_filter(member_pubnum, filter):
# Debugging
#print 'Filter matched for member:', member_pubnum
try:
bibdata = ops_biblio_documents(member_pubnum)
except:
#log.warning('Fetching bibliographic data failed for %s', member_pubnum)
request = get_current_request()
del request.errors[:]
continue
#pprint(bibdata)
if bibdata:
# TODO: Add marker that this document was swapped, display appropriately.
found = True
break
# Swap representation of document by appropriate family member
# and set a marker in the data structure containing the original
# document number(s).
if found:
representation = bibdata
#print 'representation:'; pprint(representation)
representation[0].setdefault('__meta__', {})
representation[0]['__meta__']['swapped'] = {
'canonical': representation_pubrefs_docdb[0],
'list': [representation_pubref_epodoc] + representation_pubrefs_docdb,
}
break
# TODO: Here, duplicate documents might be. Prune/deduplicate them.
# TODO: When choosing german family members (e.g. for EP666666), abstract is often missing.
# TODO: => Carry along from original representation.
"""
for result in cycles:
#pprint(result)
pubref = pointer_publication_reference.resolve(result)
#print entry, pubref
pubref_number, pubref_date = _get_document_number_date(pubref, 'docdb')
publication_numbers.append(pubref_number)
"""
chunk['exchange-document'] = representation
# Filter duplicates
seen = []
results = []
fields = ['@country', '@doc-number', '@kind', '@family-id']
for chunk in chunks:
# Prepare list of document cycles.
cycles = to_list(chunk['exchange-document'])
# Only look at first cycle slot.
doc = cycles[0]
# Compute unique document identifier.
ident = {}
for key in fields:
ident[key] = doc[key]
# Collect chunk if not seen yet.
if ident in seen:
continue
else:
seen.append(ident)
results.append(chunk)
# Overwrite reduced list of chunks in original DOM.
pointer_results.set(response, results)
return publication_numbers
