def cleanPluginDir(directory):
logger.debug('Cleaning path: {}'.format(directory))
if os.path.isdir(directory):
# Clear up directory
for doomed in os.listdir(directory):
if fnmatch.fnmatch(doomed, '[!_]*.py[co]'):
os.unlink(os.path.join(directory, doomed))
def __init__(cls, name, bases, attrs):
# Keep track of subclasses of Extension
if not attrs.get(
'__module__',
'').startswith('utopia.') and not cls.__name__.startswith('_'):
cls.__extensions[cls._typeName()] = cls
cls.__uuid__ = uuid.uuid4().urn
logger.debug(' Found {}'.format(cls))
# Give this class's module the name of its loaded plugin
if len(inspect.stack()) >= 5:
inspect.getmodule(cls).__dict__['__plugin__'] = inspect.stack(
)[-5][0].f_globals['__name__']
def loadPlugin(path):
# Split the path up
directory, filename = os.path.split(path)
basename, ext = os.path.splitext(filename)
mod_name = 'plugin_{}_{}'.format(
str(uuid.uuid4()).replace('-', ''), basename)
# Only python files and zips are acceptable
is_py_file = ext == '.py' and not os.path.isdir(path)
is_zip_file = ext == '.zip' and not os.path.isdir(path)
is_zip_dir = ext == '.zip' and os.path.isdir(path)
#logger.info((path, ext, is_py_file, is_zip_file, is_zip_dir))
if is_py_file or is_zip_file or is_zip_dir:
# Any filename beginning with an underscore is ignored
if filename.startswith('_'):
logger.info('Ignoring underscored file: {}'.format(path))
return
# Attempt to load the plugin
logger.debug('Loading extensions from: {}'.format(path))
if is_zip_dir:
entry = os.path.join(path, 'python', basename + '.py')
if os.path.exists(entry):
try:
sys.path.append(os.path.join(directory, filename,
'python'))
mod = imp.load_source(mod_name, entry)
mod.__file__ = path
mod.__loader__ = _makeLoader(path)
except Exception as e:
logger.error('Failed to load %s', filename, exc_info=True)
finally:
sys.path.pop()
else:
logger.error('Cannot find entry point %s in zip directory: %s',
os.path.join('python', basename + '.py'), path)
elif is_py_file:
try:
mod = imp.load_source(mod_name, path)
mod.__file__ = path
except Exception as e:
logger.error('Failed to load %s', filename, exc_info=True)
elif is_zip_file:
try:
importer = zipimport.zipimporter(os.path.join(path, 'python'))
mod = importer.load_module(mod_name)
except Exception as e:
logger.error('Failed to load %s', filename, exc_info=True)
from lxml import etree
from utopia.log import logger
try:
import spineapi
def scrape(document):
'''Look for a non-horizontal arXiv ID on the front page (the standard
layout of ArXiv articles has the ID printed up the left-hand side.'''
regex = r'arXiv:([\w.-]+/\d{7}|\d{4}\.\d{4,})(v\d+)+'
for match in document.search(regex, spineapi.RegExp):
if match.begin().lineArea()[1] > 0:
return match.text()[6:]
except ImportError:
logger.debug('spineapi not imported: document scraping of ArXiv IDs will be unavailable')
def url(arxivid):
'''Given an ArXiv ID, turn it into a link.'''
return 'http://arxiv.org/abs/{0}'.format(urllib.quote_plus(arxivid))
def fetchXML(arxivid):
'''Given an ArXiv ID, find its representative XML document.'''
params = urllib.urlencode({'id_list': arxivid, 'start': '0', 'max_results': '1'})
url = 'http://export.arxiv.org/api/query?{0}'.format(params)
return urllib2.urlopen(url, timeout=8).read()
def parseXML(xml):
'''Parse an ArXiv XML document and return a citation dictionary.'''
def execute_resolver(self, citations, document = None):
# Strip off the 'plugin_XXXXX_' prefix from the module's name
plugin_name = '{0}.{1}'.format(self.__module__.split('_')[-1], self.__class__.__name__)
logger.debug('Resolving from {0}: {1}'.format(plugin_name, getattr(self, '__doc__', 'Unknown resolver')))
# If a source already exists that was generated by this plugin, skip it
if len([citation for citation in citations if pick(citation, PROVENANCE_PLUGIN_KEYSPEC, None) == plugin_name]) == 0:
try:
# Just start by trying to execute the resolver's resolve() method
resolved = self.resolve(citations, document)
if resolved is not None:
if isinstance(resolved, collections.Mapping):
resolved = [resolved]
sanitised = []
for citation in resolved:
# Are there any non-provenance keys?
if len([key for key in citation.keys() if key != PROVENANCE_KEYSPEC]) > 0:
# Add any resolver provenance to the result
if hasattr(self, 'provenance') and callable(self.provenance):
provenance = self.provenance()
if not isinstance(provenance, collections.Mapping):
provenance = {}
# Record the name of the plugin that generated the citation
set_by_keyspec(provenance, PROVENANCE_PLUGIN_KEY, plugin_name)
existing = pick(citation, PROVENANCE_KEYSPEC, {})
provenance.update(existing)
set_by_keyspec(citation, PROVENANCE_KEYSPEC, provenance)
# Record when the citation was generated locally
set_by_keyspec(citation, PROVENANCE_WHEN_KEYSPEC, datetime.datetime.now().isoformat())
sanitised.append(citation)
if len(sanitised) > 0:
return sanitised
except Exception as exception:
# Unwrap timeouts
if isinstance(exception, urllib2.URLError) and hasattr(exception, 'reason') and isinstance(exception.reason, socket.timeout):
exception = exception.reason
# Record the name of the plugin that generated the error
error = {}
set_by_keyspec(error, PROVENANCE_WHEN_KEYSPEC, datetime.datetime.now().isoformat())
set_by_keyspec(error, PROVENANCE_PLUGIN_KEYSPEC, plugin_name)
# Get the provenance from the resolver
if hasattr(self, 'provenance') and callable(self.provenance):
provenance = self.provenance()
if isinstance(provenance, collections.Mapping):
existing = pick(error, PROVENANCE_KEYSPEC, {})
existing.update(provenance)
set_by_keyspec(error, PROVENANCE_KEYSPEC, existing)
# Deal with the various known errors
if isinstance(exception, socket.timeout):
category = 'timeout'
message = 'The server did not respond'
elif isinstance(exception, urllib2.HTTPError):
category = 'server'
message = unicode(getattr(exception, 'reason', 'The server did not respond as expected'))
elif isinstance(exception, urllib2.URLError):
category = 'connection'
message = unicode(getattr(exception, 'reason', 'The server could not be found'))
else:
category = 'unknown'
message = 'An unexpected error occured'
error['error'] = {
'category': category,
'message': message,
}
logger.warning('Error in resolver ({0}): {1}'.format(pick(error, PROVENANCE_WHENCE_KEYSPEC, '?'), message), exc_info=True)
return [error]
# If the result isn't what we expect, return None
logger.debug(' -> null result')
return None
## Try to initialise the Utopia bridge.
##
## This code modifies the current Python environment to make it work when
## imported from within Utopia Document's Python interpreter. This will fail
## when the package is invoked on the command-line, and that's OK.
###############################################################################
from utopia.log import logger
try:
import utopiabridge as bridge
bridge.proxyUrllib2()
import utopiaauth as auth
except (AttributeError, ImportError):
import sys
logger.debug('Could not find package utopiabridge:', exc_info=True)
logger.info(
'This script does not seem to be running inside Utopia Documents. Some functions will not work.'
)
bridge = None
auth = None
###############################################################################
## Basic classes for use in Utopia
###############################################################################
import utopia.extension
class Configurator(utopia.extension.Extension):
pass