def __init__(self, name, next_step):
step.Step.__init__(self, name, provider.pool())
self.next_step = step.pair(next_step)
outdir = config.val(config.OPT_OUTDIR)
filename = config.val(config.OPT_REPORT_FILENAME)
self.filepath = os.path.join(outdir, filename)
self.handle = open(self.filepath, 'w')
self.logger.info('started recording in %s' % self.filepath)
def get_headers():
global _headers_cache
if _headers_cache is None:
_headers_cache = {
'User-Agent':
'alviscorpus/0.0.1 (https://github.com/Bibliome/alviscorpus; mailto: %s)'
% config.val(SECTION_CROSSREF, OPT_EMAIL),
'Accept':
'*/*',
'Host':
config.val(SECTION_CROSSREF, OPT_HOST)
}
return _headers_cache
def get_dir(self):
if self.doi is None:
raise Exception()
outdir = config.val(config.SECTION_DOCS, config.OPT_OUTDIR)
r = os.path.join(outdir, self.safe_doi())
if not os.path.exists(r):
os.makedirs(r)
return r
def get_proxy():
global _proxy_cache
if _proxy_cache is None:
if config.has(SECTION_CROSSREF, OPT_PROXY):
proxy_name = config.val(SECTION_CROSSREF, OPT_PROXY)
_proxy_cache = config.proxy(proxy_name)
else:
_proxy_cache = {}
return _proxy_cache
def pool():
global _pool
if _pool is None:
ctor = lambda self: ConstantDelayProvider.__init__(self, 0)
classes = [
type('PoolProvider%02d' % i, (ConstantDelayProvider, ),
{'__init__': ctor})
for i in range(int(config.val(config.OPT_THREAD_POOL)))
]
_pool = itertools.cycle(classes)
return next(_pool)
def build_url(self, doc, arg):
return 'https://' + config.val(
SECTION_CROSSREF, OPT_HOST) + self.build_url_suffix(doc, arg)
def build_url(self, doc, arg):
return 'https://' + config.val(
SECTION_EPMC, OPT_URL_BASE) + self.build_url_suffix(doc, arg)