def test_historylogs_without_filters(self):
for document_type in ['article', 'journal']:
historylogs = json.loads(
open(
os.path.dirname(__file__) +
'/fixtures/historylogs_%s.json' % document_type).read())
mocker = Mocker()
databroker = mocker.mock()
databroker['historychanges_%s' % document_type].find(ANY).count()
mocker.result(historylogs['meta']['total'])
databroker['historychanges_%s' % document_type].find(ANY).skip(
ANY).limit(ANY).sort("date")
mocker.result(historylogs['objects'])
mocker.replay()
db = DataBroker(databroker)
result = db.historychanges(document_type)
# assert date filters are correct in meta
self.assertIn('meta', result.keys())
self.assertIn('filter', result['meta'].keys())
self.assertIn('date', result['meta']['filter'].keys())
self.assertEqual(['$lte', '$gt'],
result['meta']['filter']['date'].keys())
self.assertEqual(historylogs['meta']['total'],
result['meta']['total'])
self.assertIn('objects', result.keys())
self.assertEqual(historylogs['objects'], result['objects'])
self.assertEqual(result['objects'][0].keys(),
['date', 'code', 'event', 'collection'])
def test_log_changes_with_valid_params(self):
mocker = Mocker()
databroker = mocker.mock()
document_types = ['article', 'journal']
events = ['add', 'update', 'delete']
# db "insert" must be called len(document_types) * len(events) times
for document_type in document_types:
for event in events:
databroker['historychanges_%s' % document_type].insert(ANY)
mocker.result(123457890)
mocker.replay()
db = DataBroker(databroker)
for document_type in document_types:
for event in events:
log_data = {
'document_type': document_type,
'code': '123',
'collection': 'test_collection',
'event': event,
'date': datetime.now().isoformat(),
}
log_id = db._log_changes(**log_data)
self.assertEqual(log_id, 123457890)
def test_historylogs_without_filters(self):
for document_type in ['article', 'journal']:
historylogs = json.loads(
open(os.path.dirname(__file__) +
'/fixtures/historylogs_%s.json' % document_type).read()
)
mocker = Mocker()
databroker = mocker.mock()
databroker['historychanges_%s' % document_type].find(ANY).count()
mocker.result(historylogs['meta']['total'])
databroker['historychanges_%s' % document_type].find(ANY).skip(ANY).limit(ANY).sort("date")
mocker.result(historylogs['objects'])
mocker.replay()
db = DataBroker(databroker)
result = db.historychanges(document_type)
# assert date filters are correct in meta
self.assertIn('meta', result.keys())
self.assertIn('filter', result['meta'].keys())
self.assertIn('date', result['meta']['filter'].keys())
self.assertEqual(['$lte', '$gt'], result['meta']['filter']['date'].keys())
self.assertEqual(
historylogs['meta']['total'],
result['meta']['total']
)
self.assertIn('objects', result.keys())
self.assertEqual(historylogs['objects'], result['objects'])
self.assertEqual(
result['objects'][0].keys(),
['date', 'code', 'event', 'collection']
)
def test_log_changes_with_valid_params(self):
mocker = Mocker()
databroker = mocker.mock()
document_types = ['article', 'journal']
events = ['add', 'update', 'delete']
# db "insert" must be called len(document_types) * len(events) times
for document_type in document_types:
for event in events:
databroker['historychanges_%s' % document_type].insert(ANY)
mocker.result(123457890)
mocker.replay()
db = DataBroker(databroker)
for document_type in document_types:
for event in events:
log_data = {
'document_type': document_type,
'code': '123',
'collection': 'test_collection',
'event': event,
'date': datetime.now().isoformat(),
}
log_id = db._log_changes(**log_data)
self.assertEqual(log_id, 123457890)
def test_check_journal_meta(self):
db = DataBroker(None)
expected = db._check_journal_meta(self._raw_json['title'])
self.assertEqual(expected['code'], u'0034-8910')
self.assertEqual(expected['collection'], u'scl')
def test_check_journal_meta(self):
db = DataBroker(None)
expected = db._check_journal_meta(self._raw_json['title'])
self.assertEqual(expected['code'], [u'0034-8910'])
self.assertEqual(expected['collection'], u'scl')
def __init__(self):
config = utils.Configuration.from_env()
settings = dict(config.items())
db_dsn = os.environ.get('MONGODB_HOST', settings.get('mongo_uri', '127.0.0.1:27017'))
self._admintoken = os.environ.get('ADMIN_TOKEN', None) or settings['app:main'].get('admintoken', uuid.uuid4().hex)
db_client = get_dbconn(db_dsn)
self._databroker = DataBroker(db_client)
def test_check_article_meta(self):
db = DataBroker(None)
expected = db._check_article_meta(self._raw_json)
self.assertEqual(expected['code_issue'], u'0034-891020100004')
self.assertEqual(expected['code_title'], [u'0034-8910'])
self.assertEqual(expected['publication_year'], u'2010')
self.assertEqual(expected['collection'], u'scl')
def test_check_article_meta(self):
db = DataBroker(None)
expected = db._check_article_meta(self._raw_json)
self.assertEqual(expected['code_issue'], u'0034-891020100004')
self.assertEqual(expected['code_title'], [u'0034-8910'])
self.assertEqual(expected['publication_year'], u'2010')
self.assertEqual(expected['collection'], u'scl')
def test_get_article_available_code(self):
mocker = Mocker()
databroker = mocker.mock()
databroker['articles'].find_one(ANY)
mocker.result(self._raw_json)
mocker.replay()
db = DataBroker(databroker)
self.assertEqual(db.get_article('xx')['code'], 'S0034-89102010000400007')
def test_get_article_unavailable_code(self):
mocker = Mocker()
databroker = mocker.mock()
databroker['articles'].find_one(ANY)
mocker.result(None)
mocker.replay()
db = DataBroker(databroker)
self.assertEqual(db.get_article('xx'), None)
def test_exists_article_False(self):
mocker = Mocker()
databroker = mocker.mock()
databroker['articles'].find(ANY).count()
mocker.result(None)
mocker.replay()
db = DataBroker(databroker)
self.assertEqual(db.exists_article('xx'), False)
def test_exists_article_False(self):
mocker = Mocker()
databroker = mocker.mock()
databroker['articles'].find(ANY).count()
mocker.result(None)
mocker.replay()
db = DataBroker(databroker)
self.assertEqual(db.exists_article('xx'), False)
def __init__(self):
config = utils.Configuration.from_env()
settings = dict(config.items())
self._databroker = DataBroker.from_dsn(
settings['app:main']['mongo_uri'],
reuse_dbconn=True)
def __init__(self):
config = utils.Configuration.from_env()
settings = dict(config.items())
self._admintoken = os.environ.get('ADMIN_TOKEN', None) or settings['app:main'].get('admintoken', uuid.uuid4().hex)
self._databroker = DataBroker.from_dsn(
MONGODB_HOST or settings['app:main'].get('mongo_uri', 'mongodb://127.0.0.1:27017/articlemeta'),
reuse_dbconn=True
)
def __init__(self):
config = utils.Configuration.from_env()
settings = dict(config.items())
db_dsn = os.environ.get('MONGODB_HOST', settings.get('mongo_uri', '127.0.0.1:27017'))
self._admintoken = os.environ.get('ADMIN_TOKEN', None) or settings['app:main'].get('admintoken', uuid.uuid4().hex)
db_client = get_dbconn(db_dsn)
self._databroker = DataBroker(db_client)
def __init__(self):
config = utils.Configuration.from_env()
settings = dict(config.items())
self._databroker = DataBroker.from_dsn(
settings['app:main']['mongo_uri'], reuse_dbconn=True)
class Dispatcher(object):
def __init__(self):
config = utils.Configuration.from_env()
settings = dict(config.items())
db_dsn = os.environ.get('MONGODB_HOST',
settings.get('mongo_uri', '127.0.0.1:27017'))
self._admintoken = os.environ.get('ADMIN_TOKEN',
None) or settings['app:main'].get(
'admintoken',
uuid.uuid4().hex)
db_client = get_dbconn(db_dsn)
self._databroker = DataBroker(db_client)
def getInterfaceVersion(self):
return articlemeta_thrift.VERSION
def get_collection_identifiers(self):
try:
data = self._databroker.identifiers_collection()
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.identifiers_collection')
return [
articlemeta_thrift.collection(i['code'], i['acron'],
i.get('acron2', ''),
i.get('status', ''), i['domain'],
i['original_name'],
i['has_analytics'],
i.get('is_active', False),
i.get('type', '')) for i in data
]
def get_collection(self, code):
logger.debug('AM Thrift - get_collection(code=%s)' % code)
try:
data = self._databroker.get_collection(code)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.get_collection')
if not data:
raise articlemeta_thrift.ValueError(
'Value error: no collection %s' % code)
return articlemeta_thrift.collection(data['code'], data['acron'],
data.get('acron2', ''),
data.get('status',
''), data['domain'],
data['original_name'],
data['has_analytics'],
data.get('is_active', False),
data.get('type', ''))
def article_history_changes(self, collection, event, code, from_date,
until_date, limit, offset):
logger.debug(
'AM Thrift - article_history_changes('
'collection=%s,event=%s,code=%s,from_date=%s,'
'until_date=%s,limit=%s,offset=%s)' %
(collection, event, code, from_date, until_date, limit, offset))
from_date = from_date or '1500-01-01'
limit = limit or 1000
offset = offset or 0
try:
data = self._databroker.historychanges(document_type='article',
collection=collection,
event=event,
code=code,
from_date=from_date,
until_date=until_date,
limit=limit,
offset=offset)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.historychanges')
objs = [
articlemeta_thrift.event_document(code=i['code'],
collection=i['collection'],
event=i['event'],
date=i['date'])
for i in data['objects']
]
return objs
def issue_history_changes(self, collection, event, code, from_date,
until_date, limit, offset):
logger.debug(
'AM Thrift - issue_history_changes('
'collection=%s,event=%s,code=%s,from_date=%s,'
'until_date=%s,limit=%s,offset=%s)' %
(collection, event, code, from_date, until_date, limit, offset))
from_date = from_date or '1500-01-01'
limit = limit or 1000
offset = offset or 0
try:
data = self._databroker.historychanges(document_type='issue',
collection=collection,
event=event,
code=code,
from_date=from_date,
until_date=until_date,
limit=limit,
offset=offset)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.historychanges')
objs = [
articlemeta_thrift.event_issue(code=i['code'],
collection=i['collection'],
event=i['event'],
date=i['date'])
for i in data['objects']
]
return objs
def get_articles(self,
collection,
issn,
from_date,
until_date,
limit,
offset,
extra_filter=None,
replace_journal_metadata=False,
body=False):
logger.debug(
'AM Thrift - get_articles('
'collection=%s,issn=%s,from_date=%s,until_date=%s,limit=%s,'
'offset=%s,extra_filter=%s,replace_journal_metadata=%s,body=%s)' %
(collection, issn, from_date, until_date, limit, offset,
extra_filter, replace_journal_metadata, body))
from_date = from_date or '1500-01-01'
limit = limit or 100
offset = offset or 0
try:
data = self._databroker.get_articles_full(
collection=collection,
issn=issn,
from_date=from_date,
until_date=until_date,
limit=limit,
offset=offset,
extra_filter=extra_filter,
replace_journal_metadata=False,
body=False)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.get_articles')
return json.dumps(data)
def get_article_identifiers(self,
collection,
issn,
from_date,
until_date,
limit,
offset,
extra_filter=None):
logger.debug(
'AM Thrift - get_article_identifiers('
'collection=%s,issn=%s,from_date=%s,until_date=%s,limit=%s,'
'offset=%s,extra_filter=%s)' %
(collection, issn, from_date, until_date, limit, offset,
extra_filter))
from_date = from_date or '1500-01-01'
limit = limit or 1000
offset = offset or 0
try:
data = self._databroker.identifiers_article(
collection=collection,
issn=issn,
from_date=from_date,
until_date=until_date,
limit=limit,
offset=offset,
extra_filter=extra_filter)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.identifiers_article')
objs = [
articlemeta_thrift.article_identifiers(
code=i['code'],
collection=i['collection'],
processing_date=i['processing_date'],
aid=i.get('aid', ''),
doi=i.get('doi', '')) for i in data['objects']
]
return objs
def get_issues(self,
collection,
issn,
from_date,
until_date,
limit,
offset,
extra_filter=None):
logger.debug(
'AM Thrift - get_issues('
'collection=%s,issn=%s,from_date=%s,until_date=%s,limit=%s,'
'offset=%s,extra_filter=%s)' %
(collection, issn, from_date, until_date, limit, offset,
extra_filter))
from_date = from_date or '1500-01-01'
limit = limit or 100
offset = offset or 0
try:
data = self._databroker.get_issues_full(collection=collection,
issn=issn,
from_date=from_date,
until_date=until_date,
limit=limit,
offset=offset,
extra_filter=extra_filter)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.get_issues')
return json.dumps(data)
def get_issue_identifiers(self,
collection,
issn,
from_date,
until_date,
limit,
offset,
extra_filter=None):
logger.debug(
'AM Thrift - get_issue_identifiers('
'collection=%s,issn=%s,from_date=%s,until_date=%s,limit=%s,'
'offset=%s,extra_filter=%s)' %
(collection, issn, from_date, until_date, limit, offset,
extra_filter))
from_date = from_date or '1500-01-01'
limit = limit or 1000
offset = offset or 0
try:
data = self._databroker.identifiers_issue(
collection=collection,
issn=issn,
from_date=from_date,
until_date=until_date,
limit=limit,
offset=offset,
extra_filter=extra_filter)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.identifiers_issue')
objs = [
articlemeta_thrift.issue_identifiers(
code=i['code'],
collection=i['collection'],
processing_date=i['processing_date']) for i in data['objects']
]
return objs
def delete_journal(self, code, collection, admintoken):
logger.debug('AM Thrift - delete_journal(code=%s,collection=%s)' %
(code, collection))
if admintoken != self._admintoken:
raise articlemeta_thrift.Unauthorized(
'Unautorized Access: Invalid admin token')
try:
result = self._databroker.delete_journal(code,
collection=collection)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.delete_journal')
try:
return json.dumps(result)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.delete_journal')
def delete_issue(self, code, collection, admintoken):
logger.debug('AM Thrift - delete_issue(code=%s,collection%s)' %
(code, collection))
if admintoken != self._admintoken:
raise articlemeta_thrift.Unauthorized(
'Unautorized Access: Invalid admin token')
try:
result = self._databroker.delete_issue(code, collection=collection)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.delete_issue')
try:
return json.dumps(result)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.delete_journal')
def delete_article(self, code, collection, admintoken):
logger.debug('AM Thrift - delete_article(code=%s,collection%s)' %
(code, collection))
if admintoken != self._admintoken:
raise articlemeta_thrift.Unauthorized(
'Unautorized Access: Invalid admin token')
try:
result = self._databroker.delete_article(code,
collection=collection)
except Exception as e:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.delete_article %s' % str(e))
try:
return json.dumps(result)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.delete_journal')
def add_journal(self, metadata, admintoken):
logger.debug('AM Thrift - add_journal(metadata=%s)' % metadata)
if admintoken != self._admintoken:
raise articlemeta_thrift.Unauthorized(
'Unautorized Access: Invalid admin token')
jdata = None
try:
jdata = json.loads(metadata)
except:
raise articlemeta_thrift.ValueError(
'Value error: DataBroker.add_journal, Invalid JSON')
data = None
try:
data = self._databroker.add_journal(jdata)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.add_journal')
if data:
return json.dumps(data)
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.add_journal, Nondata inserted')
def add_article(self, metadata, admintoken):
logger.debug('AM Thrift - add_article(metadata=%s)' % metadata)
if admintoken != self._admintoken:
raise articlemeta_thrift.Unauthorized(
'Unautorized Access: Invalid admin token')
jdata = None
try:
jdata = json.loads(metadata)
except:
raise articlemeta_thrift.ValueError(
'Value error: DataBroker.add_article, Invalid JSON')
data = None
try:
data = self._databroker.add_article(jdata)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.add_article')
if data:
return json.dumps(data)
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.add_article, Nondata inserted')
def add_issue(self, metadata, admintoken):
logger.debug('AM Thrift - add_issue(metadata=%s)' % metadata)
if admintoken != self._admintoken:
raise articlemeta_thrift.Unauthorized(
'Unautorized Access: Invalid admin token')
jdata = None
try:
jdata = json.loads(metadata)
except:
raise articlemeta_thrift.ValueError(
'Value error: DataBroker.add_issue, Invalid JSON')
data = None
try:
data = self._databroker.add_issue(jdata)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.add_issue')
if data:
return json.dumps(data)
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.add_issue, Nondata inserted')
def get_article(self,
code,
collection,
replace_journal_metadata,
fmt,
body=False):
logger.debug(
'AM Thrift - get_article('
'code=%s,collection=%s,replace_journal_metadata=%s,fmt=%s,body=%s)'
% (code, collection, replace_journal_metadata, fmt, body))
try:
data = self._databroker.get_article(
code,
collection=collection,
replace_journal_metadata=replace_journal_metadata,
body=body)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.get_article')
if data:
if fmt == 'xmlwos':
return Export(data).pipeline_sci()
if fmt == 'xmldoaj':
return Export(data).pipeline_doaj()
if fmt == 'xmlrsps':
return Export(data).pipeline_rsps()
if fmt == 'xmlpubmed':
return Export(data).pipeline_pubmed()
if fmt == 'xmlcrossref':
return Export(data).pipeline_crossref()
if fmt == 'opac':
return json.dumps(Export(data).pipeline_opac())
return json.dumps(data)
def get_issue(self, code, collection, replace_journal_metadata):
logger.debug(
'AM Thrift - get_issue(code=%s,collection=%s,replace_journal_metadata=%s)'
% (code, collection, replace_journal_metadata))
try:
data = self._databroker.get_issue(
code,
collection=collection,
replace_journal_metadata=replace_journal_metadata)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.get_issue')
return json.dumps(data)
def journal_history_changes(self,
collection,
event=None,
code=None,
from_date=None,
until_date=None,
limit=None,
offset=None):
logger.debug('AM Thrift - journal_history_changes('
'collection=%s,event=%s,code=%s,from_date=%s,'
'until_date=%s,limit=%s,offset=%s)' % collection)
from_date = from_date or '1500-01-01'
limit = limit or 1000
offset = offset or 0
try:
data = self._databroker.historychanges(document_type='journal',
collection=collection,
event=event,
code=code,
from_date=from_date,
until_date=until_date,
limit=limit,
offset=offset)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.historychanges')
objs = [
articlemeta_thrift.event_journal(code=i['code'],
collection=i['collection'],
event=i['event'],
date=i['date'])
for i in data['objects']
]
return objs
def get_journal_identifiers(self,
collection,
issn=None,
limit=None,
offset=None,
extra_filter=None):
logger.debug(
'AM Thrift - get_journal_identifiers('
'collection=%s,issn=%s,limit=%s,offset=%s,extra_filter=%s)' %
(collection, issn, limit, offset, extra_filter))
limit = limit or 0
offset = offset or 0
try:
data = self._databroker.identifiers_journal(
collection=collection,
issn=issn,
limit=limit,
offset=offset,
extra_filter=extra_filter)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.identifiers_journal')
objs = [
articlemeta_thrift.journal_identifiers(
code=i['code'],
collection=i['collection'],
processing_date=i['processing_date']) for i in data['objects']
if i['code']
]
return objs
def get_journal(self, code, collection):
logger.debug('AM Thrift - get_journal(code=%s,collection=%s)' %
(code, collection))
try:
data = self._databroker.get_journal(collection=collection,
issn=code)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.get_journal')
if isinstance(data, list):
return json.dumps(data[0]) if data else ''
return json.dumps(data)
def set_doaj_id(self, code, collection, doaj_id, admintoken):
logger.debug(
'AM Thrift - set_doaj_id(code=%s,collection=%s,doaj_id=%s)' %
(code, collection, doaj_id))
if admintoken != self._admintoken:
raise articlemeta_thrift.Unauthorized(
'Unautorized Access: Invalid admin token')
try:
self._databroker.set_doaj_id(code, collection, doaj_id)
return True
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.set_doaj_id')
return False
def set_aid(self, code, collection, aid, admintoken):
logger.debug('AM Thrift - set_aid(code=%s,collection=%s,aid=%s)' %
(code, collection, aid))
if admintoken != self._admintoken:
raise articlemeta_thrift.Unauthorized(
'Unautorized Access: Invalid admin token')
try:
self._databroker.set_aid(code, collection, aid)
return True
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.set_aid')
return False
def exists_article(self, code, collection):
logger.debug('AM Thrift - exists_article(code=%s,collection=%s)' %
(code, collection))
try:
return self._databroker.exists_article(code, collection)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.exists_article')
return False
def exists_issue(self, code, collection):
logger.debug('AM Thrift - exists_issue(code=%s,collection=%s)' %
(code, collection))
try:
return self._databroker.exists_issue(code, collection)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.exists_issue')
return False
def exists_journal(self, code, collection):
logger.debug('AM Thrift - exists_journal(code=%s,collection=%s)' %
(code, collection))
try:
return self._databroker.exists_journal(code, collection)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.exists_journal')
return False
def get_issue_code_from_label(self, label, journal_code, collection):
logger.debug(
'AM Thrift - get_issue_code_from_label(label=%s,journal_code=%s,collection=%s)'
% (label, journal_code, collection))
try:
return self._databroker.get_issue_code_from_label(
label, journal_code, collection)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.get_issue_code_from_label')
class Dispatcher(object):
def __init__(self):
config = utils.Configuration.from_env()
settings = dict(config.items())
db_dsn = os.environ.get('MONGODB_HOST', settings.get('mongo_uri', '127.0.0.1:27017'))
self._admintoken = os.environ.get('ADMIN_TOKEN', None) or settings['app:main'].get('admintoken', uuid.uuid4().hex)
db_client = get_dbconn(db_dsn)
self._databroker = DataBroker(db_client)
def getInterfaceVersion(self):
return articlemeta_thrift.VERSION
def get_collection_identifiers(self):
try:
data = self._databroker.identifiers_collection()
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.identifiers_collection')
return [articlemeta_thrift.collection(i['code'], i['acron'],
i.get('acron2', ''), i.get('status', ''), i['domain'], i['original_name'],
i['has_analytics'], i.get('is_active', False), i.get('type', '')) for i in data]
def get_collection(self, code):
logger.debug('AM Thrift - get_collection(code=%s)' % code)
try:
data = self._databroker.get_collection(code)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.get_collection')
if not data:
raise articlemeta_thrift.ValueError(
'Value error: no collection %s' % code)
return articlemeta_thrift.collection(data['code'], data['acron'],
data.get('acron2', ''), data.get('status', ''),
data['domain'], data['original_name'],
data['has_analytics'],
data.get('is_active', False), data.get('type', ''))
def article_history_changes(self, collection, event, code, from_date,
until_date, limit, offset):
logger.debug(
'AM Thrift - article_history_changes('
'collection=%s,event=%s,code=%s,from_date=%s,'
'until_date=%s,limit=%s,offset=%s)'
% (collection, event, code, from_date, until_date, limit, offset)
)
from_date = from_date or '1500-01-01'
limit = limit or 1000
offset = offset or 0
try:
data = self._databroker.historychanges(document_type='article',
collection=collection,
event=event,
code=code,
from_date=from_date,
until_date=until_date,
limit=limit,
offset=offset)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.historychanges')
objs = [articlemeta_thrift.event_document(code=i['code'],
collection=i['collection'],
event=i['event'],
date=i['date'])
for i in data['objects']]
return objs
def issue_history_changes(self, collection, event, code, from_date,
until_date, limit, offset):
logger.debug(
'AM Thrift - issue_history_changes('
'collection=%s,event=%s,code=%s,from_date=%s,'
'until_date=%s,limit=%s,offset=%s)'
% (collection, event, code, from_date, until_date, limit, offset)
)
from_date = from_date or '1500-01-01'
limit = limit or 1000
offset = offset or 0
try:
data = self._databroker.historychanges(
document_type='issue',
collection=collection,
event=event,
code=code,
from_date=from_date,
until_date=until_date,
limit=limit,
offset=offset
)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.historychanges')
objs = [articlemeta_thrift.event_issue(
code=i['code'],
collection=i['collection'],
event=i['event'],
date=i['date']
) for i in data['objects']]
return objs
def get_articles(
self, collection, issn, from_date, until_date,
limit, offset, extra_filter=None, replace_journal_metadata=False,
body=False
):
logger.debug(
'AM Thrift - get_articles('
'collection=%s,issn=%s,from_date=%s,until_date=%s,limit=%s,'
'offset=%s,extra_filter=%s,replace_journal_metadata=%s,body=%s)'
% (collection, issn, from_date, until_date, limit, offset,
extra_filter, replace_journal_metadata, body)
)
from_date = from_date or '1500-01-01'
limit = limit or 100
offset = offset or 0
try:
data = self._databroker.get_articles_full(
collection=collection,
issn=issn,
from_date=from_date,
until_date=until_date,
limit=limit,
offset=offset,
extra_filter=extra_filter,
replace_journal_metadata=False,
body=False
)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.get_articles')
return json.dumps(data)
def get_article_identifiers(self, collection, issn, from_date, until_date,
limit, offset, extra_filter=None):
logger.debug(
'AM Thrift - get_article_identifiers('
'collection=%s,issn=%s,from_date=%s,until_date=%s,limit=%s,'
'offset=%s,extra_filter=%s)'
% (collection, issn, from_date, until_date, limit, offset, extra_filter)
)
from_date = from_date or '1500-01-01'
limit = limit or 1000
offset = offset or 0
try:
data = self._databroker.identifiers_article(
collection=collection,
issn=issn,
from_date=from_date,
until_date=until_date,
limit=limit,
offset=offset,
extra_filter=extra_filter
)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.identifiers_article')
objs = [articlemeta_thrift.article_identifiers(
code=i['code'],
collection=i['collection'],
processing_date=i['processing_date'],
aid=i.get('aid', ''),
doi=i.get('doi', '')) for i in data['objects']]
return objs
def get_issues(
self, collection, issn, from_date, until_date,
limit, offset, extra_filter=None
):
logger.debug(
'AM Thrift - get_issues('
'collection=%s,issn=%s,from_date=%s,until_date=%s,limit=%s,'
'offset=%s,extra_filter=%s)'
% (collection, issn, from_date, until_date, limit, offset, extra_filter)
)
from_date = from_date or '1500-01-01'
limit = limit or 100
offset = offset or 0
try:
data = self._databroker.get_issues_full(
collection=collection,
issn=issn,
from_date=from_date,
until_date=until_date,
limit=limit,
offset=offset,
extra_filter=extra_filter
)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.get_issues')
return json.dumps(data)
def get_issue_identifiers(self, collection, issn, from_date, until_date,
limit, offset, extra_filter=None):
logger.debug(
'AM Thrift - get_issue_identifiers('
'collection=%s,issn=%s,from_date=%s,until_date=%s,limit=%s,'
'offset=%s,extra_filter=%s)'
% (collection, issn, from_date, until_date, limit, offset, extra_filter)
)
from_date = from_date or '1500-01-01'
limit = limit or 1000
offset = offset or 0
try:
data = self._databroker.identifiers_issue(
collection=collection,
issn=issn,
from_date=from_date,
until_date=until_date,
limit=limit,
offset=offset,
extra_filter=extra_filter
)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.identifiers_issue')
objs = [articlemeta_thrift.issue_identifiers(
code=i['code'],
collection=i['collection'],
processing_date=i['processing_date']) for i in data['objects']]
return objs
def delete_journal(self, code, collection, admintoken):
logger.debug(
'AM Thrift - delete_journal(code=%s,collection=%s)'
% (code, collection)
)
if admintoken != self._admintoken:
raise articlemeta_thrift.Unauthorized(
'Unautorized Access: Invalid admin token')
try:
result = self._databroker.delete_journal(code, collection=collection)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.delete_journal')
try:
return json.dumps(result)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.delete_journal')
def delete_issue(self, code, collection, admintoken):
logger.debug(
'AM Thrift - delete_issue(code=%s,collection%s)'
% (code, collection)
)
if admintoken != self._admintoken:
raise articlemeta_thrift.Unauthorized(
'Unautorized Access: Invalid admin token')
try:
result = self._databroker.delete_issue(code, collection=collection)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.delete_issue')
try:
return json.dumps(result)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.delete_journal')
def delete_article(self, code, collection, admintoken):
logger.debug(
'AM Thrift - delete_article(code=%s,collection%s)'
% (code, collection)
)
if admintoken != self._admintoken:
raise articlemeta_thrift.Unauthorized(
'Unautorized Access: Invalid admin token')
try:
result = self._databroker.delete_article(code, collection=collection)
except Exception as e:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.delete_article %s' % str(e))
try:
return json.dumps(result)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.delete_journal')
def add_journal(self, metadata, admintoken):
logger.debug(
'AM Thrift - add_journal(metadata=%s)' % metadata
)
if admintoken != self._admintoken:
raise articlemeta_thrift.Unauthorized(
'Unautorized Access: Invalid admin token')
jdata = None
try:
jdata = json.loads(metadata)
except:
raise articlemeta_thrift.ValueError(
'Value error: DataBroker.add_journal, Invalid JSON')
data = None
try:
data = self._databroker.add_journal(jdata)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.add_journal')
if data:
return json.dumps(data)
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.add_journal, Nondata inserted')
def add_article(self, metadata, admintoken):
logger.debug(
'AM Thrift - add_article(metadata=%s)' % metadata
)
if admintoken != self._admintoken:
raise articlemeta_thrift.Unauthorized(
'Unautorized Access: Invalid admin token')
jdata = None
try:
jdata = json.loads(metadata)
except:
raise articlemeta_thrift.ValueError(
'Value error: DataBroker.add_article, Invalid JSON')
data = None
try:
data = self._databroker.add_article(jdata)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.add_article')
if data:
return json.dumps(data)
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.add_article, Nondata inserted')
def add_issue(self, metadata, admintoken):
logger.debug(
'AM Thrift - add_issue(metadata=%s)' % metadata
)
if admintoken != self._admintoken:
raise articlemeta_thrift.Unauthorized(
'Unautorized Access: Invalid admin token')
jdata = None
try:
jdata = json.loads(metadata)
except:
raise articlemeta_thrift.ValueError(
'Value error: DataBroker.add_issue, Invalid JSON')
data = None
try:
data = self._databroker.add_issue(jdata)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.add_issue')
if data:
return json.dumps(data)
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.add_issue, Nondata inserted')
def get_article(self, code, collection, replace_journal_metadata, fmt, body=False):
logger.debug(
'AM Thrift - get_article('
'code=%s,collection=%s,replace_journal_metadata=%s,fmt=%s,body=%s)'
% (code, collection, replace_journal_metadata, fmt, body)
)
try:
data = self._databroker.get_article(
code,
collection=collection,
replace_journal_metadata=replace_journal_metadata,
body=body
)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.get_article')
if data:
if fmt == 'xmlwos':
return Export(data).pipeline_sci()
if fmt == 'xmldoaj':
return Export(data).pipeline_doaj()
if fmt == 'xmlrsps':
return Export(data).pipeline_rsps()
if fmt == 'xmlpubmed':
return Export(data).pipeline_pubmed()
if fmt == 'xmlcrossref':
return Export(data).pipeline_crossref()
if fmt == 'opac':
return json.dumps(Export(data).pipeline_opac())
return json.dumps(data)
def get_issue(self, code, collection, replace_journal_metadata):
logger.debug(
'AM Thrift - get_issue(code=%s,collection=%s,replace_journal_metadata=%s)'
% (code, collection, replace_journal_metadata)
)
try:
data = self._databroker.get_issue(
code,
collection=collection,
replace_journal_metadata=replace_journal_metadata
)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.get_issue')
return json.dumps(data)
def journal_history_changes(self, collection, event=None, code=None, from_date=None,
until_date=None, limit=None, offset=None):
logger.debug(
'AM Thrift - journal_history_changes('
'collection=%s,event=%s,code=%s,from_date=%s,'
'until_date=%s,limit=%s,offset=%s)'
% collection
)
from_date = from_date or '1500-01-01'
limit = limit or 1000
offset = offset or 0
try:
data = self._databroker.historychanges(document_type='journal',
collection=collection,
event=event,
code=code,
from_date=from_date,
until_date=until_date,
limit=limit,
offset=offset)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.historychanges')
objs = [articlemeta_thrift.event_journal(code=i['code'],
collection=i['collection'],
event=i['event'],
date=i['date'])
for i in data['objects']]
return objs
def get_journal_identifiers(self, collection, issn=None, limit=None, offset=None, extra_filter=None):
logger.debug(
'AM Thrift - get_journal_identifiers('
'collection=%s,issn=%s,limit=%s,offset=%s,extra_filter=%s)'
% (collection, issn, limit, offset, extra_filter)
)
limit = limit or 0
offset = offset or 0
try:
data = self._databroker.identifiers_journal(collection=collection,
issn=issn,
limit=limit,
offset=offset,
extra_filter=extra_filter)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.identifiers_journal')
objs = [
articlemeta_thrift.journal_identifiers(
code=i['code'],
collection=i['collection'],
processing_date=i['processing_date']) for i in data['objects'] if i['code']
]
return objs
def get_journal(self, code, collection):
logger.debug(
'AM Thrift - get_journal(code=%s,collection=%s)' % (code, collection)
)
try:
data = self._databroker.get_journal(collection=collection,
issn=code)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.get_journal')
if isinstance(data, list):
return json.dumps(data[0]) if data else ''
return json.dumps(data)
def set_doaj_id(self, code, collection, doaj_id, admintoken):
logger.debug(
'AM Thrift - set_doaj_id(code=%s,collection=%s,doaj_id=%s)'
% (code, collection, doaj_id)
)
if admintoken != self._admintoken:
raise articlemeta_thrift.Unauthorized(
'Unautorized Access: Invalid admin token')
try:
self._databroker.set_doaj_id(code, collection, doaj_id)
return True
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.set_doaj_id')
return False
def set_aid(self, code, collection, aid, admintoken):
logger.debug(
'AM Thrift - set_aid(code=%s,collection=%s,aid=%s)'
% (code, collection, aid)
)
if admintoken != self._admintoken:
raise articlemeta_thrift.Unauthorized(
'Unautorized Access: Invalid admin token')
try:
self._databroker.set_aid(code, collection, aid)
return True
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.set_aid')
return False
def exists_article(self, code, collection):
logger.debug(
'AM Thrift - exists_article(code=%s,collection=%s)' % (code, collection)
)
try:
return self._databroker.exists_article(code, collection)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.exists_article')
return False
def exists_issue(self, code, collection):
logger.debug(
'AM Thrift - exists_issue(code=%s,collection=%s)' % (code, collection)
)
try:
return self._databroker.exists_issue(code, collection)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.exists_issue')
return False
def exists_journal(self, code, collection):
logger.debug(
'AM Thrift - exists_journal(code=%s,collection=%s)' % (code, collection)
)
try:
return self._databroker.exists_journal(code, collection)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.exists_journal')
return False
def get_issue_code_from_label(self, label, journal_code, collection):
logger.debug(
'AM Thrift - get_issue_code_from_label(label=%s,journal_code=%s,collection=%s)'
% (label, journal_code, collection)
)
try:
return self._databroker.get_issue_code_from_label(label,
journal_code, collection)
except:
raise articlemeta_thrift.ServerError(
'Server error: DataBroker.get_issue_code_from_label')
