def __init__(self, app_name, *args, **kwargs):
ADSCelery.__init__(self, app_name, *args, **kwargs)
# this is used for bulk/efficient updates to metrics db
self._metrics_engine = self._metrics_session = None
if self._config.get('METRICS_SQLALCHEMY_URL', None):
self._metrics_engine = create_engine(
self._config.get('METRICS_SQLALCHEMY_URL', 'sqlite:///'),
echo=self._config.get('SQLALCHEMY_ECHO', False))
_msession_factory = sessionmaker()
self._metrics_session = scoped_session(_msession_factory)
self._metrics_session.configure(bind=self._metrics_engine)
MetricsBase.metadata.bind = self._metrics_engine
self._metrics_table = Table('metrics', MetricsBase.metadata)
register_after_fork(self._metrics_engine,
self._metrics_engine.dispose)
self._metrics_table_insert = self._metrics_table.insert() \
.values({
'an_refereed_citations': bindparam('an_refereed_citations', required=False),
'an_citations': bindparam('an_citations', required=False),
'author_num': bindparam('author_num', required=False),
'bibcode': bindparam('bibcode'),
'citations': bindparam('citations', required=False),
'citation_num': bindparam('citation_num', required=False),
'downloads': bindparam('downloads', required=False),
'reads': bindparam('reads', required=False),
'refereed': bindparam('refereed', required=False, value=False),
'refereed_citations': bindparam('refereed_citations', required=False),
'refereed_citation_num': bindparam('refereed_citation_num', required=False),
'reference_num': bindparam('reference_num', required=False),
'rn_citations': bindparam('rn_citations', required=False),
'rn_citation_data': bindparam('rn_citation_data', required=False),
})
self._metrics_table_update = self._metrics_table.update() \
.where(self._metrics_table.c.bibcode == bindparam('bibcode')) \
.values({
'an_refereed_citations': bindparam('an_refereed_citations', required=False),
'an_citations': bindparam('an_citations', required=False),
'author_num': bindparam('author_num', required=False),
'bibcode': bindparam('bibcode'),
'citations': bindparam('citations', required=False),
'citation_num': bindparam('citation_num', required=False),
'downloads': bindparam('downloads', required=False),
'reads': bindparam('reads', required=False),
'refereed': bindparam('refereed', required=False, value=False),
'refereed_citations': bindparam('refereed_citations', required=False),
'refereed_citation_num': bindparam('refereed_citation_num', required=False),
'reference_num': bindparam('reference_num', required=False),
'rn_citations': bindparam('rn_citations', required=False),
'rn_citation_data': bindparam('rn_citation_data', required=False),
})
# solr tweaks are read in from json files
# when bibcodes match, the dict is used to update a solr doc
self.tweak_dir = './tweak_files/'
self.tweaks = {} # key=bibcode, value=dict of solr field, value
self.load_tweak_files()
self.update_timestamps = self._config.get('UPDATE_TIMESTAMPS', True)
def __init__(self, *args, **kwargs):
ADSCelery.__init__(self, *args, **kwargs)
self._client = requests.Session()
self._client.headers.update(
{'Authorization': 'Bearer:{}'.format(self.conf['API_TOKEN'])})
self._err_counter = 0
self._tmpls = {}
def test_config(self):
app = ADSCelery('test', local_config={'FOO': ['bar', {}]})
self.assertEqual(app._config['FOO'], ['bar', {}])
self.assertEqual(app.conf['FOO'], ['bar', {}])
self.assertEqual(app.conf['CELERY_RESULT_SERIALIZER'], 'adsmsg')
self.assertFalse(app._config.get('CELERY_RESULT_SERIALIZER', None))
def __init__(self, app_name, *args, **kwargs):
ADSCelery.__init__(self, app_name, *args, **kwargs)
# this is used for bulk/efficient updates to metrics db
self._metrics_engine = self._metrics_session = None
if self._config.get('METRICS_SQLALCHEMY_URL', None):
self._metrics_engine = create_engine(self._config.get('METRICS_SQLALCHEMY_URL', 'sqlite:///'),
echo=self._config.get('SQLALCHEMY_ECHO', False))
MetricsModel.metadata.bind = self._metrics_engine
self._metrics_table = Table('metrics', MetricsModel.metadata)
self._metrics_conn = self._metrics_engine.connect()
self._metrics_table_insert = self._metrics_table.insert() \
.values({
'an_refereed_citations': bindparam('an_refereed_citations', required=False),
'an_citations': bindparam('an_citations', required=False),
'author_num': bindparam('author_num', required=False),
'bibcode': bindparam('bibcode'),
'citations': bindparam('citations', required=False),
'citation_num': bindparam('citation_num', required=False),
'downloads': bindparam('downloads', required=False),
'reads': bindparam('reads', required=False),
'refereed': bindparam('refereed', required=False),
'refereed_citations': bindparam('refereed_citations', required=False),
'refereed_citation_num': bindparam('refereed_citation_num', required=False),
'reference_num': bindparam('reference_num', required=False),
'rn_citations': bindparam('rn_citations', required=False),
'rn_citation_data': bindparam('rn_citation_data', required=False),
})
self._metrics_table_update = self._metrics_table.update() \
.where(self._metrics_table.c.bibcode == bindparam('bibcode')) \
.values({
'an_refereed_citations': bindparam('an_refereed_citations', required=False),
'an_citations': bindparam('an_citations', required=False),
'author_num': bindparam('author_num', required=False),
'bibcode': bindparam('bibcode'),
'citations': bindparam('citations', required=False),
'citation_num': bindparam('citation_num', required=False),
'downloads': bindparam('downloads', required=False),
'reads': bindparam('reads', required=False),
'refereed': bindparam('refereed', required=False),
'refereed_citations': bindparam('refereed_citations', required=False),
'refereed_citation_num': bindparam('refereed_citation_num', required=False),
'reference_num': bindparam('reference_num', required=False),
'rn_citations': bindparam('rn_citations', required=False),
'rn_citation_data': bindparam('rn_citation_data', required=False),
})
def __init__(self, app_name, *args, **kwargs):
ADSCelery.__init__(self, app_name, *args, **kwargs)
# this is used for bulk/efficient updates to metrics db
self._metrics_engine = self._metrics_session = None
if self._config.get('METRICS_SQLALCHEMY_URL', None):
self._metrics_engine = create_engine(
self._config.get('METRICS_SQLALCHEMY_URL', 'sqlite:///'),
echo=self._config.get('SQLALCHEMY_ECHO', False))
_msession_factory = sessionmaker()
self._metrics_session = scoped_session(_msession_factory)
self._metrics_session.configure(bind=self._metrics_engine)
MetricsBase.metadata.bind = self._metrics_engine
self._metrics_table = Table('metrics',
MetricsBase.metadata,
autoload=True,
autoload_with=self._metrics_engine)
register_after_fork(self._metrics_engine,
self._metrics_engine.dispose)
insert_columns = {
'an_refereed_citations':
bindparam('an_refereed_citations', required=False),
'an_citations':
bindparam('an_citations', required=False),
'author_num':
bindparam('author_num', required=False),
'bibcode':
bindparam('bibcode'),
'citations':
bindparam('citations', required=False),
'citation_num':
bindparam('citation_num', required=False),
'downloads':
bindparam('downloads', required=False),
'reads':
bindparam('reads', required=False),
'refereed':
bindparam('refereed', required=False, value=False),
'refereed_citations':
bindparam('refereed_citations', required=False),
'refereed_citation_num':
bindparam('refereed_citation_num', required=False),
'reference_num':
bindparam('reference_num', required=False),
'rn_citations':
bindparam('rn_citations', required=False),
'rn_citation_data':
bindparam('rn_citation_data', required=False),
}
self._metrics_table_upsert = insert(MetricsModel).values(
insert_columns)
# on insert conflict we specify which columns update
update_columns = {
'an_refereed_citations':
getattr(self._metrics_table_upsert.excluded,
'an_refereed_citations'),
'an_citations':
getattr(self._metrics_table_upsert.excluded, 'an_citations'),
'author_num':
getattr(self._metrics_table_upsert.excluded, 'author_num'),
'citations':
getattr(self._metrics_table_upsert.excluded, 'citations'),
'citation_num':
getattr(self._metrics_table_upsert.excluded, 'citation_num'),
'downloads':
getattr(self._metrics_table_upsert.excluded, 'downloads'),
'reads':
getattr(self._metrics_table_upsert.excluded, 'reads'),
'refereed':
getattr(self._metrics_table_upsert.excluded, 'refereed'),
'refereed_citations':
getattr(self._metrics_table_upsert.excluded,
'refereed_citations'),
'refereed_citation_num':
getattr(self._metrics_table_upsert.excluded,
'refereed_citation_num'),
'reference_num':
getattr(self._metrics_table_upsert.excluded, 'reference_num'),
'rn_citations':
getattr(self._metrics_table_upsert.excluded, 'rn_citations'),
'rn_citation_data':
getattr(self._metrics_table_upsert.excluded,
'rn_citation_data')
}
self._metrics_table_upsert = self._metrics_table_upsert.on_conflict_do_update(
index_elements=['bibcode'], set_=update_columns)
def __init__(self, app_name, *args, **kwargs):
ADSCelery.__init__(self, app_name, *args, **kwargs)