def __init__(self, app_name, *args, **kwargs):
Celery.__init__(self, *args, **kwargs)
self._config = adsputils.load_config()
self._session = None
self._engine = None
self._app_name = app_name
self.logger = adsputils.setup_logging(app_name) #default logger
def __init__(self, fields, ignore_fields, new_fields):
self.fields = fields
self.ignore_fields = ignore_fields
self.new_fields = new_fields
self.logger = setup_logging('validate', 'INFO')
self.config = {}
self.config.update(load_config())
def __init__(self, file_):
self._file = file_
self.read_count = 0 # needed for logging
self.logger = setup_logging('AdsDataSqlSync', 'DEBUG')
self.logger.info('nonbib file ingest, file {}'.format(self._file))
self.config = {}
self.config.update(load_config())
self._iostream = open(file_, 'r')
def test_load_config(self):
with patch('adsputils.load_module') as load_module:
c = adsputils.load_config()
f = os.path.abspath(
os.path.join(os.path.dirname(inspect.getsourcefile(adsputils)),
'..'))
self.assertEquals((f + '/config.py', ),
load_module.call_args_list[0][0])
self.assertEquals((f + '/local_config.py', ),
load_module.call_args_list[1][0])
self.assertEqual(c['PROJ_HOME'], f)
with patch('adsputils.load_module') as load_module:
adsputils.load_config('/tmp')
self.assertEquals(('/tmp/config.py', ),
load_module.call_args_list[0][0])
self.assertEquals(('/tmp/local_config.py', ),
load_module.call_args_list[1][0])
def query_Kibana(
query='"+@log_group:\\"backoffice-orcid_pipeline-daemon\\" +@message:\\"Claim refused\\""',
n_days=7,
rows=5):
"""
Function to query Kibana for a given input query and return the response.
:param query: string query, same as would be entered in the Kibana search input (be sure to escape quotes and wrap
query in double quotes - see default query for formatting)
:param n_days: number of days backwards to query, starting now (=0 for all time)
:param rows: number of results to return. If you just need the total number of hits and not the results
themselves, can be small.
:return: JSON results
"""
config = {}
config.update(load_config())
# get start and end timestamps (in milliseconds since 1970 epoch)
now = datetime.datetime.now(tzutc())
epoch = datetime.datetime.utcfromtimestamp(0).replace(tzinfo=pytz.UTC)
end_time = (now - epoch).total_seconds() * 1000.
if n_days != 0:
start_time = (now - datetime.timedelta(days=n_days) -
epoch).total_seconds() * 1000.
else:
start_time = 0.
data = (
'{"index":["cwl-*"]}\n{"size":%.0f,"sort":[{"@timestamp":{"order":"desc","unmapped_type":"boolean"}}],'
% (rows) +
'"query":{"bool":{"must":[{"query_string":{"analyze_wildcard":true, "query":'
+ query + '}}, ' +
'{"range": {"@timestamp": {"gte": %.0f, "lte": %.0f,"format": "epoch_millis"}}}], "must_not":[]}}, '
% (start_time, end_time) + '"docvalue_fields":["@timestamp"]}\n\n')
header = {
'origin': 'https://pipeline-kibana.kube.adslabs.org',
'authorization': 'Basic ' + config['KIBANA_TOKEN'],
'content-type': 'application/x-ndjson',
'kbn-version': '5.5.2'
}
url = 'https://pipeline-kibana.kube.adslabs.org/_plugin/kibana/elasticsearch/_msearch'
# set to bypass SSL cert problem w/ Kibana
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
resp = app.client.post(url, data=data, headers=header, verify=False)
if resp.status_code == 200:
results = resp.json()
return results
logger.warn('For query {}, there was a network problem: {0}\n'.format(
query, resp))
return None
def init(cls):
if cls._initted is False:
config = load_config()
root_dir = config.get('INPUT_DATA_ROOT',
'./adsdata/tests/data1/config/')
cls._reference_network = _Network(root_dir +
data_files['reference']['path'])
cls._citation_network = _Network(root_dir +
data_files['citation']['path'])
cls._refereed_list = _Refereed(root_dir +
data_files['refereed']['path'])
cls._initted = True
def __init__(self, schema_='metrics'):
self.logger = setup_logging('AdsDataSqlSync', 'INFO')
self.schema = schema_
self.table = models.MetricsTable()
self.table.schema = self.schema
# used to buffer writes
self.upserts = []
self.tmp_update_buffer = []
self.tmp_count = 0
self.config = {}
self.config.update(load_config())
def create_app(app_name='adstb', local_config=None):
"""Builds and initializes the Celery application."""
conf = adsputils.load_config()
if local_config:
conf.update(local_config)
app = ADSTurboBeeCelery(app_name,
broker=conf.get('CELERY_BROKER', 'pyamqp://'),
include=conf.get('CELERY_INCLUDE',
['adstb.tasks']))
return app
def __init__(self, schema_='metrics'):
self.logger = setup_logging('AdsDataSqlSync', 'INFO')
self.schema = schema_
self.table = models.MetricsTable()
self.table.schema = self.schema
# used to buffer writes
self.upserts = []
self.tmp_update_buffer = []
self.tmp_count = 0
self.config = {}
self.config.update(load_config())
def __init__(self,
sqlachemy_url,
group_changes_in_chunks_of=1,
sqlalchemy_echo=False,
schema_prefix="citation_capture_",
force=False):
"""
Initializes the class and prepares DB connection.
:param sqlachemy_url: URL to connect to the DB.
:param group_changes_in_chunks_of: Number of citation changes to be
grouped when iterating.
:param sqlalchemy_echo: Print every SQL statement.
:param schema_prefix: Data is stored in schemas that correspond to a
prefix + file last access date.
:param force: If tables already exists in DB, drop them and re-ingest.
"""
self.engine = create_engine(sqlachemy_url, echo=sqlalchemy_echo)
self.connection = self.engine.connect()
self.session = sessionmaker(bind=self.engine)()
#
# - Use app logger:
#import logging
#self.logger = logging.getLogger('ads-citation-capture')
# - Or individual logger for this file:
from adsputils import setup_logging, load_config
proj_home = os.path.realpath(
os.path.join(os.path.dirname(__file__), '../'))
config = load_config(proj_home=proj_home)
self.logger = setup_logging(__name__,
proj_home=proj_home,
level=config.get('LOGGING_LEVEL', 'INFO'),
attach_stdout=config.get(
'LOG_STDOUT', False))
#
self.table_name = RawCitation.__tablename__
self.expanded_table_name = "expanded_" + self.table_name
self.recreated_previous_expanded_table_name = "recreated_previous_expanded_" + self.table_name
self.missing_previous_expanded_table_name = "not_processed_" + self.table_name
self.joint_table_name = CitationChanges.__tablename__
self.schema_prefix = schema_prefix
self.schema_name = None
self.previous_schema_name = None
self.input_refids_filename = None
self.group_changes_in_chunks_of = group_changes_in_chunks_of
self.offset = 0
self.n_changes = 0
self.force = force
self.last_modification_date = None
def __init__(self, fields, ignore_fields, new_fields):
self.fields = fields
self.ignore_fields = ignore_fields
self.new_fields = new_fields
# - Use app logger:
# import logging
# self.logger = logging.getLogger('master-pipeline')
# - Or individual logger for this file:
from adsputils import setup_logging, load_config
proj_home = os.path.realpath(os.path.join(os.path.dirname(__file__), '../'))
self.config = load_config(proj_home=proj_home)
self.logger = setup_logging(__name__, proj_home=proj_home,
level=self.config.get('LOGGING_LEVEL', 'INFO'),
attach_stdout=self.config.get('LOG_STDOUT', False))
def main():
global config
config.update(load_config())
global logger
logger = setup_logging('ADSData', config.get('LOG_LEVEL', 'INFO'))
parser = argparse.ArgumentParser(description='generate nonbib data')
ars = parser.parse_args()
load(config)
# compute metrics for a bibcode
compute_metrics('2012ApJS..199...26H')
# lots_of_metrics(config)
logger.info('end of program')
def __init__(self, schema_='nonbib'):
self.schema = schema_
self.meta = MetaData()
self.table = models.NonBibTable()
self.table.schema = self.schema
# - Use app logger:
#import logging
#logger = logging.getLogger('ads-data')
# - Or individual logger for this file:
proj_home = os.path.realpath(
os.path.join(os.path.dirname(__file__), '../'))
config = load_config(proj_home=proj_home)
self.logger = setup_logging(__name__,
proj_home=proj_home,
level=config.get('LOGGING_LEVEL', 'INFO'),
attach_stdout=config.get(
'LOG_STDOUT', False))
def __init__(self, file_):
self._file = file_
self.read_count = 0 # needed for logging
# - Use app logger:
#import logging
#logger = logging.getLogger('ads-data')
# - Or individual logger for this file:
proj_home = os.path.realpath(
os.path.join(os.path.dirname(__file__), '../'))
self.config = load_config(proj_home=proj_home)
self.logger = setup_logging(
__name__,
proj_home=proj_home,
level=self.config.get('LOGGING_LEVEL', 'INFO'),
attach_stdout=self.config.get('LOG_STDOUT', False))
self.logger.info('nonbib file ingest, file {}'.format(self._file))
self._iostream = open(file_, 'r')
def setUp(self):
unittest.TestCase.setUp(self)
config = load_config()
proj_home = os.path.abspath(
os.path.join(os.path.dirname(__file__), '../..'))
self.app = app.ADSMasterPipelineCelery('test', local_config=\
{
'SQLALCHEMY_URL': 'sqlite:///',
'METRICS_SQLALCHEMY_URL': 'postgresql://[email protected]:15678/test',
'SQLALCHEMY_ECHO': True,
'PROJ_HOME' : proj_home,
'TEST_DIR' : os.path.join(proj_home, 'adsmp/tests'),
})
Base.metadata.bind = self.app._session.get_bind()
Base.metadata.create_all()
MetricsBase.metadata.bind = self.app._metrics_engine
MetricsBase.metadata.create_all()
def setUp(self):
unittest.TestCase.setUp(self)
config = load_config()
proj_home = os.path.abspath(
os.path.join(os.path.dirname(__file__), '../..'))
self.app = app.ADSMasterPipelineCelery('test', local_config=\
{
'SQLALCHEMY_URL': config.get('METRICS_SQLALCHEMY_URL'),
'METRICS_SQLALCHEMY_URL': config.get('METRICS_SQLALCHEMY_URL'),
'SQLALCHEMY_ECHO': False,
'PROJ_HOME' : proj_home,
'TEST_DIR' : os.path.join(proj_home, 'adsmp/tests'),
})
Base.metadata.bind = self.app._session.get_bind()
Base.metadata.create_all()
MetricsBase.metadata.bind = self.app._session.get_bind()
MetricsBase.metadata.create_all()
def __init__(self, schema_='metrics'):
# - Use app logger:
#import logging
#logger = logging.getLogger('ads-data')
# - Or individual logger for this file:
proj_home = os.path.realpath(
os.path.join(os.path.dirname(__file__), '../'))
self.config = load_config(proj_home=proj_home)
self.logger = setup_logging(
__name__,
proj_home=proj_home,
level=self.config.get('LOGGING_LEVEL', 'INFO'),
attach_stdout=self.config.get('LOG_STDOUT', False))
self.schema = schema_
self.table = models.MetricsTable()
self.table.schema = self.schema
# used to buffer writes
self.upserts = []
self.tmp_update_buffer = []
self.tmp_count = 0
def __init__(self, app_name, *args, **kwargs):
"""
:param: app_name - string, name of the application (can be anything)
:keyword: local_config - dict, configuration that should be applied
over the default config (that is loaded from config.py and local_config.py)
"""
proj_home = None
if 'proj_home' in kwargs:
proj_home = kwargs.pop('proj_home')
self.config = load_config(extra_frames=1,
proj_home=proj_home,
app_name=app_name)
local_config = None
if 'local_config' in kwargs and kwargs['local_config']:
local_config = kwargs.pop('local_config')
self.config.update(local_config) #our config
if not proj_home:
proj_home = self.config.get('PROJ_HOME', None)
self.logger = setup_logging(
app_name,
proj_home=proj_home,
level=self.config.get('LOGGING_LEVEL', 'INFO'),
attach_stdout=self.config.get('LOG_STDOUT', False))
def setUp(self):
unittest.TestCase.setUp(self)
self.proj_home = tasks.app.conf['PROJ_HOME']
self._app = tasks.app
# Use a different database for unit tests since they will modify it
self.sqlalchemy_url = "{}_test".format(load_config().get(
'SQLALCHEMY_URL',
'postgres://postgres@localhost:5432/citation_capture_pipeline'))
config = {
"TESTING_MODE": False,
"CELERY_ALWAYS_EAGER": False,
"CELERY_EAGER_PROPAGATES_EXCEPTIONS": False,
"SQLALCHEMY_URL": self.sqlalchemy_url,
}
self.app = app.ADSCitationCaptureCelery('test',
proj_home=self.proj_home,
local_config=config)
tasks.app = self.app # monkey-patch the app object
self._init_mock_data()
try:
Base.metadata.create_all(bind=self.app._engine, checkfirst=True)
except:
# Database not empty!
raise
class NonbibFileReader(object):
"""reads nonbib column files
file reading/parsing is controlled by the file's properties dict in file_defs
every line must start with a bibcode
file must be sorted by bibcode
"""
bibcode_length = 19
config = load_config()
def __init__(self, filetype, file_info):
"""passed file type (e.g., canonical) and relevant part of file_defs"""
self.filetype = filetype
self.file_info = file_info
self.filename = self.config.get('INPUT_DATA_ROOT',
'./') + file_info['path']
self.logger = tasks.app.logger
self.read_count = 0 # used in logging
self.buffer = None # holds at most one line of text
self._iostream = open(self.filename, 'r', encoding='utf-8')
def __enter__(self, *args, **kwargs):
return self
def __exit__(self, *args, **kwargs):
self.close()
def __iter__(self):
return self
def next(self):
return self._iostream.next()
def close(self):
self._iostream.close()
del self._iostream
def _pushline(self, s):
"""the buffer is used when we read a line that is behond the desired bibcode
and we need to unread it"""
if self.buffer:
self.logger.error(
'error in file {}, {}, _pushline called when buffer was not empty. File line number: read line: {}, buffer: {}'
.format(self.filetype, self.filename, self.read_count, s,
self.buffer))
self.buffer = s
def _readline(self):
"""return the next valid line or empty string at eof
used to read all files"""
self.read_count += 1
if self.buffer:
line = self.buffer
self.buffer = None
return line
if self._iostream.closed:
return ''
line = self._iostream.readline()
while len(line) > 0 and len(line) < self.bibcode_length:
self.logger.error(
'error, invalid short line in readline {} filename: {} at line {}, line length less then length of bibcode, line: {}'
.format(self.filetype, self.filename, self.read_count, line))
self.read_count += 1
line = self._iostream.readline()
return line
def read_value_for(self, bibcode):
"""return the value from the file for the passed bibcode
returns default value if bibcoce is not in file
return value is a dict with the key of self.filetype
some files repeat a bibcode on consecutive lines to provide multiple values
other files do not repeat a bibcode and provide multiple values on a single line
other files (e.g., relevance/docmetrics.tab) have multiple values
some files have associated effects on values like property field
this reader handles all cases based on the file property dict
"""
# first, are we at eof?
current_line = self._readline()
if len(current_line) == 0:
# here if we are already at eof, bibcode isn't in file
return self._convert_value(self.file_info['default_value'])
# next, skip over lines in file until we:
# either find the passed bibcode or determine it isn't in the file
skip_count = 0
while len(current_line) != 0 and self._get_bibcode(
current_line) < bibcode:
current_line = self._readline()
skip_count = skip_count + 1
# at this point, we have either read to the desired bibcode
# or it doesn't exist and we read past it
if len(current_line
) == 0 or bibcode != self._get_bibcode(current_line):
# bibcode not in file
self._pushline(current_line)
return self._convert_value(self.file_info['default_value'])
if isinstance(self.file_info['default_value'], bool):
return self._convert_value(
True) # boolean files only hold bibcodes, all values are True
# at this point, we have the first line with the bibcode in it
# roll up possible other values on adjacent lines in file
value = []
value.append(self._get_rest(current_line))
current_line = self._readline()
while self.file_info.get(
'multiline', False) and (current_line is not None) and (
bibcode == self._get_bibcode(current_line)):
value.append(self._get_rest(current_line))
current_line = self._readline()
# at this point we have read beyond the desired bibcode, must back up
self._pushline(current_line)
# finally, convert raw input into something useful
return self._convert_value(value)
def _convert_value(self, value):
"""convert file string line to something more useful
return a dict with filetype as key and value converted
"""
if isinstance(value, str) and '\x00' in value:
# there should not be nulls in strings
self.logger.error(
'error string contained a null in file {} {}, line number: {}, value: {}'
.format(self.filetype, self.filename, self.read_count, value))
value = value.replace('\x00', '')
return_value = value
if isinstance(value, bool):
d = {self.filetype: return_value}
if 'extra_values' in self.file_info and value != self.file_info[
'default_value']:
d.update(self.file_info['extra_values'])
return {self.filetype: d}
elif (len(value) > 0 and '\t' in value[0]
and not self.file_info.get('tabs_to_spaces', False)):
# tab separator in string means we need to convert elements to array
z = []
for r in value:
x = r.split('\t')
if self.file_info.get('string_to_number', True):
# convert valid ints and floats to numeric representation
t = []
for y in x:
t.append(self._convert_scalar(y))
z.append(t)
return_value = z
if len(return_value) == 1:
return_value = return_value[0]
elif 'interleave' in self.file_info and value != self.file_info[
'default_value']:
# here on multi-line dict (e.g., associations)
# interleave data on successive lines e.g., merge first element in each array, second element, etc.
# since they also have subparts, these arrays will then put in dict with the cooresponding key
x = {}
for k in self.file_info['subparts']:
x[k] = []
for r in value:
# For instance, in associations 'r' should contain:
# URL title
# where title may contain spaces too
parts = r.split(' ', 1) # parts will contain [URL, title]
if len(parts) < len(self.file_info['subparts']):
self.logger.error(
'error in reader with interleave for {} file {}, incomplete value in line. value = {}, parts = {} at line'
.format(self.filetype, self.filename, value, parts,
self.read_count))
else:
for i, k in enumerate(self.file_info['subparts']):
v = parts[i].strip()
x[k].append(v)
return_value = x
elif (self.file_info.get('tabs_to_spaces', False)):
# files like simbad_objects have tabs that we simply convert to spaces
x = []
for a in value:
x.append(a.replace('\t', ' '))
return_value = x
elif (len(value) > 1):
x = []
for r in value:
x.append(r.replace('\t', ' ').strip())
return_value = x
# convert array to dict if needed
if 'subparts' in self.file_info and return_value != self.file_info[
'default_value'] and 'interleave' not in self.file_info:
if type(return_value[0]) is list:
x = []
for r in return_value:
x.append(self._convert_subparts(r))
else:
x = self._convert_subparts(return_value)
return_value = x
# are there extra_values to add to dict
if 'extra_values' in self.file_info:
self._add_extra_values(return_value)
return {self.filetype: return_value}
def _add_extra_values(self, current):
if current != self.file_info['default_value'] and type(
current) is dict:
current.update(self.file_info['extra_values'])
elif current != self.file_info['default_value'] and type(
current) is list:
# here with array of dicts, put extra_values in each dict
for x in current:
v = self.file_info['extra_values']
if type(v) is dict and type(x) is dict:
x.update(v)
else:
self.logger.error(
'serious error in reader._add_extra_values, non dict value, extra_values = {}, processing element = {}, passed current = {}'
.format(x, v, current))
def _convert_subparts(self, current):
d = {}
for i, k in enumerate(self.file_info['subparts']):
v = ''
if i < len(current):
v = current[i]
if type(k) is list:
# here if key is in a list by itself which means values should be in a list
k = k[0]
v = [v]
d[k] = v
return d
def _get_bibcode(self, s):
"""return the bibcode from the from of the line"""
if s is None:
return None
if len(s) < self.bibcode_length:
self.logger.error(
'error, invalid short line in file {} {} at line {}, line length less then length of bibcode, line = {}'
.format(self.filetype, self.filename, self.read_count, s))
return s
return s[:self.bibcode_length].strip()
def _get_rest(self, s):
"""return the text after the bibcode and first tab separator"""
if len(s) < self.bibcode_length + 1:
self.logger.error(
'error, in _get_rest with invalid short line in file {} {} at line {}, line length less then length of bibcode plus 1, line = {}'
.format(self.filetype, self.filename, self.read_count, s))
return ''
return s[self.bibcode_length + 1:].strip()
def _convert_scalar(self, s):
if s.isdigit():
return int(s)
try:
x = float(s)
return x
except ValueError:
return s.strip()
import os
import requests
import json
from adsputils import date2solrstamp
import sys
import time
from collections import OrderedDict
# ============================= INITIALIZATION ==================================== #
# - Use app logger:
#import logging
#logger = logging.getLogger('master-pipeline')
# - Or individual logger for this file:
from adsputils import setup_logging, load_config
proj_home = os.path.realpath(os.path.join(os.path.dirname(__file__), '../'))
config = load_config(proj_home=proj_home)
logger = setup_logging(__name__,
proj_home=proj_home,
level=config.get('LOGGING_LEVEL', 'INFO'),
attach_stdout=config.get('LOG_STDOUT', False))
# =============================== FUNCTIONS ======================================= #
def extract_metrics_pipeline(data, solrdoc):
citation = data.get('citations', [])
return dict(citation=citation)
def setUp(self):
self.config = {}
self.config.update(load_config())
self.assertEqual(-1, self.config['MAX_ROWS'],
'tests fail when local_config.py sets MAX_ROWS')
import collections
import copy
import datetime
import itertools
import os
import sys
import types
from aip.libs import enforce_schema, author_match
import adsputils as utils
_config = utils.load_config()
def mergeRecords(records):
completeRecords = []
e = enforce_schema.Enforcer() # TODO: no need to create new instances?
for r in copy.deepcopy(records):
r['text'] = Merger().mergeText(r['text'])
blocks = e.ensureList(r['metadata'])
#Multiply defined blocks need merging.
metadatablockCounter = collections.Counter([i['tempdata']['type'] for i in blocks])
needsMerging = dict([(k,[]) for k,v in metadatablockCounter.iteritems() if v>1])
completeMetadata = {}
#First pass: Add the singly defined blocks to the complete record
for b in blocks:
_type = b['tempdata']['type']
if _type not in needsMerging:
completeMetadata[_type] = b
else:
needsMerging[_type].append(b)
def setUp(self):
self.config = {}
self.config.update(load_config())
self.assertEqual(-1, self.config['MAX_ROWS'], 'tests fail when local_config.py sets MAX_ROWS')
def claimed_records(debug=False, test=False):
"""
Reporting function; checks SOLR for the following:
- number of records that have been claimed by at least one ORCID ID, in orcid_pub, orcid_user, orcid_other
(each reported separately)
- total number of accepted claims of each of orcid_pub, orcid_user, orcid_other (i.e. if a single record
has two separate authors who have successfully created a claim, the number reported here is 2)
- total number of bibcodes that have been claimed, of any type
The report is designed to be run regularly, and the results compared to previous report runs (via logs)
:return: None (output to logs)
"""
if test:
logger = setup_logging('test_claimed')
else:
logger = setup_logging('reporting')
config = {}
config.update(load_config())
# the first 7 digits of ORCID IDs are zero padding
orcid_wild = '000000*'
resp_pub = query_solr(config['SOLR_URL'],
'orcid_pub:"' + orcid_wild + '"',
rows=10,
sort="bibcode desc",
fl='bibcode')
resp_user = query_solr(config['SOLR_URL'],
'orcid_user:"******"',
rows=10,
sort="bibcode desc",
fl='bibcode')
resp_other = query_solr(config['SOLR_URL'],
'orcid_other:"' + orcid_wild + '"',
rows=10,
sort="bibcode desc",
fl='bibcode')
logger.info('Number of records with an orcid_pub: {}'.format(
resp_pub['response']['numFound']))
logger.info('Number of records with an orcid_user: {}'.format(
resp_user['response']['numFound']))
logger.info('Number of records with an orcid_other: {}'.format(
resp_other['response']['numFound']))
start = 0
rows = 1000
results = resp_pub['response']['docs']
num_orcid_pub = 0
num_orcid_user = 0
num_orcid_other = 0
bibcode_pub = set()
bibcode_user = set()
bibcode_other = set()
while results:
results = query_records(start=start, rows=rows)
for i in range(len(results)):
try:
results[i]['orcid_pub']
except KeyError:
pass
else:
num_p = len(
fnmatch.filter(results[i].get('orcid_pub'), '0000*'))
num_orcid_pub += num_p
bibcode_pub.add(results[i].get('bibcode'))
try:
results[i]['orcid_user']
except KeyError:
pass
else:
num_u = len(
fnmatch.filter(results[i].get('orcid_user'), '0000*'))
num_orcid_user += num_u
bibcode_user.add(results[i].get('bibcode'))
try:
results[i]['orcid_other']
except KeyError:
pass
else:
num_o = len(
fnmatch.filter(results[i].get('orcid_other'), '0000*'))
num_orcid_other += num_o
bibcode_other.add(results[i].get('bibcode'))
if debug:
if (start + rows) % 10000 == 0:
logger.info(
'Number of results processed so far: {}'.format(start +
rows))
if test:
break
else:
start += rows
logger.info('Total number of orcid_pub claims: {}'.format(num_orcid_pub))
logger.info('Total number of orcid_user claims: {}'.format(num_orcid_user))
logger.info(
'Total number of orcid_other claims: {}'.format(num_orcid_other))
orcid_bibcodes = bibcode_pub.union(bibcode_user).union(bibcode_other)
logger.info('Total number of records with any ORCID claims: {}'.format(
len(orcid_bibcodes)))
import os
import fnmatch
import datetime
import cachetools
import time
import pytz
import urllib3
# ============================= INITIALIZATION ==================================== #
# - Use app logger:
#import logging
#logger = logging.getLogger('orcid-pipeline')
# - Or individual logger for this file:
from adsputils import setup_logging, load_config
proj_home = os.path.realpath(os.path.join(os.path.dirname(__file__), '../'))
config = load_config(proj_home=proj_home)
logger = setup_logging(__name__,
proj_home=proj_home,
level=config.get('LOGGING_LEVEL', 'INFO'),
attach_stdout=config.get('LOG_STDOUT', False))
app = tasks.app
records_cache = cachetools.TTLCache(maxsize=1024,
ttl=3600,
timer=time.time,
missing=None,
getsizeof=None)
# =============================== FUNCTIONS ======================================= #
def num_missing_profile(n_days=7,test=False):
"""
Queries logs via Kibana to get the number of profiles reported missing over a given time period.
:param n_days: Number of days backwards to look, starting from now
:return: None (outputs to logs)
"""
if test:
logger = setup_logging('test_kibana')
query = '"+@log_group:\\"backoffice-orcid_pipeline-daemon\\" +@message:\\"Missing profile for\\""'
resp = query_Kibana(query=query, n_days=n_days, rows=5)
total = resp['responses'][0]['hits']['total']
logger.info('Number of missing profile errors in the last {} days: {}'.format(n_days, total))
if __name__ == '__main__':
# Runs all reporting scripts, outputs results to logs
# Before running, tunnel into SOLR and postgres and specify localhost URLs for
# SOLR_URL and SQLALCHEMY_URL, respectively, in local_config.py
config = {}
config.update(load_config())
claimed_records()
num_claims(n_days=7)
num_refused_claims(n_days=7)
num_missing_profile(n_days=7)
def main():
parser = argparse.ArgumentParser(description='process column files into Postgres')
parser.add_argument('-t', '--rowViewBaselineSchemaName', default='nonbibstaging',
help='name of old postgres schema, used to compute delta')
parser.add_argument('-d', '--diagnose', default=False, action='store_true', help='run simple test')
parser.add_argument('-f', '--filename', default='bibcodes.txt', help='name of file containing the list of bibcode for metrics comparison')
parser.add_argument('-m', '--metricsSchemaName', default='metrics', help='name of the postgres metrics schema')
parser.add_argument('-n', '--metricsSchemaName2', default='', help='name of the postgres metrics schema for comparison')
parser.add_argument('-r', '--rowViewSchemaName', default='nonbib', help='name of the postgres row view schema')
parser.add_argument('-s', '--batchSize', default=100, help='used when queuing data')
parser.add_argument('-b', '--bibcodes', default='', help='comma separate list of bibcodes send to master pipeline')
parser.add_argument('command', default='help', nargs='?',
help='ingest | verify | createIngestTables | dropIngestTables | renameSchema ' \
+ ' | createJoinedRows | createMetricsTable | dropMetricsTable ' \
+ ' | populateMetricsTable | createDeltaRows | populateMetricsTableDelta ' \
+ ' | runRowViewPipeline | runMetricsPipeline | createNewBibcodes ' \
+ ' | runRowViewPipelineDelta | runMetricsPipelineDelta '\
+ ' | runPipelines | runPipelinesDelta | nonbibToMasterPipeline | nonbibDeltaToMasterPipeline'
+ ' | metricsToMasterPipeline | metricsDeltaToMasterPipeline | metricsCompare')
args = parser.parse_args()
config.update(load_config())
global logger
logger = setup_logging('AdsDataSqlSync', config.get('LOG_LEVEL', 'INFO'))
logger.info('starting AdsDataSqlSync.app with {}'.format(args.command))
nonbib_connection_string = config.get('INGEST_DATABASE',
'postgresql://postgres@localhost:5432/postgres')
nonbib_db_engine = create_engine(nonbib_connection_string)
nonbib_db_conn = nonbib_db_engine.connect()
metrics_connection_string = config.get('METRICS_DATABASE',
'postgresql://postgres@localhost:5432/postgres')
metrics_db_engine = create_engine(metrics_connection_string)
metrics_db_conn = metrics_db_engine.connect()
sql_sync = nonbib.NonBib(args.rowViewSchemaName)
if args.command == 'help' and args.diagnose:
diagnose_nonbib()
diagnose_metrics()
elif args.command == 'createIngestTables':
sql_sync.create_column_tables(nonbib_db_engine)
elif args.command == 'dropIngestTables':
sql_sync.drop_column_tables(nonbib_db_engine)
elif args.command == 'createJoinedRows':
sql_sync.create_joined_rows(nonbib_db_conn)
elif args.command == 'createMetricsTable' and args.metricsSchemaName:
m = metrics.Metrics(args.metricsSchemaName)
m.create_metrics_table(metrics_db_engine)
elif args.command == 'dropMetricsTable' and args.metricsSchemaName:
m = metrics.Metrics(args.metricsSchemaName)
m.drop_metrics_table(metrics_db_engine)
elif args.command == 'populateMetricsTable' and args.rowViewSchemaName and args.metricsSchemaName:
m = metrics.Metrics()
m.update_metrics_all(metrics_db_conn, nonbib_db_conn, args.rowViewSchemaName)
elif args.command == 'populateMetricsTableDelta' and args.rowViewSchemaName and args.metricsSchemaName:
m = metrics.Metrics(args.metricsSchemaName)
m.update_metrics_changed(metrics_db_conn, nonbib_db_conn, args.rowViewSchemaName)
elif args.command == 'renameSchema' and args.rowViewSchemaName and args.rowViewBaselineSchemaName:
sql_sync.rename_schema(nonbib_db_conn, args.rowViewBaselineSchemaName)
elif args.command == 'createDeltaRows' and args.rowViewSchemaName and args.rowViewBaselineSchemaName:
sql_sync.create_delta_rows(nonbib_db_conn, args.rowViewBaselineSchemaName)
elif args.command == 'createNewBibcodes' and args.rowViewSchemaName and args.rowViewBaselineSchemaName:
sql_sync.build_new_bibcodes(nonbib_db_conn, args.rowViewBaselineSchemaName)
elif args.command == 'logDeltaReasons' and args.rowViewSchemaName and args.rowViewBaselineSchemaName:
sql_sync.log_delta_reasons(nonbib_db_conn, args.rowViewBaselineSchemaName)
elif args.command == 'runRowViewPipeline' and args.rowViewSchemaName:
# drop tables, create tables, load data, create joined view
sql_sync.drop_column_tables(nonbib_db_engine)
sql_sync.create_column_tables(nonbib_db_engine)
load_column_files(config, nonbib_db_engine, nonbib_db_conn, sql_sync)
elif args.command == 'runMetricsPipeline' and args.rowViewSchemaName and args.metricsSchemaName:
m = metrics.Metrics(args.metricsSchemaName)
m.drop_metrics_table(metrics_db_engine)
m.create_metrics_table(metrics_db_engine)
m.update_metrics_all(metrics_db_conn, nonbib_db_conn, args.rowViewSchemaName)
elif args.command == 'runRowViewPipelineDelta' and args.rowViewSchemaName and args.rowViewBaselineSchemaName:
# we delete the old data
baseline_sql_sync = nonbib.NonBib(args.rowViewBaselineSchemaName)
baseline_engine = create_engine(nonbib_connection_string)
baseline_sql_sync.drop_column_tables(baseline_engine)
# rename the current to be the old (for later comparison)
sql_sync.rename_schema(nonbib_db_conn, args.rowViewBaselineSchemaName)
# create the new and populate
baseline_sql_sync = None
sql_sync.create_column_tables(nonbib_db_engine)
load_column_files(config, nonbib_db_engine, nonbib_db_conn, sql_sync)
# compute delta between old and new
sql_sync.create_delta_rows(nonbib_db_conn, args.rowViewBaselineSchemaName)
sql_sync.log_delta_reasons(nonbib_db_conn, args.rowViewBaselineSchemaName)
elif args.command == 'runMetricsPipelineDelta' and args.rowViewSchemaName and args.metricsSchemaName:
m = metrics.Metrics(args.metricsSchemaName)
m.update_metrics_changed(metrics_db_conn, nonbib_db_conn, args.rowViewSchemaName)
elif args.command == 'runPipelines' and args.rowViewSchemaName and args.metricsSchemaName:
# drop tables, create tables, load data, compute metrics
sql_sync.drop_column_tables(nonbib_db_engine)
sql_sync.create_column_tables(nonbib_db_engine)
load_column_files(config, nonbib_db_engine, nonbib_db_conn, sql_sync)
m = metrics.Metrics(args.metricsSchemaName)
m.drop_metrics_table(metrics_db_engine)
m.create_metrics_table(metrics_db_engine)
m.update_metrics_all(metrics_db_conn, nonbib_db_conn, args.rowViewSchemaName)
elif args.command == 'runPipelinesDelta' and args.rowViewSchemaName and args.metricsSchemaName and args.rowViewBaselineSchemaName:
# drop tables, rename schema, create tables, load data, compute delta, compute metrics
baseline_sql_sync = nonbib.NonBib(args.rowViewBaselineSchemaName)
baseline_engine = create_engine(nonbib_connection_string)
baseline_sql_sync.drop_column_tables(baseline_engine)
sql_sync.rename_schema(nonbib_db_conn, args.rowViewBaselineSchemaName)
baseline_sql_sync = None
sql_sync.create_column_tables(nonbib_db_engine)
load_column_files(config, nonbib_db_engine, nonbib_db_conn, sql_sync)
sql_sync.create_delta_rows(nonbib_db_conn, args.rowViewBaselineSchemaName)
sql_sync.log_delta_reasons(nonbib_db_conn, args.rowViewBaselineSchemaName)
m = metrics.Metrics(args.metricsSchemaName)
m.update_metrics_changed(metrics_db_conn, nonbib_db_conn, args.rowViewSchemaName)
elif args.command == 'nonbibToMasterPipeline' and args.diagnose:
diagnose_nonbib()
elif args.command == 'nonbibToMasterPipeline' and args.bibcodes:
bibcodes = args.bibcodes.split(',')
nonbib_bibs_to_master_pipeline(nonbib_db_engine, args.rowViewSchemaName, bibcodes)
elif args.command == 'nonbibToMasterPipeline' and args.filename:
bibcodes = []
with open(args.filename, 'r') as f:
for line in f:
bibcodes.append(line.strip())
if len(bibcodes) > 100:
nonbib_bibs_to_master_pipeline(nonbib_db_engine, args.rowViewSchemaName, bibcodes)
bibcodes = []
if len(bibcodes) > 0:
nonbib_bibs_to_master_pipeline(nonbib_db_engine, args.rowViewSchemaName, bibcodes)
elif args.command == 'nonbibToMasterPipeline':
nonbib_to_master_pipeline(nonbib_db_engine, args.rowViewSchemaName, int(args.batchSize))
elif args.command == 'nonbibDeltaToMasterPipeline':
nonbib_delta_to_master_pipeline(nonbib_db_engine, args.rowViewSchemaName, int(args.batchSize))
elif args.command == 'metricsToMasterPipeline' and args.diagnose:
diagnose_metrics()
elif args.command == 'metricsToMasterPipeline' and args.bibcodes:
bibcodes = args.bibcodes.split(',')
metrics_bibs_to_master_pipeline(metrics_db_engine, args.metricsSchemaName, bibcodes)
elif args.command == 'metricsToMasterPipeline':
metrics_to_master_pipeline(metrics_db_engine, args.metricsSchemaName, int(args.batchSize))
elif args.command == 'metricsDeltaToMasterPipeline':
metrics_delta_to_master_pipeline(metrics_db_engine, args.metricsSchemaName, nonbib_db_engine, args.rowViewSchemaName, int(args.batchSize))
elif args.command == 'metricsCompare':
# compare the values in two metrics postgres tables
# useful to compare results from new pipeline to produciton pipeline
# read metrics records from both databases and compare
metrics_logger = setup_logging('metricsCompare', 'INFO')
metrics1 = metrics.Metrics(args.metricsSchemaName)
Session = sessionmaker(bind=metrics_db_engine)
session = Session()
if args.metricsSchemaName:
session.execute('set search_path to {}'.format(args.metricsSchemaName))
metrics2 = metrics.Metrics(args.metricsSchemaName2)
metrics_connection_string2 = config.get('METRICS_DATABASE2',
'postgresql://postgres@localhost:5432/postgres')
metrics_db_engine2 = create_engine(metrics_connection_string2)
Session2 = sessionmaker(bind=metrics_db_engine2)
session2 = Session2()
if args.metricsSchemaName2:
session2.execute('set search_path to {}'.format(args.metricsSchemaName2))
print 'm2', metrics_connection_string2
print 'm2 schema', args.metricsSchemaName2
with open(args.filename) as f:
for line in f:
bibcode = line.strip()
m1 = metrics1.get_by_bibcode(session, bibcode)
m2 = metrics2.get_by_bibcode(session2, bibcode)
mismatch = metrics.Metrics.metrics_mismatch(line.strip(), m1, m2, metrics_logger)
if mismatch:
metrics_logger.error('{} MISMATCHED FIELDS: {}'.format(bibcode, mismatch))
print '{} MISMATCHED FIELDS: {}'.format(bibcode, mismatch)
session.close()
session2.close()
else:
print 'app.py: illegal command or missing argument, command = ', args.command
print ' row view schema name = ', args.rowViewSchemaName
print ' row view baseline schema name = ', args.rowViewBaselineSchemaName
print ' metrics schema name = ', args.metricsSchemaName
if nonbib_db_conn:
nonbib_db_conn.close()
if metrics_db_conn:
metrics_db_conn.close()
logger.info('completed {}'.format(args.command))
# access to the values within the .ini file in use.
config = context.config
# Interpret the config file for Python logging.
# This line sets up loggers basically.
fileConfig(config.config_file_name)
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
#target_metadata = None
from adsputils import load_config
opath = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
app_conf = load_config(proj_home=opath)
if opath not in sys.path:
sys.path.insert(0, opath)
from ADSCitationCapture import models
target_metadata = models.Base.metadata
# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
def run_migrations_offline():
"""Run migrations in 'offline' mode.
class test_resolver(unittest.TestCase):
"""tests for generation of resolver"""
# Reference to testing.postgresql database instance
db = None
# Connection to the database used to set the database state before running each
# test
db_con = None
# Map of database connection parameters passed to the functions we're testing
db_conf = None
config = {}
config.update(load_config())
def setUp(self):
""" Module level set-up called once before any tests in this file are
executed. Creates a temporary database and sets it up """
global db, db_con, db_conf
db = testing.postgresql.Postgresql()
# Get a map of connection parameters for the database which can be passed
# to the functions being tested so that they connect to the correct
# database
db_conf = db.dsn()
# Create a connection which can be used by our test functions to set and
# query the state of the database
db_con = psycopg2.connect(**db_conf)
# Commit changes immediately to the database
db_con.set_isolation_level(
psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
with db_con.cursor() as cur:
# Create the initial database structure (roles, schemas, tables etc.)
# basically anything that doesn't change
cur.execute(self.slurp('tests/data/datalinks.sql'))
self.maxDiff = None
def tearDown(self):
""" Called after all of the tests in this file have been executed to close
the database connecton and destroy the temporary database """
db_con.close()
db.stop()
def slurp(self, path):
""" Reads and returns the entire contents of a file """
with open(path, 'r') as f:
return f.read()
def test_data_query(self):
with db_con.cursor() as cur:
cur.execute(self.config['DATA_QUERY'].format(
db='public', bibcode='1903BD....C......0A'))
self.assertEqual(fetch_data_link_elements_counts(cur.fetchone()),
[['CDS:1', 'Vizier:1'], 2])
def test_esource_query1(self):
with db_con.cursor() as cur:
cur.execute(self.config['ESOURCE_QUERY'].format(
db='public', bibcode='2016Atoms...4...18I'))
self.assertEqual(fetch_data_link_elements(cur.fetchone()),
['EPRINT_HTML', 'EPRINT_PDF'])
def test_esource_query2(self):
with db_con.cursor() as cur:
cur.execute(self.config['ESOURCE_QUERY'].format(
db='public', bibcode='2014MNRAS.444.1496E'))
self.assertEqual(fetch_data_link_elements(cur.fetchone()),
['PUB_PDF'])
def test_esource_query3(self):
with db_con.cursor() as cur:
cur.execute(self.config['ESOURCE_QUERY'].format(
db='public', bibcode='2014MNRAS.444.1497S'))
self.assertEqual(fetch_data_link_elements(cur.fetchone()),
['EPRINT_HTML', 'EPRINT_PDF', 'PUB_PDF'])
def test_property_query1(self):
with db_con.cursor() as cur:
cur.execute(self.config['PROPERTY_QUERY'].format(
db='public', bibcode='2004MNRAS.354L..31M'))
self.assertEqual(fetch_data_link_elements(cur.fetchone()),
['ASSOCIATED', 'ESOURCE', 'INSPIRE'])
def test_property_query2(self):
with db_con.cursor() as cur:
cur.execute(self.config['PROPERTY_QUERY'].format(
db='public', bibcode='1891opvl.book.....N'))
self.assertEqual(fetch_data_link_elements(cur.fetchone()),
['LIBRARYCATALOG'])
def test_property_query3(self):
with db_con.cursor() as cur:
cur.execute(self.config['PROPERTY_QUERY'].format(
db='public', bibcode='2018LPI....49.2177B'))
self.assertEqual(fetch_data_link_elements(cur.fetchone()),
['ESOURCE', 'TOC'])
def test_extra_property_values(self):
current_row = {}
extra_properties = [{
'pub_openaccess': True,
'private': False,
'ocrabstract': False,
'nonarticle': True,
'refereed': True
}, {
'pub_openaccess': False,
'private': True,
'ocrabstract': True,
'nonarticle': False,
'refereed': False
}]
results = [[
'NONARTICLE', 'REFEREED', 'PUB_OPENACCESS', 'ADS_OPENACCESS',
'AUTHOR_OPENACCESS', 'EPRINT_OPENACCESS', 'OPENACCESS'
], ['ARTICLE', 'NOT REFEREED', 'PRIVATE', 'OCRABSTRACT']]
esources = [['ADS_PDF', 'AUTHOR_PDF', 'EPRINT_HTML'], []]
for extra_property, result, esource in zip(extra_properties, results,
esources):
current_row['property'] = []
for key, value in extra_property.iteritems():
current_row[key] = value
current_row['esource'] = esource
current_row = add_data_link_extra_properties(current_row)
self.assertEqual(current_row['property'], result)
def test_datalinks_query(self):
with db_con.cursor() as cur:
cur.execute(self.config['DATALINKS_QUERY'].format(
db='public', bibcode='2004MNRAS.354L..31M'))
rec = fetch_data_link_record(cur.fetchall())
expected = [{
'url':
['http://articles.adsabs.harvard.edu/pdf/1825AN......4..241B'],
'title': [],
'item_count':
0,
'link_type':
'ESOURCE',
'link_sub_type':
'ADS_PDF'
}, {
'url': ['1825AN......4..241B', '2010AN....331..852K'],
'title': ['Main Paper', 'Translation'],
'item_count': 0,
'link_type': 'ASSOCIATED',
'link_sub_type': 'NA'
}, {
'url': [],
'title': [],
'item_count': 0,
'link_type': 'INSPIRE',
'link_sub_type': 'NA'
}]
self.assertEqual(rec, expected)
def test_datalinks_query_for_associated(self):
with db_con.cursor() as cur:
cur.execute(self.config['DATALINKS_QUERY'].format(
db='public', bibcode='2004MNRAS.354L..31M'))
self.assertEqual(fetch_data_link_record(cur.fetchall()), [{
'url':
['http://articles.adsabs.harvard.edu/pdf/1825AN......4..241B'],
'title': [],
'item_count':
0,
'link_type':
'ESOURCE',
'link_sub_type':
'ADS_PDF'
}, {
'url': ['1825AN......4..241B', '2010AN....331..852K'],
'title': ['Main Paper', 'Translation'],
'item_count':
0,
'link_type':
'ASSOCIATED',
'link_sub_type':
'NA'
}, {
'url': [],
'title': [],
'item_count':
0,
'link_type':
'INSPIRE',
'link_sub_type':
'NA'
}])
#!/usr/bin/python
# -*- coding: utf-8 -*-
import os
import sys
from aip.libs import read_records
from adsputils import setup_logging, load_config
from aip.models import Records
from aip import app, tasks
import time
import mmap
import argparse
from collections import OrderedDict
from sqlalchemy.orm import load_only
config = load_config()
logger = setup_logging('run.py')
def readBibcodesFromFile(
files): #rca: all here is old code, i don't see why mmap was used
"""Reads contents of the BIBFILES into memory; basically bibcode:json_fingerprint
pairs.
@param files: list of files to read from
@return: OrderedDict instance
"""
start = time.time()
records = OrderedDict()
for f in files:
def setUp(self):
super(TestXMLExtractorBase, self).setUp()
self.preferred_parser_names = load_config().get(
'PREFERRED_XML_PARSER_NAMES'
) # Iterate through all the parsers defined in config.py
from datetime import datetime, timedelta
from os import remove
from shutil import move
import subprocess
import os
from adsputils import setup_logging, load_config
logger = setup_logging('AutomatedIngestReport')
conf = load_config(proj_home='./')
# enums used to to generate file names
class FileType:
CANONICAL = 'CANONICAL'
SOLR = 'SOLR'
FULLTEXT = 'FULLTEXT'
class FileAdjective:
MISSING = 'MISSING'
DELETED = 'DELETED'
EXTRA = 'EXTRA'
NEW = 'NEW'
class Date:
TODAY = 1
YESTERDAY = 2
