def run(self):
logging.info('Start')
# Check if ElasticSearch index exists and create if needed
index = self.config['lris']['index']
logging.info('Checking for "{0}" index'.format(index))
es = pyes.ES('{0}:{1}'.format(self.config['lris']['host'], self.config['lris']['port']))
getIndicesResponse = es.get_indices()
if index not in getIndicesResponse:
logging.info('Index "{0}" not found, creating'.format(index))
settings = json.loads(helperfunctions.readFile('./lrissettings.json'))
createIndexResponse = es.create_index_if_missing(index, settings)
if 'acknowledged' in createIndexResponse and createIndexResponse['acknowledged'] and \
'ok' in createIndexResponse and createIndexResponse['ok']:
logging.info('Successfully created index "{0}"'.format(index))
else:
raise Exception('Error creating index: {0}'.format(createIndexResponse))
# The mapping uses dynamic templates so index the seed item to create the initial mapping for schema.org data
seed = json.loads(helperfunctions.readFile('./seeditem.json'))
es.index(seed, self.config['lris']['index'], self.config['lris']['index_type'], 'seed')
es.delete(self.config['lris']['index'], self.config['lris']['index_type'], 'seed')
logging.info('Index "{0}" exists, beginning harvest loop'.format(index))
# Main harvest loop
while True:
fromDate = helperfunctions.readFile('./fromDate.txt')
until = datetime.utcnow().isoformat() + "Z"
logging.debug(str.format('Beginning Harvest from: {0} until: {1}', fromDate, until))
self.doHarvest(fromDate, until)
logging.debug(str.format('Completed Harvest from: {0} until: {1}', fromDate, until))
helperfunctions.writeFile('./fromDate.txt', until)
time.sleep(self.config['interval'])
def __init__(self):
self.config = json.loads(helperfunctions.readFile('./config.json'))
logging.basicConfig(filename='./connector.log', level=getattr(logging, self.config['logging_level']), format='%(asctime)s %(levelname)s %(message)s')
self.lribhelper = lribhelper.lribhelper(self.config, logging)
self.gpg = gnupg.GPG(gpgbinary = self.config['gpg']['binary'])
self.verify = Verify_0_21(gpgbin = self.config['gpg']['binary'])
