def run(self): logging.info('Start') # Check if ElasticSearch index exists and create if needed index = self.config['lris']['index'] logging.info('Checking for "{0}" index'.format(index)) es = pyes.ES('{0}:{1}'.format(self.config['lris']['host'], self.config['lris']['port'])) getIndicesResponse = es.get_indices() if index not in getIndicesResponse: logging.info('Index "{0}" not found, creating'.format(index)) settings = json.loads(helperfunctions.readFile('./lrissettings.json')) createIndexResponse = es.create_index_if_missing(index, settings) if 'acknowledged' in createIndexResponse and createIndexResponse['acknowledged'] and \ 'ok' in createIndexResponse and createIndexResponse['ok']: logging.info('Successfully created index "{0}"'.format(index)) else: raise Exception('Error creating index: {0}'.format(createIndexResponse)) # The mapping uses dynamic templates so index the seed item to create the initial mapping for schema.org data seed = json.loads(helperfunctions.readFile('./seeditem.json')) es.index(seed, self.config['lris']['index'], self.config['lris']['index_type'], 'seed') es.delete(self.config['lris']['index'], self.config['lris']['index_type'], 'seed') logging.info('Index "{0}" exists, beginning harvest loop'.format(index)) # Main harvest loop while True: fromDate = helperfunctions.readFile('./fromDate.txt') until = datetime.utcnow().isoformat() + "Z" logging.debug(str.format('Beginning Harvest from: {0} until: {1}', fromDate, until)) self.doHarvest(fromDate, until) logging.debug(str.format('Completed Harvest from: {0} until: {1}', fromDate, until)) helperfunctions.writeFile('./fromDate.txt', until) time.sleep(self.config['interval'])
def __init__(self): self.config = json.loads(helperfunctions.readFile('./config.json')) logging.basicConfig(filename='./connector.log', level=getattr(logging, self.config['logging_level']), format='%(asctime)s %(levelname)s %(message)s') self.lribhelper = lribhelper.lribhelper(self.config, logging) self.gpg = gnupg.GPG(gpgbinary = self.config['gpg']['binary']) self.verify = Verify_0_21(gpgbin = self.config['gpg']['binary'])