def insert_into_table(self, table, raw_json, data_type, timestamp, msg_from_json): new_row = dict() gatewayTime = utils.parse_to_utc_date(msg_from_json['header']['gatewayTimestamp']) new_row['gatewayInfo'] = '{0}_{1}'.format(data_type, gatewayTime.strftime('%Y-%m-%d')) new_row['json'] = gzip.compress(bytes(raw_json, 'utf-8')) new_row['gatewayEpoch'] = utils.date_to_epoch_micro(gatewayTime) new_row['dynamoTimestamp'] = utils.date_to_epoch_micro(timestamp) logger.debug('Putting valid JSON {0}'.format(new_row)) table.put_item(Item=new_row)
def send_messages_to_table(self): try: with self.info_table.batch_writer() as batch: for entry in self.commodity_storage: self.insert_into_table(batch, entry, "c", self.commodity_storage[entry][0], self.commodity_storage[entry][1]) for entry in self.shipyard_storage: self.insert_into_table(batch, entry, "s", self.shipyard_storage[entry][0], self.shipyard_storage[entry][1]) for entry in self.outfitting_storage: self.insert_into_table(batch, entry, "o", self.outfitting_storage[entry][0], self.outfitting_storage[entry][1]) for entry in self.blackmarket_storage: self.insert_into_table(batch, entry, "b", self.blackmarket_storage[entry][0], self.blackmarket_storage[entry][1]) for entry in self.journal_storage: self.insert_into_table(batch, entry, "j", self.journal_storage[entry][0], self.journal_storage[entry][1]) except Exception as ex: logger.exception('Exception encountered in sending, sending to bad data for retry') with self.error_table.batch_writer() as batch: for entry in self.commodity_storage: batch.put_item(Item={ 'timestamp': utils.date_to_epoch_micro(utils.utcnow()), 'badData': entry, 'errorCause': repr(ex), 'source': 'DynamoRawSend' }) for entry in self.shipyard_storage: batch.put_item(Item={ 'timestamp': utils.date_to_epoch_micro(utils.utcnow()), 'badData': entry, 'errorCause': repr(ex), 'source': 'DynamoRawSend' }) for entry in self.outfitting_storage: batch.put_item(Item={ 'timestamp': utils.date_to_epoch_micro(utils.utcnow()), 'badData': entry, 'errorCause': repr(ex), 'source': 'DynamoRawSend' }) for entry in self.blackmarket_storage: batch.put_item(Item={ 'timestamp': utils.date_to_epoch_micro(utils.utcnow()), 'badData': entry, 'errorCause': repr(ex), 'source': 'DynamoRawSend' }) for entry in self.journal_storage: batch.put_item(Item={ 'timestamp': utils.date_to_epoch_micro(utils.utcnow()), 'badData': entry, 'errorCause': repr(ex), 'source': 'DynamoRawSend' })
def main(runPath, argv): try: opts, args = getopt.getopt(argv, 'ip', ['iam_role', 'profile_name']) except getopt.GetoptError as err: print(repr(err)) print('EDDNDynamoRaw.py -p <profile name> OR -r') sys.exit(2) profile_name = 'eddntest' for opt, arg in opts: if opt in ("-i", "--iam_role"): profile_name = '' if opt in ("-p", "--profile_name"): profile_name = arg logging.config.dictConfig({ 'version': 1, 'disable_existing_loggers': False, 'formatters': { 'standard': { 'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s' }, }, 'handlers': { 'default': { 'class': 'logging.StreamHandler', 'level': 'INFO', 'formatter': 'standard', 'stream': 'ext://sys.stdout' }, 'file_handler': { 'class': 'logging.handlers.RotatingFileHandler', 'formatter': 'standard', 'level': 'INFO', 'filename': 'eddn-dynamo-raw.log', 'maxBytes': 10485760, 'backupCount': 5, 'encoding': 'utf8' } }, 'loggers': { '': { 'handlers': ['default', 'file_handler'], 'level': 'INFO', 'propagate': True } } }) logger.debug('Logging configured') commodity_schema1 = requests.get('http://schemas.elite-markets.net/eddn/commodity/1', headers={'Connection': 'close'}).json() logger.info('Obtained commodity schema v1') commodity_schema2 = requests.get('https://raw.githubusercontent.com/jamesremuscat/EDDN/924c948a0233421e684145a6c40751c5a7a6bef9/schemas/commodity-v2.0.json', headers={'Connection': 'close'}).json() logger.info('Obtained commodity schema v2') commodity_schema3 = requests.get('https://raw.githubusercontent.com/jamesremuscat/EDDN/master/schemas/commodity-v3.0.json', headers={'Connection': 'close'}).json() logger.info('Obtained commodity schema v3') shipyard_schema1 = requests.get('https://raw.githubusercontent.com/jamesremuscat/EDDN/924c948a0233421e684145a6c40751c5a7a6bef9/schemas/shipyard-v1.0.json', headers={'Connection': 'close'}).json() logger.info('Obtained shipyard schema v1') shipyard_schema2 = requests.get('https://raw.githubusercontent.com/jamesremuscat/EDDN/master/schemas/shipyard-v2.0.json', headers={'Connection': 'close'}).json() logger.info('Obtained shipyard schema v2') outfitting_schema1 = requests.get('https://raw.githubusercontent.com/jamesremuscat/EDDN/924c948a0233421e684145a6c40751c5a7a6bef9/schemas/outfitting-v1.0.json', headers={'Connection': 'close'}).json() logger.info('Obtained outfitting schema v1') outfitting_schema2 = requests.get('https://raw.githubusercontent.com/jamesremuscat/EDDN/master/schemas/outfitting-v2.0.json', headers={'Connection': 'close'}).json() logger.info('Obtained outfitting schema v2') journal_schema1 = requests.get('https://raw.githubusercontent.com/jamesremuscat/EDDN/master/schemas/journal-v1.0.json', headers={'Connection': 'close'}).json() logger.info('Obtained journal schema v1') blackmarket_schema1 = requests.get('https://raw.githubusercontent.com/jamesremuscat/EDDN/master/schemas/blackmarket-v1.0.json', headers={'Connection': 'close'}).json() logger.info('Obtained blackmarket schema v1') if profile_name: boto3.setup_default_session(profile_name=profile_name) dynamodb = boto3.resource('dynamodb', region_name='eu-west-1') logger.info('Connected to Dynamo') ioloop.install() logger.info('Installed PyZMQ version of Tornado IOLoop') context = zmq.Context() message_processes = MessageProcessor(commodity_schema1, commodity_schema2, commodity_schema3, shipyard_schema1, shipyard_schema2, outfitting_schema1, outfitting_schema2, journal_schema1, blackmarket_schema1, dynamodb) # Ideally the timeout here would be coordinated with keep-alive timing from EDDN subscriber = Subscriber(context, random.randint(1500, 1800), message_processes.process_message, message_processes.send_messages) while not subscriber.shutdown_signalled: try: subscriber.start() except Exception as ex: logger.exception('Exception encountered in communications, listening again') bad_data_table.put_item(Item={ 'timestamp': utils.date_to_epoch_micro(utils.utcnow()), 'errorCause': repr(ex), 'source': 'DynamoRaw' }) sleep(0.001)
def process_message(self, msg): year = utils.utcnow().year if self.info_table is None or self.year != year: self.info_table = self.dynamodb.Table('eddn-archive-{0}'.format(year)) self.year = year try: raw_json = zlib.decompress(msg).decode(encoding='UTF-8') try: msg_from_json = simplejson.loads(raw_json) logger.debug('Raw json {0}'.format(msg_from_json)) if msg_from_json['$schemaRef'] == 'http://schemas.elite-markets.net/eddn/commodity/1': jsonschema.validate(msg_from_json, self.commodity_schema1) logger.debug('Json passed commodity schema v1 validation') storage = self.commodity_storage elif msg_from_json['$schemaRef'] == 'http://schemas.elite-markets.net/eddn/commodity/2': jsonschema.validate(msg_from_json, self.commodity_schema2) logger.debug('Json passed commodity schema v2 validation') storage = self.commodity_storage elif msg_from_json['$schemaRef'] == 'http://schemas.elite-markets.net/eddn/commodity/3': jsonschema.validate(msg_from_json, self.commodity_schema3) logger.debug('Json passed commodity schema v3 validation') storage = self.commodity_storage elif msg_from_json['$schemaRef'] == 'http://schemas.elite-markets.net/eddn/shipyard/1': jsonschema.validate(msg_from_json, self.shipyard_schema1) logger.debug('Json passed shipyard schema v1 validation') storage = self.shipyard_storage elif msg_from_json['$schemaRef'] == 'http://schemas.elite-markets.net/eddn/shipyard/2': jsonschema.validate(msg_from_json, self.shipyard_schema2) logger.debug('Json passed shipyard schema v2 validation') storage = self.shipyard_storage elif msg_from_json['$schemaRef'] == 'http://schemas.elite-markets.net/eddn/outfitting/1': jsonschema.validate(msg_from_json, self.outfitting_schema1) logger.debug('Json passed outfitting schema v1 validation') storage = self.outfitting_storage elif msg_from_json['$schemaRef'] == 'http://schemas.elite-markets.net/eddn/outfitting/2': jsonschema.validate(msg_from_json, self.outfitting_schema2) logger.debug('Json passed outfitting schema v2 validation') storage = self.outfitting_storage elif msg_from_json['$schemaRef'] == 'http://schemas.elite-markets.net/eddn/blackmarket/1': jsonschema.validate(msg_from_json, self.blackmarket_schema1) logger.debug('Json passed blackmarket schema v1 validation') storage = self.blackmarket_storage elif msg_from_json['$schemaRef'] == 'http://schemas.elite-markets.net/eddn/journal/1': jsonschema.validate(msg_from_json, self.journal_schema1) logger.debug('Json passed journal schema v1 validation') storage = self.journal_storage else: logger.debug('Data returned is not commodity, shipyard, outfitting, blackmarket, journal, ignoring {0}'.format(msg_from_json)) return timestamp = utils.utcnow() if raw_json not in storage: storage[raw_json] = (timestamp, msg_from_json) if len(storage) >= 3: self.send_messages() except Exception as ex: logger.exception('Exception encountered in parsing, listening again') self.error_table.put_item(Item={ 'timestamp': utils.date_to_epoch_micro(utils.utcnow()), 'badData': raw_json, 'errorCause': repr(ex), 'source': 'DynamoRawStore' }) sleep(0.001) except Exception as ex: logger.exception('Exception encountered in communications, listening again') self.error_table.put_item(Item={ 'timestamp': utils.date_to_epoch_micro(utils.utcnow()), 'errorCause': repr(ex), 'source': 'DynamoRawDecompress' }) sleep(0.001)