def insert_into_table(self, table, raw_json, data_type, timestamp, msg_from_json):
new_row = dict()
gatewayTime = utils.parse_to_utc_date(msg_from_json['header']['gatewayTimestamp'])
new_row['gatewayInfo'] = '{0}_{1}'.format(data_type, gatewayTime.strftime('%Y-%m-%d'))
new_row['json'] = gzip.compress(bytes(raw_json, 'utf-8'))
new_row['gatewayEpoch'] = utils.date_to_epoch_micro(gatewayTime)
new_row['dynamoTimestamp'] = utils.date_to_epoch_micro(timestamp)
logger.debug('Putting valid JSON {0}'.format(new_row))
table.put_item(Item=new_row)
def send_messages_to_table(self):
try:
with self.info_table.batch_writer() as batch:
for entry in self.commodity_storage:
self.insert_into_table(batch, entry, "c", self.commodity_storage[entry][0], self.commodity_storage[entry][1])
for entry in self.shipyard_storage:
self.insert_into_table(batch, entry, "s", self.shipyard_storage[entry][0], self.shipyard_storage[entry][1])
for entry in self.outfitting_storage:
self.insert_into_table(batch, entry, "o", self.outfitting_storage[entry][0], self.outfitting_storage[entry][1])
for entry in self.blackmarket_storage:
self.insert_into_table(batch, entry, "b", self.blackmarket_storage[entry][0], self.blackmarket_storage[entry][1])
for entry in self.journal_storage:
self.insert_into_table(batch, entry, "j", self.journal_storage[entry][0], self.journal_storage[entry][1])
except Exception as ex:
logger.exception('Exception encountered in sending, sending to bad data for retry')
with self.error_table.batch_writer() as batch:
for entry in self.commodity_storage:
batch.put_item(Item={
'timestamp': utils.date_to_epoch_micro(utils.utcnow()),
'badData': entry,
'errorCause': repr(ex),
'source': 'DynamoRawSend'
})
for entry in self.shipyard_storage:
batch.put_item(Item={
'timestamp': utils.date_to_epoch_micro(utils.utcnow()),
'badData': entry,
'errorCause': repr(ex),
'source': 'DynamoRawSend'
})
for entry in self.outfitting_storage:
batch.put_item(Item={
'timestamp': utils.date_to_epoch_micro(utils.utcnow()),
'badData': entry,
'errorCause': repr(ex),
'source': 'DynamoRawSend'
})
for entry in self.blackmarket_storage:
batch.put_item(Item={
'timestamp': utils.date_to_epoch_micro(utils.utcnow()),
'badData': entry,
'errorCause': repr(ex),
'source': 'DynamoRawSend'
})
for entry in self.journal_storage:
batch.put_item(Item={
'timestamp': utils.date_to_epoch_micro(utils.utcnow()),
'badData': entry,
'errorCause': repr(ex),
'source': 'DynamoRawSend'
})
def main(runPath, argv):
try:
opts, args = getopt.getopt(argv, 'ip', ['iam_role', 'profile_name'])
except getopt.GetoptError as err:
print(repr(err))
print('EDDNDynamoRaw.py -p <profile name> OR -r')
sys.exit(2)
profile_name = 'eddntest'
for opt, arg in opts:
if opt in ("-i", "--iam_role"):
profile_name = ''
if opt in ("-p", "--profile_name"):
profile_name = arg
logging.config.dictConfig({
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'standard': {
'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s'
},
},
'handlers': {
'default': {
'class': 'logging.StreamHandler',
'level': 'INFO',
'formatter': 'standard',
'stream': 'ext://sys.stdout'
},
'file_handler': {
'class': 'logging.handlers.RotatingFileHandler',
'formatter': 'standard',
'level': 'INFO',
'filename': 'eddn-dynamo-raw.log',
'maxBytes': 10485760,
'backupCount': 5,
'encoding': 'utf8'
}
},
'loggers': {
'': {
'handlers': ['default', 'file_handler'],
'level': 'INFO',
'propagate': True
}
}
})
logger.debug('Logging configured')
commodity_schema1 = requests.get('http://schemas.elite-markets.net/eddn/commodity/1', headers={'Connection': 'close'}).json()
logger.info('Obtained commodity schema v1')
commodity_schema2 = requests.get('https://raw.githubusercontent.com/jamesremuscat/EDDN/924c948a0233421e684145a6c40751c5a7a6bef9/schemas/commodity-v2.0.json', headers={'Connection': 'close'}).json()
logger.info('Obtained commodity schema v2')
commodity_schema3 = requests.get('https://raw.githubusercontent.com/jamesremuscat/EDDN/master/schemas/commodity-v3.0.json', headers={'Connection': 'close'}).json()
logger.info('Obtained commodity schema v3')
shipyard_schema1 = requests.get('https://raw.githubusercontent.com/jamesremuscat/EDDN/924c948a0233421e684145a6c40751c5a7a6bef9/schemas/shipyard-v1.0.json', headers={'Connection': 'close'}).json()
logger.info('Obtained shipyard schema v1')
shipyard_schema2 = requests.get('https://raw.githubusercontent.com/jamesremuscat/EDDN/master/schemas/shipyard-v2.0.json', headers={'Connection': 'close'}).json()
logger.info('Obtained shipyard schema v2')
outfitting_schema1 = requests.get('https://raw.githubusercontent.com/jamesremuscat/EDDN/924c948a0233421e684145a6c40751c5a7a6bef9/schemas/outfitting-v1.0.json', headers={'Connection': 'close'}).json()
logger.info('Obtained outfitting schema v1')
outfitting_schema2 = requests.get('https://raw.githubusercontent.com/jamesremuscat/EDDN/master/schemas/outfitting-v2.0.json', headers={'Connection': 'close'}).json()
logger.info('Obtained outfitting schema v2')
journal_schema1 = requests.get('https://raw.githubusercontent.com/jamesremuscat/EDDN/master/schemas/journal-v1.0.json', headers={'Connection': 'close'}).json()
logger.info('Obtained journal schema v1')
blackmarket_schema1 = requests.get('https://raw.githubusercontent.com/jamesremuscat/EDDN/master/schemas/blackmarket-v1.0.json', headers={'Connection': 'close'}).json()
logger.info('Obtained blackmarket schema v1')
if profile_name:
boto3.setup_default_session(profile_name=profile_name)
dynamodb = boto3.resource('dynamodb', region_name='eu-west-1')
logger.info('Connected to Dynamo')
ioloop.install()
logger.info('Installed PyZMQ version of Tornado IOLoop')
context = zmq.Context()
message_processes = MessageProcessor(commodity_schema1, commodity_schema2, commodity_schema3, shipyard_schema1, shipyard_schema2, outfitting_schema1, outfitting_schema2, journal_schema1, blackmarket_schema1, dynamodb)
# Ideally the timeout here would be coordinated with keep-alive timing from EDDN
subscriber = Subscriber(context, random.randint(1500, 1800), message_processes.process_message, message_processes.send_messages)
while not subscriber.shutdown_signalled:
try:
subscriber.start()
except Exception as ex:
logger.exception('Exception encountered in communications, listening again')
bad_data_table.put_item(Item={
'timestamp': utils.date_to_epoch_micro(utils.utcnow()),
'errorCause': repr(ex),
'source': 'DynamoRaw'
})
sleep(0.001)
def process_message(self, msg):
year = utils.utcnow().year
if self.info_table is None or self.year != year:
self.info_table = self.dynamodb.Table('eddn-archive-{0}'.format(year))
self.year = year
try:
raw_json = zlib.decompress(msg).decode(encoding='UTF-8')
try:
msg_from_json = simplejson.loads(raw_json)
logger.debug('Raw json {0}'.format(msg_from_json))
if msg_from_json['$schemaRef'] == 'http://schemas.elite-markets.net/eddn/commodity/1':
jsonschema.validate(msg_from_json, self.commodity_schema1)
logger.debug('Json passed commodity schema v1 validation')
storage = self.commodity_storage
elif msg_from_json['$schemaRef'] == 'http://schemas.elite-markets.net/eddn/commodity/2':
jsonschema.validate(msg_from_json, self.commodity_schema2)
logger.debug('Json passed commodity schema v2 validation')
storage = self.commodity_storage
elif msg_from_json['$schemaRef'] == 'http://schemas.elite-markets.net/eddn/commodity/3':
jsonschema.validate(msg_from_json, self.commodity_schema3)
logger.debug('Json passed commodity schema v3 validation')
storage = self.commodity_storage
elif msg_from_json['$schemaRef'] == 'http://schemas.elite-markets.net/eddn/shipyard/1':
jsonschema.validate(msg_from_json, self.shipyard_schema1)
logger.debug('Json passed shipyard schema v1 validation')
storage = self.shipyard_storage
elif msg_from_json['$schemaRef'] == 'http://schemas.elite-markets.net/eddn/shipyard/2':
jsonschema.validate(msg_from_json, self.shipyard_schema2)
logger.debug('Json passed shipyard schema v2 validation')
storage = self.shipyard_storage
elif msg_from_json['$schemaRef'] == 'http://schemas.elite-markets.net/eddn/outfitting/1':
jsonschema.validate(msg_from_json, self.outfitting_schema1)
logger.debug('Json passed outfitting schema v1 validation')
storage = self.outfitting_storage
elif msg_from_json['$schemaRef'] == 'http://schemas.elite-markets.net/eddn/outfitting/2':
jsonschema.validate(msg_from_json, self.outfitting_schema2)
logger.debug('Json passed outfitting schema v2 validation')
storage = self.outfitting_storage
elif msg_from_json['$schemaRef'] == 'http://schemas.elite-markets.net/eddn/blackmarket/1':
jsonschema.validate(msg_from_json, self.blackmarket_schema1)
logger.debug('Json passed blackmarket schema v1 validation')
storage = self.blackmarket_storage
elif msg_from_json['$schemaRef'] == 'http://schemas.elite-markets.net/eddn/journal/1':
jsonschema.validate(msg_from_json, self.journal_schema1)
logger.debug('Json passed journal schema v1 validation')
storage = self.journal_storage
else:
logger.debug('Data returned is not commodity, shipyard, outfitting, blackmarket, journal, ignoring {0}'.format(msg_from_json))
return
timestamp = utils.utcnow()
if raw_json not in storage:
storage[raw_json] = (timestamp, msg_from_json)
if len(storage) >= 3:
self.send_messages()
except Exception as ex:
logger.exception('Exception encountered in parsing, listening again')
self.error_table.put_item(Item={
'timestamp': utils.date_to_epoch_micro(utils.utcnow()),
'badData': raw_json,
'errorCause': repr(ex),
'source': 'DynamoRawStore'
})
sleep(0.001)
except Exception as ex:
logger.exception('Exception encountered in communications, listening again')
self.error_table.put_item(Item={
'timestamp': utils.date_to_epoch_micro(utils.utcnow()),
'errorCause': repr(ex),
'source': 'DynamoRawDecompress'
})
sleep(0.001)
