class Job:
DB_NAME = 'job_data'
SUCCESSFUL = 0
WARNINGS = 2
FAILED = 1
STATUS_MAP = {
SUCCESSFUL: 'finished successfully',
WARNINGS: 'finished with warnings',
FAILED: 'failed'
}
FIRST_PHASE = 'INITIATED'
def __init__(self, log_path=None, job_id=None):
self._db = Database(name=self.DB_NAME)
self.phase_name = None
if job_id:
# Load in an existing job from database.
job_row = self._db.get_one_row('jobs', 'id="{0}"'.format(job_id))
job_dict = query_result_to_dict([job_row], Constants.configs['tables'][self.DB_NAME]['jobs'])[0]
# Read in job phase.
phase_row = self._db.query_table('phases', 'job_id="{0}"'.format(job_dict['id']))
phase_dict = query_result_to_dict(phase_row, Constants.configs['tables'][self.DB_NAME]['phases'])[-1]
job_dict['phase_name'] = phase_dict['name']
else:
# Create new job and add it to the database.
job_dict = self._create_job_dict(log_path)
self._db.insert_row_from_dict('jobs', job_dict)
# Set instance variables.
self.id = job_dict['id']
self.name = job_dict['name']
self.script = job_dict['script']
self.version = job_dict['version']
self.log_path = job_dict['log_path']
self.elapsed_time = job_dict['elapsed_time']
self.finish_state = job_dict['finish_state']
self.start_time = job_dict['start_time']
self.phase_name = job_dict['phase_name']
# Initiate the job is no phase.
if self.phase_name is None:
self.update_phase(Job.FIRST_PHASE)
@staticmethod
def _create_job_dict(log_path):
if Constants.job_name:
name = Constants.job_name
else:
name = '{0}_manual_run'.format(Constants.script)
return {
'id': str(abs(hash(name + datetime.datetime.now().strftime(Constants.DATETIME_FORMAT)))),
'name': name.lower(),
'script': Constants.script,
'version': Constants.configs['version'],
'log_path': log_path,
'elapsed_time': None,
'finish_state': None,
'start_time': datetime.datetime.now().strftime(Constants.DATETIME_FORMAT),
'phase_name': None
}
def _add_phase(self, name):
phase_id = str(abs(hash(name + self.id)))
date_time = datetime.datetime.now().strftime(Constants.DATETIME_FORMAT)
self._db.insert_row('phases', [phase_id, self.id, date_time, name])
return phase_id
def log(self, logger=None):
if logger is None:
logger = Constants.log
logger.info('Starting job: {0}'.format(self.id))
log_hr()
def update_phase(self, phase):
self.phase_name = phase.replace(' ', '_').upper()
phase_id = self._add_phase(self.phase_name)
# self._db.update_value('jobs', 'phase_id', phase_id, 'id="{0}"'.format(self.id))
def finished(self, status=SUCCESSFUL, condition=None):
log_hr()
# Update job phase.
if condition is None:
self.update_phase('TERMINATED_SUCCESSFULLY')
else:
Constants.log.warning('Job finished early with condition: "{0}"'.format(condition))
self.update_phase('TERMINATED_{0}'.format(condition))
# Update job.
start_time = self._db.get_one_row('phases', 'job_id="{}" AND name="{}"'.format(self.id, Job.FIRST_PHASE))[2]
start_time = datetime.datetime.strptime(start_time, Constants.DATETIME_FORMAT)
run_time = round((datetime.datetime.now() - start_time).total_seconds(), 3)
self._db.update_value('jobs', 'elapsed_time', run_time, 'id="{0}"'.format(self.id))
self._db.update_value('jobs', 'finish_state', status, 'id="{0}"'.format(self.id))
# Log final status.
if status == Job.SUCCESSFUL or status == Job.WARNINGS:
Constants.log.info('Job "{0}" {1} in {2} seconds.'.format(self.name, Job.STATUS_MAP[status], run_time))
elif status == Job.FAILED:
Constants.log.error('Job "{0}" {1} after {2} seconds.'.format(self.name, Job.STATUS_MAP[status], run_time))
else:
Constants.log.info('Job "{0}" finished in {1} seconds.'.format(self.name, run_time))
def add_profile(username, user_hash):
profile_id = generate_unique_id(username)
db = Database(name='user_data')
values = [profile_id, username, user_hash]
db.insert_row('profiles', values)
return profile_id
def main():
# Setup parse options, imitate global constants and logs.
Constants.parse_arguments(Constants.APP_NAME)
# Initiate Job.
job = Job(log_path=Constants.log_path)
job.log()
# Setup connection to market data database, using the data loader's db name constant.
db = Database(name=MarketDataLoader.DB_NAME)
db.log()
# Parse subscriptions file.
job.update_phase('parsing subscriptions')
subscriptions = []
for tick_requirement in get_xml_root(Constants.xml.path):
symbol = get_xml_element_attribute(tick_requirement, 'symbol', required=True)
stale_tick_limit = get_xml_element_attribute(tick_requirement, 'stale_tick_limit')
if stale_tick_limit:
subscriptions.append({'symbol': symbol.upper(), 'stale_tick_limit': int(stale_tick_limit)})
else:
subscriptions.append({'symbol': symbol.upper()})
Constants.log.info('Loaded {0} required tickers.'.format(len(subscriptions)))
# Load data.
job.update_phase('requesting data')
ticker_data_source = TickerDataSource()
warnings = 0
for ticker in subscriptions:
data_source_data = ticker_data_source.request_quote(ticker[TickerDataSource.SYMBOL])
if data_source_data:
# Add data to ticker dictionary.
ticker['price'] = data_source_data[TickerDataSource.PRICE]
ticker['volume'] = data_source_data[TickerDataSource.VOLUME]
# Carry out checks on ticker.
ticker_warnings = ticker_checks(ticker)
# Save tick to database.
run_time_string = Constants.run_time.strftime(Constants.DATETIME_FORMAT)
db.insert_row('ticks', [generate_unique_id(ticker['symbol'] + run_time_string),
run_time_string,
ticker['symbol'],
ticker['price'],
ticker['volume']
]
)
# Log ticks.
Constants.log.info('symbol: {0}, price: {1}, volume: {2}'.format(ticker['symbol'], ticker['price'], ticker['volume']))
else:
ticker_warnings = ['no_data']
for warning_type in ticker_warnings:
warning_id = generate_unique_id(ticker['symbol'] + Constants.run_time.strftime(Constants.DATETIME_FORMAT))
db.insert_row('data_warnings', [warning_id, 'tick', warning_type, ticker['symbol']])
Constants.log.info('Could not get data for ticker {0}'.format(ticker['symbol']))
warnings += 1
if warnings:
job.finished(status=Job.WARNINGS, condition='data warnings')
return Job.WARNINGS
else:
job.finished()
return Job.SUCCESSFUL