Exemplo n.º 1
0
class Job:
    DB_NAME = 'job_data'

    SUCCESSFUL = 0
    WARNINGS = 2
    FAILED = 1
    STATUS_MAP = {
        SUCCESSFUL: 'finished successfully',
        WARNINGS: 'finished with warnings',
        FAILED: 'failed'
    }
    FIRST_PHASE = 'INITIATED'

    def __init__(self, log_path=None, job_id=None):
        self._db = Database(name=self.DB_NAME)
        self.phase_name = None

        if job_id:
            # Load in an existing job from database.
            job_row = self._db.get_one_row('jobs', 'id="{0}"'.format(job_id))
            job_dict = query_result_to_dict([job_row], Constants.configs['tables'][self.DB_NAME]['jobs'])[0]

            # Read in job phase.
            phase_row = self._db.query_table('phases', 'job_id="{0}"'.format(job_dict['id']))
            phase_dict = query_result_to_dict(phase_row, Constants.configs['tables'][self.DB_NAME]['phases'])[-1]
            job_dict['phase_name'] = phase_dict['name']

        else:
            # Create new job and add it to the database.
            job_dict = self._create_job_dict(log_path)
            self._db.insert_row_from_dict('jobs', job_dict)

        # Set instance variables.
        self.id = job_dict['id']
        self.name = job_dict['name']
        self.script = job_dict['script']
        self.version = job_dict['version']
        self.log_path = job_dict['log_path']
        self.elapsed_time = job_dict['elapsed_time']
        self.finish_state = job_dict['finish_state']
        self.start_time = job_dict['start_time']
        self.phase_name = job_dict['phase_name']

        # Initiate the job is no phase.
        if self.phase_name is None:
            self.update_phase(Job.FIRST_PHASE)

    @staticmethod
    def _create_job_dict(log_path):
        if Constants.job_name:
            name = Constants.job_name
        else:
            name = '{0}_manual_run'.format(Constants.script)
        return {
            'id': str(abs(hash(name + datetime.datetime.now().strftime(Constants.DATETIME_FORMAT)))),
            'name': name.lower(),
            'script': Constants.script,
            'version': Constants.configs['version'],
            'log_path': log_path,
            'elapsed_time': None,
            'finish_state': None,
            'start_time': datetime.datetime.now().strftime(Constants.DATETIME_FORMAT),
            'phase_name': None
        }

    def _add_phase(self, name):
        phase_id = str(abs(hash(name + self.id)))
        date_time = datetime.datetime.now().strftime(Constants.DATETIME_FORMAT)
        self._db.insert_row('phases', [phase_id, self.id, date_time, name])
        return phase_id

    def log(self, logger=None):
        if logger is None:
            logger = Constants.log
        logger.info('Starting job: {0}'.format(self.id))
        log_hr()

    def update_phase(self, phase):
        self.phase_name = phase.replace(' ', '_').upper()
        phase_id = self._add_phase(self.phase_name)
        # self._db.update_value('jobs', 'phase_id', phase_id, 'id="{0}"'.format(self.id))

    def finished(self, status=SUCCESSFUL, condition=None):
        log_hr()

        # Update job phase.
        if condition is None:
            self.update_phase('TERMINATED_SUCCESSFULLY')
        else:
            Constants.log.warning('Job finished early with condition: "{0}"'.format(condition))
            self.update_phase('TERMINATED_{0}'.format(condition))

        # Update job.
        start_time = self._db.get_one_row('phases', 'job_id="{}" AND name="{}"'.format(self.id, Job.FIRST_PHASE))[2]
        start_time = datetime.datetime.strptime(start_time, Constants.DATETIME_FORMAT)
        run_time = round((datetime.datetime.now() - start_time).total_seconds(), 3)
        self._db.update_value('jobs', 'elapsed_time', run_time, 'id="{0}"'.format(self.id))
        self._db.update_value('jobs', 'finish_state', status, 'id="{0}"'.format(self.id))

        # Log final status.
        if status == Job.SUCCESSFUL or status == Job.WARNINGS:
            Constants.log.info('Job "{0}" {1} in {2} seconds.'.format(self.name, Job.STATUS_MAP[status], run_time))
        elif status == Job.FAILED:
            Constants.log.error('Job "{0}" {1} after {2} seconds.'.format(self.name, Job.STATUS_MAP[status], run_time))
        else:
            Constants.log.info('Job "{0}" finished in {1} seconds.'.format(self.name, run_time))
def add_profile(username, user_hash):
    profile_id = generate_unique_id(username)
    db = Database(name='user_data')
    values = [profile_id, username, user_hash]
    db.insert_row('profiles', values)
    return profile_id
Exemplo n.º 3
0
def main():
    # Setup parse options, imitate global constants and logs.
    Constants.parse_arguments(Constants.APP_NAME)

    # Initiate Job.
    job = Job(log_path=Constants.log_path)
    job.log()

    # Setup connection to market data database, using the data loader's db name constant.
    db = Database(name=MarketDataLoader.DB_NAME)
    db.log()

    # Parse subscriptions file.
    job.update_phase('parsing subscriptions')
    subscriptions = []
    for tick_requirement in get_xml_root(Constants.xml.path):
        symbol = get_xml_element_attribute(tick_requirement, 'symbol', required=True)
        stale_tick_limit = get_xml_element_attribute(tick_requirement, 'stale_tick_limit')
        if stale_tick_limit:
            subscriptions.append({'symbol': symbol.upper(), 'stale_tick_limit': int(stale_tick_limit)})
        else:
            subscriptions.append({'symbol': symbol.upper()})
    Constants.log.info('Loaded {0} required tickers.'.format(len(subscriptions)))

    # Load data.
    job.update_phase('requesting data')
    ticker_data_source = TickerDataSource()
    warnings = 0
    for ticker in subscriptions:
        data_source_data = ticker_data_source.request_quote(ticker[TickerDataSource.SYMBOL])
        if data_source_data:
            # Add data to ticker dictionary.
            ticker['price'] = data_source_data[TickerDataSource.PRICE]
            ticker['volume'] = data_source_data[TickerDataSource.VOLUME]

            # Carry out checks on ticker.
            ticker_warnings = ticker_checks(ticker)

            # Save tick to database.
            run_time_string = Constants.run_time.strftime(Constants.DATETIME_FORMAT)
            db.insert_row('ticks', [generate_unique_id(ticker['symbol'] + run_time_string),
                                    run_time_string,
                                    ticker['symbol'],
                                    ticker['price'],
                                    ticker['volume']
                                    ]
                          )

            # Log ticks.
            Constants.log.info('symbol: {0}, price: {1}, volume: {2}'.format(ticker['symbol'], ticker['price'], ticker['volume']))
        else:
            ticker_warnings = ['no_data']

        for warning_type in ticker_warnings:
            warning_id = generate_unique_id(ticker['symbol'] + Constants.run_time.strftime(Constants.DATETIME_FORMAT))
            db.insert_row('data_warnings', [warning_id, 'tick', warning_type, ticker['symbol']])
            Constants.log.info('Could not get data for ticker {0}'.format(ticker['symbol']))
            warnings += 1

    if warnings:
        job.finished(status=Job.WARNINGS, condition='data warnings')
        return Job.WARNINGS
    else:
        job.finished()
        return Job.SUCCESSFUL