def main(): parser = argparse.ArgumentParser(description='Monolith Aggregator') parser.add_argument('--version', action='store_true', default=False, help='Displays version and exits.') date_group = parser.add_mutually_exclusive_group() date_group.add_argument('--date', default=None, choices=_DATES, help='Date') date_group.add_argument('--start-date', default=None, type=_mkdate, help='Start date.') parser.add_argument('--end-date', default=None, type=_mkdate, help='End date.') parser.add_argument('config', help='Configuration file.',) parser.add_argument('--log-level', dest='loglevel', default='info', choices=LOG_LEVELS.keys() + [key.upper() for key in LOG_LEVELS.keys()], help="log level") parser.add_argument('--log-output', dest='logoutput', default='-', help="log output") parser.add_argument('--sequence', dest='sequence', default=None, help='A comma-separated list of sequences.') parser.add_argument('--batch-size', dest='batch_size', default=None, type=int, help='The size of the batch when writing') parser.add_argument('--force', action='store_true', default=False, help='Forces a run') parser.add_argument('--purge-only', action='store_true', default=False, help='Only run the purge of sources.') parser.add_argument('--retries', default=3, type=int, help='Number of retries') args = parser.parse_args() if args.version: print(__version__) sys.exit(0) if args.date is not None: start, end = word2daterange(args.date) elif args.start_date is None: start, end = word2daterange('yesterday') else: start, end = args.start_date, args.end_date configure_logger(logger, args.loglevel, args.logoutput) res = extract(args.config, start, end, args.sequence, args.batch_size, args.force, args.purge_only) if res == 0: logger.info('SUCCESS') else: logger.info('ERROR') sys.exit(res)
def _run_phase(self, phase, start_date, end_date): phase, sources, targets = phase logger.info('Running phase %r' % phase) self._reset_counters() self._start_transactions(targets) self.database.start_transaction() try: greenlets = Group() # each callable will push its result in the queue for source in sources: exists = self.database.exists(source, start_date, end_date) if exists and not self.force: logger.info('Already done: %s, %s to %s' % ( source.get_id(), start_date, end_date)) continue green = greenlets.spawn(self._get_data, source, start_date, end_date) green.link_value(partial(self._log_transaction, source, start_date, end_date)) green.link_exception(partial(self._error, exception.ExtractError, source)) # looking at the queue pushed = 0 while len(greenlets) > 0 or self.queue.qsize() > 0: gevent.sleep(0) pushed += self._push_to_target(targets) # let's see if we have some errors if len(self.errors) > 0: # yeah! we need to rollback # XXX later we'll do a source-by-source rollback raise exception.RunError(self.errors) except Exception: self._rollback_transactions(targets) self.database.rollback_transaction() raise else: self._commit_transactions(targets) self.database.commit_transaction()
def main(): parser = argparse.ArgumentParser(description='Monolith Aggregator') parser.add_argument('--version', action='store_true', default=False, help='Displays version and exits.') date_group = parser.add_mutually_exclusive_group() date_group.add_argument('--date', default=None, choices=_DATES, help='Date') date_group.add_argument('--start-date', default=None, type=_mkdate, help='Start date.') parser.add_argument('--end-date', default=None, type=_mkdate, help='End date.') parser.add_argument( 'config', help='Configuration file.', ) parser.add_argument('--log-level', dest='loglevel', default='info', choices=LOG_LEVELS.keys() + [key.upper() for key in LOG_LEVELS.keys()], help="log level") parser.add_argument('--log-output', dest='logoutput', default='-', help="log output") parser.add_argument('--sequence', dest='sequence', default=None, help='A comma-separated list of sequences.') parser.add_argument('--batch-size', dest='batch_size', default=None, type=int, help='The size of the batch when writing') parser.add_argument('--force', action='store_true', default=False, help='Forces a run') parser.add_argument('--purge-only', action='store_true', default=False, help='Only run the purge of sources.') parser.add_argument('--retries', default=3, type=int, help='Number of retries') args = parser.parse_args() if args.version: print(__version__) sys.exit(0) if args.date is not None: start, end = word2daterange(args.date) elif args.start_date is None: start, end = word2daterange('yesterday') else: start, end = args.start_date, args.end_date configure_logger(logger, args.loglevel, args.logoutput) res = extract(args.config, start, end, args.sequence, args.batch_size, args.force, args.purge_only) if res == 0: logger.info('SUCCESS') else: logger.info('ERROR') sys.exit(res)