def main():
parser = argparse.ArgumentParser(description='Monolith Aggregator')
parser.add_argument('--version', action='store_true', default=False,
help='Displays version and exits.')
date_group = parser.add_mutually_exclusive_group()
date_group.add_argument('--date', default=None, choices=_DATES,
help='Date')
date_group.add_argument('--start-date', default=None, type=_mkdate,
help='Start date.')
parser.add_argument('--end-date', default=None, type=_mkdate,
help='End date.')
parser.add_argument('config', help='Configuration file.',)
parser.add_argument('--log-level', dest='loglevel', default='info',
choices=LOG_LEVELS.keys() + [key.upper() for key in
LOG_LEVELS.keys()],
help="log level")
parser.add_argument('--log-output', dest='logoutput', default='-',
help="log output")
parser.add_argument('--sequence', dest='sequence', default=None,
help='A comma-separated list of sequences.')
parser.add_argument('--batch-size', dest='batch_size', default=None,
type=int,
help='The size of the batch when writing')
parser.add_argument('--force', action='store_true', default=False,
help='Forces a run')
parser.add_argument('--purge-only', action='store_true', default=False,
help='Only run the purge of sources.')
parser.add_argument('--retries', default=3, type=int,
help='Number of retries')
args = parser.parse_args()
if args.version:
print(__version__)
sys.exit(0)
if args.date is not None:
start, end = word2daterange(args.date)
elif args.start_date is None:
start, end = word2daterange('yesterday')
else:
start, end = args.start_date, args.end_date
configure_logger(logger, args.loglevel, args.logoutput)
res = extract(args.config, start, end, args.sequence, args.batch_size,
args.force, args.purge_only)
if res == 0:
logger.info('SUCCESS')
else:
logger.info('ERROR')
sys.exit(res)
def _run_phase(self, phase, start_date, end_date):
phase, sources, targets = phase
logger.info('Running phase %r' % phase)
self._reset_counters()
self._start_transactions(targets)
self.database.start_transaction()
try:
greenlets = Group()
# each callable will push its result in the queue
for source in sources:
exists = self.database.exists(source, start_date, end_date)
if exists and not self.force:
logger.info('Already done: %s, %s to %s' % (
source.get_id(), start_date, end_date))
continue
green = greenlets.spawn(self._get_data, source,
start_date, end_date)
green.link_value(partial(self._log_transaction, source,
start_date, end_date))
green.link_exception(partial(self._error,
exception.ExtractError, source))
# looking at the queue
pushed = 0
while len(greenlets) > 0 or self.queue.qsize() > 0:
gevent.sleep(0)
pushed += self._push_to_target(targets)
# let's see if we have some errors
if len(self.errors) > 0:
# yeah! we need to rollback
# XXX later we'll do a source-by-source rollback
raise exception.RunError(self.errors)
except Exception:
self._rollback_transactions(targets)
self.database.rollback_transaction()
raise
else:
self._commit_transactions(targets)
self.database.commit_transaction()
def main():
parser = argparse.ArgumentParser(description='Monolith Aggregator')
parser.add_argument('--version',
action='store_true',
default=False,
help='Displays version and exits.')
date_group = parser.add_mutually_exclusive_group()
date_group.add_argument('--date',
default=None,
choices=_DATES,
help='Date')
date_group.add_argument('--start-date',
default=None,
type=_mkdate,
help='Start date.')
parser.add_argument('--end-date',
default=None,
type=_mkdate,
help='End date.')
parser.add_argument(
'config',
help='Configuration file.',
)
parser.add_argument('--log-level',
dest='loglevel',
default='info',
choices=LOG_LEVELS.keys() +
[key.upper() for key in LOG_LEVELS.keys()],
help="log level")
parser.add_argument('--log-output',
dest='logoutput',
default='-',
help="log output")
parser.add_argument('--sequence',
dest='sequence',
default=None,
help='A comma-separated list of sequences.')
parser.add_argument('--batch-size',
dest='batch_size',
default=None,
type=int,
help='The size of the batch when writing')
parser.add_argument('--force',
action='store_true',
default=False,
help='Forces a run')
parser.add_argument('--purge-only',
action='store_true',
default=False,
help='Only run the purge of sources.')
parser.add_argument('--retries',
default=3,
type=int,
help='Number of retries')
args = parser.parse_args()
if args.version:
print(__version__)
sys.exit(0)
if args.date is not None:
start, end = word2daterange(args.date)
elif args.start_date is None:
start, end = word2daterange('yesterday')
else:
start, end = args.start_date, args.end_date
configure_logger(logger, args.loglevel, args.logoutput)
res = extract(args.config, start, end, args.sequence, args.batch_size,
args.force, args.purge_only)
if res == 0:
logger.info('SUCCESS')
else:
logger.info('ERROR')
sys.exit(res)