Beispiel #1
0
def main():
    parser = argparse.ArgumentParser(description='Monolith Aggregator')

    parser.add_argument('--version', action='store_true', default=False,
                        help='Displays version and exits.')

    date_group = parser.add_mutually_exclusive_group()
    date_group.add_argument('--date', default=None, choices=_DATES,
                            help='Date')
    date_group.add_argument('--start-date', default=None, type=_mkdate,
                            help='Start date.')
    parser.add_argument('--end-date', default=None, type=_mkdate,
                        help='End date.')
    parser.add_argument('config', help='Configuration file.',)
    parser.add_argument('--log-level', dest='loglevel', default='info',
                        choices=LOG_LEVELS.keys() + [key.upper() for key in
                                                     LOG_LEVELS.keys()],
                        help="log level")
    parser.add_argument('--log-output', dest='logoutput', default='-',
                        help="log output")
    parser.add_argument('--sequence', dest='sequence', default=None,
                        help='A comma-separated list of sequences.')
    parser.add_argument('--batch-size', dest='batch_size', default=None,
                        type=int,
                        help='The size of the batch when writing')
    parser.add_argument('--force', action='store_true', default=False,
                        help='Forces a run')
    parser.add_argument('--purge-only', action='store_true', default=False,
                        help='Only run the purge of sources.')
    parser.add_argument('--retries', default=3, type=int,
                        help='Number of retries')
    args = parser.parse_args()

    if args.version:
        print(__version__)
        sys.exit(0)

    if args.date is not None:
        start, end = word2daterange(args.date)
    elif args.start_date is None:
        start, end = word2daterange('yesterday')
    else:
        start, end = args.start_date, args.end_date

    configure_logger(logger, args.loglevel, args.logoutput)
    res = extract(args.config, start, end, args.sequence, args.batch_size,
                  args.force, args.purge_only)

    if res == 0:
        logger.info('SUCCESS')
    else:
        logger.info('ERROR')

    sys.exit(res)
Beispiel #2
0
    def _run_phase(self, phase, start_date, end_date):
        phase, sources, targets = phase
        logger.info('Running phase %r' % phase)
        self._reset_counters()

        self._start_transactions(targets)
        self.database.start_transaction()
        try:
            greenlets = Group()
            # each callable will push its result in the queue
            for source in sources:
                exists = self.database.exists(source, start_date, end_date)
                if exists and not self.force:
                    logger.info('Already done: %s, %s to %s' % (
                        source.get_id(), start_date, end_date))
                    continue

                green = greenlets.spawn(self._get_data, source,
                                        start_date, end_date)
                green.link_value(partial(self._log_transaction, source,
                                         start_date, end_date))
                green.link_exception(partial(self._error,
                                             exception.ExtractError, source))

            # looking at the queue
            pushed = 0

            while len(greenlets) > 0 or self.queue.qsize() > 0:
                gevent.sleep(0)
                pushed += self._push_to_target(targets)
                # let's see if we have some errors
                if len(self.errors) > 0:
                    # yeah! we need to rollback
                    # XXX later we'll do a source-by-source rollback
                    raise exception.RunError(self.errors)

        except Exception:
            self._rollback_transactions(targets)
            self.database.rollback_transaction()
            raise
        else:
            self._commit_transactions(targets)
            self.database.commit_transaction()
Beispiel #3
0
def main():
    parser = argparse.ArgumentParser(description='Monolith Aggregator')

    parser.add_argument('--version',
                        action='store_true',
                        default=False,
                        help='Displays version and exits.')

    date_group = parser.add_mutually_exclusive_group()
    date_group.add_argument('--date',
                            default=None,
                            choices=_DATES,
                            help='Date')
    date_group.add_argument('--start-date',
                            default=None,
                            type=_mkdate,
                            help='Start date.')
    parser.add_argument('--end-date',
                        default=None,
                        type=_mkdate,
                        help='End date.')
    parser.add_argument(
        'config',
        help='Configuration file.',
    )
    parser.add_argument('--log-level',
                        dest='loglevel',
                        default='info',
                        choices=LOG_LEVELS.keys() +
                        [key.upper() for key in LOG_LEVELS.keys()],
                        help="log level")
    parser.add_argument('--log-output',
                        dest='logoutput',
                        default='-',
                        help="log output")
    parser.add_argument('--sequence',
                        dest='sequence',
                        default=None,
                        help='A comma-separated list of sequences.')
    parser.add_argument('--batch-size',
                        dest='batch_size',
                        default=None,
                        type=int,
                        help='The size of the batch when writing')
    parser.add_argument('--force',
                        action='store_true',
                        default=False,
                        help='Forces a run')
    parser.add_argument('--purge-only',
                        action='store_true',
                        default=False,
                        help='Only run the purge of sources.')
    parser.add_argument('--retries',
                        default=3,
                        type=int,
                        help='Number of retries')
    args = parser.parse_args()

    if args.version:
        print(__version__)
        sys.exit(0)

    if args.date is not None:
        start, end = word2daterange(args.date)
    elif args.start_date is None:
        start, end = word2daterange('yesterday')
    else:
        start, end = args.start_date, args.end_date

    configure_logger(logger, args.loglevel, args.logoutput)
    res = extract(args.config, start, end, args.sequence, args.batch_size,
                  args.force, args.purge_only)

    if res == 0:
        logger.info('SUCCESS')
    else:
        logger.info('ERROR')

    sys.exit(res)