Beispiel #1
0
def run_rule_async(rule_name, settings):
    setproctitle("inferno - %s" % rule_name)
    signal.signal(signal.SIGHUP, signal.SIG_IGN)
    signal.signal(signal.SIGINT, signal.SIG_IGN)
    signal.signal(signal.SIGTERM, signal.SIG_IGN)

    rules = get_rules_by_name(
        rule_name, settings['rules_directory'], immediate=False)
    if rules and len(rules) > 0:
        rule = rules[0]
    else:
        log.error('No rule exists with rule_name: %s' % rule_name)
        raise Exception('No rule exists with rule_name: %s' % rule_name)

    pid_dir = pid.pid_dir(settings)
    log.info("Running %s" % rule.name)
    try:
        pid.create_pid(pid_dir, rule, str(os.getpid()))
        execute_rule(rule, settings)
    except Exception as e:
        log.exception('%s: %s', rule_name, e)
        if not rule.retry:
            pid.create_last_run(pid_dir, rule)
    else:
        pid.create_last_run(pid_dir, rule)
    finally:
        pid.remove_pid(pid_dir, rule)
        os._exit(0)
Beispiel #2
0
def run_rule_async(rule_name, settings):
    setproctitle("inferno - %s" % rule_name)
    signal.signal(signal.SIGHUP, signal.SIG_IGN)
    signal.signal(signal.SIGINT, signal.SIG_IGN)
    signal.signal(signal.SIGTERM, signal.SIG_IGN)

    rules = get_rules_by_name(rule_name,
                              settings['rules_directory'],
                              immediate=False)
    if rules and len(rules) > 0:
        rule = rules[0]
    else:
        log.error('No rule exists with rule_name: %s' % rule_name)
        raise Exception('No rule exists with rule_name: %s' % rule_name)

    pid_dir = pid.pid_dir(settings)
    log.info("Running %s" % rule.name)
    try:
        pid.create_pid(pid_dir, rule, str(os.getpid()))
        execute_rule(rule, settings)
    except Exception as e:
        log.error('%s: %s', rule_name, e)
        if not rule.retry:
            pid.create_last_run(pid_dir, rule)
    else:
        pid.create_last_run(pid_dir, rule)
    finally:
        pid.remove_pid(pid_dir, rule)
        os._exit(0)
Beispiel #3
0
def run_rule_async(rule_name, settings):
    setproctitle("inferno - %s" % rule_name)
    signal.signal(signal.SIGHUP, signal.SIG_IGN)
    signal.signal(signal.SIGINT, signal.SIG_IGN)
    signal.signal(signal.SIGTERM, signal.SIG_IGN)

    rules = get_rules_by_name(rule_name,
                              settings['rules_directory'],
                              immediate=False)
    if rules and len(rules) > 0:
        rule = rules[0]
    else:
        log.error('No rule exists with rule_name: %s' % rule_name)
        raise Exception('No rule exists with rule_name: %s' % rule_name)

    pid_dir = pid.pid_dir(settings)
    log.info("Running %s" % rule.name)
    try:
        pid.create_pid(pid_dir, rule, str(os.getpid()))
        execute_rule(rule, settings)
    except Exception as e:
        log.error('%s: %s', rule_name, e)
        if rule.retry:
            if rule.retry_limit > pid.get_retry_count(pid_dir, rule):
                log.error('%s: will be retried in %s hour(s)', rule_name,
                          rule.retry_delay)
                pid.create_next_retry(pid_dir, rule)
                pid.increment_retry_count(pid_dir, rule)
            else:
                log.error('%s: failed max retry limit (%s)', rule_name,
                          rule.retry_limit)
                pid.create_failed(pid_dir, rule)
    else:
        if rule.retry:
            # Failed before, however, ran successfully this time. Clean up fail/retry files
            pid.clean_up(pid_dir, rule)
        pid.create_last_complete(pid_dir, rule)
    finally:
        pid.remove_pid(pid_dir, rule)
        os._exit(0)
Beispiel #4
0
def run_rule_async(rule_name, settings):
    setproctitle("inferno - %s" % rule_name)
    signal.signal(signal.SIGHUP, signal.SIG_IGN)
    signal.signal(signal.SIGINT, signal.SIG_IGN)
    signal.signal(signal.SIGTERM, signal.SIG_IGN)

    rules = get_rules_by_name(
        rule_name, settings['rules_directory'], immediate=False)
    if rules and len(rules) > 0:
        rule = rules[0]
    else:
        log.error('No rule exists with rule_name: %s' % rule_name)
        raise Exception('No rule exists with rule_name: %s' % rule_name)

    pid_dir = pid.pid_dir(settings)
    log.info("Running %s" % rule.name)
    try:
        pid.create_pid(pid_dir, rule, str(os.getpid()))
        execute_rule(rule, settings)
    except Exception as e:
        log.error('%s: %s', rule_name, e)
        if rule.retry:
            if rule.retry_limit > pid.get_retry_count(pid_dir, rule):
                log.error('%s: will be retried in %s hour(s)', rule_name, rule.retry_delay)
                pid.create_next_retry(pid_dir, rule)
                pid.increment_retry_count(pid_dir, rule)
            else:
                log.error('%s: failed max retry limit (%s)', rule_name, rule.retry_limit)
                pid.create_failed(pid_dir, rule)
    else:
        if rule.retry:
            # Failed before, however, ran successfully this time. Clean up fail/retry files
            pid.clean_up(pid_dir, rule)
        pid.create_last_complete(pid_dir, rule)
    finally:
        pid.remove_pid(pid_dir, rule)
        os._exit(0)
Beispiel #5
0
def main(argv=sys.argv):
    options, parser = _get_options(argv[1:])
    settings = _get_settings(options)

    if options['example_rules']:
        try:
            os.mkdir(options['example_rules'])
            here = os.path.dirname(__file__)
            src_dir = os.path.join(here, '..', 'example_rules')
            src_dir = os.path.abspath(src_dir)
            dst_dir = os.path.abspath(options['example_rules'])
            for name in os.listdir(src_dir):
                if name.endswith('.py'):
                    src = os.path.join(src_dir, name)
                    dst = os.path.join(dst_dir, name)
                    shutil.copy(src, dst)
            print '\n\tCreated example rules dir:\n\n\t\t%s' % dst_dir
            for name in os.listdir(dst_dir):
                print '\t\t\t', name
        except Exception as e:
            print 'Error creating example rules dir %r' % (e)
        finally:
            return

    _setup_logging(settings)

    for path in settings.get('extra_python_paths'):
        sys.path.insert(0, path)

    if options['process_results']:
        settings['no_purge'] = True
        rules_dir = options.get('rules_directory')
        if not rules_dir:
            rules_dir = settings.get('rules_directory')
        try:
            rule_name = options['process_results'].split('@')[0]
            job_name = options['process_results'].split('.')[1]
            rule = get_rules_by_name(rule_name, rules_dir, immediate=True)[0]
            job = InfernoJob(rule, settings)
            status, results = job.disco.results(job_name)
            if status == 'ready':
                if job.rule.rule_init_function:
                    job.rule.rule_init_function(job.params)
                rule.result_processor(rule.result_iterator(results),
                                      params=job.params,
                                      job_id=job_name)
        except Exception as e:
            import traceback
            trace = traceback.format_exc(15)
            log.error(trace)
            log.error("Error processing results for job: %s %s" %
                      (options['process_results'], e))
            raise e
    elif options['process_map']:
        settings['no_purge'] = True
        rules_dir = options.get('rules_directory')
        if not rules_dir:
            rules_dir = settings.get('rules_directory')
        try:
            rule_name = options['process_map'].split('@')[0]
            job_name = options['process_map'].split('.')[1]
            rule = get_rules_by_name(rule_name, rules_dir, immediate=True)[0]
            rule.map_function = None
            rule.source_tags = []
            disco, ddfs = get_disco_handle(settings.get('server'))
            rule.source_urls = disco.mapresults(job_name)
            job = InfernoJob(rule, settings)
            if job.start():
                job.wait()
        except Exception as e:
            import traceback
            trace = traceback.format_exc(15)
            log.error(trace)
            log.error("Error processing map results for job: %s %s" %
                      (options['process_map'], e))
            raise e
    elif options['immediate_rule']:
        # run inferno in 'immediate' mode
        settings['no_purge'] = True
        setproctitle('inferno - immediate.%s' % options['immediate_rule'])
        immed_rule = settings.get('immediate_rule')
        rules_dir = settings.get('rules_directory')
        rules = get_rules_by_name(immed_rule, rules_dir, immediate=True)
        try:
            for rule in rules:
                execute_rule(rule, settings)
        except Exception as e:
            import traceback
            trace = traceback.format_exc(15)
            log.error('Job failed: %s' % e.message)
            log.error(trace)
            exit(1)

    elif options['run_daemon']:
        # run inferno in 'daemon' mode
        from inferno.lib.daemon import InfernoDaemon
        setproctitle('inferno - master')
        InfernoDaemon(settings).start()

    else:
        # Display help when no options specified
        parser.print_help()
Beispiel #6
0
def main(argv=sys.argv):
    options, parser = _get_options(argv[1:])
    settings = _get_settings(options)

    if options['example_rules']:
        try:
            os.mkdir(options['example_rules'])
            here = os.path.dirname(__file__)
            src_dir = os.path.join(here, '..', 'example_rules')
            src_dir = os.path.abspath(src_dir)
            dst_dir = os.path.abspath(options['example_rules'])
            for name in os.listdir(src_dir):
                if name.endswith('.py'):
                    src = os.path.join(src_dir, name)
                    dst = os.path.join(dst_dir, name)
                    shutil.copy(src, dst)
            print '\n\tCreated example rules dir:\n\n\t\t%s' % dst_dir
            for name in os.listdir(dst_dir):
                print '\t\t\t', name
        except Exception as e:
            print 'Error creating example rules dir %r' % (e)
        finally:
            return

    _setup_logging(settings)

    for path in settings.get('extra_python_paths'):
        sys.path.insert(0, path)

    if options['process_results']:
        settings['no_purge'] = True
        rules_dir = options.get('rules_directory')
        if not rules_dir:
            rules_dir = settings.get('rules_directory')
        try:
            rule_name = options['process_results'].split('@')[0]
            job_name = options['process_results'].split('.')[1]
            rule = get_rules_by_name(rule_name, rules_dir, immediate=True)[0]
            job = InfernoJob(rule, settings)
            status, results = job.disco.results(job_name)
            if status == 'ready':
                if job.rule.rule_init_function:
                    job.rule.rule_init_function(job.params)
                rule.result_processor(rule.result_iterator(results), params=job.params, job_id=job_name)
        except Exception as e:
            import traceback
            trace = traceback.format_exc(15)
            log.error(trace)
            log.error("Error processing results for job: %s %s" % (options['process_results'], e))
            raise e
    elif options['process_map']:
        settings['no_purge'] = True
        rules_dir = options.get('rules_directory')
        if not rules_dir:
            rules_dir = settings.get('rules_directory')
        try:
            rule_name = options['process_map'].split('@')[0]
            job_name = options['process_map'].split('.')[1]
            rule = get_rules_by_name(rule_name, rules_dir, immediate=True)[0]
            rule.map_function = None
            rule.source_tags = []
            disco, ddfs = get_disco_handle(settings.get('server'))
            rule.source_urls = disco.mapresults(job_name)
            job = InfernoJob(rule, settings)
            if job.start():
                job.wait()
        except Exception as e:
            import traceback
            trace = traceback.format_exc(15)
            log.error(trace)
            log.error("Error processing map results for job: %s %s" % (options['process_map'], e))
            raise e
    elif options['immediate_rule']:
        # run inferno in 'immediate' mode
        settings['no_purge'] = True
        setproctitle('inferno - immediate.%s' % options['immediate_rule'])
        immed_rule = settings.get('immediate_rule')
        rules_dir = settings.get('rules_directory')
        rules = get_rules_by_name(immed_rule, rules_dir, immediate=True)
        try:
            for rule in rules:
                execute_rule(rule, settings)
        except Exception as e:
            import traceback
            trace = traceback.format_exc(15)
            log.error('Job failed: %s' % e.message)
            log.error(trace)
            exit(1)

    elif options['run_daemon']:
        # run inferno in 'daemon' mode
        from inferno.lib.daemon import InfernoDaemon
        setproctitle('inferno - master')
        InfernoDaemon(settings).start()

    else:
        # Display help when no options specified
        parser.print_help()