def run_rule_async(rule_name, settings):
setproctitle("inferno - %s" % rule_name)
signal.signal(signal.SIGHUP, signal.SIG_IGN)
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_IGN)
rules = get_rules_by_name(
rule_name, settings['rules_directory'], immediate=False)
if rules and len(rules) > 0:
rule = rules[0]
else:
log.error('No rule exists with rule_name: %s' % rule_name)
raise Exception('No rule exists with rule_name: %s' % rule_name)
pid_dir = pid.pid_dir(settings)
log.info("Running %s" % rule.name)
try:
pid.create_pid(pid_dir, rule, str(os.getpid()))
execute_rule(rule, settings)
except Exception as e:
log.exception('%s: %s', rule_name, e)
if not rule.retry:
pid.create_last_run(pid_dir, rule)
else:
pid.create_last_run(pid_dir, rule)
finally:
pid.remove_pid(pid_dir, rule)
os._exit(0)
def run_rule_async(rule_name, settings):
setproctitle("inferno - %s" % rule_name)
signal.signal(signal.SIGHUP, signal.SIG_IGN)
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_IGN)
rules = get_rules_by_name(rule_name,
settings['rules_directory'],
immediate=False)
if rules and len(rules) > 0:
rule = rules[0]
else:
log.error('No rule exists with rule_name: %s' % rule_name)
raise Exception('No rule exists with rule_name: %s' % rule_name)
pid_dir = pid.pid_dir(settings)
log.info("Running %s" % rule.name)
try:
pid.create_pid(pid_dir, rule, str(os.getpid()))
execute_rule(rule, settings)
except Exception as e:
log.error('%s: %s', rule_name, e)
if not rule.retry:
pid.create_last_run(pid_dir, rule)
else:
pid.create_last_run(pid_dir, rule)
finally:
pid.remove_pid(pid_dir, rule)
os._exit(0)
def run_rule_async(rule_name, settings):
setproctitle("inferno - %s" % rule_name)
signal.signal(signal.SIGHUP, signal.SIG_IGN)
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_IGN)
rules = get_rules_by_name(rule_name,
settings['rules_directory'],
immediate=False)
if rules and len(rules) > 0:
rule = rules[0]
else:
log.error('No rule exists with rule_name: %s' % rule_name)
raise Exception('No rule exists with rule_name: %s' % rule_name)
pid_dir = pid.pid_dir(settings)
log.info("Running %s" % rule.name)
try:
pid.create_pid(pid_dir, rule, str(os.getpid()))
execute_rule(rule, settings)
except Exception as e:
log.error('%s: %s', rule_name, e)
if rule.retry:
if rule.retry_limit > pid.get_retry_count(pid_dir, rule):
log.error('%s: will be retried in %s hour(s)', rule_name,
rule.retry_delay)
pid.create_next_retry(pid_dir, rule)
pid.increment_retry_count(pid_dir, rule)
else:
log.error('%s: failed max retry limit (%s)', rule_name,
rule.retry_limit)
pid.create_failed(pid_dir, rule)
else:
if rule.retry:
# Failed before, however, ran successfully this time. Clean up fail/retry files
pid.clean_up(pid_dir, rule)
pid.create_last_complete(pid_dir, rule)
finally:
pid.remove_pid(pid_dir, rule)
os._exit(0)
def run_rule_async(rule_name, settings):
setproctitle("inferno - %s" % rule_name)
signal.signal(signal.SIGHUP, signal.SIG_IGN)
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_IGN)
rules = get_rules_by_name(
rule_name, settings['rules_directory'], immediate=False)
if rules and len(rules) > 0:
rule = rules[0]
else:
log.error('No rule exists with rule_name: %s' % rule_name)
raise Exception('No rule exists with rule_name: %s' % rule_name)
pid_dir = pid.pid_dir(settings)
log.info("Running %s" % rule.name)
try:
pid.create_pid(pid_dir, rule, str(os.getpid()))
execute_rule(rule, settings)
except Exception as e:
log.error('%s: %s', rule_name, e)
if rule.retry:
if rule.retry_limit > pid.get_retry_count(pid_dir, rule):
log.error('%s: will be retried in %s hour(s)', rule_name, rule.retry_delay)
pid.create_next_retry(pid_dir, rule)
pid.increment_retry_count(pid_dir, rule)
else:
log.error('%s: failed max retry limit (%s)', rule_name, rule.retry_limit)
pid.create_failed(pid_dir, rule)
else:
if rule.retry:
# Failed before, however, ran successfully this time. Clean up fail/retry files
pid.clean_up(pid_dir, rule)
pid.create_last_complete(pid_dir, rule)
finally:
pid.remove_pid(pid_dir, rule)
os._exit(0)
def main(argv=sys.argv):
options, parser = _get_options(argv[1:])
settings = _get_settings(options)
if options['example_rules']:
try:
os.mkdir(options['example_rules'])
here = os.path.dirname(__file__)
src_dir = os.path.join(here, '..', 'example_rules')
src_dir = os.path.abspath(src_dir)
dst_dir = os.path.abspath(options['example_rules'])
for name in os.listdir(src_dir):
if name.endswith('.py'):
src = os.path.join(src_dir, name)
dst = os.path.join(dst_dir, name)
shutil.copy(src, dst)
print '\n\tCreated example rules dir:\n\n\t\t%s' % dst_dir
for name in os.listdir(dst_dir):
print '\t\t\t', name
except Exception as e:
print 'Error creating example rules dir %r' % (e)
finally:
return
_setup_logging(settings)
for path in settings.get('extra_python_paths'):
sys.path.insert(0, path)
if options['process_results']:
settings['no_purge'] = True
rules_dir = options.get('rules_directory')
if not rules_dir:
rules_dir = settings.get('rules_directory')
try:
rule_name = options['process_results'].split('@')[0]
job_name = options['process_results'].split('.')[1]
rule = get_rules_by_name(rule_name, rules_dir, immediate=True)[0]
job = InfernoJob(rule, settings)
status, results = job.disco.results(job_name)
if status == 'ready':
if job.rule.rule_init_function:
job.rule.rule_init_function(job.params)
rule.result_processor(rule.result_iterator(results),
params=job.params,
job_id=job_name)
except Exception as e:
import traceback
trace = traceback.format_exc(15)
log.error(trace)
log.error("Error processing results for job: %s %s" %
(options['process_results'], e))
raise e
elif options['process_map']:
settings['no_purge'] = True
rules_dir = options.get('rules_directory')
if not rules_dir:
rules_dir = settings.get('rules_directory')
try:
rule_name = options['process_map'].split('@')[0]
job_name = options['process_map'].split('.')[1]
rule = get_rules_by_name(rule_name, rules_dir, immediate=True)[0]
rule.map_function = None
rule.source_tags = []
disco, ddfs = get_disco_handle(settings.get('server'))
rule.source_urls = disco.mapresults(job_name)
job = InfernoJob(rule, settings)
if job.start():
job.wait()
except Exception as e:
import traceback
trace = traceback.format_exc(15)
log.error(trace)
log.error("Error processing map results for job: %s %s" %
(options['process_map'], e))
raise e
elif options['immediate_rule']:
# run inferno in 'immediate' mode
settings['no_purge'] = True
setproctitle('inferno - immediate.%s' % options['immediate_rule'])
immed_rule = settings.get('immediate_rule')
rules_dir = settings.get('rules_directory')
rules = get_rules_by_name(immed_rule, rules_dir, immediate=True)
try:
for rule in rules:
execute_rule(rule, settings)
except Exception as e:
import traceback
trace = traceback.format_exc(15)
log.error('Job failed: %s' % e.message)
log.error(trace)
exit(1)
elif options['run_daemon']:
# run inferno in 'daemon' mode
from inferno.lib.daemon import InfernoDaemon
setproctitle('inferno - master')
InfernoDaemon(settings).start()
else:
# Display help when no options specified
parser.print_help()
def main(argv=sys.argv):
options, parser = _get_options(argv[1:])
settings = _get_settings(options)
if options['example_rules']:
try:
os.mkdir(options['example_rules'])
here = os.path.dirname(__file__)
src_dir = os.path.join(here, '..', 'example_rules')
src_dir = os.path.abspath(src_dir)
dst_dir = os.path.abspath(options['example_rules'])
for name in os.listdir(src_dir):
if name.endswith('.py'):
src = os.path.join(src_dir, name)
dst = os.path.join(dst_dir, name)
shutil.copy(src, dst)
print '\n\tCreated example rules dir:\n\n\t\t%s' % dst_dir
for name in os.listdir(dst_dir):
print '\t\t\t', name
except Exception as e:
print 'Error creating example rules dir %r' % (e)
finally:
return
_setup_logging(settings)
for path in settings.get('extra_python_paths'):
sys.path.insert(0, path)
if options['process_results']:
settings['no_purge'] = True
rules_dir = options.get('rules_directory')
if not rules_dir:
rules_dir = settings.get('rules_directory')
try:
rule_name = options['process_results'].split('@')[0]
job_name = options['process_results'].split('.')[1]
rule = get_rules_by_name(rule_name, rules_dir, immediate=True)[0]
job = InfernoJob(rule, settings)
status, results = job.disco.results(job_name)
if status == 'ready':
if job.rule.rule_init_function:
job.rule.rule_init_function(job.params)
rule.result_processor(rule.result_iterator(results), params=job.params, job_id=job_name)
except Exception as e:
import traceback
trace = traceback.format_exc(15)
log.error(trace)
log.error("Error processing results for job: %s %s" % (options['process_results'], e))
raise e
elif options['process_map']:
settings['no_purge'] = True
rules_dir = options.get('rules_directory')
if not rules_dir:
rules_dir = settings.get('rules_directory')
try:
rule_name = options['process_map'].split('@')[0]
job_name = options['process_map'].split('.')[1]
rule = get_rules_by_name(rule_name, rules_dir, immediate=True)[0]
rule.map_function = None
rule.source_tags = []
disco, ddfs = get_disco_handle(settings.get('server'))
rule.source_urls = disco.mapresults(job_name)
job = InfernoJob(rule, settings)
if job.start():
job.wait()
except Exception as e:
import traceback
trace = traceback.format_exc(15)
log.error(trace)
log.error("Error processing map results for job: %s %s" % (options['process_map'], e))
raise e
elif options['immediate_rule']:
# run inferno in 'immediate' mode
settings['no_purge'] = True
setproctitle('inferno - immediate.%s' % options['immediate_rule'])
immed_rule = settings.get('immediate_rule')
rules_dir = settings.get('rules_directory')
rules = get_rules_by_name(immed_rule, rules_dir, immediate=True)
try:
for rule in rules:
execute_rule(rule, settings)
except Exception as e:
import traceback
trace = traceback.format_exc(15)
log.error('Job failed: %s' % e.message)
log.error(trace)
exit(1)
elif options['run_daemon']:
# run inferno in 'daemon' mode
from inferno.lib.daemon import InfernoDaemon
setproctitle('inferno - master')
InfernoDaemon(settings).start()
else:
# Display help when no options specified
parser.print_help()