def _get_settings(options):
if options['settings_file']:
settings = InfernoSettings(settings_file=options['settings_file'])
else:
settings = InfernoSettings()
for key, value in options.iteritems():
if value is not None or key not in settings:
settings[key] = value
return settings
def setUp(self):
self.rule = InfernoRule(
day_range=3,
day_offset=1,
day_start=date(2012, 12, 02),
source_tags=['tag1', 'tag2'],
result_tag='result_tag_rule')
self.settings = InfernoSettings(
day_range=4,
day_offset=2,
day_start=date(2011, 12, 02),
source_tags=['tag3', 'tag4'],
result_tag='result_tag_settings')
# expected results
self.result_tag_from_rule = 'result_tag_rule'
self.result_tag_from_settings = 'result_tag_settings'
self.tags_from_rule = [
'tag1:2012-12-01',
'tag1:2012-11-30',
'tag1:2012-11-29',
'tag2:2012-12-01',
'tag2:2012-11-30',
'tag2:2012-11-29']
self.tags_from_settings = ['tag3', 'tag4']
def test_tags_from_settings_and_rule_mix(self):
rule = InfernoRule(source_tags=['tag5'], day_range=2)
settings = InfernoSettings(day_start=date(2011, 12, 01))
actual = JobOptions(rule, settings).tags
# even though day_range=2, we only expect 1 day, as the settings will override
# the rule
expected = ['tag5:2011-12-01']
eq_(actual, expected)
def setUp(self):
settings = InfernoSettings(day_range=2, day_start=date(2011, 11, 12))
rule = InfernoRule(archive=True,
max_blobs=self.MAX_BLOBS,
name='some_rule_name',
archive_tag_prefix='archived',
source_tags=['incoming:data:chunk'])
self.job = InfernoJob(rule, settings)
self.job.disco = Disco()
self.job.ddfs = DDFS()
def expire_data(days, dryrun, extra_tags):
settings = InfernoSettings()
_, ddfs = get_disco_handle(settings["server"])
tags = extract_tags_from_infernyx()
tags = tags.union(extra_tags)
to_delete = []
date_lower = date.today() + timedelta(days=-days)
try:
all_tags = ddfs.list()
except Exception as e:
print "Can not fetch the tag list from ddfs: %s" % e
return
for tag in all_tags:
try:
prefix, ds = tag.rsplit(':', 1)
tag_date = datetime.strptime(ds, "%Y-%m-%d").date()
except:
continue # ignore the non-standard tag name
if prefix in tags and tag_date < date_lower:
to_delete.append(tag)
to_delete.sort() # delete tags with "incoming" first, then the "processed" ones
if dryrun:
if to_delete:
print "Following tags will be deleted:\n"
print "\n".join(to_delete)
else:
print "Nothing to be done"
else:
for tag in to_delete:
try:
print "Deleting tag: %s" % tag
ddfs.delete(tag)
except Exception as e:
print "Failed to delete tag %s: %s" % (tag, e)
def setUp(self):
self.settings = InfernoSettings()
self._make_temp_pid_dir()
self.job = InfernoJob(InfernoRule(name='some_rule_name'), {}, Params())
self.pid_dir = pid.pid_dir(self.settings)
def test_explict_tags_despite_day_range_on_the_rule(self):
rule = InfernoRule(source_tags=['tag:foo', 'tag:bar'], day_range=2)
settings = InfernoSettings(day_range=0)
actual = JobOptions(rule, settings).tags
expected = ['tag:foo', 'tag:bar']
eq_(actual, expected)
def test_explict_tags(self):
rule = InfernoRule(source_tags=['tag:foo', 'tag:bar'])
settings = InfernoSettings()
actual = JobOptions(rule, settings).tags
expected = ['tag:foo', 'tag:bar']
eq_(actual, expected)
def test_tags_from_rule(self):
actual = JobOptions(self.rule, InfernoSettings()).tags
eq_(actual, self.tags_from_rule)
def test_result_tag_from_rule(self):
actual = JobOptions(self.rule, InfernoSettings()).result_tag
eq_(actual, self.result_tag_from_rule)
def test_empty_rule_and_empty_settings(self):
job_options = JobOptions(InfernoRule(), InfernoSettings())
eq_(job_options.tags, [])
eq_(job_options.result_tag, None)
def setUp(self):
self.settings = InfernoSettings()
self.daemon = InfernoDaemon(self.settings)