def test_run_cleaner():
logger.debug(
os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0])
config = conf().cleaner
cleaner = Cleaner(config)
date = datetime.datetime.now() - datetime.timedelta(
days=config["retention_period"])
# check that there are no destinations
cleaner.delete_destination_keys(date)
assert len(cleaner.get_destinations(date)) == 0
# first put a few destinations and keys
prefix = config["dest_path"] + date.strftime(DATE_FORMAT) + "/"
put_key(config["url"], prefix + "EC1")
put_key(
config["url"], config["diss_path"] + "EC1/date=" +
date.strftime(DATE_FORMAT) + "/aaaa")
put_key(
config["url"], config["diss_path"] + "EC1/date=" +
date.strftime(DATE_FORMAT) + "/bbbb")
put_key(
config["url"], config["diss_path"] + "EC1/date=" +
date.strftime(DATE_FORMAT) + "/cccc")
# run the whole workflow
cleaner.run()
# check that there are no destinations
assert len(cleaner.get_destinations(date)) == 0
def test_get_current_server_rev():
logger.debug(
os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0])
compactor = Compactor(conf().compactor)
rev = compactor.get_current_server_rev()
print(rev)
assert rev is not None
assert rev > 0
def test_compact():
logger.debug(
os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0])
compactor = Compactor(conf().compactor)
# first get the current version
rev = compactor.get_current_server_rev()
# compact revision
res = compactor.compact(rev)
assert res
def test_save_rev():
logger.debug(
os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0])
compactor = Compactor(conf().compactor)
rev = random.randint(1, 100000)
res = compactor.save_rev(rev, datetime.now())
assert res
history = compactor.get_history()
rev_h = list(filter(lambda h: h["revision"] == rev, history))
assert len(rev_h) == 1
def test_clean_history():
logger.debug(
os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0])
compactor = Compactor(conf().compactor)
# first save a revision
res = compactor.save_rev(
1112,
datetime.now().replace(hour=0, minute=0, second=0, microsecond=0))
assert res
# now retrieve it back
old_rev = compactor.clean_history(datetime.now().replace(hour=0,
minute=0,
second=0,
microsecond=0))
assert old_rev == 1112
def test_get_destinations():
logger.debug(
os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0])
config = conf().cleaner
cleaner = Cleaner(config)
date = datetime.datetime.now() - datetime.timedelta(days=1)
# first put a few destinations
prefix = config["dest_path"] + date.strftime(DATE_FORMAT) + "/"
put_key(config["url"], prefix + "EC1")
put_key(config["url"], prefix + "FOO")
put_key(config["url"], prefix + "FOO2")
# retrieve destinations
dests = cleaner.get_destinations(date)
assert len(dests) == 3
def test_delete_mars_keys():
logger.debug(
os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0])
config = conf().cleaner
cleaner = Cleaner(config)
date = datetime.datetime.now() - datetime.timedelta(days=1)
# first put a few MARS keys
put_key(
config["url"],
config["mars_path"] + "date=" + date.strftime(DATE_FORMAT) + "/aaaa")
put_key(
config["url"],
config["mars_path"] + "date=" + date.strftime(DATE_FORMAT) + "/bbbb")
put_key(
config["url"],
config["mars_path"] + "date=" + date.strftime(DATE_FORMAT) + "/cccc")
# delete MARS keys
n_deleted = cleaner.delete_keys(date)
assert n_deleted == 3
def test_compact_run():
logger.debug(
os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0])
compactor = Compactor(conf().compactor)
compactor.clean_history(datetime.now())
rev_init = compactor.get_current_server_rev()
for i in range(5):
compactor.run(sec_ret_per=True)
time.sleep(1)
rev_end = compactor.get_current_server_rev()
# there are 2 revisions for every run: one for saving the rev and one for cleaning the history
assert rev_end - rev_init == 10
try:
compactor.compact(rev_end - 6)
assert False
except AssertionError as e:
assert True
try:
compactor.compact(rev_end - 4)
assert True
except AssertionError as e:
assert False