def test_run_cleaner():
logger.debug(
os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0])
config = conf().cleaner
cleaner = Cleaner(config)
date = datetime.datetime.now() - datetime.timedelta(
days=config["retention_period"])
# check that there are no destinations
cleaner.delete_destination_keys(date)
assert len(cleaner.get_destinations(date)) == 0
# first put a few destinations and keys
prefix = config["dest_path"] + date.strftime(DATE_FORMAT) + "/"
put_key(config["url"], prefix + "EC1")
put_key(
config["url"], config["diss_path"] + "EC1/date=" +
date.strftime(DATE_FORMAT) + "/aaaa")
put_key(
config["url"], config["diss_path"] + "EC1/date=" +
date.strftime(DATE_FORMAT) + "/bbbb")
put_key(
config["url"], config["diss_path"] + "EC1/date=" +
date.strftime(DATE_FORMAT) + "/cccc")
# run the whole workflow
cleaner.run()
# check that there are no destinations
assert len(cleaner.get_destinations(date)) == 0
def test_get_destinations():
logger.debug(
os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0])
config = conf().cleaner
cleaner = Cleaner(config)
date = datetime.datetime.now() - datetime.timedelta(days=1)
# first put a few destinations
prefix = config["dest_path"] + date.strftime(DATE_FORMAT) + "/"
put_key(config["url"], prefix + "EC1")
put_key(config["url"], prefix + "FOO")
put_key(config["url"], prefix + "FOO2")
# retrieve destinations
dests = cleaner.get_destinations(date)
assert len(dests) == 3
def test_delete_mars_keys():
logger.debug(
os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0])
config = conf().cleaner
cleaner = Cleaner(config)
date = datetime.datetime.now() - datetime.timedelta(days=1)
# first put a few MARS keys
put_key(
config["url"],
config["mars_path"] + "date=" + date.strftime(DATE_FORMAT) + "/aaaa")
put_key(
config["url"],
config["mars_path"] + "date=" + date.strftime(DATE_FORMAT) + "/bbbb")
put_key(
config["url"],
config["mars_path"] + "date=" + date.strftime(DATE_FORMAT) + "/cccc")
# delete MARS keys
n_deleted = cleaner.delete_keys(date)
assert n_deleted == 3
def main():
# load the configuration
config = Config()
logger.info(f"Running Aviso-admin v.{__version__}")
logger.info(f"Configuration loaded: {config}")
# instantiate the compactor and cleaner
compactor = Compactor(config.compactor)
cleaner = Cleaner(config.cleaner)
# Every day at scheduled time run the compactor
if compactor.enabled:
schedule.every().day.at(config.compactor["scheduled_time"]).do(
compactor.run)
# Every day at scheduled time run the cleaner
if cleaner.enabled:
schedule.every().day.at(config.cleaner["scheduled_time"]).do(
cleaner.run)
# create the UDP server
receiver = Receiver()
udp_server = UdpServer(config.monitoring.udp_server, receiver)
udp_server.start()
# schedule reporters
rest_reporter = AvisoRestReporter(config.monitoring, receiver)
if rest_reporter.enabled:
schedule.every(rest_reporter.frequency).minutes.do(rest_reporter.run)
auth_reporter = AvisoAuthReporter(config.monitoring, receiver)
if auth_reporter.enabled:
schedule.every(auth_reporter.frequency).minutes.do(auth_reporter.run)
etcd_reporter = EtcdReporter(config.monitoring)
if etcd_reporter.enabled:
schedule.every(etcd_reporter.frequency).minutes.do(etcd_reporter.run)
# Loop so that the scheduling task keeps on running all time.
while True:
# Checks whether a scheduled task is pending to run or not
schedule.run_pending()
time.sleep(30)
def delete_destination_keys(date):
config = conf().cleaner
cleaner = Cleaner(config)
# delete destinations
return cleaner.delete_destination_keys(date)
def delete_dissemination_keys(date, destination):
config = conf().cleaner
cleaner = Cleaner(config)
# delete dissemination keys
return cleaner.delete_keys(date, destination)