def check_default(site, seconds, config):
"""Check if time(as seconds) given while running command. If not take the default threshold
from config file (which should exist). Also when 'days' given on the command line
raise a check to make sure it was really meant to do so
:param str site: site to be cleaned and relevent date to pick
:param int seconds: Days/hours converted as seconds to check
:param dict config: config file parsed and saved as dictionary
"""
try:
default_days = config['cleanup']['milou'][site]['days']
default_seconds = misc.to_seconds(days=default_days)
except KeyError:
raise
if not seconds:
return default_seconds
elif seconds >= default_seconds:
return seconds
else:
if misc.query_yes_no(
"Seems like given time is less than the "
" default({}) days, are you sure to proceed ?".format(
default_days),
default="no"):
return seconds
else:
return None
def check_default(site, seconds, config):
"""Check if time(as seconds) given while running command. If not take the default threshold
from config file (which should exist). Also when 'days' given on the command line
raise a check to make sure it was really meant to do so
:param str site: site to be cleaned and relevent date to pick
:param int seconds: Days/hours converted as seconds to check
:param dict config: config file parsed and saved as dictionary
"""
try:
default_days = config['cleanup']['milou'][site]['days']
default_seconds = misc.to_seconds(days=default_days)
except KeyError:
raise
if not seconds:
return default_seconds
elif seconds >= default_seconds:
return seconds
else:
if misc.query_yes_no("Seems like given time is less than the "
" default({}) days, are you sure to proceed ?"
.format(default_days), default="no"):
return seconds
else:
return None
def get_closed_projects(projs, pj_con, seconds):
"""Takes list of project and gives project list that are closed
more than given time(as seconds)
:param list projs: list of projects to check
:param obj pj_con: connection object to project database
:param int seconds: Days/hours converted as seconds to check
"""
closed_projs = []
for proj in projs:
if proj not in pj_con.name_view.keys():
logger.warn(
"Project {} is not in database, so SKIPPING it..".format(proj))
continue
proj_db_obj = pj_con.get_entry(proj)
try:
proj_close_date = proj_db_obj['close_date']
except KeyError:
logger.warn(
"Project {} is either open or too old, so SKIPPING it..".
format(proj))
continue
if misc.to_seconds(days=misc.days_old(
proj_close_date, date_format='%Y-%m-%d')) > seconds:
closed_projs.append(proj)
return closed_projs
def cleanup(ctx, days, hours, site, dry_run):
""" Do appropriate cleanup on the given site i.e. NAS/processing servers/UPPMAX """
seconds = misc.to_seconds(days, hours)
if site == 'nas':
st.cleanup_nas(seconds)
if site == 'processing-server':
st.cleanup_processing(seconds)
if site in ['illumina','analysis','archive']:
st.cleanup_uppmax(site, seconds, dry_run)
def test_to_seconds(self):
"""Transform days and hours to seconds."""
with self.assertRaises(SystemExit):
misc.to_seconds(days=1, hours=1)
with self.assertRaises(SystemExit):
misc.to_seconds()
self.assertEqual(misc.to_seconds(days=1), 86400)
self.assertEqual(misc.to_seconds(hours=1), 3600)
def cleanup_swestore(seconds, dry_run=False):
"""Remove archived runs from swestore
:param int seconds: Days/hours converted as seconds to check
"""
seconds = check_default(site, seconds, CONFIG)
if not seconds:
return
runs = filesystem.list_runs_in_swestore(path=CONFIG.get('cleanup').get('swestore').get('root'))
for run in runs:
date = run.split('_')[0]
if misc.to_seconds(misc.days_old(date)) > seconds:
if dry_run:
logger.info('Will remove file {} from swestore'.format(run))
continue
misc.call_external_command('irm -f {}'.format(run))
logger.info('Removed file {} from swestore'.format(run))
def cleanup_swestore(seconds, dry_run=False):
"""Remove archived runs from swestore
:param int seconds: Days/hours converted as seconds to check
"""
seconds = check_default(site, seconds, CONFIG)
if not seconds:
return
runs = filesystem.list_runs_in_swestore(
path=CONFIG.get('cleanup').get('swestore').get('root'))
for run in runs:
date = run.split('_')[0]
if misc.to_seconds(misc.days_old(date)) > seconds:
if dry_run:
logger.info('Will remove file {} from swestore'.format(run))
continue
misc.call_external_command('irm -f {}'.format(run))
logger.info('Removed file {} from swestore'.format(run))
def get_closed_projects(projs, pj_con, seconds):
"""Takes list of project and gives project list that are closed
more than given time(as seconds)
:param list projs: list of projects to check
:param obj pj_con: connection object to project database
:param int seconds: Days/hours converted as seconds to check
"""
closed_projs = []
for proj in projs:
if proj not in pj_con.name_view.keys():
logger.warn("Project {} is not in database, so SKIPPING it.."
.format(proj))
continue
proj_db_obj = pj_con.get_entry(proj)
try:
proj_close_date = proj_db_obj['close_date']
except KeyError:
logger.warn("Project {} is either open or too old, so SKIPPING it..".format(proj))
continue
if misc.to_seconds(days=misc.days_old(proj_close_date,date_format='%Y-%m-%d')) > seconds:
closed_projs.append(proj)
return closed_projs
def milou(ctx, site, days, dry_run):
""" Do appropriate cleanup on MILOU"""
seconds = misc.to_seconds(days)
cln.cleanup_milou(site, seconds, dry_run)
def preproc(ctx, days, hours):
""" Do appropriate cleanup on PREPROC"""
seconds = misc.to_seconds(days, hours)
cln.cleanup_processing(seconds)
def nas(ctx, days, hours):
""" Do appropriate cleanup on NAS"""
seconds = misc.to_seconds(days, hours)
cln.cleanup_nas(seconds)
def milou(ctx, site, days, dry_run):
""" Do appropriate cleanup on MILOU"""
seconds = misc.to_seconds(days)
cln.cleanup_milou(site, seconds, dry_run)
def preproc(ctx, days, hours):
""" Do appropriate cleanup on PREPROC"""
seconds = misc.to_seconds(days, hours)
cln.cleanup_processing(seconds)
def nas(ctx, days, hours):
""" Do appropriate cleanup on NAS"""
seconds = misc.to_seconds(days, hours)
cln.cleanup_nas(seconds)
