def check_default(site, seconds, config): """Check if time(as seconds) given while running command. If not take the default threshold from config file (which should exist). Also when 'days' given on the command line raise a check to make sure it was really meant to do so :param str site: site to be cleaned and relevent date to pick :param int seconds: Days/hours converted as seconds to check :param dict config: config file parsed and saved as dictionary """ try: default_days = config['cleanup']['milou'][site]['days'] default_seconds = misc.to_seconds(days=default_days) except KeyError: raise if not seconds: return default_seconds elif seconds >= default_seconds: return seconds else: if misc.query_yes_no( "Seems like given time is less than the " " default({}) days, are you sure to proceed ?".format( default_days), default="no"): return seconds else: return None
def check_default(site, seconds, config): """Check if time(as seconds) given while running command. If not take the default threshold from config file (which should exist). Also when 'days' given on the command line raise a check to make sure it was really meant to do so :param str site: site to be cleaned and relevent date to pick :param int seconds: Days/hours converted as seconds to check :param dict config: config file parsed and saved as dictionary """ try: default_days = config['cleanup']['milou'][site]['days'] default_seconds = misc.to_seconds(days=default_days) except KeyError: raise if not seconds: return default_seconds elif seconds >= default_seconds: return seconds else: if misc.query_yes_no("Seems like given time is less than the " " default({}) days, are you sure to proceed ?" .format(default_days), default="no"): return seconds else: return None
def get_closed_projects(projs, pj_con, seconds): """Takes list of project and gives project list that are closed more than given time(as seconds) :param list projs: list of projects to check :param obj pj_con: connection object to project database :param int seconds: Days/hours converted as seconds to check """ closed_projs = [] for proj in projs: if proj not in pj_con.name_view.keys(): logger.warn( "Project {} is not in database, so SKIPPING it..".format(proj)) continue proj_db_obj = pj_con.get_entry(proj) try: proj_close_date = proj_db_obj['close_date'] except KeyError: logger.warn( "Project {} is either open or too old, so SKIPPING it..". format(proj)) continue if misc.to_seconds(days=misc.days_old( proj_close_date, date_format='%Y-%m-%d')) > seconds: closed_projs.append(proj) return closed_projs
def cleanup(ctx, days, hours, site, dry_run): """ Do appropriate cleanup on the given site i.e. NAS/processing servers/UPPMAX """ seconds = misc.to_seconds(days, hours) if site == 'nas': st.cleanup_nas(seconds) if site == 'processing-server': st.cleanup_processing(seconds) if site in ['illumina','analysis','archive']: st.cleanup_uppmax(site, seconds, dry_run)
def test_to_seconds(self): """Transform days and hours to seconds.""" with self.assertRaises(SystemExit): misc.to_seconds(days=1, hours=1) with self.assertRaises(SystemExit): misc.to_seconds() self.assertEqual(misc.to_seconds(days=1), 86400) self.assertEqual(misc.to_seconds(hours=1), 3600)
def cleanup_swestore(seconds, dry_run=False): """Remove archived runs from swestore :param int seconds: Days/hours converted as seconds to check """ seconds = check_default(site, seconds, CONFIG) if not seconds: return runs = filesystem.list_runs_in_swestore(path=CONFIG.get('cleanup').get('swestore').get('root')) for run in runs: date = run.split('_')[0] if misc.to_seconds(misc.days_old(date)) > seconds: if dry_run: logger.info('Will remove file {} from swestore'.format(run)) continue misc.call_external_command('irm -f {}'.format(run)) logger.info('Removed file {} from swestore'.format(run))
def cleanup_swestore(seconds, dry_run=False): """Remove archived runs from swestore :param int seconds: Days/hours converted as seconds to check """ seconds = check_default(site, seconds, CONFIG) if not seconds: return runs = filesystem.list_runs_in_swestore( path=CONFIG.get('cleanup').get('swestore').get('root')) for run in runs: date = run.split('_')[0] if misc.to_seconds(misc.days_old(date)) > seconds: if dry_run: logger.info('Will remove file {} from swestore'.format(run)) continue misc.call_external_command('irm -f {}'.format(run)) logger.info('Removed file {} from swestore'.format(run))
def get_closed_projects(projs, pj_con, seconds): """Takes list of project and gives project list that are closed more than given time(as seconds) :param list projs: list of projects to check :param obj pj_con: connection object to project database :param int seconds: Days/hours converted as seconds to check """ closed_projs = [] for proj in projs: if proj not in pj_con.name_view.keys(): logger.warn("Project {} is not in database, so SKIPPING it.." .format(proj)) continue proj_db_obj = pj_con.get_entry(proj) try: proj_close_date = proj_db_obj['close_date'] except KeyError: logger.warn("Project {} is either open or too old, so SKIPPING it..".format(proj)) continue if misc.to_seconds(days=misc.days_old(proj_close_date,date_format='%Y-%m-%d')) > seconds: closed_projs.append(proj) return closed_projs
def milou(ctx, site, days, dry_run): """ Do appropriate cleanup on MILOU""" seconds = misc.to_seconds(days) cln.cleanup_milou(site, seconds, dry_run)
def preproc(ctx, days, hours): """ Do appropriate cleanup on PREPROC""" seconds = misc.to_seconds(days, hours) cln.cleanup_processing(seconds)
def nas(ctx, days, hours): """ Do appropriate cleanup on NAS""" seconds = misc.to_seconds(days, hours) cln.cleanup_nas(seconds)