def cleanup_processing(days):
"""Cleanup runs in processing server.
:param int days: Number of days to consider a run to be old
"""
transfer_file = os.path.join(CONFIG.get('preprocessing', {}).get('status_dir'), 'transfer.tsv')
if not days:
days = CONFIG.get('cleanup', {}).get('processing-server', {}).get('days', 10)
try:
#Move finished runs to nosync
for data_dir in CONFIG.get('storage').get('data_dirs'):
logger.info('Moving old runs in {}'.format(data_dir))
with filesystem.chdir(data_dir):
for run in [r for r in os.listdir(data_dir) if re.match(filesystem.RUN_RE, r)]:
if filesystem.is_in_file(transfer_file, run):
logger.info('Moving run {} to nosync directory'
.format(os.path.basename(run)))
shutil.move(run, 'nosync')
else:
logger.info(("Run {} has not been transferred to the analysis "
"server yet, not archiving".format(run)))
#Remove old runs from archiving dirs
for archive_dir in CONFIG.get('storage').get('archive_dirs').values():
logger.info('Removing old runs in {}'.format(archive_dir))
with filesystem.chdir(archive_dir):
for run in [r for r in os.listdir(archive_dir) if re.match(filesystem.RUN_RE, r)]:
rta_file = os.path.join(run, finished_run_indicator)
if os.path.exists(rta_file):
# 1 day == 60*60*24 seconds --> 86400
if os.stat(rta_file).st_mtime < time.time() - (86400 * days) and \
filesystem.is_in_swestore("{}.tar.bz2".format(run)):
logger.info('Removing run {} to nosync directory'
.format(os.path.basename(run)))
shutil.rmtree(run)
else:
logger.info('{} file exists but is not older than {} day(s), skipping run {}'.format(
finished_run_indicator, str(days), run))
except IOError:
sbj = "Cannot archive old runs in processing server"
msg = ("Could not find transfer.tsv file, so I cannot decide if I should "
"archive any run or not.")
cnt = CONFIG.get('contact', None)
if not cnt:
cnt = "{}@localhost".format(getpass.getuser())
logger.error(msg)
misc.send_mail(sbj, msg, cnt)
def _send_to_swestore(f, dest, remove=True):
""" Send file to swestore checking adler32 on destination and eventually
removing the file from disk
:param str f: File to remove
:param str dest: Destination directory in Swestore
:param bool remove: If True, remove original file from source
"""
if not filesystem.is_in_swestore(f):
logger.info("Sending {} to swestore".format(f))
misc.call_external_command("iput -K -P {file} {dest}".format(file=f, dest=dest), with_log_files=True)
logger.info("Run {} sent correctly and checksum was okay.".format(f))
if remove:
logger.info("Removing run".format(f))
os.remove(f)
else:
logger.warn("Run {} is already in Swestore, not sending it again nor removing from the disk".format(f))
def cleanup_processing(days):
"""Cleanup runs in processing server.
:param int days: Number of days to consider a run to be old
"""
transfer_file = os.path.join(CONFIG.get('preprocessing', {}).get('status_dir'), 'transfer.tsv')
if not days:
days = CONFIG.get('cleanup', {}).get('processing-server', {}).get('days', 10)
try:
#Move finished runs to nosync
for data_dir in CONFIG.get('storage').get('data_dirs'):
logger.info('Moving old runs in {}'.format(data_dir))
with filesystem.chdir(data_dir):
for run in [r for r in os.listdir(data_dir) if re.match(filesystem.RUN_RE, r)]:
if filesystem.is_in_file(transfer_file, run):
logger.info('Moving run {} to nosync directory'
.format(os.path.basename(run)))
shutil.move(run, 'nosync')
else:
logger.info(("Run {} has not been transferred to the analysis "
"server yet, not archiving".format(run)))
#Remove old runs from archiving dirs
for archive_dir in CONFIG.get('storage').get('archive_dirs').values():
logger.info('Removing old runs in {}'.format(archive_dir))
with filesystem.chdir(archive_dir):
for run in [r for r in os.listdir(archive_dir) if re.match(filesystem.RUN_RE, r)]:
rta_file = os.path.join(run, 'RTAComplete.txt')
if os.path.exists(rta_file):
# 1 day == 60*60*24 seconds --> 86400
if os.stat(rta_file).st_mtime < time.time() - (86400 * days) and \
filesystem.is_in_swestore("{}.tar.bz2".format(run)):
logger.info('Removing run {} to nosync directory'
.format(os.path.basename(run)))
shutil.rmtree(run)
else:
logger.info('RTAComplete.txt file exists but is not older than {} day(s), skipping run {}'.format(str(days), run))
except IOError:
sbj = "Cannot archive old runs in processing server"
msg = ("Could not find transfer.tsv file, so I cannot decide if I should "
"archive any run or not.")
cnt = CONFIG.get('contact', None)
if not cnt:
cnt = "{}@localhost".format(getpass.getuser())
logger.error(msg)
misc.send_mail(sbj, msg, cnt)
def _send_to_swestore(f, dest, remove=True):
""" Send file to swestore checking adler32 on destination and eventually
removing the file from disk
:param str f: File to remove
:param str dest: Destination directory in Swestore
:param bool remove: If True, remove original file from source
"""
if not filesystem.is_in_swestore(f):
logger.info("Sending {} to swestore".format(f))
misc.call_external_command('iput -R swestoreArchCacheResc -P {file} {dest}'.format(file=f, dest=dest),
with_log_files=True, prefix=f.replace('.tar.bz2',''), log_dir="swestore_logs")
logger.info('Run {} sent to swestore.'.format(f))
if remove:
logger.info('Removing run'.format(f))
os.remove(f)
else:
logger.warn('Run {} is already in Swestore, not sending it again nor removing from the disk'.format(f))
def _send_to_swestore(f, dest, remove=True):
""" Send file to swestore checking adler32 on destination and eventually
removing the file from disk
:param str f: File to remove
:param str dest: Destination directory in Swestore
:param bool remove: If True, remove original file from source
"""
if not filesystem.is_in_swestore(f):
logger.info("Sending {} to swestore".format(f))
misc.call_external_command('iput -K -P {file} {dest}'.format(file=f, dest=dest),
with_log_files=True)
logger.info('Run {} sent correctly and checksum was okay.'.format(f))
if remove:
logger.info('Removing run'.format(f))
os.remove(f)
else:
logger.warn('Run {} is already in Swestore, not sending it again nor removing from the disk'.format(f))
def _send_to_swestore(f, dest, remove=True):
""" Send file to swestore checking adler32 on destination and eventually
removing the file from disk
:param str f: File to remove
:param str dest: Destination directory in Swestore
:param bool remove: If True, remove original file from source
"""
if not filesystem.is_in_swestore(f):
logger.info("Sending {} to swestore".format(f))
misc.call_external_command(
'iput -R swestoreArchCacheResc -P {file} {dest}'.format(
file=f, dest=dest),
with_log_files=True,
prefix=f.replace('.tar.bz2', ''),
log_dir="swestore_logs")
logger.info('Run {} sent to swestore.'.format(f))
if remove:
logger.info('Removing run'.format(f))
os.remove(f)
else:
logger.warn(
'Run {} is already in Swestore, not sending it again nor removing from the disk'
.format(f))