def cleanup_nas(seconds):
"""Will move the finished runs in NASes to nosync directory.
:param int seconds: Days/hours converted as second to consider a run to be old
"""
couch_info = CONFIG.get('statusdb')
mail_recipients = CONFIG.get('mail', {}).get('recipients')
check_demux = CONFIG.get('storage', {}).get('check_demux', False)
host_name = os.getenv('HOSTNAME', os.uname()[1]).split('.', 1)[0]
for data_dir in CONFIG.get('storage').get('data_dirs'):
logger.info('Moving old runs in {}'.format(data_dir))
with filesystem.chdir(data_dir):
for run in [r for r in os.listdir(data_dir) if re.match(filesystem.RUN_RE, r)]:
rta_file = os.path.join(run, finished_run_indicator)
if os.path.exists(rta_file):
if check_demux:
if misc.run_is_demuxed(run, couch_info):
logger.info('Moving run {} to nosync directory'.format(os.path.basename(run)))
shutil.move(run, 'nosync')
elif os.stat(rta_file).st_mtime < time.time() - seconds:
logger.warn('Run {} is older than given time, but it is not demultiplexed yet'
.format(run))
sbt = "Run not demultiplexed - {}".format(run)
msg = ("Run '{}' in '{}' is older then given threshold, but seems like it is not "
"yet demultiplexed".format(os.path.join(data_dir, run), host_name))
misc.send_mail(sbt, msg, mail_recipients)
else:
if os.stat(rta_file).st_mtime < time.time() - seconds:
logger.info('Moving run {} to nosync directory'.format(os.path.basename(run)))
shutil.move(run, 'nosync')
else:
logger.info('{} file exists but is not older than given time, skipping run {}'
.format(finished_run_indicator, run))
def cleanup_nas(seconds):
"""Will move the finished runs in NASes to nosync directory.
:param int seconds: Days/hours converted as second to consider a run to be old
"""
couch_info = CONFIG.get('statusdb')
mail_recipients = CONFIG.get('mail', {}).get('recipients')
check_demux = CONFIG.get('storage', {}).get('check_demux', False)
host_name = os.getenv('HOSTNAME', os.uname()[1]).split('.', 1)[0]
for data_dir in CONFIG.get('storage').get('data_dirs'):
if not os.path.exists(data_dir) or not os.path.isdir(data_dir):
logger.warn(
"Data directory '{}' does not exist or not a directory".format(
data_dir))
continue
logger.info('Moving old runs in {}'.format(data_dir))
with filesystem.chdir(data_dir):
for run in [
r for r in os.listdir(data_dir)
if re.match(filesystem.RUN_RE, r)
]:
rta_file = os.path.join(run, finished_run_indicator)
if os.path.exists(rta_file):
if check_demux:
if misc.run_is_demuxed(run, couch_info):
logger.info(
'Moving run {} to nosync directory'.format(
os.path.basename(run)))
shutil.move(run, 'nosync')
elif 'miseq' in data_dir:
miseq_run = MiSeq_Run(run, CONFIG)
if miseq_run.get_run_type() == 'NON-NGI-RUN':
logger.info(
'Run {} is a non-platform run, so moving it to nosync directory'
.format(os.path.basename(run)))
shutil.move(run, 'nosync')
elif os.stat(
rta_file).st_mtime < time.time() - seconds:
logger.warn(
'Run {} is older than given time, but it is not demultiplexed yet'
.format(run))
sbt = "Run not demultiplexed - {}".format(run)
msg = (
"Run '{}' in '{}' is older then given threshold, but seems like it is not "
"yet demultiplexed".format(
os.path.join(data_dir, run), host_name))
misc.send_mail(sbt, msg, mail_recipients)
else:
if os.stat(rta_file).st_mtime < time.time() - seconds:
logger.info(
'Moving run {} to nosync directory'.format(
os.path.basename(run)))
shutil.move(run, 'nosync')
else:
logger.info(
'{} file exists but is not older than given time, skipping run {}'
.format(finished_run_indicator, run))
def test_run_is_demuxed(self, mock_couch):
"""Check in StatusDB if run was demultiplexed."""
run = '200201_A00621_0032_BHHFCFDSXX'
couch_info = {
'url': 'url',
'username': '******',
'password': '******',
'port': '1234',
'db': 'db'
}
is_demultiplexed = misc.run_is_demuxed(run, couch_info=couch_info)
def encrypt_runs(cls, run, force):
"""Encrypt the runs that have been collected"""
bk = cls(run)
bk.collect_runs(ext=".tar.gz")
logger.info("In total, found {} run(s) to be encrypted".format(
len(bk.runs)))
for run in bk.runs:
run.flag = "{}.encrypting".format(run.name)
run.dst_key_encrypted = os.path.join(bk.keys_path,
run.key_encrypted)
tmp_files = [
run.zip_encrypted, run.key_encrypted, run.key, run.flag
]
logger.info("Encryption of run {} is now started".format(run.name))
# Check if there is enough space and exit if not
bk.avail_disk_space(run.path, run.name)
# Check if the run in demultiplexed
if not force and bk.check_demux:
if not misc.run_is_demuxed(run.name, bk.couch_info):
logger.warn(
"Run {} is not demultiplexed yet, so skipping it".
format(run.name))
continue
logger.info(
"Run {} is demultiplexed and proceeding with encryption".
format(run.name))
with filesystem.chdir(run.path):
# skip run if already ongoing
if os.path.exists(run.flag):
logger.warn(
"Run {} is already being encrypted, so skipping now".
format(run.name))
continue
flag = open(run.flag, 'w').close()
# zip the run directory
if os.path.exists(run.zip):
if os.path.isdir(run.name):
logger.warn(
"Both run source and zipped archive exist for run {}, skipping run as precaution"
.format(run.name))
bk._clean_tmp_files([run.flag])
continue
logger.info(
"Zipped archive already exist for run {}, so using it for encryption"
.format(run.name))
else:
logger.info("Creating zipped archive for run {}".format(
run.name))
if bk._call_commands(cmd1="tar -cf - {}".format(run.name),
cmd2="pigz --fast -c -",
out_file=run.zip,
mail_failed=True,
tmp_files=[run.zip, run.flag]):
logger.info(
"Run {} was successfully compressed, so removing the run source directory"
.format(run.name))
shutil.rmtree(run.name)
else:
logger.warn("Skipping run {} and moving on".format(
run.name))
continue
# Remove encrypted file if already exists
if os.path.exists(run.zip_encrypted):
logger.warn((
"Removing already existing encrypted file for run {}, this is a precaution "
"to make sure the file was encrypted with correct key file"
.format(run.name)))
bk._clean_tmp_files([
run.zip_encrypted, run.key, run.key_encrypted,
run.dst_key_encrypted
])
# Generate random key to use as pasphrase
if not bk._call_commands(cmd1="gpg --gen-random 1 256",
out_file=run.key,
tmp_files=tmp_files):
logger.warn("Skipping run {} and moving on".format(
run.name))
continue
logger.info("Generated randon phrase key for run {}".format(
run.name))
# Calculate md5 sum pre encryption
if not force:
logger.info("Calculating md5sum before encryption")
md5_call, md5_out = bk._call_commands(
cmd1="md5sum {}".format(run.zip),
return_out=True,
tmp_files=tmp_files)
if not md5_call:
logger.warn("Skipping run {} and moving on".format(
run.name))
continue
md5_pre_encrypt = md5_out.split()[0]
# Encrypt the zipped run file
logger.info("Encrypting the zipped run file")
if not bk._call_commands(
cmd1=
("gpg --symmetric --cipher-algo aes256 --passphrase-file {} --batch --compress-algo "
"none -o {} {}".format(run.key, run.zip_encrypted,
run.zip)),
tmp_files=tmp_files):
logger.warn("Skipping run {} and moving on".format(
run.name))
continue
# Decrypt and check for md5
if not force:
logger.info("Calculating md5sum after encryption")
md5_call, md5_out = bk._call_commands(
cmd1=
"gpg --decrypt --cipher-algo aes256 --passphrase-file {} --batch {}"
.format(run.key, run.zip_encrypted),
cmd2="md5sum",
return_out=True,
tmp_files=tmp_files)
if not md5_call:
logger.warn("Skipping run {} and moving on".format(
run.name))
continue
md5_post_encrypt = md5_out.split()[0]
if md5_pre_encrypt != md5_post_encrypt:
logger.error((
"md5sum did not match before {} and after {} encryption. Will remove temp files and "
"move on".format(md5_pre_encrypt,
md5_post_encrypt)))
bk._clean_tmp_files(tmp_files)
continue
logger.info(
"Md5sum is macthing before and after encryption")
# Encrypt and move the key file
if bk._call_commands(cmd1="gpg -e -r {} -o {} {}".format(
bk.gpg_receiver, run.key_encrypted, run.key),
tmp_files=tmp_files):
shutil.move(run.key_encrypted, run.dst_key_encrypted)
else:
logger.error("Encrption of key file failed, skipping run")
continue
bk._clean_tmp_files([run.zip, run.key, run.flag])
logger.info(
"Encryption of run {} is successfully done, removing zipped run file"
.format(run.name))
def encrypt_runs(cls, run, force):
"""Encrypt the runs that have been collected"""
bk = cls(run)
bk.collect_runs(ext=".tar.gz")
logger.info("In total, found {} run(s) to be encrypted".format(len(bk.runs)))
for run in bk.runs:
run.flag = "{}.encrypting".format(run.name)
run.dst_key_encrypted = os.path.join(bk.keys_path, run.key_encrypted)
tmp_files = [run.zip_encrypted, run.key_encrypted, run.key, run.flag]
logger.info("Encryption of run {} is now started".format(run.name))
# Check if there is enough space and exit if not
bk.avail_disk_space(run.path, run.name)
# Check if the run in demultiplexed
if not force and bk.check_demux:
if not misc.run_is_demuxed(run.name, bk.couch_info):
logger.warn("Run {} is not demultiplexed yet, so skipping it".format(run.name))
continue
logger.info("Run {} is demultiplexed and proceeding with encryption".format(run.name))
with filesystem.chdir(run.path):
# skip run if already ongoing
if os.path.exists(run.flag):
logger.warn("Run {} is already being encrypted, so skipping now".format(run.name))
continue
flag = open(run.flag, 'w').close()
# zip the run directory
if os.path.exists(run.zip):
if os.path.isdir(run.name):
logger.warn("Both run source and zipped archive exist for run {}, skipping run as precaution".format(run.name))
bk._clean_tmp_files([run.flag])
continue
logger.info("Zipped archive already exist for run {}, so using it for encryption".format(run.name))
else:
logger.info("Creating zipped archive for run {}".format(run.name))
if bk._call_commands(cmd1="tar -cf - {}".format(run.name), cmd2="pigz --fast -c -",
out_file=run.zip, mail_failed=True, tmp_files=[run.zip, run.flag]):
logger.info("Run {} was successfully compressed, so removing the run source directory".format(run.name))
shutil.rmtree(run.name)
else:
logger.warn("Skipping run {} and moving on".format(run.name))
continue
# Remove encrypted file if already exists
if os.path.exists(run.zip_encrypted):
logger.warn(("Removing already existing encrypted file for run {}, this is a precaution "
"to make sure the file was encrypted with correct key file".format(run.name)))
bk._clean_tmp_files([run.zip_encrypted, run.key, run.key_encrypted, run.dst_key_encrypted])
# Generate random key to use as pasphrase
if not bk._call_commands(cmd1="gpg --gen-random 1 256", out_file=run.key, tmp_files=tmp_files):
logger.warn("Skipping run {} and moving on".format(run.name))
continue
logger.info("Generated randon phrase key for run {}".format(run.name))
# Calculate md5 sum pre encryption
if not force:
logger.info("Calculating md5sum before encryption")
md5_call, md5_out = bk._call_commands(cmd1="md5sum {}".format(run.zip), return_out=True, tmp_files=tmp_files)
if not md5_call:
logger.warn("Skipping run {} and moving on".format(run.name))
continue
md5_pre_encrypt = md5_out.split()[0]
# Encrypt the zipped run file
logger.info("Encrypting the zipped run file")
if not bk._call_commands(cmd1=("gpg --symmetric --cipher-algo aes256 --passphrase-file {} --batch --compress-algo "
"none -o {} {}".format(run.key, run.zip_encrypted, run.zip)), tmp_files=tmp_files):
logger.warn("Skipping run {} and moving on".format(run.name))
continue
# Decrypt and check for md5
if not force:
logger.info("Calculating md5sum after encryption")
md5_call, md5_out = bk._call_commands(cmd1="gpg --decrypt --cipher-algo aes256 --passphrase-file {} --batch {}".format(run.key, run.zip_encrypted),
cmd2="md5sum", return_out=True, tmp_files=tmp_files)
if not md5_call:
logger.warn("Skipping run {} and moving on".format(run.name))
continue
md5_post_encrypt = md5_out.split()[0]
if md5_pre_encrypt != md5_post_encrypt:
logger.error(("md5sum did not match before {} and after {} encryption. Will remove temp files and "
"move on".format(md5_pre_encrypt, md5_post_encrypt)))
bk._clean_tmp_files(tmp_files)
continue
logger.info("Md5sum is macthing before and after encryption")
# Encrypt and move the key file
if bk._call_commands(cmd1="gpg -e -r {} -o {} {}".format(bk.gpg_receiver, run.key_encrypted, run.key), tmp_files=tmp_files):
shutil.move(run.key_encrypted, run.dst_key_encrypted)
else:
logger.error("Encrption of key file failed, skipping run")
continue
bk._clean_tmp_files([run.zip, run.key, run.flag])
logger.info("Encryption of run {} is successfully done, removing zipped run file".format(run.name))
