def transfer_lucky_to_stage():
"""
Get tarfile(s) from lucky and untar into stage area. This is all done from
the archive root directory.
"""
# Open lucky ftp connection and watch for tarfile(s) in '/taldcroft/eng_archive'
logger.info('ftp to lucky')
ftp = Ska.ftp.SFTP('lucky', logger=logger)
ftp.cd('/home/taldcroft')
files = ftp.ls()
if opt.ftp_dir not in files:
logger.info('mkdir {}'.format(opt.ftp_dir))
ftp.mkdir(opt.ftp_dir)
ftp.cd(opt.ftp_dir)
for _ in range(opt.timeout / opt.sleep_time):
logger.info('Directory: {}'.format(ftp.ftp.getcwd()))
logger.info('Files: {}'.format(ftp.ls()))
files = [x for x in ftp.ls() if re.match('stage_\d+\.tar', x)]
if files:
break
time.sleep(opt.sleep_time)
logger.info(
'Waiting {0} seconds for archive files to appear...'.format(
opt.sleep_time))
else:
logger.info('No tarfiles found before timeout')
ftp.close()
sys.exit(1)
# For each tarfile:
# - get file by ftp
# - untar
# - unlink local tarfile
# - delete tarfile on ftp
for tarname in sorted(files):
logger.info('Getting tarfile {0} from lucky'.format(tarname))
ftp.get(tarname)
logger.info('Extracting tarfile {0}'.format(tarname))
tar = tarfile.open(name=tarname, mode='r')
tar.extractall()
tar.close()
logger.info('Unlinking local {0}'.format(tarname))
os.unlink(tarname)
logger.info('Deleting ftp {0}'.format(tarname))
ftp.delete(tarname)
ftp.close()
def transfer_lucky_to_stage():
"""
Get tarfile(s) from lucky and untar into stage area. This is all done from
the archive root directory.
"""
# Open lucky ftp connection and watch for tarfile(s) in '/taldcroft/eng_archive'
logger.info('ftp to lucky')
ftp = Ska.ftp.SFTP('lucky', logger=logger)
ftp.cd('/home/taldcroft')
files = ftp.ls()
if opt.ftp_dir not in files:
logger.info('mkdir {}'.format(opt.ftp_dir))
ftp.mkdir(opt.ftp_dir)
ftp.cd(opt.ftp_dir)
for _ in range(opt.timeout / opt.sleep_time):
logger.info('Directory: {}'.format(ftp.ftp.getcwd()))
logger.info('Files: {}'.format(ftp.ls()))
files = [x for x in ftp.ls() if re.match('stage_\d+\.tar', x)]
if files:
break
time.sleep(opt.sleep_time)
logger.info('Waiting {0} seconds for archive files to appear...'.format(opt.sleep_time))
else:
logger.info('No tarfiles found before timeout')
ftp.close()
sys.exit(1)
# For each tarfile:
# - get file by ftp
# - untar
# - unlink local tarfile
# - delete tarfile on ftp
for tarname in sorted(files):
logger.info('Getting tarfile {0} from lucky'.format(tarname))
ftp.get(tarname)
logger.info('Extracting tarfile {0}'.format(tarname))
tar = tarfile.open(name=tarname, mode='r')
tar.extractall()
tar.close()
logger.info('Unlinking local {0}'.format(tarname))
os.unlink(tarname)
logger.info('Deleting ftp {0}'.format(tarname))
ftp.delete(tarname)
ftp.close()
def ftp_get_from_lucky(ftp_dirname, local_files, user=None, logger=None):
"""
Get files from lucky. This looks in remote ``ftp_dirname`` for files that
have basenames matching those of ``local_files``. The remote files
are copied to the corresponding local_files names. This is the converse
of ftp_put_to_lucky and thus requires unique basenames for all files.
"""
import Ska.ftp
import Ska.File
ftp = Ska.ftp.SFTP('lucky', logger=logger, user=user)
if user is None:
user = ftp.ftp.get_channel().transport.get_username()
ftp.cd('/home/{}/{}'.format(user, ftp_dirname))
for local_file in local_files:
file_dir, file_base = os.path.split(os.path.abspath(local_file))
if file_base in ftp.ls():
with Ska.File.chdir(file_dir):
ftp.get(file_base)
ftp.delete(file_base)
ftp.close()
def ftp_get_from_lucky(ftp_dirname, local_files, user=None, logger=None):
"""
Get files from lucky. This looks in remote ``ftp_dirname`` for files that
have basenames matching those of ``local_files``. The remote files
are copied to the corresponding local_files names. This is the converse
of ftp_put_to_lucky and thus requires unique basenames for all files.
"""
import Ska.ftp
import Ska.File
ftp = Ska.ftp.SFTP('lucky', logger=logger, user=user)
if user is None:
user = ftp.ftp.get_channel().transport.get_username()
ftp.cd('/home/{}/{}'.format(user, ftp_dirname))
for local_file in local_files:
file_dir, file_base = os.path.split(os.path.abspath(local_file))
if file_base in ftp.ls():
with Ska.File.chdir(file_dir):
ftp.get(file_base)
ftp.delete(file_base)
ftp.close()