def ftp_put_to_lucky(ftp_dirname, local_files, user=None, logger=None):
"""Put the ``local_files`` onto lucky in /``user``/``ftp_dirname``. First put it at the top
level, then when complete move it into a subdir eng_archive. This lets the OCC side
just watch for fully-uploaded files in that directory.
The directory paths of ``local_files`` are stripped off so they all wind up
in a flat structure within ``ftp_dirname``.
"""
import Ska.File
import Ska.ftp
import uuid
ftp = Ska.ftp.SFTP('lucky', logger=logger, user=user)
if user is None:
user = ftp.ftp.get_channel().transport.get_username()
ftp.cd('/home/{}'.format(user))
files = ftp.ls()
if ftp_dirname not in files:
ftp.mkdir(ftp_dirname)
dir_files = ftp.ls(ftp_dirname)
for local_file in local_files:
file_dir, file_base = os.path.split(os.path.abspath(local_file))
ftp_file = str(uuid.uuid4()) # random unique identifier
with Ska.File.chdir(file_dir):
ftp.put(file_base, ftp_file)
destination_file = "{}/{}".format(ftp_dirname, file_base)
# If final destination of file already exists, delete that file.
if file_base in dir_files:
ftp.delete(destination_file)
# Rename the temp/uniq-id file to the final destination
ftp.rename(ftp_file, destination_file)
ftp.close()
def ftp_put_to_lucky(ftp_dirname, local_files, user=None, logger=None):
"""Put the ``local_files`` onto lucky in /``user``/``ftp_dirname``. First put it at the top
level, then when complete move it into a subdir eng_archive. This lets the OCC side
just watch for fully-uploaded files in that directory.
The directory paths of ``local_files`` are stripped off so they all wind up
in a flat structure within ``ftp_dirname``.
"""
import Ska.File
import Ska.ftp
import uuid
ftp = Ska.ftp.SFTP('lucky', logger=logger, user=user)
if user is None:
user = ftp.ftp.get_channel().transport.get_username()
ftp.cd('/home/{}'.format(user))
files = ftp.ls()
if ftp_dirname not in files:
ftp.mkdir(ftp_dirname)
dir_files = ftp.ls(ftp_dirname)
for local_file in local_files:
file_dir, file_base = os.path.split(os.path.abspath(local_file))
ftp_file = str(uuid.uuid4()) # random unique identifier
with Ska.File.chdir(file_dir):
ftp.put(file_base, ftp_file)
destination_file = "{}/{}".format(ftp_dirname, file_base)
# If final destination of file already exists, delete that file.
if file_base in dir_files:
ftp.delete(destination_file)
# Rename the temp/uniq-id file to the final destination
ftp.rename(ftp_file, destination_file)
ftp.close()
def transfer_lucky_to_stage():
"""
Get tarfile(s) from lucky and untar into stage area. This is all done from
the archive root directory.
"""
# Open lucky ftp connection and watch for tarfile(s) in '/taldcroft/eng_archive'
logger.info('ftp to lucky')
ftp = Ska.ftp.SFTP('lucky', logger=logger)
ftp.cd('/home/taldcroft')
files = ftp.ls()
if opt.ftp_dir not in files:
logger.info('mkdir {}'.format(opt.ftp_dir))
ftp.mkdir(opt.ftp_dir)
ftp.cd(opt.ftp_dir)
for _ in range(opt.timeout / opt.sleep_time):
logger.info('Directory: {}'.format(ftp.ftp.getcwd()))
logger.info('Files: {}'.format(ftp.ls()))
files = [x for x in ftp.ls() if re.match('stage_\d+\.tar', x)]
if files:
break
time.sleep(opt.sleep_time)
logger.info(
'Waiting {0} seconds for archive files to appear...'.format(
opt.sleep_time))
else:
logger.info('No tarfiles found before timeout')
ftp.close()
sys.exit(1)
# For each tarfile:
# - get file by ftp
# - untar
# - unlink local tarfile
# - delete tarfile on ftp
for tarname in sorted(files):
logger.info('Getting tarfile {0} from lucky'.format(tarname))
ftp.get(tarname)
logger.info('Extracting tarfile {0}'.format(tarname))
tar = tarfile.open(name=tarname, mode='r')
tar.extractall()
tar.close()
logger.info('Unlinking local {0}'.format(tarname))
os.unlink(tarname)
logger.info('Deleting ftp {0}'.format(tarname))
ftp.delete(tarname)
ftp.close()
def transfer_stage_to_lucky():
# Make a tarfile of everything in stage directory
tarname = 'stage_{0:d}.tar'.format(int(DateTime().secs))
logger.info('Making tarfile {0}'.format(tarname))
tar = tarfile.open(name=tarname, mode='w')
tar.add('stage')
tar.close()
# Put the tarfile on lucky. First put it at the top level, then when
# complete move it into a subdir eng_archive. This lets the OCC side
# just watch for fully-uploaded files in that directory.
logger.info('ftp to lucky')
ftp = Ska.ftp.SFTP('lucky', logger=logger)
ftp.cd('/home/taldcroft')
files = ftp.ls()
if opt.ftp_dir not in files:
logger.info('mkdir {}'.format(opt.ftp_dir))
ftp.mkdir(opt.ftp_dir)
logger.info('put {0}'.format(tarname))
ftp.put(tarname)
logger.info('rename {0} {1}/{0}'.format(tarname, opt.ftp_dir))
ftp.rename(tarname, '{}/{}'.format(opt.ftp_dir, tarname))
ftp.close()
logger.info('unlink {0}'.format(tarname))
os.unlink(tarname)
logger.info('rmtree stage')
shutil.rmtree('stage')
def transfer_stage_to_lucky():
# Make a tarfile of everything in stage directory
tarname = 'stage_{0:d}.tar'.format(int(DateTime().secs))
logger.info('Making tarfile {0}'.format(tarname))
tar = tarfile.open(name=tarname, mode='w')
tar.add('stage')
tar.close()
# Put the tarfile on lucky. First put it at the top level, then when
# complete move it into a subdir eng_archive. This lets the OCC side
# just watch for fully-uploaded files in that directory.
logger.info('ftp to lucky')
ftp = Ska.ftp.SFTP('lucky', logger=logger)
ftp.cd('/home/taldcroft')
files = ftp.ls()
if opt.ftp_dir not in files:
logger.info('mkdir {}'.format(opt.ftp_dir))
ftp.mkdir(opt.ftp_dir)
logger.info('put {0}'.format(tarname))
ftp.put(tarname)
logger.info('rename {0} {1}/{0}'.format(tarname, opt.ftp_dir))
ftp.rename(tarname, '{}/{}'.format(opt.ftp_dir, tarname))
ftp.close()
logger.info('unlink {0}'.format(tarname))
os.unlink(tarname)
logger.info('rmtree stage')
shutil.rmtree('stage')
def transfer_lucky_to_stage():
"""
Get tarfile(s) from lucky and untar into stage area. This is all done from
the archive root directory.
"""
# Open lucky ftp connection and watch for tarfile(s) in '/taldcroft/eng_archive'
logger.info('ftp to lucky')
ftp = Ska.ftp.SFTP('lucky', logger=logger)
ftp.cd('/home/taldcroft')
files = ftp.ls()
if opt.ftp_dir not in files:
logger.info('mkdir {}'.format(opt.ftp_dir))
ftp.mkdir(opt.ftp_dir)
ftp.cd(opt.ftp_dir)
for _ in range(opt.timeout / opt.sleep_time):
logger.info('Directory: {}'.format(ftp.ftp.getcwd()))
logger.info('Files: {}'.format(ftp.ls()))
files = [x for x in ftp.ls() if re.match('stage_\d+\.tar', x)]
if files:
break
time.sleep(opt.sleep_time)
logger.info('Waiting {0} seconds for archive files to appear...'.format(opt.sleep_time))
else:
logger.info('No tarfiles found before timeout')
ftp.close()
sys.exit(1)
# For each tarfile:
# - get file by ftp
# - untar
# - unlink local tarfile
# - delete tarfile on ftp
for tarname in sorted(files):
logger.info('Getting tarfile {0} from lucky'.format(tarname))
ftp.get(tarname)
logger.info('Extracting tarfile {0}'.format(tarname))
tar = tarfile.open(name=tarname, mode='r')
tar.extractall()
tar.close()
logger.info('Unlinking local {0}'.format(tarname))
os.unlink(tarname)
logger.info('Deleting ftp {0}'.format(tarname))
ftp.delete(tarname)
ftp.close()
def ftp_get_from_lucky(ftp_dirname, local_files, user=None, logger=None):
"""
Get files from lucky. This looks in remote ``ftp_dirname`` for files that
have basenames matching those of ``local_files``. The remote files
are copied to the corresponding local_files names. This is the converse
of ftp_put_to_lucky and thus requires unique basenames for all files.
"""
import Ska.ftp
import Ska.File
ftp = Ska.ftp.SFTP('lucky', logger=logger, user=user)
if user is None:
user = ftp.ftp.get_channel().transport.get_username()
ftp.cd('/home/{}/{}'.format(user, ftp_dirname))
for local_file in local_files:
file_dir, file_base = os.path.split(os.path.abspath(local_file))
if file_base in ftp.ls():
with Ska.File.chdir(file_dir):
ftp.get(file_base)
ftp.delete(file_base)
ftp.close()
def ftp_get_from_lucky(ftp_dirname, local_files, user=None, logger=None):
"""
Get files from lucky. This looks in remote ``ftp_dirname`` for files that
have basenames matching those of ``local_files``. The remote files
are copied to the corresponding local_files names. This is the converse
of ftp_put_to_lucky and thus requires unique basenames for all files.
"""
import Ska.ftp
import Ska.File
ftp = Ska.ftp.SFTP('lucky', logger=logger, user=user)
if user is None:
user = ftp.ftp.get_channel().transport.get_username()
ftp.cd('/home/{}/{}'.format(user, ftp_dirname))
for local_file in local_files:
file_dir, file_base = os.path.split(os.path.abspath(local_file))
if file_base in ftp.ls():
with Ska.File.chdir(file_dir):
ftp.get(file_base)
ftp.delete(file_base)
ftp.close()