def ftp_put_to_lucky(ftp_dirname, local_files, user=None, logger=None): """Put the ``local_files`` onto lucky in /``user``/``ftp_dirname``. First put it at the top level, then when complete move it into a subdir eng_archive. This lets the OCC side just watch for fully-uploaded files in that directory. The directory paths of ``local_files`` are stripped off so they all wind up in a flat structure within ``ftp_dirname``. """ import Ska.File import Ska.ftp import uuid ftp = Ska.ftp.SFTP('lucky', logger=logger, user=user) if user is None: user = ftp.ftp.get_channel().transport.get_username() ftp.cd('/home/{}'.format(user)) files = ftp.ls() if ftp_dirname not in files: ftp.mkdir(ftp_dirname) dir_files = ftp.ls(ftp_dirname) for local_file in local_files: file_dir, file_base = os.path.split(os.path.abspath(local_file)) ftp_file = str(uuid.uuid4()) # random unique identifier with Ska.File.chdir(file_dir): ftp.put(file_base, ftp_file) destination_file = "{}/{}".format(ftp_dirname, file_base) # If final destination of file already exists, delete that file. if file_base in dir_files: ftp.delete(destination_file) # Rename the temp/uniq-id file to the final destination ftp.rename(ftp_file, destination_file) ftp.close()
def transfer_stage_to_lucky(): # Make a tarfile of everything in stage directory tarname = 'stage_{0:d}.tar'.format(int(DateTime().secs)) logger.info('Making tarfile {0}'.format(tarname)) tar = tarfile.open(name=tarname, mode='w') tar.add('stage') tar.close() # Put the tarfile on lucky. First put it at the top level, then when # complete move it into a subdir eng_archive. This lets the OCC side # just watch for fully-uploaded files in that directory. logger.info('ftp to lucky') ftp = Ska.ftp.SFTP('lucky', logger=logger) ftp.cd('/home/taldcroft') files = ftp.ls() if opt.ftp_dir not in files: logger.info('mkdir {}'.format(opt.ftp_dir)) ftp.mkdir(opt.ftp_dir) logger.info('put {0}'.format(tarname)) ftp.put(tarname) logger.info('rename {0} {1}/{0}'.format(tarname, opt.ftp_dir)) ftp.rename(tarname, '{}/{}'.format(opt.ftp_dir, tarname)) ftp.close() logger.info('unlink {0}'.format(tarname)) os.unlink(tarname) logger.info('rmtree stage') shutil.rmtree('stage')
def transfer_lucky_to_stage(): """ Get tarfile(s) from lucky and untar into stage area. This is all done from the archive root directory. """ # Open lucky ftp connection and watch for tarfile(s) in '/taldcroft/eng_archive' logger.info('ftp to lucky') ftp = Ska.ftp.SFTP('lucky', logger=logger) ftp.cd('/home/taldcroft') files = ftp.ls() if opt.ftp_dir not in files: logger.info('mkdir {}'.format(opt.ftp_dir)) ftp.mkdir(opt.ftp_dir) ftp.cd(opt.ftp_dir) for _ in range(opt.timeout / opt.sleep_time): logger.info('Directory: {}'.format(ftp.ftp.getcwd())) logger.info('Files: {}'.format(ftp.ls())) files = [x for x in ftp.ls() if re.match('stage_\d+\.tar', x)] if files: break time.sleep(opt.sleep_time) logger.info( 'Waiting {0} seconds for archive files to appear...'.format( opt.sleep_time)) else: logger.info('No tarfiles found before timeout') ftp.close() sys.exit(1) # For each tarfile: # - get file by ftp # - untar # - unlink local tarfile # - delete tarfile on ftp for tarname in sorted(files): logger.info('Getting tarfile {0} from lucky'.format(tarname)) ftp.get(tarname) logger.info('Extracting tarfile {0}'.format(tarname)) tar = tarfile.open(name=tarname, mode='r') tar.extractall() tar.close() logger.info('Unlinking local {0}'.format(tarname)) os.unlink(tarname) logger.info('Deleting ftp {0}'.format(tarname)) ftp.delete(tarname) ftp.close()
def transfer_lucky_to_stage(): """ Get tarfile(s) from lucky and untar into stage area. This is all done from the archive root directory. """ # Open lucky ftp connection and watch for tarfile(s) in '/taldcroft/eng_archive' logger.info('ftp to lucky') ftp = Ska.ftp.SFTP('lucky', logger=logger) ftp.cd('/home/taldcroft') files = ftp.ls() if opt.ftp_dir not in files: logger.info('mkdir {}'.format(opt.ftp_dir)) ftp.mkdir(opt.ftp_dir) ftp.cd(opt.ftp_dir) for _ in range(opt.timeout / opt.sleep_time): logger.info('Directory: {}'.format(ftp.ftp.getcwd())) logger.info('Files: {}'.format(ftp.ls())) files = [x for x in ftp.ls() if re.match('stage_\d+\.tar', x)] if files: break time.sleep(opt.sleep_time) logger.info('Waiting {0} seconds for archive files to appear...'.format(opt.sleep_time)) else: logger.info('No tarfiles found before timeout') ftp.close() sys.exit(1) # For each tarfile: # - get file by ftp # - untar # - unlink local tarfile # - delete tarfile on ftp for tarname in sorted(files): logger.info('Getting tarfile {0} from lucky'.format(tarname)) ftp.get(tarname) logger.info('Extracting tarfile {0}'.format(tarname)) tar = tarfile.open(name=tarname, mode='r') tar.extractall() tar.close() logger.info('Unlinking local {0}'.format(tarname)) os.unlink(tarname) logger.info('Deleting ftp {0}'.format(tarname)) ftp.delete(tarname) ftp.close()
def ftp_get_from_lucky(ftp_dirname, local_files, user=None, logger=None): """ Get files from lucky. This looks in remote ``ftp_dirname`` for files that have basenames matching those of ``local_files``. The remote files are copied to the corresponding local_files names. This is the converse of ftp_put_to_lucky and thus requires unique basenames for all files. """ import Ska.ftp import Ska.File ftp = Ska.ftp.SFTP('lucky', logger=logger, user=user) if user is None: user = ftp.ftp.get_channel().transport.get_username() ftp.cd('/home/{}/{}'.format(user, ftp_dirname)) for local_file in local_files: file_dir, file_base = os.path.split(os.path.abspath(local_file)) if file_base in ftp.ls(): with Ska.File.chdir(file_dir): ftp.get(file_base) ftp.delete(file_base) ftp.close()