Beispiel #1
0
def ftp_put_to_lucky(ftp_dirname, local_files, user=None, logger=None):
    """Put the ``local_files`` onto lucky in /``user``/``ftp_dirname``. First put it at the top
    level, then when complete move it into a subdir eng_archive.  This lets the OCC side
    just watch for fully-uploaded files in that directory.

    The directory paths of ``local_files`` are stripped off so they all wind up
    in a flat structure within ``ftp_dirname``.
    """
    import Ska.File
    import Ska.ftp
    import uuid

    ftp = Ska.ftp.SFTP('lucky', logger=logger, user=user)
    if user is None:
        user = ftp.ftp.get_channel().transport.get_username()
    ftp.cd('/home/{}'.format(user))
    files = ftp.ls()

    if ftp_dirname not in files:
        ftp.mkdir(ftp_dirname)
    dir_files = ftp.ls(ftp_dirname)

    for local_file in local_files:
        file_dir, file_base = os.path.split(os.path.abspath(local_file))
        ftp_file = str(uuid.uuid4())  # random unique identifier
        with Ska.File.chdir(file_dir):
            ftp.put(file_base, ftp_file)
            destination_file = "{}/{}".format(ftp_dirname, file_base)
            # If final destination of file already exists, delete that file.
            if file_base in dir_files:
                ftp.delete(destination_file)
            # Rename the temp/uniq-id file to the final destination
            ftp.rename(ftp_file, destination_file)

    ftp.close()
def transfer_stage_to_lucky():
    # Make a tarfile of everything in stage directory
    tarname = 'stage_{0:d}.tar'.format(int(DateTime().secs))
    logger.info('Making tarfile {0}'.format(tarname))
    tar = tarfile.open(name=tarname, mode='w')
    tar.add('stage')
    tar.close()

    # Put the tarfile on lucky.  First put it at the top level, then when
    # complete move it into a subdir eng_archive.  This lets the OCC side
    # just watch for fully-uploaded files in that directory.
    logger.info('ftp to lucky')
    ftp = Ska.ftp.SFTP('lucky', logger=logger)
    ftp.cd('/home/taldcroft')
    files = ftp.ls()
    if opt.ftp_dir not in files:
        logger.info('mkdir {}'.format(opt.ftp_dir))
        ftp.mkdir(opt.ftp_dir)
    logger.info('put {0}'.format(tarname))
    ftp.put(tarname)
    logger.info('rename {0} {1}/{0}'.format(tarname, opt.ftp_dir))
    ftp.rename(tarname, '{}/{}'.format(opt.ftp_dir, tarname))
    ftp.close()

    logger.info('unlink {0}'.format(tarname))
    os.unlink(tarname)
    logger.info('rmtree stage')
    shutil.rmtree('stage')
Beispiel #3
0
def ftp_put_to_lucky(ftp_dirname, local_files, user=None, logger=None):
    """Put the ``local_files`` onto lucky in /``user``/``ftp_dirname``. First put it at the top
    level, then when complete move it into a subdir eng_archive.  This lets the OCC side
    just watch for fully-uploaded files in that directory.

    The directory paths of ``local_files`` are stripped off so they all wind up
    in a flat structure within ``ftp_dirname``.
    """
    import Ska.File
    import Ska.ftp
    import uuid

    ftp = Ska.ftp.SFTP('lucky', logger=logger, user=user)
    if user is None:
        user = ftp.ftp.get_channel().transport.get_username()
    ftp.cd('/home/{}'.format(user))
    files = ftp.ls()

    if ftp_dirname not in files:
        ftp.mkdir(ftp_dirname)
    dir_files = ftp.ls(ftp_dirname)

    for local_file in local_files:
        file_dir, file_base = os.path.split(os.path.abspath(local_file))
        ftp_file = str(uuid.uuid4())  # random unique identifier
        with Ska.File.chdir(file_dir):
            ftp.put(file_base, ftp_file)
            destination_file = "{}/{}".format(ftp_dirname, file_base)
            # If final destination of file already exists, delete that file.
            if file_base in dir_files:
                ftp.delete(destination_file)
            # Rename the temp/uniq-id file to the final destination
            ftp.rename(ftp_file, destination_file)

    ftp.close()
Beispiel #4
0
def transfer_stage_to_lucky():
    # Make a tarfile of everything in stage directory
    tarname = 'stage_{0:d}.tar'.format(int(DateTime().secs))
    logger.info('Making tarfile {0}'.format(tarname))
    tar = tarfile.open(name=tarname, mode='w')
    tar.add('stage')
    tar.close()

    # Put the tarfile on lucky.  First put it at the top level, then when
    # complete move it into a subdir eng_archive.  This lets the OCC side
    # just watch for fully-uploaded files in that directory.
    logger.info('ftp to lucky')
    ftp = Ska.ftp.SFTP('lucky', logger=logger)
    ftp.cd('/home/taldcroft')
    files = ftp.ls()
    if opt.ftp_dir not in files:
        logger.info('mkdir {}'.format(opt.ftp_dir))
        ftp.mkdir(opt.ftp_dir)
    logger.info('put {0}'.format(tarname))
    ftp.put(tarname)
    logger.info('rename {0} {1}/{0}'.format(tarname, opt.ftp_dir))
    ftp.rename(tarname, '{}/{}'.format(opt.ftp_dir, tarname))
    ftp.close()

    logger.info('unlink {0}'.format(tarname))
    os.unlink(tarname)
    logger.info('rmtree stage')
    shutil.rmtree('stage')
Beispiel #5
0
def transfer_lucky_to_stage():
    """
    Get tarfile(s) from lucky and untar into stage area.  This is all done from
    the archive root directory.
    """

    # Open lucky ftp connection and watch for tarfile(s) in '/taldcroft/eng_archive'
    logger.info('ftp to lucky')
    ftp = Ska.ftp.SFTP('lucky', logger=logger)
    ftp.cd('/home/taldcroft')
    files = ftp.ls()
    if opt.ftp_dir not in files:
        logger.info('mkdir {}'.format(opt.ftp_dir))
        ftp.mkdir(opt.ftp_dir)
    ftp.cd(opt.ftp_dir)
    for _ in range(opt.timeout / opt.sleep_time):
        logger.info('Directory: {}'.format(ftp.ftp.getcwd()))
        logger.info('Files: {}'.format(ftp.ls()))
        files = [x for x in ftp.ls() if re.match('stage_\d+\.tar', x)]
        if files:
            break
        time.sleep(opt.sleep_time)
        logger.info(
            'Waiting {0} seconds for archive files to appear...'.format(
                opt.sleep_time))
    else:
        logger.info('No tarfiles found before timeout')
        ftp.close()
        sys.exit(1)

    # For each tarfile:
    # - get file by ftp
    # - untar
    # - unlink local tarfile
    # - delete tarfile on ftp
    for tarname in sorted(files):
        logger.info('Getting tarfile {0} from lucky'.format(tarname))
        ftp.get(tarname)
        logger.info('Extracting tarfile {0}'.format(tarname))
        tar = tarfile.open(name=tarname, mode='r')
        tar.extractall()
        tar.close()

        logger.info('Unlinking local {0}'.format(tarname))
        os.unlink(tarname)
        logger.info('Deleting ftp {0}'.format(tarname))
        ftp.delete(tarname)

    ftp.close()
def transfer_lucky_to_stage():
    """
    Get tarfile(s) from lucky and untar into stage area.  This is all done from
    the archive root directory.
    """

    # Open lucky ftp connection and watch for tarfile(s) in '/taldcroft/eng_archive'
    logger.info('ftp to lucky')
    ftp = Ska.ftp.SFTP('lucky', logger=logger)
    ftp.cd('/home/taldcroft')
    files = ftp.ls()
    if opt.ftp_dir not in files:
        logger.info('mkdir {}'.format(opt.ftp_dir))
        ftp.mkdir(opt.ftp_dir)
    ftp.cd(opt.ftp_dir)
    for _ in range(opt.timeout / opt.sleep_time):
        logger.info('Directory: {}'.format(ftp.ftp.getcwd()))
        logger.info('Files: {}'.format(ftp.ls()))
        files = [x for x in ftp.ls() if re.match('stage_\d+\.tar', x)]
        if files:
            break
        time.sleep(opt.sleep_time)
        logger.info('Waiting {0} seconds for archive files to appear...'.format(opt.sleep_time))
    else:
        logger.info('No tarfiles found before timeout')
        ftp.close()
        sys.exit(1)

    # For each tarfile:
    # - get file by ftp
    # - untar
    # - unlink local tarfile
    # - delete tarfile on ftp
    for tarname in sorted(files):
        logger.info('Getting tarfile {0} from lucky'.format(tarname))
        ftp.get(tarname)
        logger.info('Extracting tarfile {0}'.format(tarname))
        tar = tarfile.open(name=tarname, mode='r')
        tar.extractall()
        tar.close()

        logger.info('Unlinking local {0}'.format(tarname))
        os.unlink(tarname)
        logger.info('Deleting ftp {0}'.format(tarname))
        ftp.delete(tarname)

    ftp.close()
Beispiel #7
0
def ftp_get_from_lucky(ftp_dirname, local_files, user=None, logger=None):
    """
    Get files from lucky.  This looks in remote ``ftp_dirname`` for files that
    have basenames matching those of ``local_files``.  The remote files
    are copied to the corresponding local_files names.  This is the converse
    of ftp_put_to_lucky and thus requires unique basenames for all files.
    """
    import Ska.ftp
    import Ska.File

    ftp = Ska.ftp.SFTP('lucky', logger=logger, user=user)
    if user is None:
        user = ftp.ftp.get_channel().transport.get_username()
    ftp.cd('/home/{}/{}'.format(user, ftp_dirname))
    for local_file in local_files:
        file_dir, file_base = os.path.split(os.path.abspath(local_file))
        if file_base in ftp.ls():
            with Ska.File.chdir(file_dir):
                ftp.get(file_base)
                ftp.delete(file_base)

    ftp.close()
Beispiel #8
0
def ftp_get_from_lucky(ftp_dirname, local_files, user=None, logger=None):
    """
    Get files from lucky.  This looks in remote ``ftp_dirname`` for files that
    have basenames matching those of ``local_files``.  The remote files
    are copied to the corresponding local_files names.  This is the converse
    of ftp_put_to_lucky and thus requires unique basenames for all files.
    """
    import Ska.ftp
    import Ska.File

    ftp = Ska.ftp.SFTP('lucky', logger=logger, user=user)
    if user is None:
        user = ftp.ftp.get_channel().transport.get_username()
    ftp.cd('/home/{}/{}'.format(user, ftp_dirname))
    for local_file in local_files:
        file_dir, file_base = os.path.split(os.path.abspath(local_file))
        if file_base in ftp.ls():
            with Ska.File.chdir(file_dir):
                ftp.get(file_base)
                ftp.delete(file_base)

    ftp.close()