def copy_queue_file(tmpdir, name):
    
    rfpath = os.path.join(RepositoryRoot, 'QUEUES', name)
    if not ryw.is_valid_file(rfpath, 'copy_queue_file'):
        ryw.give_bad_news('copy_queue_file: queue file invalid: ' + rfpath,
                          logging.error)
        return (0, '')
    
    freeKB = ryw.free_MB(tmpdir) * 1024
    success,bytes = ryw.get_file_size(rfpath)
    if not success:
        return (0, '')
    kB = math.ceil(bytes / 1024.0)

    if pretty_much_out_of_space(kB, freeKB):
        ryw.give_bad_news(
            'user_error: nearly out of space while uploading queue, KB: ' + repr(kB), logging.error)
        return (0, '')

    #appendedName = name + '_selection'
    appendedName = 'saved_selection'
    tmpFileName = os.path.join(tmpdir, appendedName)
    try:
        shutil.copyfile(rfpath, tmpFileName)
    except:
        ryw.give_bad_news('copy_queue_file: failed to copy queue file: ' +
                          rfpath + ' -> ' + tmpFileName,
                          logging.critical)
        return (0, '')

    return (bytes, appendedName)
def main_nonoverlap():
    ryw.check_logging(os.path.join(RepositoryRoot, "WWW", "logs"), "upload.log")
    logging.debug("ReadIncomingCDStack: entered...")

    success, numDiscs, resources, robotJobsDir, tmpIn = get_init_vals()
    if not success:
        sys.exit(1)

    freeGB = ryw.free_MB(tmpIn) / 1000.0
    ryw.give_news("current available disk space: " + repr(freeGB) + " GB.", logging.info)

    tmpdir, objPrefix = ryw_upload.attempt_just_make_tmpdir(tmpIn, "I_", "")
    if not tmpdir:
        ryw.give_bad_news("ReadIncomingCDStack: failed to make tmpdir.", logging.critical)
        sys.exit(1)

    ryw.give_news("begin copying incoming discs...", logging.info)

    jobFile = robot_read_all(robotJobsDir, objPrefix, numDiscs, tmpdir)
    if not jobFile:
        ryw_upload.cleanup_incoming(tmpdir, jobFile)
        sys.exit(1)

    # monitor the robot's job folder for completion of job.

    ryw.give_news("", logging.info)
    oldDone = []
    while True:
        ryw.give_news2("*", logging.info)
        time.sleep(5)
        logging.debug("ReadIncomingCDStack: done sleeping...")

        success, done, doneList = check_finished(jobFile, tmpdir)
        if not success:
            ryw_upload.cleanup_incoming(tmpdir, jobFile)
            sys.exit(1)

        if not done:
            success, doneList = check_partial_completion(tmpdir)
            if not success:
                ryw_upload.cleanup_incoming(tmpdir, jobFile)
                sys.exit(1)

        # process_finished_copies(tmpdir, doneList)
        print_done_discs(doneList, oldDone)
        oldDone = doneList

        if done:
            logging.debug("ReadIncomingCDStack: done.")
            break

    print_conclusion(doneList, tmpdir, jobFile)
def check_free_disk_space(dataSize, tmpOutDir):
    logging.debug('check_free_disk_space: entered, data, dir: '
                  + repr(dataSize) + tmpOutDir)
    freeKB = ryw.free_MB(tmpOutDir) * 1024
    logging.debug('check_free_disk_space: entered, freeKB: ' + repr(freeKB))
    if ryw_upload.pretty_much_out_of_space(dataSize, freeKB):
        ryw.give_bad_news(
            'addRobotWriteRequest,check_free_disk_space:' +
            'nearly out of disk space: size, free: ' +
            repr(dataSize) + ' ' + repr(freeKB), logging.error)
        return False
    logging.debug('check_free_disk_space: dataSize, freeKB: '
                  + repr(dataSize) + ' ' + repr(freeKB))
    return True
def NOTUSED_main_overlap():
    ryw.check_logging(os.path.join(RepositoryRoot, "WWW", "logs"), "upload.log")
    logging.debug("ReadIncomingCDStack: entered...")

    success, numDiscs, resources, robotJobsDir, tmpIn = get_init_vals()
    if not success:
        sys.exit(1)

    freeGB = ryw.free_MB(tmpIn) / 1000.0
    ryw.give_news("ReadIncomingCDStack: current available disk space: " + repr(freeGB) + " GB.", logging.info)

    tmpdir, objPrefix = ryw_upload.attempt_just_make_tmpdir(tmpIn, "I_", "")
    if not tmpdir:
        ryw.give_bad_news("ReadIncomingCDStack: failed to make tmpdir.", logging.critical)
        sys.exit(1)

    jobFile = robot_read_all(robotJobsDir, objPrefix, numDiscs, tmpdir)
    if not jobFile:
        ryw_upload.cleanup_incoming(tmpdir, jobFile)
        sys.exit(1)

    # monitor the robot's job folder for completion of job
    # also periodically monitor the tmp folder for completed disk reads

    while True:
        ryw.give_news2("*", logging.info)
        time.sleep(5)
        logging.debug("ReadIncomingCDStack: done sleeping...")

        success, done, doneList = check_finished(jobFile, tmpdir)
        if not success:
            ryw_upload.cleanup_incoming(tmpdir, jobFile)
            sys.exit(1)

        if not done:
            success, doneList = check_partial_completion(tmpdir)
            if not success:
                ryw_upload.cleanup_incoming(tmpdir, jobFile)
                sys.exit(1)

        process_finished_copies(tmpdir, doneList)

        if done:
            logging.debug("ReadIncomingCDStack: done.")
            break

    logging.debug("ReadIncomingCDStack: removing robot job data: " + tmpdir + " " + jobFile)
    ryw_upload.cleanup_incoming(tmpdir, jobFile)
def check_free_space(values, path):
    """check available disk space."""

    freeKB = ryw.free_MB(path) * 1024
    logging.debug('check_free_space: freeKB, valuesKB: ' + repr(freeKB) + ' ' +
                  repr(values['kB']))
    if ryw_upload.pretty_much_out_of_space(values['kB'], freeKB):
        ryw.give_bad_news(
            'user_error: nearly out of space in the object store: ' +
            repr(values['kB']) + ' ' + repr(freeKB),
            logging.critical)
        return False

    logging.debug(
        'check_free_space: has space, kB, freekB: ' + repr(values['kB']) + ' '
        + repr(freeKB))
    return True
def check_free_space(prefix, supressWarning = False):
    """check how much free space is available and issue warnings."""

    freeMB = ryw.free_MB(prefix)
    #logging.debug('check_free_space: MB available: ' + repr(freeMB))
    
    if freeMB < FREE_SPACE_MB_GIVEUP:
        ryw.give_bad_news(
            'user_error: nearly out of space: '+repr(freeMB)+' MB. operation aborted.',
            logging.error)
        freeMB = 0
    elif freeMB < FREE_SPACE_MB_WARN2 and (not supressWarning):
        ryw.give_bad_news(
            'warning: free disk space critically low: '+repr(freeMB)+' MB. (successful operation may still be possible.)',
            logging.warning)
    elif freeMB < FREE_SPACE_MB_WARN1 and (not supressWarning):
        ryw.give_bad_news(
            'warning: free disk space low: '+repr(freeMB)+' MB. (successful operation may still be possible.)',
            logging.warning)
    return freeMB
def read_uploaded_file(form, buf, tmpdir, filename, field):
    """read the uploaded file and put it in the temp directory.
    returns the number of KB read, rounded to 4KB chunks."""

    filename = fix_IE_file_name(filename)
    tmpDataFile = open_tmp_data_file(tmpdir, filename)
    if not tmpDataFile:
        return (0, 0)

    freeKB = ryw.free_MB(tmpdir) * 1024
    kB = long(0)

    try:
        while buf:
            kB += 4
            if pretty_much_out_of_space(kB, freeKB):
                ryw.give_bad_news(
                    'user_error: nearly out of space while writing uploaded file, file kB seen so far: ' + repr(kB), logging.error)
                raise NameError('file write: out of space.')
            
            tmpDataFile.write(buf)
            buf = form[field].file.read(4096)

            #ryw.give_bad_news('read_uploaded_file: test wait for 60s.',
            #                  logging.error)
            #time.sleep(60)

        tmpDataFile.flush()
        tmpDataFile.close()
        logging.debug('temporary data file written')
        bytes = os.path.getsize(os.path.join(tmpdir, filename))
        return (kB, bytes)
    except:
        ryw.give_bad_news(
            'fatal_error: writing of temporary data file failed.',
            logging.critical)
        return (0, 0)
def try_unzip_file(form, tmpdir, filename, kB):
    """unzip the unploaded zip file.
    returns a tuple: the stuff to upload, and a temp extract directory
    that should be cleaned up."""
    
    extractDir = tmpdir + '.EXT'

    unzip = form.getfirst('unzip', '')
    
    if not unzip:
        return (os.path.join(tmpdir, filename), None)

    freeKB = ryw.free_MB(tmpdir) * 1024
    if pretty_much_out_of_space(kB, freeKB):
        ryw.give_bad_news(
            'user_error: nearly out of space while trying to unzip, zip file kB: ' + repr(kB), logging.error)
        return (None, extractDir)

    try:
        os.mkdir(extractDir)
    except:
        ryw.give_bad_news(
            'fatal_error: failed to make temporary extract directory.',
            logging.critical)
        return (None, extractDir)

    try:
        su.zipfile_extractall(os.path.join(tmpdir, filename), extractDir)
    except:
        ryw.give_bad_news('fatal_error: failed to extract zip file.',
                          logging.critical)
        return (None, extractDir)

    logging.debug('successfully extracted zipfile ' + filename + ' to ' +
                  extractDir)
    return (extractDir, extractDir)
def copy_local_file_for_upload(form, tmpdir, uploadFileName,
                               localFound, localPath, localDir,
                               isCopyingExcerpt = False):
    if not localFound:
        return (True, False, 0)

    if localDir:
        dirKB = ryw_disc.getRecursiveSizeInKB(localPath)
        bytes = dirKB * 1024
        if bytes == 0:
            ryw.give_bad_news(
                'copy_local_file_for_upload: 0-sized local directory: '+
                localPath, logging.error)
            return (False, False, 0)
    else:
        success,bytes = ryw.get_file_size(localPath)
        if not success:
            return (False, False, 0)

    #if uploadFileName != truncateLocalName:
    #    ryw.give_bad_news(
    #        'copy_local_file_for_upload: repeated local file name does<BR>' +
    #        ' not match the name of the local file to be uploaded.<BR>' +
    #        uploadFileName + '<BR>' + localName,
    #        logging.error)
    #    return (False, False, 0)
    
    freeKB = ryw.free_MB(tmpdir) * 1024
    kB = math.ceil(bytes / 1024.0)

    if pretty_much_out_of_space(kB, freeKB):
        ryw.give_bad_news(
            'copy_local_file_for_upload: nearly out of space ' +
            'while uploading queue, KB: ' + repr(kB), logging.error)
        return (False, False, 0)

    tmpFileName = os.path.join(tmpdir, uploadFileName)
    try:
        if localDir:
            ryw.give_news2('<BR>Copying local directory: ' + localPath,
                           logging.info)
            if isCopyingExcerpt:
                #
                # don't want the "excerpts" directory to contain
                # just a lone directory inside in the common case.
                # copy the content into the "excerpts" directory directly.
                # "tmpdir" in this case is the "excerpts" directory.
                # su.copytree tolerates an existing destination directory
                # and doesn't wipe out what's already in it.
                #
                tmpFileName = os.path.normpath(tmpdir)
                su.copytree(localPath, tmpFileName)
            else:
                shutil.copytree(localPath, tmpFileName)
        else:
            ryw.give_news2('<BR>Copying local file: ' + localPath,
                           logging.info)
            shutil.copyfile(localPath, tmpFileName)
            #shutil.copyfile(
            #    os.path.join(RepositoryRoot, 'Tmp-in', 'foo.txt'),
            #    tmpFileName)
    except:
        ryw.give_bad_news('copy_local_file_for_upload: ' +
                          'failed to copy data: ' +
                          localPath + ' -> ' + tmpFileName,
                          logging.critical)
        return (False, False, 0)

    logging.debug('copy_local_file_for_upload: ' +
                  'succeeded copying local data: ' +
                  localPath + ' -> ' + tmpFileName, logging.info)
    return (True, True, bytes)