def copy_queue_file(tmpdir, name):
rfpath = os.path.join(RepositoryRoot, 'QUEUES', name)
if not ryw.is_valid_file(rfpath, 'copy_queue_file'):
ryw.give_bad_news('copy_queue_file: queue file invalid: ' + rfpath,
logging.error)
return (0, '')
freeKB = ryw.free_MB(tmpdir) * 1024
success,bytes = ryw.get_file_size(rfpath)
if not success:
return (0, '')
kB = math.ceil(bytes / 1024.0)
if pretty_much_out_of_space(kB, freeKB):
ryw.give_bad_news(
'user_error: nearly out of space while uploading queue, KB: ' + repr(kB), logging.error)
return (0, '')
#appendedName = name + '_selection'
appendedName = 'saved_selection'
tmpFileName = os.path.join(tmpdir, appendedName)
try:
shutil.copyfile(rfpath, tmpFileName)
except:
ryw.give_bad_news('copy_queue_file: failed to copy queue file: ' +
rfpath + ' -> ' + tmpFileName,
logging.critical)
return (0, '')
return (bytes, appendedName)
def main_nonoverlap():
ryw.check_logging(os.path.join(RepositoryRoot, "WWW", "logs"), "upload.log")
logging.debug("ReadIncomingCDStack: entered...")
success, numDiscs, resources, robotJobsDir, tmpIn = get_init_vals()
if not success:
sys.exit(1)
freeGB = ryw.free_MB(tmpIn) / 1000.0
ryw.give_news("current available disk space: " + repr(freeGB) + " GB.", logging.info)
tmpdir, objPrefix = ryw_upload.attempt_just_make_tmpdir(tmpIn, "I_", "")
if not tmpdir:
ryw.give_bad_news("ReadIncomingCDStack: failed to make tmpdir.", logging.critical)
sys.exit(1)
ryw.give_news("begin copying incoming discs...", logging.info)
jobFile = robot_read_all(robotJobsDir, objPrefix, numDiscs, tmpdir)
if not jobFile:
ryw_upload.cleanup_incoming(tmpdir, jobFile)
sys.exit(1)
# monitor the robot's job folder for completion of job.
ryw.give_news("", logging.info)
oldDone = []
while True:
ryw.give_news2("*", logging.info)
time.sleep(5)
logging.debug("ReadIncomingCDStack: done sleeping...")
success, done, doneList = check_finished(jobFile, tmpdir)
if not success:
ryw_upload.cleanup_incoming(tmpdir, jobFile)
sys.exit(1)
if not done:
success, doneList = check_partial_completion(tmpdir)
if not success:
ryw_upload.cleanup_incoming(tmpdir, jobFile)
sys.exit(1)
# process_finished_copies(tmpdir, doneList)
print_done_discs(doneList, oldDone)
oldDone = doneList
if done:
logging.debug("ReadIncomingCDStack: done.")
break
print_conclusion(doneList, tmpdir, jobFile)
def check_free_disk_space(dataSize, tmpOutDir):
logging.debug('check_free_disk_space: entered, data, dir: '
+ repr(dataSize) + tmpOutDir)
freeKB = ryw.free_MB(tmpOutDir) * 1024
logging.debug('check_free_disk_space: entered, freeKB: ' + repr(freeKB))
if ryw_upload.pretty_much_out_of_space(dataSize, freeKB):
ryw.give_bad_news(
'addRobotWriteRequest,check_free_disk_space:' +
'nearly out of disk space: size, free: ' +
repr(dataSize) + ' ' + repr(freeKB), logging.error)
return False
logging.debug('check_free_disk_space: dataSize, freeKB: '
+ repr(dataSize) + ' ' + repr(freeKB))
return True
def NOTUSED_main_overlap():
ryw.check_logging(os.path.join(RepositoryRoot, "WWW", "logs"), "upload.log")
logging.debug("ReadIncomingCDStack: entered...")
success, numDiscs, resources, robotJobsDir, tmpIn = get_init_vals()
if not success:
sys.exit(1)
freeGB = ryw.free_MB(tmpIn) / 1000.0
ryw.give_news("ReadIncomingCDStack: current available disk space: " + repr(freeGB) + " GB.", logging.info)
tmpdir, objPrefix = ryw_upload.attempt_just_make_tmpdir(tmpIn, "I_", "")
if not tmpdir:
ryw.give_bad_news("ReadIncomingCDStack: failed to make tmpdir.", logging.critical)
sys.exit(1)
jobFile = robot_read_all(robotJobsDir, objPrefix, numDiscs, tmpdir)
if not jobFile:
ryw_upload.cleanup_incoming(tmpdir, jobFile)
sys.exit(1)
# monitor the robot's job folder for completion of job
# also periodically monitor the tmp folder for completed disk reads
while True:
ryw.give_news2("*", logging.info)
time.sleep(5)
logging.debug("ReadIncomingCDStack: done sleeping...")
success, done, doneList = check_finished(jobFile, tmpdir)
if not success:
ryw_upload.cleanup_incoming(tmpdir, jobFile)
sys.exit(1)
if not done:
success, doneList = check_partial_completion(tmpdir)
if not success:
ryw_upload.cleanup_incoming(tmpdir, jobFile)
sys.exit(1)
process_finished_copies(tmpdir, doneList)
if done:
logging.debug("ReadIncomingCDStack: done.")
break
logging.debug("ReadIncomingCDStack: removing robot job data: " + tmpdir + " " + jobFile)
ryw_upload.cleanup_incoming(tmpdir, jobFile)
def check_free_space(values, path):
"""check available disk space."""
freeKB = ryw.free_MB(path) * 1024
logging.debug('check_free_space: freeKB, valuesKB: ' + repr(freeKB) + ' ' +
repr(values['kB']))
if ryw_upload.pretty_much_out_of_space(values['kB'], freeKB):
ryw.give_bad_news(
'user_error: nearly out of space in the object store: ' +
repr(values['kB']) + ' ' + repr(freeKB),
logging.critical)
return False
logging.debug(
'check_free_space: has space, kB, freekB: ' + repr(values['kB']) + ' '
+ repr(freeKB))
return True
def check_free_space(prefix, supressWarning = False):
"""check how much free space is available and issue warnings."""
freeMB = ryw.free_MB(prefix)
#logging.debug('check_free_space: MB available: ' + repr(freeMB))
if freeMB < FREE_SPACE_MB_GIVEUP:
ryw.give_bad_news(
'user_error: nearly out of space: '+repr(freeMB)+' MB. operation aborted.',
logging.error)
freeMB = 0
elif freeMB < FREE_SPACE_MB_WARN2 and (not supressWarning):
ryw.give_bad_news(
'warning: free disk space critically low: '+repr(freeMB)+' MB. (successful operation may still be possible.)',
logging.warning)
elif freeMB < FREE_SPACE_MB_WARN1 and (not supressWarning):
ryw.give_bad_news(
'warning: free disk space low: '+repr(freeMB)+' MB. (successful operation may still be possible.)',
logging.warning)
return freeMB
def read_uploaded_file(form, buf, tmpdir, filename, field):
"""read the uploaded file and put it in the temp directory.
returns the number of KB read, rounded to 4KB chunks."""
filename = fix_IE_file_name(filename)
tmpDataFile = open_tmp_data_file(tmpdir, filename)
if not tmpDataFile:
return (0, 0)
freeKB = ryw.free_MB(tmpdir) * 1024
kB = long(0)
try:
while buf:
kB += 4
if pretty_much_out_of_space(kB, freeKB):
ryw.give_bad_news(
'user_error: nearly out of space while writing uploaded file, file kB seen so far: ' + repr(kB), logging.error)
raise NameError('file write: out of space.')
tmpDataFile.write(buf)
buf = form[field].file.read(4096)
#ryw.give_bad_news('read_uploaded_file: test wait for 60s.',
# logging.error)
#time.sleep(60)
tmpDataFile.flush()
tmpDataFile.close()
logging.debug('temporary data file written')
bytes = os.path.getsize(os.path.join(tmpdir, filename))
return (kB, bytes)
except:
ryw.give_bad_news(
'fatal_error: writing of temporary data file failed.',
logging.critical)
return (0, 0)
def try_unzip_file(form, tmpdir, filename, kB):
"""unzip the unploaded zip file.
returns a tuple: the stuff to upload, and a temp extract directory
that should be cleaned up."""
extractDir = tmpdir + '.EXT'
unzip = form.getfirst('unzip', '')
if not unzip:
return (os.path.join(tmpdir, filename), None)
freeKB = ryw.free_MB(tmpdir) * 1024
if pretty_much_out_of_space(kB, freeKB):
ryw.give_bad_news(
'user_error: nearly out of space while trying to unzip, zip file kB: ' + repr(kB), logging.error)
return (None, extractDir)
try:
os.mkdir(extractDir)
except:
ryw.give_bad_news(
'fatal_error: failed to make temporary extract directory.',
logging.critical)
return (None, extractDir)
try:
su.zipfile_extractall(os.path.join(tmpdir, filename), extractDir)
except:
ryw.give_bad_news('fatal_error: failed to extract zip file.',
logging.critical)
return (None, extractDir)
logging.debug('successfully extracted zipfile ' + filename + ' to ' +
extractDir)
return (extractDir, extractDir)
def copy_local_file_for_upload(form, tmpdir, uploadFileName,
localFound, localPath, localDir,
isCopyingExcerpt = False):
if not localFound:
return (True, False, 0)
if localDir:
dirKB = ryw_disc.getRecursiveSizeInKB(localPath)
bytes = dirKB * 1024
if bytes == 0:
ryw.give_bad_news(
'copy_local_file_for_upload: 0-sized local directory: '+
localPath, logging.error)
return (False, False, 0)
else:
success,bytes = ryw.get_file_size(localPath)
if not success:
return (False, False, 0)
#if uploadFileName != truncateLocalName:
# ryw.give_bad_news(
# 'copy_local_file_for_upload: repeated local file name does<BR>' +
# ' not match the name of the local file to be uploaded.<BR>' +
# uploadFileName + '<BR>' + localName,
# logging.error)
# return (False, False, 0)
freeKB = ryw.free_MB(tmpdir) * 1024
kB = math.ceil(bytes / 1024.0)
if pretty_much_out_of_space(kB, freeKB):
ryw.give_bad_news(
'copy_local_file_for_upload: nearly out of space ' +
'while uploading queue, KB: ' + repr(kB), logging.error)
return (False, False, 0)
tmpFileName = os.path.join(tmpdir, uploadFileName)
try:
if localDir:
ryw.give_news2('<BR>Copying local directory: ' + localPath,
logging.info)
if isCopyingExcerpt:
#
# don't want the "excerpts" directory to contain
# just a lone directory inside in the common case.
# copy the content into the "excerpts" directory directly.
# "tmpdir" in this case is the "excerpts" directory.
# su.copytree tolerates an existing destination directory
# and doesn't wipe out what's already in it.
#
tmpFileName = os.path.normpath(tmpdir)
su.copytree(localPath, tmpFileName)
else:
shutil.copytree(localPath, tmpFileName)
else:
ryw.give_news2('<BR>Copying local file: ' + localPath,
logging.info)
shutil.copyfile(localPath, tmpFileName)
#shutil.copyfile(
# os.path.join(RepositoryRoot, 'Tmp-in', 'foo.txt'),
# tmpFileName)
except:
ryw.give_bad_news('copy_local_file_for_upload: ' +
'failed to copy data: ' +
localPath + ' -> ' + tmpFileName,
logging.critical)
return (False, False, 0)
logging.debug('copy_local_file_for_upload: ' +
'succeeded copying local data: ' +
localPath + ' -> ' + tmpFileName, logging.info)
return (True, True, bytes)
