def deleteRequested(form):
image = form.getfirst("Img","")
if not image:
print "No Image specified to delete"
sys.exit(1)
success, resources = get_resources()
if not success:
ryw.give_bad_news("Error parsing resource file",logging.error)
sys.exit(1)
robotsJobDir = resources['robotsjobdir']
jobfile = os.path.join(robotsJobDir, image)
ryw.cleanup_path(jobfile+".JRQ",'deleteOutgoingImage.deleteRequested:')
ryw.cleanup_path(jobfile+".ERR",'deleteOutgoingImage.deleteRequested:')
ryw.cleanup_path(jobfile+".DON",'deleteOutgoingImage.deleteRequested:')
ryw.cleanup_path(jobfile,'deleteOutgoingImage.deleteRequested:')
tmpout = resources['tmpout']
image = os.path.join(tmpout,image)
if not os.path.exists(image):
ryw.give_bad_news("specified image doesnt exist",logging.info)
sys.exit(1)
ryw.cleanup_path(image,"deleteOutgoingImage.deleteRequested:")
sys.stdout.write("True")
sys.exit(0)
def __open_tmp_search_file(self):
try:
tmpFileName = self.searchfile + '_TMP'
if os.path.exists(tmpFileName):
ryw.cleanup_path(tmpFileName, '__open_tmp_search_file: ')
f = open(tmpFileName, 'ab')
logging.debug('__open_tmp_search_file: succeeded: ' + tmpFileName)
return f
except:
ryw.give_bad_news('__open_tmp_search_file: failed' +
self.searchfile + '_TMP', logging.critical)
return None
def __write_to_disk(self):
tmppath = self.reverseListsFile + '.TMP'
try:
su.pickdump(self.reverseDict, tmppath)
except:
ryw.give_bad_news('ReverseLists.add: pickdump failed.',
logging.critical)
return False
success,bakpath = ryw.make_tmp_file_permanent(
tmppath, self.reverseListsFile)
ryw.cleanup_path(tmppath, '_write_to_disk: cleanup, tmppath:')
ryw.cleanup_path(bakpath, '_write_to_disk: cleanup, bakpath:')
ryw.db_print('write_to_disk: written to disk...', 11)
return success
def create_view_path(path, objectID, version):
if os.path.exists(path):
ryw.give_news('create_view_path: already exists: ' + path + ', ' +
'replacing with new version.',
logging.warning)
#return True
#remove old view path and continue
if not ryw.cleanup_path(path, 'create_view_path:'):
ryw.give_news('create_view_path: fails to remove old view path.',
logging.error)
return True
try:
su.createparentdirpath(path)
except:
ryw.give_bad_news('create_view_path: failed to createparentdirpath: '+
path, logging.critical)
return False
try:
f = open(path, 'w')
f.write(objectID + '#' + str(version))
f.close()
except:
ryw.give_bad_news('create_view_path: failed to write leaf file: '+path,
logging.critical)
return False
logging.debug('create_view_path: done creating path: '+path)
return True
def write_file(self, fullFileName):
"""10/24/08, called by ChapterListFormHandle.py."""
if self.chapterList == None or self.chapterList == {}:
#
# then we remove the chapter list file.
#
ryw.db_print_info_browser('ChapterList.write_file: ' + \
'chapterList is empty.', 41)
if ryw.cleanup_path(fullFileName,
'ChapterList.write_file'):
return True
ryw.give_bad_news('ChapterList.write_file:' +
'failed to remove file', logging.error)
return False
try:
su.pickdump(self.chapterList, fullFileName)
except:
ryw.give_bad_news('ChapterList.write_file: pickdump failed: ' +
fullFileName, logging.error)
return False
ryw.db_print_info_browser('ChapterList.write_file success: ' +
fullFileName, 41)
return True
def copy_tree_diff_file_view(src, dst):
"""always copy leaves."""
assert(os.path.exists(src))
assert(os.path.isfile(src))
logging.debug('copy_tree_diff_file_view: ' + src + ' -> ' + dst)
try:
if os.path.exists(dst) and os.path.isdir(dst):
ryw.cleanup_path(dst, 'copy_tree_diff_file:')
shutil.copyfile(src, dst)
return True
except:
ryw.give_bad_news('copy_tree_diff_file_view: failed to copy file: ' +
src + ' -> ' + dst, logging.critical)
return False
def resize_thumbnails(auxDir, forced = True):
if not os.path.exists(auxDir):
return True
thumbsDir = os.path.join(auxDir, 'thumbnails')
if not os.path.exists(thumbsDir):
return True
thumbNames = os.listdir(thumbsDir)
if len(thumbNames) == 0:
return True
for thumbName in thumbNames:
if thumbName == 'Thumbs.db':
continue
thumbPath = os.path.join(thumbsDir, thumbName)
if not ryw_view.is_image(thumbPath):
ryw.give_bad_news(
'resize_thumbnails: probably not an image file: ' +
thumbPath, logging.error)
return False
scaledDir = os.path.join(auxDir, 'thumbs_scaled')
if os.path.exists(scaledDir) and not forced:
return True
ryw.cleanup_path(scaledDir, 'ryw_upload.resize_thumbnails:')
try:
su.createdirpath(scaledDir)
for thumbName in thumbNames:
if thumbName == 'Thumbs.db':
continue
thumbPath = os.path.join(thumbsDir, thumbName)
thumbDest = os.path.join(scaledDir, thumbName)
if not ryw_view.resize_image(thumbPath, thumbDest):
ryw.give_bad_news('resize_thumbnails: failed: ' +
thumbPath + ' -> ' + thumbDest,
logging.error)
return False
except:
ryw.give_bad_news('failed to resize thumbnails.', logging.error)
return True
def deleteAll():
print '<TITLE>Deleting Outing Disc Images</TITLE>'
ryw_view.print_logo()
success,resources = get_resources()
if not success:
logging.error("main: error parsing resource file")
sys.exit(1)
outdir = resources['tmpout']
for i in [os.path.join(outdir,k) for k in os.listdir(outdir)]:
ryw.cleanup_path(i,"delete outgoing image:")
robotsjobdir = resources['robotsjobdir']
for i in [os.path.join(robotsjobdir,k) for k in os.listdir(robotsjobdir)]:
if os.path.isfile(i):
ryw.cleanup_path(i,"delete outgoing image:")
ryw.give_good_news("Successfully deleted all outgoing images",logging.debug)
ryw_view.print_footer()
def remove_from_object_store(parent, datapath, metapath, auxpath, donepath):
"""remove all traces from the object store."""
# the order is somewhat significant.
# done flag first, metadata next, data last.
# then give a shot at the parent directory and beyond.
# if any part fails, we give up.
if not ryw.cleanup_path(donepath, 'remove_from_object_store, donepath:'):
return
if not ryw.cleanup_path(auxpath, 'remove_from_object_store, auxpath:'):
return
if not ryw.cleanup_path(metapath, 'remove_from_object_store, metapath:'):
return
if not ryw.cleanup_path(datapath, 'remove_from_object_store, datapath:'):
return
ryw.cleanup_partial_dir(parent, 'remove_from_object_store, parent:', True)
def cleanup_old_search_results(tmpDir, selName):
isDir = ryw.is_valid_dir(tmpDir, msg='cleanup_old_search_results')
if not isDir:
logging.debug('cleanup_old_search_results: found no result dir.')
try:
files = os.listdir(tmpDir)
latestFileName = os.path.join(tmpDir, selName)
latestFileCtime = os.path.getctime(latestFileName)
for oneFile in files:
if oneFile == selName:
continue
oneFileName = os.path.join(tmpDir, oneFile)
oneFileCtime = os.path.getctime(oneFileName)
if latestFileCtime - oneFileCtime > DELETE_OLD_SEARCH_RESULT_SECONDS:
ryw.cleanup_path(oneFileName, 'cleanup_old_search_results')
except:
ryw.give_bad_news('cleanup_old_search_results failed: ' +
tmpDir + ' -- ' + selName, logging.error)
def remove_tmpdirs_keep_n(parent, n):
"""remove a bunch of subdirectories or files inside the parent directory,
until only n of them remain"""
if n < 0:
raise NameError('remove_tmpdirs_keep_n: bad n: ' + repr(n))
if not os.path.exists(parent):
ryw.give_bad_news('remove_tmpdirs_keep_n: does not exist: '+parent,
logging.critical)
return False
if not os.path.isdir(parent):
ryw.give_bad_news('remove_tmpdirs_keep_n: not a directory: '+parent,
logging.critical)
return False
try:
os.chdir(parent)
dirContent = os.listdir('.')
except:
ryw.give_bad_news('remove_tmpdirs_keep_n: failed to list dir: '+parent,
logging.critical)
return False
length = len(dirContent)
if length == 0:
logging.debug('remove_tmpdirs_keep_n: empty dir: ' + parent)
return True
if length <= n:
logging.debug('remove_tmpdirs_keep_n: < n: ' + parent + ' ' + repr(n))
return True
try:
dirContent.sort()
except:
ryw.give_bad_news('remove_tmpdirs_keep_n: failed to sort dir: ' +
parent, logging.critical)
return False
while length > n:
try:
first = dirContent.pop(0)
except:
ryw.give_bad_news('remove_tmpdirs_keep_n: failed to pop dir: '+
repr(dirContent))
return False
if not ryw.cleanup_path(first, 'remove_tmpdirs_keep_n:'):
return False
length -= 1
return True
def remove_paths(paths):
datapath = paths[0]
metapath = paths[1]
auxipath = paths[2]
donepath = paths[3]
mdonpath = paths[4]
parent = paths[5]
viewpath = paths[6]
if viewpath:
ryw.cleanup_path(viewpath, 'DeleteObject.remove_paths view:')
dir = os.path.dirname(viewpath)
ryw.cleanup_partial_dir(dir, 'DeleteObject.remove_paths view:', False,
chmodFirst = True)
logging.debug('DeleteObject.remove_paths: killed view path: ' +
viewpath)
storePaths = (datapath,metapath,auxipath,donepath,mdonpath)
ryw.remove_all_repo_dirs(storePaths)
ryw.cleanup_partial_dir(parent, 'DeleteObject.remove_paths view::', False)
logging.debug('DeleteObject.remove_paths: killed store paths: ' +
repr(storePaths))
def copy_tree_diff_file(src, dst):
"""only copies files whose sizes have changed."""
assert(os.path.exists(src))
assert(os.path.isfile(src))
#logging.debug('copy_tree_diff_file: ' + src + ' -> ' + dst)
try:
if not os.path.exists(dst):
#logging.debug('copy_tree_diff_file: copying: ' +
# src + ' -> ' + dst)
shutil.copyfile(src, dst)
return True
if os.path.isdir(dst):
ryw.cleanup_path(dst, 'copy_tree_diff_file:')
shutil.copyfile(src, dst)
return True
assert(os.path.exists(dst))
assert(os.path.isfile(dst))
srcSize = os.path.getsize(src)
dstSize = os.path.getsize(dst)
if srcSize == dstSize:
#logging.debug('copy_tree_diff_file: skipping copying: ' +
# src + ' -> ' + dst)
return True
#logging.debug('copy_tree_diff_file: different sizes, copying: ' +
# src + ' -> ' + dst)
shutil.copyfile(src,dst)
return True
except:
ryw.give_bad_news('copy_tree_diff_file: failed to copy file: ' +
src + ' -> ' + dst, logging.critical)
return False
def cleanup_and_exit(tmpdir, metafile, extractDir, exitCode,
successMessage = 'upload completed.'):
"""wipe out temporary files and directories, and exit."""
ryw.cleanup_path(tmpdir, 'cleanup_and_exit, tmpdir:')
ryw.cleanup_path(metafile, 'cleanup_and_exit, metafile:')
ryw.cleanup_path(extractDir, 'cleanup_and_exit, extractDir:')
if tmpdir:
ryw.cleanup_path(tmpdir + '_AUXI', 'cleanup_and_exit, aux dir:')
if (exitCode == 0):
ryw.give_good_news(successMessage, logging.info)
#print '<BR><BR><A HREF="/index.html">back to home</A><br>'
ryw_view.print_footer()
sys.exit(exitCode)
def mainFunc(name, tmpdir="", onlyMeta=False, discLimit=None):
ryw.check_logging(os.path.join(RepositoryRoot, 'WWW', 'logs'),
'upload.log')
logging.debug('WriteCD.py: attempted... tmpdir: ' + tmpdir)
if discLimit:
ryw.set_disc_limit(discLimit)
#ryw.give_news2('WriteCD.py: disc size set to: ' +
# str(ryw.maxSizeInKB), logging.info)
success,username,rfpath,reqs = get_pathsFunc(name)
if not success:
ryw.give_bad_news('WriteCD failed.', logging.error)
return
success,objKB,tmpStoreDir,tmpStoreName = \
obj_store_size_inKB(tmpdir=tmpdir)
if not success:
ryw.give_bad_news('WriteCD failed.', logging.critical)
ryw.cleanup_path(tmpStoreDir, 'WriteCD.mainFunc:')
return
success,reqsize,reqpath,reqList = collect_req_info(reqs, objKB)
if not success:
ryw.give_bad_news('WriteCD failed.', logging.error)
ryw.cleanup_path(tmpStoreDir, 'WriteCD.mainFunc:')
return
success,countCDs = make_discs(reqsize, reqpath, username,
objKB, tmpStoreName, reqList,
tmpDirOption=tmpdir,
metaOnly = onlyMeta)
if not success:
ryw.give_bad_news('WriteCD failed.', logging.error)
ryw.cleanup_path(tmpStoreDir, 'WriteCD.mainFunc:')
return
# empty_download_queue(rfpath)
ryw.give_good_news('Apparent success: number of CDs made = '
+ repr(countCDs), logging.info)
# ryw.cleanup_path(tmpStoreDir, 'WriteCD.mainFunc:')
confirm_delete_queue(rfpath)
def mainFunc_old(name):
ryw.check_logging(os.path.join(RepositoryRoot, 'WWW', 'logs'),
'upload.log')
logging.debug('WriteCD.py: attempted...')
success,username,rfpath,reqs = get_pathsFunc(name)
if not success:
ryw.give_bad_news('WriteCD failed.', logging.error)
return
success,objKB,tmpStoreDir,tmpStoreName = obj_store_size_inKB()
if not success:
ryw.give_bad_news('WriteCD failed.', logging.critical)
ryw.cleanup_path(tmpStoreDir, 'WriteCD.mainFunc:')
return
success,reqsize,reqpath,reqList = collect_req_info(reqs, objKB)
if not success:
ryw.give_bad_news('WriteCD failed.', logging.error)
ryw.cleanup_path(tmpStoreDir, 'WriteCD.mainFunc:')
return
success,countCDs,finishedStuff = make_discs(reqsize, reqpath, username,
objKB, tmpStoreName, reqList)
if not success:
ryw.give_bad_news('WriteCD failed.', logging.error)
ryw.cleanup_path(tmpStoreDir, 'WriteCD.mainFunc:')
return
# AddRobotWriteRequest.wait_for_robot(finishedStuff)
# empty_download_queue(rfpath)
ryw.give_good_news('Apparent success: number of CDs made = '
+ repr(countCDs), logging.info)
# ryw.cleanup_path(tmpStoreDir, 'WriteCD.mainFunc:')
confirm_delete_queue(rfpath)
giveQManagementLink()
def restore_old_meta(paths, metaPath):
#
# first wipe out the bad current meta file.
#
ryw.cleanup_path(metaPath, 'restore_old_meta:')
attempt_repair_repo_paths(paths)
def rewrite_meta(objroot, objID, version, meta, paths=None):
"""when called by reformat_ntfs.py, we supply the paths directly.
otherwise, paths is none."""
if paths == None:
logging.debug('rewrite_meta entered: ' + objID + '#' + str(version))
paths = get_paths1(objroot, objID, version)
paths = paths[:-1]
#
# (0) sanity checking.
#
if not ryw.good_repo_paths(paths):
ryw.give_bad_news('rewrite_meta: failed ' +
'initial path sanity check: ' +
repr(paths), logging.error)
return False
dataPath,metaPath,auxiPath,donePath,mdonPath = paths
#
# (1) backup old meta file.
#
if not ryw.move_to_bak(metaPath, copyInsteadOfMove=True):
ryw.give_bad_news('rewrite_meta: failed to backup meta file: ' +
metaPath, logging.error)
return False
logging.debug('rewrite_meta: successfully backed up old meta file: ' +
metaPath)
#
# (2) backup done flag.
#
if os.path.exists(donePath):
doneFlagToMove = donePath
elif os.path.exists(mdonPath):
doneFlagToMove = mdonPath
else:
ryw.give_bad_news('rewrite_meta: unexpected missing done flag: ' +
donePath + ' or ' + mdonPath, logging.critical)
return False
if not ryw.move_to_bak(doneFlagToMove):
ryw.give_bad_news('rewrite_meta: failed to move done flag: ' +
doneFlagToMove, logging.error)
return False
logging.debug('rewrite_meta: successfully moved done flag to bak: ' +
doneFlagToMove)
#
# (3) write new meta file.
#
if not write_meta(metaPath, meta, paths):
ryw.give_bad_news('rewrite_meta: write_meta failed: ' + metaPath,
logging.error)
return False
logging.debug('rewrite_meta: successfully written meta data: ' + metaPath)
retVal = True
#
# (4) restore the done flag.
#
doneFlagBak = doneFlagToMove + '_BAK'
try:
shutil.move(doneFlagBak, doneFlagToMove)
except:
ryw.give_bad_news('rewrite_meta: failed to restore done flag: ' +
doneFlagToMove, logging.critical)
#
# last ditch effort to rewrite a new done flag.
#
if not write_new_done_flag(doneFlagToMove):
ryw.give_bad_news('rewrite_meta: failed to write new flag: ' +
doneFlagToMove, logging.critical)
retVal = False
pass # in any case we proceed to next step.
pass # in any case we proceed to next step.
#
# (5) wipe out _BAK meta.
#
ryw.cleanup_path(metaPath + '_BAK', 'rewrite_meta:')
return retVal
def __cleanup(self):
ryw.cleanup_path(self.searchfile + '_TMP', '__cleanup, tmppath:')
ryw.cleanup_path(self.searchfile + '_BAK', '__cleanup, bakpath:')
def cleanup(tmppath, bakpath):
ryw.cleanup_path(tmppath, 'cleanup, tmppath:')
ryw.cleanup_path(bakpath, 'cleanup, bakpath:')
def cleanup_incoming(tmpdir, jobFile):
#ryw.cleanup_path(tmpdir, 'ReadIncomingCDStack.cleanup:')
ryw.force_remove(tmpdir, 'ReadIncomingCDStack.cleanup:')
ryw.cleanup_path(jobFile+'.JRQ', 'ReadIncomingCDStack.cleanup:')
ryw.cleanup_path(jobFile+'.ERR', 'ReadIncomingCDStack.cleanup:')
ryw.cleanup_path(jobFile+'.DON', 'ReadIncomingCDStack.cleanup:')
def remove_view_path(path):
"""remove view path."""
ryw.cleanup_path(path, 'remove_view_path:')
dir = os.path.dirname(path)
ryw.cleanup_partial_dir(dir, 'remove_view_path, removedirs:', False)
def cleanup_image(tmpImgDir):
ryw.cleanup_path(tmpImgDir, 'cleanup_image:')
