def deleteRequested(form): image = form.getfirst("Img","") if not image: print "No Image specified to delete" sys.exit(1) success, resources = get_resources() if not success: ryw.give_bad_news("Error parsing resource file",logging.error) sys.exit(1) robotsJobDir = resources['robotsjobdir'] jobfile = os.path.join(robotsJobDir, image) ryw.cleanup_path(jobfile+".JRQ",'deleteOutgoingImage.deleteRequested:') ryw.cleanup_path(jobfile+".ERR",'deleteOutgoingImage.deleteRequested:') ryw.cleanup_path(jobfile+".DON",'deleteOutgoingImage.deleteRequested:') ryw.cleanup_path(jobfile,'deleteOutgoingImage.deleteRequested:') tmpout = resources['tmpout'] image = os.path.join(tmpout,image) if not os.path.exists(image): ryw.give_bad_news("specified image doesnt exist",logging.info) sys.exit(1) ryw.cleanup_path(image,"deleteOutgoingImage.deleteRequested:") sys.stdout.write("True") sys.exit(0)
def __open_tmp_search_file(self): try: tmpFileName = self.searchfile + '_TMP' if os.path.exists(tmpFileName): ryw.cleanup_path(tmpFileName, '__open_tmp_search_file: ') f = open(tmpFileName, 'ab') logging.debug('__open_tmp_search_file: succeeded: ' + tmpFileName) return f except: ryw.give_bad_news('__open_tmp_search_file: failed' + self.searchfile + '_TMP', logging.critical) return None
def __write_to_disk(self): tmppath = self.reverseListsFile + '.TMP' try: su.pickdump(self.reverseDict, tmppath) except: ryw.give_bad_news('ReverseLists.add: pickdump failed.', logging.critical) return False success,bakpath = ryw.make_tmp_file_permanent( tmppath, self.reverseListsFile) ryw.cleanup_path(tmppath, '_write_to_disk: cleanup, tmppath:') ryw.cleanup_path(bakpath, '_write_to_disk: cleanup, bakpath:') ryw.db_print('write_to_disk: written to disk...', 11) return success
def create_view_path(path, objectID, version): if os.path.exists(path): ryw.give_news('create_view_path: already exists: ' + path + ', ' + 'replacing with new version.', logging.warning) #return True #remove old view path and continue if not ryw.cleanup_path(path, 'create_view_path:'): ryw.give_news('create_view_path: fails to remove old view path.', logging.error) return True try: su.createparentdirpath(path) except: ryw.give_bad_news('create_view_path: failed to createparentdirpath: '+ path, logging.critical) return False try: f = open(path, 'w') f.write(objectID + '#' + str(version)) f.close() except: ryw.give_bad_news('create_view_path: failed to write leaf file: '+path, logging.critical) return False logging.debug('create_view_path: done creating path: '+path) return True
def write_file(self, fullFileName): """10/24/08, called by ChapterListFormHandle.py.""" if self.chapterList == None or self.chapterList == {}: # # then we remove the chapter list file. # ryw.db_print_info_browser('ChapterList.write_file: ' + \ 'chapterList is empty.', 41) if ryw.cleanup_path(fullFileName, 'ChapterList.write_file'): return True ryw.give_bad_news('ChapterList.write_file:' + 'failed to remove file', logging.error) return False try: su.pickdump(self.chapterList, fullFileName) except: ryw.give_bad_news('ChapterList.write_file: pickdump failed: ' + fullFileName, logging.error) return False ryw.db_print_info_browser('ChapterList.write_file success: ' + fullFileName, 41) return True
def copy_tree_diff_file_view(src, dst): """always copy leaves.""" assert(os.path.exists(src)) assert(os.path.isfile(src)) logging.debug('copy_tree_diff_file_view: ' + src + ' -> ' + dst) try: if os.path.exists(dst) and os.path.isdir(dst): ryw.cleanup_path(dst, 'copy_tree_diff_file:') shutil.copyfile(src, dst) return True except: ryw.give_bad_news('copy_tree_diff_file_view: failed to copy file: ' + src + ' -> ' + dst, logging.critical) return False
def resize_thumbnails(auxDir, forced = True): if not os.path.exists(auxDir): return True thumbsDir = os.path.join(auxDir, 'thumbnails') if not os.path.exists(thumbsDir): return True thumbNames = os.listdir(thumbsDir) if len(thumbNames) == 0: return True for thumbName in thumbNames: if thumbName == 'Thumbs.db': continue thumbPath = os.path.join(thumbsDir, thumbName) if not ryw_view.is_image(thumbPath): ryw.give_bad_news( 'resize_thumbnails: probably not an image file: ' + thumbPath, logging.error) return False scaledDir = os.path.join(auxDir, 'thumbs_scaled') if os.path.exists(scaledDir) and not forced: return True ryw.cleanup_path(scaledDir, 'ryw_upload.resize_thumbnails:') try: su.createdirpath(scaledDir) for thumbName in thumbNames: if thumbName == 'Thumbs.db': continue thumbPath = os.path.join(thumbsDir, thumbName) thumbDest = os.path.join(scaledDir, thumbName) if not ryw_view.resize_image(thumbPath, thumbDest): ryw.give_bad_news('resize_thumbnails: failed: ' + thumbPath + ' -> ' + thumbDest, logging.error) return False except: ryw.give_bad_news('failed to resize thumbnails.', logging.error) return True
def deleteAll(): print '<TITLE>Deleting Outing Disc Images</TITLE>' ryw_view.print_logo() success,resources = get_resources() if not success: logging.error("main: error parsing resource file") sys.exit(1) outdir = resources['tmpout'] for i in [os.path.join(outdir,k) for k in os.listdir(outdir)]: ryw.cleanup_path(i,"delete outgoing image:") robotsjobdir = resources['robotsjobdir'] for i in [os.path.join(robotsjobdir,k) for k in os.listdir(robotsjobdir)]: if os.path.isfile(i): ryw.cleanup_path(i,"delete outgoing image:") ryw.give_good_news("Successfully deleted all outgoing images",logging.debug) ryw_view.print_footer()
def remove_from_object_store(parent, datapath, metapath, auxpath, donepath): """remove all traces from the object store.""" # the order is somewhat significant. # done flag first, metadata next, data last. # then give a shot at the parent directory and beyond. # if any part fails, we give up. if not ryw.cleanup_path(donepath, 'remove_from_object_store, donepath:'): return if not ryw.cleanup_path(auxpath, 'remove_from_object_store, auxpath:'): return if not ryw.cleanup_path(metapath, 'remove_from_object_store, metapath:'): return if not ryw.cleanup_path(datapath, 'remove_from_object_store, datapath:'): return ryw.cleanup_partial_dir(parent, 'remove_from_object_store, parent:', True)
def cleanup_old_search_results(tmpDir, selName): isDir = ryw.is_valid_dir(tmpDir, msg='cleanup_old_search_results') if not isDir: logging.debug('cleanup_old_search_results: found no result dir.') try: files = os.listdir(tmpDir) latestFileName = os.path.join(tmpDir, selName) latestFileCtime = os.path.getctime(latestFileName) for oneFile in files: if oneFile == selName: continue oneFileName = os.path.join(tmpDir, oneFile) oneFileCtime = os.path.getctime(oneFileName) if latestFileCtime - oneFileCtime > DELETE_OLD_SEARCH_RESULT_SECONDS: ryw.cleanup_path(oneFileName, 'cleanup_old_search_results') except: ryw.give_bad_news('cleanup_old_search_results failed: ' + tmpDir + ' -- ' + selName, logging.error)
def remove_tmpdirs_keep_n(parent, n): """remove a bunch of subdirectories or files inside the parent directory, until only n of them remain""" if n < 0: raise NameError('remove_tmpdirs_keep_n: bad n: ' + repr(n)) if not os.path.exists(parent): ryw.give_bad_news('remove_tmpdirs_keep_n: does not exist: '+parent, logging.critical) return False if not os.path.isdir(parent): ryw.give_bad_news('remove_tmpdirs_keep_n: not a directory: '+parent, logging.critical) return False try: os.chdir(parent) dirContent = os.listdir('.') except: ryw.give_bad_news('remove_tmpdirs_keep_n: failed to list dir: '+parent, logging.critical) return False length = len(dirContent) if length == 0: logging.debug('remove_tmpdirs_keep_n: empty dir: ' + parent) return True if length <= n: logging.debug('remove_tmpdirs_keep_n: < n: ' + parent + ' ' + repr(n)) return True try: dirContent.sort() except: ryw.give_bad_news('remove_tmpdirs_keep_n: failed to sort dir: ' + parent, logging.critical) return False while length > n: try: first = dirContent.pop(0) except: ryw.give_bad_news('remove_tmpdirs_keep_n: failed to pop dir: '+ repr(dirContent)) return False if not ryw.cleanup_path(first, 'remove_tmpdirs_keep_n:'): return False length -= 1 return True
def remove_paths(paths): datapath = paths[0] metapath = paths[1] auxipath = paths[2] donepath = paths[3] mdonpath = paths[4] parent = paths[5] viewpath = paths[6] if viewpath: ryw.cleanup_path(viewpath, 'DeleteObject.remove_paths view:') dir = os.path.dirname(viewpath) ryw.cleanup_partial_dir(dir, 'DeleteObject.remove_paths view:', False, chmodFirst = True) logging.debug('DeleteObject.remove_paths: killed view path: ' + viewpath) storePaths = (datapath,metapath,auxipath,donepath,mdonpath) ryw.remove_all_repo_dirs(storePaths) ryw.cleanup_partial_dir(parent, 'DeleteObject.remove_paths view::', False) logging.debug('DeleteObject.remove_paths: killed store paths: ' + repr(storePaths))
def copy_tree_diff_file(src, dst): """only copies files whose sizes have changed.""" assert(os.path.exists(src)) assert(os.path.isfile(src)) #logging.debug('copy_tree_diff_file: ' + src + ' -> ' + dst) try: if not os.path.exists(dst): #logging.debug('copy_tree_diff_file: copying: ' + # src + ' -> ' + dst) shutil.copyfile(src, dst) return True if os.path.isdir(dst): ryw.cleanup_path(dst, 'copy_tree_diff_file:') shutil.copyfile(src, dst) return True assert(os.path.exists(dst)) assert(os.path.isfile(dst)) srcSize = os.path.getsize(src) dstSize = os.path.getsize(dst) if srcSize == dstSize: #logging.debug('copy_tree_diff_file: skipping copying: ' + # src + ' -> ' + dst) return True #logging.debug('copy_tree_diff_file: different sizes, copying: ' + # src + ' -> ' + dst) shutil.copyfile(src,dst) return True except: ryw.give_bad_news('copy_tree_diff_file: failed to copy file: ' + src + ' -> ' + dst, logging.critical) return False
def cleanup_and_exit(tmpdir, metafile, extractDir, exitCode, successMessage = 'upload completed.'): """wipe out temporary files and directories, and exit.""" ryw.cleanup_path(tmpdir, 'cleanup_and_exit, tmpdir:') ryw.cleanup_path(metafile, 'cleanup_and_exit, metafile:') ryw.cleanup_path(extractDir, 'cleanup_and_exit, extractDir:') if tmpdir: ryw.cleanup_path(tmpdir + '_AUXI', 'cleanup_and_exit, aux dir:') if (exitCode == 0): ryw.give_good_news(successMessage, logging.info) #print '<BR><BR><A HREF="/index.html">back to home</A><br>' ryw_view.print_footer() sys.exit(exitCode)
def mainFunc(name, tmpdir="", onlyMeta=False, discLimit=None): ryw.check_logging(os.path.join(RepositoryRoot, 'WWW', 'logs'), 'upload.log') logging.debug('WriteCD.py: attempted... tmpdir: ' + tmpdir) if discLimit: ryw.set_disc_limit(discLimit) #ryw.give_news2('WriteCD.py: disc size set to: ' + # str(ryw.maxSizeInKB), logging.info) success,username,rfpath,reqs = get_pathsFunc(name) if not success: ryw.give_bad_news('WriteCD failed.', logging.error) return success,objKB,tmpStoreDir,tmpStoreName = \ obj_store_size_inKB(tmpdir=tmpdir) if not success: ryw.give_bad_news('WriteCD failed.', logging.critical) ryw.cleanup_path(tmpStoreDir, 'WriteCD.mainFunc:') return success,reqsize,reqpath,reqList = collect_req_info(reqs, objKB) if not success: ryw.give_bad_news('WriteCD failed.', logging.error) ryw.cleanup_path(tmpStoreDir, 'WriteCD.mainFunc:') return success,countCDs = make_discs(reqsize, reqpath, username, objKB, tmpStoreName, reqList, tmpDirOption=tmpdir, metaOnly = onlyMeta) if not success: ryw.give_bad_news('WriteCD failed.', logging.error) ryw.cleanup_path(tmpStoreDir, 'WriteCD.mainFunc:') return # empty_download_queue(rfpath) ryw.give_good_news('Apparent success: number of CDs made = ' + repr(countCDs), logging.info) # ryw.cleanup_path(tmpStoreDir, 'WriteCD.mainFunc:') confirm_delete_queue(rfpath)
def mainFunc_old(name): ryw.check_logging(os.path.join(RepositoryRoot, 'WWW', 'logs'), 'upload.log') logging.debug('WriteCD.py: attempted...') success,username,rfpath,reqs = get_pathsFunc(name) if not success: ryw.give_bad_news('WriteCD failed.', logging.error) return success,objKB,tmpStoreDir,tmpStoreName = obj_store_size_inKB() if not success: ryw.give_bad_news('WriteCD failed.', logging.critical) ryw.cleanup_path(tmpStoreDir, 'WriteCD.mainFunc:') return success,reqsize,reqpath,reqList = collect_req_info(reqs, objKB) if not success: ryw.give_bad_news('WriteCD failed.', logging.error) ryw.cleanup_path(tmpStoreDir, 'WriteCD.mainFunc:') return success,countCDs,finishedStuff = make_discs(reqsize, reqpath, username, objKB, tmpStoreName, reqList) if not success: ryw.give_bad_news('WriteCD failed.', logging.error) ryw.cleanup_path(tmpStoreDir, 'WriteCD.mainFunc:') return # AddRobotWriteRequest.wait_for_robot(finishedStuff) # empty_download_queue(rfpath) ryw.give_good_news('Apparent success: number of CDs made = ' + repr(countCDs), logging.info) # ryw.cleanup_path(tmpStoreDir, 'WriteCD.mainFunc:') confirm_delete_queue(rfpath) giveQManagementLink()
def restore_old_meta(paths, metaPath): # # first wipe out the bad current meta file. # ryw.cleanup_path(metaPath, 'restore_old_meta:') attempt_repair_repo_paths(paths)
def rewrite_meta(objroot, objID, version, meta, paths=None): """when called by reformat_ntfs.py, we supply the paths directly. otherwise, paths is none.""" if paths == None: logging.debug('rewrite_meta entered: ' + objID + '#' + str(version)) paths = get_paths1(objroot, objID, version) paths = paths[:-1] # # (0) sanity checking. # if not ryw.good_repo_paths(paths): ryw.give_bad_news('rewrite_meta: failed ' + 'initial path sanity check: ' + repr(paths), logging.error) return False dataPath,metaPath,auxiPath,donePath,mdonPath = paths # # (1) backup old meta file. # if not ryw.move_to_bak(metaPath, copyInsteadOfMove=True): ryw.give_bad_news('rewrite_meta: failed to backup meta file: ' + metaPath, logging.error) return False logging.debug('rewrite_meta: successfully backed up old meta file: ' + metaPath) # # (2) backup done flag. # if os.path.exists(donePath): doneFlagToMove = donePath elif os.path.exists(mdonPath): doneFlagToMove = mdonPath else: ryw.give_bad_news('rewrite_meta: unexpected missing done flag: ' + donePath + ' or ' + mdonPath, logging.critical) return False if not ryw.move_to_bak(doneFlagToMove): ryw.give_bad_news('rewrite_meta: failed to move done flag: ' + doneFlagToMove, logging.error) return False logging.debug('rewrite_meta: successfully moved done flag to bak: ' + doneFlagToMove) # # (3) write new meta file. # if not write_meta(metaPath, meta, paths): ryw.give_bad_news('rewrite_meta: write_meta failed: ' + metaPath, logging.error) return False logging.debug('rewrite_meta: successfully written meta data: ' + metaPath) retVal = True # # (4) restore the done flag. # doneFlagBak = doneFlagToMove + '_BAK' try: shutil.move(doneFlagBak, doneFlagToMove) except: ryw.give_bad_news('rewrite_meta: failed to restore done flag: ' + doneFlagToMove, logging.critical) # # last ditch effort to rewrite a new done flag. # if not write_new_done_flag(doneFlagToMove): ryw.give_bad_news('rewrite_meta: failed to write new flag: ' + doneFlagToMove, logging.critical) retVal = False pass # in any case we proceed to next step. pass # in any case we proceed to next step. # # (5) wipe out _BAK meta. # ryw.cleanup_path(metaPath + '_BAK', 'rewrite_meta:') return retVal
def __cleanup(self): ryw.cleanup_path(self.searchfile + '_TMP', '__cleanup, tmppath:') ryw.cleanup_path(self.searchfile + '_BAK', '__cleanup, bakpath:')
def cleanup(tmppath, bakpath): ryw.cleanup_path(tmppath, 'cleanup, tmppath:') ryw.cleanup_path(bakpath, 'cleanup, bakpath:')
def cleanup_incoming(tmpdir, jobFile): #ryw.cleanup_path(tmpdir, 'ReadIncomingCDStack.cleanup:') ryw.force_remove(tmpdir, 'ReadIncomingCDStack.cleanup:') ryw.cleanup_path(jobFile+'.JRQ', 'ReadIncomingCDStack.cleanup:') ryw.cleanup_path(jobFile+'.ERR', 'ReadIncomingCDStack.cleanup:') ryw.cleanup_path(jobFile+'.DON', 'ReadIncomingCDStack.cleanup:')
def remove_view_path(path): """remove view path.""" ryw.cleanup_path(path, 'remove_view_path:') dir = os.path.dirname(path) ryw.cleanup_partial_dir(dir, 'remove_view_path, removedirs:', False)
def cleanup_image(tmpImgDir): ryw.cleanup_path(tmpImgDir, 'cleanup_image:')