def collect_req_info(reqs, objKB): logging.debug('collect_req_info: entered...') success,searchFile = ryw.open_search_file( 'collect_req_info:', os.path.join(RepositoryRoot, 'WWW', 'logs'), 'upload.log', os.path.join(RepositoryRoot, 'SearchFile'), False) if not success: return(False, None, None, None) reqsize = {} reqpath = {} reqList = [] for item in reqs: logging.debug('collect_req_info: item is: '+item) try: objname, version = item.split('#') version = int(version) except: ryw.give_bad_news( 'collect_req_info: bad format, split failed: '+item, logging.error) continue logging.debug('collect_req_info, obj, version: ' + objname + ' ' + repr(version)) success,metaData = searchFile.get_meta(objname, version) if not success: ryw.give_bad_news( 'collect_req_info: failed to get_meta.', logging.error) continue # # I'm doing this to hardwire all # places of gettting objectstoreroot. # #objroot = metaData['objectstore'] objroot = ryw.hard_wired_objectstore_root() try: itempath = objectstore.name_version_to_paths_aux(objroot, objname, version) except: ryw.give_bad_news( 'collect_req_info: nameversiontopaths failed: ' + objroot + ' ' + objname + ' ' + repr(version), logging.critical) continue logging.debug('collect_req_info: after getting itempath...' + repr(itempath)) if not ryw.good_repo_paths(itempath): ryw.give_bad_news('collect_req_info: check_obj_paths failed.', logging.error) continue success,itemSize = ryw.get_obj_size(itempath) if not success: continue logging.debug('collect_req_info, size in KB is: ' + repr(itemSize)) if (itemSize > ryw.maxSizeInKB - objKB): ryw.give_bad_news( 'collect_req_info: item size too big to fit on one disc: ' + repr(itemSize), logging.error) continue reqsize[item] = itemSize reqpath[item] = itempath logging.debug('collect_req_info: size, path: ' + repr(itemSize) + ' ' + itempath[0]) # build a list for sorting. reqItem = {} reqItem['name'] = item if metaData.has_key('upload_datetime'): reqItem['upload_datetime'] = metaData['upload_datetime'] reqList.append(reqItem) searchFile.done() reqList.sort(key = ryw.datetimesortkey, reverse = False) sortedItems = [] for r in reqList: sortedItems.append(r['name']) return (True, reqsize, reqpath, sortedItems)
def rewrite_meta(objroot, objID, version, meta, paths=None): """when called by reformat_ntfs.py, we supply the paths directly. otherwise, paths is none.""" if paths == None: logging.debug('rewrite_meta entered: ' + objID + '#' + str(version)) paths = get_paths1(objroot, objID, version) paths = paths[:-1] # # (0) sanity checking. # if not ryw.good_repo_paths(paths): ryw.give_bad_news('rewrite_meta: failed ' + 'initial path sanity check: ' + repr(paths), logging.error) return False dataPath,metaPath,auxiPath,donePath,mdonPath = paths # # (1) backup old meta file. # if not ryw.move_to_bak(metaPath, copyInsteadOfMove=True): ryw.give_bad_news('rewrite_meta: failed to backup meta file: ' + metaPath, logging.error) return False logging.debug('rewrite_meta: successfully backed up old meta file: ' + metaPath) # # (2) backup done flag. # if os.path.exists(donePath): doneFlagToMove = donePath elif os.path.exists(mdonPath): doneFlagToMove = mdonPath else: ryw.give_bad_news('rewrite_meta: unexpected missing done flag: ' + donePath + ' or ' + mdonPath, logging.critical) return False if not ryw.move_to_bak(doneFlagToMove): ryw.give_bad_news('rewrite_meta: failed to move done flag: ' + doneFlagToMove, logging.error) return False logging.debug('rewrite_meta: successfully moved done flag to bak: ' + doneFlagToMove) # # (3) write new meta file. # if not write_meta(metaPath, meta, paths): ryw.give_bad_news('rewrite_meta: write_meta failed: ' + metaPath, logging.error) return False logging.debug('rewrite_meta: successfully written meta data: ' + metaPath) retVal = True # # (4) restore the done flag. # doneFlagBak = doneFlagToMove + '_BAK' try: shutil.move(doneFlagBak, doneFlagToMove) except: ryw.give_bad_news('rewrite_meta: failed to restore done flag: ' + doneFlagToMove, logging.critical) # # last ditch effort to rewrite a new done flag. # if not write_new_done_flag(doneFlagToMove): ryw.give_bad_news('rewrite_meta: failed to write new flag: ' + doneFlagToMove, logging.critical) retVal = False pass # in any case we proceed to next step. pass # in any case we proceed to next step. # # (5) wipe out _BAK meta. # ryw.cleanup_path(metaPath + '_BAK', 'rewrite_meta:') return retVal
def copy_objects(items, itempath, repDir, tmpImgDir, metaOnly = False): ryw.give_news2('copying requested objects... ', logging.info) # # quite a bit of the copying details should be pulled out to a piece # of common code. # if len(items) == 0: ryw.give_news2('no requested objects. ', logging.info) ryw.give_news2('<BR>', logging.info) return True success,dataRoot,auxiRoot,mapDict,counter = ryw_philips.out_init(tmpImgDir) if not success: ryw.give_bad_news('copy_objects: out_init failed.', logging.error) return True for item in items: logging.debug('copy_objects: item: ' + item) try: objname,version = item.split('#') version = int(version) logging.debug('copy_objects: got name, version: ' + objname + ' ' + repr(version)) #destpath = objectstore.nameversiontopaths( # os.path.join(repDir, 'objects'), objname, version) destpath = objectstore.name_version_to_paths_aux( os.path.join(repDir, 'objects'), objname, version) logging.debug('copy_objects: got destpath: ' + repr(destpath)) if not ryw.good_repo_paths(itempath[item]): ryw.give_bad_news('copy_objects: good_repo_paths failed.', logging.error) raise('something missing in the source paths.') su.createparentdirpath(destpath[0]) logging.debug('copy_objects: created parent dir: ' + destpath[0]) # code prior to preparing for Philips DVD player # su.copytree(itempath[item][0], destpath[0]) # logging.debug('copy_objects: done copying data: ' + destpath[0]) counter,mapDict = \ ryw_philips.out_copy(item, itempath[item][0], itempath[item][1], itempath[item][2], counter, dataRoot, auxiRoot, mapDict, onlyMeta = metaOnly) shutil.copyfile(itempath[item][1], destpath[1]) logging.debug('copy_objects: done copying metadata: ' + destpath[1]) # code prior to preparing for Philips DVD player # if os.path.exists(itempath[item][2]): # su.copytree(itempath[item][2], destpath[2]) # logging.debug('copy_objects: done copying auxi files: ' + # destpath[2]) shutil.copyfile(itempath[item][3], destpath[3]) logging.debug('copy_objects: done copying done flag: ' + destpath[3]) except: ryw.give_bad_news( 'copy_objects: failed to copy data: ' + item, logging.critical) ryw.give_bad_news( 'copy_objects: skip copying an object and continue.', logging.error) continue ryw_philips.out_done(tmpImgDir, mapDict) ryw.give_news2('done. ', logging.info) ryw.give_news2('<BR>', logging.info) return True