def collect_req_info(reqs, objKB):
logging.debug('collect_req_info: entered...')
success,searchFile = ryw.open_search_file(
'collect_req_info:',
os.path.join(RepositoryRoot, 'WWW', 'logs'),
'upload.log',
os.path.join(RepositoryRoot, 'SearchFile'),
False)
if not success:
return(False, None, None, None)
reqsize = {}
reqpath = {}
reqList = []
for item in reqs:
logging.debug('collect_req_info: item is: '+item)
try:
objname, version = item.split('#')
version = int(version)
except:
ryw.give_bad_news(
'collect_req_info: bad format, split failed: '+item,
logging.error)
continue
logging.debug('collect_req_info, obj, version: ' +
objname + ' ' + repr(version))
success,metaData = searchFile.get_meta(objname, version)
if not success:
ryw.give_bad_news(
'collect_req_info: failed to get_meta.',
logging.error)
continue
#
# I'm doing this to hardwire all
# places of gettting objectstoreroot.
#
#objroot = metaData['objectstore']
objroot = ryw.hard_wired_objectstore_root()
try:
itempath = objectstore.name_version_to_paths_aux(objroot, objname,
version)
except:
ryw.give_bad_news(
'collect_req_info: nameversiontopaths failed: ' +
objroot + ' ' + objname + ' ' + repr(version),
logging.critical)
continue
logging.debug('collect_req_info: after getting itempath...' +
repr(itempath))
if not ryw.good_repo_paths(itempath):
ryw.give_bad_news('collect_req_info: check_obj_paths failed.',
logging.error)
continue
success,itemSize = ryw.get_obj_size(itempath)
if not success:
continue
logging.debug('collect_req_info, size in KB is: ' + repr(itemSize))
if (itemSize > ryw.maxSizeInKB - objKB):
ryw.give_bad_news(
'collect_req_info: item size too big to fit on one disc: ' +
repr(itemSize), logging.error)
continue
reqsize[item] = itemSize
reqpath[item] = itempath
logging.debug('collect_req_info: size, path: ' +
repr(itemSize) + ' ' + itempath[0])
# build a list for sorting.
reqItem = {}
reqItem['name'] = item
if metaData.has_key('upload_datetime'):
reqItem['upload_datetime'] = metaData['upload_datetime']
reqList.append(reqItem)
searchFile.done()
reqList.sort(key = ryw.datetimesortkey, reverse = False)
sortedItems = []
for r in reqList:
sortedItems.append(r['name'])
return (True, reqsize, reqpath, sortedItems)
def rewrite_meta(objroot, objID, version, meta, paths=None):
"""when called by reformat_ntfs.py, we supply the paths directly.
otherwise, paths is none."""
if paths == None:
logging.debug('rewrite_meta entered: ' + objID + '#' + str(version))
paths = get_paths1(objroot, objID, version)
paths = paths[:-1]
#
# (0) sanity checking.
#
if not ryw.good_repo_paths(paths):
ryw.give_bad_news('rewrite_meta: failed ' +
'initial path sanity check: ' +
repr(paths), logging.error)
return False
dataPath,metaPath,auxiPath,donePath,mdonPath = paths
#
# (1) backup old meta file.
#
if not ryw.move_to_bak(metaPath, copyInsteadOfMove=True):
ryw.give_bad_news('rewrite_meta: failed to backup meta file: ' +
metaPath, logging.error)
return False
logging.debug('rewrite_meta: successfully backed up old meta file: ' +
metaPath)
#
# (2) backup done flag.
#
if os.path.exists(donePath):
doneFlagToMove = donePath
elif os.path.exists(mdonPath):
doneFlagToMove = mdonPath
else:
ryw.give_bad_news('rewrite_meta: unexpected missing done flag: ' +
donePath + ' or ' + mdonPath, logging.critical)
return False
if not ryw.move_to_bak(doneFlagToMove):
ryw.give_bad_news('rewrite_meta: failed to move done flag: ' +
doneFlagToMove, logging.error)
return False
logging.debug('rewrite_meta: successfully moved done flag to bak: ' +
doneFlagToMove)
#
# (3) write new meta file.
#
if not write_meta(metaPath, meta, paths):
ryw.give_bad_news('rewrite_meta: write_meta failed: ' + metaPath,
logging.error)
return False
logging.debug('rewrite_meta: successfully written meta data: ' + metaPath)
retVal = True
#
# (4) restore the done flag.
#
doneFlagBak = doneFlagToMove + '_BAK'
try:
shutil.move(doneFlagBak, doneFlagToMove)
except:
ryw.give_bad_news('rewrite_meta: failed to restore done flag: ' +
doneFlagToMove, logging.critical)
#
# last ditch effort to rewrite a new done flag.
#
if not write_new_done_flag(doneFlagToMove):
ryw.give_bad_news('rewrite_meta: failed to write new flag: ' +
doneFlagToMove, logging.critical)
retVal = False
pass # in any case we proceed to next step.
pass # in any case we proceed to next step.
#
# (5) wipe out _BAK meta.
#
ryw.cleanup_path(metaPath + '_BAK', 'rewrite_meta:')
return retVal
def copy_objects(items, itempath, repDir, tmpImgDir, metaOnly = False):
ryw.give_news2('copying requested objects... ', logging.info)
#
# quite a bit of the copying details should be pulled out to a piece
# of common code.
#
if len(items) == 0:
ryw.give_news2('no requested objects. ', logging.info)
ryw.give_news2('<BR>', logging.info)
return True
success,dataRoot,auxiRoot,mapDict,counter = ryw_philips.out_init(tmpImgDir)
if not success:
ryw.give_bad_news('copy_objects: out_init failed.', logging.error)
return True
for item in items:
logging.debug('copy_objects: item: ' + item)
try:
objname,version = item.split('#')
version = int(version)
logging.debug('copy_objects: got name, version: ' +
objname + ' ' + repr(version))
#destpath = objectstore.nameversiontopaths(
# os.path.join(repDir, 'objects'), objname, version)
destpath = objectstore.name_version_to_paths_aux(
os.path.join(repDir, 'objects'), objname, version)
logging.debug('copy_objects: got destpath: ' + repr(destpath))
if not ryw.good_repo_paths(itempath[item]):
ryw.give_bad_news('copy_objects: good_repo_paths failed.',
logging.error)
raise('something missing in the source paths.')
su.createparentdirpath(destpath[0])
logging.debug('copy_objects: created parent dir: ' + destpath[0])
# code prior to preparing for Philips DVD player
# su.copytree(itempath[item][0], destpath[0])
# logging.debug('copy_objects: done copying data: ' + destpath[0])
counter,mapDict = \
ryw_philips.out_copy(item,
itempath[item][0], itempath[item][1],
itempath[item][2], counter,
dataRoot, auxiRoot, mapDict,
onlyMeta = metaOnly)
shutil.copyfile(itempath[item][1], destpath[1])
logging.debug('copy_objects: done copying metadata: ' +
destpath[1])
# code prior to preparing for Philips DVD player
# if os.path.exists(itempath[item][2]):
# su.copytree(itempath[item][2], destpath[2])
# logging.debug('copy_objects: done copying auxi files: ' +
# destpath[2])
shutil.copyfile(itempath[item][3], destpath[3])
logging.debug('copy_objects: done copying done flag: ' +
destpath[3])
except:
ryw.give_bad_news(
'copy_objects: failed to copy data: ' + item, logging.critical)
ryw.give_bad_news(
'copy_objects: skip copying an object and continue.',
logging.error)
continue
ryw_philips.out_done(tmpImgDir, mapDict)
ryw.give_news2('done. ', logging.info)
ryw.give_news2('<BR>', logging.info)
return True
