def get_paths1(objroot, objID, version):
assert(objroot)
prefix = objectstore.nameversiontoprefix(objroot, objID, version)
pair = os.path.split(os.path.normpath(prefix))
parent = pair[0]
paths = objectstore.name_version_to_paths_aux(objroot, objID, version)
paths = list(paths)
paths.append(parent)
return paths
def resize_thumbnails(meta, forced = True):
try:
ostore = ryw.get_objectstore(meta)
if not ostore:
return
paths = objectstore.name_version_to_paths_aux(
ostore, meta['id'], meta['version'])
auxiDir = paths[2]
ryw_upload.resize_thumbnails(auxiDir, forced)
except:
ryw.give_bad_news('ryw_meta.resize_thumbnails: failed, meta: ' +
repr(meta), logging.error)
def object_version_is_present(objroot, objid, version):
paths = objectstore.name_version_to_paths_aux(objroot, objid, version)
if os.path.exists(paths[3]):
#ryw.give_news('object_version_is_present: exists: ' +
# objid + '#' +
# str(version), logging.info)
ryw.give_news3('object already exists in database: ' +
objid + '#' + str(version), logging.info)
#
# place to test flushing problem
#
#time.sleep(10)
#
return True
else:
return False
def deal_with_stub(localObjRoot, objID, version,
metaData, dataDir, auxDir, searchFile = None):
"""thoughts regarding how many (unnecessary) times I'm going through
the SearchFile... if there's data, the time spent on metadata is
probably not a big deal... so only this one, when we're only dealing
with metadata, somewhat matters. I think I'm still going to write
through, to make things simpler. the only optimization to take care
of is to not to re-open the SearchFile over and over again."""
success,searchFile = EditObject.do_update_metadata(
localObjRoot, objID, version, metaData, searchFile = searchFile)
if not success:
ryw.give_bad_news('ProcessDiscs: EditObject failed: ' +
objID + '#' + str(version), logging.error)
return (False, None)
if not os.path.exists(auxDir):
return (True, searchFile)
oldPaths = objectstore.name_version_to_paths_aux(
localObjRoot, objID, version)
oldAuxDir = oldPaths[2]
try:
#ryw.give_news3('auxDir: ' + auxDir, logging.info)
#ryw.give_news3('oldAuxDir: ' + oldAuxDir, logging.info)
#ryw.give_news3('force remove: ' + oldAuxDir, logging.info)
ryw.force_remove(oldAuxDir, 'deal_with_found_objects')
#ryw.give_news3('recursive copy: ' + auxDir + ' -> ' + oldAuxDir,
# logging.info)
shutil.copytree(auxDir, oldAuxDir)
except:
ryw.give_bad_news('ProcessDiscs: failed to overwrite auxi dirs: ' +
auxDir + ' -> ' + oldAuxDir, logging.critical)
return (False, searchFile)
return (True, searchFile)
def get_DVD_URLs(item, itempaths, mapDict):
try:
objID,version = item.split('#')
version = int(version)
dvdObjRoot = '..\\objects'
paths = objectstore.name_version_to_paths_aux(
dvdObjRoot, objID, version)
datapath = os.path.normpath(paths[0])
datapath = datapath.replace('\\', '/')
auxipath = os.path.normpath(paths[2])
auxipath = auxipath.replace('\\', '/')
datapath = urllib.quote(datapath)
auxipath = urllib.quote(auxipath)
#auxidir = os.path.normpath(paths[2])
auxidir = os.path.normpath(itempaths[2])
logging.debug('get_DVD_URLs: ' + datapath + ' ' + auxipath + ' ' +
auxidir)
return (True, datapath, auxipath, auxidir)
except:
ryw.give_bad_news('get_DVD_URLs: failed for: ' + item,
logging.critical)
return (False, None, None, None)
def get_meta(objroot, objname, version):
"""uses the file system to get meta data instead of getting it from
either the SearchFile or SearchServer. should be a bit faster if
there is no need to read all the metadata. just a performance issue:
should not be a robustness issue now that we have gotten rid of the
SearchServers."""
logging.debug("ryw.get_meta: " + objroot + " " + objname + " " + str(version))
try:
# paths = objectstore.nameversiontopaths(objroot, objname, version)
paths = objectstore.name_version_to_paths_aux(objroot, objname, version)
if not good_repo_paths(paths):
logging.warning("ryw.get_meta: good_repo_paths failed.")
return (False, None)
metapath = paths[1]
meta = su.pickload(metapath)
logging.debug("ryw.get_meta: success.")
return (True, meta)
except:
logging.warning("ryw.get_meta: failed.")
return (False, None)
def process_disk_from_peer_repository(dir_name, diskRoot, overwrite=False):
objectroot = os.path.join(dir_name, 'objects')
if not os.path.exists(objectroot):
logging.info(
'process_disk_from_peer_repository: no objects directory.' +
objectroot)
return True
## Process all incoming objects
local_object_root = get_local_object_store_root()
if local_object_root is None:
return False
mapDict = ryw_philips.get_map(diskRoot)
searchFile = None
for objectId,version in objectstore.objectversioniterator(objectroot):
ryw.give_news3('----------', logging.info)
if mapDict == None:
ryw.give_bad_news(
'process_disk_from_peer_repository: failed to read map file: '+
diskRoot, logging.error)
return False
#
# We used to just skip objects already present.
# now we want to do something.
#
#if object_version_is_present(local_object_root, objectId, version):
# ## object already present
# continue
objectFound = object_version_is_present(local_object_root,
objectId,
version)
paths = objectstore.name_version_to_paths_aux(objectroot,
objectId, version)
itemName = objectId + '#' + str(version)
obdata = get_data_name_mirror(paths[0], diskRoot, mapDict, itemName)
if not obdata:
continue
success,isStub = is_data_stub(obdata)
if not success:
continue
metadata = get_metadata_mirror(paths[1])
if not metadata:
continue
auxdir = get_aux_name_mirror(paths[2], diskRoot, mapDict, itemName)
if isStub and not objectFound:
#ryw.give_bad_news(
# 'ProcessDiscs: is a stub but not found in database: '+
# itemName, logging.error)
ryw.give_news3(
'ProcessDiscs error: is a stub but not found in database: '+
itemName, logging.error)
continue
if isStub:
success,searchFile = deal_with_stub(
local_object_root, objectId, version, metadata,
obdata, auxdir, searchFile = searchFile)
if success:
ryw.give_news3(
'ProcessDiscs success: meta data processed.',
logging.info)
continue
if objectFound:
#ryw.give_bad_news(
# 'ProcessDiscs: not a stub but found in the database: '+
# itemName, logging.error)
ryw.give_news3(
'ProcessDiscs: not a stub but found in the database: '+
itemName, logging.error)
if not overwrite:
ryw.give_news3('processing skipped.', logging.error)
continue
#
# I might want to delete the old version here.
# using the code from DelObject, should be simple.
#
ryw.give_news3('deleting it...', logging.error)
#
# at one point, I thought I wanted to be clever and
# tried to reuse searchFile below. But the trouble is
# that the UploadObject.uploadobject() call below will
# change the SearchFile beneath me, and if I reuse
# the searchFile here, I end up flushing the incorrectly
# cached version back to disk. I actually would have
# expected a deadlock when UploadObject.uploadobject()
# tries to lock again but the deadlock somehow
# didn't happen...
#
success,searchFile = DeleteObject.do_delete(
objectId, version, searchFile=None)
if not success:
ryw.give_bad_news(
'ProcessDiscs: DeleteObject failed: ' +
objectId + '#' + str(version), logging.error)
continue
else:
ryw.db_print('process_disk_from_peer_repository: ' +
'do_delete succeeded.', 18)
#
# falls through to continue onto adding the object.
#
if not UploadObject.uploadobject(metadata, obdata, auxdir,
hasVersion = True):
ryw.give_bad_news(
'process_disk_from_peer_repository: ' +
'UploadObject.uploadobject failed: '+
repr(metadata) + ' ' + obdata, logging.critical)
continue
else:
ryw.give_news3('ProcessDiscs success: new data uploaded.',
logging.info)
continue
incomingReverseLists = os.path.join(dir_name, 'ReverseLists')
existingReverseLists = os.path.join(RepositoryRoot, 'ReverseLists')
ReverseLists.merge_incoming(
existingReverseLists, incomingReverseLists, RepositoryRoot,
searchFile = searchFile)
if searchFile:
searchFile.done()
return True
def collect_req_info(reqs, objKB):
logging.debug('collect_req_info: entered...')
success,searchFile = ryw.open_search_file(
'collect_req_info:',
os.path.join(RepositoryRoot, 'WWW', 'logs'),
'upload.log',
os.path.join(RepositoryRoot, 'SearchFile'),
False)
if not success:
return(False, None, None, None)
reqsize = {}
reqpath = {}
reqList = []
for item in reqs:
logging.debug('collect_req_info: item is: '+item)
try:
objname, version = item.split('#')
version = int(version)
except:
ryw.give_bad_news(
'collect_req_info: bad format, split failed: '+item,
logging.error)
continue
logging.debug('collect_req_info, obj, version: ' +
objname + ' ' + repr(version))
success,metaData = searchFile.get_meta(objname, version)
if not success:
ryw.give_bad_news(
'collect_req_info: failed to get_meta.',
logging.error)
continue
#
# I'm doing this to hardwire all
# places of gettting objectstoreroot.
#
#objroot = metaData['objectstore']
objroot = ryw.hard_wired_objectstore_root()
try:
itempath = objectstore.name_version_to_paths_aux(objroot, objname,
version)
except:
ryw.give_bad_news(
'collect_req_info: nameversiontopaths failed: ' +
objroot + ' ' + objname + ' ' + repr(version),
logging.critical)
continue
logging.debug('collect_req_info: after getting itempath...' +
repr(itempath))
if not ryw.good_repo_paths(itempath):
ryw.give_bad_news('collect_req_info: check_obj_paths failed.',
logging.error)
continue
success,itemSize = ryw.get_obj_size(itempath)
if not success:
continue
logging.debug('collect_req_info, size in KB is: ' + repr(itemSize))
if (itemSize > ryw.maxSizeInKB - objKB):
ryw.give_bad_news(
'collect_req_info: item size too big to fit on one disc: ' +
repr(itemSize), logging.error)
continue
reqsize[item] = itemSize
reqpath[item] = itempath
logging.debug('collect_req_info: size, path: ' +
repr(itemSize) + ' ' + itempath[0])
# build a list for sorting.
reqItem = {}
reqItem['name'] = item
if metaData.has_key('upload_datetime'):
reqItem['upload_datetime'] = metaData['upload_datetime']
reqList.append(reqItem)
searchFile.done()
reqList.sort(key = ryw.datetimesortkey, reverse = False)
sortedItems = []
for r in reqList:
sortedItems.append(r['name'])
return (True, reqsize, reqpath, sortedItems)
def copy_objects(items, itempath, repDir, tmpImgDir, metaOnly = False):
ryw.give_news2('copying requested objects... ', logging.info)
#
# quite a bit of the copying details should be pulled out to a piece
# of common code.
#
if len(items) == 0:
ryw.give_news2('no requested objects. ', logging.info)
ryw.give_news2('<BR>', logging.info)
return True
success,dataRoot,auxiRoot,mapDict,counter = ryw_philips.out_init(tmpImgDir)
if not success:
ryw.give_bad_news('copy_objects: out_init failed.', logging.error)
return True
for item in items:
logging.debug('copy_objects: item: ' + item)
try:
objname,version = item.split('#')
version = int(version)
logging.debug('copy_objects: got name, version: ' +
objname + ' ' + repr(version))
#destpath = objectstore.nameversiontopaths(
# os.path.join(repDir, 'objects'), objname, version)
destpath = objectstore.name_version_to_paths_aux(
os.path.join(repDir, 'objects'), objname, version)
logging.debug('copy_objects: got destpath: ' + repr(destpath))
if not ryw.good_repo_paths(itempath[item]):
ryw.give_bad_news('copy_objects: good_repo_paths failed.',
logging.error)
raise('something missing in the source paths.')
su.createparentdirpath(destpath[0])
logging.debug('copy_objects: created parent dir: ' + destpath[0])
# code prior to preparing for Philips DVD player
# su.copytree(itempath[item][0], destpath[0])
# logging.debug('copy_objects: done copying data: ' + destpath[0])
counter,mapDict = \
ryw_philips.out_copy(item,
itempath[item][0], itempath[item][1],
itempath[item][2], counter,
dataRoot, auxiRoot, mapDict,
onlyMeta = metaOnly)
shutil.copyfile(itempath[item][1], destpath[1])
logging.debug('copy_objects: done copying metadata: ' +
destpath[1])
# code prior to preparing for Philips DVD player
# if os.path.exists(itempath[item][2]):
# su.copytree(itempath[item][2], destpath[2])
# logging.debug('copy_objects: done copying auxi files: ' +
# destpath[2])
shutil.copyfile(itempath[item][3], destpath[3])
logging.debug('copy_objects: done copying done flag: ' +
destpath[3])
except:
ryw.give_bad_news(
'copy_objects: failed to copy data: ' + item, logging.critical)
ryw.give_bad_news(
'copy_objects: skip copying an object and continue.',
logging.error)
continue
ryw_philips.out_done(tmpImgDir, mapDict)
ryw.give_news2('done. ', logging.info)
ryw.give_news2('<BR>', logging.info)
return True
