def try_upload_object(meta, nameToUpload, auxDir, cloneVersion=False): """attempt to upload an object into the repository.""" # intentionally not catching exception here if not UploadObject.uploadobject(meta, nameToUpload, auxDir, cloneVersion=cloneVersion): give_bad_news("fatal_error: failed to upload object into repository.", logging.critical) return False logging.info("UploadObject.uploadobject: done.") return True
def process_uploaded_objects(mydir): uploaddir = os.path.join(mydir, 'ObjectsToUpload') if not os.path.exists(uploaddir): logging.debug('process_uploaded_objects: no ObjectsToUploaded.') return True try: allNames = os.listdir(uploaddir) except: ryw.give_bad_news('process_uploaded_objects: failed to listdir: '+ uploaddir, logging.critical) return False for n in allNames: if not n.endswith('_DONE'): continue obpref = n[:-5] obdata = get_data_name(uploaddir, obpref) if not obdata: continue metadata = get_metadata(uploaddir, obpref) if not metadata: continue auxdir = get_aux_name(uploaddir, obpref) if not UploadObject.uploadobject(metadata, obdata, auxdir): ryw.give_bad_news( 'process_uploaded_objects: UploadObject.uploadobject failed: '+ repr(metadata) + ' ' + obdata, logging.critical) continue #command = '"%s" -u %s -p %s %s' % (sys.executable, os.path.join(RepositoryRoot, 'bin', 'UploadObject.py'), os.path.join(uploaddir, obpref + '_META'), obdata) #os.system(command) return True
def process_disk_from_peer_repository(dir_name, diskRoot, overwrite=False): objectroot = os.path.join(dir_name, 'objects') if not os.path.exists(objectroot): logging.info( 'process_disk_from_peer_repository: no objects directory.' + objectroot) return True ## Process all incoming objects local_object_root = get_local_object_store_root() if local_object_root is None: return False mapDict = ryw_philips.get_map(diskRoot) searchFile = None for objectId,version in objectstore.objectversioniterator(objectroot): ryw.give_news3('----------', logging.info) if mapDict == None: ryw.give_bad_news( 'process_disk_from_peer_repository: failed to read map file: '+ diskRoot, logging.error) return False # # We used to just skip objects already present. # now we want to do something. # #if object_version_is_present(local_object_root, objectId, version): # ## object already present # continue objectFound = object_version_is_present(local_object_root, objectId, version) paths = objectstore.name_version_to_paths_aux(objectroot, objectId, version) itemName = objectId + '#' + str(version) obdata = get_data_name_mirror(paths[0], diskRoot, mapDict, itemName) if not obdata: continue success,isStub = is_data_stub(obdata) if not success: continue metadata = get_metadata_mirror(paths[1]) if not metadata: continue auxdir = get_aux_name_mirror(paths[2], diskRoot, mapDict, itemName) if isStub and not objectFound: #ryw.give_bad_news( # 'ProcessDiscs: is a stub but not found in database: '+ # itemName, logging.error) ryw.give_news3( 'ProcessDiscs error: is a stub but not found in database: '+ itemName, logging.error) continue if isStub: success,searchFile = deal_with_stub( local_object_root, objectId, version, metadata, obdata, auxdir, searchFile = searchFile) if success: ryw.give_news3( 'ProcessDiscs success: meta data processed.', logging.info) continue if objectFound: #ryw.give_bad_news( # 'ProcessDiscs: not a stub but found in the database: '+ # itemName, logging.error) ryw.give_news3( 'ProcessDiscs: not a stub but found in the database: '+ itemName, logging.error) if not overwrite: ryw.give_news3('processing skipped.', logging.error) continue # # I might want to delete the old version here. # using the code from DelObject, should be simple. # ryw.give_news3('deleting it...', logging.error) # # at one point, I thought I wanted to be clever and # tried to reuse searchFile below. But the trouble is # that the UploadObject.uploadobject() call below will # change the SearchFile beneath me, and if I reuse # the searchFile here, I end up flushing the incorrectly # cached version back to disk. I actually would have # expected a deadlock when UploadObject.uploadobject() # tries to lock again but the deadlock somehow # didn't happen... # success,searchFile = DeleteObject.do_delete( objectId, version, searchFile=None) if not success: ryw.give_bad_news( 'ProcessDiscs: DeleteObject failed: ' + objectId + '#' + str(version), logging.error) continue else: ryw.db_print('process_disk_from_peer_repository: ' + 'do_delete succeeded.', 18) # # falls through to continue onto adding the object. # if not UploadObject.uploadobject(metadata, obdata, auxdir, hasVersion = True): ryw.give_bad_news( 'process_disk_from_peer_repository: ' + 'UploadObject.uploadobject failed: '+ repr(metadata) + ' ' + obdata, logging.critical) continue else: ryw.give_news3('ProcessDiscs success: new data uploaded.', logging.info) continue incomingReverseLists = os.path.join(dir_name, 'ReverseLists') existingReverseLists = os.path.join(RepositoryRoot, 'ReverseLists') ReverseLists.merge_incoming( existingReverseLists, incomingReverseLists, RepositoryRoot, searchFile = searchFile) if searchFile: searchFile.done() return True