Ejemplo n.º 1
0
def in_copy(objname, version, dstDataPath, dstAuxiPath, driveroot, mapDict):
    itemName = objname + '#' + version
    
    success,dirName,dataDir,auxiDir = get_map_entry(driveroot, mapDict,
                                                    itemName)
    if not success:
        return False

    ryw.give_news3('  copying ' + dirName + ' ... ', logging.info)

    try:
        su.copytree(dataDir, dstDataPath)
    except:
        ryw.give_bad_news('in_copy: failed to copy data directory: ' +
                          itemName + ': ' + dataDir, logging.error)
        return False

    logging.debug('in_copy: successfully copied data directory: ' +
                  itemName + ': ' + dataDir)
    
    if os.path.exists(auxiDir):
        try:
            su.copytree(auxiDir, dstAuxiPath)
        except:
            ryw.give_bad_news('in_copy: failed to copy auxi directory: ' +
                              itemName + ': ' + auxiDir, logging.error)
            return False
        logging.debug('in_copy: successfully copied auxi directory: ' +
                      itemName + ': ' + auxiDir)

    return True
Ejemplo n.º 2
0
def get_data_name_mirror(objDataDir, diskRoot, mapDict, itemName):
    if os.path.exists(objDataDir):
        updatadir = objDataDir
        logging.debug('get_data_name_mirror: found under objects: ' +
                      objDataDir)
    else:
        success,dirName,dataDir,auxiDir = ryw_philips.get_map_entry(
            diskRoot, mapDict, itemName)
        if not success:
            return None
        #ryw.give_news('get_data_name_mirror: found data: ' + dirName,
        #              logging.info)
        ryw.give_news3('found data: ' + dirName, logging.info)
        updatadir = dataDir

    try:    
        ll = os.listdir(updatadir)
    except:
        ryw.give_bad_news('get_data_name_mirror: failed to listdir: ' +
                          updatadir, logging.error)
        return None

    if len(ll) == 1 and os.path.isfile(os.path.join(updatadir, ll[0])):
        obdata = os.path.join(updatadir, ll[0])
    else:
        obdata = updatadir

    logging.debug('get_data_name_mirror: got data name: ' + obdata)
    return obdata
Ejemplo n.º 3
0
def merge_incoming(existingName, incomingName, repositoryRoot,
                   searchFile=None):
    """called by ProcessDiscs.py"""

    logging.info('merge_incoming: ' + existingName + ' <- ' + incomingName)
    
    if not ryw.is_valid_file(incomingName, 'copy_reverse_lists:'):
        ryw.give_news3('merge_incoming: incoming ReverseLists not found.',
                       logging.info)
        return True

    if not searchFile:
        success,searchFile = ryw.open_search_file(
            'merge_incoming:',
            os.path.join(RepositoryRoot, 'WWW', 'logs'),
            'upload.log',
            os.path.join(RepositoryRoot, 'SearchFile'),
            False)
        if not success:
            if searchFile:
                searchFile.done()
            ryw.give_bad_news('merge_incoming: open search file failed. ',
                              logging.critical)
            return False

    success,existingRL = open_reverse_lists('ReverseLists.merge_incoming:',
                                            '', '', existingName, True,
                                            searchFile = searchFile,
                                            repositoryRoot = repositoryRoot)
    if not success:
        ryw.give_bad_news('merge_incoming: failed to open existing list.',
                          logging.critical)
        if existingRL:
            existingRL.done()
        return False

    success,incomingRL = open_reverse_lists('ReverseLists.merge_incoming:',
                                            '', '', incomingName, False,
                                            skipLk = True,
                                            allowNullSearchFile = True)
    if not success:
        ryw.give_bad_news('merge_incoming: failed to open incoming list.',
                          logging.error)
        if incomingRL:
            incomingRL.done()
        return False
        
    success = existingRL.merge(incomingRL, repositoryRoot)

    existingRL.done()
    incomingRL.done()

    if searchFile:
        searchFile.done()
    return success
Ejemplo n.º 4
0
def change_meta(meta, newObjStoreRoot):
    if not meta.has_key('objectstore'):
        ryw.give_bad_news3(
            'meta has no object store: ' +
            meta['id'] + '#' + str(meta['version']))
        ryw.give_bad_news3('    but continuing...')
        oldObjStore = ''
    else:
        oldObjStore = meta['objectstore']
        ryw.db_print3(meta['id'] + '#' + str(meta['version']), 64)
        ryw.db_print3(oldObjStore, 64)

    meta['objectstore'] = newObjStoreRoot
    ryw.give_news3(meta['id'] + '#' + str(meta['version']) + ': ' +
                   oldObjStore + ' -> ' + newObjStoreRoot)
Ejemplo n.º 5
0
def is_data_stub(dataDir):
    if not os.path.exists(dataDir) or not os.path.isdir(dataDir):
        return (True, False)
    try:
        lDir = os.listdir(dataDir)
        if len(lDir) == 1 and lDir[0] == ryw_philips.stubDirName:
            #ryw.give_news('is_data_stub: found stub at: ' + dataDir,
            #              logging.info)
            ryw.give_news3('found stub at: ' + dataDir, logging.info)
            return (True, True)
        else:
            return (True, False)
    except:
        ryw.give_bad_news('is_data_stub failed: ' + dataDir, logging.error)
        return (False, False)
Ejemplo n.º 6
0
def object_version_is_present(objroot, objid, version):
    paths = objectstore.name_version_to_paths_aux(objroot, objid, version)
    if os.path.exists(paths[3]):
        #ryw.give_news('object_version_is_present: exists: '  +
        #              objid + '#' +
        #              str(version), logging.info)
        ryw.give_news3('object already exists in database: '  +
                       objid + '#' + str(version), logging.info)
        #
        # place to test flushing problem
        #
        #time.sleep(10)
        #
        return True
    else:
        return False
Ejemplo n.º 7
0
def main():

    #
    # get all the path names straight.
    #
    try:
        ntfsRoot = sys.argv[1]
    except:
        ryw.give_bad_news3(
            'usage: python reformat_ntfs.py /mnt/usb0/Postmanet ' +
            '[new_object_store_path]')
        sys.exit(-1)

    if len(sys.argv) >= 3:
        newObjStoreRoot = sys.argv[2]
    else:
        newObjStoreRoot = NEW_OBJECT_STORE_ROOT

    ryw.db_print3('newObjectStoreRoot is: ' + newObjStoreRoot, 62)
        
    if (not ntfsRoot) or (not os.path.exists(ntfsRoot)):
        ryw.give_bad_news3("can't find NTFS root: " + ntfsRoot)
        sys.exit(-1)

    ryw.db_print3('NTFS root is at: ' + ntfsRoot, 61)

    repositoryRoot = os.path.join(ntfsRoot, 'repository')
    oldSearchFileName = os.path.join(repositoryRoot, 'SearchFile')
    dateTimeRand = ryw.date_time_rand()
    newSearchFileName = os.path.join(repositoryRoot,
                                     'NewSearchFile' + dateTimeRand)

    objectStoreRoot = os.path.join(repositoryRoot, 'WWW',
                                   'ObjectStore')
    if (not os.path.exists(objectStoreRoot)):
        ryw.give_bad_news3("can't find object store: " + objectStoreRoot)
        sys.exit(-1)
    ryw.db_print3('object store root is at: ' + objectStoreRoot, 61)


    #
    # open the new search file.
    #
    try:
        newSearchFileHandle = open(newSearchFileName, 'ab')
    except:
        ryw.give_bad_news3('failed to open new search file: %s\n' % \
                           (searchfile,))
        sys.exit(-1)
    

    #
    # go through all the individual meta files.
    #
    l = glob.glob(objectStoreRoot + "/?/?/?/?/*_META")
    ryw.give_news3('rewriting meta data files...')
    for filename in l:
        ryw.db_print3('found meta: ' + filename, 61)
        meta = load_meta(filename)
        change_meta(meta, newObjStoreRoot)
        rewrite_meta(filename, meta)
        append_to_new_search_file(newSearchFileHandle, meta)



    #
    # replacing the old search file.
    #

    #ryw.copy_file_carefully('/u/rywang/tmp/x1',
    #                        '/u/rywang/tmp/x2',
    #                        '/u/rywang/tmp',
    #                        None,
    #                        'SearchFile_reformat')

    newSearchFileHandle.close()
    ryw.copy_file_carefully(oldSearchFileName, newSearchFileName,
                            repositoryRoot, None, 'SearchFile_reformat')
    ryw.give_news3('replacing search file: ' + oldSearchFileName +
                   ' <- ' + newSearchFileName)
Ejemplo n.º 8
0
def process_disk_from_peer_repository(dir_name, diskRoot, overwrite=False):

    objectroot = os.path.join(dir_name, 'objects')
    if not os.path.exists(objectroot):
        logging.info(
            'process_disk_from_peer_repository: no objects directory.' +
            objectroot)
        return True
    
    ## Process all incoming objects

    local_object_root = get_local_object_store_root()
    if local_object_root is None:
        return False

    mapDict = ryw_philips.get_map(diskRoot)
    searchFile = None

    for objectId,version in objectstore.objectversioniterator(objectroot):

        ryw.give_news3('----------', logging.info)

        if mapDict == None:
            ryw.give_bad_news(
                'process_disk_from_peer_repository: failed to read map file: '+
                diskRoot, logging.error)
            return False

        #
        # We used to just skip objects already present.
        # now we want to do something.
        #
        #if object_version_is_present(local_object_root, objectId, version):
        #    ## object already present
        #    continue
        objectFound = object_version_is_present(local_object_root,
                                                objectId,
                                                version)

        paths = objectstore.name_version_to_paths_aux(objectroot,
                                                      objectId, version)

        itemName = objectId + '#' + str(version)

        obdata = get_data_name_mirror(paths[0], diskRoot, mapDict, itemName)
        if not obdata:
            continue

        success,isStub = is_data_stub(obdata)
        if not success:
            continue

        metadata = get_metadata_mirror(paths[1])
        if not metadata:
            continue

        auxdir = get_aux_name_mirror(paths[2], diskRoot, mapDict, itemName)

        if isStub and not objectFound:
            #ryw.give_bad_news(
            #    'ProcessDiscs: is a stub but not found in database: '+
            #    itemName, logging.error)
            ryw.give_news3(
                'ProcessDiscs error: is a stub but not found in database: '+
                itemName, logging.error)
            continue
        
        if isStub:
            success,searchFile = deal_with_stub(
                local_object_root, objectId, version, metadata,
                obdata, auxdir, searchFile = searchFile)
            if success:
                ryw.give_news3(
                    'ProcessDiscs success: meta data processed.',
                    logging.info)
            continue

        if objectFound:
            #ryw.give_bad_news(
            #    'ProcessDiscs: not a stub but found in the database: '+
            #    itemName, logging.error)
            ryw.give_news3(
                'ProcessDiscs: not a stub but found in the database: '+
                itemName, logging.error)

            if not overwrite:
                ryw.give_news3('processing skipped.', logging.error)
                continue
            
            #
            # I might want to delete the old version here.
            # using the code from DelObject, should be simple.
            #
            ryw.give_news3('deleting it...', logging.error)

            #
            # at one point, I thought I wanted to be clever and
            # tried to reuse searchFile below.  But the trouble is
            # that the UploadObject.uploadobject() call below will
            # change the SearchFile beneath me, and if I reuse
            # the searchFile here, I end up flushing the incorrectly
            # cached version back to disk. I actually would have
            # expected a deadlock when UploadObject.uploadobject()
            # tries to lock again but the deadlock somehow
            # didn't happen...
            #
            success,searchFile = DeleteObject.do_delete(
                objectId, version, searchFile=None)
            if not success:
                ryw.give_bad_news(
                    'ProcessDiscs: DeleteObject failed: ' +
                    objectId + '#' + str(version), logging.error)
                continue
            else:
                ryw.db_print('process_disk_from_peer_repository: ' +
                             'do_delete succeeded.', 18)
                
            #
            # falls through to continue onto adding the object.
            #
            
        if not UploadObject.uploadobject(metadata, obdata, auxdir,
                                         hasVersion = True):
            ryw.give_bad_news(
                'process_disk_from_peer_repository: ' +
                'UploadObject.uploadobject failed: '+
                repr(metadata) + ' ' + obdata, logging.critical)
            continue
        else:
            ryw.give_news3('ProcessDiscs success: new data uploaded.',
                           logging.info)
            continue

    incomingReverseLists = os.path.join(dir_name, 'ReverseLists')
    existingReverseLists = os.path.join(RepositoryRoot, 'ReverseLists')

    ReverseLists.merge_incoming(
        existingReverseLists, incomingReverseLists, RepositoryRoot,
        searchFile = searchFile)

    if searchFile:
        searchFile.done()
    return True