def merge_global_object(vineyard_endpoint, results: List[List[ObjectID]]) -> ObjectID: if results is None or len(results) == 0: raise ValueError("No available sub objects to merge") chunks = [] for subresults in results: chunks.extend(subresults) if len(chunks) == 0: raise ValueError("No available sub objects to merge") if len(chunks) == 1: # fastpath: no need to merge if not isinstance(chunks[0], ObjectID): return ObjectID(chunks[0]) else: return chunks[0] vineyard_rpc_client = vineyard.connect(vineyard_endpoint) metadatas = [] for chunk in chunks: if not isinstance(chunk, ObjectID): chunk = ObjectID(chunk) metadatas.append(vineyard_rpc_client.get_meta(chunk)) chunkmap, isglobal = dict(), False for meta in metadatas: if meta.isglobal: isglobal = True for k, v in meta.items(): if isinstance(v, ObjectMeta): chunkmap[v.id] = k else: if isglobal: raise ValueError('Not all sub objects are global objects: %s' % results) if not isglobal: raise ValueError( "Unable to merge more than one non-global objects: %s" % results) base_meta = ObjectMeta() base_meta.set_global(True) for k, v in metadatas[0].items(): if isinstance(v, ObjectMeta): continue if k in ['id', 'signature', 'instance_id']: continue base_meta[k] = v for v, k in chunkmap.items(): base_meta.add_member(k, v) meta = vineyard_rpc_client.create_metadata(base_meta) vineyard_rpc_client.persist(meta.id) return meta.id
def get(client, object_id, resolver=None, **kw): ''' Get vineyard object as python value. .. code:: python >>> id = vineyard.ObjectID('00002ec13bc81226') >>> arr = client.get(arr_id) >>> arr array([0, 1, 2, 3, 4, 5, 6, 7]) Parameters: client: IPCClient or RPCClient The vineyard client to use. object_id: ObjectID The object id that will be obtained from vineyard. resolver: When retrieving vineyard object, an optional *resolver* can be specified. If no resolver given, the default resolver context will be used. kw: User-specific argument that will be passed to the builder. Returns: A python object that return by the resolver, by resolving an vineyard object. ''' # wrap object_id if isinstance(object_id, (int, str)): object_id = ObjectID(object_id) # run resolver obj = client.get_object(object_id) meta = obj.meta if not meta.islocal and not meta.isglobal: raise ValueError( "Not a local object: for remote object, you can only get its metadata" ) if resolver is not None: value = resolver(obj, **kw) else: value = default_resolver_context.run(obj, **kw) if value is None: # if the obj has been resolved by pybind types, and there's no proper resolver, it # shouldn't be an error if type(obj) is not Object: return obj raise RuntimeError( 'Unable to construct the object: no proper resolver found: typename is %s' % obj.meta.typename) # associate a reference to the base C++ object try: setattr(value, '__vineyard_ref', obj) except AttributeError: pass # register methods default_driver_context.resolve(value, obj.typename) # return value return value
def __init__(self, vineyard_client, name=None, create=False, size=0): '''Create or obtain a shared memory block that backed by vineyard. Parameters ---------- vineyard_client: The vineyard IPC or RPC client. name: The vineyard ObjectID, could be vineyard.ObjectID, int or stringified ObjectID. create: Whether to create a new shared memory block or just obtain existing one. size: Size of the shared memory block. See Also -------- multiprocessing.shared_memory.SharedMemory ''' if not size >= 0: raise ValueError("'size' must be a positive integer") if create: if size == 0: raise ValueError("'size' must be a positive number " "different from zero") if name is not None: raise ValueError("'name' can only be None if create=True") else: if size != 0: warnings.warn("'size' will take no effect if create=False", ) if name is None: raise ValueError("'name' cannot be None if create=False") self._name = None self._size = None self._buf = None self._blob, self._blob_builder = None, None self._vineyard_client = vineyard_client if create: self._blob_builder = vineyard_client.create_blob(size) self._name = self._blob_builder.id self._size = size self._buf = memoryview(self._blob_builder) else: self._blob = vineyard_client.get_object(ObjectID(name)) self._name = self._blob.id self._size = self._blob.size self._buf = memoryview(self._blob)
def get(client, object_id, resolver=None, **kw): ''' Get vineyard object as python value. .. code:: python >>> id = vineyard.ObjectID('00002ec13bc81226') >>> arr = client.get(arr_id) >>> arr array([0, 1, 2, 3, 4, 5, 6, 7]) Parameters: client: IPCClient or RPCClient The vineyard client to use. object_id: ObjectID The object id that will be obtained from vineyard. resolver: When retrieving vineyard object, an optional *resolver* can be specified. If no resolver given, the default resolver context will be used. kw: User-specific argument that will be passed to the builder. Returns: A python object that return by the resolver, by resolving an vineyard object. ''' # wrap object_id if isinstance(object_id, (int, str)): object_id = ObjectID(object_id) # run resolver obj = client.get_object(object_id) if resolver is not None: value = resolver(obj, **kw) else: value = default_resolver_context.run(obj, **kw) if value is None: raise RuntimeError('Unable to construct the object') # associate a reference to the base C++ object try: setattr(value, '__vineyard_ref', obj) except AttributeError: pass # register methods default_driver_context.resolve(value, obj.typename) # return value return value
def make_global_dataframe(client, blocks, extra_meta=None) -> ObjectMeta: meta = ObjectMeta() meta['typename'] = 'vineyard::GlobalDataFrame' meta.set_global(True) meta['partitions_-size'] = len(blocks) if extra_meta: for k, v in extra_meta.items(): meta[k] = v for idx, block in enumerate(blocks): if not isinstance(block, (ObjectMeta, ObjectID, Object)): block = ObjectID(block) meta.add_member('partitions_-%d' % idx, block) gtensor_meta = client.create_metadata(meta) client.persist(gtensor_meta) return gtensor_meta
def get(client, object_id, resolver=None, **kw): ''' Get vineyard object as python value. .. code:: python >>> arr_id = vineyard.ObjectID('00002ec13bc81226') >>> arr = client.get(arr_id) >>> arr array([0, 1, 2, 3, 4, 5, 6, 7]) Parameters: client: IPCClient or RPCClient The vineyard client to use. object_id: ObjectID The object id that will be obtained from vineyard. resolver: When retrieving vineyard object, an optional *resolver* can be specified. If no resolver given, the default resolver context will be used. kw: User-specific argument that will be passed to the builder. Returns: A python object that return by the resolver, by resolving an vineyard object. ''' # wrap object_id if isinstance(object_id, (int, str)): object_id = ObjectID(object_id) # run resolver obj = client.get_object(object_id) meta = obj.meta if not meta.islocal and not meta.isglobal: raise ValueError( "Not a local object: for remote object, you can only get its metadata" ) if resolver is None: resolver = get_current_resolvers() return resolver(obj, __vineyard_client=client, **kw)