def merge_global_object(vineyard_endpoint,
results: List[List[ObjectID]]) -> ObjectID:
if results is None or len(results) == 0:
raise ValueError("No available sub objects to merge")
chunks = []
for subresults in results:
chunks.extend(subresults)
if len(chunks) == 0:
raise ValueError("No available sub objects to merge")
if len(chunks) == 1:
# fastpath: no need to merge
if not isinstance(chunks[0], ObjectID):
return ObjectID(chunks[0])
else:
return chunks[0]
vineyard_rpc_client = vineyard.connect(vineyard_endpoint)
metadatas = []
for chunk in chunks:
if not isinstance(chunk, ObjectID):
chunk = ObjectID(chunk)
metadatas.append(vineyard_rpc_client.get_meta(chunk))
chunkmap, isglobal = dict(), False
for meta in metadatas:
if meta.isglobal:
isglobal = True
for k, v in meta.items():
if isinstance(v, ObjectMeta):
chunkmap[v.id] = k
else:
if isglobal:
raise ValueError('Not all sub objects are global objects: %s' %
results)
if not isglobal:
raise ValueError(
"Unable to merge more than one non-global objects: %s" % results)
base_meta = ObjectMeta()
base_meta.set_global(True)
for k, v in metadatas[0].items():
if isinstance(v, ObjectMeta):
continue
if k in ['id', 'signature', 'instance_id']:
continue
base_meta[k] = v
for v, k in chunkmap.items():
base_meta.add_member(k, v)
meta = vineyard_rpc_client.create_metadata(base_meta)
vineyard_rpc_client.persist(meta.id)
return meta.id
def get(client, object_id, resolver=None, **kw):
''' Get vineyard object as python value.
.. code:: python
>>> id = vineyard.ObjectID('00002ec13bc81226')
>>> arr = client.get(arr_id)
>>> arr
array([0, 1, 2, 3, 4, 5, 6, 7])
Parameters:
client: IPCClient or RPCClient
The vineyard client to use.
object_id: ObjectID
The object id that will be obtained from vineyard.
resolver:
When retrieving vineyard object, an optional *resolver* can be specified.
If no resolver given, the default resolver context will be used.
kw:
User-specific argument that will be passed to the builder.
Returns:
A python object that return by the resolver, by resolving an vineyard object.
'''
# wrap object_id
if isinstance(object_id, (int, str)):
object_id = ObjectID(object_id)
# run resolver
obj = client.get_object(object_id)
meta = obj.meta
if not meta.islocal and not meta.isglobal:
raise ValueError(
"Not a local object: for remote object, you can only get its metadata"
)
if resolver is not None:
value = resolver(obj, **kw)
else:
value = default_resolver_context.run(obj, **kw)
if value is None:
# if the obj has been resolved by pybind types, and there's no proper resolver, it
# shouldn't be an error
if type(obj) is not Object:
return obj
raise RuntimeError(
'Unable to construct the object: no proper resolver found: typename is %s'
% obj.meta.typename)
# associate a reference to the base C++ object
try:
setattr(value, '__vineyard_ref', obj)
except AttributeError:
pass
# register methods
default_driver_context.resolve(value, obj.typename)
# return value
return value
def __init__(self, vineyard_client, name=None, create=False, size=0):
'''Create or obtain a shared memory block that backed by vineyard.
Parameters
----------
vineyard_client:
The vineyard IPC or RPC client.
name:
The vineyard ObjectID, could be vineyard.ObjectID, int or stringified
ObjectID.
create:
Whether to create a new shared memory block or just obtain existing one.
size:
Size of the shared memory block.
See Also
--------
multiprocessing.shared_memory.SharedMemory
'''
if not size >= 0:
raise ValueError("'size' must be a positive integer")
if create:
if size == 0:
raise ValueError("'size' must be a positive number "
"different from zero")
if name is not None:
raise ValueError("'name' can only be None if create=True")
else:
if size != 0:
warnings.warn("'size' will take no effect if create=False", )
if name is None:
raise ValueError("'name' cannot be None if create=False")
self._name = None
self._size = None
self._buf = None
self._blob, self._blob_builder = None, None
self._vineyard_client = vineyard_client
if create:
self._blob_builder = vineyard_client.create_blob(size)
self._name = self._blob_builder.id
self._size = size
self._buf = memoryview(self._blob_builder)
else:
self._blob = vineyard_client.get_object(ObjectID(name))
self._name = self._blob.id
self._size = self._blob.size
self._buf = memoryview(self._blob)
def get(client, object_id, resolver=None, **kw):
''' Get vineyard object as python value.
.. code:: python
>>> id = vineyard.ObjectID('00002ec13bc81226')
>>> arr = client.get(arr_id)
>>> arr
array([0, 1, 2, 3, 4, 5, 6, 7])
Parameters:
client: IPCClient or RPCClient
The vineyard client to use.
object_id: ObjectID
The object id that will be obtained from vineyard.
resolver:
When retrieving vineyard object, an optional *resolver* can be specified.
If no resolver given, the default resolver context will be used.
kw:
User-specific argument that will be passed to the builder.
Returns:
A python object that return by the resolver, by resolving an vineyard object.
'''
# wrap object_id
if isinstance(object_id, (int, str)):
object_id = ObjectID(object_id)
# run resolver
obj = client.get_object(object_id)
if resolver is not None:
value = resolver(obj, **kw)
else:
value = default_resolver_context.run(obj, **kw)
if value is None:
raise RuntimeError('Unable to construct the object')
# associate a reference to the base C++ object
try:
setattr(value, '__vineyard_ref', obj)
except AttributeError:
pass
# register methods
default_driver_context.resolve(value, obj.typename)
# return value
return value
def make_global_dataframe(client, blocks, extra_meta=None) -> ObjectMeta:
meta = ObjectMeta()
meta['typename'] = 'vineyard::GlobalDataFrame'
meta.set_global(True)
meta['partitions_-size'] = len(blocks)
if extra_meta:
for k, v in extra_meta.items():
meta[k] = v
for idx, block in enumerate(blocks):
if not isinstance(block, (ObjectMeta, ObjectID, Object)):
block = ObjectID(block)
meta.add_member('partitions_-%d' % idx, block)
gtensor_meta = client.create_metadata(meta)
client.persist(gtensor_meta)
return gtensor_meta
def get(client, object_id, resolver=None, **kw):
''' Get vineyard object as python value.
.. code:: python
>>> arr_id = vineyard.ObjectID('00002ec13bc81226')
>>> arr = client.get(arr_id)
>>> arr
array([0, 1, 2, 3, 4, 5, 6, 7])
Parameters:
client: IPCClient or RPCClient
The vineyard client to use.
object_id: ObjectID
The object id that will be obtained from vineyard.
resolver:
When retrieving vineyard object, an optional *resolver* can be specified.
If no resolver given, the default resolver context will be used.
kw:
User-specific argument that will be passed to the builder.
Returns:
A python object that return by the resolver, by resolving an vineyard object.
'''
# wrap object_id
if isinstance(object_id, (int, str)):
object_id = ObjectID(object_id)
# run resolver
obj = client.get_object(object_id)
meta = obj.meta
if not meta.islocal and not meta.isglobal:
raise ValueError(
"Not a local object: for remote object, you can only get its metadata"
)
if resolver is None:
resolver = get_current_resolvers()
return resolver(obj, __vineyard_client=client, **kw)