def create_event_xn(self, name, event_type=None, origin=None, sub_type=None, origin_type=None, pattern=None,
xp=None, auto_delete=None, **kwargs):
"""
Creates an EventExchangeName suitable for listening with an EventSubscriber.
Pass None for the name to have one automatically generated.
If you pass a pattern, it takes precedence over making a new one from event_type/origin/sub_type/origin_type.
"""
# make a name if no name exists
name = name or create_simple_unique_id()
# get event xp for the xs if not set
if not xp:
# pull from configuration
eventxp = CFG.get_safe('exchange.core.events', DEFAULT_EVENTS_XP)
xp = self.create_xp(eventxp)
node = xp.node
transport = xp._transports[0]
xn = EventExchangeName(self, transport, node, name, xp,
event_type=event_type,
sub_type=sub_type,
origin=origin,
origin_type=origin_type,
pattern=pattern,
auto_delete=auto_delete,
**kwargs)
self._register_xn(name, xn, xp)
return xn
def __init__(self, process, exchange_name=None, stream=None, exchange_point=None, callback=None):
"""
Creates a new StreamSubscriber which will listen on the specified queue (exchange_name).
@param process The IonProcess to attach to.
@param exchange_name The subscribing queue name.
@param stream (optional) Name of the stream or StreamRoute object, to subscribe to
@param callback The callback to execute upon receipt of a packet.
"""
if not isinstance(process, BaseService):
raise BadRequest("No valid process provided.")
self.queue_name = exchange_name or ("subsc_" + create_simple_unique_id())
self.streams = []
self.container = process.container
exchange_point = exchange_point or DEFAULT_DATA_XP
self.xp_name = get_streaming_xp(exchange_point)
self.xp = self.container.ex_manager.create_xp(exchange_point)
self.xn = self.container.ex_manager.create_queue_xn(self.queue_name, xs=self.xp)
self.started = False
self.callback = callback or process.call_process
super(StreamSubscriber, self).__init__(from_name=self.xn, callback=self.preprocess)
if stream:
self.add_stream_subscription(stream)
def __init__(self,
process,
exchange_name=None,
stream=None,
exchange_point=None,
callback=None):
"""
Creates a new StreamSubscriber which will listen on the specified queue (exchange_name).
@param process The IonProcess to attach to.
@param exchange_name The subscribing queue name.
@param stream (optional) Name of the stream or StreamRoute object, to subscribe to
@param callback The callback to execute upon receipt of a packet.
"""
if not isinstance(process, BaseService):
raise BadRequest("No valid process provided.")
self.queue_name = exchange_name or ("subsc_" +
create_simple_unique_id())
self.streams = []
self.container = process.container
exchange_point = exchange_point or DEFAULT_DATA_XP
self.xp_name = get_streaming_xp(exchange_point)
self.xp = self.container.ex_manager.create_xp(exchange_point)
self.xn = self.container.ex_manager.create_queue_xn(self.queue_name,
xs=self.xp)
self.started = False
self.callback = callback or process.call_process
super(StreamSubscriber, self).__init__(from_name=self.xn,
callback=self.preprocess)
if stream:
self.add_stream_subscription(stream)
def create_event_xn(self, name, event_type=None, origin=None, sub_type=None, origin_type=None, pattern=None,
xp=None, auto_delete=None, **kwargs):
"""
Creates an EventExchangeName suitable for listening with an EventSubscriber.
Pass None for the name to have one automatically generated.
If you pass a pattern, it takes precedence over making a new one from event_type/origin/sub_type/origin_type.
"""
# make a name if no name exists
name = name or create_simple_unique_id()
# get event xp for the xs if not set
if not xp:
# pull from configuration
eventxp = CFG.get_safe('exchange.core.events', DEFAULT_EVENTS_XP)
xp = self.create_xp(eventxp)
node = xp.node
transport = xp._transports[0]
xn = EventExchangeName(self, transport, node, name, xp,
event_type=event_type,
sub_type=sub_type,
origin=origin,
origin_type=origin_type,
pattern=pattern,
auto_delete=auto_delete,
**kwargs)
self._register_xn(name, xn, xp)
return xn
def start_rel(self, rel_def, reply_to=None):
command_id = create_simple_unique_id()
action_cmd = dict(command="start_rel", command_id=command_id, rel_def=rel_def,
reply_to=reply_to)
self.cmd_pub.publish(action_cmd)
return command_id
def __init__(
self, xp_name=None, event_type=None, origin=None, queue_name=None, sub_type=None, origin_type=None, pattern=None
):
self.event_type = event_type
self.sub_type = sub_type
self.origin_type = origin_type
self.origin = origin
xp_name = xp_name or get_events_exchange_point()
if pattern:
binding = pattern
else:
binding = self._topic(event_type, origin, sub_type, origin_type)
self.binding = binding
# TODO: Provide a case where we can have multiple bindings (e.g. different event_types)
# prefix the queue_name, if specified, with the sysname
if queue_name is not None:
if not queue_name.startswith(bootstrap.get_sys_name()):
queue_name = "%s.%s" % (bootstrap.get_sys_name(), queue_name)
else:
queue_name = create_simple_unique_id()
if hasattr(self, "_process") and self._process:
queue_name = "%s_%s" % (self._process._proc_name, queue_name)
queue_name = "%s.%s" % (bootstrap.get_sys_name(), queue_name)
# set this name to be picked up by inherited folks
self._ev_recv_name = (xp_name, queue_name)
def schedule(self, process_id, process_definition, schedule, configuration, name):
command_id = create_simple_unique_id()
action_cmd = dict(command="schedule", command_id=command_id,
process_id=process_id, process_definition=process_definition,
schedule=schedule, configuration=configuration, name=name)
self.cmd_pub.publish(action_cmd)
return command_id
def test_hdf5_persist_decimate(self):
# Test HDF5 writing, time indexing, array extension etc
ds_schema_str = """
type: scion_data_schema_1
description: Schema for test datasets
attributes:
basic_shape: 1d_timeseries
time_variable: time
persistence:
format: hdf5
layout: vars_individual
row_increment: 1000
time_index_step: 1000
variables:
- name: time
base_type: ntp_time
storage_dtype: i8
unit: ""
description: NTPv4 timestamp
- name: var1
base_type: float
storage_dtype: f8
unit: ""
description: Sample value
- name: random1
base_type: float
storage_dtype: f8
unit: ""
description: Random values
"""
ds_schema = yaml.load(ds_schema_str)
ds_id = create_simple_unique_id()
ds_filename = self.container.file_system.get(
"%s/%s%s.hdf5" % (DS_BASE_PATH, DS_FILE_PREFIX, ds_id))
self.hdf5_persist = DatasetHDF5Persistence.get_persistence(
ds_id, ds_schema, "hdf5")
self.hdf5_persist.require_dataset()
self.assertTrue(os.path.exists(ds_filename))
self.addCleanup(os.remove, ds_filename)
# Add 100000 values in packets of 100
for i in xrange(100):
packet = self._get_data_packet(i * 100, 100)
self.hdf5_persist.extend_dataset(packet)
data_res = self.hdf5_persist.get_data()
self.assertEqual(len(data_res), 3)
self.assertEqual(len(data_res["time"]), 10000)
data_res = self.hdf5_persist.get_data(
dict(max_rows=999, decimate=True, decimate_method="minmax"))
self.assertEqual(len(data_res), 3)
self.assertLessEqual(len(data_res["time"]), 1000)
def __init__(self, process=None):
self.process = process
self.async_res = AsyncResult()
self.wait_name = "asyncresult_" + create_simple_unique_id()
if self.process:
self.wait_name = self.wait_name + "_" + self.process.id
# TODO: Use same mechanism as pooled RPC response endpoint (without the request)
self.wait_sub = Subscriber(from_name=self.wait_name,
callback=self._result_callback,
auto_delete=True)
self.activated = False
def test_hdf5_persist_decimate(self):
# Test HDF5 writing, time indexing, array extension etc
ds_schema_str = """
type: scion_data_schema_1
description: Schema for test datasets
attributes:
basic_shape: 1d_timeseries
time_variable: time
persistence:
format: hdf5
layout: vars_individual
row_increment: 1000
time_index_step: 1000
variables:
- name: time
base_type: ntp_time
storage_dtype: i8
unit: ""
description: NTPv4 timestamp
- name: var1
base_type: float
storage_dtype: f8
unit: ""
description: Sample value
- name: random1
base_type: float
storage_dtype: f8
unit: ""
description: Random values
"""
ds_schema = yaml.load(ds_schema_str)
ds_id = create_simple_unique_id()
ds_filename = self.container.file_system.get("%s/%s%s.hdf5" % (DS_BASE_PATH, DS_FILE_PREFIX, ds_id))
self.hdf5_persist = DatasetHDF5Persistence.get_persistence(ds_id, ds_schema, "hdf5")
self.hdf5_persist.require_dataset()
self.assertTrue(os.path.exists(ds_filename))
self.addCleanup(os.remove, ds_filename)
# Add 100000 values in packets of 100
for i in xrange(100):
packet = self._get_data_packet(i * 100, 100)
self.hdf5_persist.extend_dataset(packet)
data_res = self.hdf5_persist.get_data()
self.assertEqual(len(data_res), 3)
self.assertEqual(len(data_res["time"]), 10000)
data_res = self.hdf5_persist.get_data(dict(max_rows=999, decimate=True, decimate_method="minmax"))
self.assertEqual(len(data_res), 3)
self.assertLessEqual(len(data_res["time"]), 1000)
def __init__(self, xp_name=None, event_type=None, origin=None, queue_name=None,
sub_type=None, origin_type=None, pattern=None, auto_delete=None):
self._events_xp = CFG.get_safe("exchange.core.events", DEFAULT_EVENTS_XP)
self.event_type = event_type
self.sub_type = sub_type
self.origin_type = origin_type
self.origin = origin
# Default for auto_delete is True for events, unless otherwise specified
if auto_delete is None:
auto_delete = True
self._auto_delete = auto_delete
xp_name = xp_name or self._events_xp
if pattern:
binding = pattern
else:
binding = self._topic(event_type, origin, sub_type, origin_type)
# create queue_name if none passed in
if queue_name is None:
queue_name = "subsc_" + create_simple_unique_id()
# prepend proc name to queue name if we have one
if hasattr(self, "_process") and self._process:
queue_name = "%s_%s" % (self._process._proc_name, queue_name)
# do we have a container/ex_manager?
container = (hasattr(self, '_process') and hasattr(self._process, 'container') and self._process.container) or BaseEndpoint._get_container_instance()
if container:
xp = container.create_xp(xp_name)
xne = container.create_event_xn(queue_name,
pattern=binding,
xp=xp,
auto_delete=auto_delete)
self._ev_recv_name = xne
self.binding = None
else:
# Remove this case. No container??
self.binding = binding
# prefix the queue_name, if specified, with the sysname
queue_name = "%s.system.%s" % (bootstrap.get_sys_name(), queue_name)
# set this name to be picked up by inherited folks
self._ev_recv_name = (xp_name, queue_name)
local_event_queues.append(queue_name)
def __init__(self, xp_name=None, event_type=None, origin=None, queue_name=None,
sub_type=None, origin_type=None, pattern=None):
self._events_xp = CFG.get_safe("exchange.core.events", DEFAULT_EVENTS_XP)
self.event_type = event_type
self.sub_type = sub_type
self.origin_type = origin_type
self.origin = origin
# establish names for xp, binding/pattern/topic, queue_name
xp_name = xp_name or self._events_xp
if pattern:
binding = pattern
else:
binding = self._topic(event_type, origin, sub_type, origin_type)
# create queue_name if none passed in
if queue_name is None:
queue_name = "subsc_" + create_simple_unique_id()
# prepend proc name to queue name if we have one
if hasattr(self, "_process") and self._process:
queue_name = "%s_%s" % (self._process._proc_name, queue_name)
# do we have a container/ex_manager?
container = (hasattr(self, '_process') and hasattr(self._process, 'container') and self._process.container) or BaseEndpoint._get_container_instance()
if container:
xp = container.create_xp(xp_name)
xne = container.create_xn_event(queue_name,
pattern=binding,
xp=xp)
self._ev_recv_name = xne
self.binding = None
else:
self.binding = binding
# TODO: Provide a case where we can have multiple bindings (e.g. different event_types)
# prefix the queue_name, if specified, with the sysname
queue_name = "%s.%s" % (bootstrap.get_sys_name(), queue_name)
# set this name to be picked up by inherited folks
self._ev_recv_name = (xp_name, queue_name)
local_event_queues.append(queue_name)
def list(self):
command_id = create_simple_unique_id()
action_cmd = dict(command="list", command_id=command_id)
self.cmd_pub.publish(action_cmd)
return command_id
def cancel(self, process_id):
command_id = create_simple_unique_id()
action_cmd = dict(command="cancel", command_id=command_id,
process_id=process_id)
self.cmd_pub.publish(action_cmd)
return command_id
def test_hdf5_persist(self):
# Test HDF5 writing, time indexing, array extension etc
ds_schema_str = """
type: scion_data_schema_1
description: Schema for test datasets
attributes:
basic_shape: 1d_timeseries
time_variable: time
persistence:
format: hdf5
layout: vars_individual
row_increment: 1000
time_index_step: 1000
variables:
- name: time
base_type: ntp_time
storage_dtype: i8
unit: ""
description: NTPv4 timestamp
- name: var1
base_type: float
storage_dtype: f8
unit: ""
description: Sample value
- name: random1
base_type: float
storage_dtype: f8
unit: ""
description: Random values
"""
ds_schema = yaml.load(ds_schema_str)
ds_id = create_simple_unique_id()
ds_filename = self.container.file_system.get(
"%s/%s%s.hdf5" % (DS_BASE_PATH, DS_FILE_PREFIX, ds_id))
self.hdf5_persist = DatasetHDF5Persistence.get_persistence(
ds_id, ds_schema, "hdf5")
self.hdf5_persist.require_dataset()
self.assertTrue(os.path.exists(ds_filename))
self.addCleanup(os.remove, ds_filename)
# Add 100 values in packets of 10
for i in xrange(10):
packet = self._get_data_packet(i * 10, 10)
self.hdf5_persist.extend_dataset(packet)
data_res = self.hdf5_persist.get_data()
self.assertEqual(len(data_res), 3)
self.assertEqual(len(data_res["time"]), 100)
self.assertEqual(len(data_res["var1"]), 100)
self.assertEqual(len(data_res["random1"]), 100)
self.assertEqual(data_res["var1"][1], 1.0)
with HDFLockingFile(ds_filename, "r") as hdff:
ds_time = hdff["vars/time"]
cur_idx = ds_time.attrs["cur_row"]
self.assertEqual(cur_idx, 100)
self.assertEqual(len(ds_time), 1000)
ds_tidx = hdff[DS_TIMEIDX_PATH]
cur_tidx = ds_tidx.attrs["cur_row"]
self.assertEqual(cur_tidx, 1)
self.assertEqual(len(ds_tidx), 1000)
# Add 1000 values in packets of 10
for i in xrange(100):
packet = self._get_data_packet(100 + i * 10, 10)
self.hdf5_persist.extend_dataset(packet)
data_res = self.hdf5_persist.get_data()
self.assertEqual(len(data_res["time"]), 1100)
with HDFLockingFile(ds_filename, "r") as hdff:
ds_time = hdff["vars/time"]
cur_idx = ds_time.attrs["cur_row"]
self.assertEqual(cur_idx, 1100)
self.assertEqual(len(ds_time), 2000)
ds_tidx = hdff[DS_TIMEIDX_PATH]
cur_tidx = ds_tidx.attrs["cur_row"]
self.assertEqual(cur_tidx, 2)
self.assertEqual(len(ds_tidx), 1000)
self.assertEqual(ds_time[0], ds_tidx[0][0])
self.assertEqual(ds_time[1000], ds_tidx[1][0])
info_res = self.hdf5_persist.get_data_info()
self.assertEqual(info_res["ds_rows"], 1100)
self.assertEqual(info_res["ts_first"], 1000000000.0)
self.assertEqual(info_res["ts_last"], 1000010990.0)
def test_hdf5_persist_prune(self):
# Test auto-pruning
ds_schema_str = """
type: scion_data_schema_1
description: Schema for test datasets
attributes:
basic_shape: 1d_timeseries
time_variable: time
persistence:
format: hdf5
layout: vars_individual
row_increment: 1000
time_index_step: 1000
pruning:
trigger_mode: on_ingest
prune_mode: max_age_rel
prune_action: rewrite
trigger_age: 1000.0
retain_age: 500.0
variables:
- name: time
base_type: ntp_time
storage_dtype: i8
unit: ""
description: NTPv4 timestamp
- name: var1
base_type: float
storage_dtype: f8
unit: ""
description: Sample value
- name: random1
base_type: float
storage_dtype: f8
unit: ""
description: Random values
"""
ds_schema = yaml.load(ds_schema_str)
ds_id = create_simple_unique_id()
ds_filename = self.container.file_system.get(
"%s/%s%s.hdf5" % (DS_BASE_PATH, DS_FILE_PREFIX, ds_id))
self.hdf5_persist = DatasetHDF5Persistence.get_persistence(
ds_id, ds_schema, "hdf5")
self.hdf5_persist.require_dataset()
self.assertTrue(os.path.exists(ds_filename))
self.addCleanup(os.remove, ds_filename)
# Add 100 values in packets of 10 (right up to the prune trigger)
for i in xrange(10):
packet = self._get_data_packet(i * 10, 10)
self.hdf5_persist.extend_dataset(packet)
data_res = self.hdf5_persist.get_data()
self.assertEqual(len(data_res["time"]), 100)
self.assertEqual(len(data_res["var1"]), 100)
self.assertEqual(len(data_res["random1"]), 100)
self.assertEqual(data_res["var1"][1], 1.0)
log.info("*** STEP 2: First prune")
# Add 2 values (stepping across the prune trigger - inclusive boundary)
packet = self._get_data_packet(100, 2)
self.hdf5_persist.extend_dataset(packet)
data_res = self.hdf5_persist.get_data()
self.assertEqual(len(data_res["time"]), 51)
self.assertEqual(len(data_res["var1"]), 51)
self.assertEqual(len(data_res["random1"]), 51)
self.assertEqual(data_res["var1"][0], 51.0)
self.assertEqual(data_res["var1"][50], 101.0)
log.info("*** STEP 3: Additional data")
# Add 100 values in packets of 10 (right up to the prune trigger)
packet = self._get_data_packet(102, 8)
self.hdf5_persist.extend_dataset(packet)
for i in xrange(4):
packet = self._get_data_packet(110 + i * 10, 10)
self.hdf5_persist.extend_dataset(packet)
packet = self._get_data_packet(150, 2)
self.hdf5_persist.extend_dataset(packet)
data_res = self.hdf5_persist.get_data()
self.assertEqual(len(data_res["time"]), 101)
self.assertEqual(data_res["var1"][0], 51.0)
self.assertEqual(data_res["var1"][100], 151.0)
log.info("*** STEP 4: Second prune")
packet = self._get_data_packet(152, 1)
self.hdf5_persist.extend_dataset(packet)
data_res = self.hdf5_persist.get_data()
self.assertEqual(len(data_res["time"]), 51)
self.assertEqual(data_res["var1"][0], 102.0)
self.assertEqual(data_res["var1"][50], 152.0)
log.info("*** STEP 5: Third prune")
packet = self._get_data_packet(153, 100)
self.hdf5_persist.extend_dataset(packet)
data_res = self.hdf5_persist.get_data()
self.assertEqual(len(data_res["time"]), 51)
self.assertEqual(data_res["var1"][0], 202.0)
self.assertEqual(data_res["var1"][50], 252.0)
def test_objstore_doc(self):
# Create
doc1 = dict(a="String",
b=123,
c=False,
d=None,
e=1.23,
f=["Some", "More"],
g=dict(x=[1, 2, 3], y={}, z="Str"))
doc2 = dict(a=u"String\u20ac",
b=123,
c=False,
d=None,
e=1.23,
f=[u"Some\u20ac", "More"],
g=dict(x=[1, 2, 3], y={}, z="Str"))
doc2[u"h\u20ac"] = u"Other\u20ac"
doc3 = doc1.copy()
doc4 = doc1.copy()
doc5 = doc2.copy()
did1, dv1 = self.os.create_doc(doc1)
self.assertTrue(did1)
did2, dv2 = self.os.create_doc(doc2)
self.assertTrue(did2)
did3n = create_simple_unique_id()
did3, dv3 = self.os.create_doc(doc3, object_id=did3n)
self.assertEquals(did3, did3n)
did4n, did5n = create_simple_unique_id(), create_simple_unique_id()
res = self.os.create_doc_mult([doc4, doc5], object_ids=[did4n, did5n])
_, did4, dv4 = res[0]
_, did5, dv5 = res[1]
# Read
all_doc_ids = [did1, did2, did3, did4, did5]
docs = self.os.read_doc_mult(all_doc_ids)
self.assertEquals(len(docs), len(all_doc_ids))
doc1r = self.os.read_doc(did1)
self.assertIsInstance(doc1r, dict)
self.assertIn("a", doc1r)
self.assertEquals(doc1r["g"]["x"][1], 2)
doc2r = self.os.read_doc(did2)
self.assertIsInstance(doc2r, dict)
self.assertIn("a", doc2r)
self.assertEquals(type(doc2r["a"]), str)
self.assertEquals(doc2r["a"], u"String\u20ac".encode("utf8"))
self.assertIn(u"h\u20ac".encode("utf8"), doc2r)
# Update
doc1r["a"] = "BUZZ"
doc1rc = doc1r.copy()
self.os.update_doc(doc1r)
with self.assertRaises(Conflict):
doc1rc["a"] = "ZAMM"
self.os.update_doc(doc1rc)
doc2r["a"] = u"BUZZ\u20ac"
doc2r[u"h\u20ac".encode("utf8")] = u"ZAMM\u20ac"
doc3r = self.os.read_doc(did3)
doc3r["a"] = u"BUZZ\u20ac"
self.os.update_doc_mult([doc2r, doc3r])
# Delete
self.os.delete_doc(did1)
self.os.delete_doc(did2)
self.os.delete_doc(did3)
self.os.delete_doc_mult([did4, did5])
with self.assertRaises(NotFound):
self.os.read_doc(did1)
with self.assertRaises(NotFound):
self.os.read_doc(did2)
with self.assertRaises(NotFound):
self.os.read_doc(did3)
with self.assertRaises(NotFound):
self.os.read_doc(did4)
with self.assertRaises(NotFound):
self.os.read_doc(did5)
def test_hdf5_persist(self):
# Test HDF5 writing, time indexing, array extension etc
ds_schema_str = """
type: scion_data_schema_1
description: Schema for test datasets
attributes:
basic_shape: 1d_timeseries
time_variable: time
persistence:
format: hdf5
layout: vars_individual
row_increment: 1000
time_index_step: 1000
variables:
- name: time
base_type: ntp_time
storage_dtype: i8
unit: ""
description: NTPv4 timestamp
- name: var1
base_type: float
storage_dtype: f8
unit: ""
description: Sample value
- name: random1
base_type: float
storage_dtype: f8
unit: ""
description: Random values
"""
ds_schema = yaml.load(ds_schema_str)
ds_id = create_simple_unique_id()
ds_filename = self.container.file_system.get("%s/%s%s.hdf5" % (DS_BASE_PATH, DS_FILE_PREFIX, ds_id))
self.hdf5_persist = DatasetHDF5Persistence.get_persistence(ds_id, ds_schema, "hdf5")
self.hdf5_persist.require_dataset()
self.assertTrue(os.path.exists(ds_filename))
self.addCleanup(os.remove, ds_filename)
# Add 100 values in packets of 10
for i in xrange(10):
packet = self._get_data_packet(i*10, 10)
self.hdf5_persist.extend_dataset(packet)
data_res = self.hdf5_persist.get_data()
self.assertEqual(len(data_res), 3)
self.assertEqual(len(data_res["time"]), 100)
self.assertEqual(len(data_res["var1"]), 100)
self.assertEqual(len(data_res["random1"]), 100)
self.assertEqual(data_res["var1"][1], 1.0)
with HDFLockingFile(ds_filename, "r") as hdff:
ds_time = hdff["vars/time"]
cur_idx = ds_time.attrs["cur_row"]
self.assertEqual(cur_idx, 100)
self.assertEqual(len(ds_time), 1000)
ds_tidx = hdff[DS_TIMEIDX_PATH]
cur_tidx = ds_tidx.attrs["cur_row"]
self.assertEqual(cur_tidx, 1)
self.assertEqual(len(ds_tidx), 1000)
# Add 1000 values in packets of 10
for i in xrange(100):
packet = self._get_data_packet(100 + i*10, 10)
self.hdf5_persist.extend_dataset(packet)
data_res = self.hdf5_persist.get_data()
self.assertEqual(len(data_res["time"]), 1100)
with HDFLockingFile(ds_filename, "r") as hdff:
ds_time = hdff["vars/time"]
cur_idx = ds_time.attrs["cur_row"]
self.assertEqual(cur_idx, 1100)
self.assertEqual(len(ds_time), 2000)
ds_tidx = hdff[DS_TIMEIDX_PATH]
cur_tidx = ds_tidx.attrs["cur_row"]
self.assertEqual(cur_tidx, 2)
self.assertEqual(len(ds_tidx), 1000)
self.assertEqual(ds_time[0], ds_tidx[0][0])
self.assertEqual(ds_time[1000], ds_tidx[1][0])
info_res = self.hdf5_persist.get_data_info()
self.assertEqual(info_res["ds_rows"], 1100)
self.assertEqual(info_res["ts_first"], 1000000000.0)
self.assertEqual(info_res["ts_last"], 1000010990.0)
def test_hdf5_persist_prune(self):
# Test auto-pruning
ds_schema_str = """
type: scion_data_schema_1
description: Schema for test datasets
attributes:
basic_shape: 1d_timeseries
time_variable: time
persistence:
format: hdf5
layout: vars_individual
row_increment: 1000
time_index_step: 1000
pruning:
trigger_mode: on_ingest
prune_mode: max_age_rel
prune_action: rewrite
trigger_age: 1000.0
retain_age: 500.0
variables:
- name: time
base_type: ntp_time
storage_dtype: i8
unit: ""
description: NTPv4 timestamp
- name: var1
base_type: float
storage_dtype: f8
unit: ""
description: Sample value
- name: random1
base_type: float
storage_dtype: f8
unit: ""
description: Random values
"""
ds_schema = yaml.load(ds_schema_str)
ds_id = create_simple_unique_id()
ds_filename = self.container.file_system.get("%s/%s%s.hdf5" % (DS_BASE_PATH, DS_FILE_PREFIX, ds_id))
self.hdf5_persist = DatasetHDF5Persistence.get_persistence(ds_id, ds_schema, "hdf5")
self.hdf5_persist.require_dataset()
self.assertTrue(os.path.exists(ds_filename))
self.addCleanup(os.remove, ds_filename)
# Add 100 values in packets of 10 (right up to the prune trigger)
for i in xrange(10):
packet = self._get_data_packet(i * 10, 10)
self.hdf5_persist.extend_dataset(packet)
data_res = self.hdf5_persist.get_data()
self.assertEqual(len(data_res["time"]), 100)
self.assertEqual(len(data_res["var1"]), 100)
self.assertEqual(len(data_res["random1"]), 100)
self.assertEqual(data_res["var1"][1], 1.0)
log.info("*** STEP 2: First prune")
# Add 2 values (stepping across the prune trigger - inclusive boundary)
packet = self._get_data_packet(100, 2)
self.hdf5_persist.extend_dataset(packet)
data_res = self.hdf5_persist.get_data()
self.assertEqual(len(data_res["time"]), 51)
self.assertEqual(len(data_res["var1"]), 51)
self.assertEqual(len(data_res["random1"]), 51)
self.assertEqual(data_res["var1"][0], 51.0)
self.assertEqual(data_res["var1"][50], 101.0)
log.info("*** STEP 3: Additional data")
# Add 100 values in packets of 10 (right up to the prune trigger)
packet = self._get_data_packet(102, 8)
self.hdf5_persist.extend_dataset(packet)
for i in xrange(4):
packet = self._get_data_packet(110 + i * 10, 10)
self.hdf5_persist.extend_dataset(packet)
packet = self._get_data_packet(150, 2)
self.hdf5_persist.extend_dataset(packet)
data_res = self.hdf5_persist.get_data()
self.assertEqual(len(data_res["time"]), 101)
self.assertEqual(data_res["var1"][0], 51.0)
self.assertEqual(data_res["var1"][100], 151.0)
log.info("*** STEP 4: Second prune")
packet = self._get_data_packet(152, 1)
self.hdf5_persist.extend_dataset(packet)
data_res = self.hdf5_persist.get_data()
self.assertEqual(len(data_res["time"]), 51)
self.assertEqual(data_res["var1"][0], 102.0)
self.assertEqual(data_res["var1"][50], 152.0)
log.info("*** STEP 5: Third prune")
packet = self._get_data_packet(153, 100)
self.hdf5_persist.extend_dataset(packet)
data_res = self.hdf5_persist.get_data()
self.assertEqual(len(data_res["time"]), 51)
self.assertEqual(data_res["var1"][0], 202.0)
self.assertEqual(data_res["var1"][50], 252.0)
def test_objstore_doc(self):
# Create
doc1 = dict(a="String", b=123, c=False, d=None, e=1.23,
f=["Some", "More"],
g=dict(x=[1,2,3], y={}, z="Str"))
doc2 = dict(a=u"String\u20ac", b=123, c=False, d=None, e=1.23,
f=[u"Some\u20ac", "More"],
g=dict(x=[1,2,3], y={}, z="Str"))
doc2[u"h\u20ac"] = u"Other\u20ac"
doc3 = doc1.copy()
doc4 = doc1.copy()
doc5 = doc2.copy()
did1, dv1 = self.os.create_doc(doc1)
self.assertTrue(did1)
did2, dv2 = self.os.create_doc(doc2)
self.assertTrue(did2)
did3n = create_simple_unique_id()
did3, dv3 = self.os.create_doc(doc3, object_id=did3n)
self.assertEquals(did3, did3n)
did4n, did5n = create_simple_unique_id(), create_simple_unique_id()
res = self.os.create_doc_mult([doc4, doc5], object_ids=[did4n, did5n])
_, did4, dv4 = res[0]
_, did5, dv5 = res[1]
# Read
all_doc_ids = [did1, did2, did3, did4, did5]
docs = self.os.read_doc_mult(all_doc_ids)
self.assertEquals(len(docs), len(all_doc_ids))
doc1r = self.os.read_doc(did1)
self.assertIsInstance(doc1r, dict)
self.assertIn("a", doc1r)
self.assertEquals(doc1r["g"]["x"][1], 2)
doc2r = self.os.read_doc(did2)
self.assertIsInstance(doc2r, dict)
self.assertIn("a", doc2r)
self.assertEquals(type(doc2r["a"]), str)
self.assertEquals(doc2r["a"], u"String\u20ac".encode("utf8"))
self.assertIn(u"h\u20ac".encode("utf8"), doc2r)
# Update
doc1r["a"] = "BUZZ"
doc1rc = doc1r.copy()
self.os.update_doc(doc1r)
with self.assertRaises(Conflict):
doc1rc["a"] = "ZAMM"
self.os.update_doc(doc1rc)
doc2r["a"] = u"BUZZ\u20ac"
doc2r[u"h\u20ac".encode("utf8")] = u"ZAMM\u20ac"
doc3r = self.os.read_doc(did3)
doc3r["a"] = u"BUZZ\u20ac"
self.os.update_doc_mult([doc2r, doc3r])
# Delete
self.os.delete_doc(did1)
self.os.delete_doc(did2)
self.os.delete_doc(did3)
self.os.delete_doc_mult([did4, did5] )
with self.assertRaises(NotFound):
self.os.read_doc(did1)
with self.assertRaises(NotFound):
self.os.read_doc(did2)
with self.assertRaises(NotFound):
self.os.read_doc(did3)
with self.assertRaises(NotFound):
self.os.read_doc(did4)
with self.assertRaises(NotFound):
self.os.read_doc(did5)
def test_objstore_obj(self):
# Create
doc1 = Resource(name="String", alt_ids=["Some", "More"],
addl=dict(x=[1,2,3], y={}, z="Str"))
doc2 = Resource(name=u"String\u20ac", alt_ids=[u"Some\u20ac", "More"],
addl=dict(x=[1,2,3], y={}, z="Str"))
doc3_dict = doc2.__dict__.copy()
doc3_dict.pop("type_")
doc3 = Resource(**doc3_dict)
doc4_dict = doc1.__dict__.copy()
doc4_dict.pop("type_")
doc4 = Resource(**doc4_dict)
doc5_dict = doc2.__dict__.copy()
doc5_dict.pop("type_")
doc5 = Resource(**doc5_dict)
did1, dv1 = self.os.create(doc1)
self.assertTrue(did1)
did2, dv2 = self.os.create(doc2)
self.assertTrue(did2)
did3n = create_simple_unique_id()
did3, dv3 = self.os.create(doc3, object_id=did3n)
self.assertEquals(did3, did3n)
did4n, did5n = create_simple_unique_id(), create_simple_unique_id()
res = self.os.create_mult([doc4, doc5], object_ids=[did4n, did5n])
_, did4, dv4 = res[0]
_, did5, dv5 = res[1]
# Read
all_doc_ids = [did1, did2, did3, did4, did5]
docs = self.os.read_mult(all_doc_ids)
self.assertEquals(len(docs), len(all_doc_ids))
with self.assertRaises(NotFound):
self.os.read_mult([did1, "NONEXISTING", did2])
docs1 = self.os.read_mult([did1, "NONEXISTING", did2], strict=False)
self.assertEquals(len(docs1), 3)
self.assertEquals(docs1[1], None)
docs2 = self.os.read_doc_mult([did1, "NONEXISTING", did2], strict=False)
self.assertEquals(len(docs2), 3)
self.assertEquals(docs2[1], None)
doc1r = self.os.read(did1)
self.assertIsInstance(doc1r, Resource)
self.assertEquals(doc1r.addl["x"][1], 2)
doc2r = self.os.read(did2)
self.assertIsInstance(doc2r, Resource)
self.assertEquals(type(doc2r.name), str)
self.assertEquals(doc2r.name, u"String\u20ac".encode("utf8"))
# Update
doc1r.name = "BUZZ"
doc1rc_dict = doc1r.__dict__.copy()
doc1rc_dict.pop("type_")
d1rv = doc1rc_dict.pop("_rev")
d1id = doc1rc_dict.pop("_id")
doc1rc = Resource(**doc1rc_dict)
doc1rc["_rev"] = d1rv
doc1rc["_id"] = d1id
self.os.update(doc1r)
with self.assertRaises(Conflict):
doc1rc.name = "ZAMM"
self.os.update(doc1rc)
doc2r.name = u"BUZZ\u20ac"
doc3r = self.os.read(did3)
doc3r.name = u"BUZZ\u20ac"
self.os.update_mult([doc2r, doc3r])
# Delete
self.os.delete(did1)
self.os.delete(did2)
self.os.delete(did3)
self.os.delete_mult([did4, did5] )
with self.assertRaises(NotFound):
self.os.read(did1)
with self.assertRaises(NotFound):
self.os.read(did2)
with self.assertRaises(NotFound):
self.os.read(did3)
with self.assertRaises(NotFound):
self.os.read(did4)
with self.assertRaises(NotFound):
self.os.read(did5)
def __init__(self,
xp_name=None,
event_type=None,
origin=None,
queue_name=None,
sub_type=None,
origin_type=None,
pattern=None,
auto_delete=None):
self._events_xp = CFG.get_safe("exchange.core.events",
DEFAULT_EVENTS_XP)
self.event_type = event_type
self.sub_type = sub_type
self.origin_type = origin_type
self.origin = origin
# Default for auto_delete is True for events, unless otherwise specified
if auto_delete is None:
auto_delete = True
self._auto_delete = auto_delete
xp_name = xp_name or self._events_xp
if pattern:
binding = pattern
else:
binding = self._topic(event_type, origin, sub_type, origin_type)
# create queue_name if none passed in
if queue_name is None:
queue_name = "subsc_" + create_simple_unique_id()
# prepend proc name to queue name if we have one
if hasattr(self, "_process") and self._process:
queue_name = "%s_%s" % (self._process._proc_name, queue_name)
# do we have a container/ex_manager?
container = (hasattr(self, '_process') and hasattr(
self._process, 'container') and self._process.container
) or BaseEndpoint._get_container_instance()
if container:
xp = container.create_xp(xp_name)
xne = container.create_event_xn(queue_name,
pattern=binding,
xp=xp,
auto_delete=auto_delete)
self._ev_recv_name = xne
self.binding = None
else:
# Remove this case. No container??
self.binding = binding
# prefix the queue_name, if specified, with the sysname
queue_name = "%s.system.%s" % (bootstrap.get_sys_name(),
queue_name)
# set this name to be picked up by inherited folks
self._ev_recv_name = (xp_name, queue_name)
local_event_queues.append(queue_name)
def test_objstore_obj(self):
# Create
doc1 = Resource(name="String",
alt_ids=["Some", "More"],
addl=dict(x=[1, 2, 3], y={}, z="Str"))
doc2 = Resource(name=u"String\u20ac",
alt_ids=[u"Some\u20ac", "More"],
addl=dict(x=[1, 2, 3], y={}, z="Str"))
doc3_dict = doc2.__dict__.copy()
doc3_dict.pop("type_")
doc3 = Resource(**doc3_dict)
doc4_dict = doc1.__dict__.copy()
doc4_dict.pop("type_")
doc4 = Resource(**doc4_dict)
doc5_dict = doc2.__dict__.copy()
doc5_dict.pop("type_")
doc5 = Resource(**doc5_dict)
did1, dv1 = self.os.create(doc1)
self.assertTrue(did1)
did2, dv2 = self.os.create(doc2)
self.assertTrue(did2)
did3n = create_simple_unique_id()
did3, dv3 = self.os.create(doc3, object_id=did3n)
self.assertEquals(did3, did3n)
did4n, did5n = create_simple_unique_id(), create_simple_unique_id()
res = self.os.create_mult([doc4, doc5], object_ids=[did4n, did5n])
_, did4, dv4 = res[0]
_, did5, dv5 = res[1]
# Read
all_doc_ids = [did1, did2, did3, did4, did5]
docs = self.os.read_mult(all_doc_ids)
self.assertEquals(len(docs), len(all_doc_ids))
with self.assertRaises(NotFound):
self.os.read_mult([did1, "NONEXISTING", did2])
docs1 = self.os.read_mult([did1, "NONEXISTING", did2], strict=False)
self.assertEquals(len(docs1), 3)
self.assertEquals(docs1[1], None)
docs2 = self.os.read_doc_mult([did1, "NONEXISTING", did2],
strict=False)
self.assertEquals(len(docs2), 3)
self.assertEquals(docs2[1], None)
doc1r = self.os.read(did1)
self.assertIsInstance(doc1r, Resource)
self.assertEquals(doc1r.addl["x"][1], 2)
doc2r = self.os.read(did2)
self.assertIsInstance(doc2r, Resource)
self.assertEquals(type(doc2r.name), str)
self.assertEquals(doc2r.name, u"String\u20ac".encode("utf8"))
# Update
doc1r.name = "BUZZ"
doc1rc_dict = doc1r.__dict__.copy()
doc1rc_dict.pop("type_")
d1rv = doc1rc_dict.pop("_rev")
d1id = doc1rc_dict.pop("_id")
doc1rc = Resource(**doc1rc_dict)
doc1rc["_rev"] = d1rv
doc1rc["_id"] = d1id
self.os.update(doc1r)
with self.assertRaises(Conflict):
doc1rc.name = "ZAMM"
self.os.update(doc1rc)
doc2r.name = u"BUZZ\u20ac"
doc3r = self.os.read(did3)
doc3r.name = u"BUZZ\u20ac"
self.os.update_mult([doc2r, doc3r])
# Delete
self.os.delete(did1)
self.os.delete(did2)
self.os.delete(did3)
self.os.delete_mult([did4, did5])
with self.assertRaises(NotFound):
self.os.read(did1)
with self.assertRaises(NotFound):
self.os.read(did2)
with self.assertRaises(NotFound):
self.os.read(did3)
with self.assertRaises(NotFound):
self.os.read(did4)
with self.assertRaises(NotFound):
self.os.read(did5)
