def test_qc_attachment(self):
instrument_device = InstrumentDevice(name='whatever')
instrument_device_id,_ = self.rrclient.create(instrument_device)
self.addCleanup(self.rrclient.delete, instrument_device_id)
self.client.register_instrument(instrument_device_id)
self.addCleanup(self.client.unregister_instrument, instrument_device_id)
dp = DataProduct(name='instrument output')
dp_id,_ = self.rrclient.create(dp)
self.addCleanup(self.rrclient.delete, dp_id)
parser_id = self.make_grt_parser()
attachment = Attachment(name='qc ref', attachment_type=AttachmentType.REFERENCE,content=global_range_test_document, context=ReferenceAttachmentContext(parser_id=parser_id))
att_id = self.rrclient.create_attachment(dp_id, attachment)
self.addCleanup(self.rrclient.delete_attachment, att_id)
attachment2 = Attachment(name='qc ref2', attachment_type=AttachmentType.REFERENCE, content=global_range_test_document2, context=ReferenceAttachmentContext(parser_id=parser_id))
att2_id = self.rrclient.create_attachment(dp_id, attachment2)
self.addCleanup(self.rrclient.delete_attachment, att2_id)
self.client.assign_data_product(instrument_device_id, dp_id)
self.addCleanup(self.client.unassign_data_product, instrument_device_id, dp_id)
svm = StoredValueManager(self.container)
doc = svm.read_value('grt_CE01ISSM-MF005-01-CTDBPC999_TEMPWAT')
np.testing.assert_array_almost_equal(doc['grt_min_value'], -2.)
def test_execute_acquire_data_with_stream_id_new_not_already_acquiring(self, mock):
self._bdh._semaphore = Mock()
self._bdh._semaphore.acquire.return_value = True
attachment = Attachment()
attachment.keywords = ['NewDataCheck', 'NotFound']
attachment2 = Attachment()
attachment2.keywords = ['NotTheRightKeyword']
self._rr_cli.find_objects.return_value = [attachment2, attachment], ''
self._bdh._glet_queue = Mock()
self._bdh.execute_acquire_data({'stream_id' : 'test_stream_id'})
self._bdh._semaphore.acquire.assert_called_once_with(blocking=False)
self._rr_cli.find_objects.assert_called_once()
def test_execute_acquire_data_with_stream_id_new_not_already_acquiring(self, mock):
self._bdh._semaphore = Mock()
self._bdh._semaphore.acquire.return_value = True
self._bdh._glet_queue = Mock()
attachment = Attachment()
attachment.keywords = ["NewDataCheck", "NotFound"]
attachment2 = Attachment()
attachment2.keywords = ["NotTheRightKeyword"]
self._bdh._find_new_data_check_attachment = Mock()
self._bdh._find_new_data_check_attachment.return_value = [attachment2, attachment]
self._bdh.execute_acquire_data({"stream_id": "test_stream_id"})
self._bdh._semaphore.acquire.assert_called_once_with(blocking=False)
self._bdh._find_new_data_check_attachment.assert_called_once()
def _do_test_attach(self):
binary = "\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x10\x00\x00\x00\x10\x08\x03\x00\x00\x00(-\x0fS\x00\x00\x00\x03sBIT\x08\x08\x08\xdb\xe1O\xe0\x00\x00\x00~PLTEf3\x00\xfc\xf7\xe0\xee\xcc\x00\xd3\xa0\x00\xcc\x99\x00\xec\xcdc\x9fl\x00\xdd\xb2\x00\xff\xff\xff|I\x00\xf9\xdb\x00\xdd\xb5\x19\xd9\xad\x10\xb6\x83\x00\xf8\xd6\x00\xf2\xc5\x00\xd8\xab\x00n;\x00\xff\xcc\x00\xd6\xa4\t\xeb\xb8\x00\x83Q\x00\xadz\x00\xff\xde\x00\xff\xd6\x00\xd6\xa3\x00\xdf\xaf\x00\xde\xad\x10\xbc\x8e\x00\xec\xbe\x00\xec\xd4d\xff\xe3\x00tA\x00\xf6\xc4\x00\xf6\xce\x00\xa5u\x00\xde\xa5\x00\xf7\xbd\x00\xd6\xad\x08\xdd\xaf\x19\x8cR\x00\xea\xb7\x00\xee\xe9\xdf\xc5\x00\x00\x00\tpHYs\x00\x00\n\xf0\x00\x00\n\xf0\x01B\xac4\x98\x00\x00\x00\x1ctEXtSoftware\x00Adobe Fireworks CS4\x06\xb2\xd3\xa0\x00\x00\x00\x15tEXtCreation Time\x0029/4/09Oq\xfdE\x00\x00\x00\xadIDAT\x18\x95M\x8f\x8d\x0e\x820\x0c\x84;ZdC~f\x07\xb2\x11D\x86\x89\xe8\xfb\xbf\xa0+h\xe2\x97\\\xd2^\x93\xb6\x07:1\x9f)q\x9e\xa5\x06\xad\xd5\x13\x8b\xac,\xb3\x02\x9d\x12C\xa1-\xef;M\x08*\x19\xce\x0e?\x1a\xeb4\xcc\xd4\x0c\x831\x87V\xca\xa1\x1a\xd3\x08@\xe4\xbd\xb7\x15P;\xc8\xd4{\x91\xbf\x11\x90\xffg\xdd\x8di\xfa\xb6\x0bs2Z\xff\xe8yg2\xdc\x11T\x96\xc7\x05\xa5\xef\x96+\xa7\xa59E\xae\xe1\x84cm^1\xa6\xb3\xda\x85\xc8\xd8/\x17se\x0eN^'\x8c\xc7\x8e\x88\xa8\xf6p\x8e\xc2;\xc6.\xd0\x11.\x91o\x12\x7f\xcb\xa5\xfe\x00\x89]\x10:\xf5\x00\x0e\xbf\x00\x00\x00\x00IEND\xaeB`\x82"
# Owner creation tests
instrument = IonObject("InstrumentDevice", name='instrument')
iid, _ = self.resource_registry_service.create(instrument)
att = Attachment(content=binary, attachment_type=AttachmentType.BLOB)
aid1 = self.resource_registry_service.create_attachment(iid, att)
att1 = self.resource_registry_service.read_attachment(
aid1, include_content=True)
self.assertEquals(binary, att1.content)
import base64
att = Attachment(content=base64.encodestring(binary),
attachment_type=AttachmentType.ASCII)
aid2 = self.resource_registry_service.create_attachment(iid, att)
att1 = self.resource_registry_service.read_attachment(
aid2, include_content=True)
self.assertEquals(binary, base64.decodestring(att1.content))
att_ids = self.resource_registry_service.find_attachments(iid,
id_only=True)
self.assertEquals(att_ids, [aid1, aid2])
att_ids = self.resource_registry_service.find_attachments(
iid, id_only=True, descending=True)
self.assertEquals(att_ids, [aid2, aid1])
att_ids = self.resource_registry_service.find_attachments(
iid, id_only=True, descending=True, limit=1)
self.assertEquals(att_ids, [aid2])
atts = self.resource_registry_service.find_attachments(
iid, id_only=False, include_content=True, limit=1)
self.assertEquals(atts[0].content, binary)
self.resource_registry_service.delete_attachment(aid1)
att_ids = self.resource_registry_service.find_attachments(iid,
id_only=True)
self.assertEquals(att_ids, [aid2])
self.resource_registry_service.delete_attachment(aid2)
att_ids = self.resource_registry_service.find_attachments(iid,
id_only=True)
self.assertEquals(att_ids, [])
def _update_new_data_check_attachment(cls, res_id, new_content):
"""
Update the list of data files for the external dataset resource
@param res_id the ID of the external dataset resource
@param new_content list of new files found on data host
@throws InstrumentException if external dataset resource can't be found
@retval the attachment ID
"""
rr_cli = ResourceRegistryServiceClient()
try:
# Delete any attachments with the "NewDataCheck" keyword
attachment_objs = rr_cli.find_attachments(resource_id=res_id, include_content=False, id_only=False)
for attachment_obj in attachment_objs:
kwds = set(attachment_obj.keywords)
if 'NewDataCheck' in kwds:
log.debug('Delete NewDataCheck attachment: {0}'.format(attachment_obj._id))
rr_cli.delete_attachment(attachment_obj._id)
else:
log.debug('Found attachment: {0}'.format(attachment_obj))
# Create the new attachment
att = Attachment(name='new_data_check', attachment_type=AttachmentType.ASCII, keywords=['NewDataCheck', ], content_type='text/plain', content=msgpack.packb(new_content))
att_id = rr_cli.create_attachment(resource_id=res_id, attachment=att)
except NotFound:
raise InstrumentException('ExternalDatasetResource \'{0}\' not found'.format(res_id))
return att_id
def create_attachment():
try:
resource_id = str(request.form.get('resource_id', ''))
fil = request.files['file']
content = fil.read()
# build attachment
attachment = Attachment(
name=str(request.form['attachment_name']),
description=str(request.form['attachment_description']),
attachment_type=int(request.form['attachment_type']),
content_type=str(request.form['attachment_content_type']),
content=content)
rr_client = ResourceRegistryServiceProcessClient(
node=Container.instance.node, process=service_gateway_instance)
ret = rr_client.create_attachment(resource_id=resource_id,
attachment=attachment)
ret_obj = {'attachment_id': ret}
return json_response(ret_obj)
except Exception, e:
log.exception("Error creating attachment")
return build_error_response(e)
def test_execute_acquire_sample_with_stream_id_new_not_already_acquiring(self, mock):
self._bdh._semaphore = Mock()
self._bdh._semaphore.acquire.return_value = True
self._bdh._glet_queue = Mock()
attachment = Attachment()
attachment.keywords = ['NewDataCheck', 'NotFound']
attachment2 = Attachment()
attachment2.keywords = ['NotTheRightKeyword']
self._bdh._find_new_data_check_attachment = Mock()
self._bdh._find_new_data_check_attachment.return_value =\
[attachment2, attachment]
stream_route = Mock(spec=StreamRoute)
stream_route.exchange_point = sentinel.exchange_point
stream_route.routing_key = sentinel.routing_key
self._bdh.execute_acquire_sample({'stream_id': 'test_stream_id',
'stream_route': stream_route})
self._bdh._semaphore.acquire.assert_called_once_with(blocking=False)
self._bdh._find_new_data_check_attachment.assert_called_once()
def test_execute_acquire_sample_with_stream_id_new_not_already_acquiring(self, mock, StreamPublisher_mock):
self._bdh._semaphore = Mock()
self._bdh._semaphore.acquire.return_value = True
self._bdh._glet_queue = Mock()
attachment = Attachment()
attachment.keywords = ["NewDataCheck", "NotFound"]
attachment2 = Attachment()
attachment2.keywords = ["NotTheRightKeyword"]
self._bdh._find_new_data_check_attachment = Mock()
self._bdh._find_new_data_check_attachment.return_value = [attachment2, attachment]
stream_route = Mock(spec=StreamRoute)
stream_route.exchange_point = sentinel.exchange_point
stream_route.routing_key = sentinel.routing_key
StreamPublisher_mock = MagicMock(spec=StandaloneStreamPublisher)
self._bdh.execute_acquire_sample({"stream_id": "test_stream_id", "stream_route": stream_route})
self._bdh._semaphore.acquire.assert_called_once_with(blocking=False)
self._bdh._find_new_data_check_attachment.assert_called_once()
def test_execute_acquire_sample_with_stream_id_new_not_already_acquiring(
self, mock, StreamPublisher_mock):
self._bdh._semaphore = Mock()
self._bdh._semaphore.acquire.return_value = True
self._bdh._glet_queue = Mock()
attachment = Attachment()
attachment.keywords = ['NewDataCheck', 'NotFound']
attachment2 = Attachment()
attachment2.keywords = ['NotTheRightKeyword']
self._bdh._find_new_data_check_attachment = Mock()
self._bdh._find_new_data_check_attachment.return_value =\
[attachment2, attachment]
stream_route = Mock(spec=StreamRoute)
stream_route.exchange_point = sentinel.exchange_point
stream_route.routing_key = sentinel.routing_key
StreamPublisher_mock = MagicMock(spec=StandaloneStreamPublisher)
self._bdh.execute_acquire_sample({
'stream_id': 'test_stream_id',
'stream_route': stream_route
})
self._bdh._semaphore.acquire.assert_called_once_with(blocking=False)
self._bdh._find_new_data_check_attachment.assert_called_once()
def create_attachment():
try:
payload = request.form['payload']
json_params = simplejson.loads(str(payload))
ion_actor_id, expiry = get_governance_info_from_request(
'serviceRequest', json_params)
ion_actor_id, expiry = validate_request(ion_actor_id, expiry)
headers = build_message_headers(ion_actor_id, expiry)
data_params = json_params['serviceRequest']['params']
resource_id = str(data_params.get('resource_id', ''))
fil = request.files['file']
content = fil.read()
keywords = []
keywords_str = data_params.get('keywords', '')
if keywords_str.strip():
keywords = [str(x.strip()) for x in keywords_str.split(',')]
created_by = data_params.get('attachment_created_by', 'unknown user')
modified_by = data_params.get('attachment_modified_by', 'unknown user')
# build attachment
attachment = Attachment(
name=str(data_params['attachment_name']),
description=str(data_params['attachment_description']),
attachment_type=int(data_params['attachment_type']),
content_type=str(data_params['attachment_content_type']),
keywords=keywords,
created_by=created_by,
modified_by=modified_by,
content=content)
rr_client = ResourceRegistryServiceProcessClient(
node=Container.instance.node, process=service_gateway_instance)
ret = rr_client.create_attachment(resource_id=resource_id,
attachment=attachment,
headers=headers)
return gateway_json_response(ret)
except Exception, e:
log.exception("Error creating attachment")
return build_error_response(e)
def create_attachment(self):
try:
payload = request.form[GATEWAY_ARG_JSON]
json_params = json_loads(str(payload))
actor_id, expiry = self.get_governance_info_from_request(
json_params)
actor_id, expiry = self.validate_request(actor_id, expiry)
headers = self.build_message_headers(actor_id, expiry)
data_params = json_params[GATEWAY_ARG_PARAMS]
resource_id = str(data_params.get("resource_id", ""))
fil = request.files["file"]
content = fil.read()
keywords = []
keywords_str = data_params.get("keywords", "")
if keywords_str.strip():
keywords = [str(x.strip()) for x in keywords_str.split(",")]
created_by = data_params.get("attachment_created_by",
"unknown user")
modified_by = data_params.get("attachment_modified_by",
"unknown user")
# build attachment
attachment = Attachment(
name=str(data_params["attachment_name"]),
description=str(data_params["attachment_description"]),
attachment_type=int(data_params["attachment_type"]),
content_type=str(data_params["attachment_content_type"]),
keywords=keywords,
created_by=created_by,
modified_by=modified_by,
content=content)
ret = self.rr_client.create_attachment(resource_id=resource_id,
attachment=attachment,
headers=headers)
return self.gateway_json_response(ret)
except Exception as ex:
log.exception("Error creating attachment")
return self.gateway_error_response(ex)
def test_execute_acquire_data_with_stream_id_new_not_already_acquiring(
self, mock):
self._bdh._semaphore = Mock()
self._bdh._semaphore.acquire.return_value = True
attachment = Attachment()
attachment.keywords = ['NewDataCheck', 'NotFound']
attachment2 = Attachment()
attachment2.keywords = ['NotTheRightKeyword']
self._rr_cli.find_objects.return_value = [attachment2, attachment], ''
self._bdh._glet_queue = Mock()
self._bdh.execute_acquire_data({'stream_id': 'test_stream_id'})
self._bdh._semaphore.acquire.assert_called_once_with(blocking=False)
self._rr_cli.find_objects.assert_called_once()
def test_attach(self):
binary = "\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x10\x00\x00\x00\x10\x08\x03\x00\x00\x00(-\x0fS\x00\x00\x00\x03sBIT\x08\x08\x08\xdb\xe1O\xe0\x00\x00\x00~PLTEf3\x00\xfc\xf7\xe0\xee\xcc\x00\xd3\xa0\x00\xcc\x99\x00\xec\xcdc\x9fl\x00\xdd\xb2\x00\xff\xff\xff|I\x00\xf9\xdb\x00\xdd\xb5\x19\xd9\xad\x10\xb6\x83\x00\xf8\xd6\x00\xf2\xc5\x00\xd8\xab\x00n;\x00\xff\xcc\x00\xd6\xa4\t\xeb\xb8\x00\x83Q\x00\xadz\x00\xff\xde\x00\xff\xd6\x00\xd6\xa3\x00\xdf\xaf\x00\xde\xad\x10\xbc\x8e\x00\xec\xbe\x00\xec\xd4d\xff\xe3\x00tA\x00\xf6\xc4\x00\xf6\xce\x00\xa5u\x00\xde\xa5\x00\xf7\xbd\x00\xd6\xad\x08\xdd\xaf\x19\x8cR\x00\xea\xb7\x00\xee\xe9\xdf\xc5\x00\x00\x00\tpHYs\x00\x00\n\xf0\x00\x00\n\xf0\x01B\xac4\x98\x00\x00\x00\x1ctEXtSoftware\x00Adobe Fireworks CS4\x06\xb2\xd3\xa0\x00\x00\x00\x15tEXtCreation Time\x0029/4/09Oq\xfdE\x00\x00\x00\xadIDAT\x18\x95M\x8f\x8d\x0e\x820\x0c\x84;ZdC~f\x07\xb2\x11D\x86\x89\xe8\xfb\xbf\xa0+h\xe2\x97\\\xd2^\x93\xb6\x07:1\x9f)q\x9e\xa5\x06\xad\xd5\x13\x8b\xac,\xb3\x02\x9d\x12C\xa1-\xef;M\x08*\x19\xce\x0e?\x1a\xeb4\xcc\xd4\x0c\x831\x87V\xca\xa1\x1a\xd3\x08@\xe4\xbd\xb7\x15P;\xc8\xd4{\x91\xbf\x11\x90\xffg\xdd\x8di\xfa\xb6\x0bs2Z\xff\xe8yg2\xdc\x11T\x96\xc7\x05\xa5\xef\x96+\xa7\xa59E\xae\xe1\x84cm^1\xa6\xb3\xda\x85\xc8\xd8/\x17se\x0eN^'\x8c\xc7\x8e\x88\xa8\xf6p\x8e\xc2;\xc6.\xd0\x11.\x91o\x12\x7f\xcb\xa5\xfe\x00\x89]\x10:\xf5\x00\x0e\xbf\x00\x00\x00\x00IEND\xaeB`\x82"
# Owner creation tests
instrument = IonObject("InstrumentDevice", name='instrument')
iid, _ = self.rr.create(instrument)
att = Attachment(content=binary, attachment_type=AttachmentType.BLOB)
aid1 = self.rr.create_attachment(iid, att)
att1 = self.rr.read_attachment(aid1, include_content=True)
self.assertEquals(binary, att1.content)
self.assertEquals(len(binary), att1.attachment_size)
import base64
enc_content = base64.encodestring(binary)
att = Attachment(content=enc_content, attachment_type=AttachmentType.ASCII)
aid2 = self.rr.create_attachment(iid, att)
# test that attachments are without content by default
att1 = self.rr.read_attachment(aid2)
self.assertEquals(len(enc_content), att1.attachment_size)
self.assertEquals(att1.content, '')
# tests that the attachment content can be read
att1 = self.rr.read_attachment(aid2, include_content=True)
self.assertEquals(enc_content, att1.content)
att_ids = self.rr.find_attachments(iid, id_only=True)
self.assertEquals(att_ids, [aid1, aid2])
att_ids = self.rr.find_attachments(iid, id_only=True, descending=True)
self.assertEquals(att_ids, [aid2, aid1])
att_ids = self.rr.find_attachments(iid, id_only=True, descending=True, limit=1)
self.assertEquals(att_ids, [aid2])
# test that content can be included
atts = self.rr.find_attachments(iid, id_only=False, include_content=True, limit=1)
self.assertEquals(atts[0].content, binary)
# test that content can be excluded and is the default
atts = self.rr.find_attachments(iid, id_only=False, limit=1)
self.assertEquals(atts[0].content, '')
self.rr.delete_attachment(aid1)
att_ids = self.rr.find_attachments(iid, id_only=True)
self.assertEquals(att_ids, [aid2])
self.rr.delete_attachment(aid2)
att_ids = self.rr.find_attachments(iid, id_only=True)
self.assertEquals(att_ids, [])
att = Attachment(content="SOME TEXT", attachment_type=AttachmentType.ASCII,
keywords=['BAR', 'FOO'])
aid3 = self.rr.create_attachment(iid, att)
att_ids = self.rr.find_attachments(iid, keyword="NONE", id_only=True)
self.assertEquals(att_ids, [])
att_ids = self.rr.find_attachments(iid, keyword="FOO", id_only=True)
self.assertEquals(att_ids, [aid3])
att_objs = self.rr.find_attachments(iid, keyword="FOO", id_only=False, include_content=True)
self.assertEquals(len(att_objs), 1)
self.assertEquals(att_objs[0].content, "SOME TEXT")
# tests that attachments can be retrieved without content
att_objs_without_content = self.rr.find_attachments(iid, keyword="FOO", id_only=False,
include_content=False)
self.assertEquals(len(att_objs_without_content), 1)
self.assertEquals(att_objs_without_content[0].content, '')