def test_qc_attachment(self):
        instrument_device = InstrumentDevice(name='whatever')
        instrument_device_id,_ = self.rrclient.create(instrument_device)
        self.addCleanup(self.rrclient.delete, instrument_device_id)
        self.client.register_instrument(instrument_device_id)
        self.addCleanup(self.client.unregister_instrument, instrument_device_id)
        dp = DataProduct(name='instrument output')

        dp_id,_ = self.rrclient.create(dp)
        self.addCleanup(self.rrclient.delete, dp_id)

        parser_id = self.make_grt_parser()
        attachment = Attachment(name='qc ref', attachment_type=AttachmentType.REFERENCE,content=global_range_test_document, context=ReferenceAttachmentContext(parser_id=parser_id))
        att_id = self.rrclient.create_attachment(dp_id, attachment)
        self.addCleanup(self.rrclient.delete_attachment, att_id)

        attachment2 = Attachment(name='qc ref2', attachment_type=AttachmentType.REFERENCE, content=global_range_test_document2, context=ReferenceAttachmentContext(parser_id=parser_id))
        att2_id = self.rrclient.create_attachment(dp_id, attachment2)
        self.addCleanup(self.rrclient.delete_attachment, att2_id)

        self.client.assign_data_product(instrument_device_id, dp_id)
        self.addCleanup(self.client.unassign_data_product, instrument_device_id, dp_id)
        svm = StoredValueManager(self.container)
        doc = svm.read_value('grt_CE01ISSM-MF005-01-CTDBPC999_TEMPWAT')
        np.testing.assert_array_almost_equal(doc['grt_min_value'], -2.)
    def test_execute_acquire_data_with_stream_id_new_not_already_acquiring(self, mock):
        self._bdh._semaphore = Mock()
        self._bdh._semaphore.acquire.return_value = True
        attachment = Attachment()
        attachment.keywords = ['NewDataCheck', 'NotFound']
        attachment2 = Attachment()
        attachment2.keywords = ['NotTheRightKeyword']
        self._rr_cli.find_objects.return_value = [attachment2, attachment], ''
        self._bdh._glet_queue = Mock()
        self._bdh.execute_acquire_data({'stream_id' : 'test_stream_id'})

        self._bdh._semaphore.acquire.assert_called_once_with(blocking=False)
        self._rr_cli.find_objects.assert_called_once()
    def test_execute_acquire_data_with_stream_id_new_not_already_acquiring(self, mock):
        self._bdh._semaphore = Mock()
        self._bdh._semaphore.acquire.return_value = True
        self._bdh._glet_queue = Mock()
        attachment = Attachment()
        attachment.keywords = ["NewDataCheck", "NotFound"]
        attachment2 = Attachment()
        attachment2.keywords = ["NotTheRightKeyword"]
        self._bdh._find_new_data_check_attachment = Mock()
        self._bdh._find_new_data_check_attachment.return_value = [attachment2, attachment]
        self._bdh.execute_acquire_data({"stream_id": "test_stream_id"})

        self._bdh._semaphore.acquire.assert_called_once_with(blocking=False)
        self._bdh._find_new_data_check_attachment.assert_called_once()
    def _do_test_attach(self):
        binary = "\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x10\x00\x00\x00\x10\x08\x03\x00\x00\x00(-\x0fS\x00\x00\x00\x03sBIT\x08\x08\x08\xdb\xe1O\xe0\x00\x00\x00~PLTEf3\x00\xfc\xf7\xe0\xee\xcc\x00\xd3\xa0\x00\xcc\x99\x00\xec\xcdc\x9fl\x00\xdd\xb2\x00\xff\xff\xff|I\x00\xf9\xdb\x00\xdd\xb5\x19\xd9\xad\x10\xb6\x83\x00\xf8\xd6\x00\xf2\xc5\x00\xd8\xab\x00n;\x00\xff\xcc\x00\xd6\xa4\t\xeb\xb8\x00\x83Q\x00\xadz\x00\xff\xde\x00\xff\xd6\x00\xd6\xa3\x00\xdf\xaf\x00\xde\xad\x10\xbc\x8e\x00\xec\xbe\x00\xec\xd4d\xff\xe3\x00tA\x00\xf6\xc4\x00\xf6\xce\x00\xa5u\x00\xde\xa5\x00\xf7\xbd\x00\xd6\xad\x08\xdd\xaf\x19\x8cR\x00\xea\xb7\x00\xee\xe9\xdf\xc5\x00\x00\x00\tpHYs\x00\x00\n\xf0\x00\x00\n\xf0\x01B\xac4\x98\x00\x00\x00\x1ctEXtSoftware\x00Adobe Fireworks CS4\x06\xb2\xd3\xa0\x00\x00\x00\x15tEXtCreation Time\x0029/4/09Oq\xfdE\x00\x00\x00\xadIDAT\x18\x95M\x8f\x8d\x0e\x820\x0c\x84;ZdC~f\x07\xb2\x11D\x86\x89\xe8\xfb\xbf\xa0+h\xe2\x97\\\xd2^\x93\xb6\x07:1\x9f)q\x9e\xa5\x06\xad\xd5\x13\x8b\xac,\xb3\x02\x9d\x12C\xa1-\xef;M\x08*\x19\xce\x0e?\x1a\xeb4\xcc\xd4\x0c\x831\x87V\xca\xa1\x1a\xd3\x08@\xe4\xbd\xb7\x15P;\xc8\xd4{\x91\xbf\x11\x90\xffg\xdd\x8di\xfa\xb6\x0bs2Z\xff\xe8yg2\xdc\x11T\x96\xc7\x05\xa5\xef\x96+\xa7\xa59E\xae\xe1\x84cm^1\xa6\xb3\xda\x85\xc8\xd8/\x17se\x0eN^'\x8c\xc7\x8e\x88\xa8\xf6p\x8e\xc2;\xc6.\xd0\x11.\x91o\x12\x7f\xcb\xa5\xfe\x00\x89]\x10:\xf5\x00\x0e\xbf\x00\x00\x00\x00IEND\xaeB`\x82"

        # Owner creation tests
        instrument = IonObject("InstrumentDevice", name='instrument')
        iid, _ = self.resource_registry_service.create(instrument)

        att = Attachment(content=binary, attachment_type=AttachmentType.BLOB)
        aid1 = self.resource_registry_service.create_attachment(iid, att)

        att1 = self.resource_registry_service.read_attachment(
            aid1, include_content=True)
        self.assertEquals(binary, att1.content)

        import base64
        att = Attachment(content=base64.encodestring(binary),
                         attachment_type=AttachmentType.ASCII)
        aid2 = self.resource_registry_service.create_attachment(iid, att)

        att1 = self.resource_registry_service.read_attachment(
            aid2, include_content=True)
        self.assertEquals(binary, base64.decodestring(att1.content))

        att_ids = self.resource_registry_service.find_attachments(iid,
                                                                  id_only=True)
        self.assertEquals(att_ids, [aid1, aid2])

        att_ids = self.resource_registry_service.find_attachments(
            iid, id_only=True, descending=True)
        self.assertEquals(att_ids, [aid2, aid1])

        att_ids = self.resource_registry_service.find_attachments(
            iid, id_only=True, descending=True, limit=1)
        self.assertEquals(att_ids, [aid2])

        atts = self.resource_registry_service.find_attachments(
            iid, id_only=False, include_content=True, limit=1)
        self.assertEquals(atts[0].content, binary)

        self.resource_registry_service.delete_attachment(aid1)

        att_ids = self.resource_registry_service.find_attachments(iid,
                                                                  id_only=True)
        self.assertEquals(att_ids, [aid2])

        self.resource_registry_service.delete_attachment(aid2)

        att_ids = self.resource_registry_service.find_attachments(iid,
                                                                  id_only=True)
        self.assertEquals(att_ids, [])
Exemplo n.º 5
0
    def _update_new_data_check_attachment(cls, res_id, new_content):
        """
        Update the list of data files for the external dataset resource

        @param res_id the ID of the external dataset resource
        @param new_content list of new files found on data host
        @throws InstrumentException if external dataset resource can't be found
        @retval the attachment ID
        """
        rr_cli = ResourceRegistryServiceClient()
        try:
            # Delete any attachments with the "NewDataCheck" keyword
            attachment_objs = rr_cli.find_attachments(resource_id=res_id, include_content=False, id_only=False)
            for attachment_obj in attachment_objs:
                kwds = set(attachment_obj.keywords)
                if 'NewDataCheck' in kwds:
                    log.debug('Delete NewDataCheck attachment: {0}'.format(attachment_obj._id))
                    rr_cli.delete_attachment(attachment_obj._id)
                else:
                    log.debug('Found attachment: {0}'.format(attachment_obj))

            # Create the new attachment
            att = Attachment(name='new_data_check', attachment_type=AttachmentType.ASCII, keywords=['NewDataCheck', ], content_type='text/plain', content=msgpack.packb(new_content))
            att_id = rr_cli.create_attachment(resource_id=res_id, attachment=att)

        except NotFound:
            raise InstrumentException('ExternalDatasetResource \'{0}\' not found'.format(res_id))

        return att_id
def create_attachment():

    try:
        resource_id = str(request.form.get('resource_id', ''))
        fil = request.files['file']
        content = fil.read()

        # build attachment
        attachment = Attachment(
            name=str(request.form['attachment_name']),
            description=str(request.form['attachment_description']),
            attachment_type=int(request.form['attachment_type']),
            content_type=str(request.form['attachment_content_type']),
            content=content)

        rr_client = ResourceRegistryServiceProcessClient(
            node=Container.instance.node, process=service_gateway_instance)
        ret = rr_client.create_attachment(resource_id=resource_id,
                                          attachment=attachment)

        ret_obj = {'attachment_id': ret}

        return json_response(ret_obj)

    except Exception, e:
        log.exception("Error creating attachment")
        return build_error_response(e)
    def test_execute_acquire_sample_with_stream_id_new_not_already_acquiring(self, mock):
        self._bdh._semaphore = Mock()
        self._bdh._semaphore.acquire.return_value = True
        self._bdh._glet_queue = Mock()
        attachment = Attachment()
        attachment.keywords = ['NewDataCheck', 'NotFound']
        attachment2 = Attachment()
        attachment2.keywords = ['NotTheRightKeyword']
        self._bdh._find_new_data_check_attachment = Mock()
        self._bdh._find_new_data_check_attachment.return_value =\
        [attachment2, attachment]
        stream_route = Mock(spec=StreamRoute)
        stream_route.exchange_point = sentinel.exchange_point
        stream_route.routing_key = sentinel.routing_key
        self._bdh.execute_acquire_sample({'stream_id': 'test_stream_id',
                                          'stream_route': stream_route})

        self._bdh._semaphore.acquire.assert_called_once_with(blocking=False)
        self._bdh._find_new_data_check_attachment.assert_called_once()
    def test_execute_acquire_sample_with_stream_id_new_not_already_acquiring(self, mock, StreamPublisher_mock):
        self._bdh._semaphore = Mock()
        self._bdh._semaphore.acquire.return_value = True
        self._bdh._glet_queue = Mock()
        attachment = Attachment()
        attachment.keywords = ["NewDataCheck", "NotFound"]
        attachment2 = Attachment()
        attachment2.keywords = ["NotTheRightKeyword"]
        self._bdh._find_new_data_check_attachment = Mock()
        self._bdh._find_new_data_check_attachment.return_value = [attachment2, attachment]
        stream_route = Mock(spec=StreamRoute)
        stream_route.exchange_point = sentinel.exchange_point
        stream_route.routing_key = sentinel.routing_key

        StreamPublisher_mock = MagicMock(spec=StandaloneStreamPublisher)

        self._bdh.execute_acquire_sample({"stream_id": "test_stream_id", "stream_route": stream_route})

        self._bdh._semaphore.acquire.assert_called_once_with(blocking=False)
        self._bdh._find_new_data_check_attachment.assert_called_once()
Exemplo n.º 9
0
    def test_execute_acquire_sample_with_stream_id_new_not_already_acquiring(
            self, mock, StreamPublisher_mock):
        self._bdh._semaphore = Mock()
        self._bdh._semaphore.acquire.return_value = True
        self._bdh._glet_queue = Mock()
        attachment = Attachment()
        attachment.keywords = ['NewDataCheck', 'NotFound']
        attachment2 = Attachment()
        attachment2.keywords = ['NotTheRightKeyword']
        self._bdh._find_new_data_check_attachment = Mock()
        self._bdh._find_new_data_check_attachment.return_value =\
        [attachment2, attachment]
        stream_route = Mock(spec=StreamRoute)
        stream_route.exchange_point = sentinel.exchange_point
        stream_route.routing_key = sentinel.routing_key

        StreamPublisher_mock = MagicMock(spec=StandaloneStreamPublisher)

        self._bdh.execute_acquire_sample({
            'stream_id': 'test_stream_id',
            'stream_route': stream_route
        })

        self._bdh._semaphore.acquire.assert_called_once_with(blocking=False)
        self._bdh._find_new_data_check_attachment.assert_called_once()
Exemplo n.º 10
0
def create_attachment():

    try:
        payload = request.form['payload']
        json_params = simplejson.loads(str(payload))

        ion_actor_id, expiry = get_governance_info_from_request(
            'serviceRequest', json_params)
        ion_actor_id, expiry = validate_request(ion_actor_id, expiry)
        headers = build_message_headers(ion_actor_id, expiry)

        data_params = json_params['serviceRequest']['params']
        resource_id = str(data_params.get('resource_id', ''))
        fil = request.files['file']
        content = fil.read()

        keywords = []
        keywords_str = data_params.get('keywords', '')
        if keywords_str.strip():
            keywords = [str(x.strip()) for x in keywords_str.split(',')]

        created_by = data_params.get('attachment_created_by', 'unknown user')
        modified_by = data_params.get('attachment_modified_by', 'unknown user')

        # build attachment
        attachment = Attachment(
            name=str(data_params['attachment_name']),
            description=str(data_params['attachment_description']),
            attachment_type=int(data_params['attachment_type']),
            content_type=str(data_params['attachment_content_type']),
            keywords=keywords,
            created_by=created_by,
            modified_by=modified_by,
            content=content)

        rr_client = ResourceRegistryServiceProcessClient(
            node=Container.instance.node, process=service_gateway_instance)
        ret = rr_client.create_attachment(resource_id=resource_id,
                                          attachment=attachment,
                                          headers=headers)

        return gateway_json_response(ret)

    except Exception, e:
        log.exception("Error creating attachment")
        return build_error_response(e)
Exemplo n.º 11
0
    def create_attachment(self):
        try:
            payload = request.form[GATEWAY_ARG_JSON]
            json_params = json_loads(str(payload))

            actor_id, expiry = self.get_governance_info_from_request(
                json_params)
            actor_id, expiry = self.validate_request(actor_id, expiry)
            headers = self.build_message_headers(actor_id, expiry)

            data_params = json_params[GATEWAY_ARG_PARAMS]
            resource_id = str(data_params.get("resource_id", ""))
            fil = request.files["file"]
            content = fil.read()

            keywords = []
            keywords_str = data_params.get("keywords", "")
            if keywords_str.strip():
                keywords = [str(x.strip()) for x in keywords_str.split(",")]

            created_by = data_params.get("attachment_created_by",
                                         "unknown user")
            modified_by = data_params.get("attachment_modified_by",
                                          "unknown user")

            # build attachment
            attachment = Attachment(
                name=str(data_params["attachment_name"]),
                description=str(data_params["attachment_description"]),
                attachment_type=int(data_params["attachment_type"]),
                content_type=str(data_params["attachment_content_type"]),
                keywords=keywords,
                created_by=created_by,
                modified_by=modified_by,
                content=content)

            ret = self.rr_client.create_attachment(resource_id=resource_id,
                                                   attachment=attachment,
                                                   headers=headers)

            return self.gateway_json_response(ret)

        except Exception as ex:
            log.exception("Error creating attachment")
            return self.gateway_error_response(ex)
Exemplo n.º 12
0
    def test_execute_acquire_data_with_stream_id_new_not_already_acquiring(
            self, mock):
        self._bdh._semaphore = Mock()
        self._bdh._semaphore.acquire.return_value = True
        attachment = Attachment()
        attachment.keywords = ['NewDataCheck', 'NotFound']
        attachment2 = Attachment()
        attachment2.keywords = ['NotTheRightKeyword']
        self._rr_cli.find_objects.return_value = [attachment2, attachment], ''
        self._bdh._glet_queue = Mock()
        self._bdh.execute_acquire_data({'stream_id': 'test_stream_id'})

        self._bdh._semaphore.acquire.assert_called_once_with(blocking=False)
        self._rr_cli.find_objects.assert_called_once()
Exemplo n.º 13
0
    def test_attach(self):
        binary = "\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x10\x00\x00\x00\x10\x08\x03\x00\x00\x00(-\x0fS\x00\x00\x00\x03sBIT\x08\x08\x08\xdb\xe1O\xe0\x00\x00\x00~PLTEf3\x00\xfc\xf7\xe0\xee\xcc\x00\xd3\xa0\x00\xcc\x99\x00\xec\xcdc\x9fl\x00\xdd\xb2\x00\xff\xff\xff|I\x00\xf9\xdb\x00\xdd\xb5\x19\xd9\xad\x10\xb6\x83\x00\xf8\xd6\x00\xf2\xc5\x00\xd8\xab\x00n;\x00\xff\xcc\x00\xd6\xa4\t\xeb\xb8\x00\x83Q\x00\xadz\x00\xff\xde\x00\xff\xd6\x00\xd6\xa3\x00\xdf\xaf\x00\xde\xad\x10\xbc\x8e\x00\xec\xbe\x00\xec\xd4d\xff\xe3\x00tA\x00\xf6\xc4\x00\xf6\xce\x00\xa5u\x00\xde\xa5\x00\xf7\xbd\x00\xd6\xad\x08\xdd\xaf\x19\x8cR\x00\xea\xb7\x00\xee\xe9\xdf\xc5\x00\x00\x00\tpHYs\x00\x00\n\xf0\x00\x00\n\xf0\x01B\xac4\x98\x00\x00\x00\x1ctEXtSoftware\x00Adobe Fireworks CS4\x06\xb2\xd3\xa0\x00\x00\x00\x15tEXtCreation Time\x0029/4/09Oq\xfdE\x00\x00\x00\xadIDAT\x18\x95M\x8f\x8d\x0e\x820\x0c\x84;ZdC~f\x07\xb2\x11D\x86\x89\xe8\xfb\xbf\xa0+h\xe2\x97\\\xd2^\x93\xb6\x07:1\x9f)q\x9e\xa5\x06\xad\xd5\x13\x8b\xac,\xb3\x02\x9d\x12C\xa1-\xef;M\x08*\x19\xce\x0e?\x1a\xeb4\xcc\xd4\x0c\x831\x87V\xca\xa1\x1a\xd3\x08@\xe4\xbd\xb7\x15P;\xc8\xd4{\x91\xbf\x11\x90\xffg\xdd\x8di\xfa\xb6\x0bs2Z\xff\xe8yg2\xdc\x11T\x96\xc7\x05\xa5\xef\x96+\xa7\xa59E\xae\xe1\x84cm^1\xa6\xb3\xda\x85\xc8\xd8/\x17se\x0eN^'\x8c\xc7\x8e\x88\xa8\xf6p\x8e\xc2;\xc6.\xd0\x11.\x91o\x12\x7f\xcb\xa5\xfe\x00\x89]\x10:\xf5\x00\x0e\xbf\x00\x00\x00\x00IEND\xaeB`\x82"

        # Owner creation tests
        instrument = IonObject("InstrumentDevice", name='instrument')
        iid, _ = self.rr.create(instrument)

        att = Attachment(content=binary, attachment_type=AttachmentType.BLOB)
        aid1 = self.rr.create_attachment(iid, att)

        att1 = self.rr.read_attachment(aid1, include_content=True)
        self.assertEquals(binary, att1.content)
        self.assertEquals(len(binary), att1.attachment_size)

        import base64
        enc_content = base64.encodestring(binary)
        att = Attachment(content=enc_content, attachment_type=AttachmentType.ASCII)
        aid2 = self.rr.create_attachment(iid, att)

        # test that attachments are without content by default
        att1 = self.rr.read_attachment(aid2)
        self.assertEquals(len(enc_content), att1.attachment_size)
        self.assertEquals(att1.content, '')

        # tests that the attachment content can be read
        att1 = self.rr.read_attachment(aid2, include_content=True)
        self.assertEquals(enc_content, att1.content)

        att_ids = self.rr.find_attachments(iid, id_only=True)
        self.assertEquals(att_ids, [aid1, aid2])

        att_ids = self.rr.find_attachments(iid, id_only=True, descending=True)
        self.assertEquals(att_ids, [aid2, aid1])

        att_ids = self.rr.find_attachments(iid, id_only=True, descending=True, limit=1)
        self.assertEquals(att_ids, [aid2])

        # test that content can be included
        atts = self.rr.find_attachments(iid, id_only=False, include_content=True, limit=1)
        self.assertEquals(atts[0].content, binary)

        # test that content can be excluded and is the default
        atts = self.rr.find_attachments(iid, id_only=False, limit=1)
        self.assertEquals(atts[0].content, '')

        self.rr.delete_attachment(aid1)

        att_ids = self.rr.find_attachments(iid, id_only=True)
        self.assertEquals(att_ids, [aid2])

        self.rr.delete_attachment(aid2)

        att_ids = self.rr.find_attachments(iid, id_only=True)
        self.assertEquals(att_ids, [])

        att = Attachment(content="SOME TEXT", attachment_type=AttachmentType.ASCII,
                         keywords=['BAR', 'FOO'])
        aid3 = self.rr.create_attachment(iid, att)

        att_ids = self.rr.find_attachments(iid, keyword="NONE", id_only=True)
        self.assertEquals(att_ids, [])

        att_ids = self.rr.find_attachments(iid, keyword="FOO", id_only=True)
        self.assertEquals(att_ids, [aid3])

        att_objs = self.rr.find_attachments(iid, keyword="FOO", id_only=False, include_content=True)
        self.assertEquals(len(att_objs), 1)
        self.assertEquals(att_objs[0].content, "SOME TEXT")

        # tests that attachments can be retrieved without content
        att_objs_without_content = self.rr.find_attachments(iid, keyword="FOO", id_only=False,
                                                            include_content=False)
        self.assertEquals(len(att_objs_without_content), 1)
        self.assertEquals(att_objs_without_content[0].content, '')