Ejemplo n.º 1
0
    def test_get_project__returns_none(self):
        # given
        entity_map = EntityMap()

        # when
        output = entity_map.get_project()

        # then
        self.assertEqual(output, None)
Ejemplo n.º 2
0
    def test_get_project__returns_project(self):
        # given
        entity_map = EntityMap()
        project_entity = Entity('project', 'project_0', {})
        entity_map.add_entity(project_entity)

        # when
        output = entity_map.get_project()

        # then
        self.assertEqual(output, project_entity)
Ejemplo n.º 3
0
    def test_load(self):
        # given:
        spreadsheet_json = _create_spreadsheet_json()

        # when:
        entity_map = EntityMap.load(spreadsheet_json)

        # then:
        self.assertEqual(['project', 'biomaterial', 'file', 'protocol'],
                         list(entity_map.get_entity_types()))

        # and:
        # TODO shouldn't entity id's be unique and that there's no need to specify entity type?
        biomaterial1 = entity_map.get_entity('biomaterial', 'biomaterial_id_1')
        self._assert_correct_entity(biomaterial1,
                                    entity_id='biomaterial_id_1',
                                    entity_type='biomaterial',
                                    content={'key': 'biomaterial_1'})

        # and:
        biomaterial2 = entity_map.get_entity('biomaterial', 'biomaterial_id_2')
        links = {
            'biomaterial': ['biomaterial_id_1'],
            'process': ['process_id_1']
        }
        self._assert_correct_entity(biomaterial2,
                                    entity_id='biomaterial_id_2',
                                    entity_type='biomaterial',
                                    content={'key': 'biomaterial_2'},
                                    links=links)

        # and:
        protocol1 = entity_map.get_entity('protocol', 'protocol_id_1')
        self.assertEqual({'key': 'protocol_1'}, protocol1.content)
Ejemplo n.º 4
0
 def _create_test_entity_map(self) -> EntityMap:
     product = Entity('product', 'product_1', {'k': 'v'})
     project = Entity('project', 'id', {'k': 'v'})
     user1 = Entity('user', 'user_1', {'k': 'v'})
     user2 = Entity('user',
                    'user_2', {'k': 'v'}, {
                        'content': {
                            'k': 'v0'
                        },
                        '_links': {
                            'self': {
                                'href': 'url'
                            }
                        }
                    },
                    is_reference=True)
     user3 = Entity('user',
                    'user_3', {'k': 'v'}, {
                        'content': {
                            'k': 'v0'
                        },
                        '_links': {
                            'self': {
                                'href': 'url'
                            }
                        }
                    },
                    is_reference=True)
     entity_map = EntityMap(product, user1, user2, user3, project)
     return entity_map
Ejemplo n.º 5
0
    def test_generate_direct_links_link_not_found_error(self):
        # given
        spreadsheet_json = {
            'project': {
                'dummy-project-id': {
                    'content': {
                        'key': 'project_1'
                    }
                }
            },
            'file': {
                'file_id_1': {
                    'content': {
                        'key': 'file_1'
                    },
                    'links_by_entity': {
                        'biomaterial': ['biomaterial_id_1'],
                        'protocol': ['protocol_id_1', 'protocol_id_2']
                    }
                }
            }
        }

        entity_map = EntityMap.load(spreadsheet_json)
        entity_linker = EntityLinker(self.mocked_template_manager, entity_map)

        with self.assertRaises(LinkedEntityNotFound) as context:
            entity_linker.handle_links_from_spreadsheet()

        self.assertEqual('biomaterial', context.exception.entity)
        self.assertEqual('biomaterial_id_1', context.exception.id)
Ejemplo n.º 6
0
    def add_entities(self, entity_map: EntityMap, submission_url: str) -> Submission:
        submission = Submission(self.ingest_api, submission_url)
        submission.define_manifest(entity_map)
        for e in entity_map.get_new_entities():
            self.add_entity(e, submission_url)
            submission.add_entity(e)

        return submission
Ejemplo n.º 7
0
    def dry_run_import_file(self, file_path, project_uuid=None):
        spreadsheet_json, template_mgr, errors = self.generate_json(file_path, project_uuid)

        if errors:
            return None, errors

        entity_map = EntityMap.load(spreadsheet_json)
        entity_linker = EntityLinker(template_mgr, entity_map)
        entity_linker.handle_links_from_spreadsheet()

        return entity_map, []
Ejemplo n.º 8
0
    def link_entities(self, entity_map: EntityMap, submission: Submission):
        progress = 0
        for entity in entity_map.get_entities():
            for link in entity.direct_links:
                to_entity = entity_map.get_entity(link['entity'], link['id'])
                try:
                    self.link_entity(entity, to_entity, relationship=link['relationship'],
                                     is_collection=link.get('is_collection', True))
                    progress = progress + 1
                    expected_links = int(submission.manifest.get('expectedLinks', 0))
                    if progress % self.PROGRESS_CTR == 0 or (progress == expected_links):
                        manifest_url = self.ingest_api.get_link_from_resource(submission.manifest, 'self')
                        self.ingest_api.patch(manifest_url, {'actualLinks': progress})
                        self.logger.info(f"links progress: {progress}/ {submission.manifest.get('expectedLinks')}")

                except Exception as link_error:
                    error_message = f'''The {entity.type} with id {entity.id} could not be linked to {to_entity.type} \
                    with id {to_entity.id}.'''
                    self.logger.error(error_message)
                    self.logger.error(f'{str(link_error)}')
                    raise
Ejemplo n.º 9
0
    def test_handle_links_from_spreadsheet__with_external_links(self):
        # given
        with open(
                os.path.dirname(__file__) +
                '/spreadsheet_with_external_links.json') as file:
            spreadsheet_json = json.load(file)
        mocked_template_manager = MagicMock(name='template_manager')
        mocked_template_manager.get_schema_url = MagicMock(return_value='url')
        self.mocked_template_manager = mocked_template_manager

        entity_map = EntityMap.load(spreadsheet_json)
        entity_linker = EntityLinker(self.mocked_template_manager, entity_map)

        # when
        output = entity_linker.handle_links_from_spreadsheet()

        # then
        lib_prep_protocol = output.get_entity('protocol',
                                              'librep-protocol-uuid')
        self.assertTrue(lib_prep_protocol.is_linking_reference)
        self.assertTrue(lib_prep_protocol.is_reference)

        seq_protocol = output.get_entity('protocol', 'seq-protocol-uuid')
        self.assertTrue(seq_protocol.is_linking_reference)
        self.assertTrue(seq_protocol.is_reference)

        cell_suspension = output.get_entity('biomaterial',
                                            'cell-suspension-uuid')
        self.assertTrue(cell_suspension.is_linking_reference)
        self.assertTrue(cell_suspension.is_reference)

        file1 = output.get_entity('file', 'seq-file-uuid-1')
        self.assertFalse(file1.is_linking_reference)
        self.assertTrue(file1.is_reference)

        file2 = output.get_entity('file', 'seq-file-uuid-2')
        self.assertFalse(file2.is_linking_reference)
        self.assertTrue(file2.is_reference)

        assay_process = output.get_entity('process', 'assay_process-uuid')
        self.assertTrue(assay_process.is_linking_reference)
        self.assertTrue(assay_process.is_reference)
        assay_process_content = {
            'process_core': {
                'process_description': 'desc',
                'process_id': 'assay_process'
            },
            'schema_type': 'process',
            'describedBy': 'url'
        }
        self.assertEqual(assay_process.content, assay_process_content)
Ejemplo n.º 10
0
    def test_load__is_reference(self):
        # given:
        spreadsheet_json = {
            'biomaterial': {
                'biomaterial_uuid': {
                    'content': {
                        'key': 'value'
                    },
                    'is_reference': True
                }
            }
        }

        # when:
        entity_map = EntityMap.load(spreadsheet_json)

        # then:
        self.assertEqual(['biomaterial'], list(entity_map.get_entity_types()))
Ejemplo n.º 11
0
    def test_generate_direct_links_multiple_process_links(self):
        # given
        spreadsheet_json = {
            'project': {
                'dummy-project-id': {
                    'content': {
                        'key': 'project_1'
                    }
                }
            },
            'biomaterial': {
                'biomaterial_id_1': {
                    'content': {
                        'key': 'biomaterial_1'
                    },
                    'links_by_entity': {
                        'process': ['process_id_1', 'process_id_2']
                    }
                }
            },
            'process': {
                'process_id_1': {
                    'content': {
                        'key': 'process_1'
                    }
                },
                'process_id_2': {
                    'content': {
                        'key': 'process_2'
                    }
                }
            }
        }

        entity_map = EntityMap.load(spreadsheet_json)
        entity_linker = EntityLinker(self.mocked_template_manager, entity_map)

        with self.assertRaises(MultipleProcessesFound) as context:
            entity_linker.handle_links_from_spreadsheet()

        self.assertEqual('biomaterial', context.exception.from_entity.type)
        self.assertEqual(['process_id_1', 'process_id_2'],
                         context.exception.process_ids)
Ejemplo n.º 12
0
    def test_submit(self, submission_constructor):
        # given:
        submission = self._mock_submission(submission_constructor)

        # and:
        product = Entity('product', 'product_1', {})
        project = Entity('project', 'id', {})
        user = Entity('user', 'user_1', {})
        entity_map = EntityMap(product, user, project)

        # when:
        submitter = IngestSubmitter(self.ingest_api)
        submitter.add_entity = MagicMock()
        submitter.add_entities(entity_map, submission_url='url')

        # then:
        submission_constructor.assert_called_with(self.ingest_api, 'url')
        submission.define_manifest.assert_called_with(entity_map)
        submission.add_entity.assert_has_calls(
            [call(product), call(user)], any_order=True)
Ejemplo n.º 13
0
    def test_generate_direct_links_invalid_spreadsheet_link(self):
        # given
        spreadsheet_json = {
            'project': {
                'dummy-project-id': {
                    'content': {
                        'key': 'project_1'
                    }
                }
            },
            'biomaterial': {
                'biomaterial_id_1': {
                    'content': {
                        'key': 'biomaterial_1'
                    },
                    'links_by_entity': {
                        'file': ['file_id_1']
                    }
                }
            },
            'file': {
                'file_id_1': {
                    'content': {
                        'key': 'file_1'
                    }
                }
            }
        }

        entity_map = EntityMap.load(spreadsheet_json)
        entity_linker = EntityLinker(self.mocked_template_manager, entity_map)

        with self.assertRaises(InvalidLinkInSpreadsheet) as context:
            entity_linker.handle_links_from_spreadsheet()

        self.assertEqual('biomaterial', context.exception.from_entity.type)
        self.assertEqual('file', context.exception.link_entity_type)

        self.assertEqual('biomaterial_id_1', context.exception.from_entity.id)
        self.assertEqual('file_id_1', context.exception.link_entity_id)
Ejemplo n.º 14
0
    def import_file(self, file_path, submission_url, is_update=False, project_uuid=None, update_project=False) -> Tuple[
        Submission, TemplateManager]:
        try:
            if project_uuid:
                self.submitter.link_submission_to_project(project_uuid, submission_url)

            submission = None
            template_mgr = None
            spreadsheet_json, template_mgr, errors = self.generate_json(file_path, is_update, project_uuid=project_uuid,
                                                                        update_project=update_project)
            entity_map = EntityMap.load(spreadsheet_json)
            self.ingest_api.delete_submission_errors(submission_url)

            if errors:
                self.report_errors(submission_url, errors)
            elif is_update:
                self.submitter.update_entities(entity_map)
            else:
                entity_linker = EntityLinker(template_mgr, entity_map)
                entity_linker.handle_links_from_spreadsheet()
                submission = self._submit_new_entities(entity_map, submission_url)

            project = entity_map.get_project()
            if project and project_uuid and update_project:
                self.submitter.update_entity(project)

        except HTTPError as httpError:
            self.logger.exception(httpError)
            status = httpError.response.status_code
            text = httpError.response.text
            importer_error = ImporterError(f'Received an HTTP {status} from  {httpError.request.url}: {text}')
            self.ingest_api.create_submission_error(submission_url, importer_error.getJSON())
            return None, template_mgr
        except Exception as e:
            self.ingest_api.create_submission_error(submission_url, ImporterError(str(e)).getJSON())
            self.logger.error(str(e), exc_info=True)
            return None, template_mgr
        finally:
            self.logger.info(f'Submission in {submission_url} is done!')
            return submission, template_mgr
Ejemplo n.º 15
0
    def test_load__is_linking_reference(self):
        # given:
        spreadsheet_json = {
            'biomaterial': {
                'biomaterial_id': {
                    'content': {
                        'key': 'biomaterial_3'
                    },
                    'links_by_entity': {
                        'biomaterial': ['biomaterial_id_2'],
                        'process': ['process_id_2']
                    },
                    'external_links_by_entity': {
                        'biomaterial': ['biomaterial_uuid']
                    },
                },
            }
        }

        # when:
        entity_map = EntityMap.load(spreadsheet_json)

        # then:
        self.assertEqual(['biomaterial'], list(entity_map.get_entity_types()))
Ejemplo n.º 16
0
    def define_manifest(self, entity_map: EntityMap):
        # TODO provide a better way to serialize
        manifest_json = {
            'totalCount':
            entity_map.count_total(),
            'expectedBiomaterials':
            entity_map.count_entities_of_type('biomaterial'),
            'expectedProcesses':
            entity_map.count_entities_of_type('process'),
            'expectedFiles':
            entity_map.count_entities_of_type('file'),
            'expectedProtocols':
            entity_map.count_entities_of_type('protocol'),
            'expectedProjects':
            entity_map.count_entities_of_type('project'),
            'expectedLinks':
            entity_map.count_links(),
            'actualLinks':
            0
        }

        self.manifest = self.ingest_api.create_submission_manifest(
            self.submission_url, manifest_json)
        return self.manifest
Ejemplo n.º 17
0
    def test_submit_linked_entity(self, submission_constructor):
        # given:
        submission = self._mock_submission(submission_constructor)

        # and:
        user = Entity('user', 'user_1', {})
        entity_map = EntityMap(user)

        # and:
        link_to_user = {
            'entity': 'user',
            'id': 'user_1',
            'relationship': 'wish_list'
        }
        linked_product = Entity('product',
                                'product_1', {},
                                direct_links=[link_to_user],
                                is_reference=False,
                                is_linking_reference=False)
        project = Entity('project',
                         'id', {},
                         is_reference=False,
                         is_linking_reference=False)
        entity_map.add_entity(linked_product)
        entity_map.add_entity(project)

        # when:
        submitter = IngestSubmitter(self.ingest_api)
        submitter.add_entity = MagicMock()
        submitter.link_submission_to_project = MagicMock()
        submitter.PROGRESS_CTR = 1
        submitter.add_entities(entity_map, submission_url='url')

        # then:
        submission_constructor.assert_called_with(self.ingest_api, 'url')
        submission.define_manifest.assert_called_with(entity_map)
        submission.add_entity.assert_has_calls(
            [call(user), call(linked_product)], any_order=True)
Ejemplo n.º 18
0
    def test_generate_direct_links_file_to_file_no_process(self):
        # given
        spreadsheet_json = {
            'project': {
                'dummy-project-id': {
                    'content': {
                        'key': 'project_1'
                    }
                }
            },
            'file': {
                'file_id_1': {
                    'content': {
                        'key': 'file_1'
                    }
                },
                'file_id_2': {
                    'content': {
                        'key': 'file_2'
                    },
                    'links_by_entity': {
                        'file': ['file_id_1'],
                        'protocol': ['protocol_id_1', 'protocol_id_2']
                    }
                }
            },
            'protocol': {
                'protocol_id_1': {
                    'content': {
                        'key': 'protocol_1'
                    }
                },
                'protocol_id_2': {
                    'content': {
                        'key': 'protocol_2'
                    }
                }
            }
        }

        expected_json = {
            'project': {
                'dummy-project-id': {
                    'content': {
                        'key': 'project_1'
                    }
                }
            },
            'file': {
                'file_id_1': {
                    'content': {
                        'key': 'file_1'
                    },
                    'direct_links': [{
                        'entity': 'process',
                        'id': 'process_id_1',
                        'relationship': 'inputToProcesses'
                    }, {
                        'entity': 'project',
                        'id': 'dummy-project-id',
                        'relationship': 'project',
                        'is_collection': False
                    }]
                },
                'file_id_2': {
                    'content': {
                        'key': 'file_2'
                    },
                    'links_by_entity': {
                        'file': ['file_id_1'],
                    },
                    'direct_links': [{
                        'entity': 'process',
                        'id': 'process_id_1',
                        'relationship': 'derivedByProcesses'
                    }, {
                        'entity': 'project',
                        'id': 'dummy-project-id',
                        'relationship': 'project',
                        'is_collection': False
                    }]
                }
            },
            'process': {
                'process_id_1': {
                    'content': {
                        'key': 'process_1'
                    },
                    'direct_links': [{
                        'entity': 'project',
                        'id': 'dummy-project-id',
                        'relationship': 'projects'
                    }, {
                        'entity': 'project',
                        'id': 'dummy-project-id',
                        'relationship': 'project',
                        'is_collection': False
                    }, {
                        'entity': 'protocol',
                        'id': 'protocol_id_1',
                        'relationship': 'protocols'
                    }, {
                        'entity': 'protocol',
                        'id': 'protocol_id_2',
                        'relationship': 'protocols'
                    }]
                }
            },
            'protocol': {
                'protocol_id_1': {
                    'content': {
                        'key': 'protocol_1'
                    },
                    'direct_links': [{
                        'entity': 'project',
                        'id': 'dummy-project-id',
                        'relationship': 'project',
                        'is_collection': False
                    }]
                },
                'protocol_id_2': {
                    'content': {
                        'key': 'protocol_2'
                    },
                    'direct_links': [{
                        'entity': 'project',
                        'id': 'dummy-project-id',
                        'relationship': 'project',
                        'is_collection': False
                    }]
                }
            }
        }

        entity_map = EntityMap.load(spreadsheet_json)
        entity_linker = EntityLinker(self.mocked_template_manager, entity_map)
        output = entity_linker.handle_links_from_spreadsheet()

        self._assert_equal_direct_links(expected_json, output)
Ejemplo n.º 19
0
 def update_entities(self, entity_map: EntityMap):
     updated_entities = [self.update_entity(e) for e in entity_map.get_entities() if e.is_reference]
     return updated_entities
Ejemplo n.º 20
0
    def test_count_links(self):
        entity_map = EntityMap()

        # no element
        self.assertEqual(entity_map.count_links(), 0)

        # has 1 element without links
        entity_map.add_entity(Entity('product', 'product_0', {}))
        self.assertEqual(entity_map.count_links(), 0)

        # has 1 element with links
        entity_map.add_entity(
            Entity('product', 'product_1', {}, direct_links=[{}, {}, {}]))
        self.assertEqual(entity_map.count_links(), 3)

        # has many element with links
        entity_map.add_entity(
            Entity('product', 'product_2', {}, direct_links=[{}, {}, {}, {}]))
        self.assertEqual(entity_map.count_links(), 7)
Ejemplo n.º 21
0
    def test_count_total(self):
        # given:
        zero_map = EntityMap()

        # and:
        one_map = EntityMap()
        one_map.add_entity(Entity('product', 'product_1', {}))

        # and:
        three_map = EntityMap()
        three_map.add_entity(Entity('profile', 'profile_1', {}))
        for product_id in range(0, 2):
            three_map.add_entity(Entity('product', f'product_{product_id}',
                                        {}))

        # expect:
        self.assertEqual(0, zero_map.count_total())
        self.assertEqual(1, one_map.count_total())
        self.assertEqual(3, three_map.count_total())