def _convert_to_common_format(self, format, bundle): """ Convert the bundle to the common format. This is a careful and possibly fragile process that may raise FutureFormatDetected exception. If that happens then desired_format (encoded in the function itself) must be changed and the code reviewed for any possible changes required to support the more recent format. """ while True: # Break conditions, encoded separately for clarity if format == self._desired_format: # This is our desired break condition, when format # becomes (or starts as) the desired format break if DocumentEvolution.is_latest(bundle): # This is a less desired break condition, if we # got here then the only possible explanation is # that some program started with format > desired_format # and the DocumentEvolution API is updated to understand # it but we are not. In that case let's raise an exception raise FutureFormatDetected(format) # As long as the document format is old keep upgrading it # step-by-step. Evolution is done in place DocumentEvolution.evolve_document(bundle, one_step=True)
def test_evolved_document_is_what_we_expect(self): DocumentEvolution.evolve_document(self.doc, one_step=True) fmt, evolved_doc = DocumentIO.load( resource_stream('linaro_dashboard_bundle', 'test_documents/evolution_1.7.1.json'), retain_order=False) self.assertEqual(self.doc, evolved_doc)
def _get_bundles(self, files): bundles = [] errors = [] for fname in files: if os.path.splitext(fname)[1] != ".bundle": continue content = None try: with open(fname, 'r') as f: doc = DocumentIO.load(f)[1] DocumentEvolution.evolve_document(doc) bundles.append(doc) except ValueError: msg = 'Error adding result bundle %s' % fname errors.append(msg) logging.exception(msg) if content: logging.info('Adding bundle as attachment') attachment = create_attachment(fname, content) self.context.test_data.add_attachments([attachment]) except KeyboardInterrupt: raise KeyboardInterrupt except: msg = 'Unknown error processing bundle' % fname logging.exception(msg) errors.append(msg) if len(errors) > 0: msg = ' '.join(errors) raise GatherResultsError(msg, bundles) return bundles
def _get_results_from_host(self): bundles = [] errors = [] try: bundle_list = os.listdir(self.context.host_result_dir) for bundle_name in bundle_list: bundle = "%s/%s" % (self.context.host_result_dir, bundle_name) content = None try: with open(bundle) as f: doc = DocumentIO.load(f)[1] DocumentEvolution.evolve_document(doc) bundles.append(doc) except ValueError: msg = 'Error adding host result bundle %s' % bundle errors.append(msg) logging.exception(msg) if content: logging.info('Adding bundle as attachment') attachment = create_attachment(bundle, content) self.context.test_data.add_attachments([attachment]) except: msg = 'Error getting all results from host' logging.exception(msg) raise GatherResultsError(msg, bundles) if len(errors) > 0: msg = ' '.join(errors) raise GatherResultsError(msg, bundles) return bundles
def test_sw_image_becomes_image(self): self.assertNotIn("image", self.doc["test_runs"][0]["software_context"]) self.assertIn("sw_image", self.doc["test_runs"][0]["software_context"]) DocumentEvolution.evolve_document(self.doc, one_step=True) self.assertIn("image", self.doc["test_runs"][0]["software_context"]) self.assertNotIn("sw_image", self.doc["test_runs"][0]["software_context"])
def deserialize(self, s_bundle, prefer_evolution): """ Deserializes specified Bundle. :Discussion: This method also handles internal transaction handling. All operations performed during bundle deserialization are _rolled_back_ if anything fails. If prefer_evolution is enabled then the document is first evolved to the latest known format and only then imported into the database. This operation is currently disabled to ensure that all old documents are imported exactly as before. Enabling it should be quite safe though as it passes all tests. :Exceptions raised: json_schema_validator.ValidationError When the document does not match the appropriate schema. linaro_dashboard_bundle.errors.DocumentFormatError When the document format is not in the known set of formats. ValueError When the text does not represent a correct JSON document. """ assert s_bundle.is_deserialized is False s_bundle.content.open('rb') logger = logging.getLogger(__name__) try: logger.debug("Loading document") fmt, doc = DocumentIO.load(s_bundle.content) logger.debug("Document loaded") if prefer_evolution: logger.debug("Evolving document") DocumentEvolution.evolve_document(doc) logger.debug("Document evolution complete") fmt = doc["format"] finally: s_bundle.content.close() importer = self.IMPORTERS.get(fmt) if importer is None: raise DocumentFormatError(fmt) try: logger.debug("Importing document") importer().import_document(s_bundle, doc) logger.debug("Document import complete") except Exception as exc: logger.debug("Exception while importing document: %r", exc) raise
def test_format_is_changed(self): self.assertEqual(self.doc["format"], "Dashboard Bundle Format 1.6") DocumentEvolution.evolve_document(self.doc, one_step=True) self.assertEqual(self.doc["format"], "Dashboard Bundle Format 1.7")
def test_format_is_changed(self): self.assertEqual(self.doc["format"], "Dashboard Bundle Format 1.7") DocumentEvolution.evolve_document(self.doc, one_step=True) self.assertEqual(self.doc["format"], "Dashboard Bundle Format 1.7.1")
def test_hw_context_becomes_hardware_context(self): self.assertNotIn("hardware_context", self.doc["test_runs"][0]) self.assertIn("hw_context", self.doc["test_runs"][0]) DocumentEvolution.evolve_document(self.doc, one_step=True) self.assertIn("hardware_context", self.doc["test_runs"][0]) self.assertNotIn("hw_context", self.doc["test_runs"][0])
def test_evolved_document_is_latest_format(self): self.assertFalse(DocumentEvolution.is_latest(self.doc)) DocumentEvolution.evolve_document(self.doc, one_step=True) self.assertTrue(DocumentEvolution.is_latest(self.doc))
def test_evolved_document_is_valid(self): DocumentEvolution.evolve_document(self.doc, one_step=True) self.assertEqual(DocumentIO.check(self.doc), "Dashboard Bundle Format 1.7.1")
def test_sw_image_desc_becomes_image_name(self): self.assertNotIn("name", self.doc["test_runs"][0]["software_context"]["sw_image"]) self.assertIn("desc", self.doc["test_runs"][0]["software_context"]["sw_image"]) DocumentEvolution.evolve_document(self.doc, one_step=True) self.assertIn("name", self.doc["test_runs"][0]["software_context"]["image"]) self.assertNotIn("desc", self.doc["test_runs"][0]["software_context"]["image"])
def test_evolved_document_is_valid(self): DocumentEvolution.evolve_document(self.doc, one_step=True) self.assertEqual(DocumentIO.check(self.doc), "Dashboard Bundle Format 1.7")