Example #1
0
    def _convert_to_common_format(self, format, bundle):
        """
        Convert the bundle to the common format.

        This is a careful and possibly fragile process that may
        raise FutureFormatDetected exception. If that happens
        then desired_format (encoded in the function itself) must be
        changed and the code reviewed for any possible changes
        required to support the more recent format.
        """
        while True:
            # Break conditions, encoded separately for clarity
            if format == self._desired_format:
                # This is our desired break condition, when format
                # becomes (or starts as) the desired format
                break
            if DocumentEvolution.is_latest(bundle):
                # This is a less desired break condition, if we
                # got here then the only possible explanation is
                # that some program started with format > desired_format
                # and the DocumentEvolution API is updated to understand
                # it but we are not. In that case let's raise an exception
                raise FutureFormatDetected(format)
            # As long as the document format is old keep upgrading it
            # step-by-step. Evolution is done in place
            DocumentEvolution.evolve_document(bundle, one_step=True)
Example #2
0
 def test_evolved_document_is_what_we_expect(self):
     DocumentEvolution.evolve_document(self.doc, one_step=True)
     fmt, evolved_doc = DocumentIO.load(
         resource_stream('linaro_dashboard_bundle',
                         'test_documents/evolution_1.7.1.json'),
         retain_order=False)
     self.assertEqual(self.doc, evolved_doc)
Example #3
0
    def _convert_to_common_format(self, format, bundle):
        """
        Convert the bundle to the common format.

        This is a careful and possibly fragile process that may
        raise FutureFormatDetected exception. If that happens
        then desired_format (encoded in the function itself) must be
        changed and the code reviewed for any possible changes
        required to support the more recent format.
        """
        while True:
            # Break conditions, encoded separately for clarity
            if format == self._desired_format:
                # This is our desired break condition, when format
                # becomes (or starts as) the desired format
                break
            if DocumentEvolution.is_latest(bundle):
                # This is a less desired break condition, if we
                # got here then the only possible explanation is
                # that some program started with format > desired_format
                # and the DocumentEvolution API is updated to understand
                # it but we are not. In that case let's raise an exception
                raise FutureFormatDetected(format)
            # As long as the document format is old keep upgrading it
            # step-by-step. Evolution is done in place
            DocumentEvolution.evolve_document(bundle, one_step=True)
    def _get_bundles(self, files):
        bundles = []
        errors = []
        for fname in files:
            if os.path.splitext(fname)[1] != ".bundle":
                continue
            content = None
            try:
                with open(fname, 'r') as f:
                    doc = DocumentIO.load(f)[1]
                DocumentEvolution.evolve_document(doc)
                bundles.append(doc)
            except ValueError:
                msg = 'Error adding result bundle %s' % fname
                errors.append(msg)
                logging.exception(msg)
                if content:
                    logging.info('Adding bundle as attachment')
                    attachment = create_attachment(fname, content)
                    self.context.test_data.add_attachments([attachment])
            except KeyboardInterrupt:
                raise KeyboardInterrupt
            except:
                msg = 'Unknown error processing bundle' % fname
                logging.exception(msg)
                errors.append(msg)

        if len(errors) > 0:
            msg = ' '.join(errors)
            raise GatherResultsError(msg, bundles)
        return bundles
    def _get_results_from_host(self):
        bundles = []
        errors = []
        try:
            bundle_list = os.listdir(self.context.host_result_dir)
            for bundle_name in bundle_list:
                bundle = "%s/%s" % (self.context.host_result_dir, bundle_name)
                content = None
                try:
                    with open(bundle) as f:
                        doc = DocumentIO.load(f)[1]
                    DocumentEvolution.evolve_document(doc)
                    bundles.append(doc)
                except ValueError:
                    msg = 'Error adding host result bundle %s' % bundle
                    errors.append(msg)
                    logging.exception(msg)
                    if content:
                        logging.info('Adding bundle as attachment')
                        attachment = create_attachment(bundle, content)
                        self.context.test_data.add_attachments([attachment])
        except:
            msg = 'Error getting all results from host'
            logging.exception(msg)
            raise GatherResultsError(msg, bundles)

        if len(errors) > 0:
            msg = ' '.join(errors)
            raise GatherResultsError(msg, bundles)

        return bundles
Example #6
0
 def test_sw_image_becomes_image(self):
     self.assertNotIn("image", self.doc["test_runs"][0]["software_context"])
     self.assertIn("sw_image", self.doc["test_runs"][0]["software_context"])
     DocumentEvolution.evolve_document(self.doc, one_step=True)
     self.assertIn("image", self.doc["test_runs"][0]["software_context"])
     self.assertNotIn("sw_image",
                      self.doc["test_runs"][0]["software_context"])
Example #7
0
    def _get_results_from_host(self):
        bundles = []
        errors = []
        try:
            bundle_list = os.listdir(self.context.host_result_dir)
            for bundle_name in bundle_list:
                bundle = "%s/%s" % (self.context.host_result_dir, bundle_name)
                content = None
                try:
                    with open(bundle) as f:
                        doc = DocumentIO.load(f)[1]
                    DocumentEvolution.evolve_document(doc)
                    bundles.append(doc)
                except ValueError:
                    msg = 'Error adding host result bundle %s' % bundle
                    errors.append(msg)
                    logging.exception(msg)
                    if content:
                        logging.info('Adding bundle as attachment')
                        attachment = create_attachment(bundle, content)
                        self.context.test_data.add_attachments([attachment])
        except:
            msg = 'Error getting all results from host'
            logging.exception(msg)
            raise GatherResultsError(msg, bundles)

        if len(errors) > 0:
            msg = ' '.join(errors)
            raise GatherResultsError(msg, bundles)

        return bundles
Example #8
0
    def _get_bundles(self, files):
        bundles = []
        errors = []
        for fname in files:
            if os.path.splitext(fname)[1] != ".bundle":
                continue
            content = None
            try:
                with open(fname, 'r') as f:
                    doc = DocumentIO.load(f)[1]
                DocumentEvolution.evolve_document(doc)
                bundles.append(doc)
            except ValueError:
                msg = 'Error adding result bundle %s' % fname
                errors.append(msg)
                logging.exception(msg)
                if content:
                    logging.info('Adding bundle as attachment')
                    attachment = create_attachment(fname, content)
                    self.context.test_data.add_attachments([attachment])
            except KeyboardInterrupt:
                raise KeyboardInterrupt
            except:
                msg = 'Unknown error processing bundle' % fname
                logging.exception(msg)
                errors.append(msg)

        if len(errors) > 0:
            msg = ' '.join(errors)
            raise GatherResultsError(msg, bundles)
        return bundles
Example #9
0
    def deserialize(self, s_bundle, prefer_evolution):
        """
        Deserializes specified Bundle.

        :Discussion:
            This method also handles internal transaction handling.
            All operations performed during bundle deserialization are
            _rolled_back_ if anything fails.

            If prefer_evolution is enabled then the document is first evolved
            to the latest known format and only then imported into the
            database. This operation is currently disabled to ensure that all
            old documents are imported exactly as before. Enabling it should
            be quite safe though as it passes all tests.

        :Exceptions raised:
            json_schema_validator.ValidationError
                When the document does not match the appropriate schema.
            linaro_dashboard_bundle.errors.DocumentFormatError
                When the document format is not in the known set of formats.
            ValueError
                When the text does not represent a correct JSON document.
        """
        assert s_bundle.is_deserialized is False
        s_bundle.content.open('rb')
        logger = logging.getLogger(__name__)
        try:
            logger.debug("Loading document")
            fmt, doc = DocumentIO.load(s_bundle.content)
            logger.debug("Document loaded")
            if prefer_evolution:
                logger.debug("Evolving document")
                DocumentEvolution.evolve_document(doc)
                logger.debug("Document evolution complete")
                fmt = doc["format"]
        finally:
            s_bundle.content.close()
        importer = self.IMPORTERS.get(fmt)
        if importer is None:
            raise DocumentFormatError(fmt)
        try:
            logger.debug("Importing document")
            importer().import_document(s_bundle, doc)
            logger.debug("Document import complete")
        except Exception as exc:
            logger.debug("Exception while importing document: %r", exc)
            raise
Example #10
0
    def deserialize(self, s_bundle, prefer_evolution):
        """
        Deserializes specified Bundle.

        :Discussion:
            This method also handles internal transaction handling.
            All operations performed during bundle deserialization are
            _rolled_back_ if anything fails.

            If prefer_evolution is enabled then the document is first evolved
            to the latest known format and only then imported into the
            database. This operation is currently disabled to ensure that all
            old documents are imported exactly as before. Enabling it should
            be quite safe though as it passes all tests.

        :Exceptions raised:
            json_schema_validator.ValidationError
                When the document does not match the appropriate schema.
            linaro_dashboard_bundle.errors.DocumentFormatError
                When the document format is not in the known set of formats.
            ValueError
                When the text does not represent a correct JSON document.
        """
        assert s_bundle.is_deserialized is False
        s_bundle.content.open('rb')
        logger = logging.getLogger(__name__)
        try:
            logger.debug("Loading document")
            fmt, doc = DocumentIO.load(s_bundle.content)
            logger.debug("Document loaded")
            if prefer_evolution:
                logger.debug("Evolving document")
                DocumentEvolution.evolve_document(doc)
                logger.debug("Document evolution complete")
                fmt = doc["format"]
        finally:
            s_bundle.content.close()
        importer = self.IMPORTERS.get(fmt)
        if importer is None:
            raise DocumentFormatError(fmt)
        try:
            logger.debug("Importing document")
            importer().import_document(s_bundle, doc)
            logger.debug("Document import complete")
        except Exception as exc:
            logger.debug("Exception while importing document: %r", exc)
            raise
Example #11
0
 def test_format_is_changed(self):
     self.assertEqual(self.doc["format"], "Dashboard Bundle Format 1.6")
     DocumentEvolution.evolve_document(self.doc, one_step=True)
     self.assertEqual(self.doc["format"], "Dashboard Bundle Format 1.7")
Example #12
0
 def test_format_is_changed(self):
     self.assertEqual(self.doc["format"], "Dashboard Bundle Format 1.7")
     DocumentEvolution.evolve_document(self.doc, one_step=True)
     self.assertEqual(self.doc["format"], "Dashboard Bundle Format 1.7.1")
Example #13
0
 def test_hw_context_becomes_hardware_context(self):
     self.assertNotIn("hardware_context", self.doc["test_runs"][0])
     self.assertIn("hw_context", self.doc["test_runs"][0])
     DocumentEvolution.evolve_document(self.doc, one_step=True)
     self.assertIn("hardware_context", self.doc["test_runs"][0])
     self.assertNotIn("hw_context", self.doc["test_runs"][0])
Example #14
0
 def test_evolved_document_is_latest_format(self):
     self.assertFalse(DocumentEvolution.is_latest(self.doc))
     DocumentEvolution.evolve_document(self.doc, one_step=True)
     self.assertTrue(DocumentEvolution.is_latest(self.doc))
Example #15
0
 def test_evolved_document_is_valid(self):
     DocumentEvolution.evolve_document(self.doc, one_step=True)
     self.assertEqual(DocumentIO.check(self.doc),
                      "Dashboard Bundle Format 1.7.1")
Example #16
0
 def test_evolved_document_is_latest_format(self):
     self.assertFalse(DocumentEvolution.is_latest(self.doc))
     DocumentEvolution.evolve_document(self.doc, one_step=True)
     self.assertTrue(DocumentEvolution.is_latest(self.doc))
Example #17
0
 def test_sw_image_desc_becomes_image_name(self):
     self.assertNotIn("name", self.doc["test_runs"][0]["software_context"]["sw_image"])
     self.assertIn("desc", self.doc["test_runs"][0]["software_context"]["sw_image"])
     DocumentEvolution.evolve_document(self.doc, one_step=True)
     self.assertIn("name", self.doc["test_runs"][0]["software_context"]["image"])
     self.assertNotIn("desc", self.doc["test_runs"][0]["software_context"]["image"])
Example #18
0
 def test_evolved_document_is_valid(self):
     DocumentEvolution.evolve_document(self.doc, one_step=True)
     self.assertEqual(DocumentIO.check(self.doc),
                      "Dashboard Bundle Format 1.7")
Example #19
0
 def test_hw_context_becomes_hardware_context(self):
     self.assertNotIn("hardware_context", self.doc["test_runs"][0])
     self.assertIn("hw_context", self.doc["test_runs"][0])
     DocumentEvolution.evolve_document(self.doc, one_step=True)
     self.assertIn("hardware_context", self.doc["test_runs"][0])
     self.assertNotIn("hw_context", self.doc["test_runs"][0])