def _get_bundles(self, files):
bundles = []
errors = []
for fname in files:
if os.path.splitext(fname)[1] != ".bundle":
continue
content = None
try:
with open(fname, 'r') as f:
doc = DocumentIO.load(f)[1]
DocumentEvolution.evolve_document(doc)
bundles.append(doc)
except ValueError:
msg = 'Error adding result bundle %s' % fname
errors.append(msg)
logging.exception(msg)
if content:
logging.info('Adding bundle as attachment')
attachment = create_attachment(fname, content)
self.context.test_data.add_attachments([attachment])
except KeyboardInterrupt:
raise KeyboardInterrupt
except:
msg = 'Unknown error processing bundle' % fname
logging.exception(msg)
errors.append(msg)
if len(errors) > 0:
msg = ' '.join(errors)
raise GatherResultsError(msg, bundles)
return bundles
def test_load_document(self):
# Note: resource_string uses posix-style paths
# regardless of the actual system paths
fmt, doc = DocumentIO.load(
resource_stream('linaro_dashboard_bundle',
'test_documents/' + self.filename))
self.assertIsNot(doc, None)
def test_load_document(self):
# Note: resource_string uses posix-style paths
# regardless of the actual system paths
fmt, doc = DocumentIO.load(
resource_stream('linaro_dashboard_bundle',
'test_documents/' + self.filename))
self.assertIsNot(doc, None)
def _get_bundles(self, files):
bundles = []
errors = []
for fname in files:
if os.path.splitext(fname)[1] != ".bundle":
continue
content = None
try:
with open(fname, 'r') as f:
doc = DocumentIO.load(f)[1]
DocumentEvolution.evolve_document(doc)
bundles.append(doc)
except ValueError:
msg = 'Error adding result bundle %s' % fname
errors.append(msg)
logging.exception(msg)
if content:
logging.info('Adding bundle as attachment')
attachment = create_attachment(fname, content)
self.context.test_data.add_attachments([attachment])
except KeyboardInterrupt:
raise KeyboardInterrupt
except:
msg = 'Unknown error processing bundle' % fname
logging.exception(msg)
errors.append(msg)
if len(errors) > 0:
msg = ' '.join(errors)
raise GatherResultsError(msg, bundles)
return bundles
def _get_results_from_host(self):
bundles = []
errors = []
try:
bundle_list = os.listdir(self.context.host_result_dir)
for bundle_name in bundle_list:
bundle = "%s/%s" % (self.context.host_result_dir, bundle_name)
content = None
try:
with open(bundle) as f:
doc = DocumentIO.load(f)[1]
DocumentEvolution.evolve_document(doc)
bundles.append(doc)
except ValueError:
msg = 'Error adding host result bundle %s' % bundle
errors.append(msg)
logging.exception(msg)
if content:
logging.info('Adding bundle as attachment')
attachment = create_attachment(bundle, content)
self.context.test_data.add_attachments([attachment])
except:
msg = 'Error getting all results from host'
logging.exception(msg)
raise GatherResultsError(msg, bundles)
if len(errors) > 0:
msg = ' '.join(errors)
raise GatherResultsError(msg, bundles)
return bundles
def test_evolved_document_is_what_we_expect(self):
DocumentEvolution.evolve_document(self.doc, one_step=True)
fmt, evolved_doc = DocumentIO.load(
resource_stream('linaro_dashboard_bundle',
'test_documents/evolution_1.7.1.json'),
retain_order=False)
self.assertEqual(self.doc, evolved_doc)
def _get_results_from_host(self):
bundles = []
errors = []
try:
bundle_list = os.listdir(self.context.host_result_dir)
for bundle_name in bundle_list:
bundle = "%s/%s" % (self.context.host_result_dir, bundle_name)
content = None
try:
with open(bundle) as f:
doc = DocumentIO.load(f)[1]
DocumentEvolution.evolve_document(doc)
bundles.append(doc)
except ValueError:
msg = 'Error adding host result bundle %s' % bundle
errors.append(msg)
logging.exception(msg)
if content:
logging.info('Adding bundle as attachment')
attachment = create_attachment(bundle, content)
self.context.test_data.add_attachments([attachment])
except:
msg = 'Error getting all results from host'
logging.exception(msg)
raise GatherResultsError(msg, bundles)
if len(errors) > 0:
msg = ' '.join(errors)
raise GatherResultsError(msg, bundles)
return bundles
def _load_bundle(self, local_pathname):
"""
Load the bundle from local_pathname.
There are various problems that can happen here but
they should all be treated equally, the bundle not
being used. This also transparently does schema validation
so the chance of getting wrong data is lower.
"""
with open(local_pathname, 'rt') as stream:
format, bundle = DocumentIO.load(stream)
return format, bundle
def _load_bundle(self, local_pathname):
"""
Load the bundle from local_pathname.
There are various problems that can happen here but
they should all be treated equally, the bundle not
being used. This also transparently does schema validation
so the chance of getting wrong data is lower.
"""
with open(local_pathname, 'rt') as stream:
format, bundle = DocumentIO.load(stream)
return format, bundle
def deserialize(self, s_bundle, prefer_evolution):
"""
Deserializes specified Bundle.
:Discussion:
This method also handles internal transaction handling.
All operations performed during bundle deserialization are
_rolled_back_ if anything fails.
If prefer_evolution is enabled then the document is first evolved
to the latest known format and only then imported into the
database. This operation is currently disabled to ensure that all
old documents are imported exactly as before. Enabling it should
be quite safe though as it passes all tests.
:Exceptions raised:
json_schema_validator.ValidationError
When the document does not match the appropriate schema.
linaro_dashboard_bundle.errors.DocumentFormatError
When the document format is not in the known set of formats.
ValueError
When the text does not represent a correct JSON document.
"""
assert s_bundle.is_deserialized is False
s_bundle.content.open('rb')
logger = logging.getLogger(__name__)
try:
logger.debug("Loading document")
fmt, doc = DocumentIO.load(s_bundle.content)
logger.debug("Document loaded")
if prefer_evolution:
logger.debug("Evolving document")
DocumentEvolution.evolve_document(doc)
logger.debug("Document evolution complete")
fmt = doc["format"]
finally:
s_bundle.content.close()
importer = self.IMPORTERS.get(fmt)
if importer is None:
raise DocumentFormatError(fmt)
try:
logger.debug("Importing document")
importer().import_document(s_bundle, doc)
logger.debug("Document import complete")
except Exception as exc:
logger.debug("Exception while importing document: %r", exc)
raise
def deserialize(self, s_bundle, prefer_evolution):
"""
Deserializes specified Bundle.
:Discussion:
This method also handles internal transaction handling.
All operations performed during bundle deserialization are
_rolled_back_ if anything fails.
If prefer_evolution is enabled then the document is first evolved
to the latest known format and only then imported into the
database. This operation is currently disabled to ensure that all
old documents are imported exactly as before. Enabling it should
be quite safe though as it passes all tests.
:Exceptions raised:
json_schema_validator.ValidationError
When the document does not match the appropriate schema.
linaro_dashboard_bundle.errors.DocumentFormatError
When the document format is not in the known set of formats.
ValueError
When the text does not represent a correct JSON document.
"""
assert s_bundle.is_deserialized is False
s_bundle.content.open('rb')
logger = logging.getLogger(__name__)
try:
logger.debug("Loading document")
fmt, doc = DocumentIO.load(s_bundle.content)
logger.debug("Document loaded")
if prefer_evolution:
logger.debug("Evolving document")
DocumentEvolution.evolve_document(doc)
logger.debug("Document evolution complete")
fmt = doc["format"]
finally:
s_bundle.content.close()
importer = self.IMPORTERS.get(fmt)
if importer is None:
raise DocumentFormatError(fmt)
try:
logger.debug("Importing document")
importer().import_document(s_bundle, doc)
logger.debug("Document import complete")
except Exception as exc:
logger.debug("Exception while importing document: %r", exc)
raise
def setUp(self):
super(DocumentEvolutionTests_1_6_to_1_7, self).setUp()
self.fmt, self.doc = DocumentIO.load(resource_stream(
'linaro_dashboard_bundle', 'test_documents/evolution_1.6.json'),
retain_order=False)
def test_load__with_enabled_retain_order__dict_class(self):
fmt, doc = DocumentIO.load(self.stream, retain_order=True)
observed_impl = type(doc)
# Note: VVV
self.assertNotEqual(observed_impl, dict)
def test_load__with_disabled_retain_order__dict_class(self):
fmt, doc = DocumentIO.load(self.stream, retain_order=False)
expected_impl = dict
observed_impl = type(doc)
self.assertEqual(observed_impl, expected_impl)
def test_load__with_enabled_retain_order__dict_class(self):
fmt, doc = DocumentIO.load(self.stream, retain_order=True)
observed_impl = type(doc)
# Note: VVV
self.assertNotEqual(observed_impl, dict)
def test_load__with_enabled_retain_order__key_order(self):
fmt, doc = DocumentIO.load(self.stream, retain_order=True)
observed_keys = doc.keys()
self.assertEqual(observed_keys, self.expected_keys)
def test_load__return_value(self):
fmt, doc = DocumentIO.load(self.stream)
self.assertEqual(fmt, self.expected_fmt)
self.assertEqual(doc, self.expected_doc)
def test_load__with_enabled_retain_order__key_order(self):
fmt, doc = DocumentIO.load(self.stream, retain_order=True)
observed_keys = doc.keys()
self.assertEqual(observed_keys, self.expected_keys)
def test_load__return_value(self):
fmt, doc = DocumentIO.load(self.stream)
self.assertEqual(fmt, self.expected_fmt)
self.assertEqual(doc, self.expected_doc)
def test_load__with_disabled_retain_order__dict_class(self):
fmt, doc = DocumentIO.load(self.stream, retain_order=False)
expected_impl = dict
observed_impl = type(doc)
self.assertEqual(observed_impl, expected_impl)
def setUp(self):
super(DocumentEvolutionTests_1_7_to_1_7_1, self).setUp()
self.fmt, self.doc = DocumentIO.load(
resource_stream('linaro_dashboard_bundle',
'test_documents/evolution_1.7.json'),
retain_order=False)