def _import_sources(self, c_test_run, s_test_run):
"""
Import TestRun.sources
"""
from dashboard_app.models import SoftwareSource
for c_source in self._get_sw_context(c_test_run).get("sources", []):
s_source, source_created = SoftwareSource.objects.get_or_create(
project_name=c_source["project_name"], # required by schema
branch_url=c_source["branch_url"], # required by schema
branch_vcs=c_source["branch_vcs"], # required by schema
# required by schema, may be either int or string so upconvert to string
branch_revision=str(c_source["branch_revision"]),
# optional
commit_timestamp=(
datetime_extension.from_json(
c_source["commit_timestamp"])
if "commit_timestamp" in c_source
else None),
default_params=c_source.get("default_params", []),
test_params=c_source.get("test_params", [])
)
if source_created:
s_source.save()
s_test_run.sources.add(s_source)
def test_import_sources(self):
c_test_run = {
"software_context": {
"sources": [
self.c_getUniqueSoftwareSource()
for i in range(3)
]
}
}
s_test_run = self.s_getUniqueTestRun()
importer = BundleFormatImporter_1_1()
importer._import_sources(c_test_run, s_test_run)
for c_source in c_test_run['software_context']['sources']:
filter = dict(
project_name=c_source['project_name'],
branch_url=c_source['branch_url'],
branch_vcs=c_source['branch_vcs'],
branch_revision=str(c_source['branch_revision']),
commit_timestamp=(
datetime_extension.from_json(
c_source["commit_timestamp"])
if "commit_timestamp" in c_source
else None)
)
s_source = SoftwareSource.objects.get(**filter)
self.assertTrue(s_source is not None)
self.assertTrue(s_source.pk is not None)
self.assertTrue(s_source in s_test_run.sources.all())
def _import_test_run(self, c_test_run, s_bundle):
"""
Import TestRun
"""
from dashboard_app.models import TestRun
s_test = self._import_test(c_test_run)
analyzer_assigned_uuid = UUID(c_test_run["analyzer_assigned_uuid"])
s_test_run = TestRun.objects.create(
bundle=s_bundle,
test=s_test,
analyzer_assigned_uuid=str(analyzer_assigned_uuid),
analyzer_assigned_date=datetime_extension.from_json(
# required by schema
c_test_run["analyzer_assigned_date"]
),
time_check_performed=(
# required by schema
c_test_run["time_check_performed"]
),
)
# needed for foreign key models below
s_test_run.save()
# import all the bits and pieces
self._log("starting")
self._import_test_results(c_test_run, s_test_run)
self._log("results")
self._import_attachments(c_test_run, s_test_run)
self._log("attachments")
self._import_hardware_context(c_test_run, s_test_run)
self._log("hardware")
self._import_software_context(c_test_run, s_test_run)
self._log("software")
self._import_attributes(c_test_run, s_test_run)
self._log("attributes")
# collect all the changes that happen before the previous save
s_test_run.save()
s_test_run.denormalize()
return s_test_run
def _import_test_results_pgsql(self, c_test_results, s_test_run):
cursor = connection.cursor()
# _order has been removed as order_with_respect_to and ordering
# where never compatible for the same Meta object in Django and
# django1.9 blocks use of the two elements together.
data = []
for i in range(0, len(c_test_results), 1000):
cursor.execute("""
CREATE TEMPORARY TABLE newtestresults (
relative_index INTEGER,
timestamp TIMESTAMP WITH TIME ZONE,
microseconds BIGINT,
filename TEXT,
result SMALLINT,
measurement CHARACTER VARYING(512),
message TEXT,
test_case_id TEXT,
lineno INTEGER
)
""")
data = []
for index, c_test_result in enumerate(c_test_results[i:i + 1000],
i + 1):
timestamp = c_test_result.get("timestamp")
if timestamp:
timestamp = datetime_extension.from_json(timestamp)
duration = c_test_result.get("duration", None)
if duration:
duration = timedelta_extension.from_json(duration)
duration = (duration.microseconds +
(duration.seconds * 10**6) +
(duration.days * 24 * 60 * 60 * 10**6))
result = self._translate_result_string(c_test_result["result"])
data.extend([
index,
timestamp,
duration,
c_test_result.get("log_filename", None),
result,
c_test_result.get("measurement", None),
c_test_result.get("message", None),
c_test_result.get("test_case_id", None),
c_test_result.get("log_lineno", None),
])
sequel = ',\n'.join(["(" + "%s" %
(', '.join(['%s'] * 9), ) + ")"] *
(len(data) // 9))
cursor.execute(
"""
INSERT INTO newtestresults (
relative_index,
timestamp,
microseconds,
filename,
result,
measurement,
message,
test_case_id,
lineno
) VALUES """ + sequel, data)
cursor.execute("""
INSERT INTO dashboard_app_testresult (
test_run_id,
relative_index,
timestamp,
microseconds,
filename,
result,
measurement,
message,
test_case_id,
lineno
) SELECT
%s,
relative_index,
timestamp,
microseconds,
filename,
result,
measurement,
message,
dashboard_app_testcase.id,
lineno
FROM newtestresults, dashboard_app_testcase
WHERE dashboard_app_testcase.test_id = %s
AND dashboard_app_testcase.test_case_id
= newtestresults.test_case_id
""" % (s_test_run.id, s_test_run.test.id))
cursor.execute("""
DROP TABLE newtestresults
""")
cursor.close()
def _import_test_results_sqlite(self, c_test_results, s_test_run):
cursor = connection.cursor()
# _order has been removed as order_with_respect_to and ordering
# where never compatible for the same Meta object in Django and
# django1.9 blocks use of the two elements together.
data = []
for index, c_test_result in enumerate(c_test_results, 1):
timestamp = c_test_result.get("timestamp")
if timestamp:
timestamp = datetime_extension.from_json(timestamp)
duration = c_test_result.get("duration", None)
if duration:
duration = timedelta_extension.from_json(duration)
duration = (duration.microseconds +
(duration.seconds * 10**6) +
(duration.days * 24 * 60 * 60 * 10**6))
result = self._translate_result_string(c_test_result["result"])
data.append((
s_test_run.id,
index,
timestamp,
duration,
c_test_result.get("log_filename", None),
result,
c_test_result.get("measurement", None),
c_test_result.get("message", None),
c_test_result.get("log_lineno", None),
s_test_run.test.id,
c_test_result.get("test_case_id", None),
))
cursor.executemany(
"""
INSERT INTO dashboard_app_testresult (
test_run_id,
relative_index,
timestamp,
microseconds,
filename,
result,
measurement,
message,
lineno,
test_case_id
) select
%s,
%s,
%s,
%s,
%s,
%s,
%s,
%s,
%s,
dashboard_app_testcase.id
FROM dashboard_app_testcase
WHERE dashboard_app_testcase.test_id = %s
AND dashboard_app_testcase.test_case_id
= %s
""", data)
cursor.close()
def _import_test_results_pgsql(self, c_test_results, s_test_run):
cursor = connection.cursor()
# XXX I don't understand how the _order column that Django adds is
# supposed to work. I just let it default to 0 here.
data = []
for i in range(0, len(c_test_results), 1000):
cursor.execute(
"""
CREATE TEMPORARY TABLE newtestresults (
relative_index INTEGER,
timestamp TIMESTAMP WITH TIME ZONE,
microseconds BIGINT,
filename TEXT,
result SMALLINT,
measurement CHARACTER VARYING(512),
message TEXT,
test_case_id TEXT,
lineno INTEGER
)
""")
data = []
for index, c_test_result in enumerate(c_test_results[i:i + 1000], i + 1):
timestamp = c_test_result.get("timestamp")
if timestamp:
timestamp = datetime_extension.from_json(timestamp)
duration = c_test_result.get("duration", None)
if duration:
duration = timedelta_extension.from_json(duration)
duration = (duration.microseconds +
(duration.seconds * 10 ** 6) +
(duration.days * 24 * 60 * 60 * 10 ** 6))
result = self._translate_result_string(c_test_result["result"])
data.extend([
index,
timestamp,
duration,
c_test_result.get("log_filename", None),
result,
c_test_result.get("measurement", None),
c_test_result.get("message", None),
c_test_result.get("test_case_id", None),
c_test_result.get("log_lineno", None),
])
sequel = ',\n'.join(
["(" + "%s" % (', '.join(['%s'] * 9),) + ")"] * (len(data) // 9))
cursor.execute(
"""
INSERT INTO newtestresults (
relative_index,
timestamp,
microseconds,
filename,
result,
measurement,
message,
test_case_id,
lineno
) VALUES """ + sequel, data)
cursor.execute(
"""
INSERT INTO dashboard_app_testresult (
test_run_id,
_order,
relative_index,
timestamp,
microseconds,
filename,
result,
measurement,
message,
test_case_id,
lineno
) SELECT
%s,
%s,
relative_index,
timestamp,
microseconds,
filename,
result,
measurement,
message,
dashboard_app_testcase.id,
lineno
FROM newtestresults, dashboard_app_testcase
WHERE dashboard_app_testcase.test_id = %s
AND dashboard_app_testcase.test_case_id
= newtestresults.test_case_id
""" % (s_test_run.id, 0, s_test_run.test.id))
cursor.execute(
"""
DROP TABLE newtestresults
""")
cursor.close()
def _import_test_results_sqlite(self, c_test_results, s_test_run):
cursor = connection.cursor()
# XXX I don't understand how the _order column that Django adds is
# supposed to work. I just set it to 0 here.
data = []
for index, c_test_result in enumerate(c_test_results, 1):
timestamp = c_test_result.get("timestamp")
if timestamp:
timestamp = datetime_extension.from_json(timestamp)
duration = c_test_result.get("duration", None)
if duration:
duration = timedelta_extension.from_json(duration)
duration = (duration.microseconds +
(duration.seconds * 10 ** 6) +
(duration.days * 24 * 60 * 60 * 10 ** 6))
result = self._translate_result_string(c_test_result["result"])
data.append((
s_test_run.id,
index,
timestamp,
duration,
c_test_result.get("log_filename", None),
result,
c_test_result.get("measurement", None),
c_test_result.get("message", None),
c_test_result.get("log_lineno", None),
s_test_run.test.id,
c_test_result.get("test_case_id", None),
))
cursor.executemany(
"""
INSERT INTO dashboard_app_testresult (
test_run_id,
relative_index,
timestamp,
microseconds,
filename,
result,
measurement,
message,
lineno,
_order,
test_case_id
) select
%s,
%s,
%s,
%s,
%s,
%s,
%s,
%s,
%s,
0,
dashboard_app_testcase.id
FROM dashboard_app_testcase
WHERE dashboard_app_testcase.test_id = %s
AND dashboard_app_testcase.test_case_id
= %s
""", data)
cursor.close()
def _import_test_results_pgsql(self, c_test_results, s_test_run):
cursor = connection.cursor()
# _order has been removed as order_with_respect_to and ordering
# where never compatible for the same Meta object in Django and
# django1.9 blocks use of the two elements together.
data = []
for i in range(0, len(c_test_results), 1000):
cursor.execute(
"""
CREATE TEMPORARY TABLE newtestresults (
relative_index INTEGER,
timestamp TIMESTAMP WITH TIME ZONE,
microseconds BIGINT,
filename TEXT,
result SMALLINT,
measurement CHARACTER VARYING(512),
message TEXT,
test_case_id TEXT,
lineno INTEGER
)
""")
data = []
for index, c_test_result in enumerate(c_test_results[i:i + 1000], i + 1):
timestamp = c_test_result.get("timestamp")
if timestamp:
timestamp = datetime_extension.from_json(timestamp)
duration = c_test_result.get("duration", None)
if duration:
duration = timedelta_extension.from_json(duration)
duration = (duration.microseconds +
(duration.seconds * 10 ** 6) +
(duration.days * 24 * 60 * 60 * 10 ** 6))
result = self._translate_result_string(c_test_result["result"])
data.extend([
index,
timestamp,
duration,
c_test_result.get("log_filename", None),
result,
c_test_result.get("measurement", None),
c_test_result.get("message", None),
c_test_result.get("test_case_id", None),
c_test_result.get("log_lineno", None),
])
sequel = ',\n'.join(
["(" + "%s" % (', '.join(['%s'] * 9),) + ")"] * (len(data) // 9))
cursor.execute(
"""
INSERT INTO newtestresults (
relative_index,
timestamp,
microseconds,
filename,
result,
measurement,
message,
test_case_id,
lineno
) VALUES """ + sequel, data)
cursor.execute(
"""
INSERT INTO dashboard_app_testresult (
test_run_id,
relative_index,
timestamp,
microseconds,
filename,
result,
measurement,
message,
test_case_id,
lineno
) SELECT
%s,
relative_index,
timestamp,
microseconds,
filename,
result,
measurement,
message,
dashboard_app_testcase.id,
lineno
FROM newtestresults, dashboard_app_testcase
WHERE dashboard_app_testcase.test_id = %s
AND dashboard_app_testcase.test_case_id
= newtestresults.test_case_id
""" % (s_test_run.id, s_test_run.test.id))
cursor.execute(
"""
DROP TABLE newtestresults
""")
cursor.close()
def _import_test_results_sqlite(self, c_test_results, s_test_run):
cursor = connection.cursor()
# _order has been removed as order_with_respect_to and ordering
# where never compatible for the same Meta object in Django and
# django1.9 blocks use of the two elements together.
data = []
for index, c_test_result in enumerate(c_test_results, 1):
timestamp = c_test_result.get("timestamp")
if timestamp:
timestamp = datetime_extension.from_json(timestamp)
duration = c_test_result.get("duration", None)
if duration:
duration = timedelta_extension.from_json(duration)
duration = (duration.microseconds +
(duration.seconds * 10 ** 6) +
(duration.days * 24 * 60 * 60 * 10 ** 6))
result = self._translate_result_string(c_test_result["result"])
data.append((
s_test_run.id,
index,
timestamp,
duration,
c_test_result.get("log_filename", None),
result,
c_test_result.get("measurement", None),
c_test_result.get("message", None),
c_test_result.get("log_lineno", None),
s_test_run.test.id,
c_test_result.get("test_case_id", None),
))
cursor.executemany(
"""
INSERT INTO dashboard_app_testresult (
test_run_id,
relative_index,
timestamp,
microseconds,
filename,
result,
measurement,
message,
lineno,
test_case_id
) select
%s,
%s,
%s,
%s,
%s,
%s,
%s,
%s,
%s,
dashboard_app_testcase.id
FROM dashboard_app_testcase
WHERE dashboard_app_testcase.test_id = %s
AND dashboard_app_testcase.test_case_id
= %s
""", data)
cursor.close()
def _import_test_results_pgsql(self, c_test_results, s_test_run):
cursor = connection.cursor()
# XXX I don't understand how the _order column that Django adds is
# supposed to work. I just let it default to 0 here.
data = []
for i in range(0, len(c_test_results), 1000):
cursor.execute("""
CREATE TEMPORARY TABLE newtestresults (
relative_index INTEGER,
timestamp TIMESTAMP WITH TIME ZONE,
microseconds BIGINT,
filename TEXT,
result SMALLINT,
measurement CHARACTER VARYING(512),
message TEXT,
test_case_id TEXT,
lineno INTEGER
)
""")
data = []
for index, c_test_result in enumerate(c_test_results[i:i + 1000],
i + 1):
timestamp = c_test_result.get("timestamp")
if timestamp:
timestamp = datetime_extension.from_json(timestamp)
duration = c_test_result.get("duration", None)
if duration:
duration = timedelta_extension.from_json(duration)
duration = (duration.microseconds +
(duration.seconds * 10**6) +
(duration.days * 24 * 60 * 60 * 10**6))
result = self._translate_result_string(c_test_result["result"])
data.extend([
index,
timestamp,
duration,
c_test_result.get("log_filename", None),
result,
c_test_result.get("measurement", None),
c_test_result.get("message", None),
c_test_result.get("test_case_id", None),
c_test_result.get("log_lineno", None),
])
sequel = ',\n'.join(["(" + "%s" %
(', '.join(['%s'] * 9), ) + ")"] *
(len(data) // 9))
cursor.execute(
"""
INSERT INTO newtestresults (
relative_index,
timestamp,
microseconds,
filename,
result,
measurement,
message,
test_case_id,
lineno
) VALUES """ + sequel, data)
cursor.execute("""
INSERT INTO dashboard_app_testresult (
test_run_id,
_order,
relative_index,
timestamp,
microseconds,
filename,
result,
measurement,
message,
test_case_id,
lineno
) SELECT
%s,
%s,
relative_index,
timestamp,
microseconds,
filename,
result,
measurement,
message,
dashboard_app_testcase.id,
lineno
FROM newtestresults, dashboard_app_testcase
WHERE dashboard_app_testcase.test_id = %s
AND dashboard_app_testcase.test_case_id
= newtestresults.test_case_id
""" % (s_test_run.id, 0, s_test_run.test.id))
cursor.execute("""
DROP TABLE newtestresults
""")
cursor.close()
def _import_test_results_sqlite(self, c_test_results, s_test_run):
cursor = connection.cursor()
# XXX I don't understand how the _order column that Django adds is
# supposed to work. I just set it to 0 here.
data = []
for index, c_test_result in enumerate(c_test_results, 1):
timestamp = c_test_result.get("timestamp")
if timestamp:
timestamp = datetime_extension.from_json(timestamp)
duration = c_test_result.get("duration", None)
if duration:
duration = timedelta_extension.from_json(duration)
duration = (duration.microseconds +
(duration.seconds * 10**6) +
(duration.days * 24 * 60 * 60 * 10**6))
result = self._translate_result_string(c_test_result["result"])
data.append((
s_test_run.id,
index,
timestamp,
duration,
c_test_result.get("log_filename", None),
result,
c_test_result.get("measurement", None),
c_test_result.get("message", None),
c_test_result.get("log_lineno", None),
s_test_run.test.id,
c_test_result.get("test_case_id", None),
))
cursor.executemany(
"""
INSERT INTO dashboard_app_testresult (
test_run_id,
relative_index,
timestamp,
microseconds,
filename,
result,
measurement,
message,
lineno,
_order,
test_case_id
) select
%s,
%s,
%s,
%s,
%s,
%s,
%s,
%s,
%s,
0,
dashboard_app_testcase.id
FROM dashboard_app_testcase
WHERE dashboard_app_testcase.test_id = %s
AND dashboard_app_testcase.test_case_id
= %s
""", data)
cursor.close()