def test_003_get(self):
"""This test does the following:
* Verifies error with invalid test_run_id
* Gets a valid TestRun
* verifies that the TestRun retrieves has the expected test_run_id
"""
with self.assertRaises(PylarionLibException):
TestRun(project_id=DEFAULT_PROJ, test_run_id="InValid")
tr = TestRun(project_id=DEFAULT_PROJ, test_run_id=TEST_RUN_ID)
self.assertEqual(tr.test_run_id, TEST_RUN_ID)
def revert_svn():
proj = Project(Project.default_project)
proj_grp = proj.project_group.name
svn_url = "%s/%s/%s" % (proj.repo, proj_grp, proj.project_id)
LOCAL_DIR = "/tmp/%s" % proj.project_id
USER = proj.logged_in_user_id
PASS = proj.session.password
svn = pysvn.Client()
svn.set_default_username(USER)
svn.set_default_password(PASS)
svn.set_store_passwords(False)
svn.checkout(svn_url, LOCAL_DIR)
svn_log = svn.log(LOCAL_DIR)
# first_revision = svn_log[-1].revision
for log in svn_log:
if log.revision.number == REVERT_TO:
break
first_revision = log.revision
last_revision = svn_log[0].revision
svn.merge(LOCAL_DIR, last_revision, LOCAL_DIR, first_revision, LOCAL_DIR)
svn.checkin(LOCAL_DIR, "revert to original version")
# The original Example template query has an extra colon.
# After revert, must fix that.
tmpl = TestRun(project_id=proj.project_id, test_run_id="Example")
tmpl.query = tmpl.query.replace("::", ":")
tmpl.update()
def update_all_case_results_for_run(self, run, result, user, comment):
run = run.strip()
tr = TestRun(run, None, TestRun.default_project)
print('\nUpdate %s:' % run)
if not comment:
comment = ''
print('Total records: %d' % len(tr.records))
print('Updated Date Time Result CaseID')
print('------------------- ------ -----------')
if user == 'None':
user = TestRun.logged_in_user_id
for rec in tr.records:
rec.executed = datetime.datetime.now()
rec.executed_by = user
executed_str = str(rec.executed).split('.')[0]
rec.result = result
rec.comment = comment
print('%-20s %-7s %s' % (executed_str, result, rec.test_case_id))
tr.update_test_record_by_object(rec.test_case_id, rec)
print('Done!')
def update_1_case_result_for_run(self, run, testcase, result, user,
comment):
if not comment:
comment = ''
tr = TestRun(run.strip(), None, TestRun.default_project)
print('Update %s:' % run)
if user == 'None':
user = TestRun.logged_in_user_id
is_found = False
for rec in tr.records:
if rec.test_case_id == testcase:
is_found = True
rec.executed = datetime.datetime.now()
rec.executed_by = user
rec.result = result
rec.comment = comment
tr.update_test_record_by_object(testcase, rec)
print("%4sSet %s to %s (verdict comment: '%s')" %
('', testcase, result, comment))
return 0
if not is_found:
print('Test case %s is not found in run.' % testcase)
def test_run(path, test_run_id, test_template_id, user, project):
"""Execute a test run based on jUnit XML file."""
results = parse_junit(path)
try:
test_run = TestRun(test_run_id, project_id=project)
click.echo('Test run {0} found.'.format(test_run_id))
except PylarionLibException as err:
click.echo(err, err=True)
click.echo('Creating test run {0}.'.format(test_run_id))
test_run = TestRun.create(project, test_run_id, test_template_id)
for result in results:
test_case_id = '{0}.{1}'.format(result['classname'], result['name'])
test_case = TestCase.query(test_case_id)
if len(test_case) == 0:
click.echo(
'Was not able to find test case with id {0}, skipping...'.
format(test_case_id))
continue
status = POLARION_STATUS[result['status']]
work_item_id = test_case[0].work_item_id
click.echo(
'Adding test record for test case {0} with status {1}.'.format(
work_item_id, status))
try:
test_run.add_test_record_by_fields(
test_case_id=work_item_id,
test_result=status,
test_comment=result.get('message'),
executed_by=user,
executed=datetime.datetime.now(),
duration=float(result.get('time', '0')))
except PylarionLibException as err:
click.echo('Skipping test case {0}.'.format(work_item_id))
click.echo(err, err=True)
def test_custom_testrun(self):
testrun = TestRun(project_id=DEFAULT_PROJ, test_run_id=TEST_RUN_ID)
self.assertIsNotNone(testrun.arch)
with self.assertRaises(PylarionLibException):
testrun.arch = "bad"
valid_values = testrun.get_valid_field_values("arch")
testrun.arch = valid_values[0]
self.assertEqual(valid_values[0], testrun.arch)
def update_run(self,
run,
template=None,
plannedin=None,
assignee=None,
status=None,
description=None,
is_template=False):
run = run.strip()
query_ful = 'project.id:%s AND id:%s' % (TestRun.default_project, run)
fields = [
'query', 'created', 'test_run_id', 'select_test_cases_by',
'status', 'plannedin', 'assignee', 'author'
]
st = TestRun.search(query_ful, fields, 'created', -1, is_template)
# Update run if exists, otherwise create it.
if st:
print('Update the existing run: %s' % run)
tr = TestRun(run, None, TestRun.default_project)
# set fields
if assignee != 'None':
tr.assignee = assignee
print('%4sSet Assignee to %s' % ('', assignee))
if plannedin is not None:
tr.plannedin = plannedin
print('%4sSet Plannedin to %s' % ('', plannedin))
if status is not None:
tr.status = status
print('%4sSet Status to %s' % ('', status))
if description is not None:
tr.description = description
print('%4sSet Description to %s' % ('', description))
tr.update()
else:
tr = TestRun.create(TestRun.default_project,
run,
template,
assignee=assignee,
plannedin=plannedin,
status=status,
description=description)
# display fields
if assignee != 'None':
print('%4sSet Assignee to %s' % ('', assignee))
if plannedin is not None:
print('%4sSet Plannedin to %s' % ('', plannedin))
if status is not None:
print('%4sSet Status to %s' % ('', status))
if description is not None:
print('%4sSet Description to %s' % ('', description))
print('Created %s:' % run)
def test_uri_obj(self):
testrun2 = TestRun(project_id=DEFAULT_PROJ, test_run_id=TEST_RUN_ID)
self.assertEqual(testrun2.template, "example")
new_template = TestRun.create_template(DEFAULT_PROJ,
TEMPLATE_ID,
"example",
title=TEMPLATE_TITLE)
testrun2.template = new_template
self.assertEqual(testrun2.template, new_template.test_run_id)
self.assertNotEqual(testrun2.template, "example")
def test_011_customfield_object(self):
"""This test does the following:
* gets a TestRun
* Adds a Plan to it
* Verifies that the plan was added
* Verifies that a non valid plan cant be added
"""
tr = TestRun(project_id=DEFAULT_PROJ, test_run_id=TEST_RUN_ID)
with self.assertRaises(PylarionLibException):
tr.plannedin = "not_valid"
tr.plannedin = self.NEW_PLAN
self.assertEquals(tr.plannedin, self.NEW_PLAN)
tr.update()
def test_008_update(self):
"""This test does the following:
* gets a TestRun
* modifies an attribute
* updates the TestRun
* reloads the TestRun
* verifies that the TestRun attribute has changed
"""
tr = TestRun(project_id=DEFAULT_PROJ, test_run_id=TEST_RUN_ID)
tr.type = "featureverification"
tr.update()
tr.reload()
self.assertEqual(tr.type, "featureverification")
def print_testcases_from_run(self, run):
tr = TestRun(run, None, TestRun.default_project)
print('(Only CaseID can be displayed when --run=$template)')
print('List cases for: %s\n' % run)
ttstr = ('Created Time %8sStatus %1sExecutedBy %2sCaseID' %
('', '', ''))
lnstr = ('------------%9s------%2s----------%3s------' % ('', '', ''))
print(ttstr)
print(lnstr)
for rec in tr.records:
time = str(rec.executed).split('.')[0]
print('%-21s%-9s%-12s%-10s' %
(time, rec.result, rec.executed_by, rec.test_case_id))
def test_custom_testrun2(self):
"""this test does the following:
* Instantiate an empty test run.
* verify that the custom field exists and is None
* test an invalid value
* test a valid value
* verify that it saves it in the attribute
"""
testrun = TestRun()
self.assertIsNone(testrun.arch)
with self.assertRaises(PylarionLibException):
testrun.arch = "bad"
valid_values = testrun.get_valid_field_values("arch")
testrun.arch = valid_values[0]
self.assertEqual(valid_values[0], testrun.arch)
def polarion_collect_items(config):
polarion_run = config.getoption('polarion_run')
polarion_proj = config.getoption('polarion_project')
tr = TestRun(project_id=polarion_proj, test_run_id=polarion_run)
# caching TestRun
config.option.test_run_obj = tr
items = {rec.test_case_id: rec for rec in tr.records}
if not items:
pytest.fail('Failed to collect items from polarion {} run'.format(polarion_run))
# caching test records
config.option.test_run_records = items
return items
def test_run(
context, path, source_code_path, test_run_id, test_run_type,
test_template_id, user, custom_fields, project):
"""Execute a test run based on jUnit XML file."""
custom_fields = load_custom_fields(custom_fields)
test_run_id = re.sub(INVALID_CHARS_REGEX, '', test_run_id)
testcases = {
generate_test_id(test): test.tokens.get('id')
for test in itertools.chain(
*testimony.get_testcases([source_code_path]).values()
)
}
results = parse_junit(path)
try:
test_run = TestRun(test_run_id, project_id=project)
click.echo('Test run {0} found.'.format(test_run_id))
except PylarionLibException as err:
click.echo(err, err=True)
click.echo('Creating test run {0}.'.format(test_run_id))
test_run = TestRun.create(
project, test_run_id, test_template_id, type=test_run_type,
**custom_fields)
update = False
if test_run.type != test_run_type:
test_run.type = test_run_type
update = True
for field, value in custom_fields.items():
if getattr(test_run, field) != value:
setattr(test_run, field, value)
click.echo(
'Test Run {0} updated with {1}={2}.'.format(
test_run_id, field, value)
)
update = True
if update:
test_run.update()
OBJ_CACHE['test_run'] = test_run
OBJ_CACHE['user'] = user
OBJ_CACHE['testcases'] = testcases
TestRun.session.tx_begin()
pool = multiprocessing.Pool(context.obj['jobs'])
pool.map(add_test_record, results)
pool.close()
pool.join()
TestRun.session.tx_commit()
def test_005_test_record_by_fields(self):
"""This test does the following:
* gets a TestRun object
* Adds a TestRecord to it
** verifies that it fails with an invalid result
** verifies that it fails if it adds a duplicate case.
* Adds an attachment to the record.
** verifies that the attachment is there
* deletes the attachment
** verifies the attachment is not there
* updates the test record.
"""
tr = TestRun(project_id=DEFAULT_PROJ, test_run_id=TEST_RUN_ID)
with self.assertRaises(PylarionLibException):
tr.add_test_record_by_fields(self.NEW_TEST_CASE, "invalid",
"No Comment", tr.logged_in_user_id,
datetime.datetime.now(), "50.5")
tr.add_test_record_by_fields(self.NEW_TEST_CASE, "passed",
"No Comment", tr.logged_in_user_id,
datetime.datetime.now(), "50.5")
tr.reload()
self.assertEqual(tr.status, "finished")
# test that the same case cannot be added multiple times.
with self.assertRaises(PylarionLibException):
tr.add_test_record_by_fields(self.NEW_TEST_CASE, "passed",
"No Comment", tr.logged_in_user_id,
datetime.datetime.now(), "50.5")
tr.reload()
rec = tr.records[-1]
tr.add_attachment_to_test_record(rec.test_case_id, ATTACH_PATH,
ATTACH_TITLE)
tr.reload()
rec = tr.records[-1]
self.assertTrue(len(rec.attachments) == 1)
self.assertEqual(rec.attachments[0].title, ATTACH_TITLE)
tr.delete_attachment_from_test_record(rec.test_case_id,
rec.attachments[0].filename)
tr.reload()
rec = tr.records[-1]
self.assertEqual(rec.attachments, [])
tr.update_test_record_by_fields(rec.test_case_id, rec.result,
"Yes Comment", rec.executed_by,
rec.executed, rec.duration)
tr.reload()
rec = tr.records[-1]
self.assertEqual(rec.comment, "Yes Comment")
def test_013_incident_report_test(self):
"""This test does the following:
* gets a TestRun
* gets a TestCase
* adds test_steps
* creates a TestRecord
* populates the TestRecord
* Fail the testRecord
* reloads the TestCase
* Verifies that an Incident Report was Created
"""
tr = TestRun(project_id=DEFAULT_PROJ, test_run_id=TEST_RUN_ID)
tests = [["Test 1", "Result 1"], ["Test 2", "Result 2"],
["Test 3", "Result 3"]]
set_steps = []
for test in tests:
ts = TestStep()
ts.values = test
set_steps.append(ts)
tc = TestCase(work_item_id=self.NEW_TEST_CASE)
tc.set_test_steps(set_steps)
tc.update()
tc.reload()
steps = tc.test_steps.steps
results = []
for step in steps:
res = TestStepResult()
res.result = "failed"
res.comment = "This is the result"
results.append(res)
rec = TestRecord()
rec.test_step_results = results
rec.test_case_id = self.NEW_TEST_CASE
rec.comment = "Incident Report was Created"
rec.duration = "50.5"
rec.result = "failed"
rec.executed_by = tr.logged_in_user_id
rec.executed = datetime.datetime.now()
tr.update_test_record_by_object(self.NEW_TEST_CASE, rec)
tc.reload()
linked_work_items = tc.linked_work_items_derived
idx = len(linked_work_items) - 1
incident = Incident(project_id="proj1",
work_item_id=linked_work_items[idx].work_item_id)
self.assertIsNotNone(incident)
def test_arr_obj(self):
testrun = TestRun(project_id=DEFAULT_PROJ, test_run_id=TEST_RUN_ID)
recs = testrun.records
if not isinstance(recs, list):
recs = []
cnt = 0
else:
cnt = len(recs)
rec = TestRecord()
rec.test_case_id = self.TEST_CASE_ID
rec.executed = datetime.datetime.now()
rec.executed_by = USER
rec.duration = "5.0"
rec.result = "passed"
recs.append(rec)
testrun.records = recs
self.assertEqual(cnt + 1, len(testrun.records))
recs = testrun.records
rec = recs[-1]
self.assertEqual(rec.result, "passed")
def test_007_attachment(self):
"""This test does the following:
* add an attachment to the TestRun.
* verify that there is 1 attachment with the correct title
* verify the get_attachment function
* verify the get_attachments function
* delete the attachment
* verify that there are no attachments.
"""
tr = TestRun(project_id=DEFAULT_PROJ, test_run_id=TEST_RUN_ID)
tr.add_attachment(ATTACH_PATH, ATTACH_TITLE)
tr.reload()
self.assertEqual(len(tr.attachments), 1)
self.assertEqual(tr.attachments[0].title, ATTACH_TITLE)
attach = tr.get_attachment(tr.attachments[0].filename)
self.assertEqual(tr.attachments[0].title, attach.title)
lst_attach = tr.get_attachments()
self.assertEqual(lst_attach[0].title, attach.title)
tr.delete_attachment(tr.attachments[0].filename)
tr.reload()
self.assertEqual(tr.attachments, [])
def test_006_test_record_by_object(self):
"""This test does the following:
* gets a TestRun
* creates a TestRecord
* populates the TestRecord
* Tries to add a duplicate TestRecord (should fail)
* Adds a TestRecord
* Reloads the TestRun
* Verifies the TestRecord was added
* Updates the TestRecord
* Reloads the TestRun
* Verifies the TestRecord was modified
"""
tr = TestRun(project_id=DEFAULT_PROJ, test_run_id=TEST_RUN_ID)
rec = TestRecord()
rec.test_case_id = self.NEW_TEST_CASE
rec.comment = "No Comment"
rec.duration = "50.5"
# verify that it does not allow duplicate records.
# (same record was added in previous test)
with self.assertRaises(PylarionLibException):
tr.add_test_record_by_object(rec)
rec.test_case_id = self.NEW_TEST_CASE2
tr.add_test_record_by_object(rec)
tr.reload()
check_rec = tr.records[-1]
self.assertEqual(tr.status, "inprogress")
self.assertEqual(check_rec.test_case_id, self.NEW_TEST_CASE2)
rec.result = "blocked"
rec.executed_by = tr.logged_in_user_id
rec.executed = datetime.datetime.now()
tr.update_test_record_by_object(self.NEW_TEST_CASE2, rec)
tr.reload()
self.assertEqual(tr.status, "finished")
check_rec = tr.records[-1]
self.assertEqual(check_rec.result, "blocked")
def make_test_run():
return TestRun(project_id=TR_PROJECT_ID,
test_run_id='4_0_Node_0622_AutoInstallWithKickstart_Must')
def main():
isUpdateAutomationValue = False
# access excel file and update results
credentials = get_credentials()
http = credentials.authorize(httplib2.Http())
discoveryUrl = ('https://sheets.googleapis.com/$discovery/rest?version=v4')
service = discovery.build('sheets',
'v4',
http=http,
discoveryServiceUrl=discoveryUrl)
# https://docs.google.com/spreadsheets/d/1y4eBJhcZ0HsB5JUH5MPcFXWtc2zbP9XbgdCIF0S4iHs/edit#gid=1503195790
spreadsheetId = '1y4eBJhcZ0HsB5JUH5MPcFXWtc2zbP9XbgdCIF0S4iHs'
# Get all test runs by Polarion query, extract test run id and test run results (pass, fail, pending block, total...)
test_runs_uris = TestRun.search(
'NOT status:invalid AND plannedin.KEY:RHOS16'
) #updated:[20190627 TO 20190630]') #
# test_runs_uris = TestRun.search('20180625-0836')
print("Number of items %s" % len(test_runs_uris))
loop_counter = 1
missing_test_run_in_excel = ''
non_test_cases_item = 0
for test_run_uri in test_runs_uris:
# for i in range(106,130):
# test_run_uri = test_runs_uris[i]
#get excel values
rangeName = 'RHOS 16!A2:X'
result = service.spreadsheets().values().get(
spreadsheetId=spreadsheetId, range=rangeName).execute()
values = result.get('values', [])
value_input_option = 'RAW'
print('Updating test run number: ' + str(loop_counter))
loop_counter += 1
print(test_run_uri.uri)
test_run = TestRun(uri=test_run_uri.uri)
test_run_id = test_run.test_run_id
print('Test run title: ' + test_run.title)
print('Test run ID: ' + test_run.test_run_id)
records = test_run.records
pass_counter = 0
fail_counter = 0
pending_counter = 0
automation_counter = 0.0
critical_counter = 0
critical_auto_counter = 0
#automation_percentage = 0
blocked_counter = 0
total_counter = 0
#Collect inforamtion about test runs, how many test pass
if test_run.TestRunType == 'Acceptance':
for record in records:
if record.result == 'passed':
pass_counter += 1
elif record.result == 'failed':
fail_counter += 1
elif record.result == 'blocked':
blocked_counter += 1
else:
pending_counter += 1
else:
for record in records:
# print record.result
#check if test is automated
test = TestCase.query(record.test_case_id)
# print('Test case ID: ' + record.test_case_id)
# Check if the object type is a testcase and not a header for example!
if test and not Requirement.query(record.test_case_id):
#calculate critical automated and rest automated
if isUpdateAutomationValue:
if test[0].caseautomation.lower() == 'automated':
automation_counter += 1
if test[0].caseimportance.lower() == 'critical':
critical_auto_counter += 1
#count number of critical cases
if test[0].caseimportance.lower() == 'critical':
critical_counter += 1
if record.result == 'passed':
pass_counter += 1
elif record.result == 'failed':
fail_counter += 1
elif record.result == 'blocked':
blocked_counter += 1
else:
pending_counter += 1
else:
non_test_cases_item += 1
total_counter = pass_counter + fail_counter + blocked_counter + pending_counter
# if total_counter > 0:
# automation_percentage = int(float(automation_counter)/float(total_counter)) #*100
print('Total pass:'******'Total fail:', fail_counter)
print('Total blocked:', blocked_counter)
print('Total pending:', pending_counter)
print('Total automated:', automation_counter)
print('Number of critical:', critical_counter)
print('Number of critical auto:', critical_auto_counter)
#print ('Automation percentage:', automation_percentage)
print('Total number of test cases:', total_counter)
#column number in excel file and thier representation as hard coded value
row_counter = 1 # offset due to headers
title_column_number = 2
total_column_number = 8
pass_column_number = 9
fail_column_number = 10
blocked_column_number = 11
test_run_id_column_number = 20
automation_percentage_column_number = 18
critical_test_number = 22
is_test_run_exist_in_excel = None
if not values:
print('No data found.')
else:
for row in values:
is_test_run_exist_in_excel = False
row_counter += 1
# print(row_counter)
# if(row_counter==134):
# print('stop')
# Check that row contains test run id in cell R AND check that test_run_id is match
if row.__len__() >= test_run_id_column_number and row[
test_run_id_column_number] == test_run_id:
print('Row number is: ' + str(row_counter))
is_test_run_exist_in_excel = True
# print('%s, %s, %s, %s, %s, %s, %s :' % (row[title_column_number], row[total_column_number], row[pass_column_number], row[fail_column_number], row[blocked_column_number],row[automation_percentage_column_number], row[critical_test_number]))
values = [[
total_counter, total_counter, pass_counter,
fail_counter, blocked_counter
]]
body = {'values': values}
rangeName = 'RHOS 15!H' + str(row_counter) + ':L' + str(
row_counter)
result = service.spreadsheets().values().update(
spreadsheetId=spreadsheetId,
range=rangeName,
valueInputOption=value_input_option,
body=body).execute()
# # update automation percentage field
# values = [
# [automation_percentage]
# ]
# body = {
# 'values': values
# }
# rangeName = 'RHOS 13!S' + str(row_counter)
# result = service.spreadsheets().values().update(spreadsheetId=spreadsheetId, range=rangeName, valueInputOption='USER_ENTERED',body=body).execute()
# update PQI values...
if isUpdateAutomationValue and test_run.TestRunType != 'Acceptance':
values = [[
automation_counter, critical_counter,
critical_auto_counter
]]
body = {'values': values}
rangeName = 'RHOS 15!V' + str(
row_counter) + ':X' + str(row_counter)
result = service.spreadsheets().values().update(
spreadsheetId=spreadsheetId,
range=rangeName,
valueInputOption=value_input_option,
body=body).execute()
# done with update, move to next test run
break
#Check if test run exist in excel file and was updated
if not is_test_run_exist_in_excel:
missing_test_run_in_excel += test_run_id + ", "
print("Missing Test Runs in Excel: " + missing_test_run_in_excel)
print("Number of headers or requirements in test runs: ",
non_test_cases_item)
