def __init__(self):
self.shishito_support = ShishitoSupport()
self.api_token = self.shishito_support.get_opt('circleci_api_token')
self.circle_username = self.shishito_support.get_opt(
'circleci_username')
self.circle_project = self.shishito_support.get_opt('circleci_project')
self.circle_branch = self.shishito_support.get_opt('circleci_branch')
def __init__(self, driver):
self.driver = driver
self.shishito_support = ShishitoSupport()
self.base_url = self.shishito_support.get_opt('base_url')
self.default_implicit_wait = int(
self.shishito_support.get_opt('default_implicit_wait'))
self.timeout = int(self.shishito_support.get_opt('timeout'))
def setup_class(self):
# shishito / selenium support
self.tc = ShishitoSupport().get_test_control()
self.driver = self.tc.start_browser()
self.ts = SeleniumTest(self.driver)
self.etsy_host = os.environ['ETSY_HOST']
self.etsy_port = os.environ['ETSY_PORT']
self.etsy_url = 'http://' + self.etsy_host + ':' + self.etsy_port
# etsy url - create sql script to update cfg
with open(DB_DIR + "/tmp-etsy-location.sql", "w+") as f:
f.write("INSERT INTO config (id, value) values ('etsy_host', '" +
self.etsy_host + "');\n")
f.write("INSERT INTO config (id, value) values ('etsy_port', '" +
self.etsy_port + "');\n")
self.run_sql(self, '00', 'clean_db_00')
self.run_sql(self, '00', 'tmp-etsy-location')
# restart node
if call([BIN_DIR + '/restart_node']) != 0:
raise Exception("Error: Node restart failed (" + BIN_DIR +
"/restart_node)")
sleep(2)
# Set base URL - host taken from env.
prod_host = os.environ[
'PRODUCT_HOST'] if 'PRODUCT_HOST' in os.environ else '127.0.0.1'
self.base_url = 'http://' + prod_host + '/'
class BaseTestClass():
""" Contextual help test """
def run_sql(self, db_instance, script):
"run psql <script.sql> on db_instance (eg '00')"
for i in ['PGHOST', 'PGPORT', 'PGDATABASE', 'PGUSER', 'PGPASSWORD']:
os.environ[i] = os.environ[i + '_' + db_instance]
returncode = call([
'sh', '-c', 'psql -v ON_ERROR_STOP=1 < ' + DB_DIR + '/' + script +
'.sql > /dev/null'
])
if returncode != 0:
raise Exception("Error: DB script failed (" + DB_DIR + '/' +
script + ".sql)")
def set_etsy_testcase(self, tc_name):
url = self.etsy_url + '/set_test_id?test_id=' + tc_name
r = requests.get(url)
sleep(1)
if r.status_code != 200:
raise Exception("Error: cannot set test case on " + url)
def setup_class(self):
# shishito / selenium support
self.tc = ShishitoSupport().get_test_control()
self.driver = self.tc.start_browser()
self.ts = SeleniumTest(self.driver)
self.etsy_host = os.environ['ETSY_HOST']
self.etsy_port = os.environ['ETSY_PORT']
self.etsy_url = 'http://' + self.etsy_host + ':' + self.etsy_port
# etsy url - create sql script to update cfg
with open(DB_DIR + "/tmp-etsy-location.sql", "w+") as f:
f.write("INSERT INTO config (id, value) values ('etsy_host', '" +
self.etsy_host + "');\n")
f.write("INSERT INTO config (id, value) values ('etsy_port', '" +
self.etsy_port + "');\n")
self.run_sql(self, '00', 'clean_db_00')
self.run_sql(self, '00', 'tmp-etsy-location')
# restart node
if call([BIN_DIR + '/restart_node']) != 0:
raise Exception("Error: Node restart failed (" + BIN_DIR +
"/restart_node)")
sleep(2)
# Set base URL - host taken from env.
prod_host = os.environ[
'PRODUCT_HOST'] if 'PRODUCT_HOST' in os.environ else '127.0.0.1'
self.base_url = 'http://' + prod_host + '/'
def setup_method(self, method):
# reset DB instance 00
self.run_sql('00', 'clean_db_00')
self.run_sql('00', 'tmp-etsy-location')
self.driver.get(self.base_url)
def teardown_class(self):
self.tc.stop_browser()
pass
def teardown_method(self, method):
self.tc.stop_test(
get_test_info()) # save screenshot in case test fails
class ShishitoControlTest(object):
""" Base class for ControlTest objects. """
def __init__(self):
self.shishito_support = ShishitoSupport()
# create control environment object
control_env_obj = self.shishito_support.get_module('test_environment')
self.test_environment = control_env_obj(self.shishito_support)
self.driver = None
def start_browser(self):
""" Webdriver startup function.
:return: initialized webdriver
"""
base_url = self.shishito_support.get_opt('base_url')
config_section = self.shishito_support.get_opt('environment_configuration')
# call browser from proper environment
self.driver = self.test_environment.call_browser(config_section)
# load init url
if base_url:
self.test_init(base_url)
return self.driver
def start_test(self, reload_page=None):
""" To be executed before every test-case (test function).
:param reload_page:
"""
def stop_browser(self):
""" Webdriver termination function. """
self.driver.quit()
def stop_test(self, test_info):
""" To be executed after every test-case (test function). If test failed, function saves
screenshots created during test.
:param test_info: information about test
"""
if test_info.test_status not in ('passed', None):
# save screenshot in case test fails
screenshot_folder = os.path.join(self.shishito_support.project_root, 'screenshots')
if not os.path.exists(screenshot_folder):
os.makedirs(screenshot_folder)
file_name = re.sub('[^A-Za-z0-9_. ]+', '', test_info.test_name)
self.driver.save_screenshot(os.path.join(screenshot_folder, file_name + '.png'))
def test_init(self, url):
""" Executed only once after browser starts.
def __init__(self):
self.shishito_support = ShishitoSupport()
# create control environment object
control_env_obj = self.shishito_support.get_module('test_environment')
self.test_environment = control_env_obj(self.shishito_support)
self.drivers = []
def __init__(self):
self.shishito_support = ShishitoSupport()
self.email_address = self.shishito_support.get_opt('email_address')
self.email_imap = self.shishito_support.get_opt('email_imap')
self.email_password = self.shishito_support.get_opt('email_password')
self.email_mailbox = self.shishito_support.get_opt('email_mailbox')
self.mail = imaplib.IMAP4_SSL(self.email_imap)
self.mail.login(self.email_address, self.email_password)
def setup_class(self):
self.tc = ShishitoSupport().get_test_control()
self.driver = self.tc.start_browser()
self.ts = SeleniumTest(self.driver)
self.welcome_guide = WelcomeGuide(self.driver)
self.bottom_bar = BottomBar(self.driver)
self.url_bar = UrlBar(self.driver)
self.tabs_page = TabsPage(self.driver)
self.main_page = MainPage(self.driver)
self.bookmarks_page = BookmarksPage(self.driver)
self.dashobard_page = DashboardPage(self.driver)
self.history_page = HistoryPage(self.driver)
self.test_url = "google.com"
class CircleAPI(object):
"""Handles communication with Circle CI via REST API"""
def __init__(self):
self.shishito_support = ShishitoSupport()
self.api_token = self.shishito_support.get_opt('circleci_api_token')
self.circle_username = self.shishito_support.get_opt(
'circleci_username')
self.circle_project = self.shishito_support.get_opt('circleci_project')
self.circle_branch = self.shishito_support.get_opt('circleci_branch')
def collect_artifacts(self, destination_folder):
"""downloads build artifacts from CircleCI for latest build from specific branch"""
if not os.path.exists(destination_folder):
os.makedirs(destination_folder)
artifact_data = self.get_artifact_data()
for artifact in artifact_data:
self.save_artifact(artifact, destination_folder)
return bool(os.listdir(destination_folder))
def save_artifact(self, artifact, destination_folder):
""" saves artifact into specified folder """
file_name = artifact['url'].split('/')[-1]
file_request_url = artifact['url'] + '?circle-token=' + self.api_token
response = requests.get(file_request_url, stream=True)
response.raise_for_status()
with open(os.path.join(destination_folder, file_name),
'wb') as extension:
for block in response.iter_content(1024):
extension.write(block)
def get_artifact_data(self):
"""returns json with artifact urls"""
latest_dev_build_url = (
'https://circleci.com/api/v1/project/{circle_username}/{circle_project}/tree/'
'{circle_branch}?circle-token={api_token}&limit=1').format(
**self.__dict__)
headers = {'Accept': 'application/json'}
response = requests.get(latest_dev_build_url, headers=headers)
builds_json = json.loads(response.text)
build_number = builds_json[0]['build_num']
artifacts_url = (
'https://circleci.com/api/v1/project/{circle_username}/'
'{circle_project}/{build_number}/artifacts?circle-token={api_token}'
).format(build_number=build_number, **self.__dict__)
response = requests.get(artifacts_url, headers=headers)
return json.loads(response.text)
class CircleAPI(object):
"""Handles communication with Circle CI via REST API"""
def __init__(self):
self.shishito_support = ShishitoSupport()
self.api_token = self.shishito_support.get_opt("circleci_api_token")
self.circle_username = self.shishito_support.get_opt("circleci_username")
self.circle_project = self.shishito_support.get_opt("circleci_project")
self.circle_branch = self.shishito_support.get_opt("circleci_branch")
def collect_artifacts(self, destination_folder):
"""downloads build artifacts from CircleCI for latest build from specific branch"""
if not os.path.exists(destination_folder):
os.makedirs(destination_folder)
artifact_data = self.get_artifact_data()
for artifact in artifact_data:
self.save_artifact(artifact, destination_folder)
return bool(os.listdir(destination_folder))
def save_artifact(self, artifact, destination_folder):
""" saves artifact into specified folder """
file_name = artifact["url"].split("/")[-1]
file_request_url = artifact["url"] + "?circle-token=" + self.api_token
response = requests.get(file_request_url, stream=True)
response.raise_for_status()
with open(os.path.join(destination_folder, file_name), "wb") as extension:
for block in response.iter_content(1024):
extension.write(block)
def get_artifact_data(self):
"""returns json with artifact urls"""
latest_dev_build_url = (
"https://circleci.com/api/v1/project/{circle_username}/{circle_project}/tree/"
"{circle_branch}?circle-token={api_token}&limit=1"
).format(**self.__dict__)
headers = {"Accept": "application/json"}
response = requests.get(latest_dev_build_url, headers=headers)
builds_json = json.loads(response.text)
build_number = builds_json[0]["build_num"]
artifacts_url = (
"https://circleci.com/api/v1/project/{circle_username}/"
"{circle_project}/{build_number}/artifacts?circle-token={api_token}"
).format(build_number=build_number, **self.__dict__)
response = requests.get(artifacts_url, headers=headers)
return json.loads(response.text)
class EmailIMAP(object):
def __init__(self):
self.shishito_support = ShishitoSupport()
self.email_address = self.shishito_support.get_opt('email_address')
self.email_imap = self.shishito_support.get_opt('email_imap')
self.email_password = self.shishito_support.get_opt('email_password')
self.email_mailbox = self.shishito_support.get_opt('email_mailbox')
self.mail = imaplib.IMAP4_SSL(self.email_imap)
self.mail.login(self.email_address, self.email_password)
def retrieve_latest_email(self, timeout=36):
""" tries to retrieve latest email. If there are no emails,
it waits for 5 seconds and then tries again, until total (default 36*5 = 180s) timeout is reached """
count = 0
while count < timeout:
id_list = self.get_all_email_ids()
if id_list:
return self.get_message(id_list[-1])
time.sleep(5)
count += 1
def cleanup_emails(self):
""" cleans up email folder """
for num in self.get_all_email_ids():
self.mail.store(num, '+FLAGS', '\\Deleted')
self.mail.expunge()
def is_pattern_in_message(self, pattern, message):
""" return Boolean if pattern is to be found in message """
return bool(re.match(pattern, message))
# SUPPORT METHODS
def get_message(self, message_id):
""" gets email message """
data = self.mail.fetch(
message_id,
"(RFC822)")[1] # fetch the email body (RFC822) for the given ID
raw_email = data[0][
1] # here's the body, which is raw text of the whole email
return email.message_from_string(raw_email)
def get_all_email_ids(self):
""" connects to gmail account and return list of all email ids """
self.mail.select(self.email_mailbox)
data = self.mail.search(None, "ALL")[1]
return data[0].split() #ids split
def __init__(self, project_root=None, test_timestamp=None):
self.project_root = project_root or ShishitoSupport().project_root
self.current_folder = os.path.dirname(os.path.abspath(__file__))
self.timestamp = test_timestamp or time.strftime("%Y-%m-%d_%H-%M-%S")
self.result_folder = os.path.join(self.project_root, 'results',
self.timestamp)
def __init__(self):
self.shishito_support = ShishitoSupport()
# create control environment object
control_env_obj = self.shishito_support.get_module('test_environment')
self.test_environment = control_env_obj(self.shishito_support)
self.drivers = []
def __init__(self, project_root):
# set project root
self.project_root = project_root
# test timestamp - for storing results
self.test_timestamp = time.strftime("%Y-%m-%d_%H-%M-%S")
self.epoch = int(time.time())
# parse cmd args
self.cmd_args = self.handle_cmd_args()
# Get SUT build for use in reporting
self.test_build = self.cmd_args['build']
self.reporter = Reporter(project_root, self.test_timestamp)
self.shishito_support = ShishitoSupport(cmd_args=self.cmd_args,
project_root=self.project_root)
class EmailIMAP(object):
def __init__(self):
self.shishito_support = ShishitoSupport()
self.email_address = self.shishito_support.get_opt('email_address')
self.email_imap = self.shishito_support.get_opt('email_imap')
self.email_password = self.shishito_support.get_opt('email_password')
self.email_mailbox = self.shishito_support.get_opt('email_mailbox')
self.mail = imaplib.IMAP4_SSL(self.email_imap)
self.mail.login(self.email_address, self.email_password)
def retrieve_latest_email(self, timeout=36):
""" tries to retrieve latest email. If there are no emails,
it waits for 5 seconds and then tries again, until total (default 36*5 = 180s) timeout is reached """
count = 0
while count < timeout:
id_list = self.get_all_email_ids()
if id_list:
return self.get_message(id_list[-1])
time.sleep(5)
count += 1
def cleanup_emails(self):
""" cleans up email folder """
for num in self.get_all_email_ids():
self.mail.store(num, '+FLAGS', '\\Deleted')
self.mail.expunge()
def is_pattern_in_message(self, pattern, message):
""" return Boolean if pattern is to be found in message """
return bool(re.match(pattern, message))
# SUPPORT METHODS
def get_message(self, message_id):
""" gets email message """
data = self.mail.fetch(message_id, "(RFC822)")[1] # fetch the email body (RFC822) for the given ID
raw_email = data[0][1] # here's the body, which is raw text of the whole email
return email.message_from_string(raw_email)
def get_all_email_ids(self):
""" connects to gmail account and return list of all email ids """
self.mail.select(self.email_mailbox)
data = self.mail.search(None, "ALL")[1]
return data[0].split() #ids split
def setup_class(self):
self.tc = ShishitoSupport().get_test_control()
self.driver = self.tc.start_browser()
self.ts = SeleniumTest(self.driver)
# Page Objects
self.search_page = GoogleSearch(self.driver)
self.doodles = GoogleDoodles(self.driver)
def __init__(self):
self.shishito_support = ShishitoSupport()
self.email_address = self.shishito_support.get_opt('email_address')
self.email_imap = self.shishito_support.get_opt('email_imap')
self.email_password = self.shishito_support.get_opt('email_password')
self.email_mailbox = self.shishito_support.get_opt('email_mailbox')
self.mail = imaplib.IMAP4_SSL(self.email_imap)
self.mail.login(self.email_address, self.email_password)
def __init__(self, user, password, timestamp, epoch, build):
self.shishito_support = ShishitoSupport()
self.qastats_base_url = self.shishito_support.get_opt('qastats_url')
self.user = user
self.password = password
self.timestamp = timestamp
self.epoch = epoch
self.build = build
# project specific config
self.project_id = self.shishito_support.get_opt('qastats_project_id')
# shishito results
self.reporter = Reporter()
self.shishito_results = self.reporter.get_xunit_test_cases(timestamp)
self.default_headers = {'Content-Type': 'application/json'}
self.result_url = self.qastats_base_url + '/api/v1/results'
self.project_url = self.shishito_support.get_opt('base_url')
def setup_class(self):
# shishito / selenium support
self.tc = ShishitoSupport().get_test_control()
try:
# Set base URL - host taken from env.
prod_host = os.environ[
'QA_PRODUCT_HOST'] if 'QA_PRODUCT_HOST' in os.environ else '127.0.0.1'
prod_port = os.environ[
'QA_WEB_HTTPS_PORT'] if 'QA_WEB_HTTPS_PORT' in os.environ else '80'
login_host = os.environ[
'QA_PRODUCT_HOST_AUTH'] if 'QA_PRODUCT_HOST_AUTH' in os.environ else prod_host
self.base_url = 'https://' + prod_host + ':' + prod_port + '/'
# Login URLs
self.login_url_http = 'http://' + login_host + ':' + os.environ[
'QA_AUTH_HTTP_PORT'] + '/'
self.login_url_https = 'https://' + login_host + ':' + os.environ[
'QA_AUTH_HTTPS_PORT'] + '/'
self.web_api_url_https = 'https://' + prod_host + ':' + prod_port + '/api/v1/'
except:
print('*** setup_class failed ***')
raise
def __init__(self, user, password, timestamp, build):
self.shishito_support = ShishitoSupport()
self.test_rail_instance = self.shishito_support.get_opt(
'test_rail_url')
self.user = user
self.password = password
self.timestamp = timestamp
# project specific config
self.project_id = self.shishito_support.get_opt('test_rail_project_id')
self.section_id = self.shishito_support.get_opt('test_rail_section_id')
self.test_plan_id = self.shishito_support.get_opt(
'test_rail_test_plan_id')
self.test_plan_name = self.shishito_support.get_opt(
'test_rail_test_plan_name') or build
self.suite_id = self.shishito_support.get_opt('test_rail_suite_id')
# shishito results
self.reporter = Reporter()
self.shishito_results = self.reporter.get_xunit_test_cases(timestamp)
self.default_headers = {'Content-Type': 'application/json'}
self.uri_base = self.test_rail_instance + '/index.php?/api/v2/'
def __init__(self, project_root):
# set project root
self.project_root = project_root
# test timestamp - for storing results
self.test_timestamp = time.strftime("%Y-%m-%d_%H-%M-%S")
# parse cmd args
self.cmd_args = self.handle_cmd_args()
self.reporter = Reporter(project_root, self.test_timestamp)
self.shishito_support = ShishitoSupport(
cmd_args=self.cmd_args,
project_root=self.project_root
)
def __init__(self, user, password, timestamp):
self.shishito_support = ShishitoSupport()
self.test_rail_instance = self.shishito_support.get_opt('test_rail_url')
self.user = user
self.password = password
self.timestamp = timestamp
# project specific config
self.project_id = self.shishito_support.get_opt('test_rail_project_id')
self.section_id = self.shishito_support.get_opt('test_rail_section_id')
self.test_plan_id = self.shishito_support.get_opt('test_rail_test_plan_id')
self.suite_id = self.shishito_support.get_opt('test_rail_suite_id')
# shishito results
self.reporter = Reporter()
self.shishito_results = self.reporter.get_xunit_test_cases(timestamp)
self.default_headers = {'Content-Type': 'application/json'}
self.uri_base = self.test_rail_instance + '/index.php?/api/v2/'
def __init__(self, user, password, timestamp, epoch, build):
self.shishito_support = ShishitoSupport()
self.qastats_base_url = self.shishito_support.get_opt('qastats_url')
self.user = user
self.password = password
self.timestamp = timestamp
self.epoch = epoch
self.build = build
# project specific config
self.project_id = self.shishito_support.get_opt('qastats_project_id')
# shishito results
self.reporter = Reporter()
self.shishito_results = self.reporter.get_xunit_test_cases(timestamp)
self.default_headers = {'Content-Type': 'application/json'}
self.result_url = self.qastats_base_url + '/api/v1/results'
self.project_url = self.shishito_support.get_opt('base_url')
def __init__(self, project_root):
# set project root
self.project_root = project_root
# test timestamp - for storing results
self.test_timestamp = time.strftime("%Y-%m-%d_%H-%M-%S")
self.epoch = int(time.time())
# parse cmd args
self.cmd_args = self.handle_cmd_args()
# Get SUT build for use in reporting
self.test_build = self.cmd_args['build']
self.reporter = Reporter(project_root, self.test_timestamp)
self.shishito_support = ShishitoSupport(
cmd_args=self.cmd_args,
project_root=self.project_root
)
class TestMainPage():
""" Contextual help test """
def setup_class(self):
self.tc = ShishitoSupport().get_test_control()
self.driver = self.tc.start_browser()
self.ts = SeleniumTest(self.driver)
# Page Objects
self.search_page = GoogleSearch(self.driver)
self.doodles = GoogleDoodles(self.driver)
def teardown_class(self):
self.tc.stop_browser()
def setup_method(self, method):
self.tc.start_test(True)
def teardown_method(self, method):
test_info = get_test_info()
self.tc.stop_test(test_info)
### Tests ###
@pytest.mark.smoke
def test_google_search(self):
""" test google search """
self.ts.click_and_wait(self.search_page.luck)
self.ts.click_and_wait(self.doodles.doodle_archive)
Assert.equal(self.driver.title, 'Google Doodles')
def test_failing(self):
""" test google title """
# self.execution_id = self.tc.get_execution_id("MET-11")
Assert.equal(self.driver.title, 'Yahoo')
def test_good_title(self):
""" test google title """
self.search_page.search_field.send_keys('Jaromir Jagr')
self.ts.click_and_wait(self.search_page.search_button)
time.sleep(3)
Assert.equal(self.search_page.jagr_title.text, u'Jaromír Jágr')
class TestMainPage():
""" Contextual help test """
def setup_class(self):
self.tc = ShishitoSupport().get_test_control()
self.driver = self.tc.start_browser()
self.ts = SeleniumTest(self.driver)
def teardown_class(self):
self.tc.stop_browser()
def setup_method(self, method):
self.tc.start_test(True)
def teardown_method(self, method):
self.tc.stop_test(get_test_info())
### Tests ###
@pytest.mark.smoke
def test_google_search(self):
""" test google search """
Assert.equal(self.driver.title, 'Google')
class TestMainPage():
""" Contextual help test """
def setup_class(self):
self.tc = ShishitoSupport().get_test_control()
self.driver = self.tc.start_browser()
self.ts = SeleniumTest(self.driver)
def teardown_class(self):
self.tc.stop_browser()
def setup_method(self, method):
self.tc.start_test(True)
def teardown_method(self, method):
self.tc.stop_test(get_test_info())
### Tests ###
@pytest.mark.smoke
def test_google_search(self):
""" test google search """
Assert.equal(self.driver.title, 'Google')
class TestWelcomeGuide():
""" Test Adblock Browser Welcome Guide """
def setup_class(self):
self.tc = ShishitoSupport().get_test_control()
self.driver = self.tc.start_browser()
self.ts = SeleniumTest(self.driver)
self.welcome_guide = WelcomeGuide(self.driver)
self.bottom_bar = BottomBar(self.driver)
self.url_bar = UrlBar(self.driver)
self.tabs_page = TabsPage(self.driver)
self.main_page = MainPage(self.driver)
self.bookmarks_page = BookmarksPage(self.driver)
self.dashobard_page = DashboardPage(self.driver)
self.history_page = HistoryPage(self.driver)
self.test_url = "google.com"
def teardown_class(self):
self.tc.stop_browser()
def setup_method(self, method):
self.tc.start_test(True)
def teardown_method(self, method):
test_info = get_test_info()
self.tc.stop_test(test_info)
### Tests ###
@pytest.mark.smoke
def test_1welcome_guide(self):
self.welcome_guide.skip_guide()
Assert.true(self.ts.is_element_visible(self.bottom_bar._menu_icon))
def test_2open_site(self):
"""Test check if I can load page in app"""
self.ts.click_and_wait(self.url_bar.address_field)
Assert.true(self.ts.is_element_visible(self.url_bar._cancel))
Assert.true(
self.ts.is_element_visible(self.main_page._keyboard_locator))
url_replace_text = self.url_bar.address_field_text.text
self.url_bar.address_field_text.send_keys(self.test_url)
self.ts.click_and_wait(self.main_page.go_button)
url_text = self.url_bar.address_field_text.text
self.ts.click_and_wait(self.bottom_bar.bookmarks_icon)
Assert.not_equal(url_text, url_replace_text, 'Site is not loaded')
def test_3new_tab_dialog(self):
#Test check if user click on new tab button - dialog is shown with plus icon
self.ts.click_and_wait(self.bottom_bar.tabs_icon)
self.ts.click_and_wait(self.tabs_page.plus_button)
self.ts.click_and_wait(self.bottom_bar.tabs_icon)
Assert.equal(len(self.tabs_page.tabs_list), 2,
'New tab is not created')
def test_4bookmark_dialog(self):
#Test open bookmarks dialig, need to add later assertion for added site
self.ts.click_and_wait(self.bottom_bar.tabs_icon)
self.ts.click_and_wait(self.main_page.bookmark_button)
Assert.true(
self.ts.is_element_visible(self.bookmarks_page._bookmark_dialog))
#TODO Assertions checking that site added to bookmarks
def test_5dashboard_dialog(self):
#Test open dashboard dialig, need to add later assertion for added site
self.ts.click_and_wait(self.main_page.dashboard_button)
Assert.true(
self.ts.is_element_visible(self.dashobard_page._dashboard_dialog))
#TODO Assertions checking that site added to dashboards
def test_6history_dialog(self):
#Test open history dialig, need to add later assertion for loaded site
self.ts.click_and_wait(self.main_page.history_button)
Assert.true(
self.ts.is_element_visible(self.history_page._history_dialog))
def setup_class(self):
self.tc = ShishitoSupport().get_test_control()
self.driver = self.tc.start_browser()
self.ts = SeleniumTest(self.driver)
class ShishitoRunner(object):
""" Base shishito test runner.
- runs python selenium tests on customizable configurations (using PyTest)
- archive the test results in .zip file """
def __init__(self, project_root):
# set project root
self.project_root = project_root
# test timestamp - for storing results
self.test_timestamp = time.strftime("%Y-%m-%d_%H-%M-%S")
self.epoch = int(time.time())
# parse cmd args
self.cmd_args = self.handle_cmd_args()
# Get SUT build for use in reporting
self.test_build = self.cmd_args['build']
self.reporter = Reporter(project_root, self.test_timestamp)
self.shishito_support = ShishitoSupport(
cmd_args=self.cmd_args,
project_root=self.project_root
)
def handle_cmd_args(self):
""" Retrieve command line arguments passed to the script.
:return: dict with parsed command line arguments
"""
parser = argparse.ArgumentParser(description='Selenium Python test runner execution arguments.')
parser.add_argument('--platform',
help='Platform on which run tests.',
dest='test_platform')
parser.add_argument('--environment',
help='Environment for which run tests.',
dest='test_environment')
parser.add_argument('--test_directory',
help='Directory where to lookup for tests')
parser.add_argument('--smoke',
help='Run only smoke tests',
action='store_true')
parser.add_argument('--browserstack',
help='BrowserStack credentials; format: "username:token"')
parser.add_argument('--saucelabs',
help='Saucelabs credentials; format: "username:token"')
parser.add_argument('--test_rail',
help='TestRail Test Management tool credentials; format: "username:password"')
parser.add_argument('--qastats',
help='QAStats Test Management tool credentials; format: "token"')
parser.add_argument('--node_webkit_chromedriver_path',
help='Path to chromedriver located in same directory as node-webkit application')
parser.add_argument('--app',
help='Path to appium application')
parser.add_argument('--test',
help='Run specified test (PyTest string expression)')
parser.add_argument('--build',
help='Specify build number for reporting purposes')
parser.add_argument('--maxfail',
help='stop after x failures')
args = parser.parse_args()
# return args dict --> for use in other classes
return vars(args)
def run_tests(self):
""" Execute tests for given platform and environment. Platform and Environment can be passed as command lines
argument or settings in config file.
"""
if __name__ == "__main__":
sys.exit('The runner cannot be executed directly.'
' You need to import it within project specific runner. Session terminated.')
# cleanup previous results
self.reporter.cleanup_results()
# import execution class
executor_class = self.shishito_support.get_module('platform_execution')
# executor_class = getattr(import_module(platform_path), 'ControlExecution')
executor = executor_class(self.shishito_support, self.test_timestamp)
# run test
exit_code = executor.run_tests()
# archive results + generate combined report
self.reporter.archive_results()
self.reporter.generate_combined_report()
# upload results to QAStats test management app
qastats_credentials = self.shishito_support.get_opt('qastats')
if qastats_credentials:
try:
qas_user, qas_password = qastats_credentials.split(':', 1)
except (AttributeError, ValueError):
raise ValueError('QAStats credentials were not specified! Unable to connect to QAStats.')
qastats = QAStats(qas_user, qas_password, self.test_timestamp, self.epoch, self.test_build)
qastats.post_results()
# upload results to TestRail test management app
test_rail_credentials = self.shishito_support.get_opt('test_rail')
if test_rail_credentials:
try:
tr_user, tr_password = test_rail_credentials.split(':', 1)
except (AttributeError, ValueError):
raise ValueError('TestRail credentials were not specified! Unable to connect to TestRail.')
test_rail = TestRail(tr_user, tr_password, self.test_timestamp, self.test_build)
test_rail.post_results()
return exit_code
class ShishitoControlTest(object):
""" Base class for ControlTest objects. """
def __init__(self):
self.shishito_support = ShishitoSupport()
# create control environment object
control_env_obj = self.shishito_support.get_module('test_environment')
self.test_environment = control_env_obj(self.shishito_support)
self.drivers = []
def start_browser(self, base_url = None):
""" Webdriver startup function.
:return: initialized webdriver
"""
config_section = self.shishito_support.get_opt('environment_configuration')
# call browser from proper environment
driver = self.test_environment.call_browser(config_section)
self.drivers.append(driver)
# load init url
if not base_url:
base_url = self.shishito_support.get_opt('base_url')
if base_url:
self.test_init(driver, base_url)
else:
self.test_init(driver)
return driver
def start_test(self, reload_page=None):
""" To be executed before every test-case (test function).
:param reload_page:
"""
def stop_browser(self):
""" Webdriver termination function. """
for driver in self.drivers:
driver.quit() # Cleanup the driver info
del self.drivers[:]
def stop_test(self, test_info, debug_events=None):
""" To be executed after every test-case (test function). If test failed, function saves
screenshots created during test.
:param test_info: information about test
"""
if test_info.test_status not in ('passed', None):
# save screenshot in case test fails
test_name = re.sub('[^A-Za-z0-9_.]+', '_', test_info.test_name)
# Capture screenshot and debug info from driver(s)
for driver in self.drivers:
if(self.shishito_support.test_platform == 'mobile'):
browser_name = 'appium'
else:
browser_name = driver.name
file_name = browser_name + '_' + test_name
ts = SeleniumTest(driver)
ts.save_screenshot(name=file_name)
#Save debug info to file
if debug_events is not None:
debugevent_folder = os.path.join(self.shishito_support.project_root, 'debug_events')
if not os.path.exists(debugevent_folder):
os.makedirs(debugevent_folder)
with open(os.path.join(debugevent_folder, file_name + '.json'), 'w') as logfile:
json.dump(debug_events, logfile)
def test_init(self, driver, url=None):
""" Executed only once after browser starts.
class SeleniumTest(object):
def __init__(self, driver):
self.driver = driver
self.shishito_support = ShishitoSupport()
self.base_url = self.shishito_support.get_opt('base_url')
self.default_implicit_wait = int(
self.shishito_support.get_opt('default_implicit_wait'))
self.timeout = int(self.shishito_support.get_opt('timeout'))
def save_screenshot(self, name=None, project_root=None):
""" Saves application screenshot """
if not name:
# Use the name of browser and caller function (e.g. 'chrome_test_google_search'
name = self.driver.name + "_" + inspect.stack()[1][3]
if not project_root:
project_root = self.shishito_support.project_root
screenshot_folder = os.path.join(project_root, 'screenshots')
if not os.path.exists(screenshot_folder):
os.makedirs(screenshot_folder)
existing_images = glob.glob(
os.path.join(screenshot_folder, name + '_*.png'))
actual_pic_nr = len(existing_images) + 1
self.driver.save_screenshot(
os.path.join(screenshot_folder,
'{}_{}.png'.format(name, actual_pic_nr)))
def save_file_from_url(self, file_path, url):
""" Saves file from url """
if os.path.isfile(file_path):
print('File %s already exists.' % file_path)
return
response = requests.get(url, stream=True)
response.raise_for_status()
with open(file_path, 'wb') as save_file:
for block in response.iter_content(1024):
if not block:
break
save_file.write(block)
# Deprecated use property directly
def get_base_url(self):
return self.base_url
# Deprecated use property directly
def get_current_url(self):
return self.current_url
@property
def current_url(self):
""" Return the url for the current page."""
return self.driver.current_url
def hover_on(self, element):
""" Mouse over specific element """
mouse_over = ActionChains(self.driver).move_to_element(element)
mouse_over.perform()
def go_to_page(self, url):
""" Opens url in currently active window """
self.driver.get(url)
self.driver.implicitly_wait(self.default_implicit_wait)
def click_and_wait(self, element, locator=None):
""" clicks on a element and then waits for specific element to be present or simply waits implicitly """
element.click()
if locator:
self.wait_for_element_ready(locator)
else:
self.driver.implicitly_wait(10)
def check_images_are_loaded(self):
""" checks all images on the pages and verifies if they are properly loaded """
script = 'return arguments[0].complete && typeof arguments[0].naturalWidth' \
' != "undefined" && arguments[0].naturalWidth > 0'
images_not_loaded = []
for image in self.driver.find_elements_by_tag_name('img'):
loaded = self.driver.execute_script(script, image)
if not loaded and image.get_attribute('src'):
images_not_loaded.append(
'%s: %s' % (self.driver.title, image.get_attribute('src')))
return images_not_loaded
def is_element_present(self, locator):
"""
True if the element at the specified locator is present in the DOM.
Note: It returns false immediately if the element is not found.
"""
self.driver.implicitly_wait(0)
try:
self.driver.find_element(*locator)
return True
except NoSuchElementException:
return False
finally:
# set the implicit wait back
self.driver.implicitly_wait(self.default_implicit_wait)
def is_element_visible(self, locator):
"""
True if the element at the specified locator is visible in the browser.
Note: It uses an implicit wait if element is not immediately found.
"""
try:
return self.driver.find_element(*locator).is_displayed()
except (NoSuchElementException, ElementNotVisibleException):
return False
def is_element_not_visible(self, locator):
"""
True if the element at the specified locator is not visible.
Note: It returns true immediately if the element is not found.
"""
self.driver.implicitly_wait(0)
try:
return not self.driver.find_element(*locator).is_displayed()
except (NoSuchElementException, ElementNotVisibleException):
return True
finally:
# set the implicit wait back
self.driver.implicitly_wait(self.default_implicit_wait)
def wait_for_element_present(self, locator, timeout=None):
""" Wait for the element at the specified locator
to be present in the DOM. """
timeout = timeout or self.timeout
count = 0
while not self.is_element_present(locator):
time.sleep(1)
count += 1
if count == timeout:
raise Exception('{0} has not loaded'.format(locator))
def wait_for_element_visible(self, locator, timeout=None):
"""
Wait for the element at the specified locator to be visible.
"""
timeout = timeout or self.timeout
count = 0
while not self.is_element_visible(locator):
time.sleep(1)
count += 1
if count == timeout:
raise Exception("{0} is not visible".format(locator))
def wait_for_element_not_visible(self, locator, timeout=None):
"""
Wait for the element at the specified locator not to be visible anymore.
"""
timeout = timeout or self.timeout
count = 0
while self.is_element_visible(locator):
time.sleep(1)
count += 1
if count == timeout:
raise Exception("{0} is still visible".format(locator))
def wait_for_element_not_present(self, locator, timeout=None):
""" Wait for the element at the specified locator
not to be present in the DOM. """
timeout = timeout or self.timeout
self.driver.implicitly_wait(0)
try:
WebDriverWait(
self.driver,
timeout).until(lambda s: len(self.find_elements(*locator)) < 1)
return True
except TimeoutException:
Assert.fail(TimeoutException)
finally:
self.driver.implicitly_wait(self.default_implicit_wait)
def wait_for_text_to_match(self, text, locator, max_count=20, delay=0.25):
""" Waits for element text to match specified text, until certain deadline """
element = self.driver.find_element(*locator)
counter = 0
while element.text != text:
if counter < max_count:
time.sleep(delay)
counter += 1
element = self.driver.find_element(*locator)
else:
Assert.fail('"' + text + '" text did not match "' +
element.text + '" after ' + str(counter * delay) +
' seconds')
break
def wait_for_attribute_value(self,
attribute,
attribute_text,
locator,
max_count=20,
delay=0.25):
""" Waits for element attribute value to match specified text, until certain deadline """
element = self.driver.find_element(*locator)
counter = 0
while element.get_attribute(attribute) != attribute_text:
if counter < max_count:
time.sleep(delay)
counter += 1
else:
Assert.fail('"' + attribute_text + '" text did not match "' +
element.get_attribute(attribute) + '" after ' +
str(counter * delay) + ' seconds')
break
def wait_for_element_ready(self, locator, timeout=None):
""" Waits until certain element is present and clickable """
timeout = timeout or self.timeout
WebDriverWait(self.driver, timeout).until(
EC.presence_of_element_located(locator),
'Element specified by {0} was not present!'.format(locator))
WebDriverWait(self.driver, timeout).until(
EC.element_to_be_clickable(locator),
'Element specified by {0} did not become clickable!'.format(
locator))
def find_element(self, locator):
""" Return the element at the specified locator."""
return self.driver.find_element(*locator)
def find_elements(self, locator):
""" Return a list of elements at the specified locator."""
return self.driver.find_elements(*locator)
def find_elements_with_text(self, text, locator):
""" Find elements that have specified text """
elements = self.driver.find_elements(*locator)
selected = [item for item in elements if item.text == text]
return selected[0] if len(selected) == 1 else selected
def link_destination(self, locator):
""" Return the href attribute of the element at the specified locator."""
link = self.driver.find_element(*locator)
return link.get_attribute('href')
def image_source(self, locator):
""" Return the src attribute of the element at the specified locator."""
link = self.driver.find_element(*locator)
return link.get_attribute('src')
def select_dropdown_value(self, select, value):
""" Set 'select' dropdown value """
select = Select(select)
option = [option for option in select.options
if option.text == value][0]
option.click()
def upload_file(self, file_path, input_field_locator, delay=5):
""" uploads file through the file input field
@file_path: path to file (including the file name) relative to test project root
@input_field_locator: locator of input element with type="file"
@delay: seconds to wait for file to upload
"""
file_path = os.path.join(self.shishito_support.project_root, file_path)
self.driver.find_element(*input_field_locator).send_keys(file_path)
time.sleep(delay)
def execute_js_script(self, script, arguments=None):
"""execute any js command with arguments or without it"""
script_value = self.driver.execute_script(script, arguments)
return script_value
def open_new_tab(self, url):
"""Open new tab using keyboard, for now work only in Firefox and IE, in Chrome use js script to open tab """
ActionChains(self.driver).send_keys(Keys.CONTROL, "t").perform()
windows = self.driver.window_handles
self.driver.switch_to_window(windows[-1])
self.driver.get(url)
def switch_new_tab(self):
"""switch to new tab/window"""
windows = self.driver.window_handles
self.driver.switch_to_window(windows[-1])
def switch_first_tab(self):
"""Close current tab, switch to first tab/window"""
windows = self.driver.window_handles
self.driver.close()
self.driver.switch_to_window(windows[0])
class QAStats(object):
""" QAStats object """
def __init__(self, user, password, timestamp, epoch, build):
self.shishito_support = ShishitoSupport()
self.qastats_base_url = self.shishito_support.get_opt('qastats_url')
self.user = user
self.password = password
self.timestamp = timestamp
self.epoch = epoch
self.build = build
# project specific config
self.project_id = self.shishito_support.get_opt('qastats_project_id')
# shishito results
self.reporter = Reporter()
self.shishito_results = self.reporter.get_xunit_test_cases(timestamp)
self.default_headers = {'Content-Type': 'application/json'}
self.result_url = self.qastats_base_url + '/api/v1/results'
self.project_url = self.shishito_support.get_opt('base_url')
def post_results(self):
""" Create test-cases on QAStats, adds a new test run and update results for the run
{
"project_id": 123,
"timestamp": 1470133472,
"build": "773", // optional
"environment": "Firefox" // optional
"branch": "develop", // optional
"git": "ae232a", // optional
"results": [
{ "test": "test_login", "result": "pass" }, // [pass fail err nr]
...
],
}
"""
for (i, run) in enumerate(self.shishito_results):
environment = run['name']
m = re.match('^(.*)\.xml$', environment)
if m != None: environment = m.group(1)
payload = {
'project_id': self.project_id,
'timestamp': self.epoch + i,
'environment': environment
}
if self.build:
payload['build'] = self.build
if 'QA_BRANCH_TO_TEST' in os.environ:
payload['branch'] = os.environ['QA_BRANCH_TO_TEST']
if 'QA_GIT_COMMIT' in os.environ:
payload['git'] = os.environ["QA_GIT_COMMIT"]
elif 'CIRCLE_REPOSITORY_URL' in os.environ:
print('github url is known')
payload['git'] = os.environ['CIRCLE_REPOSITORY_URL']
if 'CIRCLE_TEST_REPORTS' in os.environ:
print('result url is known')
payload['reporturl'] = os.environ.get('CIRCLE_TEST_REPORTS')
if self.project_url is not None:
payload['testurl'] = self.project_url
status_map = {
'error': 'err',
'failure': 'fail',
'success': 'pass',
'skipped': 'nr'
}
results = [{
'test': t['name'],
'result': status_map[t['result']]
} for t in run['cases']]
payload['results'] = results
print(payload)
json_payload = json.dumps(payload)
r = requests.post(self.result_url,
auth=(self.user, self.password),
data=json_payload,
headers=self.default_headers)
if r.status_code == requests.codes.ok:
try:
resp = r.json()
if 'result' in resp and resp['result'] == 'OK':
print("Results uploaded to QAStats")
return True
except (ValueError, AttributeError):
pass
print("Error: uploading tests to QAStats\n\n", json_payload, "\n")
print("\tStatus-code:\t" + str(r.status_code) + "\n")
for n, v in r.headers.items():
print("\t" + n + "\t" + v)
print("")
print(r.text)
class TestRail(object):
""" TestRail object """
def __init__(self, user, password, timestamp, build):
self.shishito_support = ShishitoSupport()
self.test_rail_instance = self.shishito_support.get_opt('test_rail_url')
self.user = user
self.password = password
self.timestamp = timestamp
# project specific config
self.project_id = self.shishito_support.get_opt('test_rail_project_id')
self.section_id = self.shishito_support.get_opt('test_rail_section_id')
self.test_plan_id = self.shishito_support.get_opt('test_rail_test_plan_id')
self.test_plan_name = self.shishito_support.get_opt('test_rail_test_plan_name') or build
self.suite_id = self.shishito_support.get_opt('test_rail_suite_id')
# shishito results
self.reporter = Reporter()
self.shishito_results = self.reporter.get_xunit_test_cases(timestamp)
self.default_headers = {'Content-Type': 'application/json'}
self.uri_base = self.test_rail_instance + '/index.php?/api/v2/'
def post_results(self):
""" Create test-cases on TestRail, adds a new test run and update results for the run """
self.create_missing_test_cases()
if self.test_plan_name:
test_plan_id = self.add_test_plan()
else:
test_plan_id = self.test_plan_id
test_run = self.add_test_run(test_plan_id)
self.add_test_results(test_run)
def tr_get(self, url):
""" GET request for TestRail API
:param url: url endpoint snippet
:return: response JSON
"""
response = requests.get(self.uri_base + url, auth=(self.user, self.password), headers=self.default_headers)
#print(self.uri_base + url, response, response.text)
return response.json()
def tr_post(self, url, payload):
""" GET request for TestRail API
:param url: url endpoint snippet
:param payload: payload for the POST api call
:return: response object
"""
return requests.post(self.uri_base + url, auth=(self.user, self.password), data=json.dumps(payload),
headers=self.default_headers)
def get_all_test_plans(self):
""" Gets list of all test-plans from certain project
:return: list of test-plans (names = strings)
"""
test_plans_list = self.tr_get('get_plans/{}'.format(self.project_id))
return [{'name': test_plan['name'], 'id': test_plan['id']} for test_plan in test_plans_list]
def get_all_test_cases(self):
""" Gets list of all test-cases from certain project
:return: list of test-cases (names = strings)
"""
test_case_list = self.tr_get('get_cases/{}§ion_id={}&suite_id={}'.format(self.project_id, self.section_id, self.suite_id))
return [{'title': test_case['title'], 'id': test_case['id']} for test_case in test_case_list]
def create_test_case(self, title):
""" Creates a new test case in TestRail
:param title: Title of the test-case
:return: response object
"""
return self.tr_post('add_case/{}'.format(self.section_id), {"title": title})
def create_missing_test_cases(self):
""" Creates new test-cases on TestRail for those in test project (those run by pytest).
Does not create test-cases if already existed on TestRail.
:return: list of test-cases that could not be created on TestRail (post failure)
"""
post_errors = []
test_case_names = [item['title'] for item in self.get_all_test_cases()]
# Iterate over results for each environment combination
for result_combination in self.shishito_results:
# Create TestRail entry for every test-case in combination (if missing)
for item in result_combination['cases']:
if item['name'] not in test_case_names:
response = self.create_test_case(item['name'])
if response.status_code != requests.codes.ok:
post_errors.append(item['name'])
else:
test_case_names.append(item['name'])
return post_errors
def add_test_plan(self):
test_plan_id = 0
# Check if already exists
for plan in self.get_all_test_plans():
if plan['name'] == self.test_plan_name:
return plan['id']
result = self.tr_post('add_plan/{}'.format(self.project_id), {"name": self.test_plan_name})
return json.loads(result.text)['id']
def add_test_run(self, test_plan_id = None):
""" Adds new test run under certain test plan into TestRail
:return: dictionary of TestRail run names & IDs
"""
test_plan_id = test_plan_id or self.test_plan_id
runs_created = []
# Iterate over results for each environment combination
for result_combination in self.shishito_results:
run_name = '{} ({})'.format(result_combination['name'][:-4], self.timestamp)
test_run = {"case_ids": [case['id'] for case in self.get_all_test_cases()]}
result = self.tr_post('add_plan_entry/{}'.format(test_plan_id),
{"suite_id": self.suite_id, "name": run_name, "runs": [test_run]}).json()
# lookup test run id
for run in result['runs']:
if run['name'] == run_name:
runs_created.append({'combination': result_combination['name'], 'id': run['id']})
return runs_created
def add_test_results(self, test_runs):
""" Add test results for specific test run based on parsed xUnit results
:return: list of test run IDs for which results could not be added (post failure)
"""
post_errors = []
run_ids = {r['combination']: r['id'] for r in test_runs}
# Iterate over results for each environment combination
for result in self.shishito_results:
run_id = run_ids.get(result['name'])
if not run_id:
continue
test_results = []
tr_tests = {t['title']: t['id'] for t in self.tr_get('get_tests/{}'.format(run_id))}
# Create TestRail entry for every test-case in combination (if missing)
for xunit_test in result['cases']:
tr_test_id = tr_tests.get(xunit_test['name'])
result_id = {'success': 1, 'failure': 5}.get(xunit_test['result'])
if tr_test_id and result_id:
# Add result content into the payload list
result = {'test_id': tr_test_id, 'status_id': result_id}
if result_id == 5:
result['comment'] = xunit_test['failure_message']
test_results.append(result)
response = self.tr_post('add_results/{}'.format(run_id), {'results': test_results})
if response.status_code != requests.codes.ok:
post_errors.append(run_id)
return post_errors
def __init__(self, driver):
self.driver = driver
self.shishito_support = ShishitoSupport()
self.base_url = self.shishito_support.get_opt("base_url")
self.default_implicit_wait = int(self.shishito_support.get_opt("default_implicit_wait"))
self.timeout = int(self.shishito_support.get_opt("timeout"))
class SeleniumTest(object):
def __init__(self, driver):
self.driver = driver
self.shishito_support = ShishitoSupport()
self.base_url = self.shishito_support.get_opt("base_url")
self.default_implicit_wait = int(self.shishito_support.get_opt("default_implicit_wait"))
self.timeout = int(self.shishito_support.get_opt("timeout"))
def save_screenshot(self, name, project_root):
""" Saves application screenshot """
screenshot_folder = os.path.join(project_root, "screenshots")
if not os.path.exists(screenshot_folder):
os.makedirs(screenshot_folder)
self.driver.save_screenshot(os.path.join(screenshot_folder, name + ".png"))
def save_file_from_url(self, file_path, url):
""" Saves file from url """
if os.path.isfile(file_path):
print "File %s already exists." % file_path
return
response = requests.get(url, stream=True)
response.raise_for_status()
with open(file_path, "wb") as save_file:
for block in response.iter_content(1024):
if not block:
break
save_file.write(block)
# Deprecated use property directly
def get_base_url(self):
return self.base_url
# Deprecated use property directly
def get_current_url(self):
return self.current_url
@property
def current_url(self):
""" Return the url for the current page."""
return self.driver.current_url
def hover_on(self, element):
""" Mouse over specific element """
mouse_over = ActionChains(self.driver).move_to_element(element)
mouse_over.perform()
def go_to_page(self, url):
""" Opens url in currently active window """
self.driver.get(url)
self.driver.implicitly_wait(self.default_implicit_wait)
def click_and_wait(self, element, locator=None, wait_time=None):
""" clicks on a element and then waits for specific element to be present or simply waits implicitly """
element.click()
if locator:
self.wait_for_element_ready(locator, wait_time)
else:
self.driver.implicitly_wait(10)
def check_images_are_loaded(self):
""" checks all images on the pages and verifies if they are properly loaded """
script = (
"return arguments[0].complete && typeof arguments[0].naturalWidth"
' != "undefined" && arguments[0].naturalWidth > 0'
)
images_not_loaded = []
for image in self.driver.find_elements_by_tag_name("img"):
loaded = self.driver.execute_script(script, image)
if not loaded and image.get_attribute("src"):
images_not_loaded.append("%s: %s" % (self.driver.title, image.get_attribute("src")))
return images_not_loaded
def is_element_present(self, locator):
"""
True if the element at the specified locator is present in the DOM.
Note: It returns false immediately if the element is not found.
"""
self.driver.implicitly_wait(0)
try:
self.driver.find_element(*locator)
return True
except NoSuchElementException:
return False
finally:
# set the implicit wait back
self.driver.implicitly_wait(self.default_implicit_wait)
def is_element_visible(self, locator):
"""
True if the element at the specified locator is visible in the browser.
Note: It uses an implicit wait if element is not immediately found.
"""
try:
return self.driver.find_element(*locator).is_displayed()
except (NoSuchElementException, ElementNotVisibleException):
return False
def is_element_not_visible(self, locator):
"""
True if the element at the specified locator is not visible.
Note: It returns true immediately if the element is not found.
"""
self.driver.implicitly_wait(0)
try:
return not self.driver.find_element(*locator).is_displayed()
except (NoSuchElementException, ElementNotVisibleException):
return True
finally:
# set the implicit wait back
self.driver.implicitly_wait(self.default_implicit_wait)
def wait_for_element_present(self, locator, timeout=None):
""" Wait for the element at the specified locator
to be present in the DOM. """
timeout = timeout or self.timeout
count = 0
while not self.is_element_present(locator):
time.sleep(1)
count += 1
if count == timeout:
raise Exception("{0} has not loaded".format(locator))
def wait_for_element_visible(self, locator, timeout=None):
"""
Wait for the element at the specified locator to be visible.
"""
timeout = timeout or self.timeout
count = 0
while not self.is_element_visible(locator):
time.sleep(1)
count += 1
if count == timeout:
raise Exception("{0} is not visible".format(locator))
def wait_for_element_not_visible(self, locator, timeout=None):
"""
Wait for the element at the specified locator not to be visible anymore.
"""
timeout = timeout or self.timeout
count = 0
while self.is_element_visible(locator):
time.sleep(1)
count += 1
if count == timeout:
raise Exception("{0} is still visible".format(locator))
def wait_for_element_not_present(self, locator, timeout=None):
""" Wait for the element at the specified locator
not to be present in the DOM. """
timeout = timeout or self.timeout
self.driver.implicitly_wait(0)
try:
WebDriverWait(self.driver, timeout).until(lambda s: len(self.find_elements(*locator)) < 1)
return True
except TimeoutException:
Assert.fail(TimeoutException)
finally:
self.driver.implicitly_wait(self.default_implicit_wait)
def wait_for_text_to_match(self, text, locator, max_count=20, delay=0.25):
""" Waits for element text to match specified text, until certain deadline """
element = self.driver.find_element(*locator)
counter = 0
while element.text != text:
if counter < max_count:
time.sleep(delay)
counter += 1
element = self.driver.find_element(*locator)
else:
Assert.fail(
'"'
+ text
+ '" text did not match "'
+ element.text
+ '" after '
+ str(counter * delay)
+ " seconds"
)
break
def wait_for_attribute_value(self, attribute, attribute_text, locator, max_count=20, delay=0.25):
""" Waits for element attribute value to match specified text, until certain deadline """
element = self.driver.find_element(*locator)
counter = 0
while element.get_attribute(attribute) != attribute_text:
if counter < max_count:
time.sleep(delay)
counter += 1
else:
Assert.fail(
'"'
+ attribute_text
+ '" text did not match "'
+ element.get_attribute(attribute)
+ '" after '
+ str(counter * delay)
+ " seconds"
)
break
def wait_for_element_ready(self, locator, timeout=None):
""" Waits until certain element is present and clickable """
timeout = timeout or self.timeout
WebDriverWait(self.driver, timeout).until(
EC.presence_of_element_located(locator), "Element specified by {0} was not present!".format(locator)
)
WebDriverWait(self.driver, timeout).until(
EC.element_to_be_clickable(locator), "Element specified by {0} did not become clickable!".format(locator)
)
def find_element(self, locator):
""" Return the element at the specified locator."""
return self.driver.find_element(*locator)
def find_elements(self, locator):
""" Return a list of elements at the specified locator."""
return self.driver.find_elements(*locator)
def find_elements_with_text(self, text, locator):
""" Find elements that have specified text """
elements = self.driver.find_elements(*locator)
selected = [item for item in elements if item.text == text]
return selected[0] if len(selected) == 1 else selected
def link_destination(self, locator):
""" Return the href attribute of the element at the specified locator."""
link = self.driver.find_element(*locator)
return link.get_attribute("href")
def image_source(self, locator):
""" Return the src attribute of the element at the specified locator."""
link = self.driver.find_element(*locator)
return link.get_attribute("src")
def select_dropdown_value(self, select, value):
""" Set 'select' dropdown value """
select = Select(select)
option = [option for option in select.options if option.text == value][0]
option.click()
def upload_file(self, file_path, input_field_locator, delay=5):
""" uploads file through the file input field
@file_path: path to file (including the file name) relative to test project root
@input_field_locator: locator of input element with type="file"
@delay: seconds to wait for file to upload
"""
file_path = os.path.join(self.shishito_support.project_root, file_path)
self.driver.find_element(*input_field_locator).send_keys(file_path)
time.sleep(delay)
def execute_js_script(self, script, arguments=None):
"""execute any js command with arguments or without it"""
script_value = self.driver.execute_script(script, arguments)
return script_value
def open_new_tab(self, url):
"""Open new tab using keyboard, for now work only in Firefox and IE, in Chrome use js script to open tab """
ActionChains(self.driver).send_keys(Keys.CONTROL, "t").perform()
windows = self.driver.window_handles
self.driver.switch_to_window(windows[-1])
self.driver.get(url)
def switch_new_tab(self):
"""switch to new tab/window"""
windows = self.driver.window_handles
self.driver.switch_to_window(windows[-1])
def switch_first_tab(self):
"""Close current tab, switch to first tab/window"""
windows = self.driver.window_handles
self.driver.close()
self.driver.switch_to_window(windows[0])
def __init__(self):
self.shishito_support = ShishitoSupport()
self.api_token = self.shishito_support.get_opt("circleci_api_token")
self.circle_username = self.shishito_support.get_opt("circleci_username")
self.circle_project = self.shishito_support.get_opt("circleci_project")
self.circle_branch = self.shishito_support.get_opt("circleci_branch")
class QAStats(object):
""" QAStats object """
def __init__(self, user, password, timestamp, epoch, build):
self.shishito_support = ShishitoSupport()
self.qastats_base_url = self.shishito_support.get_opt('qastats_url')
self.user = user
self.password = password
self.timestamp = timestamp
self.epoch = epoch
self.build = build
# project specific config
self.project_id = self.shishito_support.get_opt('qastats_project_id')
# shishito results
self.reporter = Reporter()
self.shishito_results = self.reporter.get_xunit_test_cases(timestamp)
self.default_headers = {'Content-Type': 'application/json'}
self.result_url = self.qastats_base_url + '/api/v1/results'
self.project_url = self.shishito_support.get_opt('base_url')
def post_results(self):
""" Create test-cases on QAStats, adds a new test run and update results for the run
{
"project_id": 123,
"timestamp": 1470133472,
"build": "773", // optional
"environment": "Firefox" // optional
"branch": "develop", // optional
"git": "ae232a", // optional
"results": [
{ "test": "test_login", "result": "pass" }, // [pass fail err nr]
...
],
}
"""
for (i, run) in enumerate(self.shishito_results):
environment = run['name'];
m =re.match('^(.*)\.xml$', environment)
if m != None: environment = m.group(1)
payload = {
'project_id': self.project_id,
'timestamp': self.epoch + i,
'environment': environment
}
if self.build:
payload['build'] = self.build
if 'QA_BRANCH_TO_TEST' in os.environ:
payload['branch'] = os.environ['QA_BRANCH_TO_TEST']
if 'QA_GIT_COMMIT' in os.environ:
payload['git'] = os.environ["QA_GIT_COMMIT"]
elif 'CIRCLE_REPOSITORY_URL' in os.environ:
print('github url is known')
payload['git'] = os.environ['CIRCLE_REPOSITORY_URL']
if 'CIRCLE_TEST_REPORTS' in os.environ:
print('result url is known')
payload['reporturl'] = os.environ.get('CIRCLE_TEST_REPORTS')
if self.project_url is not None:
payload['testurl']=self.project_url
status_map = {
'error': 'err',
'failure': 'fail',
'success': 'pass',
'skipped': 'nr'
}
results = [ {'test': t['name'], 'result': status_map[t['result']]} for t in run['cases'] ]
payload['results'] = results
print(payload)
json_payload = json.dumps(payload)
r = requests.post(self.result_url, auth=(self.user, self.password), data=json_payload,
headers=self.default_headers)
if r.status_code == requests.codes.ok:
try:
resp = r.json()
if 'result' in resp and resp['result'] == 'OK':
print("Results uploaded to QAStats")
return True
except (ValueError, AttributeError):
pass
print("Error: uploading tests to QAStats\n\n", json_payload, "\n")
print("\tStatus-code:\t" + str(r.status_code) + "\n")
for n, v in r.headers.items():
print("\t" + n + "\t" + v)
print("")
print(r.text)
class ShishitoRunner(object):
""" Base shishito test runner.
- runs python selenium tests on customizable configurations (using PyTest)
- archive the test results in .zip file """
def __init__(self, project_root):
# set project root
self.project_root = project_root
# test timestamp - for storing results
self.test_timestamp = time.strftime("%Y-%m-%d_%H-%M-%S")
self.epoch = int(time.time())
# parse cmd args
self.cmd_args = self.handle_cmd_args()
# Get SUT build for use in reporting
self.test_build = self.cmd_args['build']
self.reporter = Reporter(project_root, self.test_timestamp)
self.shishito_support = ShishitoSupport(cmd_args=self.cmd_args,
project_root=self.project_root)
def handle_cmd_args(self):
""" Retrieve command line arguments passed to the script.
:return: dict with parsed command line arguments
"""
parser = argparse.ArgumentParser(
description='Selenium Python test runner execution arguments.')
parser.add_argument('--platform',
help='Platform on which run tests.',
dest='test_platform')
parser.add_argument('--environment',
help='Environment for which run tests.',
dest='test_environment')
parser.add_argument('--test_directory',
help='Directory where to lookup for tests')
parser.add_argument('--smoke',
help='Run only smoke tests',
action='store_true')
parser.add_argument(
'--browserstack',
help='BrowserStack credentials; format: "username:token"')
parser.add_argument(
'--saucelabs',
help='Saucelabs credentials; format: "username:token"')
parser.add_argument(
'--test_rail',
help=
'TestRail Test Management tool credentials; format: "username:password"'
)
parser.add_argument(
'--qastats',
help='QAStats Test Management tool credentials; format: "token"')
parser.add_argument(
'--node_webkit_chromedriver_path',
help=
'Path to chromedriver located in same directory as node-webkit application'
)
parser.add_argument('--app', help='Path to appium application')
parser.add_argument(
'--test', help='Run specified test (PyTest string expression)')
parser.add_argument('--build',
help='Specify build number for reporting purposes')
args = parser.parse_args()
# return args dict --> for use in other classes
return vars(args)
def run_tests(self):
""" Execute tests for given platform and environment. Platform and Environment can be passed as command lines
argument or settings in config file.
"""
if __name__ == "__main__":
sys.exit(
'The runner cannot be executed directly.'
' You need to import it within project specific runner. Session terminated.'
)
# cleanup previous results
self.reporter.cleanup_results()
# import execution class
executor_class = self.shishito_support.get_module('platform_execution')
# executor_class = getattr(import_module(platform_path), 'ControlExecution')
executor = executor_class(self.shishito_support, self.test_timestamp)
# run test
exit_code = executor.run_tests()
# archive results + generate combined report
self.reporter.archive_results()
self.reporter.generate_combined_report()
# upload results to QAStats test management app
qastats_credentials = self.shishito_support.get_opt('qastats')
if qastats_credentials:
try:
qas_user, qas_password = qastats_credentials.split(':', 1)
except (AttributeError, ValueError):
raise ValueError(
'QAStats credentials were not specified! Unable to connect to QAStats.'
)
qastats = QAStats(qas_user, qas_password, self.test_timestamp,
self.epoch, self.test_build)
qastats.post_results()
# upload results to TestRail test management app
test_rail_credentials = self.shishito_support.get_opt('test_rail')
if test_rail_credentials:
try:
tr_user, tr_password = test_rail_credentials.split(':', 1)
except (AttributeError, ValueError):
raise ValueError(
'TestRail credentials were not specified! Unable to connect to TestRail.'
)
test_rail = TestRail(tr_user, tr_password, self.test_timestamp,
self.test_build)
test_rail.post_results()
return exit_code
class ShishitoControlTest(object):
""" Base class for ControlTest objects. """
def __init__(self):
self.shishito_support = ShishitoSupport()
# create control environment object
control_env_obj = self.shishito_support.get_module('test_environment')
self.test_environment = control_env_obj(self.shishito_support)
self.drivers = []
def start_browser(self, base_url=None):
""" Webdriver startup function.
:return: initialized webdriver
"""
config_section = self.shishito_support.get_opt(
'environment_configuration')
# call browser from proper environment
driver = self.test_environment.call_browser(config_section)
self.drivers.append(driver)
# load init url
if not base_url:
base_url = self.shishito_support.get_opt('base_url')
if base_url:
self.test_init(driver, base_url)
return driver
def start_test(self, reload_page=None):
""" To be executed before every test-case (test function).
:param reload_page:
"""
def stop_browser(self):
""" Webdriver termination function. """
for driver in self.drivers:
driver.quit()
def stop_test(self, test_info, debug_events=None):
""" To be executed after every test-case (test function). If test failed, function saves
screenshots created during test.
:param test_info: information about test
"""
if test_info.test_status not in ('passed', None):
# save screenshot in case test fails
test_name = re.sub('[^A-Za-z0-9_.]+', '_', test_info.test_name)
# Capture screenshot and debug info from driver(s)
for driver in self.drivers:
browser_name = driver.name
file_name = browser_name + '_' + test_name
ts = SeleniumTest(driver)
ts.save_screenshot(file_name)
#Save debug info to file
if debug_events is not None:
debugevent_folder = os.path.join(
self.shishito_support.project_root, 'debug_events')
if not os.path.exists(debugevent_folder):
os.makedirs(debugevent_folder)
with open(
os.path.join(debugevent_folder,
file_name + '.json'), 'w') as logfile:
json.dump(debug_events, logfile)
def test_init(self, driver, url):
""" Executed only once after browser starts.
def setup_class(self):
self.tc = ShishitoSupport().get_test_control()
self.driver = self.tc.start_browser()
self.ts = SeleniumTest(self.driver)
class BaseTestClass():
""" Contextual help test """
def set_etsy_testcase(self, tc_name):
etsy_emulator_interface = EtsyEmulatorInterface()
for _ in range(5):
try:
etsy_emulator_interface.set_test_case(tc_name)
break
except EtsyEmulatorRequestError:
etsy_emulator_interface.restart_emulator()
else:
# last attempt
etsy_emulator_interface.set_test_case(tc_name)
def restart_node(self, instance='00'):
if call([BIN_DIR + '/restart_node', instance]) != 0:
raise Exception("Error: Node restart failed (" + BIN_DIR +
"/restart_node)")
sleep(2)
def stop_all(self):
cmd = os.path.join(BIN_DIR, 'default', 'restart-product')
for _ in range(3):
if call([cmd, '--kill']) == 0:
sleep(2)
return
raise Exception("Error: Nodes kill failed (" + cmd + " --kill)")
def restart_all(self):
os.environ[
'QA_CURRENT_TEST'] = self.__module__ + '.py ' + self.test_id if hasattr(
self, 'test_id') else self.__module__ + '.py'
cmd = os.path.join(BIN_DIR, 'default', 'restart-product')
for full_restart in range(5):
for _ in range(10):
if call([cmd]) == 0:
sleep(2)
break
else:
raise Exception("Error: Nodes restart failed (" + cmd + ")")
# check if we can connect
for connect_attempt in range(10, -1, -1):
try:
r1 = requests.get(self.base_url, allow_redirects=False)
r2 = requests.get(self.login_url_http,
allow_redirects=False)
if r1.status_code == 302 and r2.status_code == 302:
print("restart_all: servers are running",
self.base_url, self.login_url_http)
sleep(1)
return
print("restart_all: cannot get", self.base_url,
"status_code =", r1.status_code, self.login_url_http,
"status_code =", r2.status_code)
except Exception as e:
print("restart_all: error getting", self.base_url,
self.login_url_http, e.args)
if connect_attempt > 0:
sleep(1)
raise Exception('Error: restart_all failed, no attempt left')
def setup_class(self):
# shishito / selenium support
self.tc = ShishitoSupport().get_test_control()
try:
# Set base URL - host taken from env.
prod_host = os.environ[
'QA_PRODUCT_HOST'] if 'QA_PRODUCT_HOST' in os.environ else '127.0.0.1'
prod_port = os.environ[
'QA_WEB_HTTPS_PORT'] if 'QA_WEB_HTTPS_PORT' in os.environ else '80'
login_host = os.environ[
'QA_PRODUCT_HOST_AUTH'] if 'QA_PRODUCT_HOST_AUTH' in os.environ else prod_host
self.base_url = 'https://' + prod_host + ':' + prod_port + '/'
# Login URLs
self.login_url_http = 'http://' + login_host + ':' + os.environ[
'QA_AUTH_HTTP_PORT'] + '/'
self.login_url_https = 'https://' + login_host + ':' + os.environ[
'QA_AUTH_HTTPS_PORT'] + '/'
self.web_api_url_https = 'https://' + prod_host + ':' + prod_port + '/api/v1/'
except:
print('*** setup_class failed ***')
raise
def setup_method(self, method):
self.test_id = method.__name__
# start browser
for attempt in range(20):
try:
print('*** setup_method: starting browser ***')
self.driver = self.tc.start_browser()
break
except WebDriverException as e:
print("Cannot connect to Webdriver.\n" + str(e))
sleep(5)
else:
assert False, "Test could not be run due to the Webdriver error"
self.ts = SeleniumTest(self.driver)
def teardown_method(self, method):
print('*** teardown_method: stopping browser ***')
try:
console_events = self.driver.get_log('browser')
except Exception as e:
print('ERROR: failed to get console events: ' + str(e))
console_events = {}
try:
self.tc.stop_test(get_test_info(), debug_events=console_events
) # save screenshot in case test fails
except Exception as e:
print('ERROR: failed to stop the test: ' + str(e))
try:
self.tc.stop_browser()
except Exception as e:
print('ERROR: failed to stop the browser: ' + str(e))
print('*** teardown_method: ended successfully ***')
def close_intercom(self):
"""
try to find intercom close button (if exists), click it
"""
d = self.driver
window_selectors = ['div.intercom-chat-composer', 'intercom-block']
button_selectors = [
'div.intercom-chat-dismiss-button',
'div.intercom-launcher-hovercard-close'
]
# move to the IC window, so that the close button is displayed
for selector in window_selectors:
try:
win = d.find_element_by_css_selector(selector)
action = ActionChains(d)
action.move_to_element(win)
action.perform()
except:
pass
for selector in button_selectors:
try:
b = d.find_element_by_css_selector(selector)
click(b)
except:
pass
class TestRail(object):
""" TestRail object """
def __init__(self, user, password, timestamp, build):
self.shishito_support = ShishitoSupport()
self.test_rail_instance = self.shishito_support.get_opt(
'test_rail_url')
self.user = user
self.password = password
self.timestamp = timestamp
# project specific config
self.project_id = self.shishito_support.get_opt('test_rail_project_id')
self.section_id = self.shishito_support.get_opt('test_rail_section_id')
self.test_plan_id = self.shishito_support.get_opt(
'test_rail_test_plan_id')
self.test_plan_name = self.shishito_support.get_opt(
'test_rail_test_plan_name') or build
self.suite_id = self.shishito_support.get_opt('test_rail_suite_id')
# shishito results
self.reporter = Reporter()
self.shishito_results = self.reporter.get_xunit_test_cases(timestamp)
self.default_headers = {'Content-Type': 'application/json'}
self.uri_base = self.test_rail_instance + '/index.php?/api/v2/'
def post_results(self):
""" Create test-cases on TestRail, adds a new test run and update results for the run """
self.create_missing_test_cases()
if self.test_plan_name:
test_plan_id = self.add_test_plan()
else:
test_plan_id = self.test_plan_id
test_run = self.add_test_run(test_plan_id)
self.add_test_results(test_run)
def tr_get(self, url):
""" GET request for TestRail API
:param url: url endpoint snippet
:return: response JSON
"""
response = requests.get(self.uri_base + url,
auth=(self.user, self.password),
headers=self.default_headers)
#print(self.uri_base + url, response, response.text)
return response.json()
def tr_post(self, url, payload):
""" GET request for TestRail API
:param url: url endpoint snippet
:param payload: payload for the POST api call
:return: response object
"""
return requests.post(self.uri_base + url,
auth=(self.user, self.password),
data=json.dumps(payload),
headers=self.default_headers)
def get_all_test_plans(self):
""" Gets list of all test-plans from certain project
:return: list of test-plans (names = strings)
"""
test_plans_list = self.tr_get('get_plans/{}'.format(self.project_id))
return [{
'name': test_plan['name'],
'id': test_plan['id']
} for test_plan in test_plans_list]
def get_all_test_cases(self):
""" Gets list of all test-cases from certain project
:return: list of test-cases (names = strings)
"""
test_case_list = self.tr_get(
'get_cases/{}§ion_id={}&suite_id={}'.format(
self.project_id, self.section_id, self.suite_id))
return [{
'title': test_case['title'],
'id': test_case['id']
} for test_case in test_case_list]
def create_test_case(self, title):
""" Creates a new test case in TestRail
:param title: Title of the test-case
:return: response object
"""
return self.tr_post('add_case/{}'.format(self.section_id),
{"title": title})
def create_missing_test_cases(self):
""" Creates new test-cases on TestRail for those in test project (those run by pytest).
Does not create test-cases if already existed on TestRail.
:return: list of test-cases that could not be created on TestRail (post failure)
"""
post_errors = []
test_case_names = [item['title'] for item in self.get_all_test_cases()]
# Iterate over results for each environment combination
for result_combination in self.shishito_results:
# Create TestRail entry for every test-case in combination (if missing)
for item in result_combination['cases']:
if item['name'] not in test_case_names:
response = self.create_test_case(item['name'])
if response.status_code != requests.codes.ok:
post_errors.append(item['name'])
else:
test_case_names.append(item['name'])
return post_errors
def add_test_plan(self):
test_plan_id = 0
# Check if already exists
for plan in self.get_all_test_plans():
if plan['name'] == self.test_plan_name:
return plan['id']
result = self.tr_post('add_plan/{}'.format(self.project_id),
{"name": self.test_plan_name})
return json.loads(result.text)['id']
def add_test_run(self, test_plan_id=None):
""" Adds new test run under certain test plan into TestRail
:return: dictionary of TestRail run names & IDs
"""
test_plan_id = test_plan_id or self.test_plan_id
runs_created = []
# Iterate over results for each environment combination
for result_combination in self.shishito_results:
run_name = '{} ({})'.format(result_combination['name'][:-4],
self.timestamp)
test_run = {
"case_ids": [case['id'] for case in self.get_all_test_cases()]
}
result = self.tr_post('add_plan_entry/{}'.format(test_plan_id), {
"suite_id": self.suite_id,
"name": run_name,
"runs": [test_run]
}).json()
# lookup test run id
for run in result['runs']:
if run['name'] == run_name:
runs_created.append({
'combination':
result_combination['name'],
'id':
run['id']
})
return runs_created
def add_test_results(self, test_runs):
""" Add test results for specific test run based on parsed xUnit results
:return: list of test run IDs for which results could not be added (post failure)
"""
post_errors = []
run_ids = {r['combination']: r['id'] for r in test_runs}
# Iterate over results for each environment combination
for result in self.shishito_results:
run_id = run_ids.get(result['name'])
if not run_id:
continue
test_results = []
tr_tests = {
t['title']: t['id']
for t in self.tr_get('get_tests/{}'.format(run_id))
}
# Create TestRail entry for every test-case in combination (if missing)
for xunit_test in result['cases']:
tr_test_id = tr_tests.get(xunit_test['name'])
result_id = {
'success': 1,
'failure': 5
}.get(xunit_test['result'])
if tr_test_id and result_id:
# Add result content into the payload list
result = {'test_id': tr_test_id, 'status_id': result_id}
if result_id == 5:
result['comment'] = xunit_test['failure_message']
test_results.append(result)
response = self.tr_post('add_results/{}'.format(run_id),
{'results': test_results})
if response.status_code != requests.codes.ok:
post_errors.append(run_id)
return post_errors
