def run():
"""
:param app_name: HPC application
:param perf_coln: performance name to be optimized
:param num_core: number of CPU cores
:param num_node: number of computing nodes
:param rand_seed: random seed
:param num_smpl: number of samples
:param pool_size: pool size
:param num_iter: number of iterations
:param prec_rand: precentage of random samples
"""
try:
cm.init()
app_name = cm.app_name
perf_coln = cm.perf_coln
num_smpl = cm.num_smpl
pool_size = cm.pool_size
num_iter = cm.num_iter
prec_rand = cm.prec_rand
if (app_name == "lv"):
conf_colns = data.lv_conf_colns
elif (app_name == "hs"):
conf_colns = data.hs_conf_colns
num_rand = int(num_smpl * prec_rand)
nspi = int((num_smpl - num_rand) / num_iter)
# pool_df = data.gen_smpl(app_name, pool_size)
# conf_df = pool_df.head(num_rand)
conf_df = data.gen_smpl(app_name, num_rand)
train_df = cm.measure_perf(conf_df)
for iter_idx in range(num_iter):
num_curr = num_smpl - nspi * (num_iter - 1 - iter_idx)
pool_df = data.gen_smpl(app_name, pool_size)
pred_top_smpl = learn.whl_pred_top_eval(train_df, pool_df, conf_colns, perf_coln, num_smpl, 0)
pred_top_smpl = pred_top_smpl.sort_values([perf_coln]).reset_index(drop=True)
new_conf_df = pred_top_smpl[conf_colns].head(nspi)
conf_df = tool.df_union(conf_df, new_conf_df)
last = nspi
while (conf_df.shape[0] < num_curr):
last = last + 1
new_conf_df = pred_top_smpl[conf_colns].head(last)
conf_df = tool.df_union(conf_df, new_conf_df)
new_train_df = cm.measure_perf(new_conf_df)
train_df = tool.df_union(train_df, new_train_df)
data.df2csv(train_df, app_name + "_train.csv")
mdl_chk, mdl = learn.train_mdl_chk(train_df, conf_colns, perf_coln)
top_df = cm.find_top('ALe', (mdl_chk, mdl, ), conf_colns, perf_coln, train_df)
cm.test(train_df, conf_colns, perf_coln)
cm.finish(train_df, top_df)
except:
traceback.print_exc()
def check_multi_windows(browser):
# Done: 1) зайти в админку
# Done: 2) открыть пункт меню Countries (или страницу http://localhost/litecart/admin/?app=countries&doc=countries)
# Done: 3) открыть на редактирование какую-нибудь страну или начать создание новой
# Done: 4) возле некоторых полей есть ссылки с иконкой в виде квадратика со стрелкой -- они ведут на внешние страницы и открываются в новом окне, именно это и нужно проверить.
# Done: требуется именно кликнуть по ссылке, чтобы она открылась в новом окне, потом переключиться в новое окно, закрыть его, вернуться обратно, и повторить эти действия для всех таких ссылок.
common.start("Checking multiple windows on countries")
function_result = True
wait10 = WebDriverWait(browser, 10)
common_litecart.do_login_admin(browser)
browser.get("http://localhost/litecart/admin/?app=countries&doc=countries")
wait10.until(
EC.element_to_be_clickable((By.CSS_SELECTOR, "button[name=disable]")))
common.success("Countries page opened")
browser.find_element_by_link_text("Add New Country").click()
common.success("New country add page opened")
# find all links targeting new window and go through them
for i in range(
len(browser.find_elements_by_css_selector(".fa-external-link"))):
common.info("")
# take another link
current_element = browser.find_elements_by_css_selector(
".fa-external-link")[i]
# get handles of all windows and current window
start_handles = set(browser.window_handles)
start_active_handle = browser.current_window_handle
# click on the link to open new window
current_element.click()
WebDriverWait(browser,
30).until(EC.new_window_is_opened(list(start_handles)))
# get handles of windows and find new one
next_handles = set(browser.window_handles)
new_handle = next_handles ^ start_handles
common.info("New window id = %s" % list(new_handle)[0])
# switch to the new window
browser.switch_to_window(list(new_handle)[0])
common.success("Switched to window '%s'" % list(new_handle)[0])
common.info("Window title = '%s'" % browser.title)
# close active (new) window using javascript
browser.execute_script("close();")
final_handles = set(browser.window_handles)
if final_handles == start_handles:
call_function = common.success
else:
common.info("Cannot close window '%s'" % browser.title)
call_function = common.fail
call_function("Closed window '%s'" % list(new_handle)[0])
# switch back to the main window
browser.switch_to_window(start_active_handle)
common.success("Switched to main window '%s'" % start_active_handle)
common.finish(function_result, "Checking multiple windows")
return function_result
def register_user(browser):
common.start("Starting registration")
function_result = True
browser.get("http://www.localhost/litecart/")
browser.find_element_by_link_text("New customers click here").click()
common.success("Get to the registration page")
user = get_user_data('en')
# set registration data
registration_data = [
{"location": "input[name=firstname]", "value": user["first_name"] },
{"location": "input[name=lastname]", "value": user["last_name"] },
{"location": "input[name=address1", "value": user["street"] },
{"location": "input[name=city]", "value": user["city"] },
{"location": "#box-create-account input[name=email]", "value": user["email"] },
{"location": "#box-create-account input[name=password]", "value": user["password"] },
{"location": "input[name=confirmed_password]", "value": user["password"] },
{"location": "input[name=postcode]", "value": user["postcode"] }
]
for field in registration_data:
function_result = input_value_to_input_field(browser, field) and function_result
# select country and state from dropdown list
registration_data = [
{"location": "select[name=country_code]", "value": user["country_code"], "description": user["country"] },
{"location": "select[name=zone_code]", "value": user["state_code"], "description": user["state"] }
]
for field in registration_data:
function_result = select_from_dropdown_list(browser, field) and function_result
# unsubscribe from newsletters
function_result = set_check_box(browser, "input[name=newsletter]", False) and function_result
common.info("Uncheck '%s' to unsubscribe from newsletters: ok" % "input[name=newsletter]")
# click registration button
browser.find_element_by_css_selector("button[name=create_account]").click()
common.success("Login successfull")
# make logout with first appropriate link
browser.find_element_by_link_text("Logout").click()
common.success("Logout successfull")
# make login back using left form
registration_data = [
{"location": "#box-account-login input[name=email]", "value": user["email"] },
{"location": "#box-account-login input[name=password]", "value": user["password"] }
]
for field in registration_data:
function_result = input_value_to_input_field(browser, field) and function_result
browser.find_element_by_css_selector("button[name=login]").click()
common.success("Second login successfull")
# make logout again to get out of store
browser.find_element_by_link_text("Logout").click()
common.success("Logout successfull")
common.finish(function_result, "Registration")
return function_result
def test_function_Chrome(browser_Chrome):
try:
print()
test_name = "Checking multiple windows operationing"
test_start_time = time.time()
common.startTest(test_name)
overall_result = True
overall_result = check_multi_windows(browser_Chrome) and overall_result
except Exception as e:
overall_result = False
common.finish(False, "Exception detected (%s)" % e.__str__())
raise e
finally:
common.finishTest(test_name, overall_result, test_start_time)
def test_function_Chrome(browser_Chrome):
try:
print()
test_name = "User registration"
test_start_time = time.time()
common.startTest(test_name)
overall_result = True
overall_result = register_user(browser_Chrome) and overall_result
except Exception as e:
overall_result = False
common.finish("Exception detected")
raise e
finally:
common.finishTest(test_name, overall_result, test_start_time)
def test_function_Chrome(browser_Chrome):
try:
print()
test_name = "Add new goods"
test_start_time = time.time()
common.startTest(test_name)
overall_result = True
overall_result = add_new_goods(browser_Chrome) and overall_result
except Exception as e:
overall_result = False
common.finish(False, "Exception detected")
raise e
finally:
common.finishTest(test_name, overall_result, test_start_time)
def browse_catalog(browser):
common.start("Start browsing catalog")
function_result = True
common_litecart.do_login_admin(browser)
wait10 = WebDriverWait(browser, 10)
# go to the 'Catalog' page
browser.get(
"http://localhost/litecart/admin/?app=catalog&doc=catalog&category_id=1"
)
wait10.until(
EC.presence_of_element_located((By.CSS_SELECTOR, ".data-table")))
length = len(browser.find_elements_by_css_selector("tbody tr"))
for i in range(1, length):
common.info("")
# go to the 'Catalog' page
browser.get(
"http://localhost/litecart/admin/?app=catalog&doc=catalog&category_id=1"
)
wait10.until(
EC.presence_of_element_located((By.CSS_SELECTOR, ".data-table")))
common.success("Open 'Catalog' page")
check_log(browser)
table_lines = browser.find_elements_by_css_selector("tbody tr")
category = table_lines[i].find_elements_by_css_selector(
"input")[0].get_property("name")
if "products" in category:
product = table_lines[i].find_elements_by_css_selector("a")[0]
product_name = product.get_property("innerText")
common.info("Process product '%s'" % product_name)
product.click()
wait10.until(
EC.presence_of_element_located(
(By.CSS_SELECTOR, "button[name=delete]")))
common.success("Process product '%s'" % product_name)
# Замечание: если раскомментировать эту строку, то получим замечание в логе драйвера и аварийный останов теста,
# но эта строка использовалась для просмотра, что лог доступен и можно вывести его сообщения
#browser.execute_script("alert('OK');")
check_log(browser)
# Done: зайти в админскую часть
# Done: открыть каталог, категорию, которая содержит товары
# Done: последовательно открывать страницы товаров
# Done: проверять, не появляются ли в логе браузера сообщения (любого уровня)
common.finish(function_result, "Browse catalog")
return function_result
def test_function_Chrome(browser_Chrome):
try:
print()
test_name = "Processing Shopping Cart"
test_start_time = time.time()
common.startTest(test_name)
overall_result = True
overall_result = process_shopping_cart(
browser_Chrome) and overall_result
except Exception as e:
overall_result = False
common.finish(False, "Exception detected (%s)" % e.__str__())
raise e
finally:
common.finishTest(test_name, overall_result, test_start_time)
def get_term_frequency(doc, nprocs, min_seq_len=1):
result = State()
with common.finish(
concurrent.futures.ThreadPoolExecutor(max_workers=nprocs),
accumulator=result) as executor:
for pkg in sal.get_packages(doc=doc):
for seq in sal.get_sequences(pkg=pkg):
if len(sal.get_calls(seq=seq)) >= min_seq_len:
executor.submit(seq_term_frequency, seq)
return result.get()
def main():
game = Game(create(), idle_time=20, debug=True)
common.handle_common_interruption(game)
game.addAction(SimpleAction('ok'))
game.addAction(common.finish(False))
game.addAction(fight)
game.addAction(box)
game.addAction(SimpleAction('boss'))
game.addAction(isfighting)
game.idle = move
game.start()
def test_function_Chrome(browser_Chrome):
try:
print()
test_name = "Browser log access check"
test_start_time = time.time()
common.startTest(test_name)
overall_result = True
overall_result = browse_catalog(browser_Chrome) and overall_result
except Exception as e:
overall_result = False
common.finish(False, "Exception detected")
raise e
finally:
common.finishTest(test_name, overall_result, test_start_time)
#попробуйте взять Chrome и увеличить уровень логирования до максимального
#DesiredCapabilities cap = DesiredCapabilities.chrome();
#LoggingPreferences logPrefs = new LoggingPreferences();
#logPrefs.enable(LogType.BROWSER, Level.ALL);
#cap.setCapability(CapabilityType.LOGGING_PREFS, logPrefs);
def process_shopping_cart(browser):
common.start("Process shopping")
function_result = True
wait10 = WebDriverWait(browser, 10)
items_to_buy = ["Yellow Duck", "Green Duck", "Purple Duck"]
for i in range(3):
browser.get("http://localhost/litecart/en/")
wait10.until(
EC.presence_of_element_located((By.LINK_TEXT, "Rubber Ducks")))
common.success("Open main page")
browser.find_elements_by_link_text("Rubber Ducks")[0].click()
common.success("Open 'Rubber Ducks' sales page")
time.sleep(5)
product_list = browser.find_elements_by_css_selector(".product")
for product in product_list:
item = product.find_element_by_css_selector(".info")
if (items_to_buy[i] in item.get_property("innerText")):
common.success("Found product '%s'" % items_to_buy[i])
item.click()
wait10.until(
EC.presence_of_element_located(
(By.CSS_SELECTOR, "div#view-full-page a")))
full_page_link_object = browser.find_elements_by_css_selector(
"div#view-full-page a")
if (len(full_page_link_object)):
common.info(
"Popup window with desciption opened, looking for full page.."
)
full_page_link_object[0].click()
common.success("Product '%s' page opened" % items_to_buy[i])
common.info("Set parameters of the product to buy..")
if (browser.find_elements_by_css_selector(
"select[name='options[Size]']")):
common_litecart.select_from_dropdown_list(
browser, {
"location": "select[name='options[Size]']",
"value": "Small",
"description": "Small"
})
common_litecart.input_value_to_scrollable_field(
browser, {
"location": "input[name='quantity']",
"value": "1"
})
# save the text for the shopping cart dut to it contains price before product adding
previous_amount = browser.find_element_by_css_selector(
".quantity").get_property("innerText")
previous_summ = browser.find_element_by_css_selector(
".formatted_value").get_property("innerText")
common.info(
"Shopping cart before operation contains %d products with summ = '%s'"
% (int(previous_amount), previous_summ))
browser.find_element_by_css_selector(
"button[name='add_cart_product']").click()
# wait for the summ for shopping cart to be changed, i.e. previous text to disappear
wait10.until_not(
EC.text_to_be_present_in_element(
(By.CSS_SELECTOR, ".quantity"), previous_amount))
# check the data of the shopping cart
operation_amount = browser.find_element_by_css_selector(
".quantity").get_property("innerText")
operation_summ = browser.find_element_by_css_selector(
".formatted_value").get_property("innerText")
common.info(
"Shopping cart after adding contains %d products with summ = '%s'"
% (int(operation_amount), operation_summ))
if (int(operation_amount) - 1) == int(previous_amount):
common.success("Product '%s' added to the Shopping Cart" %
items_to_buy[i])
else:
common.fail("Product '%s' added to the Shopping Cart" %
items_to_buy[i])
function_result = False
break # break from loop [for product in product_list]
# process Shopping Cart
browser.find_element_by_css_selector("div#cart a").click()
wait10.until(
EC.presence_of_element_located((By.CSS_SELECTOR, ".table-bordered")))
common.success("Open Shopping Cart checkout page")
items_to_remove = browser.find_elements_by_css_selector(".item")
while (len(items_to_remove) > 1):
# save total payment summ to watch for its change
previous_payment_due = str.split(
browser.find_element_by_css_selector(
".table-bordered .footer").get_property("innerText"))[2]
# get the name of the product to delete from shopping cart
item_name = items_to_remove[1].find_element_by_css_selector(
"strong").get_property("innerText")
common.info("Delete product '%s' from Shopping Cart.." % item_name)
common.info("Payment summ before deleting of the product '%s' is: %s" %
(item_name, previous_payment_due))
remove_button = items_to_remove[1].find_element_by_css_selector(
"button[name=remove_cart_item]")
remove_button.click()
wait10.until_not(
EC.text_to_be_present_in_element(
(By.CSS_SELECTOR, ".table-bordered .footer"),
previous_payment_due))
if (len(
browser.find_elements_by_css_selector(
".table-bordered .footer"))):
final_payment_due = str.split(
browser.find_element_by_css_selector(
".table-bordered .footer").get_property("innerText"))[2]
else:
final_payment_due = '$00.00 (There are no items in the cart)'
common.info("Payment summ after deleting of the product '%s' is: %s" %
(item_name, final_payment_due))
common.success("Delete product '%s' from Shopping Cart" % item_name)
items_to_remove = browser.find_elements_by_css_selector(".item")
# Done: открыть главную страницу
# Done: открыть первый товар из списка (Замечание: сделал по другому, ищу товар с заданным именем, а не какой попался первым)
# Done: добавить первый товар в корзину (при этом может случайно добавиться товар, который там уже есть, ничего страшного)
# Done: подождать, пока счётчик товаров в корзине обновится (Замечание: отслеживаю сумму для корзины, а не счётчик товаров)
# Done: вернуться на главную страницу, повторить предыдущие шаги ещё два раза, чтобы в общей сложности в корзине было 3 единицы товара
# Done: открыть корзину (в правом верхнем углу кликнуть по ссылке Checkout)
# Done: удалить все товары из корзины один за другим, после каждого удаления подождать, пока внизу обновится таблица
common.finish(function_result, "Processing to buy")
return function_result
common.set_motors(100, 100)
time.sleep(5)
print("Spinning left 3 seconds")
common.set_motors(-100, 100)
time.sleep(3)
print("Spinning right 3 seconds")
common.set_motors(100, -100)
time.sleep(3)
print("Reversing 3 second")
common.set_motors(-100, -100)
time.sleep(3)
print("Wide right arc for 10 seconds")
common.set_motors(100, 90)
time.sleep(10)
common.finish()
def add_new_goods(browser):
common.start("Start add new goods")
function_result = True
common_litecart.do_login_admin(browser)
wait = WebDriverWait(browser, 10)
# go to the 'Add New Product' page
browser.find_element_by_link_text("Catalog").click()
wait.until(
EC.presence_of_element_located((By.CSS_SELECTOR, "ul.pull-right")))
common.success("Open 'Add New Product' page")
browser.find_elements_by_css_selector("ul.pull-right a")[1].click()
common.success("Open 'General' tab")
# fill the data on the 'General' tab
# Status
browser.execute_script(
"arguments[0].click();",
browser.find_element_by_css_selector("input[name=status][value='1']"))
#browser.execute_script("document.querySelector(\"input[name=status][value='1']\").click()")
common.info("Set %s = '%s': ok" % ("input[name=status]", "Enabled"))
# Categories
common_litecart.set_checkbox(browser,
"input[name='categories[]'][value='0']", True)
common_litecart.set_checkbox(browser,
"input[name='categories[]'][value='1']", True)
# Default category -- skipped
# Product Groups
common_litecart.set_checkbox(
browser, "input[name='product_groups[]'][value='1-1']", False)
common_litecart.set_checkbox(
browser, "input[name='product_groups[]'][value='1-2']", False)
common_litecart.set_checkbox(
browser, "input[name='product_groups[]'][value='1-3']", True)
# Date Valid From
browser.find_element_by_css_selector(
"input[name=date_valid_from]").send_keys("01.05.2017")
common.info("Set %s = '%s': ok" %
("input[name=date_valid_from]", "01.05.2017"))
# Date Valid To
browser.find_element_by_css_selector(
"input[name=date_valid_to]").send_keys("01.09.2017")
common.info("Set %s = '%s': ok" %
("input[name=date_valid_to]", "01.09.2017"))
# Code
common_litecart.input_value_to_input_field(browser, {
"location": "input[name=code]",
"value": "fd001"
})
# Name
common_litecart.input_value_to_input_field(browser, {
"location": "input[name='name[en]']",
"value": "Fried Duck"
})
# SKU
common_litecart.input_value_to_input_field(browser, {
"location": "input[name=sku]",
"value": "FD001"
})
# GTIN -- skipped
# TARIC -- skipped
# Quantity
common_litecart.input_value_to_input_field(browser, {
"location": "input[name=quantity]",
"value": 70
})
common_litecart.select_from_dropdown_list(
browser, {
"location": "select[name=quantity_unit_id]",
"value": "1",
"description": "pcs"
})
# Weight
common_litecart.input_value_to_scrollable_field(browser, {
"location": "input[name=weight]",
"value": "4.5"
})
common_litecart.select_from_dropdown_list(
browser, {
"location": "select[name=weight_class]",
"value": "lb",
"description": "lb"
})
# Width x Height x Length
common_litecart.input_value_to_scrollable_field(browser, {
"location": "input[name=dim_x]",
"value": "20,0"
})
common_litecart.input_value_to_scrollable_field(browser, {
"location": "input[name=dim_y]",
"value": "30,0"
})
common_litecart.input_value_to_scrollable_field(browser, {
"location": "input[name=dim_z]",
"value": "40,0"
})
common_litecart.select_from_dropdown_list(browser, {
"location": "select[name=dim_class]",
"value": "cm",
"description": "cm"
})
# Delivery Status
common_litecart.select_from_dropdown_list(
browser, {
"location": "select[name=delivery_status_id]",
"value": "1",
"description": "3-5 days"
})
# Sold Out Status
common_litecart.select_from_dropdown_list(
browser, {
"location": "select[name=sold_out_status_id]",
"value": "1",
"description": "Sold out"
})
# Images
common_litecart.input_value_to_input_field(
browser, {
"location": "input[name='new_images[]']",
"value": os.getcwd() + "\\images\\fried_duck.jpg"
})
common.info("Set %s = '%s': ok" %
("input[name='new_images[]']",
os.getcwd() + "\\images\\fried_duck.jpg"))
# go to the 'Information' tab
browser.find_element_by_css_selector("a[href='#tab-information']").click()
wait.until(
EC.presence_of_element_located(
(By.CSS_SELECTOR, "input[name='meta_description[en]']")))
common.success("Open 'Information' tab")
# fill the data on the 'Information' tab
# Manufacturer
common_litecart.select_from_dropdown_list(
browser, {
"location": "select[name=manufacturer_id]",
"value": "1",
"description": "ACME Corp."
})
# Supplier -- skipped
# Keywords
common_litecart.input_value_to_input_field(browser, {
"location": "input[name=keywords]",
"value": "fried duck food"
})
# Short Description
common_litecart.input_value_to_input_field(
browser, {
"location": "input[name='short_description[en]']",
"value": "short description"
})
# Description
common_litecart.input_value_to_input_field(
browser, {
"location": "textarea[name='description[en]']",
"value": "Full description of the fried duck"
})
# Attributes -- skipped
# Head Title -- skipped
# Meta Description -- skipped
# go to the 'Prices' tab
browser.find_element_by_css_selector("a[href='#tab-prices']").click()
wait.until(
EC.presence_of_element_located(
(By.CSS_SELECTOR, "input[name='gross_prices[EUR]']")))
common.success("Open 'Prices' tab")
# fill the data on the 'Prices' tab
# Purchase Price
common_litecart.input_value_to_scrollable_field(browser, {
"location": "input[name=purchase_price]",
"value": "25"
})
common_litecart.select_from_dropdown_list(
browser, {
"location": "select[name=purchase_price_currency_code]",
"value": "USD",
"description": "US Dollars"
})
# Tax Class -- skipped
# Price
common_litecart.input_value_to_input_field(browser, {
"location": "input[name='prices[USD]']",
"value": "25"
})
# Price Incl. Tax -- skipped
# press 'SAVE' button
browser.find_element_by_css_selector("button[name=save]").click()
# go to the main store page, tab 'NEW' and check if new goods are there
browser.get("http://localhost/litecart/")
browser.find_element_by_css_selector("a[href='#latest-products']").click()
if (browser.find_elements_by_css_selector("a[title='Fried Duck']")):
common.success("Found new item in the store")
common.finish(function_result, "Add new goods")
return function_result
def run():
"""
:param app_name: HPC application
:param perf_coln: performance name to be optimized
:param num_core: number of CPU cores
:param num_node: number of computing nodes
:param rand_seed: random seed
:param num_smpl: number of samples
:param pool_size: pool size
:param num_iter: number of iterations
:param prec_rand: precentage of random samples
:param prec_init: precentage of initial samples replaced by equivalent samples
"""
try:
cm.init()
app_name = cm.app_name
perf_coln = cm.perf_coln
num_smpl = cm.num_smpl
pool_size = cm.pool_size
num_iter = cm.num_iter
prec_rand = cm.prec_rand
prec_init = cm.prec_init
if (app_name == "lv"):
conf_colns = data.lv_conf_colns
conf1_colns = data.lmp_conf_colns
conf2_colns = data.vr_conf_colns
elif (app_name == "hs"):
conf_colns = data.hs_conf_colns
conf1_colns = data.ht_conf_colns
conf2_colns = data.sw_conf_colns
num_rand = int(num_smpl * prec_rand)
# pool_df = data.gen_smpl(app_name, pool_size)
# conf_df = pool_df.head(num_rand)
conf_df = data.gen_smpl(app_name, num_rand)
train_df = cm.measure_perf(conf_df)
num_init = int(num_smpl * prec_init)
pool1_df = data.gen_smpl(cm.app1_name(app_name), num_init * 100)
conf1_df = pool1_df.head(num_init)
train1_df = cm.measure_perf(conf1_df)
pool2_df = data.gen_smpl(cm.app2_name(app_name), num_init * 100)
conf2_df = pool2_df.head(num_init)
train2_df = cm.measure_perf(conf2_df)
avg_mach_time = data.sa_mach_time(train_df) / num_rand
avg_sprt_mach_time = (data.app_mach_time(train1_df) + \
data.app_mach_time(train2_df)) / num_init
factor = max(1, avg_mach_time / avg_sprt_mach_time)
if (factor > 1):
num_sprt = int(num_init * factor)
new_conf1_df = pool1_df.head(num_sprt).tail(num_sprt - num_init)
new_train1_df = cm.measure_perf(new_conf1_df)
train1_df = tool.df_union(train1_df, new_train1_df)
new_conf2_df = pool2_df.head(num_sprt).tail(num_sprt - num_init)
new_train2_df = cm.measure_perf(new_conf2_df)
train2_df = tool.df_union(train2_df, new_train2_df)
pool_df = data.gen_smpl(app_name, pool_size)
pred_top_smpl = learn.sprt_pred_top_eval(train1_df, train2_df, pool_df,
conf1_colns, conf2_colns,
conf_colns, perf_coln,
num_smpl, 0)
nspi = int((num_smpl - num_init - num_rand) / num_iter)
for iter_idx in range(num_iter):
num_curr = num_smpl - num_init - nspi * (num_iter - 1 - iter_idx)
pred_top_smpl = pred_top_smpl.sort_values(
[perf_coln]).reset_index(drop=True)
new_conf_df = pred_top_smpl[conf_colns].head(nspi)
conf_df = tool.df_union(conf_df, new_conf_df)
last = nspi
while (conf_df.shape[0] < num_curr):
last = last + 1
new_conf_df = pred_top_smpl[conf_colns].head(last)
conf_df = tool.df_union(conf_df, new_conf_df)
new_train_df = cm.measure_perf(new_conf_df)
train_df = tool.df_union(train_df, new_train_df)
if (iter_idx < num_iter - 1):
pool_df = data.gen_smpl(app_name, pool_size)
pred_top_smpl = learn.whl_pred_top_eval(
train_df, pool_df, conf_colns, perf_coln, num_smpl, 0)
data.df2csv(train_df, app_name + "_train.csv")
mdl_chk, mdl = learn.train_mdl_chk(train_df, conf_colns, perf_coln)
top_df = cm.find_top('TaLeC', (
mdl_chk,
mdl,
), conf_colns, perf_coln, train_df)
cm.test(train_df, conf_colns, perf_coln)
cm.finish(train_df, top_df)
except:
traceback.print_exc()
def run():
"""
:param app_name: HPC application
:param perf_coln: performance name to be optimized
:param num_core: number of CPU cores
:param num_node: number of computing nodes
:param rand_seed: random seed
:param num_smpl: number of samples
"""
try:
cm.init()
app_name = cm.app_name
perf_coln = cm.perf_coln
num_smpl = cm.num_smpl
if (app_name == "lv"):
conf_colns = data.lv_conf_colns
in_conf_colns = data.lv_in_conf_colns
in_params = data.lmp_in_params
elif (app_name == "hs"):
conf_colns = data.hs_conf_colns
in_conf_colns = data.hs_in_conf_colns
in_params = data.ht_in_params
train_in_fn = cm.app_in_name(app_name) + "_time.csv"
train_in_df = data.csv2df(train_in_fn, in_conf_colns)
train_in_df = data.get_exec_mach_df(
data.get_runnable_df(train_in_df, in_conf_colns))
in_mdl_chk, in_mdl = learn.train_mdl_chk(train_in_df, in_conf_colns,
perf_coln)
conf_df = data.gen_smpl(app_name, num_smpl)
train_df = cm.measure_perf(conf_df)
data.df2csv(train_df, app_name + "_train.csv")
train_intrmdt_df = learn.add_layer_shrt_pred(train_df, in_params, conf_colns, perf_coln, \
in_mdl_chk, in_mdl)
mdl_chk, mdl = learn.train_mdl_chk(train_intrmdt_df,
conf_colns + [perf_coln + '0'],
perf_coln)
top_df = cm.find_top('cmtl_wh', (
in_mdl_chk,
in_mdl,
mdl_chk,
mdl,
), conf_colns, perf_coln, train_df)
test_df = data.csv2df(app_name + "_time.csv", conf_colns)
test_df = data.get_exec_mach_df(
data.get_runnable_df(test_df, conf_colns))
test_intrmdt_df = learn.add_layer_shrt_pred(test_df, in_params, conf_colns, perf_coln, \
in_mdl_chk, in_mdl)
'''
pred_top, err, rs = learn.whl_pred_top_eval(train_intrmdt_df, test_intrmdt_df, \
conf_colns + [perf_coln+'0'], perf_coln, 10)
data.df2csv(pred_top, app_name + "_test.csv")
data.df2csv(rs, app_name + "_rs")
data.df2csv(err, app_name + "_err.csv")
'''
cm.finish(train_df, top_df)
except:
traceback.print_exc()
def main():
import argparse
parser = argparse.ArgumentParser(
description=
"Given an archive of C files, generates Salento Package JSON files.")
parser.add_argument("-v",
dest="verbose",
help="Verbose output.",
action="store_true")
parser.add_argument(
"-i",
dest="infile",
nargs='?',
type=str,
default="/dev/stdin",
help=
"A filename of the APISAN file format (.as). DEFAULT: '%(default)s'")
parser.add_argument(
"-p",
dest="prefix",
nargs='?',
type=str,
default="as-out",
help="The directory where we are locating. DEFAULT: '%(default)s'")
parser.add_argument("-d",
help="Show commands instead of user friendly label.",
dest="debug",
action="store_true")
parser.add_argument(
"--accept-file",
help=
"A file that contains the C file names to be accepted; one file per line.",
metavar="filename",
nargs='?',
type=argparse.FileType('r'),
default=None,
dest="accept_file")
parser.add_argument(
"--skip-file",
help=
"A file that contains the C file names to be ignored; one file per line.",
metavar="filename",
nargs='?',
type=argparse.FileType('r'),
default=None,
dest="skip_file")
parser.add_argument(
"--log-ignored",
help=
"Log C files which could not be parsed to target filename (which will be appended).",
nargs='?',
type=argparse.FileType('a'),
default=None,
dest="log_ignored")
parser.add_argument("-t",
dest="timeout",
nargs='?',
type=str,
default="1h",
help="The timeout. DEFAULT: '%(default)s'")
parser.add_argument("-r",
"--run",
type=Run.from_string,
choices=list(Run),
dest='run',
default=Run.SALENTO,
help="Run until the following state. %(default)s")
parser.add_argument(
"-k",
"--keep",
nargs="+",
type=lambda x: map(Run.from_string, x),
dest='keep',
default=[Run.APISAN, Run.SALENTO],
help=
"Keep the following files, remove any files not listed. %(default)s")
parser.add_argument(
"--apisan-translator",
default=None,
help=
"Set the Apisan-to-Salento translator algorithm. Check `apisan-to-salento.py` for options."
)
parser.add_argument("--exit-on-fail",
action="store_true",
help="Stop executing when a failure occurs.")
get_nprocs = common.parser_add_parallelism(parser)
args = parser.parse_args()
with finish(
fifo(
concurrent.futures.ThreadPoolExecutor(
max_workers=get_nprocs(args)),
get_nprocs(args))) as executor:
env = Env(args=args, executor=executor)
try:
if os.path.splitext(args.infile)[1] == '.c':
process_file(env)
else:
process_tar(env)
except KeyboardInterrupt:
# Cleanup pending commands
print("Caught a Ctrl-c! Cancelling running tasks.",
file=sys.stderr)
env.cancel()
if args.exit_on_fail and len(env.failed) > 0:
sys.exit(1)