def runTests( driver, url, timeout, framework, output = None, oneByOne = False ): if oneByOne: log.writeln( "Running tests one by one" ) results = framework.runTests( driver, url, timeout ) jsonResults = results[ "json" ] xmlResults = results[ "junit" ] else: log.writeln( "Running tests ..." ) driver.get( url ) WebDriverWait( driver, timeout ).until( framework.isFinished ) log.writeln( "Retrieving results ..." ) jsonResults = framework.getResults( driver ) if ( output ): xmlResults = framework.getXmlResults( driver ) printResults( jsonResults ) if ( output ): saveResults( xmlResults, output ) log.writeln( "JUnit xml saved to: " + output )
def runTests( driver, url, timeout ): # timeout is ignored log.write( "Calculating number of tests ... " ) tests = getTests( driver, url ) log.writeln( "%d tests found" % len( tests ) ) testResults = [] counter = 0 for test in tests: retries = 5 passed = False testResult = None counter += 1 log.write( "Running test %d: %s ... " % ( counter, test ) ) while not passed and retries: retries -= 1 try: testResult = runTest( driver, "%s?spec=%s" % ( url, urllib.quote( test ) ) ) if isPassed( testResult[ "json" ] ): passed = True except Exception: log.writeln( "fatal error" ) raise log.writeln( "passed" ) if passed else log.writeln( "failed" ) testResults.append( testResult ) suites = groupTestSuites( map( lambda x: x[ "json" ], testResults ) ) xmlSuites = map( lambda x: ElementTree.tostring( x ), groupXmlSuites( map( lambda x: x[ "junit" ], testResults ) ) ) testResults = { "json": { "suites": suites, "durationSec": sum( map( lambda x: x[ "durationSec" ], suites ) ), "passed": all( map( lambda x: x[ "passed" ], suites ) ) }, "junit": '<?xml version="1.0" encoding="UTF-8" ?>\n<testsuites>\n%s\n</testsuites>\n' % "\n".join( xmlSuites ) } return testResults
def Main( testsUrl, browser, framework, seleniumServer = None, platform = None, browserVersion = None, screenResolution = None, maxDuration = None, tunnelId = None, output = None, chromeOptions = None, prerunScriptUrl = None, oneByOne = False, avoidProxy = False ): driver = None framework = __import__( "lib.frameworks." + framework, fromlist = [ "lib.frameworks" ] ) if ( seleniumServer is None ): log.writeln( "Starting selenium ..." ) seleniumServer = selenium_process.run_selenium_process() waitSeleniumPort( seleniumServer ) try: driver_browser = getattr( webdriver.DesiredCapabilities, browser.upper() ) if chromeOptions: opts = chromeOptions.split( "," ) driver_browser[ "chromeOptions" ] = { "args": opts } if not ( browserVersion is None ): driver_browser[ "version" ] = browserVersion if not ( screenResolution is None ): driver_browser[ "screenResolution" ] = screenResolution if not ( platform is None ): driver_browser[ "platform" ] = platform if not ( maxDuration is None ): driver_browser[ "maxDuration" ] = maxDuration if not ( tunnelId is None ): driver_browser[ "tunnelIdentifier" ] = tunnelId if not ( prerunScriptUrl is None ): driver_browser[ "prerun" ] = { "executable": prerunScriptUrl, "background": "false" } if avoidProxy: driver_browser[ "avoidProxy" ] = True log.writeln( "Connecting to selenium ..." ) driver = webdriver.Remote( seleniumServer, driver_browser ) driver.set_page_load_timeout( webDriverWaitTimeout ) log.writeln( "Selenium session id: %s" % ( driver.session_id ) ) runTests( driver = driver, url = testsUrl, timeout = maxDuration, framework = framework, output = output, oneByOne = oneByOne ) finally: if driver: driver.quit() selenium_process.stop_selenium_process()
def convertAnalogyToMatrices(analogy, setting, emb_wrapper, max_answers): '''Convert an analogy into 1) A matrix of indices in the term vocabulary 2) A matrix of term embeddings ''' valid = True # analogy is a:b::c:d analogy_ixes, analogy_embeds = [], [] for i in range(4): # if using multi_both and looking at `b`, just ignore it if setting == settings.ALL_INFO and i == 1: analogy_ixes.append(-3) # if only using single answers, handle all 4 components the same elif setting == settings.SINGLE_ANSWER or i < 3: key = analogy[i] ix = emb_wrapper.index(key) analogy_ixes.append(ix) # if using multiple answers, add on the array of valid answers elif setting in [settings.ALL_INFO, settings.MULTI_ANSWER] and i == 3: found_at_least_one_answer = False for j in range(max_answers): if j < len(analogy[3]): ix = emb_wrapper.index(analogy[3][j]) if ix > -1: found_at_least_one_answer = True analogy_ixes.append(ix) else: analogy_ixes.append( -2 ) # emb_wrapper can return -1; use -2 to signal All Done # if none of the valid answers are in the vocabulary, then skip this analogy if not found_at_least_one_answer: valid = False try: # if using multi_both, use the average of the offsets between # `a` and all the possible `b`s if setting == settings.ALL_INFO and i == 1: keys = analogy[i] ixes = [emb_wrapper.index(key) for key in keys] avg_embed, valid_bs = 0, 0 for j in range(len(ixes)): ix, key = ixes[j], keys[j] # if there's a b_k we can't model, skip it try: if ix > -1: avg_embed += emb_wrapper[ix] else: log.writeln('[ERROR] USING BAD ACCESS 1: %s' % key) avg_embed += emb_wrapper[key] valid_bs += 1 except (KeyError, AttributeError): pass # as long as we were able to model at least one b_k, keep this analogy if valid_bs > 0: analogy_embeds.append(avg_embed / valid_bs) else: valid = False # only grab the embeddings for a,b,c elif i < 3: if ix > -1: analogy_embeds.append(emb_wrapper[ix]) else: log.writeln('[ERROR] USING BAD ACCESS 2: %s' % key) analogy_embeds.append(emb_wrapper[key]) except (KeyError, AttributeError) as e: # thrown if we can't get an embedding for something in the analogy; # skip this analogy valid = False print('COULD NOT FIND "%s"' % (analogy[i])) break return (valid, analogy_ixes, analogy_embeds)
with open(predictions_file, 'w') as stream: pass for (embedf, label, vocab_is_dirty) in config.LABELED_EMBEDDINGS: if type(embedf) is tuple: (embedf, glove_vocabf) = embedf else: glove_vocabf = None these_results_dir = os.path.join( results_dir, os.path.splitext(os.path.basename(embedf))[0]) if not os.path.isdir(these_results_dir): os.mkdir(these_results_dir) log.writeln( ('\n\n\n{0}\nEmbeddings: %s\n{0}\n\n' % label).format('-' * 79)) if predictions_file: with open(predictions_file, 'a') as stream: stream.write(('\n\n\n{0}\nEmbeddings: %s\n{0}\n\n\n' % label).format('-' * 79)) results = evaluate(embedf, analogy_file, setting, freqtermf, unigrams, analogy_method, log=log, predictions_file=predictions_file, predictions_file_mode='a',
def printResults( results ): log.writeln( "Results:" ) log.writeln( " Passed: %r" % results[ "passed" ] ) log.writeln( " Duration: %f" % results[ "durationSec" ] ) log.writeln( " Suites: %d" % len( results[ "suites" ] ) )