def runTests( driver, url, timeout, framework, output = None, oneByOne = False ):
if oneByOne:
log.writeln( "Running tests one by one" )
results = framework.runTests( driver, url, timeout )
jsonResults = results[ "json" ]
xmlResults = results[ "junit" ]
else:
log.writeln( "Running tests ..." )
driver.get( url )
WebDriverWait( driver, timeout ).until( framework.isFinished )
log.writeln( "Retrieving results ..." )
jsonResults = framework.getResults( driver )
if ( output ):
xmlResults = framework.getXmlResults( driver )
printResults( jsonResults )
if ( output ):
saveResults( xmlResults, output )
log.writeln( "JUnit xml saved to: " + output )
def runTests( driver, url, timeout ):
# timeout is ignored
log.write( "Calculating number of tests ... " )
tests = getTests( driver, url )
log.writeln( "%d tests found" % len( tests ) )
testResults = []
counter = 0
for test in tests:
retries = 5
passed = False
testResult = None
counter += 1
log.write( "Running test %d: %s ... " % ( counter, test ) )
while not passed and retries:
retries -= 1
try:
testResult = runTest( driver, "%s?spec=%s" % ( url, urllib.quote( test ) ) )
if isPassed( testResult[ "json" ] ):
passed = True
except Exception:
log.writeln( "fatal error" )
raise
log.writeln( "passed" ) if passed else log.writeln( "failed" )
testResults.append( testResult )
suites = groupTestSuites( map( lambda x: x[ "json" ], testResults ) )
xmlSuites = map( lambda x: ElementTree.tostring( x ), groupXmlSuites( map( lambda x: x[ "junit" ], testResults ) ) )
testResults = {
"json": {
"suites": suites,
"durationSec": sum( map( lambda x: x[ "durationSec" ], suites ) ),
"passed": all( map( lambda x: x[ "passed" ], suites ) )
},
"junit": '<?xml version="1.0" encoding="UTF-8" ?>\n<testsuites>\n%s\n</testsuites>\n' % "\n".join( xmlSuites )
}
return testResults
def Main( testsUrl, browser, framework, seleniumServer = None, platform = None, browserVersion = None, screenResolution = None,
maxDuration = None, tunnelId = None, output = None, chromeOptions = None, prerunScriptUrl = None, oneByOne = False, avoidProxy = False ):
driver = None
framework = __import__( "lib.frameworks." + framework, fromlist = [ "lib.frameworks" ] )
if ( seleniumServer is None ):
log.writeln( "Starting selenium ..." )
seleniumServer = selenium_process.run_selenium_process()
waitSeleniumPort( seleniumServer )
try:
driver_browser = getattr( webdriver.DesiredCapabilities, browser.upper() )
if chromeOptions:
opts = chromeOptions.split( "," )
driver_browser[ "chromeOptions" ] = { "args": opts }
if not ( browserVersion is None ):
driver_browser[ "version" ] = browserVersion
if not ( screenResolution is None ):
driver_browser[ "screenResolution" ] = screenResolution
if not ( platform is None ):
driver_browser[ "platform" ] = platform
if not ( maxDuration is None ):
driver_browser[ "maxDuration" ] = maxDuration
if not ( tunnelId is None ):
driver_browser[ "tunnelIdentifier" ] = tunnelId
if not ( prerunScriptUrl is None ):
driver_browser[ "prerun" ] = { "executable": prerunScriptUrl, "background": "false" }
if avoidProxy:
driver_browser[ "avoidProxy" ] = True
log.writeln( "Connecting to selenium ..." )
driver = webdriver.Remote( seleniumServer, driver_browser )
driver.set_page_load_timeout( webDriverWaitTimeout )
log.writeln( "Selenium session id: %s" % ( driver.session_id ) )
runTests( driver = driver, url = testsUrl, timeout = maxDuration, framework = framework, output = output, oneByOne = oneByOne )
finally:
if driver:
driver.quit()
selenium_process.stop_selenium_process()
def convertAnalogyToMatrices(analogy, setting, emb_wrapper, max_answers):
'''Convert an analogy into
1) A matrix of indices in the term vocabulary
2) A matrix of term embeddings
'''
valid = True
# analogy is a:b::c:d
analogy_ixes, analogy_embeds = [], []
for i in range(4):
# if using multi_both and looking at `b`, just ignore it
if setting == settings.ALL_INFO and i == 1:
analogy_ixes.append(-3)
# if only using single answers, handle all 4 components the same
elif setting == settings.SINGLE_ANSWER or i < 3:
key = analogy[i]
ix = emb_wrapper.index(key)
analogy_ixes.append(ix)
# if using multiple answers, add on the array of valid answers
elif setting in [settings.ALL_INFO, settings.MULTI_ANSWER] and i == 3:
found_at_least_one_answer = False
for j in range(max_answers):
if j < len(analogy[3]):
ix = emb_wrapper.index(analogy[3][j])
if ix > -1: found_at_least_one_answer = True
analogy_ixes.append(ix)
else:
analogy_ixes.append(
-2
) # emb_wrapper can return -1; use -2 to signal All Done
# if none of the valid answers are in the vocabulary, then skip this analogy
if not found_at_least_one_answer: valid = False
try:
# if using multi_both, use the average of the offsets between
# `a` and all the possible `b`s
if setting == settings.ALL_INFO and i == 1:
keys = analogy[i]
ixes = [emb_wrapper.index(key) for key in keys]
avg_embed, valid_bs = 0, 0
for j in range(len(ixes)):
ix, key = ixes[j], keys[j]
# if there's a b_k we can't model, skip it
try:
if ix > -1: avg_embed += emb_wrapper[ix]
else:
log.writeln('[ERROR] USING BAD ACCESS 1: %s' % key)
avg_embed += emb_wrapper[key]
valid_bs += 1
except (KeyError, AttributeError):
pass
# as long as we were able to model at least one b_k, keep this analogy
if valid_bs > 0: analogy_embeds.append(avg_embed / valid_bs)
else: valid = False
# only grab the embeddings for a,b,c
elif i < 3:
if ix > -1:
analogy_embeds.append(emb_wrapper[ix])
else:
log.writeln('[ERROR] USING BAD ACCESS 2: %s' % key)
analogy_embeds.append(emb_wrapper[key])
except (KeyError, AttributeError) as e:
# thrown if we can't get an embedding for something in the analogy;
# skip this analogy
valid = False
print('COULD NOT FIND "%s"' % (analogy[i]))
break
return (valid, analogy_ixes, analogy_embeds)
with open(predictions_file, 'w') as stream:
pass
for (embedf, label, vocab_is_dirty) in config.LABELED_EMBEDDINGS:
if type(embedf) is tuple:
(embedf, glove_vocabf) = embedf
else:
glove_vocabf = None
these_results_dir = os.path.join(
results_dir,
os.path.splitext(os.path.basename(embedf))[0])
if not os.path.isdir(these_results_dir):
os.mkdir(these_results_dir)
log.writeln(
('\n\n\n{0}\nEmbeddings: %s\n{0}\n\n' % label).format('-' * 79))
if predictions_file:
with open(predictions_file, 'a') as stream:
stream.write(('\n\n\n{0}\nEmbeddings: %s\n{0}\n\n\n' %
label).format('-' * 79))
results = evaluate(embedf,
analogy_file,
setting,
freqtermf,
unigrams,
analogy_method,
log=log,
predictions_file=predictions_file,
predictions_file_mode='a',
def printResults( results ): log.writeln( "Results:" ) log.writeln( " Passed: %r" % results[ "passed" ] ) log.writeln( " Duration: %f" % results[ "durationSec" ] ) log.writeln( " Suites: %d" % len( results[ "suites" ] ) )