def scanprops_from_text(text, fname):
base_text_props = [
'timedependent',
'interactive',
'uses_random',
'has_testinfo',
]
if text is None:
propdict = {'IOerror': True}
return propdict
# get the props that are simple string matchs in text
propdict = text_props_simple(text, base_text_props)
# calc additional props mixing the traits discovered above
propdict['static'] = ( not propdict['timedependent'] and
not propdict['interactive'] )
# testinfo related props
testinfo = testinfo_payload(text)
if testinfo is not None:
propdict['testinfo'] = testinfo
basename = os.path.basename(fname)
diagnostic, expected_snapshots = st.ScreenSampler.validated_info(testinfo, basename)
propdict['testinfo_diagnostic'] = diagnostic
propdict['expected_snapshots'] = expected_snapshots
propdict['md5_at_testinfo'] = doers.md5_hex(text)
return propdict
def scanprops_from_text(text, fname):
base_text_props = [
'timedependent',
'interactive',
'uses_random',
'has_testinfo',
]
if text is None:
propdict = {'IOerror': True}
return propdict
# get the props that are simple string matchs in text
propdict = text_props_simple(text, base_text_props)
# calc additional props mixing the traits discovered above
propdict['static'] = (not propdict['timedependent']
and not propdict['interactive'])
# testinfo related props
testinfo = testinfo_payload(text)
if testinfo is not None:
propdict['testinfo'] = testinfo
basename = os.path.basename(fname)
diagnostic, expected_snapshots = st.ScreenSampler.validated_info(
testinfo, basename)
propdict['testinfo_diagnostic'] = diagnostic
propdict['expected_snapshots'] = expected_snapshots
propdict['md5_at_testinfo'] = doers.md5_hex(text)
return propdict
def info_outdated_strong(db, candidates=None):
"""candidates wose testrun info is not uptodate with testinfo
the db is not modified
candidates=None means 'all know scripts'
if a script can't be read, then is considered outdated
"""
# un loop comparando el md5 actual con el de testrun
have_testrun, no_comply = get_scripts(db, 'testrun_present')
outdated = set()
for name in have_testrun:
fname = db.fname_from_canonical(name)
try:
text = doers.load_text(fname)
except Exception:
outdated.add(name)
continue
actual_md5 = doers.md5_hex(text)
if actual_md5 != db.get_prop_value(name,'testrun_md5'):
outdated.add(name)
return outdated
def info_outdated_strong(db, candidates=None):
"""candidates wose testrun info is not uptodate with testinfo
the db is not modified
candidates=None means 'all know scripts'
if a script can't be read, then is considered outdated
"""
# un loop comparando el md5 actual con el de testrun
have_testrun, no_comply = get_scripts(db, 'testrun_present')
outdated = set()
for name in have_testrun:
fname = db.fname_from_canonical(name)
try:
text = doers.load_text(fname)
except Exception:
outdated.add(name)
continue
actual_md5 = doers.md5_hex(text)
if actual_md5 != db.get_prop_value(name, 'testrun_md5'):
outdated.add(name)
return outdated
def scanprops_from_text(text, fname):
base_text_props = ["timedependent", "interactive", "uses_random", "has_testinfo"]
if text is None:
propdict = {"IOerror": True}
return propdict
# get the props that are simple string matchs in text
propdict = text_props_simple(text, base_text_props)
# calc additional props mixing the traits discovered above
propdict["static"] = not propdict["timedependent"] and not propdict["interactive"]
# testinfo related props
testinfo = testinfo_payload(text)
if testinfo is not None:
propdict["testinfo"] = testinfo
basename = os.path.basename(fname)
diagnostic, expected_snapshots = st.ScreenSampler.validated_info(testinfo, basename)
propdict["testinfo_diagnostic"] = diagnostic
propdict["expected_snapshots"] = expected_snapshots
propdict["md5_at_testinfo"] = doers.md5_hex(text)
return propdict
def measure_repeteability(db, candidates, limit, samples_dir, required_md5=None):
"""
:Parameters:
limit : int or float
int means count of snapdhots runs
float means times in minutes, will be surpased to end the round
"""
start_time = time.time()
scripts, unknowns = db.entities(fn_allow=fn_allow_testinfo_valid,
candidates=candidates)
f = db.get_prop_value
# flat list of all snapshots
snapshots = [ snap for name in scripts
for snap in f(name, 'expected_snapshots')]
# each snapshot name will hold a dict of md5(snap): count key-value pairs
stats_by_snapshot_name = dict( [ (snap, {}) for snap in snapshots])
stats_by_script_name = dict( [(name, { 'timeouts':0, 'errs':0 })
for name in scripts if f(name, 'expected_snapshots')])
# get rid of scripts that don't expect snapshots
scripts = [name for name in stats_by_script_name]
# build a hash to limit mismatchs when combining runs. Caveat: if files
# edited and testinfo not updated mismatch happens.
# It is recomended to run continuations from a clean checkout for safe
# combination.
hasher = hashlib.md5()
for name in stats_by_script_name:
hasher.update(compat.asciibytes(db.get_prop_value(name, 'md5_at_testinfo')))
overall_md5 = hasher.hexdigest()
if required_md5:
assert required_md5==overall_md5
if not os.path.exists(samples_dir):
os.makedirs(samples_dir)
snapshots_abspath = os.path.abspath(samples_dir)
proxy_abspath = os.path.abspath('proxy_snapshots.py')
if not os.path.exists(proxy_abspath):
raise ValueError("proxy script not found:%s"%proxy_abspath)
rounds = 0
if isinstance(limit, float):
limit_seconds = limit * 60
f_continue = lambda: (time.time() - start_time) < limit_seconds
elif isinstance(limit, int):
f_continue = lambda: rounds < limit
else:
raise ValueError
while f_continue():
for name in scripts:
# exercise the script acording to testinfo, snapshots would be taken
fname = db.fname_from_canonical(name)
stored_testinfo = db.get_prop_value(name, 'testinfo')
timeout_hit, err = proxy.proxy_run(
proxy_abspath,
fname,
[stored_testinfo, snapshots_abspath])
# count errors
if timeout_hit:
stats_by_script_name[name]['timeouts'] += 1
if err:
stats_by_script_name[name]['errs'] += 1
# update stats by snapshots
sbs = stats_by_snapshot_name
for snap in stats_by_snapshot_name:
sname = os.path.join(snapshots_abspath, snap)
if os.path.exists(sname):
try:
f = open(sname, 'rb')
data = f.read()
f.close()
md5 = doers.md5_hex(data)
sbs[snap][md5] = sbs[snap].setdefault(md5, 0) + 1
except Exception:
pass
try:
os.remove(sname)
except Exception:
pass
rounds += 1
elapsed = time.time() - start_time
return ( overall_md5, elapsed, rounds, stats_by_script_name,
stats_by_snapshot_name )
def measure_repeteability(db,
candidates,
limit,
samples_dir,
required_md5=None):
"""
:Parameters:
limit : int or float
int means count of snapdhots runs
float means times in minutes, will be surpased to end the round
"""
start_time = time.time()
scripts, unknowns = db.entities(fn_allow=fn_allow_testinfo_valid,
candidates=candidates)
f = db.get_prop_value
# flat list of all snapshots
snapshots = [
snap for name in scripts for snap in f(name, 'expected_snapshots')
]
# each snapshot name will hold a dict of md5(snap): count key-value pairs
stats_by_snapshot_name = dict([(snap, {}) for snap in snapshots])
stats_by_script_name = dict([(name, {
'timeouts': 0,
'errs': 0
}) for name in scripts if f(name, 'expected_snapshots')])
# build a hash to limit mismatchs when combining runs. Caveat: if files
# edited and testinfo not updated mismatch happens.
# It is recomended to run continuations from a clean checkout for safe
# combination.
hasher = hashlib.md5()
for name in stats_by_script_name:
hasher.update(db.get_prop_value(name, 'md5_at_testinfo'))
overall_md5 = hasher.hexdigest()
if required_md5:
assert required_md5 == overall_md5
if not os.path.exists(samples_dir):
os.makedirs(samples_dir)
snapshots_abspath = os.path.abspath(samples_dir)
proxy_abspath = os.path.abspath('proxy_snapshots.py')
if not os.path.exists(proxy_abspath):
raise ValueError("proxy script not found:%s" % proxy_abspath)
rounds = 0
if isinstance(limit, float):
limit_seconds = limit * 60
f_continue = lambda: (time.time() - start_time) < limit_seconds
elif isinstance(limit, int):
f_continue = lambda: rounds < limit
else:
raise ValueError
while f_continue():
for name in scripts:
# exercise the script acording to testinfo, snapshots would be taken
fname = db.fname_from_canonical(name)
stored_testinfo = db.get_prop_value(name, 'testinfo')
timeout_hit, err = proxy.proxy_run(
proxy_abspath, fname, [stored_testinfo, snapshots_abspath])
# count errors
if timeout_hit:
stats_by_script_name[name]['timeouts'] += 1
if err:
stats_by_script_name[name]['errs'] += 1
# update stats by snapshots
sbs = stats_by_snapshot_name
for snap in stats_by_snapshot_name:
sname = os.path.join(snapshots_abspath, snap)
if os.path.exists(sname):
try:
f = open(sname, 'rb')
data = f.read()
f.close()
md5 = doers.md5_hex(data)
sbs[snap][md5] = sbs[snap].setdefault(md5, 0) + 1
except Exception:
pass
try:
os.remove(sname)
except Exception:
pass
rounds += 1
elapsed = time.time() - start_time
return (overall_md5, elapsed, rounds, stats_by_script_name,
stats_by_snapshot_name)