def measure_repeteability(db, candidates, limit, samples_dir, required_md5=None):
"""
:Parameters:
limit : int or float
int means count of snapdhots runs
float means times in minutes, will be surpased to end the round
"""
start_time = time.time()
scripts, unknowns = db.entities(fn_allow=fn_allow_testinfo_valid,
candidates=candidates)
f = db.get_prop_value
# flat list of all snapshots
snapshots = [ snap for name in scripts
for snap in f(name, 'expected_snapshots')]
# each snapshot name will hold a dict of md5(snap): count key-value pairs
stats_by_snapshot_name = dict( [ (snap, {}) for snap in snapshots])
stats_by_script_name = dict( [(name, { 'timeouts':0, 'errs':0 })
for name in scripts if f(name, 'expected_snapshots')])
# get rid of scripts that don't expect snapshots
scripts = [name for name in stats_by_script_name]
# build a hash to limit mismatchs when combining runs. Caveat: if files
# edited and testinfo not updated mismatch happens.
# It is recomended to run continuations from a clean checkout for safe
# combination.
hasher = hashlib.md5()
for name in stats_by_script_name:
hasher.update(compat.asciibytes(db.get_prop_value(name, 'md5_at_testinfo')))
overall_md5 = hasher.hexdigest()
if required_md5:
assert required_md5==overall_md5
if not os.path.exists(samples_dir):
os.makedirs(samples_dir)
snapshots_abspath = os.path.abspath(samples_dir)
proxy_abspath = os.path.abspath('proxy_snapshots.py')
if not os.path.exists(proxy_abspath):
raise ValueError("proxy script not found:%s"%proxy_abspath)
rounds = 0
if isinstance(limit, float):
limit_seconds = limit * 60
f_continue = lambda: (time.time() - start_time) < limit_seconds
elif isinstance(limit, int):
f_continue = lambda: rounds < limit
else:
raise ValueError
while f_continue():
for name in scripts:
# exercise the script acording to testinfo, snapshots would be taken
fname = db.fname_from_canonical(name)
stored_testinfo = db.get_prop_value(name, 'testinfo')
timeout_hit, err = proxy.proxy_run(
proxy_abspath,
fname,
[stored_testinfo, snapshots_abspath])
# count errors
if timeout_hit:
stats_by_script_name[name]['timeouts'] += 1
if err:
stats_by_script_name[name]['errs'] += 1
# update stats by snapshots
sbs = stats_by_snapshot_name
for snap in stats_by_snapshot_name:
sname = os.path.join(snapshots_abspath, snap)
if os.path.exists(sname):
try:
f = open(sname, 'rb')
data = f.read()
f.close()
md5 = doers.md5_hex(data)
sbs[snap][md5] = sbs[snap].setdefault(md5, 0) + 1
except Exception:
pass
try:
os.remove(sname)
except Exception:
pass
rounds += 1
elapsed = time.time() - start_time
return ( overall_md5, elapsed, rounds, stats_by_script_name,
stats_by_snapshot_name )
def measure_repeteability(db,
candidates,
limit,
samples_dir,
required_md5=None):
"""
:Parameters:
limit : int or float
int means count of snapdhots runs
float means times in minutes, will be surpased to end the round
"""
start_time = time.time()
scripts, unknowns = db.entities(fn_allow=fn_allow_testinfo_valid,
candidates=candidates)
f = db.get_prop_value
# flat list of all snapshots
snapshots = [
snap for name in scripts for snap in f(name, 'expected_snapshots')
]
# each snapshot name will hold a dict of md5(snap): count key-value pairs
stats_by_snapshot_name = dict([(snap, {}) for snap in snapshots])
stats_by_script_name = dict([(name, {
'timeouts': 0,
'errs': 0
}) for name in scripts if f(name, 'expected_snapshots')])
# build a hash to limit mismatchs when combining runs. Caveat: if files
# edited and testinfo not updated mismatch happens.
# It is recomended to run continuations from a clean checkout for safe
# combination.
hasher = hashlib.md5()
for name in stats_by_script_name:
hasher.update(db.get_prop_value(name, 'md5_at_testinfo'))
overall_md5 = hasher.hexdigest()
if required_md5:
assert required_md5 == overall_md5
if not os.path.exists(samples_dir):
os.makedirs(samples_dir)
snapshots_abspath = os.path.abspath(samples_dir)
proxy_abspath = os.path.abspath('proxy_snapshots.py')
if not os.path.exists(proxy_abspath):
raise ValueError("proxy script not found:%s" % proxy_abspath)
rounds = 0
if isinstance(limit, float):
limit_seconds = limit * 60
f_continue = lambda: (time.time() - start_time) < limit_seconds
elif isinstance(limit, int):
f_continue = lambda: rounds < limit
else:
raise ValueError
while f_continue():
for name in scripts:
# exercise the script acording to testinfo, snapshots would be taken
fname = db.fname_from_canonical(name)
stored_testinfo = db.get_prop_value(name, 'testinfo')
timeout_hit, err = proxy.proxy_run(
proxy_abspath, fname, [stored_testinfo, snapshots_abspath])
# count errors
if timeout_hit:
stats_by_script_name[name]['timeouts'] += 1
if err:
stats_by_script_name[name]['errs'] += 1
# update stats by snapshots
sbs = stats_by_snapshot_name
for snap in stats_by_snapshot_name:
sname = os.path.join(snapshots_abspath, snap)
if os.path.exists(sname):
try:
f = open(sname, 'rb')
data = f.read()
f.close()
md5 = doers.md5_hex(data)
sbs[snap][md5] = sbs[snap].setdefault(md5, 0) + 1
except Exception:
pass
try:
os.remove(sname)
except Exception:
pass
rounds += 1
elapsed = time.time() - start_time
return (overall_md5, elapsed, rounds, stats_by_script_name,
stats_by_snapshot_name)
def update_snapshots(db, filename_persist, target_scripts, snapshots_dir):
"""
runs the scripts in target scripts, taking snapshots as indicated by the
testinfo in the script, and updating the snapshots related info in the db.
Params:
db:
a remembercases.TestbedEntityPropDB object
filename_persist:
filename to persist the db after updating
target_scripts:
iterable yielding scripts. If None, all scripts known in the
default testbed are assumed.
snapshots_dir:
directory to store the snapshots
Returns (valid_scripts, rejected) where:
valid_scripts :
scripts in target_scripts that are known in the default testbed and
have valid testinfo
rejected :
scripts in target_scripts that are unknown in the default testbed or
dont have valid testinfo
Db operations are done over the default testbed, which should have been
set bejore calling here.
For each valid entity the following props are set:
'snapshots_success':
bool, True if (no traceback or timeout when running the script,
also all expected snapshots had been produced), else False
'snapshots_diagnostic':
string, '' means no errors, else description of failure while
trying to take snapshots
'missing_snapshots':
list of missing snapshots filenames
"""
proxy_abspath = os.path.abspath('proxy_snapshots.py')
if not os.path.exists(proxy_abspath):
raise ValueError("proxy script not found:%s"%proxy_abspath)
snapshots_abspath = os.path.abspath(snapshots_dir)
if not os.path.exists(snapshots_abspath):
os.makedirs(snapshots_abspath)
valid_scripts, rejected = db.entities(fn_allow=fn_allow_testinfo_valid,
candidates=target_scripts)
for script in valid_scripts:
# get the exercise plan
stored_testinfo = db.get_prop_value(script, 'testinfo')
# delete old snapshots if they exist
expected_snapshots = db.get_prop_value(script, 'expected_snapshots')
for name in expected_snapshots:
p = os.path.join(snapshots_abspath, name)
if os.path.exists(p):
os.remove(p)
# exercise the script acording to testinfo, snapshots would be taken
fname = db.fname_from_canonical(script)
timeout_hit, err = proxy.proxy_run(proxy_abspath, fname, [stored_testinfo, snapshots_abspath])
# calc and store missing_snapshots prop
missing = [ s for s in expected_snapshots if not os.path.exists(os.path.join(snapshots_abspath, s))]
db.set_prop_value(script, 'missing_snapshots', missing)
# calc and store snapshots_success prop
snapshots_success = (err=='' and
not timeout_hit and
len(missing)==0)
db.set_prop_value(script, 'snapshots_success', snapshots_success)
# calc and store snapshots_diagnostic prop
snapshots_diagnostic = ''
if not snapshots_success:
if timeout_hit:
err = ('Timeout hit. Remember in this case it is posible not all stderr captured.\n' +
err)
if len(missing):
missing = [ os.path.basename(s) for s in missing ]
err += '\nNot all snapshots captured - missing snapshots:\n'
err += '\n'.join(missing)
snapshots_diagnostic = err
print('err:', err)
db.set_prop_value(script, 'snapshots_diagnostic', snapshots_diagnostic)
# update history
text = '\n'.join(valid_scripts)
db.history_add('update_snapshots', text)
if filename_persist:
dbm.db_save(db, filename_persist)
return valid_scripts, rejected
def update_snapshots(db, filename_persist, target_scripts, snapshots_dir):
"""
runs the scripts in target scripts, taking snapshots as indicated by the
testinfo in the script, and updating the snapshots related info in the db.
Params:
db:
a remembercases.TestbedEntityPropDB object
filename_persist:
filename to persist the db after updating
target_scripts:
iterable yielding scripts. If None, all scripts known in the
default testbed are assumed.
snapshots_dir:
directory to store the snapshots
Returns (valid_scripts, rejected) where:
valid_scripts :
scripts in target_scripts that are known in the default testbed and
have valid testinfo
rejected :
scripts in target_scripts that are unknown in the default testbed or
dont have valid testinfo
Db operations are done over the default testbed, which should have been
set bejore calling here.
For each valid entity the following props are set:
'snapshots_success':
bool, True if (no traceback or timeout when running the script,
also all expected snapshots had been produced), else False
'snapshots_diagnostic':
string, '' means no errors, else description of failure while
trying to take snapshots
'missing_snapshots':
list of missing snapshots filenames
"""
proxy_abspath = os.path.abspath('proxy_snapshots.py')
if not os.path.exists(proxy_abspath):
raise ValueError("proxy script not found:%s" % proxy_abspath)
snapshots_abspath = os.path.abspath(snapshots_dir)
if not os.path.exists(snapshots_abspath):
os.makedirs(snapshots_abspath)
valid_scripts, rejected = db.entities(fn_allow=fn_allow_testinfo_valid,
candidates=target_scripts)
for script in valid_scripts:
# get the exercise plan
stored_testinfo = db.get_prop_value(script, 'testinfo')
# delete old snapshots if they exist
expected_snapshots = db.get_prop_value(script, 'expected_snapshots')
for name in expected_snapshots:
p = os.path.join(snapshots_abspath, name)
if os.path.exists(p):
os.remove(p)
# exercise the script acording to testinfo, snapshots would be taken
fname = db.fname_from_canonical(script)
timeout_hit, err = proxy.proxy_run(
proxy_abspath, fname, [stored_testinfo, snapshots_abspath])
# calc and store missing_snapshots prop
missing = [
s for s in expected_snapshots
if not os.path.exists(os.path.join(snapshots_abspath, s))
]
db.set_prop_value(script, 'missing_snapshots', missing)
# calc and store snapshots_success prop
snapshots_success = (err == '' and not timeout_hit
and len(missing) == 0)
db.set_prop_value(script, 'snapshots_success', snapshots_success)
# calc and store snapshots_diagnostic prop
snapshots_diagnostic = ''
if not snapshots_success:
if timeout_hit:
err = (
'Timeout hit. Remember in this case it is posible not all stderr captured.\n'
+ err)
if len(missing):
missing = [os.path.basename(s) for s in missing]
err += '\nNot all snapshots captured - missing snapshots:\n'
err += '\n'.join(missing)
snapshots_diagnostic = err
print 'err:', err
db.set_prop_value(script, 'snapshots_diagnostic', snapshots_diagnostic)
# update history
text = '\n'.join(valid_scripts)
db.history_add('update_snapshots', text)
if filename_persist:
dbm.db_save(db, filename_persist)
return valid_scripts, rejected
