def scanprops_from_text(text, fname): base_text_props = [ 'timedependent', 'interactive', 'uses_random', 'has_testinfo', ] if text is None: propdict = {'IOerror': True} return propdict # get the props that are simple string matchs in text propdict = text_props_simple(text, base_text_props) # calc additional props mixing the traits discovered above propdict['static'] = ( not propdict['timedependent'] and not propdict['interactive'] ) # testinfo related props testinfo = testinfo_payload(text) if testinfo is not None: propdict['testinfo'] = testinfo basename = os.path.basename(fname) diagnostic, expected_snapshots = st.ScreenSampler.validated_info(testinfo, basename) propdict['testinfo_diagnostic'] = diagnostic propdict['expected_snapshots'] = expected_snapshots propdict['md5_at_testinfo'] = doers.md5_hex(text) return propdict
def scanprops_from_text(text, fname): base_text_props = [ 'timedependent', 'interactive', 'uses_random', 'has_testinfo', ] if text is None: propdict = {'IOerror': True} return propdict # get the props that are simple string matchs in text propdict = text_props_simple(text, base_text_props) # calc additional props mixing the traits discovered above propdict['static'] = (not propdict['timedependent'] and not propdict['interactive']) # testinfo related props testinfo = testinfo_payload(text) if testinfo is not None: propdict['testinfo'] = testinfo basename = os.path.basename(fname) diagnostic, expected_snapshots = st.ScreenSampler.validated_info( testinfo, basename) propdict['testinfo_diagnostic'] = diagnostic propdict['expected_snapshots'] = expected_snapshots propdict['md5_at_testinfo'] = doers.md5_hex(text) return propdict
def info_outdated_strong(db, candidates=None): """candidates wose testrun info is not uptodate with testinfo the db is not modified candidates=None means 'all know scripts' if a script can't be read, then is considered outdated """ # un loop comparando el md5 actual con el de testrun have_testrun, no_comply = get_scripts(db, 'testrun_present') outdated = set() for name in have_testrun: fname = db.fname_from_canonical(name) try: text = doers.load_text(fname) except Exception: outdated.add(name) continue actual_md5 = doers.md5_hex(text) if actual_md5 != db.get_prop_value(name,'testrun_md5'): outdated.add(name) return outdated
def info_outdated_strong(db, candidates=None): """candidates wose testrun info is not uptodate with testinfo the db is not modified candidates=None means 'all know scripts' if a script can't be read, then is considered outdated """ # un loop comparando el md5 actual con el de testrun have_testrun, no_comply = get_scripts(db, 'testrun_present') outdated = set() for name in have_testrun: fname = db.fname_from_canonical(name) try: text = doers.load_text(fname) except Exception: outdated.add(name) continue actual_md5 = doers.md5_hex(text) if actual_md5 != db.get_prop_value(name, 'testrun_md5'): outdated.add(name) return outdated
def scanprops_from_text(text, fname): base_text_props = ["timedependent", "interactive", "uses_random", "has_testinfo"] if text is None: propdict = {"IOerror": True} return propdict # get the props that are simple string matchs in text propdict = text_props_simple(text, base_text_props) # calc additional props mixing the traits discovered above propdict["static"] = not propdict["timedependent"] and not propdict["interactive"] # testinfo related props testinfo = testinfo_payload(text) if testinfo is not None: propdict["testinfo"] = testinfo basename = os.path.basename(fname) diagnostic, expected_snapshots = st.ScreenSampler.validated_info(testinfo, basename) propdict["testinfo_diagnostic"] = diagnostic propdict["expected_snapshots"] = expected_snapshots propdict["md5_at_testinfo"] = doers.md5_hex(text) return propdict
def measure_repeteability(db, candidates, limit, samples_dir, required_md5=None): """ :Parameters: limit : int or float int means count of snapdhots runs float means times in minutes, will be surpased to end the round """ start_time = time.time() scripts, unknowns = db.entities(fn_allow=fn_allow_testinfo_valid, candidates=candidates) f = db.get_prop_value # flat list of all snapshots snapshots = [ snap for name in scripts for snap in f(name, 'expected_snapshots')] # each snapshot name will hold a dict of md5(snap): count key-value pairs stats_by_snapshot_name = dict( [ (snap, {}) for snap in snapshots]) stats_by_script_name = dict( [(name, { 'timeouts':0, 'errs':0 }) for name in scripts if f(name, 'expected_snapshots')]) # get rid of scripts that don't expect snapshots scripts = [name for name in stats_by_script_name] # build a hash to limit mismatchs when combining runs. Caveat: if files # edited and testinfo not updated mismatch happens. # It is recomended to run continuations from a clean checkout for safe # combination. hasher = hashlib.md5() for name in stats_by_script_name: hasher.update(compat.asciibytes(db.get_prop_value(name, 'md5_at_testinfo'))) overall_md5 = hasher.hexdigest() if required_md5: assert required_md5==overall_md5 if not os.path.exists(samples_dir): os.makedirs(samples_dir) snapshots_abspath = os.path.abspath(samples_dir) proxy_abspath = os.path.abspath('proxy_snapshots.py') if not os.path.exists(proxy_abspath): raise ValueError("proxy script not found:%s"%proxy_abspath) rounds = 0 if isinstance(limit, float): limit_seconds = limit * 60 f_continue = lambda: (time.time() - start_time) < limit_seconds elif isinstance(limit, int): f_continue = lambda: rounds < limit else: raise ValueError while f_continue(): for name in scripts: # exercise the script acording to testinfo, snapshots would be taken fname = db.fname_from_canonical(name) stored_testinfo = db.get_prop_value(name, 'testinfo') timeout_hit, err = proxy.proxy_run( proxy_abspath, fname, [stored_testinfo, snapshots_abspath]) # count errors if timeout_hit: stats_by_script_name[name]['timeouts'] += 1 if err: stats_by_script_name[name]['errs'] += 1 # update stats by snapshots sbs = stats_by_snapshot_name for snap in stats_by_snapshot_name: sname = os.path.join(snapshots_abspath, snap) if os.path.exists(sname): try: f = open(sname, 'rb') data = f.read() f.close() md5 = doers.md5_hex(data) sbs[snap][md5] = sbs[snap].setdefault(md5, 0) + 1 except Exception: pass try: os.remove(sname) except Exception: pass rounds += 1 elapsed = time.time() - start_time return ( overall_md5, elapsed, rounds, stats_by_script_name, stats_by_snapshot_name )
def measure_repeteability(db, candidates, limit, samples_dir, required_md5=None): """ :Parameters: limit : int or float int means count of snapdhots runs float means times in minutes, will be surpased to end the round """ start_time = time.time() scripts, unknowns = db.entities(fn_allow=fn_allow_testinfo_valid, candidates=candidates) f = db.get_prop_value # flat list of all snapshots snapshots = [ snap for name in scripts for snap in f(name, 'expected_snapshots') ] # each snapshot name will hold a dict of md5(snap): count key-value pairs stats_by_snapshot_name = dict([(snap, {}) for snap in snapshots]) stats_by_script_name = dict([(name, { 'timeouts': 0, 'errs': 0 }) for name in scripts if f(name, 'expected_snapshots')]) # build a hash to limit mismatchs when combining runs. Caveat: if files # edited and testinfo not updated mismatch happens. # It is recomended to run continuations from a clean checkout for safe # combination. hasher = hashlib.md5() for name in stats_by_script_name: hasher.update(db.get_prop_value(name, 'md5_at_testinfo')) overall_md5 = hasher.hexdigest() if required_md5: assert required_md5 == overall_md5 if not os.path.exists(samples_dir): os.makedirs(samples_dir) snapshots_abspath = os.path.abspath(samples_dir) proxy_abspath = os.path.abspath('proxy_snapshots.py') if not os.path.exists(proxy_abspath): raise ValueError("proxy script not found:%s" % proxy_abspath) rounds = 0 if isinstance(limit, float): limit_seconds = limit * 60 f_continue = lambda: (time.time() - start_time) < limit_seconds elif isinstance(limit, int): f_continue = lambda: rounds < limit else: raise ValueError while f_continue(): for name in scripts: # exercise the script acording to testinfo, snapshots would be taken fname = db.fname_from_canonical(name) stored_testinfo = db.get_prop_value(name, 'testinfo') timeout_hit, err = proxy.proxy_run( proxy_abspath, fname, [stored_testinfo, snapshots_abspath]) # count errors if timeout_hit: stats_by_script_name[name]['timeouts'] += 1 if err: stats_by_script_name[name]['errs'] += 1 # update stats by snapshots sbs = stats_by_snapshot_name for snap in stats_by_snapshot_name: sname = os.path.join(snapshots_abspath, snap) if os.path.exists(sname): try: f = open(sname, 'rb') data = f.read() f.close() md5 = doers.md5_hex(data) sbs[snap][md5] = sbs[snap].setdefault(md5, 0) + 1 except Exception: pass try: os.remove(sname) except Exception: pass rounds += 1 elapsed = time.time() - start_time return (overall_md5, elapsed, rounds, stats_by_script_name, stats_by_snapshot_name)