Beispiel #1
0
def measure_repeteability(db, candidates, limit, samples_dir, required_md5=None):
    """
    :Parameters:
        limit : int or float
            int means count of snapdhots runs
            float means times in minutes, will be surpased to end the round
    """
    start_time = time.time()
    scripts, unknowns = db.entities(fn_allow=fn_allow_testinfo_valid,
                                    candidates=candidates)
    
    f = db.get_prop_value
    # flat list of all snapshots
    snapshots = [ snap for name in scripts
                       for snap in f(name, 'expected_snapshots')]
    # each snapshot name will hold a dict of md5(snap): count key-value pairs
    stats_by_snapshot_name = dict( [ (snap, {}) for snap in snapshots])

    stats_by_script_name = dict( [(name, { 'timeouts':0, 'errs':0 })
                        for name in scripts if f(name, 'expected_snapshots')])

    # get rid of scripts that don't expect snapshots
    scripts = [name for name in stats_by_script_name]

    # build a hash to limit mismatchs when combining runs. Caveat: if files
    # edited and testinfo not updated mismatch happens.
    # It is recomended to run continuations from a clean checkout for safe
    # combination.    
    hasher = hashlib.md5()
    for name in stats_by_script_name:
        hasher.update(compat.asciibytes(db.get_prop_value(name, 'md5_at_testinfo')))
    overall_md5 = hasher.hexdigest()
    if required_md5:
        assert required_md5==overall_md5

    if not os.path.exists(samples_dir):
        os.makedirs(samples_dir)
    snapshots_abspath = os.path.abspath(samples_dir)

    proxy_abspath = os.path.abspath('proxy_snapshots.py')
    if not os.path.exists(proxy_abspath):
        raise ValueError("proxy script not found:%s"%proxy_abspath)

    rounds = 0
    if isinstance(limit, float):
        limit_seconds = limit * 60
        f_continue = lambda: (time.time() - start_time) < limit_seconds
    elif isinstance(limit, int):
        f_continue = lambda: rounds < limit
    else:
        raise ValueError


    while f_continue():
        for name in scripts:
            # exercise the script acording to testinfo, snapshots would be taken 
            fname = db.fname_from_canonical(name)
            stored_testinfo = db.get_prop_value(name, 'testinfo')
            timeout_hit, err = proxy.proxy_run(
                                        proxy_abspath,
                                        fname,
                                        [stored_testinfo, snapshots_abspath])
            # count errors
            if timeout_hit:
                stats_by_script_name[name]['timeouts'] += 1
            if err:
                stats_by_script_name[name]['errs'] += 1

            # update stats by snapshots
            sbs = stats_by_snapshot_name
            for snap in stats_by_snapshot_name:
                sname = os.path.join(snapshots_abspath, snap)
                if os.path.exists(sname):
                    try:
                        f = open(sname, 'rb')
                        data = f.read()
                        f.close()
                        md5 = doers.md5_hex(data)
                        sbs[snap][md5] = sbs[snap].setdefault(md5, 0) + 1 
                    except Exception:
                        pass
                    try:
                        os.remove(sname)
                    except Exception:
                        pass
        rounds += 1

    elapsed = time.time() - start_time
    return ( overall_md5, elapsed, rounds, stats_by_script_name,
                                                     stats_by_snapshot_name ) 
Beispiel #2
0
def measure_repeteability(db,
                          candidates,
                          limit,
                          samples_dir,
                          required_md5=None):
    """
    :Parameters:
        limit : int or float
            int means count of snapdhots runs
            float means times in minutes, will be surpased to end the round
    """
    start_time = time.time()
    scripts, unknowns = db.entities(fn_allow=fn_allow_testinfo_valid,
                                    candidates=candidates)

    f = db.get_prop_value
    # flat list of all snapshots
    snapshots = [
        snap for name in scripts for snap in f(name, 'expected_snapshots')
    ]
    # each snapshot name will hold a dict of md5(snap): count key-value pairs
    stats_by_snapshot_name = dict([(snap, {}) for snap in snapshots])

    stats_by_script_name = dict([(name, {
        'timeouts': 0,
        'errs': 0
    }) for name in scripts if f(name, 'expected_snapshots')])

    # build a hash to limit mismatchs when combining runs. Caveat: if files
    # edited and testinfo not updated mismatch happens.
    # It is recomended to run continuations from a clean checkout for safe
    # combination.
    hasher = hashlib.md5()
    for name in stats_by_script_name:
        hasher.update(db.get_prop_value(name, 'md5_at_testinfo'))
    overall_md5 = hasher.hexdigest()
    if required_md5:
        assert required_md5 == overall_md5

    if not os.path.exists(samples_dir):
        os.makedirs(samples_dir)
    snapshots_abspath = os.path.abspath(samples_dir)

    proxy_abspath = os.path.abspath('proxy_snapshots.py')
    if not os.path.exists(proxy_abspath):
        raise ValueError("proxy script not found:%s" % proxy_abspath)

    rounds = 0
    if isinstance(limit, float):
        limit_seconds = limit * 60
        f_continue = lambda: (time.time() - start_time) < limit_seconds
    elif isinstance(limit, int):
        f_continue = lambda: rounds < limit
    else:
        raise ValueError

    while f_continue():
        for name in scripts:
            # exercise the script acording to testinfo, snapshots would be taken
            fname = db.fname_from_canonical(name)
            stored_testinfo = db.get_prop_value(name, 'testinfo')
            timeout_hit, err = proxy.proxy_run(
                proxy_abspath, fname, [stored_testinfo, snapshots_abspath])
            # count errors
            if timeout_hit:
                stats_by_script_name[name]['timeouts'] += 1
            if err:
                stats_by_script_name[name]['errs'] += 1

            # update stats by snapshots
            sbs = stats_by_snapshot_name
            for snap in stats_by_snapshot_name:
                sname = os.path.join(snapshots_abspath, snap)
                if os.path.exists(sname):
                    try:
                        f = open(sname, 'rb')
                        data = f.read()
                        f.close()
                        md5 = doers.md5_hex(data)
                        sbs[snap][md5] = sbs[snap].setdefault(md5, 0) + 1
                    except Exception:
                        pass
                    try:
                        os.remove(sname)
                    except Exception:
                        pass
        rounds += 1

    elapsed = time.time() - start_time
    return (overall_md5, elapsed, rounds, stats_by_script_name,
            stats_by_snapshot_name)
Beispiel #3
0
def update_snapshots(db, filename_persist, target_scripts, snapshots_dir):
    """
    runs the scripts in target scripts, taking snapshots as indicated by the
    testinfo in the script, and updating the snapshots related info in the db.

    Params:
        db:
            a remembercases.TestbedEntityPropDB object
        filename_persist:
            filename to persist the db after updating
        target_scripts:
            iterable yielding scripts. If None, all scripts known in the
            default testbed are assumed.
        snapshots_dir:
            directory to store the snapshots

    Returns (valid_scripts, rejected) where:

        valid_scripts :
            scripts in target_scripts that are known in the default testbed and
            have valid testinfo

        rejected :
            scripts in target_scripts that are unknown in the default testbed or
            dont have valid testinfo
        
    Db operations are done over the default testbed, which should have been
    set bejore calling here.
    
    For each valid entity the following props are set:
        'snapshots_success':
            bool, True if (no traceback or timeout when running the script,
            also all expected snapshots had been produced), else False
        'snapshots_diagnostic':
            string, '' means no errors, else description of failure while
            trying to take snapshots
        'missing_snapshots':
            list of missing snapshots filenames
    """
    proxy_abspath = os.path.abspath('proxy_snapshots.py')
    if not os.path.exists(proxy_abspath):
        raise ValueError("proxy script not found:%s"%proxy_abspath)

    snapshots_abspath = os.path.abspath(snapshots_dir)
    if not os.path.exists(snapshots_abspath):
        os.makedirs(snapshots_abspath)
        
    valid_scripts, rejected = db.entities(fn_allow=fn_allow_testinfo_valid,
                                          candidates=target_scripts)
    
    for script in valid_scripts:
        # get the exercise plan
        stored_testinfo = db.get_prop_value(script, 'testinfo')

        # delete old snapshots if they exist
        expected_snapshots = db.get_prop_value(script, 'expected_snapshots')
        for name in expected_snapshots:
            p = os.path.join(snapshots_abspath, name)
            if os.path.exists(p):
                os.remove(p)
        
        # exercise the script acording to testinfo, snapshots would be taken 
        fname = db.fname_from_canonical(script)
        timeout_hit, err = proxy.proxy_run(proxy_abspath, fname, [stored_testinfo, snapshots_abspath])

        # calc and store missing_snapshots prop
        missing = [ s for s in expected_snapshots if not os.path.exists(os.path.join(snapshots_abspath, s))]
        db.set_prop_value(script, 'missing_snapshots', missing)

        # calc and store snapshots_success prop
        snapshots_success = (err=='' and
                            not timeout_hit and
                            len(missing)==0)
        db.set_prop_value(script, 'snapshots_success', snapshots_success)

        # calc and store snapshots_diagnostic prop
        snapshots_diagnostic = ''
        if not snapshots_success:
            if timeout_hit:
                err = ('Timeout hit. Remember in this case it is posible not all stderr captured.\n' +
                       err)
            if len(missing):
                missing = [ os.path.basename(s) for s in missing ]
                err += '\nNot all snapshots captured - missing snapshots:\n'
                err += '\n'.join(missing)
            snapshots_diagnostic = err
            print('err:', err)
        db.set_prop_value(script, 'snapshots_diagnostic', snapshots_diagnostic)


    # update history
    text = '\n'.join(valid_scripts)
    db.history_add('update_snapshots', text)

    if filename_persist:
        dbm.db_save(db, filename_persist)
    return valid_scripts, rejected
Beispiel #4
0
def update_snapshots(db, filename_persist, target_scripts, snapshots_dir):
    """
    runs the scripts in target scripts, taking snapshots as indicated by the
    testinfo in the script, and updating the snapshots related info in the db.

    Params:
        db:
            a remembercases.TestbedEntityPropDB object
        filename_persist:
            filename to persist the db after updating
        target_scripts:
            iterable yielding scripts. If None, all scripts known in the
            default testbed are assumed.
        snapshots_dir:
            directory to store the snapshots

    Returns (valid_scripts, rejected) where:

        valid_scripts :
            scripts in target_scripts that are known in the default testbed and
            have valid testinfo

        rejected :
            scripts in target_scripts that are unknown in the default testbed or
            dont have valid testinfo
        
    Db operations are done over the default testbed, which should have been
    set bejore calling here.
    
    For each valid entity the following props are set:
        'snapshots_success':
            bool, True if (no traceback or timeout when running the script,
            also all expected snapshots had been produced), else False
        'snapshots_diagnostic':
            string, '' means no errors, else description of failure while
            trying to take snapshots
        'missing_snapshots':
            list of missing snapshots filenames
    """
    proxy_abspath = os.path.abspath('proxy_snapshots.py')
    if not os.path.exists(proxy_abspath):
        raise ValueError("proxy script not found:%s" % proxy_abspath)

    snapshots_abspath = os.path.abspath(snapshots_dir)
    if not os.path.exists(snapshots_abspath):
        os.makedirs(snapshots_abspath)

    valid_scripts, rejected = db.entities(fn_allow=fn_allow_testinfo_valid,
                                          candidates=target_scripts)

    for script in valid_scripts:
        # get the exercise plan
        stored_testinfo = db.get_prop_value(script, 'testinfo')

        # delete old snapshots if they exist
        expected_snapshots = db.get_prop_value(script, 'expected_snapshots')
        for name in expected_snapshots:
            p = os.path.join(snapshots_abspath, name)
            if os.path.exists(p):
                os.remove(p)

        # exercise the script acording to testinfo, snapshots would be taken
        fname = db.fname_from_canonical(script)
        timeout_hit, err = proxy.proxy_run(
            proxy_abspath, fname, [stored_testinfo, snapshots_abspath])

        # calc and store missing_snapshots prop
        missing = [
            s for s in expected_snapshots
            if not os.path.exists(os.path.join(snapshots_abspath, s))
        ]
        db.set_prop_value(script, 'missing_snapshots', missing)

        # calc and store snapshots_success prop
        snapshots_success = (err == '' and not timeout_hit
                             and len(missing) == 0)
        db.set_prop_value(script, 'snapshots_success', snapshots_success)

        # calc and store snapshots_diagnostic prop
        snapshots_diagnostic = ''
        if not snapshots_success:
            if timeout_hit:
                err = (
                    'Timeout hit. Remember in this case it is posible not all stderr captured.\n'
                    + err)
            if len(missing):
                missing = [os.path.basename(s) for s in missing]
                err += '\nNot all snapshots captured - missing snapshots:\n'
                err += '\n'.join(missing)
            snapshots_diagnostic = err
            print 'err:', err
        db.set_prop_value(script, 'snapshots_diagnostic', snapshots_diagnostic)

    # update history
    text = '\n'.join(valid_scripts)
    db.history_add('update_snapshots', text)

    if filename_persist:
        dbm.db_save(db, filename_persist)
    return valid_scripts, rejected