Ejemplo n.º 1
0
def evaluate_file(args, inconsistencies):
	print("Prediction file:  ", args.prediction)
	print("Gold file:        ", args.gold)
	pred_file = load_predictions(args)
	gold_file = pyconll.load_from_file(args.gold)

	if len(pred_file) != len(gold_file):
		print("Number of sentences does not match!")
		print("Prediction: {}    Gold: {}".format(len(pred_file), len(gold_file)))
		return
	
	upos_evaluator = evaluator.Evaluator(mode="exact")
	xpos_evaluator = evaluator.Evaluator(mode="exact")
	feats_evaluator = evaluator.Evaluator(mode="by_feats")
	ufeats_evaluator = evaluator.Evaluator(mode="exact", only_univ=True)
	upos_feats_evaluator = evaluator.Evaluator(mode="by_feats")
	incons_count = 0
	token_count = 0
	
	for pred_sent, gold_sent in zip(pred_file, gold_file):
		if len(pred_sent) != len(gold_sent):
			print("Number of words in sentence does not match!")
			print("Prediction: {}    Gold: {}".format(len(pred_sent), len(gold_sent)))
			print("Prediction:", pred_sent._meta)
			print("Gold:", gold_sent._meta)
			continue
		
		for pred_token, gold_token in zip(pred_sent, gold_sent):
			if args.upos:
				upos_evaluator.add_instance({POS: gold_token.upos}, {POS: pred_token.upos})
			if args.xpos:
				xpos_evaluator.add_instance({POS: gold_token.xpos}, {POS: pred_token.xpos})
			if args.feats:
				gold_feats = {x: ",".join(gold_token.feats[x]) for x in gold_token.feats}
				pred_feats = {x: ",".join(pred_token.feats[x]) for x in pred_token.feats}
				feats_evaluator.add_instance(gold_feats, pred_feats)
				ufeats_evaluator.add_instance(gold_feats, pred_feats)
				if args.upos:
					if args.incons:
						token_count += 1
						if len(set(pred_feats.keys()) & inconsistencies[pred_token.upos]) > 0:
							incons_count += 1
					gold_feats.update({POS: gold_token.upos})
					pred_feats.update({POS: pred_token.upos})
					upos_feats_evaluator.add_instance(gold_feats, pred_feats)
	
	if upos_evaluator.instance_count > 0:
		print("UPOS accuracy          {:.2f}%".format(100*upos_evaluator.acc()))
	if xpos_evaluator.instance_count > 0:
		print("XPOS accuracy          {:.2f}%".format(100*xpos_evaluator.acc()))
	if feats_evaluator.instance_count > 0:
		print("FEATS micro-F1         {:.2f}%".format(100*feats_evaluator.micro_f1()))
	if upos_feats_evaluator.instance_count > 0:
		print("UPOS+FEATS micro-F1    {:.2f}%".format(100*upos_feats_evaluator.micro_f1()))
	if ufeats_evaluator.instance_count > 0:
		print("UFEATS accuracy        {:.2f}%".format(100*ufeats_evaluator.acc()))
	if token_count > 0:
		print("UFEATS inconsistencies {:.2f}%".format(100*incons_count / token_count))
	print()
Ejemplo n.º 2
0
def test_evaluator_external_server(logger):
    """
    Tests evaluator skip_empty flag.
    """
    tester = evaluator.Evaluator(
        logger,  # logger for the session
        None,  # no internal censor
        "http://facebook.com",  # try to talk to facebook
        actions.utils.RUN_DIRECTORY,  # directory to log
        workers=1,  # workers to use
        runs=1,  # only need 1 run
        test_type="http",  # use http test
        skip_empty=False)  # don't skip empty strats

    population = [
        "\/",
    ]
    population = [actions.utils.parse(ind, logger) for ind in population]

    inds = tester.evaluate(population)
    assert inds[0].fitness == 400

    tester = evaluator.Evaluator(
        logger,  # logger for the session
        None,  # no internal censor
        "http://facebook.com",  # try to talk to facebook
        actions.utils.RUN_DIRECTORY,  # directory to log
        workers=1,  # workers to use
        runs=1,  # only need 1 run
        test_type="http",  # use http test
        skip_empty=True)  # don't skip empty strats

    population = [
        "\/",
    ]
    population = [actions.utils.parse(ind, logger) for ind in population]

    inds = tester.evaluate(population)
    assert inds[0].fitness == -1000

    tester = evaluator.Evaluator(
        logger,  # logger for the session
        None,  # no internal censor
        None,  # no external server
        actions.utils.RUN_DIRECTORY,  # directory to log
        use_external_sites=True,
        workers=1,  # workers to use
        runs=1,  # only need 1 run
        test_type="http",  # use http test
        skip_empty=False)  # don't skip empty strats

    population = [
        "\/",
    ]
    population = [actions.utils.parse(ind, logger) for ind in population]

    inds = tester.evaluate(population)
    assert inds[0].fitness == 400
Ejemplo n.º 3
0
def main():
    wordbank2 = {
        'Det': ['the', 'this', 'that', 'each', 'every', 'another'],
        'Ord': ['first', 'second', 'last', 'middle', 'third', 'final', 'penultimate'],
        'Adj': ['green', 'blue', 'illiterate', 'cheerful', 'elated', 'ethereal', 'transliterated', 'randy', 'greasy'],
        'SingNoun': ['duck', 'ant', 'egg', 'window', 'cereal', 'AI project', 'ulna', 'trenchcoat', 'denture'],
        'SingPropNoun': ['Jack', 'Jill', 'Fred', 'George', 'Doris', 'Illuminati', 'Big Brother', 'Arcady Ivanovich'],
        'AdvPlace': ['here', 'there', 'everywhere'],
        'AdvTimePres': ['today', 'Friday', 'this week', 'sometime'],
        'AdvTimePast': ['yesterday', 'last Wednesday', 'last month']
    }

    wordbank3 = {
        'Det': ['the', 'this', 'that', 'every', 'another', 'a single'],
        'Ord': ['fourth', 'eleventh', 'last', 'middle', 'third', 'final', 'penultimate'],
        'Adj': ['red', 'blue', 'winning', 'cheerful', 'elated', 'big', 'little', 'scared', 'rare', 'mucosal',
                'phallic'],
        'SingNoun': ['spider', 'bucket', 'steak', 'milk', 'computer', 'snowman', 'milksteak', 'funnel', 'babycarrot'],
        'SingPropNoun': ['Brendan', 'Ed', 'Jimmy', 'Hannah', 'Ryan', 'Helena', 'Svetlana', 'Pamela', 'Vladimir'],
        'AdvDeg': ['very', 'mostly', 'rarely', 'never']
    }

    skeleton_gram_str2 = ''.join(open('species2.pcfg'))

    skeleton_gram_str3 = ''.join(open('species3.pcfg'))

    for w in wordbank2.keys():
        skeleton_gram_str2 = addterminals(skeleton_gram_str2, w, wordbank2[w])

    for w in wordbank3.keys():
        skeleton_gram_str3 = addterminals(skeleton_gram_str3, w, wordbank3[w])

    user1 = environment.User('/u/_junebug_', None, True, None, True)
    prod1 = Producer(user1, skeleton_gram_str2, wordbank2)
    user1.producer = prod1
    user2 = environment.User('/u/_gstring_', None, True, None, True)
    prod2 = Producer(user2, skeleton_gram_str3, wordbank3)
    user2.producer = prod2
    user1.evaluator = evaluator.Evaluator([A(), L(), T(), W(['spider', 'Ryan', 'Doris'], ['little', 'elated']), B(user1)], user1)
    user2.evaluator = evaluator.Evaluator([A(), L(), T(), W(['Ed', 'steak', 'never'], ['funnel', 'milk', 'green']), B(user2)], user2)

    # run 5 'iterations'
    posts = []
    for i in range(50):
        if i % 10 == 0:
            prod1.parent_grammar.mutate()
            prod2.parent_grammar.mutate()
            user1.evaluate_iteration(posts)
            user2.evaluate_iteration(posts)
            posts = []
        posts.append(prod1.parent_grammar.make_post('1', i//10))
        posts.append(prod2.parent_grammar.make_post('2', i//10))
Ejemplo n.º 4
0
def test_evaluator_external_dns_client(client_worker, logger):
    """
    Tests evaluator server side with external client with --use-external-sites.
    """
    population = [
        "\/",
    ]

    population = [actions.utils.parse(ind, logger) for ind in population]
    cmd = [
        "--test-type",
        "dns",
        "--external-server",
        "--external-client",
        client_worker["worker"],
        "--log-on-fail",  # this test should not fail, so log if it does
        "--log",
        actions.utils.CONSOLE_LOG_LEVEL,
        "--no-skip-empty",
        "--output-directory",
        actions.utils.RUN_DIRECTORY
    ]
    tester = evaluator.Evaluator(cmd, logger)

    inds = tester.evaluate(population)
    assert len(inds) == 1
    assert str(inds[0]).strip() == "\/"
Ejemplo n.º 5
0
def test_evaluator_http_client_external_sites(logger):
    """
    Tests http plugin client.
    """
    with tempfile.TemporaryDirectory() as output_dir:
        cmd = [
            "--test-type", "http", "--external-server", "--use-external-sites",
            "--no-canary", "--log", actions.utils.CONSOLE_LOG_LEVEL,
            "--no-skip-empty", "--output-directory", output_dir
        ]
        tester = evaluator.Evaluator(cmd, logger)
        population = [
            "\/ [UDP:dport:100]-drop-|",  # strategy with an unused action tree
            "\/",
            "[TCP:flags:PA]-drop-|"  # strategy that will break TCP connection
        ]
        population = [actions.utils.parse(ind, logger) for ind in population]
        inds = tester.evaluate(population)
        assert len(inds) == 3
        assert inds[0].fitness == 389  # -10 for unused, -1 for size
        assert inds[1].fitness == 400
        assert inds[2].fitness == -480
        for ind in inds:
            assert os.path.exists(
                os.path.join(output_dir, "logs",
                             ind.environment_id + ".client.log"))
            assert os.path.exists(
                os.path.join(output_dir, "logs",
                             ind.environment_id + ".engine.log"))
            assert os.path.exists(
                os.path.join(output_dir, "flags",
                             ind.environment_id + ".fitness"))
Ejemplo n.º 6
0
def test_evaluator_censor_log_on_debug(logger):
    """
    Tests http plugin client.
    """
    print(
        "Test testing a failing strategy and a successful strategies, dumping logs on success and failure."
    )
    cmd = [
        "--test-type", "http", "--port", "80", "--censor", "censor2", "--log",
        actions.utils.CONSOLE_LOG_LEVEL, "--no-skip-empty", "--bad-word",
        "facebook", "--log-on-fail", "--log-on-success", "--output-directory",
        actions.utils.RUN_DIRECTORY
    ]
    try:
        tester = evaluator.Evaluator(cmd, logger)
        population = [
            "\/",
            "\/ [TCP:flags:R]-drop-|",
        ]
        population = [actions.utils.parse(ind, logger) for ind in population]
        inds = tester.evaluate(population)
        assert len(inds) == 2

    finally:
        print("Test shutting down any lingering containers.")
        common.clean_containers()
Ejemplo n.º 7
0
def main(start, end, interval, kfold):
    tpr_lists = []
    pth_lists = []
    tpr_lists_not_ign = []
    pth_lists_not_ign = []
    ckpts = range(start, end + 1, interval)
    
    best_score = 0
    best_epoch = "100"

    for i in ckpts:
        e = evaluator.Evaluator('val/%s/%s/bbox/test_%s' % (kfold, i, i), 'val', 'split/val_%s.npy' % (kfold), ckpt=i,pbb_cutoff=0.5)
#     fp_per_scan, tprs, pths, score = e.froc(ignore=ignore_val)
#     tpr_lists.append(tprs)
#     pth_lists.append(pths)
    
        _, tprs_not_ign, pths_not_ign, score = e.froc()
        tpr_lists_not_ign.append(tprs_not_ign)
        pth_lists_not_ign.append(pths_not_ign)
        
        if score > best_score:
	    best_score, best_epoch = score, i
        print i, 'all:', score

    print "Best score: ", best_score, " Best epoch: ", best_epoch
    return best_epoch
Ejemplo n.º 8
0
 def test_list(self):
     parser = futhon_parser.FuthonParser()
     env = environment.GlobalEnvironment({})
     ev = evaluator.Evaluator()
     self.assertEqual(
         "", ""
     )
Ejemplo n.º 9
0
def test_evaluator_external_client(client_worker, logger):
    """
    Tests evaluator server side with external client.
    """
    population = [
        "\/",
    ]
    print(client_worker["worker"])
    population = [actions.utils.parse(ind, logger) for ind in population]
    cmd = [
        "--test-type",
        "http",
        "--port",
        "80",
        "--external-server",
        "--external-client",
        client_worker["worker"],
        "--server",
        "http://google.com",
        "--log-on-fail",  # this test should not fail, so log when it does
        "--no-canary",
        "--log",
        actions.utils.CONSOLE_LOG_LEVEL,
        "--no-skip-empty",
        "--output-directory",
        actions.utils.RUN_DIRECTORY
    ]
    tester = evaluator.Evaluator(cmd, logger)

    inds = tester.evaluate(population)
    assert len(inds) == 1
    assert str(inds[0]).strip() == "\/"
    assert inds[0].fitness == 400
Ejemplo n.º 10
0
def test_get_from_fuzzed_or_real(logger):
    """
    Tests utils.get_from_fuzzed_or_real_packet(environment_id, real_packet_probability):
    """
    # Create an evaluator
    cmd = [
        "--test-type", "echo", "--censor", "censor2", "--log",
        actions.utils.CONSOLE_LOG_LEVEL, "--no-skip-empty", "--bad-word",
        "facebook", "--output-directory", actions.utils.RUN_DIRECTORY
    ]

    tester = evaluator.Evaluator(cmd, logger)

    canary = evolve.generate_strategy(logger, 0, 0, 0, 0, None)
    environment_id = tester.canary_phase(canary)
    for i in range(0, 100):
        proto, field, value = actions.utils.get_from_fuzzed_or_real_packet(
            environment_id, 1)
        assert proto
        assert field
        assert value is not None
        proto, field, value = actions.utils.get_from_fuzzed_or_real_packet(
            environment_id, 0)
        assert proto
        assert field
        assert value is not None
Ejemplo n.º 11
0
def test_evaluator_skip_empty(logger):
    """
    Tests evaluator skip_empty flag.
    """
    population = [
        "\/",
    ]
    population = [actions.utils.parse(ind, logger) for ind in population]
    # Create an evaluator
    tester = evaluator.Evaluator(
        logger,  # logger for the session
        "censor2",  # internal censor
        None,  # no external server
        actions.utils.RUN_DIRECTORY,  # directory to log
        workers=4,  # workers to use
        runs=1,  # only need 1 run
        test_type="echo",  # use echo test
        skip_empty=True)  # skip empty strats
    inds = tester.evaluate(population)
    assert len(inds) == 1
    assert str(inds[0]).strip() == "\/"
    assert inds[0].fitness == -1000

    tester.skip_empty = False
    inds = tester.evaluate(population)
    assert len(inds) == 1
    assert str(inds[0]).strip() == "\/"
    assert inds[0].fitness == -40
Ejemplo n.º 12
0
def calculate_file(filename):
    lines = []
    with open(filename, 'r') as f:
        for line in f:
            lines.append(parser.tokenizer(line))

    return evaluator.Evaluator(lines).run()
Ejemplo n.º 13
0
 def test_reset(self):
     """Test we can reset okay"""
     evaluator_ = evaluator.Evaluator()
     evaluator_.set_variable("x", 21.)
     self.assertAlmostEqual(evaluator_.evaluate("x"), 21.)
     evaluator_.reset()
     self.assertRaises(NameError, evaluator_.evaluate, "x")
Ejemplo n.º 14
0
    def __init__(self, configfile=None):
        '''
            Initialization method of Choronzon. Reads the configuration,
            instantiates objects of the vital classes, builds and
            analyzes the first generation of chromosomes by reading
            the initial population provided to the fuzzer.
        '''
        # configuration is a singleton
        self.configuration = configuration.Configuration(configfile)
        self.campaign = campaign.Campaign(self.configuration['CampaignName'])

        seedpath = self.campaign.copy_directory(
            self.configuration['InitialPopulation'], name='seedfiles')

        self.tracer = tracer.Tracer()
        self.strategy = strategy.FuzzingStrategy()
        self.population = world.Population(self.tracer.cache)
        self.evaluator = evaluator.Evaluator(self.tracer.cache)

        try:
            self.sharedpath = self.campaign.create_shared_directory(
                self.configuration['ChromosomeShared'])
        except:
            self.sharedpath = None

        # Initialize factory for building chromosome
        # and the proxy for computing the fitness.
        chromosomes = chromosome.Factory.build(seedpath)
        for chromo in chromosomes:
            self.population.add_chromosome(chromo)

        self.analyze()
Ejemplo n.º 15
0
def test_evaluator_client_dns_test(client_worker, protocol, logger):
    """
    Tests DNS evaluation with external client.
    """
    # Setup the population and test type
    test_type = "dns_tcp"
    if protocol == "udp":
        test_type = "dns"

    population = ["\/"]

    population = [actions.utils.parse(ind, logger) for ind in population]
    tester = evaluator.Evaluator(
        logger,  # logger for the session
        None,  # no internal censor
        None,  # no external server
        actions.utils.RUN_DIRECTORY,  # directory to log
        workers=1,  # workers to use
        runs=1,  # only need 1 run for testing
        external_client=False,  # testing an external client
        test_type=test_type,
        skip_empty=False)  # don't skip empty strats

    inds = tester.evaluate(population)
    assert len(inds) == 1
    assert str(inds[0]).strip() == "\/"
    assert inds[0].fitness > 0
Ejemplo n.º 16
0
    def __init__(self):
        # Init from settings
        self.max_pop_size = int(settings.Get_Setting("max_pop_size"))
        self.mutation_rate = float(settings.Get_Setting("mutation_rate"))
        self.num_cross_points = int(
            settings.Get_Setting("number_crossover_points"))
        self.intra_swap_rate = int(settings.Get_Setting("intra_swap_rate"))

        # Init the rest...
        self.is_init = false
        self.average_fitness = 0
        self.current_pop = {}
        self.culled_pop = {}
        self.eval_cache = {}
        self.breeders = []
        self.offspring = []
        self.evaluators = []

        # Init the offspring and current population
        self.Genesis()

        # Init the evaluators
        num_procs = int(settings.Get_Setting("number_cpus"))
        eval_type = "smp"
        for i in range(0, num_procs):
            self.evaluators.append(eval.Evaluator(eval_type))
Ejemplo n.º 17
0
def main():
    file_name = __get_file_name(sys.argv)
    file_ext = __get_file_extension(file_name)

    if file_ext != ".fyp":
        eh.print_and_exit("error: invalid file extension: must be .fyp")

    if not __file_exists(file_name):
        eh.print_and_exit("error: file does not exist!")

    _file = open(file_name)
    _lex = lexer.Lexer(' '.join(
            [line.rstrip("\n") for line in _file if not __is_comment(line)]))
    _tokens = _lex.tokenize().tokens

    is_test = False

    if len(_tokens) > 1 and _tokens[2] == "test":
        is_test = True
        _tokens = _tokens[0:2] + _tokens[3:]

    _parser = parser_.Parser(_tokens)
    _eval = evaluator.Evaluator(_parser, is_test)

    if _eval.tokens[0] != "begin":
        eh.print_and_exit("error: first function must be 'begin'!")

    _eval.eval()
Ejemplo n.º 18
0
    def __init__(self, model):
        self.model = model
        self.reader = data_input.DataInput()
        self.split = split.Split(self.reader)
        self.evaluator = evaluator.Evaluator()
        self.writer = output.Output()

        self.model.setup(self.reader)
Ejemplo n.º 19
0
 def test_set_variable(self):
     """Test we can set evaluator variables"""
     evaluator_ = evaluator.Evaluator()
     # check we can assign numbers
     evaluator_.set_variable("x", 21.)
     self.assertEqual(evaluator_.variables["x"], 21.)
     # check we throw an error if value can't be overloaded to float 
     self.assertRaises(ValueError, evaluator_.set_variable, "x", "value")
Ejemplo n.º 20
0
def test_evaluator_worker_ip_lookup(logger):
    """
    Tests worker IP lookup by specifying a worker name instead of a public IP
    """
    cmd = [
        "--test-type", "http", "--public-ip", "example", "--external-client",
        "example", "--output-directory", actions.utils.RUN_DIRECTORY
    ]
    test_evaluator = evaluator.Evaluator(cmd, logger)
    assert test_evaluator.public_ip == "0.0.0.0"

    cmd = [
        "--test-type", "http", "--output-directory",
        actions.utils.RUN_DIRECTORY
    ]
    test_evaluator = evaluator.Evaluator(cmd, logger)
    assert not test_evaluator.get_ip()
Ejemplo n.º 21
0
 def __init__(self, **kargs):
     super().__init__(**kargs)
     self.data_queue = []
     self.model = None
     print("Load Evaluator")
     import evaluator
     print("Instantiate Evaluator")
     self.model = evaluator.Evaluator()
     print("Evaluator Loaded")
Ejemplo n.º 22
0
 def __init__(self, network, training_data, validation_data):
     self.network = network
     self.aux = None
     self.training_data = training_data
     self.validation_data = validation_data
     self.evaluator = eva.Evaluator(self.training_data,
                                    self.validation_data)
     self.mu = 1
     super(Mac, self).__init__()
Ejemplo n.º 23
0
def test_evaluator_dns_client_external_server(logger, extra_args):
    """
    Tests http plugin client.
    """
    with tempfile.TemporaryDirectory() as output_dir:
        cmd = [
            "--test-type", "dns", "--external-server", "--log",
            actions.utils.CONSOLE_LOG_LEVEL, "--no-skip-empty",
            "--output-directory", output_dir
        ]
        cmd += extra_args
        tester = evaluator.Evaluator(cmd, logger)
        if "--use-tcp" not in cmd:
            population = [
                "\/ [UDP:dport:100]-drop-|",  # strategy with an unused action tree
                "\/",
                "[UDP:dport:53]-sleep{1}-|",
                "[UDP:dport:53]-drop-|",  # strategy that will break query
                "[UDP:dport:53]-tamper{DNS:qd:compress}-|"
            ]
        else:
            population = [
                "\/ [UDP:dport:100]-drop-|",  # strategy with an unused action tree
                "\/",
                "[TCP:flags:PA]-sleep{1}-|",
                "[TCP:flags:PA]-drop-|",  # strategy that will break query
                # "[TCP:flags:PA]-tamper{DNS:qd:compress}-|" # Not implemented due to TCP protocol limitations
            ]

        population = [actions.utils.parse(ind, logger) for ind in population]
        inds = tester.evaluate(population)

        # Special case for UDP
        assert len(inds) == 5 if "--use-tcp" not in cmd else 4

        assert inds[0].fitness == 389  # -10 for unused, -1 for size
        assert inds[1].fitness == 400
        assert inds[2].fitness == 399  # -1 for size
        assert inds[3].fitness == -400

        if "--use-tcp" not in cmd:
            assert inds[4].fitness > 0

        for ind in inds:
            assert os.path.exists(
                os.path.join(output_dir, "logs",
                             ind.environment_id + ".client.log"))
            assert os.path.exists(
                os.path.join(output_dir, "logs",
                             ind.environment_id + ".engine.log"))
            if ind.fitness > 0:
                assert os.path.exists(
                    os.path.join(output_dir, "flags",
                                 ind.environment_id + ".dnsresult"))
            assert os.path.exists(
                os.path.join(output_dir, "flags",
                             ind.environment_id + ".fitness"))
Ejemplo n.º 24
0
def test_evaluator_external_client_server_side(client_worker, logger, args):
    """
    Tests evaluator server side with external client.
    """
    if "http" in args or "--use-tcp" in args:
        population = [
            "\/ [UDP:dport:100]-drop-|",  # strategy with an unused action tree
            "\/",
            "[TCP:flags:SA]-drop-|"  # strategy that will break TCP connection
        ]
    else:
        population = [
            "\/ [UDP:dport:100]-drop-|",  # strategy with an unused action tree
            "\/",
            "\/ [UDP:dport:53]-drop-|"  # strategy that will break query
        ]

    population = [actions.utils.parse(ind, logger) for ind in population]
    cmd = [
        "--test-type", "http", "--external-client", client_worker["worker"],
        "--server-side", "--public-ip",
        get_ip(), "--timeout", "15", "--no-canary", "--log",
        actions.utils.CONSOLE_LOG_LEVEL, "--no-skip-empty",
        "--output-directory", actions.utils.RUN_DIRECTORY
    ]
    cmd += args
    tester = evaluator.Evaluator(cmd, logger)

    inds = tester.evaluate(population)
    assert len(inds) == 3
    assert inds[0].fitness == 389
    assert inds[1].fitness == 400
    assert inds[2].fitness < 0

    # Request a server side without specifying the public ip - should raise an exception
    cmd = [
        "--test-type", "http", "--port", "80", "--external-client",
        client_worker["worker"], "--server-side", "--no-canary", "--log",
        actions.utils.CONSOLE_LOG_LEVEL, "--no-skip-empty",
        "--output-directory", actions.utils.RUN_DIRECTORY
    ]
    with pytest.raises(AssertionError):
        tester = evaluator.Evaluator(cmd, logger)
Ejemplo n.º 25
0
def test_evaluator_init_nat(logger):
    """
    Sets up evaluator with NAT
    """
    cmd = [
        "--test-type", "http", "--sender-ip", "1.1.1.1", "--forward-ip",
        "2.2.2.2", "--routing-ip", "3.3.3.3", "--output-directory",
        actions.utils.RUN_DIRECTORY
    ]
    test_evaluator = evaluator.Evaluator(cmd, logger)
    assert not test_evaluator.forwarder, "Evaluator set up a forwarder without --act-as-middlebox"
    cmd += ["--act-as-middlebox"]

    test_evaluator = evaluator.Evaluator(cmd, logger)

    assert test_evaluator.forwarder
    assert test_evaluator.forwarder["sender_ip"] == "1.1.1.1"
    assert test_evaluator.forwarder["forward_ip"] == "2.2.2.2"
    assert test_evaluator.forwarder["routing_ip"] == "3.3.3.3"
Ejemplo n.º 26
0
def run_test(logger, solution, censor, test_type, log_on_success=False, log_on_fail=False):
    """
    Tests a given solution against a given censor under a given test type
    using the given log level.
    """
    # Test if docker is running
    try:
        subprocess.check_output(['docker', 'ps'])
    except subprocess.CalledProcessError:
        pytest.fail("Docker is not running")

    try:
        # Parse the string representation of the solution
        strat = actions.utils.parse(solution, logger)
        logger.info("Parsed strategy %s" % (str(strat)))

        # Confirm the parsing was correct
        assert str(strat).strip() == solution, "Failed to correctly parse given strategy"

        logger.info("Testing %s" % censor)

        # Setup the external server to test with, if an http test is done
        if test_type == "echo":
            external_server = None
        elif test_type == "http":
            external_server = "facebook.com"

        # Create an evaluator
        cmd = [
            "--test-type", "echo",
            "--censor", censor,
            "--log", actions.utils.CONSOLE_LOG_LEVEL,
            "--no-skip-empty",
            "--bad-word", "facebook",
            "--output-directory", actions.utils.RUN_DIRECTORY
        ]
        if log_on_success:
            cmd += ["--log-on-success"]
        if log_on_fail:
            cmd += ["--log-on-fail"]
        tester = evaluator.Evaluator(cmd, logger)

        # Use the fitness function to evaluate the strategy
        population = evolve.fitness_function(logger, [strat], tester)

        # Check that we got back the same number of individuals we gave
        assert len(population) == 1, "Population size changed"

        # Shutdown the evaluator
        tester.shutdown()

        # Retrieve the fitness from the individual
        return population[0].fitness
    finally:
        clean_containers()
Ejemplo n.º 27
0
def serve():
    server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
    evaluator_pb2_grpc.add_EvaluatorServicer_to_server(evaluator.Evaluator(),
                                                       server)
    server.add_insecure_port('[::]:50051')
    server.start()
    try:
        while True:
            time.sleep(_ONE_DAY_IN_SECONDS)
    except KeyboardInterrupt:
        server.stop(0)
Ejemplo n.º 28
0
    def evaluate(self):
        """Sets up and shows the evaluation of schedule"""

        evalworker = evaluator.Evaluator(self.mainLayout.tab.currentWidget())
        empty = evalworker.countEmptyPercentforColumns()
        constr = []
        for i in range(len(empty)):
            constr.append(10 - int(empty[i] / 10))

        evald = EvaluationDialog(self.mainLayout.tab.currentWidget().getWeeknum(), constr)
        evald.exec_()
Ejemplo n.º 29
0
	def __init__(self):
		if Computor.instance is None:
			Computor.instance = self
			self.__parser = Parser()
			self.__simplifier = Simplifier()
			self.__evaluator = evaluator.Evaluator()
			self.__solver = Solver()
			self.__vars = {}
			self.__funcs = {}
			self.__local_vars = []		# list of tuple (name, value), to be used as stack
		else:
			raise ComputorException('Computor.instance already instantiated')
Ejemplo n.º 30
0
    def test_evaluate(self):
        """Test we can evaluate expressions okay"""
        evaluator_ = evaluator.Evaluator()
        evaluator_.set_variable("x", 21.)
        value = evaluator_.evaluate("cos(pi*x)*m")
        self.assertAlmostEqual(value, -1.*xboa.Common.units["m"])
        value = evaluator_.evaluate("sin(pi*x)")
        self.assertAlmostEqual(value, 0.)
        self.assertRaises(NameError, evaluator_.evaluate, "sin(pi*y)")

        value = evaluator_.evaluate("sin(pi*x)")
        self.assertAlmostEqual(value, 0.)