def find_failure_cases(dataset, r_types, times=500, console_output=False):
failures = set()
individual_logs = log.read_individual_log(dataset)
for i in xrange(times):
traces, config = log.random_mix(individual_logs, num=2, min_n=100)
label = log.label_of_config(config)
detector = DriftDetector()
detector.r_types = r_types
if console_output:
print '----------------------------testing two models-----------------------------------'
print 'dataset: %s, model_1: %s, model_2: %s' % (dataset, config[0]['name'], config[1]['name'])
print 'relations: ', r_types
result = detector.detect(traces)
if console_output:
print 'label: ', label
print 'result: ', result
if len(result) < \
len(label):
if config[0]['name'] < config[1]['name']:
failures.add((config[0]['name'], config[1]['name']))
else:
failures.add((config[1]['name'], config[0]['name']))
return failures
def generate_mixed_log(dataset, model_count=10, min_n=100, times=1):
individual_logs = log.read_individual_log(dataset)
for i in xrange(times):
traces, config = log.random_mix(individual_logs, num=model_count, min_n=min_n)
filename = log.name_of_config(config) + '.mxml'
filepath = os.path.join(log.mixed_logs_dir, dataset, filename)
mxml.write(traces, filepath)
def test_two_models(dataset, model_1, model_2, r_types, console_output=True):
if console_output:
print '----------------------------testing two models-----------------------------------'
print 'dataset: %s, model_1: %s, model_2: %s' % (dataset, model_1, model_2)
print 'relations: ', r_types
individual_logs = log.read_individual_log(dataset)
model_1_trace_length = 500
model_2_trace_length = 500
config = [
{
'name': model_1,
'length': model_1_trace_length
},
{
'name': model_2,
'length': model_2_trace_length
}
]
traces = log.mix(individual_logs, config)
detector = DriftDetector()
detector.r_types = r_types
detector.console_output = True
result = detector.detect(traces)
if console_output:
print '---------------'
print 'label: ', 0, model_1_trace_length, model_1_trace_length + model_2_trace_length
print 'result: ', result
if len(result) == 3:
print 'successfully'
print '---------------'
return len(result) == 3
def generate_logs(dataset):
model_counts = [2, 4, 6, 8, 10, 12, 14, 16]
trace_ranges = [(200, 300), (450, 550), (700, 800), (950, 1050)]
individual_logs = log.read_individual_log(dataset)
for model_count in model_counts:
for min_n, max_n in trace_ranges:
traces, config = log.random_mix(individual_logs, num=model_count, min_n=min_n, max_n=max_n)
filename = '[%d %d]-' % (model_count, (min_n + max_n) / 2) + str(log.label_of_config(config)) + '.mxml'
print filename
filepath = os.path.join(log_dir, filename)
mxml.write(traces, filepath)