def test_data_instream():
for file_size in test_filesize:
newfile = False
for size in test_size:
result_file = 'report/test_data_instream_{}_{}.txt'.format(size, file_size)
if os.path.exists(result_file):
print('skip {}'.format(result_file))
continue
test_file = 'testfile.csv'
dfmaker = DataFrameMaker(size, file_size, 100)
multip = len(dfmaker)
if multip == 0:
continue
print('testing {}'.format(result_file))
if not newfile:
df = next(dfmaker)
df.to_csv('data/'+test_file)
newfile = True
timer = Timer()
kwargs = {
'read_number':'0',
'header':'True',
'path':test_file,
}
while timer.total_cost < 2:
timer(data_instream, **kwargs)
cost = timer.avg_cost * multip
with open(result_file, 'w') as f:
f.write('total,each,number\n')
f.write(str(cost)+","+str(timer.avg_cost)+","+str(multip))
def test_kmeans():
for size in size_list:
for file_size in filesize_list:
kwargs = {
'n_cluster':'2',
'max_iter':'300',
'predict_labels':'label',
'store_origin':'False',
'n_jobs':'10',
}
result_file = 'report/test_kmeans_{}_{}.txt'.format(size, file_size)
if os.path.exists(result_file):
print('skip {}'.format(result_file))
continue
dfmaker = LabelDataMaker(size, file_size, 100, 2)
multip = len(dfmaker)
if multip == 0:
continue
timer = Timer()
print('running {}'.format(result_file))
timer(kmeans, next(dfmaker), **kwargs)
cost = timer.avg_cost * multip
with open(result_file, 'w') as f:
f.write('total,each,number\n')
f.write(str(cost)+","+str(timer.avg_cost)+","+str(multip))
def test_accuracy():
for size in size_list:
for file_size in filesize_list:
kwargs = {
'true_posi':'right',
'average':'binary',
'truth_column':'label',
'pred_column':'label',
}
result_file = 'report/test_accuracy_{}_{}.txt'.format(size, file_size)
if os.path.exists(result_file):
print('skip {}'.format(result_file))
continue
dfmaker = LabelDataMaker(size, file_size, 100, 2)
multip = len(dfmaker)
if multip == 0:
continue
timer = Timer()
print('running {}'.format(result_file))
new_data = next(dfmaker)
timer(accuracy, new_data, new_data, **kwargs)
cost = timer.avg_cost * multip
with open(result_file, 'w') as f:
f.write('total,each,number\n')
f.write(str(cost)+","+str(timer.avg_cost)+","+str(multip))
def test_iforest():
for size in size_list:
for file_size in filesize_list:
kwargs = {
'contamination': '0.1',
'n_jobs': '10',
}
result_file = 'report/test_iforest_{}_{}.txt'.format(
size, file_size)
if os.path.exists(result_file):
print('skip {}'.format(result_file))
continue
dfmaker = LabelDataMaker(size, file_size, 100, 2)
multip = len(dfmaker)
if multip == 0:
continue
timer = Timer()
print('running {}'.format(result_file))
new_data = next(dfmaker)
timer(outlier_iforest, new_data, **kwargs)
cost = timer.avg_cost * multip
with open(result_file, 'w') as f:
f.write('total,each,number\n')
f.write(
str(cost) + "," + str(timer.avg_cost) + "," + str(multip))
def test_cut():
for size in size_list:
kwargs = {
'left': "0",
'width': '960',
'top': '0',
'height': '540',
}
result_file = 'report/test_image_cut_{}.txt'.format(size)
if os.path.exists(result_file):
print('skip {}'.format(result_file))
continue
import cv2
data_in = [[cv2.imread('data/1.png')], ['1.png']]
filesize = 2690941
totalsize = 1024 * 1024 * 1024 * 1024 #gb*mb*kb*b
multip = totalsize // filesize
timer = Timer()
print('running {}'.format(result_file))
while timer.count < 20 or timer.total_cost < 2:
timer(image_cut, data_in, **kwargs)
cost = timer.avg_cost * multip
with open(result_file, 'w') as f:
f.write('total,each,number\n')
f.write(str(cost) + "," + str(timer.avg_cost) + "," + str(multip))
def test_random():
for size in size_list:
for file_size in filesize_list:
kwargs = {}
result_file = 'report/test_random_{}_{}.txt'.format(
size, file_size)
if os.path.exists(result_file):
print('skip {}'.format(result_file))
continue
dfmaker = LabelDataMaker(size, file_size, 100, 2)
multip = len(dfmaker)
if multip == 0:
continue
timer = Timer()
print('running {}'.format(result_file))
timer(random, next(dfmaker), **kwargs)
cost = timer.avg_cost * multip
with open(result_file, 'w') as f:
f.write('total,each,number\n')
f.write(
str(cost) + "," + str(timer.avg_cost) + "," + str(multip))
def test_dilate():
for size in size_list:
kwargs = {
'kernel_size': '3,3',
'iterations': 1,
}
result_file = 'report/test_image_dilate_{}.txt'.format(size)
if os.path.exists(result_file):
print('skip {}'.format(result_file))
continue
import cv2
data_in = [[cv2.imread('data/1.png')], ['1.png']]
filesize = 2690941
totalsize = 1024 * 1024 * 1024 * 1024 #gb*mb*kb*b
multip = totalsize // filesize
timer = Timer()
print('running {}'.format(result_file))
while timer.count < 20 or timer.total_cost < 2:
timer(image_dilate, data_in, **kwargs)
cost = timer.avg_cost * multip
with open(result_file, 'w') as f:
f.write('total,each,number\n')
f.write(str(cost) + "," + str(timer.avg_cost) + "," + str(multip))
def test_monorec():
for size in size_list:
kwargs = {
'class_': 'people,car',
}
result_file = 'report/test_image_monorec_{}.txt'.format(size)
if os.path.exists(result_file):
print('skip {}'.format(result_file))
continue
import cv2
data_in = [[cv2.imread('data/1.png')], ['1.png']]
filesize = 2690941
totalsize = 1024 * 1024 * 1024 * 1024 #gb*mb*kb*b
multip = totalsize // filesize
timer = Timer()
print('running {}'.format(result_file))
image_monorec(data_in, **kwargs)
while timer.count < 20:
timer(image_monorec, data_in, **kwargs)
cost = timer.avg_cost * multip
with open(result_file, 'w') as f:
f.write('total,each,number\n')
f.write(str(cost) + "," + str(timer.avg_cost) + "," + str(multip))
def test_sort():
for size in size_list:
for file_size in filesize_list:
kwargs = {
'columns': '1',
'ascending': 'True',
'na_position': 'last',
}
result_file = 'report/test_sort_{}_{}.txt'.format(size, file_size)
if os.path.exists(result_file):
print('skip {}'.format(result_file))
continue
dfmaker = LabelDataMaker(size, file_size, 100, 2)
multip = len(dfmaker)
if multip == 0:
continue
timer = Timer()
print('running {}'.format(result_file))
timer(sort, next(dfmaker), **kwargs)
cost = timer.avg_cost * multip
with open(result_file, 'w') as f:
f.write('total,each,number\n')
f.write(
str(cost) + "," + str(timer.avg_cost) + "," + str(multip))