def test_dump_to_file_like(self):
class FileLike(object):
def __init__(self):
self.bytes = ''
def write(self, data_bytes):
self.bytes += data_bytes
f = FileLike()
ujson.dump([1, 2, 3], f)
assert "[1,2,3]" == f.bytes
def test_dump_to_file_like(self):
class FileLike(object):
def __init__(self):
self.bytes = ''
def write(self, data_bytes):
self.bytes += data_bytes
f = FileLike()
ujson.dump([1, 2, 3], f)
assert "[1,2,3]" == f.bytes
def test_dump_file_args_error(self):
with pytest.raises(TypeError):
ujson.dump([], "")
def test_dump_to_file(self):
f = StringIO()
ujson.dump([1, 2, 3], f)
assert "[1,2,3]" == f.getvalue()
def train(argv):
''' If you want to do data augmentation, please setting aug=True, otherwise, there is no data augmentation
:param argv:
:return:
'''
# Params
# Params
paras = {
'train_test_percent': 0.33,
'crop_spectra': False,
'aug': False,
'reshape': True,
'one_hot': True,
'X_min': 800,
'X_max': 2000,
'n_components': 2,
'model_path': '../result/CNN/CNN_MODEL/weights/CNN_Noise_DataAug/',
'num_classes': 2,
'epochs': 1000,
'batch_size': 512,
'seed': 7
}
# Writing paras into .json file
with open(paras['model_path'] + "paras_record.json", "w") as f:
json.dump(paras, f)
print("File successfully written... ...")
# load dataset and preprocess it, formatting it to a readable tensor
# Splitting data into training set and testing set
X_train, X_test, y_train, y_test = load_data_preprocess(
paras['train_test_percent'],
crop_spectra=paras['crop_spectra'],
aug=paras['aug'],
reshape=paras['reshape'],
one_hot=paras['one_hot'],
X_min=paras['X_min'],
X_max=paras['X_max'])
if argv[1] == 'base_cnn':
model = base_cnn((paras['X_max'] - paras['X_min'], 1),
paras['num_classes'])
if argv[1] == 'fully_cnn':
model = fully_connected_cnn((paras['X_max'] - paras['X_min'], 1),
paras['num_classes'])
if argv[1] == 'cnn':
model = without_fully_connected_cnn(
(paras['X_max'] - paras['X_min'], 1), paras['num_classes'])
if argv[1] == 'mlp':
model = build_mlp_architecture((paras['X_max'] - paras['X_min'], 1),
paras['num_classes'])
# fit and run our model
np.random.seed(paras['seed'])
best_model_file = \
"../result/CNN/CNN_MODEL/weights/CNN_Noise_DataAug/" \
"highest_val_acc_weights_epoch{epoch:02d}-val_acc{val_acc:.3f}_" \
+ str(argv[1]) + ".h5"
best_model = ModelCheckpoint(best_model_file,
monitor='val_acc',
verbose=1,
save_best_only=True)
hist = model.fit(X_train,
y_train,
validation_data=(X_test, y_test),
nb_epoch=paras['epochs'],
batch_size=paras['batch_size'],
callbacks=[best_model],
shuffle=True,
verbose=1)
print("done training")
training_graphs(hist)
def test_dump_file_args_error(self):
with pytest.raises(TypeError):
ujson.dump([], "")
'reshape': False,
'one_hot': False,
'X_min': 2700,
'X_max': 3100,
'n_components': 2,
'model_path': '../result/SVM/SVM_MODEL/',
'print_raw': True,
'show_interactive_plot': True,
'show_pca_loadings': False,
'show_tsne': False
}
plt.ioff()
# Writing paras into .json file
with open(paras['model_path'] + "paras_record.json", "w") as f:
json.dump(paras, f)
print("File successfully written... ...")
nor_Fat = area_normalization(
reshape_data('../Data/Trainging Data/20%Fat/',
X_max=paras['X_max'],
X_min=paras['X_min'])[0])
nor_Tumor = area_normalization(
reshape_data('../Data/Trainging Data/Tumor/',
X_max=paras['X_max'],
X_min=paras['X_min'])[0])
# Splitting data into training set and testing set
X_train, X_test, y_train, y_test, X_train_path, X_test_path = load_data_preprocess(
paras['train_test_percent'],
crop_spectra=paras['crop_spectra'],
def test_dump_to_file(self):
f = StringIO()
ujson.dump([1, 2, 3], f)
assert "[1,2,3]" == f.getvalue()
def update_codeConffile(dict):
with open(code_conf_file, 'w+') as outfile:
json.dump(dict, outfile)
def update_dataConffile(dict):
with open(data_Conf_file, 'w+') as outfile:
json.dump(dict, outfile)
return ps.stem(list)
with open('C:/Users/upadh/Desktop/domains.txt',
'r') as csvFile, open('current.json', 'w') as f:
csvReader = csv.DictReader(csvFile)
string = {}
# list1 = ["word"]
count = 0
for row in csvReader:
for row in csvReader:
#if row is 0 :
for header, value in row.items():
#print(tldextract.extract(value))
ext = tldextract.extract(value)
list = ext.domain
list = name(list)
#print(list)
try:
string[header].append("{0}".format(list))
except KeyError:
string[header] = [list]
#for w in string:
# print(w, " : ", ps.stem(w))
f.write(str(json.dump((string), f)))
# json.dump(doubleQString, f)
#fin = (string)
base_dir = args.path
dset = os.listdir(base_dir)
dset_list = [d for d in dset]
print(dset_list)
for dset in tqdm(dset_list):
parse_data_sub_multi_process(dset, base_dir)
elif args.job == 'create_json':
base_dir = args.path
dset = os.listdir(base_dir)
dset_list = [d for d in dset]
print(dset_list)
data = []
endvid = []
for subset in dset:
jsonpath = os.path.join(base_dir, subset, 'track.json')
if os.path.exists(jsonpath):
subdata = json.load(open(jsonpath, 'r'))
for idx, frame in enumerate(subdata):
if idx % args.skip == 0 and frame['object'] != []:
data.append(frame)
endvid.append(False)
elif frame['object'] != []:
print('found no veh in %d' % (frame['timestamp']))
endvid[-1] = True
json.dump(data, open(os.path.join(base_dir, 'track.json'), 'w'))
json.dump(endvid, open(os.path.join(base_dir, 'endvid.json'), 'w'))
else:
raise NotImplementedError(
'Please specify a valid job: parse_data/filter_for_detection_and_'
'split/filter_for_tracking_and_split')
# fit the model
svc = svm.SVC(kernel='linear', C=1.0).fit(irisDF[features], irisDF["Species"])
#array = irisDF.values
#X = array[:,0:4]
#Y = array[:,4]
#validation_size = 0.20
#seed = 7
#X_train, X_validation, Y_train, Y_validation = model_selection.train_test_split(X, Y, test_size=validation_size, random_state=seed)
# output a text description of the model
f = open(os.path.join(output_datadir, 'model.txt'), 'w')
f.write(str(svc))
f.close()
#medium
x_train, x_test, y_train, y_test = model_selection.train_test_split(
irisDF[features], irisDF["Species"], test_size=.5)
predictions = svc.predict(x_test)
#from sklearn.metrics import accuracy_score
auc = (accuracy_score(y_test, predictions))
with open(metrics_file, 'w') as fd:
fd.write('AUC: {:4f}\n'.format(auc))
with open(metrics_json, 'w') as outfile:
json.dump(auc, outfile)
#kfold = svc.KFold(n_splits=10, random_state=seed)
# cv_results = model_selection.cross_val_score(model.pkl, X_train, Y_train, cv=kfold, scoring=scoring)
# msg = "%f (%f)" % (cv_results.mean(), cv_results.std())
# print(msg)
# persist the model
joblib.dump(svc, os.path.join(output_datadir, 'model.pkl'))
