def test_load_file(self):
data = "[1,2,3,4]"
exp_data = [1, 2, 3, 4]
f = StringIO(data)
assert exp_data == ujson.load(f)
f = StringIO(data)
tm.assert_numpy_array_equal(np.array(exp_data),
ujson.load(f, numpy=True))
def test_load_file(self):
data = "[1,2,3,4]"
exp_data = [1, 2, 3, 4]
f = StringIO(data)
assert exp_data == ujson.load(f)
f = StringIO(data)
tm.assert_numpy_array_equal(np.array(exp_data),
ujson.load(f, numpy=True))
def dvcRepro():
print(
"\033[93m\n--------Repo initialized, Start building pipeline as per the steps mentioned in user inputs-------- \033[0m"
)
dvcCommands = []
with open(sys.argv[1], 'rb') as json_file:
user_inputs = json.load(json_file)
for key in user_inputs:
if (key != Constants.TAG):
dvcCommands.append(
automationService.generateCommand(key, user_inputs))
for cmd in dvcCommands:
print(Fore.RED)
print("\n " + cmd)
print(Style.RESET_ALL)
p = subprocess.Popen(cmd,
stdin=subprocess.PIPE,
stdout=commandOutput,
stderr=subprocess.PIPE,
universal_newlines=True,
shell=True)
p.communicate()
with open(path_to_output_file, "r+") as f:
print(f.read())
f.truncate(0)
def test_load_file_like(self):
class FileLike(object):
def read(self):
try:
self.end
except AttributeError:
self.end = True
return "[1,2,3,4]"
exp_data = [1, 2, 3, 4]
f = FileLike()
assert exp_data == ujson.load(f)
f = FileLike()
tm.assert_numpy_array_equal(np.array(exp_data),
ujson.load(f, numpy=True))
def test_load_file_like(self):
class FileLike(object):
def read(self):
try:
self.end
except AttributeError:
self.end = True
return "[1,2,3,4]"
exp_data = [1, 2, 3, 4]
f = FileLike()
assert exp_data == ujson.load(f)
f = FileLike()
tm.assert_numpy_array_equal(np.array(exp_data),
ujson.load(f, numpy=True))
def test_load_file_args_error(self):
with pytest.raises(TypeError):
ujson.load("[]")
def test_load_file_args_error(self):
with pytest.raises(TypeError):
ujson.load("[]")
'''
import numpy as np
import pandas as pd
from pandas._libs import json
from basic.predictor import Predictor
if __name__ == '__main__':
predictor = Predictor('../result/SVM/SVM_MODEL/model.pkl')
# Printing info about model and data
print("classifier model info: ", predictor.model)
with open("../result/SVM/SVM_MODEL/paras_record.json", 'r') as paras_f:
paras = json.load(paras_f)
print(paras)
data = predictor.read_data(
'../Data/Testing Data/2_7-17-2019/20190716/S19-19485',
crop_spectra=paras['crop_spectra'],
X_min=paras['X_min'],
X_max=paras['X_max'])
labels = predictor.predict(data)
decision = predictor.decision_function(data)
proba = np.around(predictor.predict_proba(data), decimals=2)
print('labels.shape: ', labels.shape, '\n', labels)
print('decision.shape: ', decision.shape, '\n', decision)
print('proba.shape: ', proba.shape, '\n', proba)
result = pd.concat([pd.DataFrame(labels), pd.DataFrame(proba)])
data_dict = get_checksum(data_files, data_dir_path)
update_dataConffile(data_dict)
files = []
code_files = dir_structure(code_dir_path)
code_dict = get_checksum(code_files, code_dir_path)
update_codeConffile(code_dict)
else:
print("Looking for updated files")
updated_data_files = dir_structure(data_dir_path)
updated_data_dict = get_checksum(updated_data_files, data_dir_path)
files = []
with open(data_Conf_file, 'rb') as json_file:
last_data_info = json.load(json_file)
for key in updated_data_dict:
if key in last_data_info:
if (updated_data_dict[key] != last_data_info[key]):
print("updated----", key, updated_data_dict[key])
print("last-----", key, last_data_info[key])
commands = ['dvc add data\Posts.csv', 'dvc push']
p = subprocess.Popen('cmd.exe',
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
for cmd in commands:
p.stdin.write(cmd + "\n")
base_dir = args.path
dset = os.listdir(base_dir)
dset_list = [d for d in dset]
print(dset_list)
for dset in tqdm(dset_list):
parse_data_sub_multi_process(dset, base_dir)
elif args.job == 'create_json':
base_dir = args.path
dset = os.listdir(base_dir)
dset_list = [d for d in dset]
print(dset_list)
data = []
endvid = []
for subset in dset:
jsonpath = os.path.join(base_dir, subset, 'track.json')
if os.path.exists(jsonpath):
subdata = json.load(open(jsonpath, 'r'))
for idx, frame in enumerate(subdata):
if idx % args.skip == 0 and frame['object'] != []:
data.append(frame)
endvid.append(False)
elif frame['object'] != []:
print('found no veh in %d' % (frame['timestamp']))
endvid[-1] = True
json.dump(data, open(os.path.join(base_dir, 'track.json'), 'w'))
json.dump(endvid, open(os.path.join(base_dir, 'endvid.json'), 'w'))
else:
raise NotImplementedError(
'Please specify a valid job: parse_data/filter_for_detection_and_'
'split/filter_for_tracking_and_split')
from functools import partial
import pandas as pd
import numpy as np
import multiprocessing as mp
from pandas._libs import json
import helper
parametersFilePath = "parameters/data_parameters.json"
#Loading parameters file
print("========= Loading Parameters")
parameters = None
with open(parametersFilePath, 'r') as parametersFileHandler:
parameters = json.load(parametersFileHandler)
if parameters is None:
exit(1)
mimic_data_path = parameters['mimicDataPath']
events_files_path = parameters['dataPath']
new_events_files_path = parameters['dataPathBinary']
if not os.path.exists(new_events_files_path):
os.mkdir(new_events_files_path)
all_features, features_types = helper\
.get_attributes_from_arff(parameters['parametersArffFile'])
categorical_features_chartevents = set([itemid for itemid in features_types.keys()
if features_types[itemid] == helper.CATEGORICAL_LABEL])
dataset_csv = pd.read_csv('dataset.csv')
def checkForUpdates():
data_changes = False
code_changes = False
updated_data_files = dir_structure(data_dir_path)
updated_data_dict = get_checksum(updated_data_files, data_dir_path)
updated_code_files = dir_structure(code_dir_path)
updated_code_dict = get_checksum(updated_code_files, code_dir_path)
with open(data_Conf_file, 'rb') as json_file:
last_data_info = json.load(json_file)
for key in updated_data_dict:
if key in last_data_info:
if (updated_data_dict[key] != last_data_info[key]):
print("updated----", key, updated_data_dict[key])
print("last-----", key, last_data_info[key])
data_changes = True
with open(code_conf_file, 'rb') as json_file:
last_code_info = json.load(json_file)
for key in updated_code_dict:
if key in last_code_info:
if (updated_code_dict[key] != last_code_info[key]):
code_changes = True
print("updated----", key, updated_code_dict[key])
print("last-----", key, last_code_info[key])
if (code_changes and data_changes):
update_dataConffile(updated_data_dict)
update_codeConffile(updated_code_dict)
dvc_dataupdate = [
'dvc add data',
'dvc push']
print("\n \033[93m Both Code and Data files are updated,push changes to dvc remote and github\033[0m")
for cmd in dvc_dataupdate:
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True,shell=True)
print(Fore.RED +"\n"+ cmd)
print(Style.RESET_ALL)
output=p.communicate()
print(output)
gitCommands()
elif (code_changes):
print("\n\033[93m Only Code is updated, push changes to github \033[0m")
update_codeConffile(updated_code_dict)
gitCommands()
elif (data_changes):
update_dataConffile(updated_data_dict)
dvc_dataupdate = ['dvc add data',
'dvc push']
print("\n \033[93mData files are updated so add updated data to dvc cache and push files to remote server \033[0m")
for cmd in dvc_dataupdate:
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True,
shell=True)
print(Fore.RED +"\n"+ cmd)
print(Style.RESET_ALL)
output = p.communicate()
print(output)
gitCommands()
else:
print("\033[91m No updates found \033[0m")
import sys
from colorama import init
from pandas._libs import json
from os import path, walk
import hashlib
import subprocess
import Constants
from colorama import Fore, Style
# Read conf.json file"
with open(Constants.CONF_FILE, 'rb') as json_file:
conf_info = json.load(json_file)
data_Conf_file = conf_info['dataConf_path']
code_conf_file = conf_info['codeConf_path']
data_dir_path = conf_info['data_path']
code_dir_path = conf_info['code_path']
Key_Id = conf_info['Key']
Secret_Access_Key = conf_info['Secret_Access_Key']
with open(sys.argv[1], 'rb') as json_file:
user_inputs = json.load(json_file)
# Generate dvc commands by reading the input from user_inputs.json file for various dvc stages.
def generateCommand(stage,user_inputs):
dependentVariable = 0
outputVariable = 0
command = Constants.DVC_INITIAL_COMMAND
if (stage == Constants.EVALUATION_STAGE):
command = Constants.DVC_FINAL_COMMAND
while dependentVariable < (len((user_inputs[stage])[Constants.DEPENDENCY])):
