def test_load_file(self): data = "[1,2,3,4]" exp_data = [1, 2, 3, 4] f = StringIO(data) assert exp_data == ujson.load(f) f = StringIO(data) tm.assert_numpy_array_equal(np.array(exp_data), ujson.load(f, numpy=True))
def test_load_file(self): data = "[1,2,3,4]" exp_data = [1, 2, 3, 4] f = StringIO(data) assert exp_data == ujson.load(f) f = StringIO(data) tm.assert_numpy_array_equal(np.array(exp_data), ujson.load(f, numpy=True))
def dvcRepro(): print( "\033[93m\n--------Repo initialized, Start building pipeline as per the steps mentioned in user inputs-------- \033[0m" ) dvcCommands = [] with open(sys.argv[1], 'rb') as json_file: user_inputs = json.load(json_file) for key in user_inputs: if (key != Constants.TAG): dvcCommands.append( automationService.generateCommand(key, user_inputs)) for cmd in dvcCommands: print(Fore.RED) print("\n " + cmd) print(Style.RESET_ALL) p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=commandOutput, stderr=subprocess.PIPE, universal_newlines=True, shell=True) p.communicate() with open(path_to_output_file, "r+") as f: print(f.read()) f.truncate(0)
def test_load_file_like(self): class FileLike(object): def read(self): try: self.end except AttributeError: self.end = True return "[1,2,3,4]" exp_data = [1, 2, 3, 4] f = FileLike() assert exp_data == ujson.load(f) f = FileLike() tm.assert_numpy_array_equal(np.array(exp_data), ujson.load(f, numpy=True))
def test_load_file_like(self): class FileLike(object): def read(self): try: self.end except AttributeError: self.end = True return "[1,2,3,4]" exp_data = [1, 2, 3, 4] f = FileLike() assert exp_data == ujson.load(f) f = FileLike() tm.assert_numpy_array_equal(np.array(exp_data), ujson.load(f, numpy=True))
def test_load_file_args_error(self): with pytest.raises(TypeError): ujson.load("[]")
def test_load_file_args_error(self): with pytest.raises(TypeError): ujson.load("[]")
''' import numpy as np import pandas as pd from pandas._libs import json from basic.predictor import Predictor if __name__ == '__main__': predictor = Predictor('../result/SVM/SVM_MODEL/model.pkl') # Printing info about model and data print("classifier model info: ", predictor.model) with open("../result/SVM/SVM_MODEL/paras_record.json", 'r') as paras_f: paras = json.load(paras_f) print(paras) data = predictor.read_data( '../Data/Testing Data/2_7-17-2019/20190716/S19-19485', crop_spectra=paras['crop_spectra'], X_min=paras['X_min'], X_max=paras['X_max']) labels = predictor.predict(data) decision = predictor.decision_function(data) proba = np.around(predictor.predict_proba(data), decimals=2) print('labels.shape: ', labels.shape, '\n', labels) print('decision.shape: ', decision.shape, '\n', decision) print('proba.shape: ', proba.shape, '\n', proba) result = pd.concat([pd.DataFrame(labels), pd.DataFrame(proba)])
data_dict = get_checksum(data_files, data_dir_path) update_dataConffile(data_dict) files = [] code_files = dir_structure(code_dir_path) code_dict = get_checksum(code_files, code_dir_path) update_codeConffile(code_dict) else: print("Looking for updated files") updated_data_files = dir_structure(data_dir_path) updated_data_dict = get_checksum(updated_data_files, data_dir_path) files = [] with open(data_Conf_file, 'rb') as json_file: last_data_info = json.load(json_file) for key in updated_data_dict: if key in last_data_info: if (updated_data_dict[key] != last_data_info[key]): print("updated----", key, updated_data_dict[key]) print("last-----", key, last_data_info[key]) commands = ['dvc add data\Posts.csv', 'dvc push'] p = subprocess.Popen('cmd.exe', stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) for cmd in commands: p.stdin.write(cmd + "\n")
base_dir = args.path dset = os.listdir(base_dir) dset_list = [d for d in dset] print(dset_list) for dset in tqdm(dset_list): parse_data_sub_multi_process(dset, base_dir) elif args.job == 'create_json': base_dir = args.path dset = os.listdir(base_dir) dset_list = [d for d in dset] print(dset_list) data = [] endvid = [] for subset in dset: jsonpath = os.path.join(base_dir, subset, 'track.json') if os.path.exists(jsonpath): subdata = json.load(open(jsonpath, 'r')) for idx, frame in enumerate(subdata): if idx % args.skip == 0 and frame['object'] != []: data.append(frame) endvid.append(False) elif frame['object'] != []: print('found no veh in %d' % (frame['timestamp'])) endvid[-1] = True json.dump(data, open(os.path.join(base_dir, 'track.json'), 'w')) json.dump(endvid, open(os.path.join(base_dir, 'endvid.json'), 'w')) else: raise NotImplementedError( 'Please specify a valid job: parse_data/filter_for_detection_and_' 'split/filter_for_tracking_and_split')
from functools import partial import pandas as pd import numpy as np import multiprocessing as mp from pandas._libs import json import helper parametersFilePath = "parameters/data_parameters.json" #Loading parameters file print("========= Loading Parameters") parameters = None with open(parametersFilePath, 'r') as parametersFileHandler: parameters = json.load(parametersFileHandler) if parameters is None: exit(1) mimic_data_path = parameters['mimicDataPath'] events_files_path = parameters['dataPath'] new_events_files_path = parameters['dataPathBinary'] if not os.path.exists(new_events_files_path): os.mkdir(new_events_files_path) all_features, features_types = helper\ .get_attributes_from_arff(parameters['parametersArffFile']) categorical_features_chartevents = set([itemid for itemid in features_types.keys() if features_types[itemid] == helper.CATEGORICAL_LABEL]) dataset_csv = pd.read_csv('dataset.csv')
def checkForUpdates(): data_changes = False code_changes = False updated_data_files = dir_structure(data_dir_path) updated_data_dict = get_checksum(updated_data_files, data_dir_path) updated_code_files = dir_structure(code_dir_path) updated_code_dict = get_checksum(updated_code_files, code_dir_path) with open(data_Conf_file, 'rb') as json_file: last_data_info = json.load(json_file) for key in updated_data_dict: if key in last_data_info: if (updated_data_dict[key] != last_data_info[key]): print("updated----", key, updated_data_dict[key]) print("last-----", key, last_data_info[key]) data_changes = True with open(code_conf_file, 'rb') as json_file: last_code_info = json.load(json_file) for key in updated_code_dict: if key in last_code_info: if (updated_code_dict[key] != last_code_info[key]): code_changes = True print("updated----", key, updated_code_dict[key]) print("last-----", key, last_code_info[key]) if (code_changes and data_changes): update_dataConffile(updated_data_dict) update_codeConffile(updated_code_dict) dvc_dataupdate = [ 'dvc add data', 'dvc push'] print("\n \033[93m Both Code and Data files are updated,push changes to dvc remote and github\033[0m") for cmd in dvc_dataupdate: p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True,shell=True) print(Fore.RED +"\n"+ cmd) print(Style.RESET_ALL) output=p.communicate() print(output) gitCommands() elif (code_changes): print("\n\033[93m Only Code is updated, push changes to github \033[0m") update_codeConffile(updated_code_dict) gitCommands() elif (data_changes): update_dataConffile(updated_data_dict) dvc_dataupdate = ['dvc add data', 'dvc push'] print("\n \033[93mData files are updated so add updated data to dvc cache and push files to remote server \033[0m") for cmd in dvc_dataupdate: p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, shell=True) print(Fore.RED +"\n"+ cmd) print(Style.RESET_ALL) output = p.communicate() print(output) gitCommands() else: print("\033[91m No updates found \033[0m")
import sys from colorama import init from pandas._libs import json from os import path, walk import hashlib import subprocess import Constants from colorama import Fore, Style # Read conf.json file" with open(Constants.CONF_FILE, 'rb') as json_file: conf_info = json.load(json_file) data_Conf_file = conf_info['dataConf_path'] code_conf_file = conf_info['codeConf_path'] data_dir_path = conf_info['data_path'] code_dir_path = conf_info['code_path'] Key_Id = conf_info['Key'] Secret_Access_Key = conf_info['Secret_Access_Key'] with open(sys.argv[1], 'rb') as json_file: user_inputs = json.load(json_file) # Generate dvc commands by reading the input from user_inputs.json file for various dvc stages. def generateCommand(stage,user_inputs): dependentVariable = 0 outputVariable = 0 command = Constants.DVC_INITIAL_COMMAND if (stage == Constants.EVALUATION_STAGE): command = Constants.DVC_FINAL_COMMAND while dependentVariable < (len((user_inputs[stage])[Constants.DEPENDENCY])):