def save_model(self, model_path=None): if model_path is None: model_path = self.model_path utils.check_dir(model_path+"a") np.save(os.path.join(model_path,"Weight"),self.weight) np.save(os.path.join(model_path,"Bias"),self.bias) logging.info('Save model to ' + model_path[:50] + '...')
def get_daily_data(self, code, expire=60 * 6): """ 获取一支股票所有历史数据保存到本地 """ UT.check_dir(CT.DAILY_DIR) file_path = CT.DAILY_DIR + code expired = UT.check_file_expired(file_path, expire) if expired or not os.path.exists(file_path): symbol = self._code_to_symbol(code) start_date = CT.START end_date = date_time.get_today_str() adjust = 'qfq' if is_index(code): d = ak.stock_zh_index_daily(symbol) else: d = ak.stock_zh_a_daily(symbol, start_date, end_date, adjust) if d is None: return d d.to_csv(file_path, sep='\t') if not os.path.exists(file_path): return None #d = pd.read_csv(file_path, sep='\t', index_col=0) d = pd.read_csv(file_path, sep='\t', skiprows=0, parse_dates=True, header=0, index_col=0) return d
def controller_plots(model_dir, ds, ds_eval, groundtruth, prediction, communication): """ :param model_dir: directory containing the trained model :param ds: name of the dataset :param ds_eval: name of the dataset for the evaluation (usually the manual one) :param groundtruth: evidence :param prediction: output control :param communication: states if the communication is used by the network """ model_img = '%s/images/' % model_dir utils.check_dir(model_img) # Plot R^2 of the regressor between prediction and ground truth title = 'Regression %s vs %s' % (ds_eval, ds) file_name = 'regression-%svs%s' % (ds_eval, ds) if not communication: groundtruth = np.array(groundtruth).flatten() prediction = np.array(prediction).flatten() x_label = 'groundtruth' y_label = 'prediction' my_plots.plot_regressor(groundtruth, prediction, x_label, y_label, model_img, title, file_name)
def get_tick_data(self, code, trade_date, expire=60 * 24 * 365 * 10): """ 获取一支股票一天的tick数据保存到本地 """ UT.check_dir(CT.TICK_DIR + code) file_path = CT.TICK_DIR + code + '/' + trade_date symbol = self._code_to_symbol(code) trade_date = date_time.date_to_str(date_time.str_to_date(trade_date), '%Y%m%d') expired = UT.check_file_expired(file_path, expire) if expired or not os.path.exists(file_path): d = ak.stock_zh_a_tick_tx(symbol, trade_date) #过掉当天没数据的 if d is None or len(d) < 10: return None d.to_csv(file_path, sep='\t') if not os.path.exists(file_path): return None #d = pd.read_csv(file_path, sep='\t', index_col=1) d = pd.read_csv(file_path, sep='\t', skiprows=0, parse_dates=True, header=0, index_col=0) #过掉当天没数据的 if d is None or len(d) < 10: return None return d
def get_minute_data(self, code, period='1', adjust="", expire=60 * 6): """ 获取一支股票分钟级数据保存到本地 """ UT.check_dir(CT.MINUTE_DIR + '/' + period) file_path = CT.MINUTE_DIR + '/' + period + '/' + code expired = UT.check_file_expired(file_path, expire) if expired or not os.path.exists(file_path): symbol = self._code_to_symbol(code) start_date = CT.START end_date = date_time.get_today_str() adjust = 'qfq' # d = ak.stock_zh_a_minute(symbol, period, adjust) d = self.stock_zh_a_minute(symbol, period, adjust) if d is None: return d d.to_csv(file_path, sep='\t') if not os.path.exists(file_path): return None #d = pd.read_csv(file_path, sep='\t', index_col=1) d = pd.read_csv(file_path, sep='\t', skiprows=0, parse_dates=True, header=0, index_col=0) return d
def yolo_to_mask_images(path, save_path, pedestrian_ratio=0.4, vehicle_ratio=0.5): vehicle_mask, pedestrian_mask = get_mask_filename(path, pedestrian_ratio, vehicle_ratio) all_mask = np.array(vehicle_mask + pedestrian_mask) all_mask = np.unique(all_mask) img_save_path = os.path.join(save_path, 'images') label_save_path = os.path.join(save_path, 'labels') check_dir(img_save_path) check_dir(label_save_path) img_root = os.path.join(path, 'images') label_root = os.path.join(path, 'labels') for filename in all_mask: img_path = os.path.join(img_root, filename) label_path = os.path.join(label_root, filename[:-4] + ".txt") mask_vehicle = filename in vehicle_mask mask_pedestrian = filename in pedestrian_mask mask_img, mask_labels = transform_mask(img_path, label_path, mask_pedestrian=True, mask_vehicle=True, background_color=(114, 114, 114)) if len(mask_labels) == 0: continue save_name = filename[:-4] + '_mask' with open(os.path.join(label_save_path, save_name + ".txt"),"w") as weiter: for mask_label in mask_labels: weiter.write(mask_label) Image.fromarray(mask_img).save(os.path.join(img_save_path, save_name + '.jpg'))
def ivs_to_yolo(path, save_path, sampling_margin=200): img_dirs = os.listdir(path) img_save_path = os.path.join(save_path, 'images') label_save_path = os.path.join(save_path, 'labels') check_dir(img_save_path) check_dir(label_save_path) for img_dir in img_dirs: img_root = os.path.join(path, img_dir) filenames = os.listdir(img_root) for filename in filenames[0::sampling_margin]: source = os.path.join(img_root, filename) dest = os.path.join(img_save_path, filename) shutil.copyfile(source, dest) source = source.replace('JPEGImages', 'Annotations').replace('jpg', 'xml') bboxes = xml_convert_yolo(source) filename = filename.replace('jpg', 'txt') dest = os.path.join(label_save_path, filename) with open(dest, "w") as weiter: for bboxe in bboxes: msg = msg_format % bboxe weiter.write(msg)
def bdd_to_yolo(path='./bdd100k', save_path='./bdd100k/val', mode='val'): labels_path = os.path.join( path, 'labels/bdd100k_labels_images_' + mode + '.json') with open(labels_path) as f: all_labels = json.load(f) img_root = os.path.join(path, 'images/100k/' + mode) img_save_path = os.path.join(save_path, 'images') label_save_path = os.path.join(save_path, 'labels') check_dir(img_save_path) check_dir(label_save_path) rider_names, motor_names, bike_names = get_bdd_fixed_filename(all_labels) for labels in all_labels: filename = labels['name'] if filename in rider_names: continue shutil.copyfile(os.path.join(img_root, filename), os.path.join(img_save_path, filename)) bboxes = json_convert_yolo(labels) dest = os.path.join(label_save_path, filename[:-4] + ".txt") with open(dest, "w") as weiter: for bboxe in bboxes: msg = msg_format % bboxe weiter.write(msg)
def evaluate(args): """ Evaluate model on testset """ # get image paths and split dataset testset_dir = os.path.join(args.dataset_dir, 'test') check_dir(testset_dir, report_error=True) _, test_img_paths, _, _ = \ get_and_split_dataset(testset_dir, test_size=1.0) with open(args.label_path, 'r') as fid: label2idx = json.loads(fid.read()) idx2label = defaultdict(str) for k, v in label2idx.items(): idx2label[v] = k # create dataset testset = Dataset(test_img_paths, label2idx=label2idx, idx2label=idx2label, target_shape=TARGET_SHAPE, batch_size=args.batch_size, shuffle=False) # load model model = load_model(args.model_path) # get prediction error_cnt = 0 for images, labels in testset.generate(epoch_stop=True): # predict gt_labels = np.argmax(labels, axis=1) pred_probs = model.predict(images) print(pred_probs.shape) pred_labels = np.argmax(pred_probs, axis=1) # print results for idx, gt in enumerate(gt_labels): pred = pred_labels[idx] print('gt={}, pred={}'.format(idx2label[gt], idx2label[pred])) true_or_false = True if gt != pred: error_cnt += 1 true_or_false = False if args.save_samples is True: img_path = 'output/test_{}_gt_{}-pred_{}.png'.format( true_or_false, idx2label[gt], idx2label[pred]) img = images[idx, ...] * 255 img = img.astype(np.uint8) cv2.imwrite(img_path, cv2.cvtColor(img, cv2.COLOR_RGB2BGR)) num_samples = testset.get_dataset_size() print('total acc={}%'.format( (num_samples - error_cnt) * 100 / float(num_samples)))
def main(): detector = YoloV3Detection(graph_path=args.graph_path) utils.check_dir(args.out_dir) args.mode = "video" args.video_pth = "hiv00418.mp4" if args.mode == "video": cap = cv2.VideoCapture(args.video_pth) total_frame = cap.get(cv2.CAP_PROP_FRAME_COUNT) vid_w, vid_h = cap.get(cv2.CAP_PROP_FRAME_WIDTH), cap.get( cv2.CAP_PROP_FRAME_HEIGHT) fps = cap.get(cv2.CAP_PROP_FPS) fourcc = cv2.VideoWriter_fourcc(*'mp4v') output_video = os.path.join(args.out_dir, os.path.split(args.video_pth)[-1]) out_res = cv2.VideoWriter(output_video, fourcc, fps, (int(vid_w), int(vid_h))) count = 0 while cap.isOpened() and count <= total_frame: ret, frame = cap.read() if ret: boxes, labels, labels_idx = process(detector, frame) print(boxes) frame = utils.draw_boxes_with_texts(frame, boxes, labels, labels_idx) out_res.write(frame) cv2.imshow("Demo", frame) count += 1 if cv2.waitKey(1) & 0xFF == ord('q'): break cap.release() cv2.destroyAllWindows() elif args.mode == "images": list_ims = os.listdir(args.im_dir) for im_n in tqdm(list_ims): if im_n.split(".")[-1] in ["jpg", "png"]: im_pth = os.path.join(args.im_dir, im_n) im = cv2.imread(im_pth) boxes, labels, labels_idx = process(detector, im) im = utils.draw_boxes_with_texts(im, boxes, labels, labels_idx) # save im_save_pth = os.path.join(args.out_dir, im_n) cv2.imwrite(im_save_pth, im) else: print("[ERRO] Mode is not exits, check mode pls ... ")
def parse_args(): """ Parse arguments from command line """ ap = argparse.ArgumentParser() ap.add_argument('-d', '--dataset-dir', type=str, default=DATASET_PATH, help="path to input dataset") ap.add_argument('-t', '--test-ratio', type=float, default=0.2, help='ratio of samples in testset over the whole dataset') ap.add_argument('-m', '--model-path', type=str, default=MODEL_SAVE_PATH, help="path to output model") ap.add_argument('-l', '--label_path', type=str, default=LABELBIN_SAVE, help="path to output label binarizer") ap.add_argument('-p', '--plot-path', type=str, default=LOSS_PLOT_PATH, help="path to output accuracy/loss plot") ap.add_argument('-b', '--batch-size', type=int, default=128, help='batch size') ap.add_argument('-e', '--epochs', type=int, default=25, help='') ap.add_argument('-i', '--init-lr', type=float, default=1e-3) ap.add_argument('--phase', type=str, default='train', choices=['train', 'evaluate'], help='specify operations, train or evaluate') ap.add_argument('--save-samples', type=bool, default=True, help='flag to indicate whether save samples on evaluating') args = ap.parse_args() # check args check_dir(args.dataset_dir, report_error=True) check_dir(os.path.dirname(args.model_path)) check_dir(os.path.dirname(args.label_path)) check_dir(os.path.dirname(args.plot_path)) return args
def run_iterations(args, augmentator, augmentator_name, tmp_output_directory, iterations, datasets_dict, classifier_name, epochs, start): print('\t\twithout augmentation: ', augmentator_name) for dataset_name in dataset_names_for_archive[ARCHIVE_NAMES[0]]: print('\t\t\tdataset_name: ', dataset_name) upper_dir = tmp_output_directory + augmentator_name + '/' + dataset_name done = check_dir(upper_dir) if not done: #save all the predictions and the corresponding true class predicted_y = [] expected_y = [] for iter in range(iterations): print('\t\t\t\titer', iter) trr = '_itr_' + str(iter) output_directory = upper_dir + '/' + trr + '/' #print(output_directory) create_directory(output_directory) y_pred, y_true = fit_classifier_aug(args, augmentator, datasets_dict, dataset_name, classifier_name, epochs, output_directory) print('\t\t\t\tDONE') # the creation of this directory means create_directory(output_directory + '/DONE') if (y_pred.shape == y_true.shape): predicted_y.extend(y_pred) expected_y.extend(y_true) else: raise Exception("FALSE: y_pred.shape==y_true.shape.") totalduration = time.time() - start df_metrics = calculate_metrics(expected_y, predicted_y, totalduration) df_metrics.to_csv(upper_dir + '/avg_metrics.csv', index=False) create_directory(upper_dir + '/DONE') print('iterations DONE!') print(df_metrics)
def parse_args(): """ Parse arguments from command line """ ap = argparse.ArgumentParser() ap.add_argument('-d', '--dataset-dir', type=str, default=DATASET_PATH, help="path to input dataset") ap.add_argument('-m', '--model-path', type=str, default=MODEL_SAVE_PATH, help="path to output model") ap.add_argument('-l', '--label_path', type=str, default=LABELBIN_SAVE, help="path to output label binarizer") ap.add_argument('-p', '--plot-path', type=str, default=LOSS_PLOT_PATH, help="path to output accuracy/loss plot") ap.add_argument('--trainlog-path', type=str, default=TRAIN_LOG_PATH, help='path to training log') ap.add_argument('-b', '--batch-size', type=int, default=64, help='batch size') ap.add_argument('-e', '--epochs', type=int, default=20, help='number of epochs') ap.add_argument('-i', '--init-lr', type=float, default=1e-5) ap.add_argument('--phase', type=str, default='train', choices=['train', 'evaluate'], help='specify operations, train or evaluate') ap.add_argument('--goon-train', type=bool, default=False, help='load old model and go on training') ap.add_argument('--batch-normalize', type=bool, default=True, help='add batch normalization layers after activations') ap.add_argument('--dropout', type=float, default=0.25, help='dropout probability on training') args = ap.parse_args() # check args check_dir(args.dataset_dir, report_error=True) check_dir(os.path.dirname(args.model_path)) check_dir(os.path.dirname(args.label_path)) check_dir(os.path.dirname(args.plot_path)) check_dir(os.path.dirname(args.trainlog_path)) return args
def run_cv(args, augmentator, augmentator_name, tmp_output_directory, datasets_dict, classifier_name, epochs, start, cv): print('\t\taugmentator_name: ', augmentator_name) for dataset_name in dataset_names_for_archive[ARCHIVE_NAMES[0]]: print('\t\t\tdataset_name: ', dataset_name) output_directory = tmp_output_directory + augmentator_name + '/' + dataset_name done = check_dir(output_directory) if not done: create_directory(output_directory) cv_fit_classifier_aug(args, augmentator, datasets_dict, dataset_name, classifier_name, epochs, output_directory, cv) create_directory(output_directory + '/DONE')
def main(): ############################################################################# # 0. # # Check if tmp folder exists, otherwise create it if check_dir(settings.tmp_dir) == True: sys.exit(0) else: os.makedirs(settings.tmp_dir) # Run some checks on the source folder with core data. if not get_years(): # Is there anything in the source folder to begin with? print "We were not able to find a XLSX file with core data in the folder: "\ "%s. Make sure this folder contains at least one XLSX file named "\ "after the year (eg. 2014.xlsx). Check the readme for more info "\ "about the required structure of these files.\n"\ "Quiting..." % (settings.src_core) sys.exit(0) # Provide feedback that the script only processes XLSX files with properly # formatted filenames. (eg. 2014.xlsx) fn_pattern = re.compile('^20[0-9]{2}$') for f in os.listdir(settings.src_core): fn = os.path.splitext(f)[0] ext = os.path.splitext(f)[-1].lower() path = os.path.join(settings.src_core, fn) if not os.path.isdir(path): # Only check files if ext == ".xlsx": if not fn_pattern.match(fn): print "The XLSX file %s doesn't have a properly formatted year as "\ "filename and will be ignored." % (f) else: print "The script only processes XLSX files. %s will be ignored." % (f) print "Loading the core and meta data..." # Build the different sets of admin areas with things we have to loop over. countries = build_set('country','type','iso',settings.src_meta_aa) states = build_set('state','type','iso',settings.src_meta_aa) admin_areas = countries | states # Build sets for the variables we loop over global index_param index_param = build_set('param','type','id',settings.src_meta_index) index_score = build_set('score','type','id',settings.src_meta_index) sp = list(index_score | index_param) # Build set for the years we're interested in global years years = get_years() global current_yr current_yr = max(years) # Read in the files with meta-data and set the scope to global global df_meta_aa df_meta_aa = pd.read_csv(settings.src_meta_aa,index_col='iso') global df_meta_index df_meta_index = pd.read_csv(settings.src_meta_index,index_col='id') ############################################################################# # 1. Store the relevant core data in one DF (df_full) # # # Output: df_full # # 2014 2015 # iso ind value data value data # AR 0 1.2420 NaN 1.2235 NaN # 1.01 0.1802 78.17 0.1795 75.16 # ... first_yr = True for yr in years: # All core data files are named after the year of the edition fn = settings.src_core + yr + '.xlsx' df_yr = pd.DataFrame() for sheet in settings.core_data_sheets: # Build an index to parse only the relevant columns cols_index = build_col_index(fn,sheet) # Read Excel (parsing only relevant cols) df_sheet = pd.read_excel(fn,sheet,parse_cols=cols_index) # Ensure that the iso codes don't contain strange characters. They can only # contain letters, numbers and hyphens. (eg. CN, CN-65 or IN-MP) df_sheet['iso'].replace(to_replace='[^a-zA-Z0-9-]', value='',inplace=True,regex=True) # Append each sheet to a dataframe holding the data for that year df_yr = df_yr.append(df_sheet) # Set the index of the DF to the ISO code and ID of the indicator df_yr.set_index(['iso','id'],inplace=True) # Make sure the index is sorted so the slicing works well df_yr.sortlevel(inplace=True) # Rename the column 'score' to value df_yr.rename(columns={'score':'value'}, inplace=True) # Add an extra level in the hierarchy of the columns (Mutli-index) # containing an indication of the year # Create list that repeats 'value' for the amount of years available c = [yr] * len(df_yr.columns) # Add a level to the cols df_yr.columns = [c, df_yr.columns] if first_yr: # If it's the first year, we initialize the full DataFrame df_full = df_yr first_yr = False else: # Every subsequent year will have to be merged into df_full df_full = pd.merge(df_full,df_yr,how='outer',left_index=True,right_index=True) df_full.sortlevel(axis=1,inplace=True) ############################################################################# # 2. CSV downloads # # For all the CSV exports, prepare a dataframe that combines the data with # the meta. print "Building the CSV files for the download section..." # For the CSV, we're only interested in the value column of each year df_full_csv = df_full.loc[:,(slice(None),'value')] df_full_csv.columns = df_full_csv.columns.get_level_values(0) # The full DF is a multi-index. Since the meta-files have a single index, # it is necessary to reset the indexes before joining on the column. df_full_csv = df_full_csv.reset_index() df_meta_aa_csv = df_meta_aa.reset_index() df_meta_index_csv = df_meta_index.reset_index() # Merge the country meta df_full_csv = pd.merge(df_full_csv,df_meta_aa_csv,on='iso') # Merge the index meta data df_full_csv = pd.merge(df_full_csv,df_meta_index_csv,on='id',suffixes=('_aa','_var')) # Re-index the DF on iso & id and make sure it's sorted df_full_csv.set_index(['iso','id'],inplace=True) df_full_csv.sortlevel(inplace=True) # 2.0 Export the full dataset to CSV for lang in settings.langs: # Build a list with the meta-data that needs to be included columns = ['name:' + lang + '_aa','name:' + lang + '_var','type_var'] columns = columns + list(years) file_path = (settings.exp_full_csv).format(lang=lang) df_full_csv.loc[slice(None),columns].to_csv(file_path,encoding='UTF-8',index=False) # 2.1 Generate the main CSV files # Slice the DF to only contain the score and parameters for the current year. df_main_csv = df_full_csv.loc[(slice(None),sp),:] for lang in settings.langs: # Pivot the DF and export it file_path = (settings.exp_current_csv).format(lang=lang, yr=current_yr) pivot_df(df_main_csv,'name:' + lang + '_aa','name:' + lang + '_var',current_yr).to_csv(file_path,encoding='UTF-8') # 2.3 Generate the country + state CSV files for aa in admin_areas: # Select the data of this admin area df_aa_csv = df_full_csv.loc[(aa,slice(None)),:] for lang in settings.langs: # Include the name of the var, its type and the years columns = ['name:' + lang + '_var','type_var'] + list(years) # Select the proper columns and generate the CSV file_path = (settings.exp_aa_csv).format(lang = lang, aa = aa.lower()) df_aa_csv.loc[slice(None),columns].to_csv(file_path,encoding='UTF-8',index=False) ############################################################################# # 3. Calculate the rankings # # # Output: df_full # # 2014 2015 # value data gr sr value data gr sr # iso id # AR 0 1.2420 NaN 13 NaN 1.2235 NaN 12 NaN # 1.01 0.1802 73.1 5 NaN 0.1795 75.8 6 NaN # ... print "Calculating the ranking..." # 3.0 Prepare the structure # Add placeholder cols with NaN that can be updated later with df.update() for year in years: for rank in ('gr', 'sr'): df_full[(year,rank)] = np.nan # Make sure its sorted df_full.sortlevel(axis=1,inplace=True) # 3.1 Global rank # The global rank (gr) is a rank of all the COUNTRIES in the project df_full = get_rank(countries,df_full,'gr') # 3.3 State rank # The state rank ('sr') ranks the STATES of a particular country for country in countries: # Check if there are any states or provinces for this country cs = build_set(country,'country','iso',settings.src_meta_aa) if cs: df_full = get_rank(cs,df_full,'sr') ############################################################################# # 4. JSON api # print "Building the JSON files for the API..." # 4.1 Generate the main JSON file for lang in settings.langs: # The JSON will contain a list with dicts json_data = [] # Loop over the countries list for country in countries: country_data = build_json_aa(country,df_full,lang, historic=True) # Sort the list of states / provinces if country_data['states']: country_data['states'] = sorted(country_data['states'], key=lambda k: k['name']) json_data.append(country_data) # Sort the list of countries by name sorted_data = sorted(json_data, key=lambda k: k['name']) # Write the list to a JSON file file_path = (settings.exp_core).format(lang=lang) write_json(file_path, sorted_data) # 4.3 Generate the country + state JSON files for aa in admin_areas: for lang in settings.langs: # Get the data for this admin area in a dict json_data = build_json_aa(aa,df_full,lang,indicators=True,historic=True) # Write the dict to a JSON file file_path = (settings.exp_aa).format(lang=lang,aa=aa.lower()) write_json(file_path, json_data) # Fully remove the temp directory clean_dir(settings.tmp_dir , True) print "All done. The data has been prepared for use on global-climatescope.org."
parser.add_argument('--inner_lr', type=float, default=1e-2) parser.add_argument('--inner_opt', type=str, default='SGD') parser.add_argument('--outer_lr', type=float, default=1e-3) parser.add_argument('--outer_opt', type=str, default='Adam') parser.add_argument('--lr_sched', type=lambda x: (str(x).lower() == 'true'), default=False) # imaml specific settings parser.add_argument('--lambda', type=float, default=2.0) parser.add_argument('--version', type=str, default='GD') parser.add_argument('--cg_steps', type=int, default=5) # network settings parser.add_argument('--net', type=str, default='ConvNet') parser.add_argument('--n_conv', type=int, default=4) parser.add_argument('--n_dense', type=int, default=0) parser.add_argument('--hidden_dim', type=int, default=64) parser.add_argument('--in_channels', type=int, default=1) parser.add_argument('--hidden_channels', type=int, default=64, help='Number of channels for each convolutional layer (default: 64).') args = parser.parse_args() return args if __name__ == '__main__': args = parse_args() set_seed(args.seed) set_gpu(args.device) check_dir(args) main(args)
plt.xlabel('time (seconds)', fontsize=11) plt.ylabel('x position', fontsize=11) plt.plot(time, positions, label='Kp = %s, Ki = %s, Kd = %s' % (round(P, 2), round(I, 2), round(D, 2))) plt.plot(time, [myt.goal_position[0]] * len(positions), label='Setpoint', ls='--', color='black') plt.title('Step response') plt.ylim(top=myt.goal_position[0] + 10) plt.xlim(0, 7) plt.legend() my_plots.save_visualisation( 'Step-responsep=kp%ski%skd%s' % (round(P, 2), round(I, 2), round(D, 2)), directory) if __name__ == '__main__': dir_ = os.path.join('controllers', 'images') utils.check_dir(dir) Prop = 100 Int = 0 Der = 0 main(dir_, Prop, Int, Der)
def save_model(self, model_path=None): if model_path is None: model_path = self.model_path utils.check_dir(os.path.join(model_path, "LR.pkl")) joblib.dump(self.model, os.path.join(model_path, "LR.pkl")) logging.info('Save model to ' + model_path[:50] + '...')
for data in data_array: this_date = data.get_date() if this_date < check: check = this_date # Add a relative time field to the objects for data in data_array: data.set_relative_time(data.get_date() - check) # Sort by date for nice plotting data_array.sort(key=lambda x: x.get_concentration()) data_array.sort(key=lambda x: x.get_date()) ####################### # Make output file and put the raw # histo's in it. # out_dir = utils.check_dir(args.out_dir) outFile = ROOT.TFile("{0}stabaliser_plots.root".format(out_dir), "recreate") for dKey in sorted(dates): for sKey in samples: # Make title strings and root objects for this loop hist = False date_str = dKey.strftime("%d/%m/%Y") base_name = "{0}_{1}".format(date_str, sKey) stack_name = "THStack_{0}".format(base_name) stack_title = "{0}% Te, {1}g/l PPO, {2}, {3}, {4}".format( args.te, args.ppo, args.stabaliser, date_str, sKey) canvas_name = "TCanvas_{0}".format(base_name) stack = ROOT.THStack(stack_name, stack_title) legend = ROOT.TLegend(0.65, 0.65, 0.9, 0.9)
def main(distances, front_prox_values, back_prox_values, front_prox_comms, back_prox_comms, myt_quantity, min_distance): """ :param distances: array containing the distances among the agents for each experiment :param front_prox_values: array containing the value of the frontal sensor using prox_values :param back_prox_values: array containing the value corresponding to the average of the rear sensor readings using prox_values :param front_prox_comms: array containing the value of the frontal sensor using prox_comm :param back_prox_comms: array containing the value corresponding to the average of the rear sensor readings using prox_comm :param myt_quantity: number of agents :param min_distance: length of the agents """ for idx, distance in enumerate(distances): initial_positions = np.array( [0, min_distance + distance, (min_distance + distance) * 2], dtype=np.float64) world = pyenki.World() myts = [] for i in range(myt_quantity): myt = Thymio(name='myt%d' % (i + 1), index=i, use_aseba_units=False) myt.position = (initial_positions[i], 0) myt.initial_position = myt.position # Reset the parameters myt.angle = 0 myt.prox_comm_tx = 0 myt.prox_comm_enable = True myts.append(myt) world.add_object(myt) print('distance = %d' % distance) # for m in myts: # print(m.name, round(m.initial_position[0], 1), round(m.position[0], 1)) sensors = dict() a = [] b = [] c = [] d = [] for _ in range(5): world.step(dt=0.1) sensing = myts[1].get_input_sensing() front_prox_value = sensing[2] back_prox_value = np.mean([sensing[5], sensing[6]]) front_prox_comm = sensing[9] back_prox_comm = np.mean([sensing[12], sensing[13]]) a.append(int(front_prox_value)) b.append(int(back_prox_value)) c.append(int(front_prox_comm)) d.append(int(back_prox_comm)) front_prox_values[idx] = int(np.mean(a)) back_prox_values[idx] = int(np.mean(b)) front_prox_comms[idx] = int(np.mean(c)) back_prox_comms[idx] = int(np.mean(d)) sensors['front_prox_values'] = int(np.mean(a)) sensors['back_prox_values'] = int(np.mean(b)) sensors['front_prox_comm'] = int(np.mean(c)) sensors['back_prox_comm'] = int(np.mean(d)) print(sensors) sensing_to_distances = [ distances, front_prox_values, back_prox_values, front_prox_comms, back_prox_comms ] file = os.path.join('controllers', 'sensing_to_distances.pkl') with open(file, 'wb') as f: pickle.dump(sensing_to_distances, f) plt.figure() plt.xlabel('distance', fontsize=11) plt.ylabel('sensing', fontsize=11) plt.plot(distances, front_prox_values, label='front_prox_values') plt.plot(distances, back_prox_values, label='back_prox_values') plt.plot(distances, front_prox_comms, label='front_prox_comm') plt.plot(distances, back_prox_comms, label='back_prox_comm') plt.legend() dir = os.path.join('controllers', 'images') utils.check_dir(dir) my_plots.save_visualisation('sensing_to_distances', dir)
def save_model(self, model_path=None): if model_path is None: model_path = self.model_path utils.check_dir(model_path) torch.save(self.state_dict(), model_path)
def save_model(self, model_path=None) -> NoReturn: if model_path is None: model_path = self.model_path utils.check_dir(model_path) torch.save(self.state_dict(), model_path) logging.info('Save model to ' + model_path[:50] + '...')
def train(args): """ Train model on trainset and validate on valset """ # get image paths and split dataset trainval_dir = os.path.join(args.dataset_dir, 'train_val') check_dir(trainval_dir, report_error=True) train_img_paths, test_img_paths, label2idx, idx2label = \ get_and_split_dataset(trainval_dir, test_size=args.test_ratio) with open(args.label_path, 'w') as wfid: wfid.write(json.dumps(label2idx)) # create augumentation operations aug = ImageDataGenerator(rotation_range=25, width_shift_range=0.1, height_shift_range=0.1, shear_range=0.2, zoom_range=0.2, horizontal_flip=True, fill_mode="nearest") # create generators of trainset and testset trainset = Dataset(image_paths=train_img_paths, label2idx=label2idx, idx2label=idx2label, target_shape=TARGET_SHAPE, batch_size=args.batch_size, shuffle=True) valset = Dataset(image_paths=test_img_paths, label2idx=label2idx, idx2label=idx2label, target_shape=TARGET_SHAPE, batch_size=args.batch_size, shuffle=False) # build model model = SmallerVGGNet.build(width=TARGET_SHAPE[1], height=TARGET_SHAPE[0], depth=TARGET_SHAPE[2], classes=len(label2idx)) opt = Adam(lr=args.init_lr, decay=args.init_lr / args.epochs) model.compile(loss="categorical_crossentropy", optimizer=opt, metrics=["accuracy"]) model.summary() # create callback callbacks = [ ModelCheckpoint(args.model_path, monitor='val_loss', save_best_only=True, verbose=1), # EarlyStopping(patience=50), ReduceLROnPlateau(patience=10), CSVLogger("training.log") ] # train model H = model.fit_generator( trainset.generate(), steps_per_epoch=trainset.get_dataset_size() // args.batch_size, validation_data=valset.generate(), validation_steps=valset.get_dataset_size() // args.batch_size, epochs=args.epochs, verbose=1, callbacks=callbacks) # plot curve plt.style.use("ggplot") plt.figure() N = args.epochs plt.plot(np.arange(0, N), H.history['loss'], label='train_loss') plt.plot(np.arange(0, N), H.history['val_loss'], label='val_loss') plt.plot(np.arange(0, N), H.history['acc'], label='train_acc') plt.plot(np.arange(0, N), H.history['val_acc'], label='val_acc') plt.title("Training Loss and Accuracy") plt.xlabel("Epoch #") plt.ylabel("Loss/Accuracy") plt.legend(loc="upper left") plt.savefig(args.plot_path)
def fixed_bdd_to_yolo(path='./bdd100k', save_path='./bdd100k/val', mode='val', x_scale=1. / 1280., y_scale=1. / 720.): img_root = os.path.join(path, 'images/100k/' + mode) img_save_path = os.path.join(save_path, 'images') label_save_path = os.path.join(save_path, 'labels') check_dir(img_save_path) check_dir(label_save_path) labels_path = os.path.join(path, 'labels/bdd100k_labels_images_' + mode + '.json') with open(labels_path) as f: all_labels = json.load(f) names = [] rider_labels = [] ###### for labels in all_labels: filename = labels['name'] for label in labels['labels']: if label['category'] == "rider": names.append(filename) rider_labels.append(labels) break ##### index = 0 while index < len(rider_labels): filename = names[index] labels = rider_labels[index] bbox_labels = {"rider" : [], "bike" : [], "motor":[], "normal": []} for label in labels['labels']: category = label['category'] if category in trans_dict.keys() or category == 'rider': xmin = int(label['box2d']['x1']) ymin = int(label['box2d']['y1']) xmax = int(label['box2d']['x2']) ymax = int(label['box2d']['y2']) if category in bbox_labels.keys(): if category == 'rider': bbox_labels[category].append([xmin,ymin,xmax,ymax + 20]) else: bbox_labels[category].append([xmin,ymin,xmax,ymax]) else: bbox_labels['normal'].append([trans_dict[category], xmin, ymin, xmax, ymax]) datas = rider_iou_pair(bbox_labels) if datas == None: rider_labels.pop(index) names.pop(index) continue dest = os.path.join(label_save_path, filename[:-4] + ".txt") with open(dest, "w") as weiter: for bbox in datas + bbox_labels['normal']: bbox[1], bbox[2], bbox[3], bbox[4] = xy2cxcywh(bbox[1], bbox[2], bbox[3], bbox[4], x_scale, y_scale) bbox = tuple(bbox) msg = msg_format % bbox weiter.write(msg) shutil.copyfile(os.path.join(img_root, filename), os.path.join(img_save_path, filename)) index += 1
init_parser.add_argument('--model_name', type=str, default='BPR', help='Choose a model to run.') init_args, init_extras = init_parser.parse_known_args() model_name = eval('{0}.{0}'.format(init_args.model_name)) reader_name = eval('{0}.{0}'.format(model_name.reader)) runner_name = eval('{0}.{0}'.format(model_name.runner)) # Args parser = argparse.ArgumentParser(description='') parser = parse_global_args(parser) parser = reader_name.parse_data_args(parser) parser = runner_name.parse_runner_args(parser) parser = model_name.parse_model_args(parser) args, extras = parser.parse_known_args() # Logging configuration log_args = [init_args.model_name, args.dataset, str(args.random_seed)] for arg in ['lr', 'l2'] + model_name.extra_log_args: log_args.append(arg + '=' + str(eval('args.' + arg))) log_file_name = '__'.join(log_args).replace(' ', '__') if args.log_file == '': args.log_file = '../log/{}/{}.txt'.format(init_args.model_name, log_file_name) if args.model_path == '': args.model_path = '../model/{}/{}.pt'.format(init_args.model_name, log_file_name) utils.check_dir(args.log_file) logging.basicConfig(filename=args.log_file, level=args.verbose) logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) logging.info(init_args) main()
def save_model(self, model_path=None): if model_path is None: model_path = self.model_path utils.check_dir(model_path) logging.info('Save model to ' + model_path[:50] + '...')
plot_compared_distance_compressed(dataset_folders_dist, runs_img_dir, datasets_dist, 'Robot distances from goal', 'distances-from-goal-compressed-distributed') plot_compared_distance_compressed(dataset_folders_comm, runs_img_dir, datasets_comm, 'Robot distances from goal', 'distances-from-goal-compressed-communication') # Evaluate the learned controllers by passing a specific initial position configuration and compare them with # the omniscient and the manual controllers test_controller_given_init_positions(runs_img_dir, args.model, args.net_input) if args.generate_animations: from utils.my_plots import animate_simulation, plot_simulations, visualise_position_over_time from utils.utils import generate_fake_simulations, generate_init_positions, check_dir import numpy as np animations_dir = os.path.join(d, 'animations') check_dir(animations_dir) plots_dir = os.path.join(d, 'plots') check_dir(plots_dir) # Create a simulation for each of the controller using the same initial position if args.myt_quantity == 'variable': myt_quantities = np.arange(5, 11) else: myt_quantities = [int(args.myt_quantity)] for N in myt_quantities: dir = os.path.join(plots_dir, 'N%d' % N) check_dir(dir) _ = generate_fake_simulations(dir, args.model, N, simulations=100) out_dirs = [os.path.join(dir, 'omniscient'), os.path.join(dir, 'manual'), os.path.join(dir, 'distributed'), os.path.join(dir, 'communication')]