def target(ostream, params, parallel_tests, sorted_mode): global PROCESSED, REPORT_LOCK, TOTAL for result in process(params, pool_size=parallel_tests, sorted_mode=sorted_mode): with REPORT_LOCK: PROCESSED += 1 print_progress_info(PROCESSED, TOTAL) save_result(ostream, strip_ephemeral(result))
def target(ostream, params, parallel_tests): global PROCESSED, REPORT_LOCK, TOTAL for result in process(params, pool_size=parallel_tests): with REPORT_LOCK: PROCESSED += 1 print_progress_info(PROCESSED, TOTAL) save_result(ostream, strip_ephemeral(result))
def main(conf, istream, ostream, test_whitelist, test_blacklist, stage_whitelist, stage_blacklist, tags_whitelist, tags_blacklist, no_action): ''' main worker function performs particular stages handling generates particular stage/test result list ''' params = dict(test_whitelist=test_whitelist, test_blacklist=test_blacklist, stage_whitelist=stage_whitelist, stage_blacklist=stage_blacklist, tags_whitelist=tags_whitelist, tags_blacklist=tags_blacklist, enabled=not no_action) params = load(istream, config_file=conf, augment=params) total = required_actions_count(params) processed = 0 print_progress_info(processed, total) for item in params: for result in process(item): processed += 1 save_result(ostream, strip_ephemeral(result)) print_progress_info(processed, total)
def main(conf, istream, ostream, test_whitelist, test_blacklist, stage_whitelist, stage_blacklist, tags_whitelist, tags_blacklist, no_action, parallel_instances=1): ''' main worker function performs particular stages handling generates particular stage/test result list ''' params = dict(test_whitelist=test_whitelist, test_blacklist=test_blacklist, stage_whitelist=stage_whitelist, stage_blacklist=stage_blacklist, tags_whitelist=tags_whitelist, tags_blacklist=tags_blacklist, enabled=not no_action) params = load(istream, config_file=conf, augment=params) progress_info = ProgressInfo(params) tests = [] for item in params: for test_stage in item['test_stages']: for test in item['test_stages'][test_stage]: tests.append( dict(test=dict(name=test, stage=test_stage), **item)) from gevent.pool import Pool pool = Pool(size=parallel_instances) for result in pool.imap_unordered( lambda test: process(progress_info, test), tests): save_result(ostream, strip_ephemeral(result))
def run_analyse(script, codes, start, end): open_time = "09:30:00" close_time = "15:00:00" start_time = "{0} {1}".format(start, open_time) end_time = "{0} {1}".format(end, close_time) sim_params = create_simulation_parameters( start=pd.to_datetime(start_time).tz_localize("Asia/Shanghai").tz_convert("UTC"), end=pd.to_datetime(end_time).tz_localize("Asia/Shanghai").tz_convert("UTC"), data_frequency="daily", emission_rate="daily", sids=codes) with open(script, 'r') as f: algo_text = f.read() zp_algo = zipline.TradingAlgorithm(script=algo_text, namespace={}, capital_base=10e6, sim_params=sim_params) stocks = Market.get_stocks(codes, start, end) d = pd.Panel(stocks) res = zp_algo.run(d) results = {} results['parameters'] = { 'time': datetime.datetime.now(), 'algorithm': script, } results['results'] = res results['report'] = zp_algo.risk_report results['orders'] = zp_algo.blotter.orders results['benchmark'] = zp_algo.perf_tracker.all_benchmark_returns job = get_current_job(connection=Redis()) data.save_result(job.id, results) return results
def main(conf, istream, ostream, test_whitelist, test_blacklist, stage_whitelist, stage_blacklist, tags_whitelist, tags_blacklist, no_action, parallel_instances=1): ''' main worker function performs particular stages handling generates particular stage/test result list ''' params = dict(test_whitelist=test_whitelist, test_blacklist=test_blacklist, stage_whitelist=stage_whitelist, stage_blacklist=stage_blacklist, tags_whitelist=tags_whitelist, tags_blacklist=tags_blacklist, enabled=not no_action) params = load(istream, config_file=conf, augment=params) progress_info = ProgressInfo(params) tests = [] for item in params: for test_stage in item['test_stages']: for test in item['test_stages'][test_stage]: tests.append(dict(test=dict(name=test, stage=test_stage), **item)) from gevent.pool import Pool pool = Pool(size=parallel_instances) for result in pool.imap_unordered(lambda test: process(progress_info, test), tests): save_result(ostream, strip_ephemeral(result))
] # Training history_dilated = model_dilated.fit_generator( generator=trainGen, steps_per_epoch=NUM_TRAINING_STEP, validation_data=valGen, validation_steps=NUM_VALIDATION_STEP, epochs=NUM_EPOCH, callbacks=callbacks) print("Predict and save results...") print("...For U-Net with 32 filters...") testGene = testGenerator(test_path) result_1 = model_32.predict_generator(testGene, TEST_SIZE, verbose=1) print("...For U-Net with 64 filters...") testGene = testGenerator(test_path) result_2 = model_64.predict_generator(testGene, TEST_SIZE, verbose=1) print("...For U-Net with dilated convolution...") testGene = testGenerator(test_path) result_3 = model_dilated.predict_generator(testGene, TEST_SIZE, verbose=1) print("...Averaging the prediction results...") result = (result_1 + result_2 + result_3) / 3 save_result(predict_path, result) print("Make submission...") make_submission(predict_path, test_size=TEST_SIZE, submission_filename=os.path.join(submission_path, "submission.csv")) print("Done!")
# If the --verbose argument is not supplied, suppress all of the TensorFlow startup messages. if not args.verbose: import os os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3" import tensorflow as tf tf.logging.set_verbosity(tf.logging.ERROR) import models import data from datetime import datetime from keras.callbacks import TensorBoard, ModelCheckpoint, EarlyStopping from keras.optimizers import Adam # Compile the model model = models.unet2D(size=args.size, ablated=args.ablated) model.compile(optimizer=Adam(lr=0.0001), loss="binary_crossentropy", metrics=["accuracy"]) model.load_weights(args.weights) # Process the images in the test set and save the results to the test/ directory. test_gen = data.test_generator(f"{args.dir}", num_image=args.tests, target_size=(args.size, args.size)) results = model.predict_generator(test_gen, args.tests, verbose=1) # Use the custom save_result() method defined in data.py to save the results as # greyscale .png images. print("Saving results in ./test/") data.save_result("test", results)
else: pth = Path('data') data_gen_args = dict(rotation_range=0.2, width_shift_range=0.05, height_shift_range=0.05, shear_range=0.05, zoom_range=0.05, horizontal_flip=True, fill_mode='nearest') myGene = train_generator(batch_size=2, train_path=pth / 'membrane/train', image_folder='image', mask_folder='label', aug_dict=data_gen_args, save_to_dir=None) model = unet() model_checkpoint = ModelCheckpoint('unet_membrane.hdf5', monitor='loss', verbose=1, save_best_only=True) model.fit_generator(myGene, steps_per_epoch=300, epochs=1, callbacks=[model_checkpoint]) testGene = test_generator(test_path=pth / 'membrane/test') results = model.predict_generator(testGene, 30, verbose=1) save_result(pth / 'membrane/test', results)
else: print('File {} does not exists'.format(filename)) print(len(test_imgs)) predict_path = "predict_images" submission_path = "submission" weight_path = "weights" weight_list = ["weights_32.h5", "weights_64.h5", "weights_dilated.h5" ] # weight_list = ["weights_32_dice.h5"] print("Check weights...") missing_weight = list(set(weight_list) - set(os.listdir(weight_path))) if len(missing_weight): raise FileNotFoundError("Can not find: " + str(missing_weight)) print("Load models and predict...") results = 0 for w in weight_list: print("...Load " + w + "...") model = load_model(os.path.join(weight_path, w), custom_objects={"dice_loss": dice_loss, "f1": f1}) print("...Predict...") testGene = testGenerator(test_imgs) results += model.predict_generator(testGene, TEST_SIZE, verbose=1) results /= len(weight_list) save_result(predict_path, results, test_index) print("Make submission...") make_submission(predict_path, test_size=TEST_SIZE, indices=test_index, submission_filename=os.path.join(submission_path, "submission.csv")) print("Done!")
print(f" Augmentations: \n{indent(pformat(data_gen_args), ' '*18)}") train, validate = train_generator(batch_size, 'data/filament/train', 'image', 'label', data_gen_args, save_to_dir=None) model = unet(learning_rate=learning_rate) model_checkpoint = tf.keras.callbacks.ModelCheckpoint(str( output_dir / 'unet_filament.hdf5'), monitor='val_loss', verbose=1, save_best_only=True) es = tf.keras.callbacks.EarlyStopping(monitor='val_loss', verbose=1, patience=10) tensorboard = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1, write_images=True) model.fit(train, steps_per_epoch=steps_per_epoch, epochs=epochs, callbacks=[model_checkpoint, es, tensorboard], validation_data=validate, validation_steps=1) testGene = test_generator("data/filament/test") results = model.predict(testGene, verbose=1) save_result(output_dir, results)
predict_path = "predict_images" submission_path = "submission" weight_path = "weights" weight_list=["weights_segnet.h5", "weights_unet.h5", "weights_dunet.h5","weights_unet_attention.h5" ,"weights_linknet.h5","weights_dlinknet.h5",'weights_linknet_attention.h5'] method=int(input("choose method:1.validate single modle 2.calculate the voting")) vote_result=0 if method==1: num=int(input("choose model number you want to validate: 1.segnet 2.unet 3. dunet 4.unet_attention 5.linknet 6.dlinknet 7.link_attention")) w=weight_list[num-1] model = load_model(os.path.join(weight_path, w), custom_objects={"dice_loss": dice_loss, "f1": f1}) test = testGenerator(test_path) predict_result= model.predict_generator(test, TEST_SIZE, verbose=1) print(type(predict_result)) save_result(predict_path, predict_result) if method==2: print("calculate the voting of different models") for w in weight_list: model= load_model(os.path.join(weight_path, w), custom_objects={"dice_loss": dice_loss, "f1": f1}) test=testGenerator(test_path) vote_result+=model.predict_generator(test, TEST_SIZE, verbose=1) save_result(predict_path,vote_result) print("make submission") make_submission(predict_path, test_size=TEST_SIZE, submission_filename=os.path.join(submission_path, "submission.csv"))