def run_unfinished(max_retries, tasks_module): from runner import DirectRunner from filenames import fn_formatter # user's fn_formatter from logs import setup_logger logger = setup_logger("rerunner", "%s/_logs/rerunner.txt" % os.environ["PREFIX"]) num_rerun = 0 for p in db.Pipeline.select(): pruns = db.PipelineRun.select() pruns = pruns.where(db.PipelineRun.pipeline_id == p.id and db.PipelineRun.scheduled) pruns = pruns.where((db.PipelineRun.state == RUNNING) or (db.PipelineRun.state == FAILED)) pruns = pruns.where(db.PipelineRun.retries <= max_retries) pruns = pruns.order_by(db.PipelineRun.start_time.desc()) for pr in pruns.limit(1): num_rerun += 1 ppl = Pipeline(p.id, pr.run_id) dst_task_ids = json.loads(ppl.final_tasks) kw = json.loads(ppl.args) logger.info("re-run %s (%s) with args %s" % (pr.run_id, ", ".join(dst_task_ids), ppl.args)) runner = DirectRunner(tasks_module, kw, fn_formatter(kw)) ppl.run(dst_task_ids, runner) logger.info("re-ran %i pipelines" % num_rerun)
config = configparser.ConfigParser() config.read('config.ini') optimizer_type = config['Parameter']['optimizer'] lr = float(config['Parameter']['lr']) batch_size = int(config['Parameter']['batch_size']) epochs = int(config['Parameter']['epochs']) train_size = int(config['Parameter']['train_size']) augumentation = config['Parameter'].getboolean('augumentation') clip = float(config['Parameter']['clip']) train_path = config['Paths']['train_path'] test_path = config['Paths']['test_path'] test_kvasir_path = f'{test_path}/Kvasir' model_name = config['Parameter']['model'] log_file = 'logs/train_' + model_name + '_' + datetime.now().strftime( '%Y%m%d%H') + '.log' train_logger = setup_logger('train_logger', log_file) model = ResNetPD().cuda() print('Params: ', get_n_params(model)) train_logger.info(f'Params: {get_n_params(model)}') train_logger.info( f'optimizer: {optimizer_type}, lr: {lr}, batch_size: {batch_size}, image_size: {train_size}' ) params = model.parameters() if optimizer_type == 'Adam': optimizer = torch.optim.Adam(params, lr) else: optimizer = torch.optim.SGD(params, lr, weight_decay=1e-4, momentum=0.9)
dice = 2 * TP / (num_obj + num_pred) specif = TN / (TN + FP) fmeasure = ((2.0 * pre * recall) / (pre + recall)) # beta = 1.0 return dice, iou if __name__ == '__main__': config = configparser.ConfigParser() config.read('config.ini') test_path = config['Paths']['test_path'] # specify model_result_path eval_result_path = config['Paths']['eval_result_path'] print(eval_result_path) log_file = 'logs/eval_cpd' + '.log' eval_logger = setup_logger('eval_logger', log_file) datasets = [ 'CVC-ClinicDB', 'CVC-ColonDB', 'ETIS-LaribPolypDB', 'Kvasir', 'CVC-300' ] # datasets = ['Kvasir', 'CVC-ClinicDB'] print('Model: ', eval_result_path) for data_name in datasets: image_root = f'{eval_result_path}/{data_name}/' gt_root = f'{test_path}/{data_name}/masks/' eval_loader = eval_dataset(image_root, gt_root) # thresholds 1:0:-1/255 thresholds = np.linspace(1, 0, 256) # thresholds = [0.5] threshold_dice, threshold_IoU = np.zeros( (eval_loader.size, len(thresholds))), np.zeros( (eval_loader.size, len(thresholds)))
if __name__ == '__main__': config = configparser.ConfigParser() config.read('config.ini') optimizer_type = config['Parameter']['optimizer'] lr = float(config['Parameter']['lr']) batch_size = int(config['Parameter']['batch_size']) epochs = int(config['Parameter']['epochs']) train_size = int(config['Parameter']['train_size']) augumentation = config['Parameter'].getboolean('augumentation') clip = float(config['Parameter']['clip']) train_path = config['Paths']['train_path'] test_path = config['Paths']['test_path'] test_kvasir_path = f'{test_path}/Kvasir' train_logger = setup_logger( 'train_logger', 'logs/train_prahardnet_0805.log') model = PraHarDNet().cuda() params = model.parameters() if optimizer_type == 'Adam': optimizer = torch.optim.Adam(params, lr) else: optimizer = torch.optim.SGD( params, lr, weight_decay=1e-4, momentum=0.9) image_root = '{}/images/'.format(train_path) gt_root = '{}/masks/'.format(train_path) train_loader = get_loader(image_root, gt_root, batchsize=batch_size, trainsize=train_size, augmentation=augumentation) total_step = len(train_loader)
if __name__ == '__main__': config = configparser.ConfigParser() config.read('config.ini') optimizer_type = config['Parameter']['optimizer'] lr = float(config['Parameter']['lr']) batch_size = int(config['Parameter']['batch_size']) epochs = int(config['Parameter']['epochs']) train_size = int(config['Parameter']['train_size']) augumentation = config['Parameter'].getboolean('augumentation') clip = float(config['Parameter']['clip']) train_path = config['Paths']['train_path'] test_path = config['Paths']['test_path'] test_kvasir_path = f'{test_path}/Kvasir' train_logger = setup_logger('train_logger', 'logs/train_mseg.log') model = SwinCPD().cuda() params = model.parameters() if optimizer_type == 'Adam': optimizer = torch.optim.Adam(params, lr) else: optimizer = torch.optim.SGD(params, lr, weight_decay=1e-4, momentum=0.9) image_root = '{}/images/'.format(train_path) gt_root = '{}/masks/'.format(train_path) train_loader = get_loader(image_root, gt_root, batchsize=batch_size, trainsize=train_size, augmentation=augumentation) total_step = len(train_loader) print('#'*20, 'Start Training', '#'*20)
DRY_RUN_ENABLED = "--dry-run" in sys.argv DEBUG_ENABLED = "--debug" in sys.argv MATCH_VODS_ONLY = "--match-vods-only" in sys.argv IGNORE_FILE_SIZE_AND_AGE = "--no-size-age" in sys.argv # Google APIs reset quota at midnight PT pacific_tz = pytz.timezone("America/Los_Angeles") ROOT_DIR = os.path.dirname(os.path.abspath(__file__ + "/..")) STATE_FILE_PATH = ROOT_DIR + "/data/state.json" CONFIG_FILE_PATH = ROOT_DIR + "/data/config.json" UPLOAD_HISTORY_PATH = ROOT_DIR + "/data/upload_history.txt" logger = setup_logger(debug_enabled=DEBUG_ENABLED) def watch_recordings_folder(google: dict): """ Watches the recodings folder for new video files to show up that need to be uploaded. Once a Twitch VOD corresponding to a video file is found, the video is uploaded using the metadata from the Twitch VOD as it's own. Refreshes the Twitch VOD information every twitch_vod_refresh_rate seconds (specified in config.json). If no YouTube API quota remains, sleeps until midnight PT (+ 10 minutes to be safe). """ logger.debug(f"config: {config}") folder_to_move_completed_uploads = config[
if __name__ == '__main__': config = configparser.ConfigParser() config.read('config.ini') optimizer_type = config['Parameter']['optimizer'] lr = float(config['Parameter']['lr']) batch_size = int(config['Parameter']['batch_size']) epochs = int(config['Parameter']['epochs']) train_size = int(config['Parameter']['train_size']) augumentation = config['Parameter'].getboolean('augumentation') clip = float(config['Parameter']['clip']) train_path = config['Paths']['train_path'] test_path = config['Paths']['test_path'] test_kvasir_path = f'{test_path}/Kvasir' train_logger = setup_logger('train_logger', 'logs/train_effnetv2_2704.log') model = EffNetV2SCPD() weights = torch.load('checkpoints/effnetv2scpd_') model.load_state_dict(weights) model = model.cuda() params = model.parameters() if optimizer_type == 'Adam': optimizer = torch.optim.Adam(params, lr) else: optimizer = torch.optim.SGD(params, lr, weight_decay=1e-4, momentum=0.9) image_root = '{}/images/'.format(train_path)