def perform_validation_on_ft_esdr(db, fpaths, mask=None):
for f in fpaths:
validate_file_path(f)
pf = WMOValidationPointFetcher(db)
pg = PointsGridder(*eg.v1_get_full_grid_lonlat(eg.ML), invalid_mask=mask)
data = load_ft_esdr_data_from_files(fpaths)
dates_am = [d.dt for d in data if d.am_grid is not None]
dates_pm = [d.dt for d in data if d.pm_grid is not None]
grids_am = [d.am_grid for d in data if d.am_grid is not None]
grids_pm = [d.pm_grid for d in data if d.pm_grid is not None]
grids_am = {k: v for k, v in zip(dates_am, grids_am)}
grids_pm = {k: v for k, v in zip(dates_pm, grids_pm)}
perform_default_am_pm_validation(grids_am, grids_pm, pf, pg)
def main(args):
cfile = os.path.join(args.target_dir, "config.yaml")
if args.config is not None:
# Already validated by parser
cfile = args.config
else:
try:
utils.validate_file_path(cfile)
except IOError:
cfile = os.path.join(args.target_dir, "config")
utils.validate_file_path(cfile)
config = load_config(cfile)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
transform = REGION_TO_TRANS[config.region]
lon, lat = [transform(i) for i in eg.v1_get_full_grid_lonlat(eg.ML)]
model_path = os.path.join(args.target_dir, "model.pt")
utils.validate_file_path(model_path)
model_class = UNet
if args.legacy_model:
print("Using legacy model")
model_class = UNetLegacy
model = model_class(
in_chan=config.in_chan,
n_classes=config.n_classes,
depth=config.depth,
base_filter_bank_size=config.base_filters,
skip=config.skips,
bndry_dropout=config.bndry_dropout,
bndry_dropout_p=config.bndry_dropout_p,
)
if torch.cuda.device_count() > 1:
model = torch.nn.DataParallel(model)
model = model.to(device)
model_dict = torch.load(model_path)
model_load(model, model_dict)
model.eval()
input_ds = build_input_dataset_form_config(config, False, lat)
batch_size = args.batch_size if args.batch_size > 0 else config.batch_size
input_dl = torch.utils.data.DataLoader(input_ds,
batch_size=batch_size,
shuffle=False,
drop_last=False)
preds, probs = get_predictions(
input_dl,
model,
transform(np.load("../data/masks/ft_esdr_water_mask.npy")),
LABEL_OTHER,
device,
config,
)
# TODO
pred_path = os.path.join(args.target_dir, "pred.npy")
prob_path = os.path.join(args.target_dir, "prob.npy")
print(f"Saving probabilities: '{prob_path}'")
np.save(prob_path, probs)
if args.save_predictions:
print(f"Saving predictions: '{pred_path}'")
np.save(preds, pred_path)
def set_filepaths(self, filepaths):
for filepath in filepaths:
validate_file_path(filepath)
self.filepaths = filepaths
#!/usr/bin/env python3
import yaml
from utils import validate_file_path
def set_bungee_mode(file_path: str):
with open(file_path, 'r') as cfg:
spig_yml = yaml.safe_load(cfg)
# except yaml.YAMLError as exc:
spig_yml['settings']['bungeecord'] = True
with open(file_path, 'w') as cfg:
yaml.dump(spig_yml, cfg)
if __name__ == '__main__':
path = validate_file_path('spigot.yml')
set_bungee_mode(path)
config = configparser.ConfigParser()
config.read_string(config_string)
config['placeholder']['online-mode'] = 'false'
if localhost:
config['placeholder']['server-ip'] = "127.0.0.1"
# Write contents to mock IO
with StringIO("") as sIO: # Trick into giving us config as string
config.write(sIO, space_around_delimiters=False)
sIO.seek(0)
new_config_str = sIO.read()
new_config_str = new_config_str[len(placeholder):] # Remove placeholder
with open(file_path, 'w') as cfg:
cfg.write(new_config_str)
if __name__ == '__main__':
import sys
args = sys.argv
localhost = False
if len(args) >= 3 and str(args[2]).lower() == 'localhost':
localhost = True
path = validate_file_path('server.properties')
set_offline_mode(path, localhost)
def load_persisted_data_object(path):
utils.validate_file_path(path)
with open(path, "rb") as fd:
return pickle.load(fd)
def __init__(self, path, creation_func):
utils.validate_file_path(path)
self.path = path
self.creation_func = creation_func
def __init__(self, path, n):
self.path = utils.validate_file_path(path)
self.n = n
self.data = None