def test_data_gaps(self):
test_csv_filepath = "little_melbourne_has_gaps.csv"
with open(os.path.join(CONFIG_DIR, "tmy-config.json")) as f:
config = json.load(f)
d = tmy.loadBomCsvFile(test_csv_filepath, config["params"])
d_no_null = tmy.removeMonthsWithNulls(config["params"].keys(), d)
validator = validate_data.DataValidator(d, d_no_null, False, 9)
self.assertFalse(validator._dataHasNoGaps())
def test_data_timestamps_not_monotonic_increase(self):
test_csv_filepath = "little_melbourne_no_monotonic.csv"
with open(os.path.join(CONFIG_DIR, "tmy-config.json")) as f:
config = json.load(f)
d = tmy.loadBomCsvFile(test_csv_filepath, config["params"])
d_no_null = tmy.removeMonthsWithNulls(config["params"].keys(), d)
validator = validate_data.DataValidator(d, d_no_null, False, 9)
self.assertFalse(validator._timestampsAreMonotonicIncreasing())
def test_sufficient_data_3(self):
# 1 has nine or more but rest have 1 (ok)
test_csv_filepath = "sufficient_data_03.csv"
with open(os.path.join(CONFIG_DIR, "tmy-config.json")) as f:
config = json.load(f)
config["verbose"] = True
d = tmy.loadBomCsvFile(test_csv_filepath, config["params"])
d_no_null = tmy.removeMonthsWithNulls(config["params"].keys(), d)
validator = validate_data.DataValidator(d, d_no_null, False, 9)
self.assertTrue(validator._sufficientDataAvailable())
def doOne(args, config, continue_on_fail):
d = loadBomCsvFile(config['bomfile'], config['params'])
# Keeping this in addition to d, because we want to report on any nulls that exist to the user.
d_no_nulls = removeMonthsWithNulls(config["params"].keys(), d)
validator = validate_data.DataValidator(d, d_no_nulls, config["verbose"],
config["min_years_required"])
success = validator.validate()
if not success:
if not continue_on_fail:
print("Cannot continue with invalid data. Exiting...")
sys.exit(1)
else:
return (None, None, None, None)
tmys = calculateTmy(d_no_nulls, config)
merged_csv_filepath = mergeMonths(d, tmys, config)
station_num = int(d.loc[:, config["params"]["station"]][-1])
return (tmys, merged_csv_filepath, validator, station_num)