from utils.settings_builder import Settings
# -----------------------------------------------------------------------------
# *****************
# *****************
json_path = "settings/settings_example.json"
json_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
json_path)
# *****************
# *****************
timestamp = datetime.datetime.fromtimestamp(int(
time.time())).strftime('%Y_%m_%d_%H_%M_%S')
s = Settings()
s.load(json_path)
base_path = s.base_path
version = s.config["version"]
# final merged output
merge_out_path = os.path.join(
base_path, "output/s1_merge/merge_{}_{}.csv".format(version, timestamp))
# find input data based on models version tag
regex_str = os.path.join(base_path,
"output/s1_train/train_*_{}.json".format(version))
regex_search = glob.glob(regex_str)
qlist = regex_search
# *****************
json_path = "settings/settings_example.json"
json_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
json_path)
# *****************
# *****************
print('-' * 40)
print("\nInitializing...")
timestamp = datetime.datetime.fromtimestamp(int(
time.time())).strftime('%Y_%m_%d_%H_%M_%S')
# date_str = datetime.datetime.now().strftime("%Y%m%d")
s = Settings()
s.load(json_path)
base_path = s.base_path
s.set_param_count()
s.build_dirs()
job_dir = os.path.basename(os.path.dirname(json_path))
shutil.copyfile(
json_path,
os.path.join(
base_path,
"output/s0_settings/settings_{}_{}.json".format(job_dir, timestamp)))
s.save_params()
tasks = s.hashed_iter()
from utils.settings_builder import Settings
from utils.data_prep import make_dir
from utils.create_grid import PointGrid
# from utils.load_ntl_data import NTL_Reader
# *****************
# *****************
json_path = "settings/settings_example.json"
json_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
json_path)
# *****************
# *****************
s = Settings()
s.load(json_path)
s.build_dirs()
boundary_path = os.path.join(s.base_path, "data/boundary",
s.data["static"]["grid_boundary_file"])
pixel_size = s.data["third_stage"]["grid"]["pixel_size"]
# ntl_calibrated = s.data["third_stage"]["predict"]["ntl_calibrated"]
# ntl_year = s.data["third_stage"]["predict"]["ntl_year"]
# ntl_dim = s.data["third_stage"]["predict"]["ntl_dim"]
surface_tag = s.config["surface_tag"]
# fname = os.path.basename(json_path, ".json")
fname = ".".join(os.path.basename(boundary_path).split(".")[:-1])
import sklearn.metrics
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from utils.settings_builder import Settings
# *****************
# *****************
json_path = "settings/nigeria_acled.json"
# json_path = "settings/settings_example.json"
# json_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), json_path)
# *****************
# *****************
s = Settings()
s.load(json_path)
predict_settings = s.data[s.config["predict"]]
predict_hash = s.build_hash(predict_settings, nchar=7)
tasks = s.hashed_iter()
def make_dir(path):
try:
os.makedirs(path)
except OSError as exception:
if exception.errno != errno.EEXIST:
raise
try:
return self.fp / float(self.fp + self.tn)
except:
return None
timestamp = datetime.datetime.fromtimestamp(int(
time.time())).strftime('%Y_%m_%d_%H_%M_%S')
json_path_list = glob.glob("../*acled/settings/nigeria_acled.json")
# group versions of same temporal
version_groups = {}
for json_path in json_path_list:
temporal = json_path.split("/")[1].split("_")[1]
s = Settings()
s.load(json_path)
tag = "{}_{}".format(s.config["version"], s.config["predict_tag"])
if temporal not in version_groups:
version_groups[temporal] = []
version_groups[temporal].append(json_path)
cm_list = []
thresh_val_list = [0.3, 0.35, 0.4, 0.45, 0.5]
for ix, temporal in enumerate(version_groups.keys()):
# ==============
# WARNING: THIS WILL NOT CURRENTLY WORK IF YOU HAVE MORE THAN ONE PARAM COMBO
# it will just put all the params from same temporal group in same plot
# ==============
from utils.settings_builder import Settings
from utils.model_prep import (pearson_r2, ModelHelper, run_cv, find_best_alpha,
predict_inner_test_fold, scale_features, train,
train_and_predict, run_models, run_tasks)
# *****************
# *****************
json_path = "settings/settings_example.json"
json_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
json_path)
# *****************
# *****************
s = Settings()
s.load(json_path)
base_path = s.base_path
s.build_dirs()
mode = s.config["second_stage_mode"]
predict_hash = s.build_hash(s.data[s.config["predict"]], nchar=7)
# timestamp = datetime.datetime.fromtimestamp(int(time.time())).strftime(
# '%Y_%m_%d_%H_%M_%S')
mh = ModelHelper(settings=s)
tasks = s.hashed_iter()