Esempio n. 1
0
from utils.settings_builder import Settings

# -----------------------------------------------------------------------------

# *****************
# *****************
json_path = "settings/settings_example.json"
json_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                         json_path)
# *****************
# *****************

timestamp = datetime.datetime.fromtimestamp(int(
    time.time())).strftime('%Y_%m_%d_%H_%M_%S')

s = Settings()
s.load(json_path)
base_path = s.base_path

version = s.config["version"]

# final merged output
merge_out_path = os.path.join(
    base_path, "output/s1_merge/merge_{}_{}.csv".format(version, timestamp))

# find input data based on models version tag
regex_str = os.path.join(base_path,
                         "output/s1_train/train_*_{}.json".format(version))
regex_search = glob.glob(regex_str)

qlist = regex_search
Esempio n. 2
0
# *****************
json_path = "settings/settings_example.json"
json_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                         json_path)
# *****************
# *****************

print('-' * 40)
print("\nInitializing...")

timestamp = datetime.datetime.fromtimestamp(int(
    time.time())).strftime('%Y_%m_%d_%H_%M_%S')

# date_str = datetime.datetime.now().strftime("%Y%m%d")

s = Settings()
s.load(json_path)
base_path = s.base_path
s.set_param_count()
s.build_dirs()

job_dir = os.path.basename(os.path.dirname(json_path))
shutil.copyfile(
    json_path,
    os.path.join(
        base_path,
        "output/s0_settings/settings_{}_{}.json".format(job_dir, timestamp)))

s.save_params()
tasks = s.hashed_iter()
Esempio n. 3
0
from utils.settings_builder import Settings
from utils.data_prep import make_dir

from utils.create_grid import PointGrid
# from utils.load_ntl_data import NTL_Reader

# *****************
# *****************
json_path = "settings/settings_example.json"
json_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                         json_path)
# *****************
# *****************

s = Settings()
s.load(json_path)
s.build_dirs()

boundary_path = os.path.join(s.base_path, "data/boundary",
                             s.data["static"]["grid_boundary_file"])

pixel_size = s.data["third_stage"]["grid"]["pixel_size"]

# ntl_calibrated = s.data["third_stage"]["predict"]["ntl_calibrated"]
# ntl_year = s.data["third_stage"]["predict"]["ntl_year"]
# ntl_dim = s.data["third_stage"]["predict"]["ntl_dim"]

surface_tag = s.config["surface_tag"]
# fname = os.path.basename(json_path, ".json")
fname = ".".join(os.path.basename(boundary_path).split(".")[:-1])
Esempio n. 4
0
import sklearn.metrics
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt

from utils.settings_builder import Settings

# *****************
# *****************
json_path = "settings/nigeria_acled.json"
# json_path = "settings/settings_example.json"
# json_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), json_path)
# *****************
# *****************

s = Settings()
s.load(json_path)

predict_settings = s.data[s.config["predict"]]
predict_hash = s.build_hash(predict_settings, nchar=7)

tasks = s.hashed_iter()


def make_dir(path):
    try:
        os.makedirs(path)
    except OSError as exception:
        if exception.errno != errno.EEXIST:
            raise
        try:
            return self.fp / float(self.fp + self.tn)
        except:
            return None


timestamp = datetime.datetime.fromtimestamp(int(
    time.time())).strftime('%Y_%m_%d_%H_%M_%S')

json_path_list = glob.glob("../*acled/settings/nigeria_acled.json")

# group versions of same temporal
version_groups = {}
for json_path in json_path_list:
    temporal = json_path.split("/")[1].split("_")[1]
    s = Settings()
    s.load(json_path)
    tag = "{}_{}".format(s.config["version"], s.config["predict_tag"])
    if temporal not in version_groups:
        version_groups[temporal] = []
    version_groups[temporal].append(json_path)

cm_list = []

thresh_val_list = [0.3, 0.35, 0.4, 0.45, 0.5]

for ix, temporal in enumerate(version_groups.keys()):
    # ==============
    # WARNING: THIS WILL NOT CURRENTLY WORK IF YOU HAVE MORE THAN ONE PARAM COMBO
    # it will just put all the params from same temporal group in same plot
    # ==============
Esempio n. 6
0
from utils.settings_builder import Settings

from utils.model_prep import (pearson_r2, ModelHelper, run_cv, find_best_alpha,
                              predict_inner_test_fold, scale_features, train,
                              train_and_predict, run_models, run_tasks)

# *****************
# *****************
json_path = "settings/settings_example.json"
json_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                         json_path)
# *****************
# *****************

s = Settings()
s.load(json_path)
base_path = s.base_path
s.build_dirs()

mode = s.config["second_stage_mode"]

predict_hash = s.build_hash(s.data[s.config["predict"]], nchar=7)

# timestamp = datetime.datetime.fromtimestamp(int(time.time())).strftime(
#     '%Y_%m_%d_%H_%M_%S')

mh = ModelHelper(settings=s)

tasks = s.hashed_iter()