from deepcovidnet.config.base_config import Config
import sys
import os, getpass
from datetime import datetime
from pathlib import Path
config = Config('BaseRunnerConfig')
config.print_freq = 500
config.intermittent_output_freq = 5 # Num batches between outputs
config.save_freq = 5
file_dir = os.path.dirname(os.path.abspath(__file__))
config.get_tensorboard_dir = \
lambda exp_name: os.path.join(
Path(file_dir).parent.parent,
'runs',
datetime.now().strftime('%b%d_%H-%M') + '_' + exp_name
)
config.models_base_dir = os.path.join(Path(file_dir).parent.parent, 'models')
config.min_save_acc = 0.730
if not os.path.exists(config.models_base_dir):
os.mkdir(config.models_base_dir)
sys.modules[__name__] = config
from deepcovidnet.config.base_config import Config
import sys
import deepcovidnet.config.global_config as global_config
import os
from datetime import timezone, date, timedelta
from pandas.tseries import offsets
import pandas as pd
import pickle
config = Config('Config for RawFeatureExtractor')
class ReaderConfig(Config):
def __init__(self,
file_granularity: str,
file_path_format: str,
is_timezone_variable: bool,
timezone: timezone = None,
part_prefix: str = None,
file_extension='.csv'):
super(ReaderConfig,
self).__init__('Config for reading data from different mediums')
assert file_granularity in ['daily', 'monthly', 'weekly']
self.file_granularity = file_granularity
if self.file_granularity == 'daily':
self.date_offset = timedelta(days=1)
elif self.file_granularity == 'monthly':
from deepcovidnet.config.base_config import Config
import sys
import os
from datetime import date, datetime
config = Config('Global config parameters')
data_base_dir = '/data'
data_save_dir = '/saved_covid_data'
config.set_static_val('data_base_dir', data_base_dir)
config.set_static_val('data_save_dir', data_save_dir)
config.data_start_date = date(2020, 4, 5)
config.data_end_date = date(2020, 6, 29)
config.train_end_date = date(2020, 6, 2)
config.val_end_date = date(2020, 6, 12)
def get_best_tune_file(exp_name):
now = datetime.now()
fl = f'{now.year}-{now.month}-{now.day}_{now.hour}-{now.minute}_{exp_name}.pickle'
dr = os.path.join(config.data_save_dir, 'tunes')
if not os.path.exists(dr):
os.mkdir(dr)
return os.path.join(dr, fl)
config.get_best_tune_file = get_best_tune_file
from deepcovidnet.config.base_config import Config
import sys
import pandas as pd
import os
import deepcovidnet.config.global_config as global_config
config = Config('general features config parameters')
config.counties_save_path = os.path.join(global_config.data_save_dir,
'features_config_counties.csv')
def get_county_info(county_info_link):
if os.path.exists(config.counties_save_path):
df = pd.read_csv(config.counties_save_path, dtype={
'FIPS': str
}).set_index('FIPS')
return df
df = pd.read_html(county_info_link)[0].iloc[:-1].set_index('FIPS')
duplicates = df.loc[df.index.duplicated()].index
df = df.drop(duplicates)
df = df.drop(df.index[df.index.str.startswith(('6', '7'))])
if not os.path.exists(config.counties_save_path):
df.to_csv(config.counties_save_path)
return df
def get_iloc_to_county(county_df):
from deepcovidnet.config.base_config import Config
import sys
import deepcovidnet.config.global_config as global_config
import os
import logging
from datetime import date
config = Config('Config for CovidCountyDataset')
config.labels_key = 'labels'
config.labels_class_boundaries = [1, 13, 93] # 0.31, 0.67, 0.9: 4/5 - 6/29
# [2, 11, 80] # 0.33, 0.67, 0.9 percentiles 1/28 - 6/7
config.label_to_str_range = {}
config.label_to_range = {}
for i in range(len(config.labels_class_boundaries) + 1):
low, high = 0, None
if i > 0:
low = config.labels_class_boundaries[i - 1] + 1
if i < len(config.labels_class_boundaries):
high = config.labels_class_boundaries[i]
config.label_to_str_range[i] = f'{low}{"+" if high is None else f"-{high}"}'
config.label_to_range[i] = (low, high)
# assert global_config.data_start_date == date(2020, 4, 5), 'change label_class_boundaries accordingly before use or explicitly comment this out'
# assert global_config.data_end_date == date(2020, 6, 29), 'change label_class_boundaries accordingly or explicitly comment this out'
config.num_classifiers = len(config.labels_class_boundaries)
config.num_classes = len(config.labels_class_boundaries) + 1
from deepcovidnet.config.base_config import Config
import sys
import deepcovidnet.config.global_config as global_config
import os
from datetime import timezone, date, timedelta
from pandas.tseries import offsets
config = Config('GeometryUtilityConfig')
config.core_poi_apr_data_path = os.path.join(
global_config.data_base_dir,
"core_places/CoreApr2020Release-CORE_POI-2020_03-2020-04-07/")
config.svi_data_us_county_data_path = os.path.join(global_config.data_base_dir,
"SVI2018_US_COUNTY/")
sys.modules[__name__] = config
from deepcovidnet.config.base_config import Config
import sys
import os
from datetime import date, datetime
config = Config('Global config parameters')
# data_base_dir = 'data'
# data_save_dir = 'saved_covid_data'
data_base_dir = '/home/lews/PycharmProjects/covid_deepcovidnet/deepcovidnet/data'
data_save_dir = '/home/lews/PycharmProjects/covid_deepcovidnet/deepcovidnet/saved_covid_data'
config.set_static_val('data_base_dir', data_base_dir)
config.set_static_val('data_save_dir', data_save_dir)
# config.data_start_date = date(2020, 4, 5)
# config.data_end_date = date(2020, 6, 29)
#
# config.train_end_date = date(2020, 6, 2)
# config.val_end_date = date(2020, 6, 12)
# see CovidCountryDataset how actual start and end are calculated (depends on hyperparameters)
# projection days = 7
# past days to consider = 13
config.data_start_date = date(2020, 1, 1 + 13 + 7)
config.data_end_date = date(2020, 9, 11)
config.train_end_date = date(2020, 7, 1)
config.val_end_date = date(2020, 8, 1)
from deepcovidnet.config.base_config import Config
import sys
config = Config('Config for CountyWiseTimeDependentFeatures')
config.cross_type = 'CROSS'
config.const_type = 'CONSTANT'
config.types = [config.cross_type, config.const_type]
sys.modules[__name__] = config
from deepcovidnet.config.base_config import Config
import sys
import deepcovidnet.config.global_config as global_config
import datetime
import os
config = Config('Config for DataSaver')
class FeatureSaver:
def __init__(self, root, keyword, time_dependence=True):
self.root = os.path.join(global_config.data_save_dir, root)
self.keyword = keyword
self.time_dependence = time_dependence
if self.time_dependence:
self.file_format = f'%Y-%m-%d-{self.keyword}.csv'
self.mean_path = os.path.join(self.root,
f'{self.keyword}-mean.pickle')
self.std_path = os.path.join(self.root,
f'{self.keyword}-std.pickle')
self.save_file = None
else:
self.file_format = f'{self.keyword}.csv'
self.mean_path = None
self.std_path = None
self.save_file = os.path.join(self.root, f'{self.keyword}.csv')
def get_file_func(self):
return lambda d: d.strftime(os.path.join(self.root, self.file_format))
from deepcovidnet.config.base_config import Config
import sys
from enum import Enum, auto
config = Config('RawFeatures config parameters')
class FeatureType(Enum):
CONSTANTS = auto() # time independent
TIME_DEPENDENT = auto()
COUNTY_WISE_TIME_DEPENDENT = auto()
config.feature_type = FeatureType
sys.modules[__name__] = config
from deepcovidnet.config.base_config import Config
import deepcovidnet.config.global_config as global_config
import os, sys
config = Config('Global config parameters')
config.training_mean_std_file = os.path.join(global_config.data_save_dir,
'train_mean_std.pickle')
config.get_spatial_csv = lambda dt: os.path.join(global_config.data_save_dir,
f'viz/spatial_{dt}.csv')
sys.modules[__name__] = config
from deepcovidnet.config.base_config import Config
import deepcovidnet.config.global_config as global_config
import sys
from datetime import datetime
import os
config = Config('FeatureAnalyzerConfig')
def get_ranks_file(exp):
n = datetime.now()
fl = f'rank_{exp}_{str(n.date())}_{n.hour}-{n.minute}.csv'
dr = os.path.join(global_config.data_save_dir, 'ranks')
if not os.path.exists(dr):
os.mkdir(dr)
return os.path.join(dr, fl)
config.get_ranks_file = get_ranks_file
sys.modules[__name__] = config
from deepcovidnet.config.base_config import Config
import sys
config = Config('Config for Hyperparameters')
config.total_trials = 30
sys.modules[__name__] = config