def __init__(self, data_set, abits, wbits, network_type, seed):
self.network_type = network_type
self.abits = abits
self.wbits = wbits
self.data_set = data_set
self.seed = seed
self.model = Sequential()
cfDeep = self.myCF(self)
if self.data_set == 'mnist':
cfg = 'config_MNIST'
if self.data_set == 'fashion':
cfg = 'config_FASHION'
if self.data_set == 'cifar10':
cfg = 'config_CIFAR-10'
self.cf = Config(cfg, cmd_args=cfDeep.myDict)
print("Dataset: " + str("%s_pic/" % self.data_set))
assure_path_exists("%s_pic/" % self.data_set)
default=None,
help='Configuration file')
parser.add_argument('-o', '--override', action='store', nargs='*', default=[])
arguments = parser.parse_args()
override_dir = {}
for s in arguments.override:
s_s = s.split("=")
k = s_s[0].strip()
v = "=".join(s_s[1:]).strip()
override_dir[k] = v
arguments.override = override_dir
cfg = arguments.config_path
cf = Config(cfg, cmd_args=arguments.override)
# if necessary, only use the CPU for debugging
if cf.cpu:
os.environ["CUDA_VISIBLE_DEVICES"] = ""
else:
os.environ["CUDA_VISIBLE_DEVICES"] = cf.cuda
# ## Construct the network
print('Construct the Network\n')
model = build_model(cf)
print('loading data\n')
train_data, val_data, test_data = load_dataset(cf.dataset, cf)
print('setting up the network and creating callbacks\n')
from sklearn import preprocessing
import scipy.special as special
from pandas import DataFrame, Series
from tqdm import tqdm
import time
# import sys
# sys.path.extend('../')
from utils.data_utils import preprocess
from utils.config_utils import Config
from sklearn.feature_extraction.text import TfidfVectorizer
import scipy.io as scio
cfg = Config()
np.random.seed(cfg.seed)
random.seed(cfg.seed)
"""
Feature Extraction Tools
TF-IDF + W2V + Multi-label + Onehot + Click multiply + Time Sequence + Shuffle
"""
def tfidf(log, pivot_key, out_key, flag, max_df=0.99):
"""
TF-IDF Features
override_dir = {}
#arguments.override=
#for s in arguments.override:
# s_s = s.split("=")
# k = s_s[0].strip()
# v = "=".join(s_s[1:]).strip()
# override_dir[k]=v
#arguments.override = override_dir
override_dir['lr'] = 0.01
override_dir['wbits'] = 4
override_dir['abits'] = 4
override_dir['network_type'] = 'full-qnn'
#config_oath
cfg = "config_CIFAR-10"
cf = Config(cfg, cmd_args=override_dir)
# if necessary, only use the CPU for debugging
#if cf.cpu:
# os.environ["CUDA_VISIBLE_DEVICES"] = ""
# ## Construct the network
print('Construct the Network\n')
# In[4]:
model = build_model(cf)
print('setting up the network and creating callbacks\n')
early_stop = EarlyStopping(monitor='loss',
min_delta=0.001,
print(checkpoint_name)
#print(len(name_list))
#exit()
train_from_config(lr,
batch_size,
num_nodes,
dataset_size,
teacher_forcing,
checkpoint_name,
log_dir_num,
log_dir_path,
train_option,
sys.argv)
log_dir_num += 1
#print(checkpoint_name_idx)
if checkpoint_name_idx < len(name_list)-1:
checkpoint_name_idx += 1
else:
checkpoint_name_idx = 0
if __name__=="__main__":
config_path = os.getcwd()
config = Config(config_path)
yml_args = config.config_parse_yaml()
sess_args = flat_dict(yml_args)
train_many_jobs(sess_args)
