'learning_rate' : 1e-2,
'epochs' : 100
},
'finetune' : {
'batch_size' : 20,
'learning_rate' : 1e-2,
'epochs' : 100
}
}
}
### grid searchのパラメータ格納
params['STEP3']['brandcode'] = ['0101', '7203', '6758', '6502', '7201', '6501', '6702', '6753', '8058', '8031', '7751']
# params['STEP3']['brandcode'] = ['0101']
# params['STEP3']['brandcode'] = getNikkei225()
ALL_brandcodes = getNikkei225()
NG_brandcodes = ['2768', '3382', '3893', '4188', '4324', '4568', '4689', '4704', '5411', '6674', '8303', '8306', '8308', '8309', '8316', '8411', '8766', '8795', '9983', '6796', '9984', '6366', '2282', '7004', '7013', '9020', '9432']
brandcodes = list(set(ALL_brandcodes) - set(NG_brandcodes))
params['STEP3']['brandcode'] = brandcodes
params['STEP4']['hidden_layers_sizes'] = [
[1000]
# [500],
# [1000, 500]
# [params['STEP1']['n_hidden'] , params['STEP1']['n_hidden'], params['STEP1']['n_hidden'] / 2]
]
params['STEP4']['pretrain'] = {
'batch_size' : [100, 50],
lines = hadoop_res.readlines()
vectors = {"brand" : {}, "category" : {}}
def checkEnoughData(vec):
flag = True
threshold = 5
for element in vec:
print str(len(element[0])) + ', ' + str(len(element[1]) )
if (len(element[0]) < threshold) or (len(element[1]) < threshold):
flag = False
return flag
print 'start to make vectors'
a = 0
brandlist = nikkei225.getNikkei225()
for t in ['brand', 'category']:
for bc in brandlist:
a += 1
n = 0
utils.Progress.print_progress(a, len(brandlist) * 2)
vectors = [[], []], [[], []], [[], []] # train[train_x, train_y], valid[valid_x, valid_y], test[test_x, test_y]
c = 0
for line in lines:
# n += 1
# if utils.Progress.get_progress(n, len(lines) * 2):
# print '\t',
# utils.Progress.print_progress(n, len(lines))
type, brand_code, date, rate, wordid_list = line.strip().split('\t')
if type == t and brand_code == str(bc):
c += 1
'finetune': {
'batch_size': 20,
'learning_rate': 1e-2,
'epochs': 100
}
}
}
### grid searchのパラメータ格納
params['STEP3']['brandcode'] = [
'0101', '7203', '6758', '6502', '7201', '6501', '6702', '6753', '8058',
'8031', '7751'
]
# params['STEP3']['brandcode'] = ['0101']
# params['STEP3']['brandcode'] = getNikkei225()
ALL_brandcodes = getNikkei225()
NG_brandcodes = [
'2768', '3382', '3893', '4188', '4324', '4568', '4689', '4704', '5411',
'6674', '8303', '8306', '8308', '8309', '8316', '8411', '8766', '8795',
'9983', '6796', '9984', '6366', '2282', '7004', '7013', '9020', '9432'
]
brandcodes = list(set(ALL_brandcodes) - set(NG_brandcodes))
params['STEP3']['brandcode'] = brandcodes
params['STEP4']['hidden_layers_sizes'] = [
[1000]
# [500],
# [1000, 500]
# [params['STEP1']['n_hidden'] , params['STEP1']['n_hidden'], params['STEP1']['n_hidden'] / 2]
]
