def gen_data(cfg):
files = glob.glob(os.path.join(cfg['location'].format(HOME), 'L_*.csv'))
print len(files)
ids = l_utils.rdm_ids(files)
file2 = []
for f in ids:
if ids[f] > -1:
file2.append(f)
gen = 5
step = 10000
SEG = cfg['SEG']
for f in files:
basename = os.path.basename(f)[:-4]
basename = os.path.join(HOME, 'tmp', basename)
x, y = l_utils.load_data(f)
t0 = y[0]
print f, len(x) / 150000, basename, t0
N = len(x) / step
for g in range(gen):
T = []
F = []
rps = np.random.randint(0, len(x) - SEG - 1, N)
for r in rps:
t, f = create_T_F(y[r:r + SEG], x[r:r + SEG], t0)
T.append(t)
F.append(f)
fn = basename + '_{}.p'.format(g)
print g, len(T)
if len(T) > 0:
with open(fn, 'w') as fp:
pickle.dump((T, F), fp)
def process(c):
f = c[0]
step = c[1]
gen = c[2]
cfg = c[3]
SEG = cfg['SEG']
L_step = cfg['L_step']
length = SEG * L_step
#length = c[3]
#L_step = c[4]
tmp = cfg['tmp']
basename = os.path.basename(f)[:-4]
basename = os.path.join(HOME, tmp, basename)
x, y = l_utils.load_data(f)
t0 = 1 # y[0]
print f, len(x) / l_utils.SEGMENT, basename, t0
N = len(x) / step
for g in range(gen):
T = []
F = []
rps = np.random.randint(0, len(x) - length - 1, N)
for r in rps:
t, f = create_T_F(y[r:r + length], x[r:r + length], t0)
T.append(t)
F.append(create_feature(f, length, L_step))
fn = basename + '_{}.p'.format(g)
print g, len(T)
if len(T) > 0:
with open(fn, 'w') as fp:
pickle.dump((T, F), fp)
def get_testing_data(files, cfg):
L_step = cfg['L_step']
SEG = cfg['SEG']
out = []
for f in files:
x, y = l_utils.load_data(f)
t0 = 1 #y[0]
print f, len(x) / l_utils.SEGMENT, t0
for r in range(0, len(x), l_utils.SEGMENT):
t, f = create_T_F(y[r:r + l_utils.SEGMENT],
x[r:r + l_utils.SEGMENT], t0)
if len(f) != l_utils.SEGMENT:
break
fs = []
if L_step != 300:
step = SEG #int(len(f)/200)
for a in range(0, len(f), step):
f0 = f[a:a + SEG * L_step]
if len(f0) == SEG * L_step:
f0 = abs(fft(f0))[1:len(f0) / 2 + 1]
d = f0.reshape((SEG / 2, L_step))
d = np.mean(d, 1)
fs.append(d)
# fs.append(abs(fft(f0))[1:len(f0) / 2])
else:
break
else:
length = L_step * SEG
fs.append(create_feature(f, length, L_step))
out.append((t, np.array(fs)))
return out
def main5(cfg, filename, scale=10):
x, y = l_utils.load_data(filename)
if y is not None:
length = len(x)
step = length/100
for a in range(0, length, step):
X = np.array(x[a: a+l_utils.SEGMENT])
Y = np.array(y[a: a+l_utils.SEGMENT])
if len(X)==l_utils.SEGMENT:
avg = np.mean(Y)
A, B = net_run(cfg, X)
print avg, A*scale, B*scale
else:
A, B = net_run(cfg, x)
return A*scale, B*scale
def generate_test_data(files, cfg):
SEG = cfg['SEG']
Ts = {}
for f in files:
x0, y0 = l_utils.load_data(f)
t0 = y0[0]
T = []
print f, len(y0), t0
for r in range(0, len(y0), l_utils.SEGMENT):
x = x0[r:r + l_utils.SEGMENT]
y = y0[r:r + l_utils.SEGMENT]
t, _ = create_T_F(y, x, t0)
F = []
if len(y) == l_utils.SEGMENT:
for rr in range(0, l_utils.SEGMENT, SEG / 15):
if len(y[rr:rr + SEG]) == SEG:
# _, fs = create_T_F(y[rr:rr + SEG], x[rr:rr + SEG], t0)
_, fs = create_T_F(y[rr:rr + SEG], x[rr:rr + SEG], 1)
F.append(fs)
if len(F) > 0:
T.append((t, F))
Ts[f] = T
return Ts