############################################################
if opt.DEBUG_MODE_ON:
from testingtools import gen_new_fake_data
opt.RAINFALLDATA_FN = 'dummy_rainfalldata.csv'
opt.SAMPLEDATA_FN = 'dummy_sampledata.csv'
opt.RAIN_DATA_HAS_HEADERS = False
opt.SAMPLE_DATA_HAS_HEADERS = False
if opt.GENERATE_NEW_FAKE_DATA:
args = (opt.N_RAIN, opt.N_SAMPLES, opt.REL_NOISE, opt.MEAN_SLOPE)
gen_new_fake_data(*args)
############################################################
# Data Pre-Processing
############################################################
assert 0 <= opt.MIN_OVERLAP <= len(csv2arr(opt.RAINFALLDATA_FN))
# format and normalize rainfall data
rain = csv2arr(opt.RAINFALLDATA_FN, hasheaders=opt.RAIN_DATA_HAS_HEADERS)
rain = list(np.array(rain).T[0]) # convert Nx1 array to list
rain = [float(x) for x in rain]
# read and format sample data and take the transpose
# (so now each sample is a row)
sampledata = csv2arr(opt.SAMPLEDATA_FN, hasheaders=opt.SAMPLE_DATA_HAS_HEADERS)
# take the transpose (so now each sample is a row)
sampledata = transpose(sampledata)
# convert any strings to floats
# if None, defaults to 'summary_data.csv' (in the folder containing this
# script)
outf = None
#####################################################
# set defaults
if not rows2grab:
rows2grab = [2, 3]
if not trans2grab:
trans2grab = range(10)
if not summary_dir:
summary_dir = os.path.join(os.getcwd(), 'summaries')
if not outf:
outf = 'summary_data.csv'
# do stuff
data = []
data_guide = []
for fn in os.listdir(summary_dir):
path2summary = os.path.join(summary_dir, fn)
summary = csv2arr(path2summary)[4:] # throw out first 4 rows
for m in trans2grab:
for k in rows2grab:
data.append(summary[11*m + k][3:])
x = 10*m + k + 4 # in original csv (not counting blank rows)
data_guide.append(fn[:-4] + '_({}-{}-{}))'.format(m, k, x))
data = transpose(data)
data.insert(0, data_guide)
arr2csv(data, filename=outf)
# if None, defaults to 'summary_data.csv' (in the folder containing this
# script)
outf = None
#####################################################
# set defaults
if not rows2grab:
rows2grab = [2, 3]
if not trans2grab:
trans2grab = range(10)
if not summary_dir:
summary_dir = os.path.join(os.getcwd(), 'summaries')
if not outf:
outf = 'summary_data.csv'
# do stuff
data = []
data_guide = []
for fn in os.listdir(summary_dir):
path2summary = os.path.join(summary_dir, fn)
summary = csv2arr(path2summary)[4:] # throw out first 4 rows
for m in trans2grab:
for k in rows2grab:
data.append(summary[11 * m + k][3:])
x = 10 * m + k + 4 # in original csv (not counting blank rows)
data_guide.append(fn[:-4] + '_({}-{}-{}))'.format(m, k, x))
data = transpose(data)
data.insert(0, data_guide)
arr2csv(data, filename=outf)
if opt.DEBUG_MODE_ON:
from testingtools import gen_new_fake_data
opt.RAINFALLDATA_FN = 'dummy_rainfalldata.csv'
opt.SAMPLEDATA_FN = 'dummy_sampledata.csv'
opt.RAIN_DATA_HAS_HEADERS = False
opt.SAMPLE_DATA_HAS_HEADERS = False
if opt.GENERATE_NEW_FAKE_DATA:
args = (opt.N_RAIN, opt.N_SAMPLES, opt.REL_NOISE, opt.MEAN_SLOPE)
gen_new_fake_data(*args)
############################################################
# Data Pre-Processing
############################################################
assert 0 <= opt.MIN_OVERLAP <= len(csv2arr(opt.RAINFALLDATA_FN))
# format and normalize rainfall data
rain = csv2arr(opt.RAINFALLDATA_FN, hasheaders=opt.RAIN_DATA_HAS_HEADERS)
rain = list(np.array(rain).T[0]) # convert Nx1 array to list
rain = [float(x) for x in rain]
# read and format sample data and take the transpose
# (so now each sample is a row)
sampledata = csv2arr(opt.SAMPLEDATA_FN, hasheaders=opt.SAMPLE_DATA_HAS_HEADERS)
# take the transpose (so now each sample is a row)
sampledata = transpose(sampledata)
def get_fake_data_params():
shifts, slopes, constants = csv2arr('dummy_params.csv', dtype=float)
shifts = [int(x) for x in shifts]
return shifts, slopes, constants
def get_fake_data_params():
shifts, slopes, constants = csv2arr('dummy_params.csv', dtype=float)
shifts = [int(x) for x in shifts]
return shifts, slopes, constants
