def read_file(self):
"""
Get and return specific variables from specific file type.
"""
if '.h5' in self.fname:
h5f = tb.openFile(self.fname, 'r')
#data = h5f.root.data.read() # in-memory
data = h5f.root.data # out-of-memory
if data.shape[0] < 1:
return None
secs = data[:,self.col]
dt = SecsToDateTime(secs, since_year=1985)
if self.N is not None:
print self.fname
print_dates(dt.dates(), self.N) # to test the output
h5f.close()
return None
# dates -> months, years
return [dt.months(), dt.years(), data, h5f]
def main(args):
files = args.files
months = args.months
seasons = args.seasons
intervals = args.intervals
window = args.window
usedir = args.usedir
N = args.N
print 'files to read:', len(files)
print 'create directories:', usedir
if months:
print 'separate by: months'
elif seasons:
print 'separate by: seasons'
elif intervals:
print 'separate by: intervals'
elif window is not None:
print 'separate by: window', window
else:
print 'separate by: years'
print 'reading and processing files ...'
# input
#-----------------------------------------------------------------
ndirs = 0
nfiles = 0
for fname in files:
f = tb.openFile(fname, 'r')
try:
secs = f.root.idr.cols.utc85[:]
since_year = 1985
except:
secs = f.root.idr.cols.utc00[:]
since_year = 2000
if secs.shape[0] < 1: continue
# processing
#-------------------------------------------------------------
dt = SecsToDateTime(secs, since_year=since_year)
if months:
res = get_months(dt.years(), dt.months())
elif seasons:
res = get_seasons(dt.years(), dt.months())
elif intervals:
res = get_intervals(dt.datenum(), T1, T2)
elif window is not None:
m1, m2 = window
res = get_window(dt.years(), dt.months(), m1, m2)
else:
res = get_years(dt.years())
# output -> dir or file
#-------------------------------------------------------------
path, file_name = os.path.split(fname)
fname2, ext = os.path.splitext(file_name)
# one output per result
for r in res:
ind, year, month = r[:]
if ind.shape[0] < 1:
continue
if month is None:
month_name = ''
elif np.ndim(month) > 0:
month_name = MONTH_NAME[month[0]] + MONTH_NAME[month[1]]
else:
month_name = MONTH_NAME[month]
if dir:
dname = str(year) + month_name
outdir = os.path.join(path, dname)
if not os.path.exists(outdir): # if dir doesn't exist
os.mkdir(outdir) # create one
ndirs += 1
fname_out = os.path.join(outdir, file_name)
else:
file_name = ''.join([fname2, SEP, str(year), month_name, ext])
fname_out = os.path.join(path, file_name)
# tables to save
try:
table1_out = f.root.idr[ind]
except:
table1_out = f.root.gla[ind]
table2_out = f.root.mask[ind]
save_tbl(fname_out, table1_out)
save_tbl(fname_out, table2_out)
nfiles += 1
for fid in tb.file._open_files.values():
fid.close()
print 'done!'
if dir is True:
print 'directories created:', ndirs
print 'files created:', nfiles
def main(args):
files = args.files
months = args.months
seasons = args.seasons
intervals = args.intervals
window = args.window
struct = args.struct
usedir = args.usedir
N = args.N
print 'files to read:', len(files)
print 'data structure:', struct
print 'create directories:', usedir
if months:
print 'separate by: months'
elif seasons:
print 'separate by: seasons'
elif intervals:
print 'separate by: intervals'
elif window is not None:
print 'separate by: window', window
else:
print 'separate by: years'
print 'reading and processing files ...'
# input
#-----------------------------------------------------------------
ndirs = 0
nfiles = 0
for fname in files:
f = tb.openFile(fname, 'r')
if struct == 'idr':
secs = f.root.idr.cols.secs85[:]
since_year = 1985
elif struct == 'gla':
secs = f.root.gla.cols.secs00[:]
since_year = 2000
else:
raise IOError('-s must be idr/gla')
if secs.shape[0] < 1: continue
# processing
#-------------------------------------------------------------
dt = SecsToDateTime(secs, since_year=since_year)
if months:
res = get_months(dt.years(), dt.months())
elif seasons:
res = get_seasons(dt.years(), dt.months())
elif intervals:
res = get_intervals(dt.datenum(), T1, T2)
elif window is not None:
m1, m2 = window
res = get_window(dt.years(), dt.months(), m1, m2)
else:
res = get_years(dt.years())
# output -> dir or file
#-------------------------------------------------------------
path, fname2 = os.path.split(fname)
fname3, ext = os.path.splitext(fname2)
# one output per result
for r in res:
ind, time = r[:]
if ind.shape[0] < 1:
continue
# dir
if usedir:
outdir = os.path.join(path, time)
if not os.path.exists(outdir): # if dir doesn't exist
os.mkdir(outdir) # create one
ndirs += 1
fname_out = os.path.join(outdir, fname2)
# file
else:
fname4 = ''.join([fname3, SEP, time, ext])
fname_out = os.path.join(path, fname4)
# tables to save
#---------------------------------------------------------
try:
table1_out = f.root.idr
except:
table1_out = f.root.gla
table2_out = f.root.mask
table3_out = f.root.trk
save_tbl(fname_out, table1_out, ind)
save_tbl(fname_out, table2_out, ind)
save_tbl(fname_out, table3_out, ind)
nfiles += 1
#---------------------------------------------------------
close_files()
print 'done!'
if dir is True:
print 'directories created:', ndirs
print 'files created:', nfiles
close_files()
def main(args):
files = args.files
months = args.months
seasons = args.seasons
intervals = args.intervals
window = args.window
struct = args.struct
usedir = args.usedir
N = args.N
print 'files to read:', len(files)
print 'data structure:', struct
print 'create directories:', usedir
if months:
print 'separate by: months'
elif seasons:
print 'separate by: seasons'
elif intervals:
print 'separate by: intervals'
elif window is not None:
print 'separate by: window', window
else:
print 'separate by: years'
print 'reading and processing files ...'
# input
#-----------------------------------------------------------------
ndirs = 0
nfiles = 0
for fname in files:
f = tb.openFile(fname, 'r')
if struct == 'idr':
secs = f.root.idr.cols.secs85[:]
since_year = 1985
elif struct == 'gla':
secs = f.root.gla.cols.secs00[:]
since_year = 2000
else:
raise IOError('-s must be idr/gla')
if secs.shape[0] < 1: continue
# processing
#-------------------------------------------------------------
dt = SecsToDateTime(secs, since_year=since_year)
if months:
res = get_months(dt.years(), dt.months())
elif seasons:
res = get_seasons(dt.years(), dt.months())
elif intervals:
res = get_intervals(dt.datenum(), T1, T2)
elif window is not None:
m1, m2 = window
res = get_window(dt.years(), dt.months(), m1, m2)
else:
res = get_years(dt.years())
# output -> dir or file
#-------------------------------------------------------------
path, fname2 = os.path.split(fname)
fname3, ext = os.path.splitext(fname2)
# one output per result
for r in res:
ind, time = r[:]
if ind.shape[0] < 1:
continue
# dir
if usedir:
outdir = os.path.join(path, time)
if not os.path.exists(outdir): # if dir doesn't exist
os.mkdir(outdir) # create one
ndirs += 1
fname_out = os.path.join(outdir, fname2)
# file
else:
fname4 = ''.join([fname3, SEP, time, ext])
fname_out = os.path.join(path, fname4)
# tables to save
#---------------------------------------------------------
try:
table1_out = f.root.idr
except:
table1_out = f.root.gla
table2_out = f.root.mask
table3_out = f.root.trk
save_tbl(fname_out, table1_out, ind)
save_tbl(fname_out, table2_out, ind)
save_tbl(fname_out, table3_out, ind)
nfiles += 1
#---------------------------------------------------------
close_files()
print 'done!'
if dir is True:
print 'directories created:', ndirs
print 'files created:', nfiles
close_files()
