def merge_csv(data_dir, out_dir, plot=False, ver=False):
if not os.path.exists(out_dir):
os.makedirs(out_dir)
os.chdir(out_dir)
files = sorted(os.listdir(data_dir))
syear,eyear = int(files[0].split('_')[3]), int(files[-1].split('_')[3])
smonth,emonth = int(files[0].split('_')[4][0:-4]), int(files[-1].split('_')[4][0:-4])
name = files[0].split('_')[0:3]
if os.path.exists(out_dir+'/'+name[0]+'_'+name[1]+'_'+name[2]+'_merged.csv'):
return name,'no errors'
# Merge csv files into one pandas dataframe
def read_append(data_dir,names,name,year,month,ver=False):
path =data_dir+'/'+name[0]+'_'+name[1]+'_'+name[2]+'_%s_%s.csv' % (year,month)
frame = pd.read_csv(path, header=14, parse_dates=['Date/Time'], index_col=['Date/Time'])
names = names.append(frame)
if ver==True: print path
return names
years = range(syear, eyear+1)
months,smonths,emonths = range(1,12+1), range(smonth,12+1), range(1,emonth+1)
names = pd.DataFrame()
for year in years:
if year==eyear:
for month in emonths:
try:
names = read_append(data_dir,names,name,year,month)
except ValueError, e:
print e
return name,e
except IOError, e:
print e
dd.downloader(data_dir,name[0],name[1],name[2],1,month,year,verbose='on')
names = read_append(data_dir,names,name,year,month)
return name, e
return name,e
except IOError, e:
print e
dd.downloader(data_dir,name[0],name[1],name[2],1,month,year,verbose='on')
names = read_append(data_dir,names,name,year,month)
return name, e
elif year==syear:
for month in smonths:
try:
names = read_append(data_dir,names,name,year,month)
except ValueError, e:
print e
return name,e
except IOError, e:
print e
dd.downloader(data_dir,name[0],name[1],name[2],1,month,year,verbose='on')
names = read_append(data_dir,names,name,year,month)
return name, e
else:
for month in months:
try:
names = read_append(data_dir,names,name,year,month)
except ValueError, e:
print e
return name,e
except IOError, e:
print e
dd.downloader(data_dir,name[0],name[1],name[2],1,month,year,verbose='on')
names = read_append(data_dir,names,name,year,month)
return name, e
names.to_csv(out_dir+'/'+name[0]+'_'+name[1]+'_'+name[2]+'_merged.csv')
