def calculateaveflow(self):
utils.start_waiting_cursor()
date_from = self.FromDateTime.dateTime().toPyDateTime()
date_to = self.ToDateTime.dateTime().toPyDateTime()
#Identify distinct set of obsid and instrumentid with Accvol-data and within the user-defined date_time-interval:
sql= """SELECT DISTINCT obsid, instrumentid FROM (SELECT * FROM w_flow WHERE flowtype = 'Accvol' AND date_time >= '%s' AND date_time <= '%s' AND obsid IN (%s))"""%(date_from,date_to, utils.sql_unicode_list(self.observations))
#utils.pop_up_info(sql)#debug
uniqueset = db_utils.sql_load_fr_db(sql)[1] # The unique set of obsid and instrumentid is kept in uniqueset
negativeflow = False
for pyobsid, pyinstrumentid in uniqueset:
sql= """select date_time, reading from w_flow where flowtype = 'Accvol' and obsid='%s' and instrumentid='%s' and date_time >='%s' and date_time <='%s' order by date_time"""%(pyobsid,pyinstrumentid,date_from,date_to)
recs = db_utils.sql_load_fr_db(sql)[1]
"""Transform data to a numpy.recarray"""
My_format = [('date_time', datetime.datetime), ('values', float)] #Define format with help from function datetime
table = np.array(recs, dtype=My_format) #NDARRAY
table2=table.view(np.recarray) # RECARRAY Makes the two columns into callable objects, i.e. write table2.values
for j, row in enumerate(table2):#This is where Aveflow is calculated for each obs and also written to db
if j>0:#first row is "start-value" for Accvol and there is no Aveflow to be calculated
Volume = (table2.values[j] - table2.values[j-1])*1000#convert to L since Accvol is supposed to be in m3
""" Get help from function datestr2num to get date and time into float"""
DeltaTime = 24*3600*(datestr2num(table2.date_time[j]) - datestr2num(table2.date_time[j-1]))#convert to seconds since numtime is days
Aveflow = Volume/DeltaTime#L/s
if Aveflow<0:
negativeflow = True
sql = """insert or ignore into w_flow(obsid,instrumentid,flowtype,date_time,reading,unit) values('%s','%s','Aveflow','%s','%s','l/s')"""%(pyobsid,pyinstrumentid,table2.date_time[j],Aveflow)
db_utils.sql_alter_db(sql)
if negativeflow:
utils.MessagebarAndLog.info(bar_msg=ru(QCoreApplication.translate('Calcave', "Please notice that negative flow was encountered.")))
utils.stop_waiting_cursor()
self.close()
def plot_data(self):
if not self.has_graph:
new_x = [mdates.datestr2num(x) for x in self.graph_x]
plt.xlabel('Time')
plt.ylabel('Count')
ax = self.fig.add_subplot(111)
ax.xaxis.set_major_formatter(mdates.DateFormatter('%H:%M'))
ax.plot(new_x, self.graph_y, 'r-')
ax.get_xaxis().set_ticks([])
#ax.get_yaxis().set_ticks([])
self.canvas = FigureCanvas(self.fig)
self.graph_layout.addWidget(self.canvas)
self.canvas.draw()
self.has_graph = True
else:
self.graph_layout.removeWidget(self.canvas)
self.canvas.close()
self.fig.clf()
plt.xlabel('Time')
plt.ylabel('Count')
ax = self.fig.add_subplot(111)
ax.xaxis.set_major_formatter(mdates.DateFormatter('%H:%M'))
new_x = [mdates.datestr2num(x) for x in self.graph_x]
ax.plot(new_x, self.graph_y, 'r-')
ax.get_xaxis().set_ticks([])
#ax.get_yaxis().set_ticks([])
self.canvas = FigureCanvas(self.fig)
self.graph_layout.addWidget(self.canvas)
self.canvas.draw()
def test_autofmt_xdate(which):
date = ['3 Jan 2013', '4 Jan 2013', '5 Jan 2013', '6 Jan 2013',
'7 Jan 2013', '8 Jan 2013', '9 Jan 2013', '10 Jan 2013',
'11 Jan 2013', '12 Jan 2013', '13 Jan 2013', '14 Jan 2013']
time = ['16:44:00', '16:45:00', '16:46:00', '16:47:00', '16:48:00',
'16:49:00', '16:51:00', '16:52:00', '16:53:00', '16:55:00',
'16:56:00', '16:57:00']
angle = 60
minors = [1, 2, 3, 4, 5, 6, 7]
x = mdates.datestr2num(date)
y = mdates.datestr2num(time)
fig, ax = plt.subplots()
ax.plot(x, y)
ax.yaxis_date()
ax.xaxis_date()
ax.xaxis.set_minor_locator(AutoMinorLocator(2))
ax.xaxis.set_minor_formatter(FixedFormatter(minors))
fig.autofmt_xdate(0.2, angle, 'right', which)
if which in ('both', 'major', None):
for label in fig.axes[0].get_xticklabels(False, 'major'):
assert int(label.get_rotation()) == angle
if which in ('both', 'minor'):
for label in fig.axes[0].get_xticklabels(True, 'minor'):
assert int(label.get_rotation()) == angle
def calculateaveflow(self):
PyQt4.QtGui.QApplication.setOverrideCursor(PyQt4.QtCore.Qt.WaitCursor)
date_from = self.FromDateTime.dateTime().toPyDateTime()
date_to = self.ToDateTime.dateTime().toPyDateTime()
#Identify distinct set of obsid and instrumentid with Accvol-data and within the user-defined date_time-interval:
sql= """select distinct obsid, instrumentid from(select * from w_flow where flowtype = "Accvol" and date_time >="%s" and date_time <="%s" and obsid IN %s)"""%(date_from,date_to,(str(self.observations)).encode('utf-8').replace('[','(').replace(']',')'))
#utils.pop_up_info(sql)#debug
uniqueset = utils.sql_load_fr_db(sql)[1] # The unique set of obsid and instrumentid is kept in uniqueset
negativeflow = False
for pyobsid, pyinstrumentid in uniqueset:
sql= """select date_time, reading from w_flow where flowtype = 'Accvol' and obsid='%s' and instrumentid='%s' and date_time >='%s' and date_time <='%s' order by date_time"""%(pyobsid,pyinstrumentid,date_from,date_to)
recs = utils.sql_load_fr_db(sql)[1]
"""Transform data to a numpy.recarray"""
My_format = [('date_time', datetime.datetime), ('values', float)] #Define format with help from function datetime
table = np.array(recs, dtype=My_format) #NDARRAY
table2=table.view(np.recarray) # RECARRAY Makes the two columns into callable objects, i.e. write table2.values
j=0
for row in table2:#This is where Aveflow is calculated for each obs and also written to db
if j>0:#first row is "start-value" for Accvol and there is no Aveflow to be calculated
Volume = (table2.values[j] - table2.values[j-1])*1000#convert to L since Accvol is supposed to be in m3
""" Get help from function datestr2num to get date and time into float"""
DeltaTime = 24*3600*(datestr2num(table2.date_time[j]) - datestr2num(table2.date_time[j-1]))#convert to seconds since numtime is days
Aveflow = Volume/DeltaTime#L/s
if Aveflow<0:
negativeflow = True
sql = """insert or ignore into w_flow(obsid,instrumentid,flowtype,date_time,reading,unit) values('%s','%s','Aveflow','%s','%s','l/s')"""%(pyobsid,pyinstrumentid,table2.date_time[j],Aveflow)
utils.sql_alter_db(sql)
j = j + 1
if negativeflow:
self.iface.messageBar().pushMessage("Info","Please notice that negative flow was encountered.", 0, duration=5)
PyQt4.QtGui.QApplication.restoreOverrideCursor()
self.close()
def main_loop(self):
"""
Single iteration of the application's main loop.
"""
# Get current image frame from the camera
frame = self.camera.get_frame()
self.h,self.w,_c = frame.shape
#display unaltered frame
#imshow("Original",frame)
#set current image frame to the processor's input
self.processor.frame_in = frame
#process the image frame to perform all needed analysis
self.processor.run(frame)
#collect the output frame for display
output_frame = self.processor.frame_out
#show the processed/annotated output frame
imshow("Processed",output_frame)
#create and/or update the raw data display if needed
global smileneighbour, mqLoop, smilecount, eyetot
#if self.bpm_plot:
#self.make_bpm_plot()
if mqLoop >= 1:
x = str(datetime.datetime.now())
sm.write(str(md.datestr2num(x)) + " " + str(smileneighbour) + "\n")
e.write(str(md.datestr2num(x)) + " " + str(eyetot) + "\n")
#hr.write(str(md.datestr2num(x)) + " " + str(self.processor.show_bpm_text.bpm) + "\n")
hr.write(str(md.datestr2num(x)) + " " + str(self.processor.bpm) + "\n")
pulse_estimation_log.write(str(int(round(time.time() * 1000))) + " " + str(self.processor.bpm) + "\n")
smileneighbour+= 2*eyetot
smileneighbour/=100
print "bpm: " + str(self.processor.bpm)
#if (self.processor.show_bpm_text.bpm) > dhr:
if (self.processor.bpm) > dhr:
#print (self.processor.fft.samples[-1]/2, self.processor.fft.samples[-1]-dhr/2)
#overbeat = (self.processor.fft.samples[-1]-dhr)*(self.processor.fft.samples[-1]-dhr)
#smileneighbour += (self.processor.show_bpm_text.bpm-dhr)
smileneighbour += (self.processor.bpm - dhr)
f.write(str(md.datestr2num(x)) + " " + str(smileneighbour) + "\n")
mqLoop = 0
else:
mqLoop+= 0.9
img = cv.QueryFrame(capture)
smileneighbour = 0
eyetot = 0
#if img:
# image = DetectRedEyes(img, faceCascade, smileCascade, eyeCascade)
# cv.ShowImage("camera", image)
#handle any key presses
self.key_handler()
def getDate(self,startDate,endDate,date2hot):
date = []
hot = []
minDate = mdates.datestr2num(startDate)
maxDate = mdates.datestr2num(endDate)
for item in date2hot:
value = item[1]
dt = mdates.datestr2num(item[0])
if dt >= minDate and dt <= maxDate:
date.append(dt)
hot.append(value)
return date , hot
def calc_frequency(self,table2):
freqs = np.zeros(len(table2.values),dtype=float)
for j, row in enumerate(table2):
if j>0:#we can not calculate frequency for first row
try:
diff = (table2.values[j] - table2.values[j-1])
""" Get help from function datestr2num to get date and time into float"""
delta_time = 24*3600*(datestr2num(table2.date_time[j]) - datestr2num(table2.date_time[j-1]))#convert to seconds since numtime is days
except:
pass #just skip inaccurate data values and use previous frequency
freqs[j] = diff/delta_time
freqs[0]=freqs[1]#assuming start frequency to get a nicer plot
return freqs
def load_ctd_mod(filename):
tmp=np.genfromtxt(filename,dtype=str)
out={}
out[tmp[0,0]]=tmp[1,0].astype(int)
out[tmp[0,1]]=tmp[1:,1].astype(int)
out[tmp[0,2]]=tmp[1,2].astype(float)
out[tmp[0,3]]=tmp[1,3].astype(float)
out["{}{}".format(tmp[0,4],tmp[0,5])]=np.array(["{} {}".format(d,t) for d,t in zip(tmp[1:,4],tmp[1:,5])])
out['time']=dates.datestr2num(out["{}{}".format(tmp[0,4],tmp[0,5])])
out[tmp[0,6]]=tmp[1:,6].astype(float)
out[tmp[0,7]]=tmp[1:,7].astype(float)
out[tmp[0,8]]=tmp[1:,8].astype(float)
uit=np.unique(out['it'])
for i,it in enumerate(uit):
if i==0:
idx=np.argwhere(out['it']==it)
else:
idx=np.hstack([idx,np.argwhere(out['it']==it)])
aout={}
for key in out:
if idx.shape[0]*idx.shape[1] == np.atleast_2d(out[key]).shape[0]*np.atleast_2d(out[key]).shape[1]:
aout[key]=out[key][idx]
out['arrays']=aout
return out
def Converter(val):
if val.count(".") == 2:
return dates.datestr2num(val)
else:
try:
val.index("/")
return dates.datestr2num(val)
except ValueError:
try:
val.index(":")
return dates.datestr2num(val) % 1
except ValueError:
try:
return float(val or "nan")
except ValueError:
return np.nan
def ForecastDraw(self):
"""
at the day-level
:return:
"""
pl.title("aqi/time(day)")# give plot a title
pl.xlabel('time')# make axis labels
pl.ylabel('aqi')
data = np.loadtxt(StringIO(self._xyArrayStr), dtype=np.dtype([("t", "S13"), ("v", float)]))
datestr = np.char.replace(data["t"], "T", " ")
t = dt.datestr2num(datestr)
# k = pl.num2date(t)
# k2 = dt.num2date(t)
v = data["v"]
if len(t) > 30:
t = t[-30:]
v = v[-30:]
pl.plot_date(t, v, fmt="-o")
self.polyfit(t, v)
pl.subplots_adjust(bottom=0.3)
# pl.legend(loc=4)#指定legend的位置,读者可以自己help它的用法
ax = pl.gca()
ax.fmt_xdata = pl.DateFormatter('%Y-%m-%d %H:%M:%S')
pl.xticks(rotation=70)
# pl.xticks(t, datestr) # 如果以数据点为刻度，则注释掉这一行
ax.xaxis.set_major_formatter(pl.DateFormatter('%Y-%m-%d %H:%M'))
# pl.xlim(('2016-03-09 00:00', '2016-03-12 00:00'))
pl.grid() # 有格子
pl.show()# show the plot on the screen
return self._forecast_Value
def _parse_date(datestring):
'''
takes a date string and returns a datetime.datetime object
'''
datenum = mdates.datestr2num(datestring)
dateval = mdates.num2date(datenum)
return dateval
def gd2jd(datestr):
""" Convert a string Gregorian date into a Julian date using Pylab.
If no time is given (i.e., only a date), then noon is assumed.
Timezones can be given, but UTC is assumed otherwise.
:EXAMPLES:
::
print gd2jd('Aug 11 2007') #---------------> 2454324.5
print gd2jd('Aug 11 2007, 12:00 PST') #-----> 2454324.29167
print gd2jd('12:00 PM, January 1, 2000') #--> 2451545.0
:REQUIREMENTS: :doc:`matplotlib`
:SEE ALSO: :func:`jd2gd`
"""
# 2008-08-26 14:03 IJC: Created
# 2010-12-08 13:00 IJC: Removed "+ 3442850" from num2julian call
# 2011-05-19 11:37 IJMC: Put the factor back in for error-catching...
import matplotlib.dates as dates
if datestr.__class__==str:
d = dates.datestr2num(datestr)
jd = dates.num2julian(d)
if jd<0:
jd = dates.num2julian(d + 3442850)
print "You are probably using an old version of Matplotlib..."
else:
jd = []
return jd
def generate_plot(dictionary, title, labelX, labelY, filename,ids, flag):
import numpy as np
figure=plt.figure(figsize=(6*3.13,4*3.13))
plt.title(title)
hspace = 1.0
nrow=1
plt.subplots_adjust( hspace=hspace )
figure.autofmt_xdate()
for hashtag in dictionary:
x = dictionary[hashtag]['x']
x = dates.datestr2num(x)
y = dictionary[hashtag]['y']
plt.subplot(3,3,nrow)
nrow+=1
plt.ylabel(labelY)
plt.xlabel(labelX)
plt.xticks(rotation=30)
plt.plot_date(x, y, '-',color='green', linewidth=2.0, label=hashtag.decode('utf8'))
plt.legend(loc='best',prop={'size':10})
plt.show()
figure.savefig(filename+ids,dpi=(1200))
plt.close()
def dayHisto(timeList, minDate, maxDate, yLabel, labels=(), filename='gitfig.png', highlightLo=None, highlightHi=None, highlightLabel=None):
'''
take a list of lists 'timeList' of datetime objects, and histogram them w/ 1 bin per day
minDate and maxDate == datetime.date objects; stack histo for each element in timeList.
'''
timeNum = []
for times in timeList:
nums = []
for time in times:
nums.append(dates.date2num(time))
timeNum.append(nums)
days = dates.DayLocator()
fmt = dates.DateFormatter('%b-%d')
fig = plt.figure()
ax = fig.add_subplot(111)
plt.xlim(minDate, maxDate)
ax.hist(timeNum,
bins=range( int(dates.date2num(minDate)), int(dates.date2num(maxDate)) +1 ),
align='left',
stacked=True,
label = labels )
ax.xaxis_date()
ax.xaxis.set_major_locator(days)
ax.xaxis.set_major_formatter(fmt)
fig.autofmt_xdate()
plt.ylabel(yLabel)
if len(timeList) > 1:
plt.legend(loc='best')
# shade area nicely
if highlightHi is not None and highlightLo is not None:
xlo = dates.datestr2num(highlightLo)
xhi = dates.datestr2num(highlightHi)
extra = (xhi-xlo)/2
xlo -= extra
xhi += extra
ax.axvspan(xlo, xhi, color='black', alpha=0.1, hatch='.')
shade_patch = patches.Patch(color='black', alpha=0.1, hatch='.', label=highlightLabel)
handles, labels = ax.get_legend_handles_labels()
handles.append(shade_patch)
plt.legend(handles=handles, loc='best')
plt.savefig(filename, bbox_inches='tight')
def createsingleplotobject(self,sql,i,My_format,curs,plottype='line'):
rs = curs.execute(sql) #Send SQL-syntax to cursor
recs = rs.fetchall() # All data are stored in recs
# late fix for xy-plots
My_format2 = [('numx', float), ('values', float)]#define a format for xy-plot (to use if not datetime on x-axis)
#Transform data to a numpy.recarray
try:
table = np.array(recs, dtype=My_format) #NDARRAY
table2=table.view(np.recarray) # RECARRAY transform the 2 cols into callable objects
myTimestring = [] #LIST
FlagTimeXY = 'time'
j = 0
for row in table2:
myTimestring.append(table2.date_time[j])
j = j + 1
numtime=datestr2num(myTimestring) #conv list of strings to numpy.ndarray of floats
except:
table = np.array(recs, dtype=My_format2) #NDARRAY
table2=table.view(np.recarray) # RECARRAY transform the 2 cols into callable objects
myXYstring = [] #LIST
FlagTimeXY = 'XY'
j = 0
for row in table2: #
myXYstring.append(table2.numx[j])
j = j + 1
numtime = myXYstring
# from version 0.2 there is a possibility to make discontinuous plot if timestep bigger than maxtstep
if self.spnmaxtstep.value() > 0: # if user selected a time step bigger than zero than thre may be discontinuous plots
pos = np.where(np.abs(np.diff(numtime)) >= self.spnmaxtstep.value())[0]
numtime[pos] = np.nan
table2.values[pos] = np.nan
if plottype == "marker":
MarkVar = 'o'
elif plottype == "line":
MarkVar = '-'
elif plottype == "line and cross":
MarkVar = '+-'
else:
MarkVar = 'o-'
if FlagTimeXY == "time" and plottype == "step-pre":
self.p[i], = self.axes.plot_date(numtime, table2.values, drawstyle='steps-pre', linestyle='-', marker='None',c=np.random.rand(3,1),label=self.plabels[i])# 'steps-pre' best for precipitation and flowmeters, optional types are 'steps', 'steps-mid', 'steps-post'
elif FlagTimeXY == "time" and plottype == "step-post":
self.p[i], = self.axes.plot_date(numtime, table2.values, drawstyle='steps-post', linestyle='-', marker='None',c=np.random.rand(3,1),label=self.plabels[i])
elif FlagTimeXY == "time" and plottype == "line and cross":
self.p[i], = self.axes.plot_date(numtime, table2.values, MarkVar,markersize = 6, label=self.plabels[i])
elif FlagTimeXY == "time":
self.p[i], = self.axes.plot_date(numtime, table2.values, MarkVar,label=self.plabels[i])
elif FlagTimeXY == "XY" and plottype == "step-pre":
self.p[i], = self.axes.plot(numtime, table2.values, drawstyle='steps-pre', linestyle='-', marker='None',c=np.random.rand(3,1),label=self.plabels[i])
elif FlagTimeXY == "XY" and plottype == "step-post":
self.p[i], = self.axes.plot(numtime, table2.values, drawstyle='steps-post', linestyle='-', marker='None',c=np.random.rand(3,1),label=self.plabels[i])
elif FlagTimeXY == "XY" and plottype == "line and cross":
self.p[i], = self.axes.plot(numtime, table2.values, MarkVar,markersize = 6, label=self.plabels[i])
else:
self.p[i], = self.axes.plot(numtime, table2.values, MarkVar,label=self.plabels[i])
def save_tgnc(tg, filename):
ncid = n4.Dataset(filename, 'w',format='NETCDF3_CLASSIC')
#create dimensions
ncid.createDimension('time',None)
ncid.createDimension('one',1)
ncid.createDimension('DateStrLen',19)
#define variables
cast = ncid.createVariable('tgnumber','i',('one',))
lon = ncid.createVariable('lon','d',('one',))
lat = ncid.createVariable('lat','d',('one',))
h = ncid.createVariable('h','d',('one',))
dist = ncid.createVariable('dist','d',('one',))
time = ncid.createVariable('time','d',('time',))
timestamp = ncid.createVariable('Times','c',('time','DateStrLen'))
zeta = ncid.createVariable('zeta','d',('time',))
cast[:] = tg['tg_number']
cast.__setattr__('long_name','TG Deployment Number matched to model location')
lon[:] = tg['lon']
lon.__setattr__('long_name','Longitude')
lon.__setattr__('units','degrees')
lat[:] = tg['lat']
lat.__setattr__('long_name','latitude')
lat.__setattr__('units','degrees')
time[:] = tg['time']
time.__setattr__('long_name','time')
time.__setattr__('units','days')
time.__setattr__('comments','python datenum')
tstr=dates.num2date(dates.datestr2num(tg['Time']))
tnew=np.array([ t.strftime('%Y-%m-%dT%H:%M:%S') for t in tstr])
timestamp[:]=np.array([list(tt) for tt in tnew])[:]
timestamp.__setattr__('long_name','Time string')
timestamp.__setattr__('units','yyyy-mm-dd HH:MM:SS')
h[:]=tg['h']
h.__setattr__('long_name','Depth')
h.__setattr__('units','meters')
dist[:]=tg['dist']
dist.__setattr__('long_name','Distance between obs and model')
dist.__setattr__('units','meters')
zeta[:] = tg['zeta']
zeta.__setattr__('long_name','Water Elevation')
zeta.__setattr__('units','meters')
ncid.__setattr__('type','TG-like ncfile')
ncid.__setattr__('history','Created ' +ttime.ctime(ttime.time()) )
ncid.close()
def plot_time_series(self, fig, is_timeseries, ax, x, y, fill=False, title='', xlabel='', ylabel='',
title_font={}, axis_font={}, tick_font={}, scatter=False, qaqc=[], events={}, **kwargs):
if not title_font:
title_font = title_font_default
if not axis_font:
axis_font = axis_font_default
if scatter:
ppl.scatter(ax, x, y, **kwargs)
else:
h = ppl.plot(ax, x, y, **kwargs)
if is_timeseries:
self.get_time_label(ax, x)
fig.autofmt_xdate()
else:
ax.set_xlabel(xlabel.replace("_", " "), **axis_font)
if ylabel:
ax.set_ylabel(ylabel.replace("_", " "), **axis_font)
if title:
ax.set_title(title.replace("_", " "), **title_font)
ax.grid(True)
if fill:
miny = min(ax.get_ylim())
if not scatter:
ax.fill_between(x, y, miny+1e-7, facecolor = h[0].get_color(), alpha=0.15)
else:
ax.fill_between(x, y, miny+1e-7, facecolor = axis_font_default['color'], alpha=0.15)
if events:
ylim = ax.get_ylim()
for event in events['events']:
time = datestr2num(event['start_date'])
x = np.array([time, time])
h = ax.plot(x, ylim, '--', label=event['class'])
legend = ax.legend()
if legend:
for label in legend.get_texts():
label.set_fontsize(10)
if len(qaqc) > 0:
bad_data = np.where(qaqc > 0)
h = ppl.plot(ax, x[bad_data], y[bad_data],
marker='o',
mfc='none',
linestyle='None',
markersize=6,
markeredgewidth=2,
mec='r')
# plt.tick_params(axis='both', which='major', labelsize=10)
if tick_font:
ax.tick_params(**tick_font)
plt.tight_layout()
def deal_with_results(self,res):
"""Handles results gotten from API and formatted, plots them with matplotlib tools and saves plot img"""
view_nums = [x[1] for x in res] # y axis
date_strs = [mdates.datestr2num(x[0]) for x in res]
fig, ax = plt.subplots(1)
ax.plot_date(date_strs, view_nums, fmt="g-")
fig.autofmt_xdate()
ax.fmt_xdata = mdates.DateFormatter('%Y-%m-%d')
total = sum(view_nums)
plt.title("%d total Course Views over past %s days" % (total, len(date_strs)-1)) # should get title of course
return fig
def loadcur(filename,exact=False):
files=glob.glob(filename)
files.sort()
returndic={}
for j,fname in enumerate(files):
fp=open(fname,'r')
numlines = len(fp.readlines())
fp.seek(0)
headerdone=False
indata={}
for i,line in enumerate(fp.readlines()):
if '||' in line:
if '!Observed' in line:
sline=line.split()
indata['lon']=-1*(float(sline[3])+float(sline[4][:-1])/60)
indata['lat']=1*(float(sline[1])+float(sline[2][:-1])/60)
elif 'Computed from spatial average bin' in line:
sline=line.split()
indata['bin']=int(sline[5][:-1])
indata['range']=np.array([int(val) for val in sline[6][:-1].split('-')])
else:
if headerdone==False:
headerdone=True
arrstart=i
indata['time']=np.empty((numlines-i,))
indata['timestr']=np.empty((numlines-i,),dtype='S16')
indata['u']=np.empty((numlines-i,))
indata['v']=np.empty((numlines-i,))
sline=line.split()
if len(sline)==2:
indata['timestr'][i-arrstart]=sline[0]+' '+sline[1][:5]
indata['u'][i-arrstart]=np.nan
indata['v'][i-arrstart]=np.nan
elif len(sline)==4:
indata['timestr'][i-arrstart]=sline[0]+' '+sline[1]
indata['u'][i-arrstart]=float(sline[2])
indata['v'][i-arrstart]=float(sline[3])
else:
print('Unhandled Case')
fp.close()
returndic[j+1]=indata
returndic[j+1]['time']=dates.datestr2num(returndic[j+1]['timestr'])
returndic[j+1]['filename']=fname
return returndic
def load_ctd_zeta_mod(filename):
tmp=np.genfromtxt(filename,dtype=str)
out={}
out[tmp[0,0]]=tmp[1,0].astype(int)
out[tmp[0,1]]=tmp[1:,1].astype(int)
out[tmp[0,2]]=tmp[1,2].astype(float)
out[tmp[0,3]]=tmp[1,3].astype(float)
out["{}{}".format(tmp[0,4],tmp[0,5])]=np.array(["{} {}".format(d,t) for d,t in zip(tmp[1:,4],tmp[1:,5])])
out['time']=dates.datestr2num(out["{}{}".format(tmp[0,4],tmp[0,5])])
out[tmp[0,6]]=tmp[1:,6].astype(float)
return out
def generate_user_activity(user_contribution_ddict):
import numpy as np
figure=plt.figure(figsize=(6*3.13,4*3.13))
for hashtag in user_contribution_ddict:
xlist=user_contribution_ddict[hashtag]['x']
ylist=user_contribution_ddict[hashtag]['y']
for index in range(len(xlist)):
x=xlist[index]
y=ylist[index]
y = dates.datestr2num(y)
plt.plot(x,y)
plt.show()
plt.close()
def plot(self, x=[], y=[], label = None):
def _float(value):
try:
return float(value)
except:
return -9999
if self.xdate:
x = [mpdates.datestr2num(v) for v in x]
else:
x = [_float(v) for v in x]
if self.ydate:
y = [mpdates.datestr2num(v) for v in y]
else:
y = [_float(v) for v in y]
line, = self.ax.plot (x, y, label = label, picker=5)
if self.xdate:
self.ax.xaxis_date()
if self.ydate:
self.ax.yaxis_date()
if self.oldLines.has_key(label):
line.update_from(self.oldLines.pop(label))
def load_ctd_obs(filename):
tmp=np.genfromtxt(filename,dtype=str)
out={}
out[tmp[0,0]]=tmp[1,0].astype(int)
out[tmp[0,1]]=tmp[1,1]
out['time']=dates.datestr2num(tmp[1,1])
out[tmp[0,2]]=tmp[1,2].astype(float)
out[tmp[0,3]]=tmp[1,3].astype(float)
out[tmp[0,4]]=tmp[1:,4].astype(float)
out[tmp[0,5]]=tmp[1:,5].astype(float)
out[tmp[0,6]]=tmp[1:,6].astype(float)
out[tmp[0,7]]=tmp[1:,7].astype(float)
return out
def deal_with_results(self, res):
"""Handles results gotten from API and formatted, plots them with matplotlib tools and saves plot img"""
view_nums = [x[1] for x in res] # y axis
view_nums_orig = [x[1] for x in self.return_results(self.get_results_other(self.service,self.profile_id))] ## let's see
date_strs = [mdates.datestr2num(x[0]) for x in res] # x axis
fig, ax = plt.subplots(1)
ax.plot_date(date_strs, view_nums, fmt="b-", label="Downloads")
ax.plot_date(date_strs, view_nums_orig, fmt="g-", label="Views")
fig.autofmt_xdate()
ax.fmt_xdata = mdates.DateFormatter('%Y-%m-%d')
#total = sum(view_nums)
plt.legend(loc='upper left')
plt.title("Course Views vs Bulk Material Downloads over past %s days" % (len(date_strs)-1)) # should get title of course
#savefig('test4.png')
return fig
def Main():
with open ('C:\\Users\\Paul\\Documents\\Spring 2014\\DSV_Data_Files\\HW_3\\bios.csv','r') as doc:
for line in csv.DictReader(doc):
#print type(line)
date = line["signup_date"][:19]
gender.append(line["gender"])
age = int(line["age"])
if age >= 8 and age <= 80:
ages.append(age)
tstamps.append(date) #grabbing dates when signedup
dt = dates.datestr2num(date)
tstamps_formatted.append(dt)
#Graph_SignUp_Date(tstamps_formatted)
#Graph_Gender(gender)
Graph_Age(ages)
def plot_stats(stats):
import numpy as np
import matplotlib
# prevent interactive pop-up
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from matplotlib.dates import datestr2num, MonthLocator, DateFormatter
dates = datestr2num(stats)
# 2d array to store totals by release and by date
all = np.array([[stats[x][r] for x in stats] for r in releases])
fig = plt.figure(figsize=(30,5))
a=1. # alpha
kwargs = dict(linewidth=0,color='g', alpha=a, align='center')
ax = plt.subplot(1,1,1)
ax.xaxis.set_major_locator(MonthLocator())
ax.xaxis.set_major_formatter(DateFormatter('%b'))
ax.xaxis_date()
for i,r in enumerate(releases):
if i==0:
prev = all.sum(0)
prev=prev-all[i]
kwargs.update(alpha=a,label=r,bottom=prev)
ax.bar(dates, all[i],**kwargs)
a*=.72
plt.gcf().autofmt_xdate(rotation=70, ha='right')
plt.setp(ax.xaxis.get_majorticklabels(), rotation_mode='anchor', va='center')
l = plt.legend()
l.set_frame_on(False)
# Liberate axis!
ax.yaxis.tick_left()
ax.xaxis.tick_bottom()
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
plt.draw()
image_file = "_".join([owner_name, ppa, 'downloads.png'])
plt.savefig(image_file)
print "Graph saved as %s" % image_file
print "Totals by date:"
for d,t in zip(stats,all.sum(0))[::-1]:
print "%4d :"%t,d
print "Totals by release:"
for r,t in zip(releases,all.sum(1)):
print '%4d :'%t,r
print "Total downloads: ", all.sum()
def read_variable(self, data, var):
"""Reads variable data from NetCDF file pointer."""
# Checks for special variable syntax. If the first variable has only
# invalid values, uses the second, third, fourth, ...
if var.find('|') >= 0: # Checks spetial variable syntax
var_list = var.split('|')
else:
var_list = [var]
try:
for item in var_list:
dat = data.variables[item]
if not (dat.data == dat._FillValue).all():
break
except:
pass
# Determines if variable has a quality flag
try:
qf = data.variables[data.variables[var].quality_flag]
except:
qf = None
# Loads bathymetry data
H = data.variables['bathymetry']
#
dat0 = 0
#
try:
scale_factor = dat.scale_factor
except:
scale_factor = 1.
#
if var == 'time':
if dat.units == 'seconds since 1985-01-01 00:00:00.0':
dat0 = dates.datestr2num('1985-01-01 00:00:00.0')
scale_factor = 1. / 86400. # seconds per day!
else:
raise Warning("I don't know what to do!")
#
values = array(dat.data, dtype=float)
# Masks invalid values and data over land
try:
values[(dat.data == dat._FillValue) | (H.data > 0)] = nan
except:
pass
#
return values * scale_factor + dat0
def getEvents():
fromfile = open("events.txt", 'r')
fileData = csv.reader(fromfile)
dates = []
symbols = []
closep = []
percRets = []
e_vwap = []
pos = []
for line in fileData:
dates.append(mdates.datestr2num(line[0]))
symbols.append(line[1])
closep.append(float(line[2]))
percRets.append(float(line[3]))
e_vwap.append(line[4])
pos.append(line[5])
return dates, symbols, closep, percRets, e_vwap, pos
def createsingleplotobject(self,sql,i,My_format,curs,plottype='line', factor=1.0, offset=0.0, remove_mean=False):
rs = curs.execute(sql) #Send SQL-syntax to cursor
recs = rs.fetchall() # All data are stored in recs
# late fix for xy-plots
#Transform data to a numpy.recarray
try:
table = np.array(recs, dtype=My_format) #NDARRAY
table2=table.view(np.recarray) # RECARRAY transform the 2 cols into callable objects
FlagTimeXY = 'time'
myTimestring = list(table2.date_time)
numtime=datestr2num(myTimestring) #conv list of strings to numpy.ndarray of floats
except Exception, e:
utils.MessagebarAndLog.info(log_msg=u"Customplot, transforming to recarray with date_time as x-axis failed, msg: " + utils.returnunicode(str(e)))
table = np.array(recs, dtype=[('numx', float), ('values', float)]) #NDARRAY #define a format for xy-plot (to use if not datetime on x-axis)
table2=table.view(np.recarray) # RECARRAY transform the 2 cols into callable objects
FlagTimeXY = 'XY'
numtime = list(table2.numx)
def loadslev(filename):
with open(filename) as fp:
numlines = len(fp.readlines())
fp.seek(0)
cnt=0
rd={}
rd['timestr']=np.empty((numlines-8,),dtype='S16')
rd['zeta']=np.zeros((numlines-8,))
for i,line in enumerate(fp.readlines()):
line=line.replace('\r','').replace('\n','')
if '' == line:
continue
if i<8:
if 'Station_Name' in line:
rd['Station_Name']=line[13:]
if 'Station_Number' in line:
rd['Station_Number']=line[15:]
if 'Latitude_Decimal_Degrees' in line:
rd['lat']=float(line[25:])
if 'Longitude_Decimal_Degrees' in line:
rd['lon']=-1*float(line[26:])
if 'Datum' in line:
rd['datum']=line[6:]
if 'Time_zone' in line:
rd['tz']=line[10:]
else:
lsplit=line.split(',')
rd['timestr'][cnt]=lsplit[0]
rd['zeta'][cnt]=lsplit[1]
cnt+=1
#remove empty values due to extra spaces in file
idx=np.argwhere(rd['timestr']=='')
rd['timestr']=np.delete(rd['timestr'],idx)
rd['zeta']=np.delete(rd['zeta'],idx)
rd['time']=dates.datestr2num(rd['timestr'])
return rd
"plotting dates, e.g., with ax.xaxis_date()")
dt = self.num2date(x, self.tz)
ms = dt.strftime("%f")[:self.precision]
return dt.strftime(self.fmt).format(ms=ms)
def set_tzinfo(self, tz):
self.tz = tz
dates_str = [dt.strftime('%Y-%m-%d-%H:%M:%S.%f') for dt in dates]
y = [0, 1, 3, 4, 6, 5, 7]
with plt.style.context(['science', 'grid', 'no-latex']):
dates = mpl_dates.datestr2num(dates_str) # convert string dates to numbers
# plt.plot_date(dates, y, linestyle='solid')
plt.plot(dates, y)
plt.gcf().autofmt_xdate()
# date_format = mpl_dates.DateFormatter(
# '%M_%S_%f') # custom date formatter
plt.gca().xaxis.set_major_formatter(
PrecisionDateFormatter(fmt="%M:%S.{ms}", precision=1))
plt.title('Bitcoin Prices')
plt.xlabel('Date')
plt.ylabel('Closing Price')
plt.tight_layout()
plt.show()
###
### Plot (Bar-Chart)
###
# Datatypes of the returning data: column 1(col1) --> integer, column 2(date) --> string
datatypes = [('col1', 'i4'), ('date', 'S20')]
# Data-tuple and datatype
data = np.array(data_tuples, dtype=datatypes)
# Date comes from 'col1'
col1 = data['col1']
# Converts date to a manageable date-format for matplotlib
dates = mdates.num2date(mdates.datestr2num(data['date']))
fig, ax = plt.subplots()
# Create barchart (x-axis=dates, y-axis=col1,
ax.bar(dates, col1, width=15, align='center', color='#2dd700')
# Place a gray dashed grid behind the thicks (only for y-axis)
ax.yaxis.grid(color='gray', linestyle='dashed')
# Set this grid behind the thicks
ax.set_axisbelow(True)
# Rotate x-labels on the x-axis
fig.autofmt_xdate()
# Label x and y axis