def main():
stream_series = py.Stream(stream_token_series)
stream_series.open()
stream_anomaly = py.Stream(stream_token_anomaly)
stream_anomaly.open()
end_date=END_DATE
while(True):
checkfile = open("checkfile.txt", 'a')
dm=connect2MODIS(end_date)
existsFiles=filesDownloaded()
availableFiles=filesAvailable(dm)
downloads=[(f,date) for (f,date) in availableFiles if f not in existsFiles]
downloads.sort()
for filename, date in downloads:
print "DL: " + filename
download(dm,filename, date)
if filename.endswith('.hdf'):
tifFile=convert2Tif(filename)
gdalProcessing(tifFile)
stats=retrieveStats(tifFile,date)
stream_series.write(dict(x=str(stats[0]), y=stats[3]))
anomaly=anomalyNDVI(date[5:])
print anomaly,stats[3]
stream_anomaly.write(dict(x=str(stats[0]), y=anomaly))
end_date=date
checkfile.write(filename+"\n")
cleaner()
checkfile.close()
stream_series.close()
stream_anomaly.close()
return 0
def open_streams(stream_id, gpumap):
""" Open each given stream and start streaming"""
for s in stream_id:
st = py.Stream(s)
st.open()
streamdict = {}
start = time.time()
for gpu in gpu_info():
listofstreams = gpumap.get(gpu.id)
for i in range(3):
st0 = py.Stream(listofstreams[0]['token'])
st1 = py.Stream(listofstreams[1]['token'])
st2 = py.Stream(listofstreams[2]['token'])
st0.open()
st1.open()
st2.open()
streamdict[gpu.id] = [st0, st1, st2]
a = time.time() - start
while True:
x = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')
for gpu in gpu_info():
streamdict[gpu.id][0].write(dict(x=x, y=memory(gpu.id)))
streamdict[gpu.id][1].write(dict(x=x, y=temperature(gpu.id)))
streamdict[gpu.id][2].write(dict(x=x, y=load(gpu.id)))
time.sleep(3)
def init_interview_stream(self):
"""initialize stream object of interview sentiment
INPUT: None
OUTPUT:
- s_line: line stream object
- s_bar: bar stream object
- url: url to the plotly graph
"""
s_line_id=self.traces['line']['stream_id']
s_bar_id=self.traces['bar']['stream_id']
trace_line = self.traces['line']['trace']
trace_bar = self.traces['bar']['trace']
title = 'Performance Metrics'
filename = 'perf-metric'
data = go.Data([trace_line, trace_bar])
# Send fig to Plotly, initialize streaming plot, open new tab
url = py.plot(data, filename=filename, auto_open=False)
self.stream_line = py.Stream(s_line_id)
self.stream_bar = py.Stream(s_bar_id)
self.stream_line.open()
self.stream_bar.open()
self.chart_url = url
def stream_to_plotly():
with open('./config.json') as config_file:
plotly_user_config = json.load(config_file)
py.sign_in(plotly_user_config["plotly_username"], plotly_user_config["plotly_api_key"])
url = py.plot([
{
'x': [], 'y': [], 'type': 'scatter',
'stream': {
'token': plotly_user_config['plotly_streaming_tokens'][0],
'maxpoints': 200
}
},
{
'x': [], 'y': [], 'type': 'scatter',
'stream': {
'token': plotly_user_config['plotly_streaming_tokens'][1],
'maxpoints': 200
}
},
{
'x': [], 'y': [], 'type': 'scatter',
'stream': {
'token': plotly_user_config['plotly_streaming_tokens'][2],
'maxpoints': 200
}
}], filename='Raspberry Pi Light Intensity Streaming')
print "View your streaming graph here: ", url
stream = py.Stream(plotly_user_config['plotly_streaming_tokens'][0])
stream.open()
stream1 = py.Stream(plotly_user_config['plotly_streaming_tokens'][1])
stream1.open()
stream2 = py.Stream(plotly_user_config['plotly_streaming_tokens'][2])
stream2.open()
while True:
light_value = get_intensity(1)
print light_value
if(light_value<=AMBIENT_DARK and light_value>=0):
led_when_low();
elif (light_value>AMBIENT_DARK and light_value<AMBIENT_BRIGHT):
led_when_mid();
elif(light_value>=AMBIENT_BRIGHT and light_value<=100):
led_when_high();
else:
led_error(3);
stream.write({'x': datetime.datetime.now(), 'y': ("%.1f" % light_value)})
stream1.write({'x': datetime.datetime.now(), 'y': get_temp(0)})
#stream2.write({'x': datetime.datetime.now(), 'y': read_temp(sensor2_file)[0]})
time.sleep(1)
def __init__(self):
self.username = '******'
self.apikey = 'csRULP61Y5yZ8qYVjTN9'
self.light_token = 'pajbficbtq'
self.sound_token = 'xnqm2wcrgx'
self.k_light_token = 's96vqjowbq'
plotly.sign_in(self.username, self.apikey)
self.sound_stream = plotly.Stream(self.sound_token)
self.light_stream = plotly.Stream(self.light_token)
self.k_light_stream = plotly.Stream(self.k_light_token)
def stream(self):
# Initialize traces
cpu = go.Scatter(
x=[],
y=[],
mode='lines',
stream=self.stream_dicts['cpu']
)
memory = go.Scatter(
x=[],
y=[],
mode='lines',
stream=self.stream_dicts['memory']
)
data = go.Data([cpu,memory])
# Add title to layout object
layout = go.Layout(title='Plot for process %s, %s' % (self.metric.pid, self.metric.label))
# Make a figure object
fig = go.Figure(data=data, layout=layout)
# Send fig to Plotly, initialize streaming plot, open new tab
py.iplot(fig, filename='pypidplot-%s' % self.metric.label)
# We will provide the stream link object the same token that's associated with the trace we wish to stream to
cpu_stream = py.Stream(self.stream_dicts['cpu']['token'])
memory_stream = py.Stream(self.stream_dicts['memory']['token'])
# We then open a connection
cpu_stream.open()
memory_stream.open()
# Delay start of stream by 5 sec (time to switch tabs)
time.sleep(5)
while True:
# Current time on x-axis, random numbers on y-axis
x = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')
# Send data to your plot
cpu_stream.write(dict(x=x, y=self.metric.value()[0]))
memory_stream.write(dict(x=x, y=self.metric.value()[1]))
time.sleep(self.delay) # plot a point every second
# Close the stream when done plotting
cpu_stream.close()
memory_stream.close()
def dataPlotlyHandler():
configuration = ConfigParser.ConfigParser()
configuration.read('credentials.config')
username = configuration.get('plotly', 'username')
apikey = configuration.get('plotly', 'apikey')
streamtokentx = configuration.get('plotly', 'streamtokentx')
streamtokenrx = configuration.get('plotly', 'streamtokenrx')
py.sign_in(username, apikey)
trace_network_tx = Scatter(x=[],
y=[],
stream=Stream(token=streamtokentx, ),
yaxis='tx')
trace_network_rx = Scatter(x=[],
y=[],
stream=Stream(token=streamtokenrx, ),
yaxis='rx')
layout = Layout(title='IoT Lab Network Health System',
yaxis=YAxis(title='Bytes'),
yaxis2=YAxis(title='%', side='right', overlaying="y"))
data = Data([trace_network_tx, trace_network_rx])
fig = Figure(data=data, layout=layout)
print py.plot(fig,
filename='IoT Lab Network Health System',
auto_open=False)
stream_network_tx = py.Stream(streamtokentx)
stream_network_tx.open()
stream_network_rx = py.Stream(streamtokenrx)
stream_network_rx.open()
counter = 0
while True:
output = psutil.net_io_counters()
randoma = randint(0, 1000)
randomb = randint(0, 1000)
stream_network_tx.write({'x': counter, 'y': randoma})
stream_network_rx.write({'x': counter, 'y': randomb})
counter += 1
time.sleep(0.25)
stream_network_tx.close()
stream_network_rx.close()
def create_plot(username, api_key,
weather_token,
sensor_tokens, sensor_names,
title, units,
max_points):
py.sign_in(username, api_key)
traces = [
Scatter(
x=[],
y=[],
name=n,
stream={
'token': t,
# 'maxpoints': max_points,
}
)
for t, n in zip(sensor_tokens, sensor_names)
]
traces.append(
Scatter(
x=[],
y=[],
name='Outside Temperature',
stream={
'token': weather_token,
# 'maxpoints': max_points,
}
)
)
layout = Layout(
title=title,
yaxis=YAxis(
title='Degrees %s' % units.title(),
),
showlegend=True,
)
fig = Figure(data=traces, layout=layout)
plot_url = py.plot(fig, filename=title, extend=True, auto_open=False)
LOG.info('Output graph visible at %s', plot_url)
sensor_streams = {
t: py.Stream(t)
for t in sensor_tokens
}
for s in sensor_streams.values():
s.open()
weather_stream = py.Stream(weather_token)
weather_stream.open()
return sensor_streams, weather_stream
def stream_from_file(self, f_path):
s1 = py.Stream(const.PY_STEPH_STREAM)
s2 = py.Stream(const.PY_PRESS_STREAM)
s3 = py.Stream(const.PY_STEPA_STREAM)
s1.open()
s2.open()
s3.open()
k = 5 # some shape parameter
with open(f_path, 'rb') as csvfile:
print("Opened file")
spamreader = csv.reader(csvfile)
# row_count = sum(1 for row in spamreader)
count = 0
print "reading rows"
for row in spamreader:
if count is not 0:
t = float(row[0])
step_length = float(row[1])
step_height = float(row[2])
pitch = float(row[3])
fsr_frontl = float(row[4]) * const.PY_PRESS_MULTIPLIER
fsr_frontr = float(row[5]) * const.PY_PRESS_MULTIPLIER
fsr_back = float(row[6]) * const.PY_PRESS_MULTIPLIER
s1.write(dict(x=t, y=step_height))
s3.write(dict(x=t, y=pitch))
trace = dict(
x=[-1, 0, 1],
y=[8, 3, 8],
marker=dict(size=[fsr_frontl, fsr_back, fsr_frontr], ))
s2.write(trace)
time.sleep(t - last_t)
if t > 20:
"Stopping stream after 20 seconds"
break
last_t = float(row[0])
count = count + 1
s1.close()
s2.close()
s3.close()
print f_path
def __init__(self):
print "Starting up Plot.ly, please wait"
with open('./config.json') as config_file:
plotly_user_config = json.load(config_file)
pltly.sign_in(plotly_user_config["plotly_username"],
plotly_user_config["plotly_api_key"])
tempTrace = Scatter(
x=[],
y=[],
name='Temperature [deg C]',
stream={
'token': plotly_user_config["plotly_streaming_tokens"][0],
'maxpoints': 100
},
yaxis='y')
presTrace = Scatter(
x=[],
y=[],
name='Pressure [kPa]',
stream={
'token': plotly_user_config["plotly_streaming_tokens"][1],
'maxpoints': 100
},
yaxis='y2')
data = Data([tempTrace, presTrace])
layout = Layout(title='Raspberry Pi Temp/Pressure Graph',
yaxis=YAxis(title='deg C', range=[0, 50]),
yaxis2=YAxis(title='kPa',
overlaying='y',
side='right',
range=[100.87, 101.085]))
fig = Figure(data=data, layout=layout)
plot_url = pltly.plot(fig, filename='Raspberry Pi Streaming Example')
self.tempStream = pltly.Stream(
plotly_user_config["plotly_streaming_tokens"][0])
self.tempStream.open()
self.presStream = pltly.Stream(
plotly_user_config["plotly_streaming_tokens"][1])
self.presStream.open()
print "--Ready to go!"
def test_stream_validate_data(self):
with self.assertRaises(exceptions.PlotlyError):
py.sign_in(un, ak)
my_stream = py.Stream(tk)
my_stream.open()
my_stream.write(dict(x=1, y=10, z=[1])) # assumes scatter...
my_stream.close()
def test_stream_validate_layout(self):
with self.assertRaises(exceptions.PlotlyError):
py.sign_in(un, ak)
my_stream = py.Stream(tk)
my_stream.open()
my_stream.write(Scatter(x=1, y=10), layout=Layout(legend=True))
my_stream.close()
def __init__(self, subject, plotly_config, name):
# call parent init
super(plotlyObserver, self).__init__(subject)
# load config
with open('./%s' % plotly_config) as config_file:
self.plotly_user_config = json.load(config_file)
py.sign_in(self.plotly_user_config["plotly_username"],
self.plotly_user_config["plotly_api_key"])
# set up
self.url = py.plot([{
'x': [],
'y': [],
'type': 'scatter',
'stream': {
'token': self.plotly_user_config['plotly_streaming_tokens'][0],
'maxpoints': 1440
}
}],
filename=name)
# print "View your streaming graph here: ", url
# open stream
self.stream = py.Stream(
self.plotly_user_config['plotly_streaming_tokens'][0])
self.stream.open()
def init_stream(self, chart_type=None):
# Read params
stream_id = self.traces[chart_type]['stream_id']
trace = self.traces[chart_type]['trace']
title = self.traces[chart_type]['title']
filename = self.traces[chart_type]['filename']
# prepare data
data = go.Data([trace])
# Add title to layout object
layout = go.Layout(title=title)
# Make a figure object
fig = go.Figure(data=data, layout=layout)
# Send fig to Plotly, initialize streaming plot, open new tab
url = py.plot(fig, filename=filename, auto_open=False)
s = py.Stream(stream_id)
s.open()
# TODO: now support only one chart per type
self.streams[chart_type] = s
self.chart_url = url
def test_stream_validate_layout(self):
with self.assertRaises(ValueError):
py.sign_in(un, ak)
my_stream = py.Stream(tk)
my_stream.open()
my_stream.write(Scatter(x=[1], y=[10]), layout=Layout(legend=True))
my_stream.close()
def __init__(self,
fn='test',
token='',
key='',
username='',
plotUrl='',
sharing=['public', 'private', 'secret'],
fileopt=['new', 'overwrite', 'extend', 'append']):
''' Constructor.
fn (string) -- the name that will be associated with this figure
fileopt ('new' | 'overwrite' | 'extend' | 'append') -- 'new' creates a
'new': create a new, unique url for this plot
'overwrite': overwrite the file associated with `filename` with this
'extend': add additional numbers (data) to existing traces
'append': add additional traces to existing data lists
world_readable (default=True) -- make this figure private/public
auto_open (default=True) -- Toggle browser options
True: open this plot in a new browser tab
False: do not open plot in the browser, but do return the unique url
sharing ('public' | 'private' | 'sharing') -- Toggle who can view this graph
- 'public': Anyone can view this graph. It will appear in your profile
and can appear in search engines. You do not need to be
logged in to Plotly to view this chart.
- 'private': Only you can view this plot. It will not appear in the
Plotly feed, your profile, or search engines. You must be
logged in to Plotly to view this graph. You can privately
share this graph with other Plotly users in your online
Plotly account and they will need to be logged in to
view this plot.
- 'secret': Anyone with this secret link can view this chart. It will
not appear in the Plotly feed, your profile, or search
engines. If it is embedded inside a webpage or an IPython
notebook, anybody who is viewing that page will be able to
view the graph. You do not need to be logged in to view
this plot.
'''
DomainBehavior.__init__(self)
if token != '' and key != '' and username != '':
py.sign_in(username, key)
trace1 = Scatter(x=[], y=[], stream=dict(token=token))
data = Data([trace1])
self.plotUrl = py.plot(data,
filename=fn,
auto_open=False,
sharing=sharing[0],
fileopt=fileopt[0])
#print(self.plotUrl)
self.s = py.Stream(token)
self.s.open()
# patch TIC
self.sTime = time.time()
self.sNbData = 0
self.x = []
self.y = []
else:
self.s = None
self.state = {'status': 'IDLE', 'sigma': INFINITY}
def setup_metric_streams(self, local_stream_ids, metric_name, num_channels):
for i in range(num_channels):
stream_id = local_stream_ids[i]
self.stream_ids.append(stream_id)
py_stream = py.Stream(stream_id)
py_stream.open()
self.py_streams.append(py_stream)
go_stream = go.Stream(token=stream_id, maxpoints=self.max_points)
self.go_streams.append(go_stream)
traces = []
for i in range(num_channels):
channel_name = "channel_%s" % i
go_stream = self.go_streams[i]
trace = go.Scatter(
x=[],
y=[],
mode='splines',
stream=go_stream,
name=channel_name
)
traces.append(trace)
data = go.Data(traces)
layout = go.Layout(title=metric_name)
fig = go.Figure(data=data, layout=layout)
py.iplot(fig, filename=metric_name)
def test_stream_validate_data(self):
with self.assertRaises(ValueError):
py.sign_in(un, ak)
my_stream = py.Stream(tk)
my_stream.open()
my_stream.write(dict(x=[1], y=[10], z=[1])) # assumes scatter...
my_stream.close()
def initialize_graph():
global stream_1
global s
tls.set_credentials_file(username='******',
api_key='9pjer55x5e',
stream_ids=['efzmil96bw'])
stream_id = tls.get_credentials_file()['stream_ids'][0]
stream_1 = go.Stream(
token=stream_id, # link stream id to 'token' key
maxpoints=80 # keep a max of 80 pts on screen
)
# pick up where we left off
x_data = ab.get_column('time', 'prices', 'test_table.sqlite')
y_data = ab.get_column('last', 'prices', 'test_table.sqlite')
# some magic that makes the plot and uploads it to the plotly hosting site
trace0 = go.Scatter(x=x_data, y=y_data, stream=stream_1)
data = [trace0]
layout = go.Layout(xaxis=dict(showticklabels=False))
fig = go.Figure(data=data, layout=layout)
unique_url = py.plot(fig, filename='basic-line', auto_open=False)
# open stream connection
s = py.Stream(stream_id)
s.open()
def openStream(self):
# Make instance of the Stream link object,
# with same stream id as Stream id object
self.s = py.Stream(self.my_stream_id)
# Open the stream
self.s.open()
def init_plotly_stream():
# init credentials
with open('./credentials.json') as credentials_file:
plotly_user_config = json.load(credentials_file)
# authenticate with provided credentials
plot.sign_in(plotly_user_config["plotly_username"], plotly_user_config["plotly_api_key"])
# configure the plot
url = plot.plot([
{
'x': [],
'y': [],
'type': 'scatter',
'mode':'lines+markers',
'stream': {
'token': plotly_user_config['plotly_streaming_tokens'][0],
'maxpoints': 200
},
}], filename='MS-Temperature')
print "View your streaming graph here: ", url
# attach a stream to the plot
stream = plot.Stream(plotly_user_config['plotly_streaming_tokens'][0])
return stream
def __init__(self,
stream_token,
data_source,
name,
maxpoints,
subplot_id,
parse_data=None,
mode='lines+markers',
xaxis='x',
yaxis='y'):
self.old_value = None
self.data_source = int(data_source.getsockopt_string(zmq.IDENTITY))
self.name = name
self.trace = Scatter(x=[],
y=[],
stream=dict(token=stream_token,
maxpoints=maxpoints),
name=name,
mode=mode,
connectgaps=True,
xaxis=xaxis,
yaxis=yaxis)
self.stream = ply.Stream(stream_token)
self.subplot_id = subplot_id
if parse_data is not None:
self.parse_data = parse_data
def __init__(self, title):
self.title = title
self.directorycurrent = os.path.dirname(os.path.realpath(__file__))
self.directoryconfiguration = self.directorycurrent + '/../configuration/'
self.configuration = ConfigParser.ConfigParser()
self.credentialspath = self.directoryconfiguration + "credentials.config"
self.configuration.read(self.credentialspath)
self.username = self.configuration.get('plotly', 'username')
self.apikey = self.configuration.get('plotly', 'apikey')
self.streamtoken = self.configuration.get('plotly', 'streamtoken')
py.sign_in(self.username, self.apikey)
stream_data = Scatter(x=[],
y=[],
stream=dict(token=self.streamtoken, ))
layout = Layout(title=self.title)
this = Figure(data=[stream_data], layout=layout)
py.plot(this, filename=self.title, auto_open=False)
self.stream = py.Stream(self.streamtoken)
self.stream.open()
time.sleep(5)
def readdata():
n = 0
f = mce_data.SmallMCEFile(
'/data/TIME_Software/cryo/20171009/data_test4_2.858Arms_3hz')
h = f.Read(row_col=True, unfilter='DC').data
d = [[[] for i in range(32)] for j in range(31)]
stream_ids = tls.get_credentials_file()['stream_ids']
stream_id = stream_ids[0]
heatmap = go.Heatmap(stream=dict(token=stream_id))
plot_data = go.Data([heatmap])
py.plot(plot_data)
s = py.Stream(stream_id)
s.open()
time.sleep(5)
while n < 5000:
t = time.time()
for i in range(32):
for j in range(31):
d[j][i] = (np.std(h[j, i, n:n + 1000]))
n = n + 1000
print(d[27][4])
z = ([[d[23][0], d[23][1], d[23][2], d[23][3], d[23][4]],
[d[24][0], d[24][1], d[24][2], d[24][3], d[24][4]],
[d[25][0], d[25][1], d[25][2], d[25][3], d[25][4]],
[d[26][0], d[26][1], d[26][2], d[26][3], d[26][4]],
[d[27][0], d[27][1], d[27][2], d[27][3], d[27][4]]])
s.write(dict(z=z, type='heatmap'))
print("time elapsed for loop: %s" % (time.time() - t))
t = 0
time.sleep(2)
s.close()
def graph(self):
stream_randomizer = Scatter(
x=[],
y=[],
stream=dict(
token=self.streamtoken,
)
)
layout = Layout(
title="IoTPy Randomizer"
)
this = Figure(data=[stream_randomizer], layout=layout)
py.plot(this, filename='IotPy Randomizer', auto_open=False)
stream = py.Stream(self.streamtoken)
stream.open()
time.sleep(5)
counter = 0
while True:
randomizerdata = randint(0,100)
stream.write({'x': counter, 'y': randomizerdata})
counter += 1
time.sleep(0.25)
def __init__(self, stream_tokens):
''' Store the stream tokens in an object
so that the message handler can access them
'''
self.streams = {}
for token in stream_tokens:
self.streams[token] = py.Stream(token)
self.streams[token].open()
def __enter__(self):
self._entry_count += 1
if self.is_open:
return
if self.py_stream is None:
self.py_stream = py.Stream(self.stream.id)
self.py_stream.open()
self.is_open = True
def test_stream_unstreamable():
# even though `name` isn't streamable, we don't validate it --> should pass
py.sign_in(un, ak)
my_stream = py.Stream(tk)
my_stream.open()
my_stream.write(Scatter(x=1, y=10, name='nope'))
my_stream.close()
def getGraphStream(stream_id):
## 2. Setup Stream Link Object
# We will provide the stream link object the same token that's associated with the trace we wish to stream to
graph_stream = py.Stream(stream_id)
# We then open a connection
graph_stream.open()
return graph_stream
def graph(self):
trace_network_tx = Scatter(x=[],
y=[],
stream=Stream(token=self.streamtokentx, ),
yaxis='tx')
trace_network_rx = Scatter(x=[],
y=[],
stream=Stream(token=self.streamtokenrx, ),
yaxis='rx')
layout = Layout(title='IoTPy Network Health System',
yaxis=YAxis(title='Bytes'),
yaxis2=YAxis(title='%', side='right', overlaying="y"))
data = Data([trace_network_tx, trace_network_rx])
fig = Figure(data=data, layout=layout)
print py.plot(fig,
filename='IoTPy Network Health System',
auto_open=False)
stream_network_tx = py.Stream(self.streamtokentx)
stream_network_tx.open()
stream_network_rx = py.Stream(self.streamtokenrx)
stream_network_rx.open()
time.sleep(5)
counter = 0
while True:
output = psutil.net_io_counters()
print "Network Bytes Tx %s" % output.bytes_sent
print "Network Bytes Rx %s" % output.bytes_recv
stream_network_tx.write({'x': counter, 'y': output.bytes_sent})
stream_network_rx.write({'x': counter, 'y': output.bytes_recv})
counter += 1
time.sleep(0.25)
stream_network_tx.close()
stream_network_rx.close()
