def main():
streamer = Streamer(bucket_name=BUCKET_NAME,
bucket_key=BUCKET_KEY,
access_key=ACCESS_KEY)
while True:
cpu_temperature = get_cpu_temperature()
if METRIC_UNITS:
streamer.log("CPU Temperature(C)", cpu_temperature)
else:
cpu_temperature = cpu_temperature * 9.0 / 5.0 + 32.0
streamer.log("CPU Temperature(F)",
str("{0:.2f}".format(cpu_temperature)))
cpu_percents = psutil.cpu_percent(percpu=True)
streamer.log_object(cpu_percents, key_prefix="cpu")
cpu_percent = psutil.cpu_percent(percpu=False)
streamer.log("CPU Usage", cpu_percent)
disk = psutil.disk_usage('/')
disk_total = disk.total / 2**30
streamer.log("Disk Total(GB)", str("{0:.2f}".format(disk_total)))
disk_used = disk.used / 2**30
streamer.log("Disk Used(GB)", str("{0:.2f}".format(disk_used)))
disk_free = disk.free / 2**30
streamer.log("Disk Free(GB)", str("{0:.2f}".format(disk_free)))
disk_percent_used = disk.percent
streamer.log("Disk Used(%)", str("{0:.2f}".format(disk_percent_used)))
mem = psutil.virtual_memory()
mem_total = mem.total / 2**20
streamer.log("Memory Total(MB)", str("{0:.2f}".format(mem_total)))
mem_avail = mem.available / 2**20
streamer.log("Memory Available(MB)", str("{0:.2f}".format(mem_avail)))
mem_percent_used = mem.percent
streamer.log("Memory Used(%)", str("{0:.2f}".format(mem_percent_used)))
mem_used = mem.used / 2**20
streamer.log("Memory Used(MB)", str("{0:.2f}".format(mem_used)))
mem_free = mem.free / 2**20
streamer.log("Memory Free(MB)", str("{0:.2f}".format(mem_free)))
net = psutil.net_io_counters()
net_bytes_sent = net.bytes_sent / 2**20
streamer.log("Network MB Sent", str("{0:.2f}".format(net_bytes_sent)))
net_bytes_recv = net.bytes_recv / 2**20
streamer.log("Network MB Received",
str("{0:.2f}".format(net_bytes_recv)))
net_errin = net.errin
streamer.log("Network Errors Receiving", str(net_errin))
net_errout = net.errout
streamer.log("Network Errors Sending", str(net_errout))
net_dropin = net.dropin
streamer.log("Incoming Packets Dropped", str(net_dropin))
net_dropout = net.dropout
streamer.log("Outgoing Packets Dropped", str(net_dropout))
streamer.flush()
time.sleep(60 * MINUTES_BETWEEN_READS)
def main():
streamer = Streamer(bucket_name=BUCKET_NAME, bucket_key=BUCKET_KEY, access_key=ACCESS_KEY)
while True:
cpu_temperature = get_cpu_temperature()
if METRIC_UNITS:
streamer.log("CPU Temperature(C)",cpu_temperature)
else:
cpu_temperature = cpu_temperature * 9.0 / 5.0 + 32.0
streamer.log("CPU Temperature(F)",str("{0:.2f}".format(cpu_temperature)))
cpu_percents = psutil.cpu_percent(percpu=True)
streamer.log_object(cpu_percents, key_prefix="cpu")
cpu_percent = psutil.cpu_percent(percpu=False)
streamer.log("CPU Usage",cpu_percent)
disk = psutil.disk_usage('/')
disk_total = disk.total / 2**30
streamer.log("Disk Total(GB)",str("{0:.2f}".format(disk_total)))
disk_used = disk.used / 2**30
streamer.log("Disk Used(GB)",str("{0:.2f}".format(disk_used)))
disk_free = disk.free / 2**30
streamer.log("Disk Free(GB)",str("{0:.2f}".format(disk_free)))
disk_percent_used = disk.percent
streamer.log("Disk Used(%)",str("{0:.2f}".format(disk_percent_used)))
mem = psutil.virtual_memory()
mem_total = mem.total / 2**20
streamer.log("Memory Total(MB)",str("{0:.2f}".format(mem_total)))
mem_avail = mem.available / 2**20
streamer.log("Memory Available(MB)",str("{0:.2f}".format(mem_avail)))
mem_percent_used = mem.percent
streamer.log("Memory Used(%)",str("{0:.2f}".format(mem_percent_used)))
mem_used = mem.used / 2**20
streamer.log("Memory Used(MB)",str("{0:.2f}".format(mem_used)))
mem_free = mem.free / 2**20
streamer.log("Memory Free(MB)",str("{0:.2f}".format(mem_free)))
net = psutil.net_io_counters()
net_bytes_sent = net.bytes_sent / 2**20
streamer.log("Network MB Sent",str("{0:.2f}".format(net_bytes_sent)))
net_bytes_recv = net.bytes_recv / 2**20
streamer.log("Network MB Received",str("{0:.2f}".format(net_bytes_recv)))
net_errin = net.errin
streamer.log("Network Errors Receiving",str(net_errin))
net_errout = net.errout
streamer.log("Network Errors Sending",str(net_errout))
net_dropin = net.dropin
streamer.log("Incoming Packets Dropped",str(net_dropin))
net_dropout = net.dropout
streamer.log("Outgoing Packets Dropped",str(net_dropout))
streamer.flush()
time.sleep(60*MINUTES_BETWEEN_READS)
async def send_data():
stream_data = deepcopy(data.statsDict)
stream_data['user_count'] = len(data.userSet)
stream_data['server_count'] = len(bot.guilds)
stream_data['total_command_count'] = await bot.cogs.get('StatsCog').get_total_helper(data.statsDict)
streamer = Streamer(bucket_name="Feyre", bucket_key=bucket_key, access_key=access_key, buffer_size=200)
streamer.log_object(stream_data)
streamer.flush()
streamer.close()
class InitialStateLogObject(TerminatorBlock):
""" Initial State block for logging objects
"""
version = VersionProperty("1.0.0")
access_key = StringProperty(title='Access Key',
default='[[INITIAL_STATE_ACCESS_KEY]]')
bucket_name = StringProperty(title='Bucket Name', default='New Bucket')
bucket_key = StringProperty(title='Bucket Key', default='')
object = Property(title='Object', default='{{ $.to_dict() }}')
buffer_size = IntProperty(title='Buffer Size', default=10)
def __init__(self):
super().__init__()
self._streamer = None
def configure(self, context):
super().configure(context)
try:
kwargs = {'access_key': self.access_key()}
if self.bucket_name():
kwargs['bucket_name'] = self.bucket_name()
if self.bucket_key():
kwargs['bucket_key'] = self.bucket_key()
if self.buffer_size():
kwargs['buffer_size'] = self.buffer_size()
self._streamer = Streamer(**kwargs)
except Exception as e:
self.logger.error("Failed to create streamer: {}".format(e))
raise e
def process_signals(self, signals):
for s in signals:
try:
self._streamer.log_object(self.object(s))
except Exception as e:
self.logger.warning("Failed to log object: {}".format(e))
self._streamer.flush()
def stop(self):
super().stop()
self._streamer.close()
import psutil
from ISStreamer.Streamer import Streamer
streamer = Streamer(bucket_name="test object logging", debug_level=2)
# Example dict
streamer.log_object({"foo": "1", "bar": "2"})
# Example lists
cpu_percents = psutil.cpu_percent(percpu=True)
streamer.log_object(cpu_percents, key_prefix="cpu")
streamer.log_object(['1', '2', '3'])
# Example objects with attributes
streamer.log_object(psutil.virtual_memory())
streamer.close()
ini_file_location="./isstreamer.ini",
debug_level=1,
)
sample_rate_in_ms = 100
for x in range(1000):
streamer.log("sample", x)
# Get total CPU usage
cpu_percent = psutil.cpu_percent()
streamer.log("cpu_total", cpu_percent)
# Get individual CPU usage
cpu_percents = psutil.cpu_percent(percpu=True)
streamer.log_object(cpu_percents, key_prefix="cpu")
# Get the virtual memory usage
memory = psutil.virtual_memory()
streamer.log_object(memory, key_prefix="virtual_mem")
# Get the swap memory usage
swap = psutil.swap_memory()
streamer.log_object(swap, key_prefix="swap_mem")
# Get the network usage
network = psutil.net_io_counters()
streamer.log_object(network, key_prefix="net_io")
# flush the stream to ensure optimal buffer and consumption experience
streamer.flush()
from ISStreamer.Streamer import Streamer
def get_cpu_temperature():
process = Popen(['vcgencmd', 'measure_temp'], stdout=PIPE)
output, _error = process.communicate()
return float(output[output.index('=') + 1:output.rindex("'")])
streamer = Streamer(bucket_name=":computer: Pi3 Performance", bucket_key="piperf1", access_key="INSERT YOUR INITIAL STATE ACCESS KEY HERE")
while True:
cpu_temperature = get_cpu_temperature()
cpu_temperature = cpu_temperature * 9.0 / 5.0 + 32.0
streamer.log("CPU Temperature(F)",format(cpu_temperature,".1f"))
cpu_percents = psutil.cpu_percent(percpu=True)
streamer.log_object(cpu_percents, key_prefix="cpu")
cpu_percent = psutil.cpu_percent(percpu=False)
streamer.log("CPU Usage",cpu_percent)
disk = psutil.disk_usage('/')
disk_total = disk.total / 2**30
streamer.log("Disk Total(GB)",format(disk_total,".1f"))
disk_used = disk.used / 2**30
streamer.log("Disk Used(GB)",format(disk_used,".1f"))
disk_free = disk.free / 2**30
streamer.log("Disk Free(GB)",format(disk_free,".1f"))
disk_percent_used = disk.percent
streamer.log("Disk Used(%)",format(disk_percent_used,".1f"))
mem = psutil.virtual_memory()
try:
while 1:
quelist = (QUE_NAME)
jobque = r.brpop(quelist, 10)
if jobque:
redis_val = json.loads(jobque[1].decode("utf-8"))
kprefix = redis_val.get('key_prefix')
epoch00 = int(redis_val.get('epoch'))
bucket = redis_val.get('bucket')
# print(epoch00, kprefix, bucket)
# streamer.log_object(bucket, key_prefix=kprefix)
streamer.log_object(bucket, key_prefix=kprefix, epoch=epoch00)
time.sleep(0.1)
else:
b = { "tstamp": time.time() }
streamer.log_object(b)
streamer.flush()
except Exception as e:
print(e)
sys.exit()
except KeyboardInterrupt:
print('[dequeue] intterupted by keyboard')
sys.exit()
# redis
try:
pipe = r.pipeline()
pipe.set(r_KEY_DASH_BTC_PRICE, json.dumps(dashbtc, sort_keys=True))
pipe.set(r_KEY_DASH_USD_PRICE, json.dumps(dashusd, sort_keys=True))
pipe.zadd(r_SS_DASH_BTC_PRICE, epoch00, str(epoch00) + ':' + str(dashbtc['avg']))
pipe.zadd(r_SS_DASH_USD_PRICE, epoch00, str(epoch00) + ':' + str(dashusd['avg']))
response = pipe.execute()
except Exception as e:
print(e.args[0])
pass
# ISS
try:
streamer.log_object(dashbtc, key_prefix=ISS_PREFIX_BTC, epoch=epoch00)
streamer.log_object(dashusd, key_prefix=ISS_PREFIX_USD, epoch=epoch00)
streamer.flush()
streamer.close()
except Exception as e:
print(e.args[0])
pass
except Exception as e:
print(e.args[0])
except KeyboardInterrupt:
sys.exit()
# Provide a client_key from local ini file, override buffer and flush for optimal streaming
streamer = Streamer(bucket_name="Example Performance Metrics",bucket_key="compute_metrics", buffer_size=100, ini_file_location="./isstreamer.ini", debug_level=1)
sample_rate_in_ms=100
for x in range(1000):
streamer.log("sample", x)
# Get total CPU usage
cpu_percent = psutil.cpu_percent()
streamer.log("cpu_total", cpu_percent)
# Get individual CPU usage
cpu_percents = psutil.cpu_percent(percpu=True)
streamer.log_object(cpu_percents, key_prefix="cpu")
# Get the virtual memory usage
memory = psutil.virtual_memory()
streamer.log_object(memory, key_prefix="virtual_mem")
# Get the swap memory usage
swap = psutil.swap_memory()
streamer.log_object(swap, key_prefix="swap_mem")
# Get the network usage
network = psutil.net_io_counters()
streamer.log_object(network, key_prefix="net_io")
# flush the stream to ensure optimal buffer and consumption experience
streamer.flush()
btcusd['avg'] = round(mean(sorted(l_btcusd)[1:-1]), 2)
btcusd['tstamp'] = epoch00
# redis
try:
pipe = r.pipeline()
pipe.zadd(r_SS_BTC_PRICE, epoch00,
str(epoch00) + ':' + str(btcusd['avg']))
pipe.set(r_KEY_BTC_PRICE, json.dumps(btcusd, sort_keys=True))
response = pipe.execute()
except Exception as e:
print(e.args[0])
pass
# ISS
try:
streamer.log_object(btcusd, key_prefix=ISS_PREFIX, epoch=epoch00)
streamer.flush()
streamer.close()
except Exception as e:
print(e.args[0])
pass
except Exception as e:
print(e.args[0])
except KeyboardInterrupt:
sys.exit()
import psutil
from ISStreamer.Streamer import Streamer
streamer = Streamer(bucket_name="test object logging", debug_level=2)
# Example dict
streamer.log_object({"foo": "1", "bar": "2"})
# Example lists
cpu_percents = psutil.cpu_percent(percpu=True)
streamer.log_object(cpu_percents, key_prefix="cpu")
streamer.log_object(['1', '2', '3'])
# Example objects with attributes
streamer.log_object(psutil.virtual_memory())
streamer.close()
if (checkFormat(str(columns[7]), "$iiiiii.ii") == False):
print("DATA INVALID %s COLUMN 7" % str(columns[7]))
else:
Epoch = convertTime(columns[7])
if (checkFormat(str(columns[1]), "iiiiiiii") == False
and checkFormat(str(columns[1]), "-iiiiiii") == False):
print("DATA INVALID %s COLUMN 1" % str(columns[1]))
if (checkFormat(str(columns[2]), "iiii") == False
and checkFormat(str(columns[2]), "-iii") == False):
print("DATA INVALID %s COLUMN 2" % str(columns[2]))
else:
streamer.log_object(int(columns[2]),
key_prefix="Wind Speed/Hz",
epoch=Epoch)
print("Wind Speed/Hz: %s" % columns[2])
if (checkFormat(str(columns[3]), "-iiii") == False
and checkFormat(str(columns[2]), "iiiii") == False):
print("DATA INVALID %s COLUMN 3" % str(columns[3]))
else:
streamer.log_object(int(columns[3]),
key_prefix="Magnetometer",
epoch=Epoch)
print("Magnetometer: %s" % columns[3])
if (checkFormat(str(columns[4]), "iiiii") == False
and checkFormat(str(columns[4]), "-iiii") == False):
print("DATA INVALID %s COLUMN 4" % str(columns[4]))