def app():
#tokenize = []
auth = Auth()
auth = auth.connect()
streaming_data = DataCollector()
streaming_data = tweepy.Stream(auth, DataCollector())
streaming_data.filter(track=['#python'])
def parallel_wrapper(self, x, currentDate, positive_scans):
try:
Stock_data = DataCollector.getStockData(x)
df = self.getRSI(Stock_data)
if (not 'RSI' in df.columns):
return
df = EMA_Calc.computeEMA(df, "fast_EMA", config.fast_ema_days)
df = EMA_Calc.computeEMA(df, "slow_EMA", config.slow_ema_days)
print(df)
entry_point = self.checkForEntryPoint(df)
if (not entry_point):
return
RSI = df.iloc[-1]["RSI"]
RSI_Calc.Price_Graph(df)
self.customPrint(df, x, RSI)
if (config.SEND_EMAIL):
EmailResults.SendResults(x, config.send_to, RSI)
#Allows you to keep all data for the end of the Bots run. Not doing anything with it yet
stonk = dict()
stonk['date'] = df['Date'].iloc[-1]
stonk['stock'] = x
stonk['Adj Close'] = df['Adj Close'].iloc[-1]
print(stonk['date'])
Testing.log_stock_pick_CSV(stonk)
positive_scans.append(stonk)
except Exception as e:
print("Exception occurred in Parallel_wrapper: " + e)
def __init__(self, ui):
print "hi there"
super(LoadManager, self).__init__()
#QtCore.QObject.__init__()
# store the UI
self.ui = ui
# store load data, system history, commands, and launched processes
self.load_data = {}
self.position = None
self.goal = None
self.signal = None
PATH = "/home/ubuntu/thesis/LoadManager/data/"
HIST_FILE = PATH + "history_file_" + str(time.time()) + ".pkl"
self.history = DataCollector(HIST_FILE)
self.command_queue = deque()
self.process_queue = deque()
# system time
self.start_time = time.time()
# set up signals and slots with GUI
self.cpu_signal.connect(self.ui.updateCPU)
self.process_signal.connect(self.ui.updateProcesses)
self.idle_signal.connect(self.ui.updateIdleness)
# launch the load manager
print "hi"
self.main()
def compute_probas(directory_tagged, directory_untagged, percentage=0.8, number_blocks=10):
d = DataCollector(directory_tagged)
nB = NaifBayes(d, percentage, number_blocks )
results = nB.compute_all(directory_tagged, directory_untagged)
pos = 0
neg = 0
mean = 0
for name, tupleResult in sorted(results.items(), key=lambda key: key[0]):
print(name, " %Pos:", tupleResult[0], " %Neg", tupleResult[1], " %Tot",tupleResult[2], " Time",tupleResult[3])
if(str(name).__contains__('cross')):
if (str(name).__contains__('0')):
pos = tupleResult[0]
neg = tupleResult[1]
else:
pos += tupleResult[0]
neg += tupleResult[1]
if (str(name).__contains__('9')):
print("pos %d",(pos/10))
print("neg %d",(neg/10))
print("mean %d",(((pos/10)+(neg/10))/2))
else:
print("mean %d",(tupleResult[0] + tupleResult[1])/2)
from config import Config
personData = View.collectPersonData(u'INDUKCIÓ', True)
male = (personData['nem'] == u"férfi")
right = (personData['kez'] == u"jobb")
trialszam = personData['trialszam']
session = personData['session']
# pin number 2 for right hand and port number 3 for left hand
if right:
pinNumber = 2
else:
pinNumber = 3
dataCollector = DataCollector('indukcio', personData['sorszam'], personData['nem'], personData['kez'], personData['session'])
if not dataCollector.openFile():
View.showErrorAndQuit(u'Létező beállítások ennél a személynél!\nAz adott sorszámú személynél korábban már elindult ez a blokk. Ha a blokkot újra kell kezdeni ennél a személynél, töröld ki a személy adott blokkjához tartozó .txt fájlt a scriptet tartalmazó mappából.')
size = [1366, 768]
view = View(size, Config.fullscreen)
controller = Controller(view)
view.quit = controller.quit
view.setHands(male, right)
controller.loadThresholds()
if Config.parallelDisabled:
signal.disable()
personData = View.collectPersonData(u'AKTÍV')
male = (personData['nem'] == u"férfi")
right = (personData['kez'] == u"jobb")
# pin number 2 for right hand and port number 3 for left hand
if right:
pinNumber = 2
else:
pinNumber = 3
dataCollector = DataCollector('aktiv', personData['sorszam'],
personData['nem'], personData['kez'])
if not dataCollector.openFile():
View.showErrorAndQuit(
u'Létező beállítások ennél a személynél!\nAz adott sorszámú személynél korábban már elindult ez a blokk. Ha a blokkot újra kell kezdeni ennél a személynél, töröld ki a személy adott blokkjához tartozó .txt fájlt a scriptet tartalmazó mappából.'
)
size = [1366, 768]
view = View(size, Config.fullscreen)
controller = Controller(view)
view.quit = controller.quit
view.setHands(male, right)
import csv
from dataCollector import DataCollector
from random import shuffle
data = DataCollector()
newsAgencies = ["the-economist", "bloomberg", "financial-times", "crypto-coin-news"]
clickbait = ["buzzfeed", "entertainment-weekly", "mirror", "mtv-news"]
def create_all_news_CSV():
with open('BTCheadlines.csv', 'w', newline='') as csvfile:
writer = csv.writer(csvfile, delimiter=';',
dialect = "excel-tab")
writer.writerow(["id", "name", "author", "title", "description", "url", "urlToImage",
"publishedAt"])
for article in allData:
fields = []
try:
if article[header[0]]['id'] == None:
fields.append("None")
else:
fields.append(article[header[0]]['id'])
if article[header[0]]['name'] == None:
fields.append("None")
else:
fields.append(article[header[0]]['name'])
#!/usr/bin/python
import Adafruit_DHT
import time
from dataCollector import DataCollector
dataCollector = DataCollector()
dataCollector.saveSensorData(dataCollector.cpuTempSensorId,
dataCollector.getCpuTemp())
# sendSensorData(cpuTempSensorId, cpuTemp)
# Sensor should be set to Adafruit_DHT.DHT11,
# Adafruit_DHT.DHT22, or Adafruit_DHT.AM2302.
sensor = Adafruit_DHT.DHT11
# Example using a Raspberry Pi with DHT sensor
# connected to GPIO23.
pin = 25
# Try to grab a sensor reading. Use the read_retry method which will retry up
# to 15 times to get a sensor reading (waiting 2 seconds between each retry).
humidity, temperature = Adafruit_DHT.read_retry(sensor, pin)
# Note that sometimes you won't get a reading and
# the results will be null (because Linux can't
# guarantee the timing of calls to read the sensor).
# If this happens try again!
if humidity is not None and temperature is not None:
print 'Temp={0:0.1f}*C Humidity={1:0.1f}%'.format(temperature, humidity)
dataCollector.saveSensorData(dataCollector.tempSensorId, temperature)
dataCollector.saveSensorData(dataCollector.humiditySensorId, humidity)
Flag("new_data", {"image": stream, "THR": THR, "STR": STR})
stream.seek(0)
stream.truncate()
time.sleep(.0001)
carlos = car() #initialize car object
tc = termCondition()
js_source = SocketReader(commands_in_sock) #joystick input from socket
server_thread = threading.Thread(target=server_process,
args=[tc, stream_out_sock, stream])
controller = ControllerObject(js_source) #controller handler
controller.start_thread()
streamer = Streamer(stream_out_sock)
collector = DataCollector()
try:
camera = picamera.PiCamera()
camera.resolution = (128, 96)
camera.framerate = 20
server_thread.start()
time.sleep(2)
camera.start_recording(stream, format='rgb')
while not tc.isSet():
commands = controller.carpoll() #get commands from controller
carlos.go(commands[1], commands[0]) #tell car to execute commands
commands_lock.acquire()
car_commands = commands #put commands in shared variable so they can be read by other thread
commands_lock.release()
time.sleep(.01)
#!/usr/bin/python
import Adafruit_DHT;
import time;
from dataCollector import DataCollector
dataCollector = DataCollector()
dataCollector.saveSensorData(dataCollector.cpuTempSensorId, dataCollector.getCpuTemp());
# sendSensorData(cpuTempSensorId, cpuTemp)
# Sensor should be set to Adafruit_DHT.DHT11,
# Adafruit_DHT.DHT22, or Adafruit_DHT.AM2302.
sensor = Adafruit_DHT.DHT11
# Example using a Raspberry Pi with DHT sensor
# connected to GPIO23.
pin = 25
# Try to grab a sensor reading. Use the read_retry method which will retry up
# to 15 times to get a sensor reading (waiting 2 seconds between each retry).
humidity, temperature = Adafruit_DHT.read_retry(sensor, pin)
# Note that sometimes you won't get a reading and
# the results will be null (because Linux can't
# guarantee the timing of calls to read the sensor).
# If this happens try again!
if humidity is not None and temperature is not None:
print 'Temp={0:0.1f}*C Humidity={1:0.1f}%'.format(temperature, humidity)
dataCollector.saveSensorData(dataCollector.tempSensorId, temperature)
dataCollector.saveSensorData(dataCollector.humiditySensorId, humidity)
class LoadManager(QtCore.QObject):
cpu_signal = QtCore.pyqtSignal((str), (float))
process_signal = QtCore.pyqtSignal((str), (str))
idle_signal = QtCore.pyqtSignal((str), (bool))
def __init__(self, ui):
print "hi there"
super(LoadManager, self).__init__()
#QtCore.QObject.__init__()
# store the UI
self.ui = ui
# store load data, system history, commands, and launched processes
self.load_data = {}
self.position = None
self.goal = None
self.signal = None
PATH = "/home/ubuntu/thesis/LoadManager/data/"
HIST_FILE = PATH + "history_file_" + str(time.time()) + ".pkl"
self.history = DataCollector(HIST_FILE)
self.command_queue = deque()
self.process_queue = deque()
# system time
self.start_time = time.time()
# set up signals and slots with GUI
self.cpu_signal.connect(self.ui.updateCPU)
self.process_signal.connect(self.ui.updateProcesses)
self.idle_signal.connect(self.ui.updateIdleness)
# launch the load manager
print "hi"
self.main()
def init(self):
""" Create SSH shells and launch roscore on SQUIRREL."""
for machine_id in MACHINES.keys():
machine = MACHINES[machine_id]
if machine["ssh"] is None:
machine["ssh"] = self.openSSH(machine["usr"], machine["ip"])
print "Launching roscore..."
self.executeCommand({"machine" : SQUIRREL,
"command" : ROSCORE,
"catchOut" : CATCH_NODES,
"isMovable" : False})
def openSSH(self, usr, ip, catch_output=False):
""" Open an SSH connection to the specified machine. """
SSH = ["ssh", "-t", "-t", usr + "@" + ip]
if catch_output:
ssh = psutil.Popen(SSH, stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
else:
ssh = psutil.Popen(SSH, stdin=subprocess.PIPE)
return ssh
def executeCommand(self, command, delay=WAIT_TIME):
""" Execute a command on the specified machine. """
oldssh = command["machine"]["ssh"]
oldssh.stdin.write(command["command"])
# log to history
self.history.updateProcess(command["machine"]["id"], command["command"])
# add to process_queue
command["process"] = oldssh
self.process_queue.append(command)
# get new ssh shell for this machine
newssh = self.openSSH(command["machine"]["usr"],
command["machine"]["ip"],
command["catchOut"])
command["machine"]["ssh"] = newssh
# sleep
time.sleep(delay)
def monitorCPUs(self):
""" Launch CPU monitoring node on all machines. """
for machine_id in MACHINES.keys():
print "Launching CPU monitor code for machine: " + machine_id
self.executeCommand({"machine" : MACHINES[machine_id],
"command" : ACTIVITY_LAUNCH,
"catchOut" : True,
"isMovable" : False},
delay=0.0)
def isIdle(self, machine_id):
"""
Bang-bang control loop to avoid hysteresis. Set high and low
thresholds of CPU activity and change load_data's "isIdle"
parameter accordingly.
"""
cpu = self.load_data[machine_id]["activity"].output()
idle = self.load_data[machine_id]["isIdle"]
if ((idle and (cpu > CPU_HI)) or ((not idle) and (cpu < CPU_LO))):
self.idle_signal.emit(machine_id, not idle)
return not idle
else:
self.idle_signal.emit(machine_id, idle)
return idle
def findIdleMachine(self):
"""
Return (the most) idle machine.
As implemented, this method simply returns the machine in
MACHINES that is currently idle, with the lowest CPU
utilization.
"""
lowest_cpu = float("inf")
most_idle = None
for machine_id in MACHINES.keys():
# check if idle
if self.load_data[machine_id]["isIdle"]:
# check if lowest CPU utilization so far
activity = self.load_data[machine_id]["activity"].output()
if activity < lowest_cpu:
lowest_cpu = activity
most_idle = MACHINES[machine_id]
# return None if no idle machines
return most_idle
def launchTasks(self):
"""
Set up a polling loop. On each tic:
1) Check process_queue and see if there are any processes
running on non-idle machines. Mark those processes and add
them back to the command_queue.
2) Kill all marked processes.
3) Check command_queue and launch any pending commands, making
sure to edit the amcl launch file if needed.
"""
while True:
# keep a list of processes to be killed
marked_processes = []
# check process_queue
for task in self.process_queue:
# check if movable
if task["isMovable"]:
# check if machine is not idle anymore and remove
machine = task["machine"]
if not self.load_data[machine["id"]]["isIdle"] or self.signal.message() == "switch":
# find (the most) idle machine
idle_machine = self.findIdleMachine()
self.signal.clear()
# only move the process if there is an idle machine on the network
if idle_machine is not None:
# mark and remove later
marked_processes.append(task)
# change info
new_task = task.copy()
new_task["machine"] = idle_machine
new_task["process"] = None
self.command_queue.append(new_task)
# now kill all marked processes
for task in marked_processes:
print ("********************** Killing process on " +
task["machine"]["id"] + ": " + task["command"])
self.process_signal.emit(task["machine"]["id"],
"Killing: " + task["command"] + "\n")
self.process_queue.remove(task)
task["process"].terminate() # don't bother waiting
# now check command_queue and launch
while len(self.command_queue) > 0:
command = self.command_queue.popleft()
# adjust amcl launch file if needed
print self.goal.goal()
if command["command"] == AMCL_LAUNCH:
self.initPosition(command["machine"])
print ("********************** Launching process on " +
command["machine"]["id"] + ": " + command["command"])
self.process_signal.emit(command["machine"]["id"],
"Launching: " + command["command"] + "\n")
# execute
self.executeCommand(command)
if command["command"] == AMCL_LAUNCH and self.goal.goal() != None:
self.pubgoal.simple_move(goal.goal())
# print state of all processes
self.printState()
print "Robot At:"
print self.position.position()
# tic
print "--------------------------------------------------------------------------"
time.sleep(UPDATE_INTERVAL)
def initPosition(self, machine):
"""
SSH into machine and alter amcl_demo_edited.launch file to include
current position as default.
"""
print "Setting initial position on " + machine["id"] + " :"
print self.position.position()
x = self.position.position()["x"]
y = self.position.position()["y"]
a = self.position.position()["a"]
print "executing command " + INIT_POSITION + " %s %s %s" % (x, y, a)
self.executeCommand({"machine" : machine,
"command" : INIT_POSITION + " %s %s %s" % (x, y, a)+"\n",
"catchOut" : CATCH_NODES,
"isMovable" : False},
delay=0.5)
def initPositionTopic(self):
"""
Publish to initialPose the current position
"""
print "Publising to Initial Pose"
print self.position.position()
x = self.position.position()["x"]
y = self.position.position()["y"]
a = self.position.position()["a"]
rate = rospy.Rate(1) # 10hz
pose = geometry_msgs.msg.PoseWithCovarianceStamped()
pose.header.frame_id = 'map'
pose.pose.pose.position.x=-1
pose.pose.pose.position.y=0
pose.pose.pose.position.z=0
pose.pose.covariance=[0.25, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.25, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.06853891945200942]
pose.pose.pose.orientation.z=0.0267568523876
pose.pose.pose.orientation.w=0.999641971333
def printState(self):
for task in self.process_queue:
print "Machine: " + task["machine"]["id"]
print "CPU %: " + str(self.load_data[task["machine"]["id"]]["activity"].output())
print "isIdle: " + str(self.load_data[task["machine"]["id"]]["isIdle"])
print "isMovable: " + str(task["isMovable"])
print "Command: " + task["command"]
def navigationSetup(self):
#Add the requisite commands for autonomous navigation
#to the command queue.
self.position = PositionTracker()
self.goal = GoalTracker()
self.signal = ManualSignal()
print "setting up navigation with initial pose"
print self.position.position()
self.command_queue.append({"machine" : ASDF,
"command" : MIN_LAUNCH,
"catchOut" : CATCH_NODES,
"isMovable" : False})
self.command_queue.append({"machine" : ASDF,
"command" : SENSE_LAUNCH,
"catchOut" : CATCH_NODES,
"isMovable" : False})
self.command_queue.append({"machine" : ASDF, #SQUIRREL,
"command" : AMCL_LAUNCH,
"catchOut" : CATCH_NODES,
"isMovable" : True})
def mappingSetup(self):
"""
Add the requisite commands for mapping to the command queue.
Note that this will not start keyboard_teleop, since that
requires additional user interaction (which is outside the scope
of this module). That command may be launched on ASDF as follows:
$ roslaunch turtlebot_teleop keyboard_teleop.launch
Similarly, this does not start RViz. You may do so manually:
$ roslaunch turtlebot_rviz_launchers view_navigation.launch
"""
self.command_queue.append({"machine" : ASDF,
"command" : MIN_LAUNCH,
"catchOut" : CATCH_NODES,
"isMovable" : False})
self.command_queue.append({"machine" : ASDF,
"command" : MAPPING_LAUNCH,
"catchOut" : CATCH_NODES,
"isMovable" : True})
def genericCPUCallback(self, data, machine_id):
"""
Generic callback function for handling CPU utilization data.
This function is extendable to future implementations of the
activity_monitor package, which may include data other than
simple CPU utilization percentage.
"""
self.load_data[machine_id]["activity"].update(float(data.data))
self.load_data[machine_id]["isIdle"] = self.isIdle(machine_id)
self.history.updateMachine(machine_id,
raw_cpu=float(data.data),
filtered_cpu=self.load_data[machine_id]["activity"].output())
self.cpu_signal.emit(machine_id,
self.load_data[machine_id]["activity"].output())
# rospy.loginfo("CPU activity for " + machine_id + ": " +
# str((load_data[machine_id]["activity"].output(),
# float(data.data))))
def nice(self, command, niceness=0):
return "nice -n " + str(niceness) + " " + command
def onTerminateCallback(self, process):
""" Callback function for process termination. """
print("Process {} terminated".format(process))
def killAll(self):
""" Kill all running processes. """
print "\nKeyboardInterrupt detected."
print "Terminating all processes cleanly."
process_list = []
# gently terminate all processes on the process_queue
for task in self.process_queue:
process = task["process"]
process_list.append(process)
process.terminate()
#Add roscore to the list of processes to terminate
# wait, then forceably kill all remaining processes
dead, alive = psutil.wait_procs(process_list, timeout=5,
callback=self.onTerminateCallback)
for process in alive:
process.kill()
# save system history
print "\nSaving system history to file."
self.history.save()
# main script
def main(self):
try:
# launch roscore
self.init()
# set up CPU monitoring
rospy.init_node("load_manager", anonymous=True,
disable_signals=True)
self.monitorCPUs()
# set up callbacks
callbacks = {}
for machine_id in MACHINES.keys():
callbacks[machine_id] = functools.partial(self.genericCPUCallback,
machine_id=machine_id)
# set ROS subscriptions
for machine_id in MACHINES.keys():
# initialize to empty filter
self.load_data[machine_id] = {"activity" : FilterCPU(FILTER_INIT, FILTER_TAP),
"isIdle" : False}
# subscribe
rospy.Subscriber("cpu_util/" + machine_id, String,
callbacks[machine_id])
# set up and launch all tasks
self.navigationSetup()
self.launchTasks()
except KeyboardInterrupt:
# terminate all processes gently
self.killAll()
sys.exit()