def _test_runUDPThreadCorrectly(self):
"""
@return:
"""
threads = Threads(self.CONNECTION_INFO, self.gsi)
return self.assertIsNone(threads.runUDPThreadReceive())
def _test_runKISSThreadCorrectly(self):
"""
@return:
"""
threads = Threads(self.CONNECTION_INFO, self.gsi)
print threads.runKISSThreadReceive()
def _mount_volume(self):
'''
directories on hosts mounted to containers
'''
if self.volume_mounted:
return
# create work dir
(work_dir, host_work_dir, name) = self.work_volume
cmd = 'ssh ' + self.node + ' "sudo rm -rf ' + host_work_dir + '; mkdir -p ' + host_work_dir + '"'
os.system(cmd)
self.volume_mounted = True
# if it is worker task or data not mounted from local host, then read data from HDFS
if self.role == "ps":
return
if self.local_mounted:
return
if self.hdfs_url is None or self.hdfs_url == '':
raise ValueError('HDFS data URL is not specified')
(data_dir, host_data_dir, name) = self.data_volume
pool = Threads()
for data in self.hdfs_url:
fn = data.split("/")[-1]
local_file = host_data_dir + fn
# force copy even exist: some file may be broken due to interruption
cmd = 'ssh ' + self.node + ' "/usr/local/hadoop/bin/hadoop fs -copyToLocal -f ' + data + ' ' + local_file + '"'
os.system(cmd)
thread = threading.Thread(target=(lambda cmd=cmd: os.system(cmd)),
args=())
pool.add(thread)
pool.start()
pool.wait()
def __init__(self, argumentsdict, parent=None):
QtGui.QWidget.__init__(self, parent)
QtGui.QToolTip.setFont(QtGui.QFont('SansSerif', 18))
self.enviromentDesktop = os.environ.get('DESKTOP_SESSION')
self.connection = ''
self.settingsfile = '.settings'
self.setArguments(argumentsdict)
self.initUI()
self.initButtons()
self.initFields()
self.setParameters()
self.initLogo()
self.initConfiguration()
self.initConsole()
self.gsi = GroundStationInterface(self.CONNECTION_INFO, "Vigo",
client_amp.ClientProtocol)
self.threads = Threads(self.CONNECTION_INFO, self.gsi)
# Initialize the reactor parameters needed for the pyqt enviroment
client_amp.Client(self.CONNECTION_INFO, self.gsi, self.threads).createconnection(test=False)
def delete(self, all):
'''delete the task'''
pool = Threads()
thread = threading.Thread(target=(os.system('kubectl delete -f ' +
self.yaml)),
args=())
pool.add(thread)
if all:
(work_dir, host_work_dir, name) = self.work_volume
cmd = 'timeout 10 ssh ' + self.node + ' "sudo rm -r ' + host_work_dir + '"'
thread = threading.Thread(target=(lambda cmd=cmd: os.system(cmd)),
args=())
pool.add(thread)
pool.start()
pool.wait()
def getLongestThreads(startdate, enddate, identities_db):
main_topics = Threads(startdate, enddate, identities_db)
longest_threads = main_topics.topLongestThread(10)
print "Top longest threads: "
l_threads = {}
l_threads['message_id'] = []
l_threads['length'] = []
l_threads['subject'] = []
l_threads['date'] = []
l_threads['initiator_name'] = []
l_threads['initiator_id'] = []
for email in longest_threads:
l_threads['message_id'].append(email.message_id)
l_threads['length'].append(main_topics.lenThread(email.message_id))
l_threads['subject'].append(email.subject)
l_threads['date'].append(str(email.date))
l_threads['initiator_name'].append(email.initiator_name)
l_threads['initiator_id'].append(email.initiator_id)
return l_threads
def getLongestThreads(startdate, enddate, identities_db):
main_topics = Threads(startdate, enddate, identities_db)
longest_threads = main_topics.topLongestThread(10)
print "Top longest threads: "
l_threads = {}
l_threads['message_id'] = []
l_threads['length'] = []
l_threads['subject'] = []
l_threads['date'] = []
l_threads['initiator_name'] = []
l_threads['initiator_id'] = []
for email in longest_threads:
l_threads['message_id'].append(email.message_id)
l_threads['length'].append(main_topics.lenThread(email.message_id))
l_threads['subject'].append(email.subject)
l_threads['date'].append(str(email.date))
l_threads['initiator_name'].append(email.initiator_name)
l_threads['initiator_id'].append(email.initiator_id)
return l_threads
fileName = download_links[download_links.rfind('/') + 1: ]
print(fileName)
urllib.request.urlretrieve(download_links, fileName)
file = drive.CreateFile({'title': fileName})
file.SetContentFile(fileName)
file.Upload()
song["link"] = 'https://docs.google.com/uc?export=download&id=' + file['id']
print('Inserting: ' + name)
songs.insert_one(song)
except Exception as e:
print("type error: " + str(e))
threads = Threads(3, data)
threads.run(func, None)
threads.join()
from threads import Threads
import time
import urllib
import threading
def handler(data, extra):
ex, lock = extra
f = urllib.urlopen(data)
response = f.read()
with lock:
with open(ex, "a") as f:
f.write(response)
lock = threading.Lock()
data = ['http://www.google.com/'] * 100
extra = ('./text.txt', lock)
threads = Threads(100, data)
threads.run(handler, extra)
threads.join()
# Done
from connect import Connect as connect
from openhab_data import Openhab
from threads import Threads
import json
import socket
import threading
running = True
a = Openhab()
b = Threads()
a.initialize()
while True:
openhab_sending = threading.Thread(
target=b.sending_thread
) #thread for getting data from openhab sending to wappsto
wappsto_sending = threading.Thread(
target=b.receiving_thread
) # thread for getting data from wappsto sending to openhab
if running:
openhab_sending.start()
wappsto_sending.start()
openhab_sending.join()
wappsto_sending.join()
def StartThreads():
thr = t.Threads(0)
global datfile
print COLORSBASH["BROWN"] + "Open dat file:" + datfile + COLORSBASH["END"]
datfile = open(datfile, "rb+")
thr.BlocksAdder(datfile)
class SatNetUI(QtGui.QWidget):
def __init__(self, argumentsdict, parent=None):
QtGui.QWidget.__init__(self, parent)
QtGui.QToolTip.setFont(QtGui.QFont('SansSerif', 18))
self.enviromentDesktop = os.environ.get('DESKTOP_SESSION')
self.connection = ''
self.settingsfile = '.settings'
self.setArguments(argumentsdict)
self.initUI()
self.initButtons()
self.initFields()
self.setParameters()
self.initLogo()
self.initConfiguration()
self.initConsole()
self.gsi = GroundStationInterface(self.CONNECTION_INFO, "Vigo",
client_amp.ClientProtocol)
self.threads = Threads(self.CONNECTION_INFO, self.gsi)
# Initialize the reactor parameters needed for the pyqt enviroment
client_amp.Client(self.CONNECTION_INFO, self.gsi, self.threads).createconnection(test=False)
# Create a new connection by loading the connection parameters
# from the interface window
def NewConnection(self, test=False):
self.CONNECTION_INFO['username'] = str(self.LabelUsername.text())
self.CONNECTION_INFO['password'] = str(self.LabelPassword.text())
self.CONNECTION_INFO['connection'] =\
str(self.LabelConnection.currentText())
if self.AutomaticReconnection.isChecked():
self.CONNECTION_INFO['reconnection'] = 'yes'
else:
self.CONNECTION_INFO['reconnection'] = 'no'
self.openInterface()
return client_amp.Client(self.CONNECTION_INFO, self.gsi,
self.threads).setconnection(test=False)
def initUI(self):
"""
self.CONNECTION_INFO = misc.get_data_local_file(
settingsFile='.settings')
"""
self.CONNECTION_INFO = misc.get_data_local_file(
settingsFile=self.settingsfile)
QtGui.QToolTip.setFont(QtGui.QFont('SansSerif', 10))
self.setFixedSize(1300, 800)
self.setWindowTitle("SatNet client - %s" %
(self.CONNECTION_INFO['name']))
if self.CONNECTION_INFO['parameters'] == 'yes':
self.CONNECTION_INFO = {}
self.UpdateFields()
elif self.CONNECTION_INFO['parameters'] == 'no':
pass
else:
log.msg("No parameters configuration found." +
" Using default parameter - Yes")
def initButtons(self):
buttons = QtGui.QGroupBox(self)
grid = QtGui.QGridLayout(buttons)
buttons.setLayout(grid)
self.ButtonNew = QtGui.QPushButton("Connection")
self.ButtonNew.setToolTip("Start a new connection using " +
" the selected connection")
self.ButtonNew.setFixedWidth(145)
self.ButtonNew.clicked.connect(self.NewConnection)
self.ButtonCancel = QtGui.QPushButton("Disconnection")
self.ButtonCancel.setToolTip("End current connection")
self.ButtonCancel.setFixedWidth(145)
self.ButtonCancel.clicked.connect(self.CloseConnection)
self.ButtonCancel.setEnabled(False)
self.ButtonLoad = QtGui.QPushButton("Load parameters from file")
self.ButtonLoad.setToolTip("Load parameters from <i>.settings</i> file")
self.ButtonLoad.setFixedWidth(296)
self.ButtonLoad.clicked.connect(self.UpdateFields)
self.ButtonConfiguration = QtGui.QPushButton("Configuration")
self.ButtonConfiguration.setToolTip("Open configuration window")
self.ButtonConfiguration.setFixedWidth(145)
self.ButtonConfiguration.clicked.connect(self.SetConfiguration)
ButtonHelp = QtGui.QPushButton("Help")
ButtonHelp.setToolTip("Click for help")
ButtonHelp.setFixedWidth(145)
ButtonHelp.clicked.connect(self.usage)
grid.addWidget(self.ButtonNew, 0, 0, 1, 1)
grid.addWidget(self.ButtonCancel, 0, 1, 1, 1)
grid.addWidget(self.ButtonLoad, 1, 0, 1, 2)
grid.addWidget(self.ButtonConfiguration, 2, 0, 1, 1)
grid.addWidget(ButtonHelp, 2, 1, 1, 1)
buttons.setTitle("Connection")
buttons.move(10, 10)
self.dialogTextBrowser = configurationWindow.ConfigurationWindow(self)
def initFields(self):
# Connection parameters group
connectionParameters = QtGui.QGroupBox(self)
gridConnection = QtGui.QFormLayout()
connectionParameters.setLayout(gridConnection)
LabelAttemps = QtGui.QLabel("Reconnection tries:")
LabelAttemps.setFixedWidth(145)
self.FieldLabelAttemps = QtGui.QLineEdit()
self.FieldLabelAttemps.setFixedWidth(145)
gridConnection.addRow(LabelAttemps, self.FieldLabelAttemps)
connectionParameters.setTitle("Connection parameters")
connectionParameters.move(10, 140)
# Configuration group.
configuration = QtGui.QGroupBox(self)
configurationLayout = QtGui.QVBoxLayout()
configuration.setLayout(configurationLayout)
self.LoadDefaultSettings =\
QtGui.QCheckBox("Automatically load settings from file")
configurationLayout.addWidget(self.LoadDefaultSettings)
self.AutomaticReconnection =\
QtGui.QCheckBox("Reconnect after a failure")
configurationLayout.addWidget(self.AutomaticReconnection)
configuration.move(10, 180)
# User parameters group
parameters = QtGui.QGroupBox(self)
self.layout = QtGui.QFormLayout()
parameters.setLayout(self.layout)
self.LabelUsername = QtGui.QLineEdit()
self.LabelUsername.setFixedWidth(190)
self.layout.addRow(QtGui.QLabel("Username: "******"Password: "******"Interface: "),
self.LabelConnection)
parameters.setTitle("User data")
parameters.move(10, 265)
# User interface group
interfaceControl = QtGui.QGroupBox(self)
gridControl = QtGui.QGridLayout(interfaceControl)
interfaceControl.setLayout(gridControl)
self.stopInterfaceButton = QtGui.QPushButton("Stop interface")
self.stopInterfaceButton.setToolTip("Stop the actual interface")
self.stopInterfaceButton.setFixedWidth(145)
self.stopInterfaceButton.clicked.connect(self.stopInterface)
self.stopInterfaceButton.setEnabled(False)
gridControl.addWidget(self.stopInterfaceButton, 0, 0, 1, 1)
interfaceControl.move(155, 380)
def initLogo(self):
LabelLogo = QtGui.QLabel(self)
LabelLogo.move(20, 450)
pic = QtGui.QPixmap(os.getcwd() + "/logo.png")
pic = pic.scaledToWidth(300)
# pic = pic.scaled(400, 400, QtCore.Qt.KeepAspectRatio)
LabelLogo.setPixmap(pic)
LabelLogo.show()
def initConfiguration(self):
if self.CONNECTION_INFO['reconnection'] == 'yes':
self.AutomaticReconnection.setChecked(True)
elif self.CONNECTION_INFO['reconnection'] == 'no':
self.AutomaticReconnection.setChecked(False)
if self.CONNECTION_INFO['parameters'] == 'yes':
self.LoadDefaultSettings.setChecked(True)
elif self.CONNECTION_INFO['parameters'] == 'no':
self.LoadDefaultSettings.setChecked(False)
def initConsole(self):
self.console = QtGui.QTextBrowser(self)
self.console.move(340, 10)
self.console.resize(950, 780)
self.console.setFont(QtGui.QFont('SansSerif', 11))
# Set parameters form arguments list.
def setArguments(self, argumentsdict):
try:
if argumentsdict['username'] != "":
self.LabelUsername.setText(argumentsdict['username'])
if argumentsdict['connection'] != "":
index = self.LabelConnection.findText(argumentsdict['connection'])
self.LabelConnection.setCurrentIndex(index)
except KeyError:
self.settingsfile = argumentsdict['file']
# Set parameters from CONNECTION_INFO dict.
def setParameters(self):
self.FieldLabelAttemps.setText(self.CONNECTION_INFO['attempts'])
self.LabelUsername.setText(self.CONNECTION_INFO['username'])
try:
index = self.LabelConnection.findText(
self.CONNECTION_INFO['connection'])
self.LabelConnection.setCurrentIndex(index)
except Exception as e:
log.err(e)
def CloseConnection(self):
self.gsi.clear_slots()
self.ButtonNew.setEnabled(True)
self.ButtonCancel.setEnabled(False)
def UpdateFields(self):
self.CONNECTION_INFO = misc.get_data_local_file(
settingsFile=self.settingsfile)
log.msg("Parameters loaded from .setting file.")
@QtCore.pyqtSlot()
def SetConfiguration(self):
self.dialogTextBrowser.exec_()
self.UpdateFields()
def openInterface(self):
if str(self.LabelConnection.currentText()) == 'udp':
self.threads.runUDPThreadReceive()
self.threads.runUDPThreadSend()
self.connection = 'udp'
self.LabelConnection.setEnabled(False)
self.stopInterfaceButton.setEnabled(True)
elif str(self.LabelConnection.currentText()) == 'serial':
self.threads.runKISSThreadReceive()
self.connection = 'serial'
self.LabelConnection.setEnabled(False)
self.stopInterfaceButton.setEnabled(True)
def stopInterface(self):
if self.connection == 'udp':
self.threads.stopUDPThreadReceive()
self.LabelConnection.setEnabled(True)
self.stopInterfaceButton.setEnabled(False)
elif self.connection == 'serial':
self.threads.stopKISSThread()
self.LabelConnection.setEnabled(True)
self.stopInterfaceButton.setEnabled(False)
def usage(self):
log.msg("USAGE of client_amp.py")
log.msg("")
log.msg("python client_amp.py")
log.msg(" [-n <username>] # Set SATNET username to login")
log.msg(" [-p <password>] # Set SATNET user password to login")
log.msg(" [-c <connection>] # Set the type of interface with "
"the GS (serial, udp or tcp)")
log.msg(" [-s <serialport>] # Set serial port")
log.msg(" [-b <baudrate>] # Set serial port baudrate")
log.msg(" [-i <ip>] # Set ip direction")
log.msg(" [-u <udpport>] # Set port address")
log.msg("")
log.msg("Example for serial config:")
log.msg("python client_amp.py -g -n crespo -p cre.spo -t 2 -c serial"
"-s /dev/ttyS1 -b 115200")
log.msg("Example for udp config:")
log.msg("python client_amp.py -g -n crespo -p cre.spo -t 2 -c udp -i"
"127.0.0.1 -u 5001")
log.msg("")
log.msg("[User]")
log.msg("username: test-sc-user")
log.msg("password: password")
log.msg("connection: udp")
log.msg("[Serial]")
log.msg("serialport: /dev/ttyUSB0")
log.msg("baudrate: 500000")
log.msg("[UDP]")
log.msg("ip: 127.0.0.1")
log.msg("udpport: 5005")
def center(self):
frameGm = self.frameGeometry()
screen_pos = QtGui.QApplication.desktop().cursor().pos()
screen = QtGui.QApplication.desktop().screenNumber(screen_pos)
centerPoint = QtGui.QApplication.desktop().screenGeometry(
screen).center()
frameGm.moveCenter(centerPoint)
self.move(frameGm.topLeft())
# Functions designed to output information
@QtCore.pyqtSlot(str)
def append_text(self, text):
self.console.moveCursor(QtGui.QTextCursor.End)
self.console.insertPlainText(text)
if "Connection lost" in str(text):
self.ButtonNew.setEnabled(True)
self.ButtonCancel.setEnabled(False)
if "Connection failed" in str(text):
self.ButtonNew.setEnabled(True)
self.ButtonCancel.setEnabled(False)
if "Connection sucessful" in str(text):
self.ButtonNew.setEnabled(False)
self.ButtonCancel.setEnabled(True)
filename = ("log-" + self.CONNECTION_INFO['name'] +
"-" + time.strftime("%Y.%m.%d") + ".csv")
with open(filename, "a+") as f:
f.write(text)
def closeEvent(self, event):
self.reply = QtGui.QMessageBox.question(self, 'Exit confirmation',
"Are you sure to quit?",
QtGui.QMessageBox.Yes |
QtGui.QMessageBox.No,
QtGui.QMessageBox.No)
# Non asynchronous way. Need to re implement this. TO-DO
if self.reply == QtGui.QMessageBox.Yes:
self.stopInterface()
self.gsi.clear_slots()
client_amp.Client(self.CONNECTION_INFO, self.gsi, self.threads).destroyconnection()
elif self.reply == QtGui.QMessageBox.No:
event.ignore()
return l_threads
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
logging.info("Starting Studies analysis")
opts = read_options()
GrimoireSQL.SetDBChannel(database=opts.dbmls,
user=opts.dbuser,
password=opts.dbpassword)
startdate = opts.initdate
enddate = opts.enddate
identities_db = opts.dbidentities
main_topics = Threads(startdate, enddate, identities_db)
# Example of use for the most verbose thread
#email = main_topics.verboseThread()
#print "The most verbose thread: "
#print """
# message_id: %s
# subject: %s
# date: %s
# """ % (email.message_id, email.subject, email.date)
# WARNING: This code for senders overwrite previous mls-top.json file
bots = [
'wikibugs', 'gerrit-wm', 'wikibugs_', 'wm-bot', '',
'Translation updater bot', 'jenkins-bot'
]