def doIt(self):
lookingList = self.lookingList
newsRdd = self.filenameRdd.flatMap(
lambda x: _fct(lookingList, x)).filter(lambda x: x != [])
MessageManager.debugMessage(
"ReutersNewsSourceHDFS : stop reading Reuters corpus")
return newsRdd
def lookingAt(self, symbole, startDate, endDate, keywords):
upperKeywords = [x.upper() for x in keywords]
MessageManager.debugMessage("ReutersNewsSourceHDFS : start reading Reuters corpus")
def hasAnyofTheresKeywords(keywords, text):
for word in keywords:
if(word in text):
return True
return False
def fct(line):
try:
lines = line.split(',')
date = datetime.datetime.strptime(lines[0], "%Y-%m-%d %H:%M:%S")
if(date >= startDate and date <= endDate):
head = lines[1]
msg = ''.join(lines[2:])
if(hasAnyofTheresKeywords(upperKeywords, head.upper()) or hasAnyofTheresKeywords(upperKeywords, msg.upper())):
#MessageManager.debugMessage("ReutersNewsSource : head or msg has keywords")
return News(pubDate=date, symbole=symbole, publication=head+msg, pubSource="Reuters")
except:
pass # explicative line or empty
return None
newsRdd = self.filenameRdd.map(fct).filter(lambda x: x != None)
MessageManager.debugMessage("ReutersNewsSourceHDFS : stop reading Reuters corpus")
return newsRdd
def crossvalidation(self, nbSplits=5):
'''
return precision min, precision max, moyenne des precisions
'''
precMin = 1.0
precMax = 0.0
prec = 0.0
listPrec = []
aSplit = [1 for x in range(0,nbSplits)]
rdds = self.dataset.randomSplit(aSplit,random.randint(1,100000))
for cpt in range(0, nbSplits):
MessageManager.debugMessage("DataClassifier : start new cross-validation iteration %d/%d" % (cpt, nbSplits))
trainSet ,testSet = self._giveTrainAndtest(rdds, cpt)
#one = trainSet.take(1)[0]
#print('size of vect : %d' % len(one.features))
print('trainset size : %d' % trainSet.count())
print('testset size : %d' % testSet.count())
# cheat because we can't use self.predict in a map here...
toto = DataClassifier(None,self.classifier)
toto.train(trainSet)
#TODO test here
evaluatorRdd = testSet.map(lambda p: (p.label, toto.predict(p.features)))
#evaluatorRdd = testSet.map(lambda p: (p.label, 1))
print('evaluatorRdd size : %d' % evaluatorRdd.count())
rddOK = evaluatorRdd.filter(lambda (a, b): a == b)
nbOK = rddOK.count()
nbTOT = testSet.count()
prec = nbOK/float(nbTOT)
if(prec < precMin):
precMin = prec
if(prec > precMax):
precMax = prec
listPrec.append(prec)
print('iteration precision : %f' % prec)
return precMin, precMax, mean(listPrec)
def lookingAt(self, symbole, startDate, endDate, keywords):
upperKeywords = [x.upper() for x in keywords]
MessageManager.debugMessage(
"ReutersNewsSourceHDFS : start reading Reuters corpus")
def hasAnyofTheresKeywords(keywords, text):
for word in keywords:
if (word in text):
return True
return False
def fct(line):
try:
lines = line.split(',')
date = datetime.datetime.strptime(lines[0],
"%Y-%m-%d %H:%M:%S")
if (date >= startDate and date <= endDate):
head = lines[1]
msg = ''.join(lines[2:])
if (hasAnyofTheresKeywords(upperKeywords, head.upper())
or hasAnyofTheresKeywords(upperKeywords,
msg.upper())):
#MessageManager.debugMessage("ReutersNewsSource : head or msg has keywords")
return News(pubDate=date,
symbole=symbole,
publication=head + msg,
pubSource="Reuters")
except:
pass # explicative line or empty
return None
newsRdd = self.filenameRdd.map(fct).filter(lambda x: x != None)
MessageManager.debugMessage(
"ReutersNewsSourceHDFS : stop reading Reuters corpus")
return newsRdd
def requestForMarkets(self, symbole):
params = {
'q': symbole,
'startdate': '2000-01-01',
'enddate': time.strftime('%Y-%m-%d'),
'num': 30,
'output': 'csv'
}
r = requests.get(self.url, params=params)
MessageManager.debugMessage("GoogleFinanceMarketSource : request")
return r.text.encode('utf-8').split('\n')
def __init__(self,dir):
MessageManager.__init__(self)
import os
path = os.path.join(dir,"messages.shelf")
# print "Will open shelf at " + path
from shelve import open
self.shelf = open(path)
for k in self.__dict__:
if k != "shelf":
self.__dict__[k] = self.shelf[k]
# print " loaded " + k + " = " + str(self.shelf[k])
self.synchronize()
def addMarketStatusToNews(self, news):
for new in news:
self.addIfNotExist(new.symbole)
enddate = new.pubDate + datetime.timedelta(days=7)
isFirstLline = True
new.marketStatus = []
for line in self.symboles[new.symbole]:
if (isFirstLline):
isFirstLline = False
else:
try:
date_m, open_m, high_m, low_m, close_m, volume_m = line.split(
',')
date_m = datetime.datetime.strptime(date_m, "%d-%b-%y")
if (date_m >= new.pubDate and date_m <= enddate):
MessageManager.debugMessage(
"GoogleFinanceMarketSource : add marketStatus")
for machin in [
date_m, open_m, high_m, low_m, close_m,
volume_m
]:
MessageManager.debugMessage(str(machin))
new.marketStatus.append(
MarketStatus(date_m, open_m, high_m, low_m,
close_m, volume_m))
MessageManager.debugMessage(
"GoogleFinanceMarketSource : marketStatus added"
)
except:
pass # empty line
MessageManager.debugMessage(
"GoogleFinanceMarketSource : exception")
new.marketStatus = sorted(new.marketStatus,
key=lambda x: x.market_date)[:3]
def lookingAt(self, symbole, startDate, endDate, keywords):
upperKeywords = [x.upper() for x in keywords]
MessageManager.debugMessage(
"ReutersNewsSource : start reading Reuters corpus")
f = open(self.filename, 'r')
for line in f:
try:
lines = line.split(',')
date = datetime.datetime.strptime(lines[0],
"%Y-%m-%d %H:%M:%S")
if (date >= startDate and date <= endDate):
head = lines[1]
msg = ''.join(lines[2:])
if (self.hasAnyofTheresKeywords(upperKeywords,
head.upper())
or self.hasAnyofTheresKeywords(
upperKeywords, msg.upper())):
MessageManager.debugMessage(
"ReutersNewsSource : head or msg has keywords")
self.news.append(
News(pubDate=date,
symbole=symbole,
publication=head,
pubSource="Reuters"))
except:
pass # explicative line or empty
f.close()
MessageManager.debugMessage(
"ReutersNewsSource : stop reading Reuters corpus")
MessageManager.debugMessage("ReutersNewsSource : %d news found" %
len(self.news))
def classification(filepath='/media/droz/KIKOOLOL HDD/Corpus/dataset/dataset.txt', sc=None):
MessageManager.debugMessage("classification : start open file %s" % filepath)
lines = sc.textFile(filepath)
fullDataSet = lines.map(lambda line: literal_eval(line)).map(lambda (data,label): LabeledPoint((1 if label else 0), data)).cache()
MessageManager.debugMessage("classification : start split dataset")
trainRdd, testRdd = fullDataSet.randomSplit([80,20], 17)
dc = DataClassifier()
MessageManager.debugMessage("classification : start training")
dc.train(trainRdd)
MessageManager.debugMessage("classification : stop training")
MessageManager.debugMessage("classification : start prediction")
evaluatorRdd = testRdd.map(lambda p: (p.label, dc.predict(p.features)))
nbOK = evaluatorRdd.filter(lambda (a, b): a == b).count()
nbTOT = testRdd.count()
precision = nbOK/float(nbTOT)
print('precision : %f' % precision)
def crossvalidation(self, nbSplits=5):
'''
Cross validate the algorithmes for show/select the best
return best algorithme name and mean precision
'''
dicoPrec = {}
for (classifier, name) in self.classifier:
dicoPrec[name] = {'min': 1.0, 'max': 0.0, 'mean': []}
aSplit = [1 for x in range(0, nbSplits)]
rdds = self.dataset.randomSplit(aSplit, random.randint(1, 100000))
matrixList = []
for cpt in range(0, nbSplits):
MessageManager.debugMessage(
"DataClassifierEvaluator : start new cross-validation iteration %d/%d"
% (cpt + 1, nbSplits))
trainSet, testSet = self._giveTrainAndtest(rdds, cpt)
#one = trainSet.take(1)[0]
#print('size of vect : %d' % len(one.features))
print('trainset size : %d' % trainSet.count())
print('testset size : %d' % testSet.count())
for (classifier, name) in self.classifier:
currentModel = DataClassifier(None, classifier)
currentModel.train(trainSet)
def f(iterator):
yield ((p.label, currentModel.predict(p.features))
for p in iterator)
#evaluatorRdd = testSet.mapPartitions(lambda p: (p.label, currentModel.predict(p.features)))
evaluatorRdd = testSet.mapPartitions(f).flatMap(lambda x: x)
matrix = self._createConfusionMatrix(evaluatorRdd)
matrixList.append(matrix)
self._showMatrix(matrix)
prec = self._results(evaluatorRdd)
if (prec < dicoPrec[name]['min']):
dicoPrec[name]['min'] = prec
if (prec > dicoPrec[name]['max']):
dicoPrec[name]['max'] = prec
dicoPrec[name]['mean'].append(prec)
print('=== Result of iteration ===')
self.showResultConsole(dicoPrec)
print('+++=== mean confusion matrix ===+++')
self.matrixConfusion = self._meanMatrix(matrixList)
self._showMatrix(self.matrixConfusion)
print('+++=== Final Result ===+++')
return self.showResultConsole(dicoPrec)
def run(self):
port = serial.Serial(self.serial_port,
baudrate=self.serial_baudrate,
timeout=self.serial_timeout,
parity=self.serial_parity,
stopbits=serial.STOPBITS_TWO,
bytesize=serial.SEVENBITS)
# database creation
conn = DatabaseConnector.connectSQLiteDB('meshDB.db')
# MessageManager creation
self.messageManager = MessageManager(conn)
# reading loop
print("Python version detected : " + str(sys.version_info.major))
while True:
print("--- Listening...")
rline = self.readlineCR(port)
currentDate = datetime.datetime.utcnow()
try:
print("Parsing line: " + rline)
message = self.messageManager.parse_line(currentDate, rline)
# CSV writing
#self.df.loc[self.row_iter] = [datetime.datetime.utcnow(), node_number, sensor_type, float(sensor_value)]
#self.df.to_csv('./test.csv',index = False)
# DB writing
self.messageManager.postMessage(message)
except ValueError:
print("ValueError Exception")
print(str(len(rline)) + " " + rline)
except Exception as e:
print("Erreur" + e)
conn.rollback()
# raise e
print("--- Received: " + rline + "\n")
# closing of the database
conn.close()
def crossvalidation(self, nbSplits=5):
'''
Cross validate the algorithmes for show/select the best
return best algorithme name and mean precision
'''
dicoPrec = {}
for (classifier, name) in self.classifier:
dicoPrec[name] = {'min' : 1.0, 'max' : 0.0, 'mean' : []}
aSplit = [1 for x in range(0,nbSplits)]
rdds = self.dataset.randomSplit(aSplit,random.randint(1,100000))
matrixList = []
for cpt in range(0, nbSplits):
MessageManager.debugMessage("DataClassifierEvaluator : start new cross-validation iteration %d/%d" % (cpt+1, nbSplits))
trainSet ,testSet = self._giveTrainAndtest(rdds, cpt)
#one = trainSet.take(1)[0]
#print('size of vect : %d' % len(one.features))
print('trainset size : %d' % trainSet.count())
print('testset size : %d' % testSet.count())
for (classifier, name) in self.classifier:
currentModel = DataClassifier(None, classifier)
currentModel.train(trainSet)
def f(iterator): yield ((p.label, currentModel.predict(p.features)) for p in iterator)
#evaluatorRdd = testSet.mapPartitions(lambda p: (p.label, currentModel.predict(p.features)))
evaluatorRdd = testSet.mapPartitions(f).flatMap(lambda x: x)
matrix = self._createConfusionMatrix(evaluatorRdd)
matrixList.append(matrix)
self._showMatrix(matrix)
prec = self._results(evaluatorRdd)
if(prec < dicoPrec[name]['min']):
dicoPrec[name]['min'] = prec
if(prec > dicoPrec[name]['max']):
dicoPrec[name]['max'] = prec
dicoPrec[name]['mean'].append(prec)
print('=== Result of iteration ===')
self.showResultConsole(dicoPrec)
print('+++=== mean confusion matrix ===+++')
self.matrixConfusion = self._meanMatrix(matrixList)
self._showMatrix(self.matrixConfusion)
print('+++=== Final Result ===+++')
return self.showResultConsole(dicoPrec)
def lookingAt(self, symbole, startDate, endDate, keywords):
'''
méthode pour rechercher des news
-symbole ce qu'on cherche depuis startDate jusqu'à endDate
avec les keywords dans le contenu
'''
hasMoreQuote = True
params = {
'q': symbole,
'startdate': str(startDate.strftime('%Y-%m-%d')),
'enddate': str(endDate.strftime('%Y-%m-%d')),
'start': 0,
'num': self.num
}
while (hasMoreQuote):
r = requests.get(self.url, params=params)
MessageManager.debugMessage("GoogleFinanceNewsSource : request")
text = self.h.unescape(r.text).encode('utf-8')
quotes = re.findall(self.expNews, text)
dates = re.findall(self.expDate, text)
sources = re.findall(self.expPubSource, text)
if (len(quotes) < self.num):
hasMoreQuote = False
for cpt in xrange(len(quotes)):
try:
#Feb 26, 2015
date = datetime.datetime.strptime(dates[cpt], "%b %d, %Y")
self.news.append(
News(pubDate=date,
symbole=symbole,
publication=quotes[cpt],
pubSource=sources[cpt]))
except:
MessageManager.debugMessage(
"new recent, ... set for today")
self.news.append(
News(pubDate=datetime.datetime.now(),
symbole=symbole,
publication=quotes[cpt],
pubSource=sources[cpt]))
params['start'] += self.num
def setUpClass( self ):
self.msg = MessageManager( 'debug', None )
self.msg.show( "Info! SetUp started", 'info', True )
#Initialize QGIS app
app = QgsApplication([], True)
QgsApplication.setPrefixPath("/usr", True)
QgsApplication.initQgis()
#Configure QSettings (organization and application name)
self.settings = QSettings("GeoTux", "QGIS-Plugin-Test")
#Load layer, add field f1, and add layer to Registry
baseDir = os.path.dirname( os.path.realpath( __file__ ) )
self.layerPath = os.path.join( baseDir, 'test_data', 'test_points.shp' )
self.layer = QgsVectorLayer( self.layerPath, 'puntos', 'ogr' )
self.layer.dataProvider().addAttributes([QgsField('f1', QVariant.Double)])
self.layer.updateFields()
QgsMapLayerRegistry.instance().addMapLayer( self.layer )
#Instantiate AutoFieldManager
self.autoFieldManager = AutoFieldManager( self.msg, None, '/AutoFieldsTest', 'GeoTux', 'QGIS-Plugin-Test' )
def lookingAt(self, symbole, startDate, endDate, keywords):
hasMoreQuote=True
params = {'q' : symbole, 'startdate' : str(startDate.strftime('%Y-%m-%d')), 'enddate' : str(endDate.strftime('%Y-%m-%d')), 'start' : 0, 'num' : self.num}
while(hasMoreQuote):
r = requests.get(self.url, params=params)
MessageManager.debugMessage("GoogleFinanceNewsSource : request")
text = self.h.unescape(r.text).encode('utf-8')
quotes = re.findall(self.expNews, text)
dates = re.findall(self.expDate, text)
sources = re.findall(self.expPubSource, text)
if(len(quotes) < self.num):
hasMoreQuote=False
for cpt in xrange(len(quotes)):
try:
#Feb 26, 2015
date = datetime.datetime.strptime(dates[cpt], "%b %d, %Y")
self.news.append(News(pubDate=date, symbole=symbole, publication=quotes[cpt], pubSource=sources[cpt]))
except:
MessageManager.debugMessage("new recent, ... set for today")
self.news.append(News(pubDate=datetime.datetime.now(), symbole=symbole, publication=quotes[cpt], pubSource=sources[cpt]))
params['start'] += self.num
def message_actions():
# Parse the request payload
payload = json.loads(request.form["payload"])
selection_item = payload["actions"][0]["name"]
selection_value = payload["actions"][0]["value"]
reply_action = MessageManager(payload)
# get tag using trained model
if selection_item == "isQuestion":
if selection_value == "yes":
question_content = payload["original_message"]["attachments"][0][
"fallback"]
# print(question_content)
tag = prediction_model.get_Question_tags(question_content)
print(type(tag))
linklist, titlelist = stackOverFlowApi.get_Frequent_Questions_of_a_tag(
tag)
taglist = stackOverFlowApi.get_related_tag(tag)
message = reply_action.selectIsQuestion(tag, linklist, titlelist,
taglist)
response = slack_client.api_call(
"chat.postMessage",
**message,
attachments=reply_action.getSearchingBlock(question_content))
return ""
elif selection_value == 'no':
#reply_is_question = MessageActions(payload)
message = reply_action.selectIsnotQuestion()
response = slack_client.api_call("chat.update",
**message,
attachments=[])
return ""
else:
return ""
if selection_item == "search":
if selection_value == "yes":
# reply_is_selection = MessageActions(payload)
question_content = payload["original_message"]["attachments"][0][
"fallback"]
# get search link
link = stackOverFlowApi.search_question_on_stackoverflow(
question_content)
message = reply_action.selectIsSearch(link)
response = slack_client.api_call("chat.postMessage", **message)
else:
response = slack_client.api_call(
"chat.postMessage",
channel=payload["channel"]["id"],
ts=payload["message_ts"],
text="It's great to help you! :smile:")
return ""
return ""
def crossvalidation(self, nbSplits=5):
'''
return precision min, precision max, moyenne des precisions
'''
precMin = 1.0
precMax = 0.0
prec = 0.0
listPrec = []
aSplit = [1 for x in range(0, nbSplits)]
rdds = self.dataset.randomSplit(aSplit, random.randint(1, 100000))
for cpt in range(0, nbSplits):
MessageManager.debugMessage(
"DataClassifier : start new cross-validation iteration %d/%d" %
(cpt, nbSplits))
trainSet, testSet = self._giveTrainAndtest(rdds, cpt)
#one = trainSet.take(1)[0]
#print('size of vect : %d' % len(one.features))
print('trainset size : %d' % trainSet.count())
print('testset size : %d' % testSet.count())
# cheat because we can't use self.predict in a map here...
toto = DataClassifier(None, self.classifier)
toto.train(trainSet)
#TODO test here
evaluatorRdd = testSet.map(lambda p:
(p.label, toto.predict(p.features)))
#evaluatorRdd = testSet.map(lambda p: (p.label, 1))
print('evaluatorRdd size : %d' % evaluatorRdd.count())
rddOK = evaluatorRdd.filter(lambda (a, b): a == b)
nbOK = rddOK.count()
nbTOT = testSet.count()
prec = nbOK / float(nbTOT)
if (prec < precMin):
precMin = prec
if (prec > precMax):
precMax = prec
listPrec.append(prec)
print('iteration precision : %f' % prec)
return precMin, precMax, mean(listPrec)
def __init__(self, sourlient):
AFTNPaths.normalPaths(sourlient.name)
PXPaths.normalPaths()
self.sysman = SystemManager() # General system manager
self.sourlient = sourlient # Sourlient (Source/Client) object containing configuration infos.
self.logger = sourlient.logger # Logger object
self.subscriber = sourlient.subscriber # Determine if it will act like a subscriber or a provider(MHS)
self.host = sourlient.host # Remote host (name or ip)
self.portR = sourlient.portR # Receiving port
self.portS = sourlient.portS # Sending port
self.batch = sourlient.batch # Number of files we read in a pass (20)
self.timeout = sourlient.timeout # Timeout time in seconds (default = 10 seconds)
self.sleepBetweenConnect = int('10') # Time (in seconds) between connection trials
self.slow = sourlient.slow # Sleeps are added when we want to be able to decrypt log entries
self.igniter = None # Igniter object (link to pid)
self.writePath = AFTNPaths.RECEIVED # Where we write messages we receive
self.archivePath = AFTNPaths.SENT # Where we put sent messages
self.specialOrdersPath = AFTNPaths.SPECIAL_ORDERS # Where we put special orders
# Paths creation
self.sysman.createDir(PXPaths.TXQ + self.sourlient.name)
self.sysman.createDir(self.writePath)
self.sysman.createDir(self.archivePath)
self.sysman.createDir(self.specialOrdersPath)
self.mm = MessageManager(self.logger, self.sourlient) # AFTN Protocol is implemented in MessageManager Object
self.remoteAddress = None # Remote address (where we will connect())
self.socket = None # Socket object
self.dataFromFiles = [] # A list of tuples (content, filename) obtained from a DiskReader
self.reader = DiskReader(PXPaths.TXQ + self.sourlient.name, self.sourlient.batch,
self.sourlient.validation, self.sourlient.diskReaderPatternMatching,
self.sourlient.mtime, True, self.logger, eval(self.sourlient.sorter), self.sourlient)
self.debug = True # Debugging switch
self.justConnect = False # Boolean that indicates when a connexion just occur
self.totBytes = 0
#self.printInitInfos()
self.makeConnection()
def lookingAt(self, symbole, startDate, endDate, keywords):
upperKeywords = [x.upper() for x in keywords]
MessageManager.debugMessage("ReutersNewsSource : start reading Reuters corpus")
f = open(self.filename, 'r')
for line in f:
try:
lines = line.split(',')
date = datetime.datetime.strptime(lines[0], "%Y-%m-%d %H:%M:%S")
if(date >= startDate and date <= endDate):
head = lines[1]
msg = ''.join(lines[2:])
if(self.hasAnyofTheresKeywords(upperKeywords, head.upper()) or self.hasAnyofTheresKeywords(upperKeywords, msg.upper())):
MessageManager.debugMessage("ReutersNewsSource : head or msg has keywords")
self.news.append(News(pubDate=date, symbole=symbole, publication=head, pubSource="Reuters"))
except:
pass # explicative line or empty
f.close()
MessageManager.debugMessage("ReutersNewsSource : stop reading Reuters corpus")
MessageManager.debugMessage("ReutersNewsSource : %d news found" % len(self.news))
def run(self):
port = serial.Serial(self.serial_port,
baudrate = self.serial_baudrate,
timeout = self.serial_timeout,
parity = self.serial_parity,
stopbits=serial.STOPBITS_TWO,
bytesize=serial.SEVENBITS)
# database creation
conn = DatabaseConnector.connectSQLiteDB('meshDB.db')
# MessageManager creation
self.messageManager = MessageManager(conn)
# reading loop
print("Python version detected : "+str(sys.version_info.major))
while True:
print("--- Listening...")
rline = self.readlineCR(port)
currentDate = datetime.datetime.utcnow()
try:
print("Parsing line: "+rline)
message = self.messageManager.parse_line(currentDate, rline)
# CSV writing
#self.df.loc[self.row_iter] = [datetime.datetime.utcnow(), node_number, sensor_type, float(sensor_value)]
#self.df.to_csv('./test.csv',index = False)
# DB writing
self.messageManager.postMessage(message)
except ValueError:
print("ValueError Exception")
print(str(len(rline))+" "+rline)
except Exception as e:
print("Erreur"+e)
conn.rollback()
# raise e
print("--- Received: "+rline+"\n")
# closing of the database
conn.close()
def useDataClassifier(filepath='/media/droz/KIKOOLOL HDD/Corpus/dataset/dataset.txt', sc=None):
MessageManager.debugMessage("useDataClassifier : start open file %s" % filepath)
lines = sc.textFile(filepath)
fullDataSet = lines.map(lambda line: literal_eval(line)).map(lambda (data,label): LabeledPoint((1 if label else 0), data))
fullDataSet.cache()
#fullDataSet = sc.parallelize(fullDataSet)
dc = DataClassifier(fullDataSet, SVMWithSGD)
MessageManager.debugMessage("useDataClassifier : start crossvalidation")
precMin, precMax, prec = dc.crossvalidation(5)
MessageManager.debugMessage("useDataClassifier : train full dataset")
dc.train(fullDataSet)
dc.saveModel()
print('min : %f, max : %f, mean : %f' % (precMin, precMax, prec))
def selectBestModel(self):
nameBest = ''
bestPrec = 0.0
bestClassifier = None
for (classifier, name) in self.classifier:
MessageManager.debugMessage('DataClassifierEvaluator : Start evaluation of %s' % name)
dc = DataClassifier(self.dataset, classifier)
precMin, precMax, precMean = dc.crossvalidation()
MessageManager.debugMessage('DataClassifierEvaluator : Results for %s : \n\tPrecMin : %f\n\tPrecMax : %f\n\tPrecMean : %f' % (name, precMin, precMax, precMean))
if(precMean > bestPrec):
bestPrec = precMean
nameBest = name
bestClassifier = classifier
MessageManager.debugMessage('DataClassifierEvaluator : best classifier is %s with precision of %f' % (nameBest, bestPrec))
return bestClassifier, nameBest
def addMarketStatusToNews(self, news):
for new in news:
self.addIfNotExist(new.symbole)
enddate = new.pubDate + datetime.timedelta(days=7)
isFirstLline=True
new.marketStatus = []
for line in self.symboles[new.symbole]:
if(isFirstLline):
isFirstLline=False
else:
try:
date_m,open_m,high_m,low_m,close_m,volume_m = line.split(',')
date_m = datetime.datetime.strptime(date_m, "%d-%b-%y")
if(date_m >= new.pubDate and date_m <= enddate):
MessageManager.debugMessage("GoogleFinanceMarketSource : add marketStatus")
for machin in [date_m,open_m,high_m,low_m,close_m,volume_m]:
MessageManager.debugMessage(str(machin))
new.marketStatus.append(MarketStatus(date_m,open_m,high_m,low_m,close_m,volume_m))
MessageManager.debugMessage("GoogleFinanceMarketSource : marketStatus added")
except:
pass # empty line
MessageManager.debugMessage("GoogleFinanceMarketSource : exception")
new.marketStatus = sorted(new.marketStatus, key=lambda x:x.market_date)[:3]
def selectBestModel(self):
nameBest = ''
bestPrec = 0.0
bestClassifier = None
for (classifier, name) in self.classifier:
MessageManager.debugMessage(
'DataClassifierEvaluator : Start evaluation of %s' % name)
dc = DataClassifier(self.dataset, classifier)
precMin, precMax, precMean = dc.crossvalidation()
MessageManager.debugMessage(
'DataClassifierEvaluator : Results for %s : \n\tPrecMin : %f\n\tPrecMax : %f\n\tPrecMean : %f'
% (name, precMin, precMax, precMean))
if (precMean > bestPrec):
bestPrec = precMean
nameBest = name
bestClassifier = classifier
MessageManager.debugMessage(
'DataClassifierEvaluator : best classifier is %s with precision of %f'
% (nameBest, bestPrec))
return bestClassifier, nameBest
return News(pubDate=date, symbole='NASDAQ:GOOGL', publication=txt, pubSource='Reuteurs', marketStatus=[m1,m2,m3])
if __name__ == "__main__":
allNews = [createRandomNews() for x in range(30)]
sc = SparkContext()
newsRDD = sc.parallelize(allNews).distinct()
dataSetMaker = DataSetMakerV2()
fullDataSet = dataSetMaker.process(newsRDD)
fullDataSet.cache()
myClassifier = ClassifiersWrapper()
myClassifier.addClassifier(classifier=SVMWithSGD, trainParameters={}, weight=0.3)
myClassifier.addClassifier(classifier=LogisticRegressionWithSGD, trainParameters={}, weight=0.3)
myClassifier.addClassifier(classifier=LogisticRegressionWithLBFGS, trainParameters={}, weight=0.3)
dc = DataClassifier(fullDataSet, myClassifier)
MessageManager.debugMessage("main : start crossvalidation")
precMin, precMax, prec = dc.crossvalidation(5)
print('min : %f, max : %f, mean : %f' % (precMin, precMax, prec))
'''
featuresRDD = newsRDD.map(lambda x: FeaturesV2(x))
allBg2 = featuresRDD.map(lambda x: list(x.bg2)).reduce(lambda a,b : a+b)
allBg3 = featuresRDD.map(lambda x: list(x.bg3)).reduce(lambda a,b : a+b)
setAllBg2 = set(allBg2)
setAllBg3 = set(allBg3)
print('size of setAllBg2 : %d' % len(setAllBg2))
print('size of setAllBg3 : %d' % len(setAllBg3))
allBg2Flat = featuresRDD.flatMap(lambda x: list(x.bg2))
allBg2FlatUnique = allBg2Flat.intersection(allBg2Flat).collect()
print('size of allBg2FlatUnique %d' % len(allBg2FlatUnique))
'''
class Server():
def __init__(self, serial_port):
'''Constructor with default values only
TODO have default values and user values as input
'''
self.serial_port = serial_port
self.serial_baudrate = 115200
self.serial_parity = serial.PARITY_ODD
self.serial_timeout = None
def readlineCR(self, port):
'''Listen on the serial port and construct a string until \r\n is met
'''
print("--- Reading line")
rline = ""
prevch = ''
while True:
if (sys.version_info.major == 3):
ch = str(port.read(), encoding="utf-7") # for Python 3
elif (sys.version_info.major == 2):
ch = str(port.read()) # for python 2
else:
print("The used version of Python is too old : version " +
str(sys.version_info.major))
return rline
rline += ch
if (prevch == '\r' or ch == '\n') or ch == '':
return rline
prevch = ch
def run(self):
port = serial.Serial(self.serial_port,
baudrate=self.serial_baudrate,
timeout=self.serial_timeout,
parity=self.serial_parity,
stopbits=serial.STOPBITS_TWO,
bytesize=serial.SEVENBITS)
# database creation
conn = DatabaseConnector.connectSQLiteDB('meshDB.db')
# MessageManager creation
self.messageManager = MessageManager(conn)
# reading loop
print("Python version detected : " + str(sys.version_info.major))
while True:
print("--- Listening...")
rline = self.readlineCR(port)
currentDate = datetime.datetime.utcnow()
try:
print("Parsing line: " + rline)
message = self.messageManager.parse_line(currentDate, rline)
# CSV writing
#self.df.loc[self.row_iter] = [datetime.datetime.utcnow(), node_number, sensor_type, float(sensor_value)]
#self.df.to_csv('./test.csv',index = False)
# DB writing
self.messageManager.postMessage(message)
except ValueError:
print("ValueError Exception")
print(str(len(rline)) + " " + rline)
except Exception as e:
print("Erreur" + e)
conn.rollback()
# raise e
print("--- Received: " + rline + "\n")
# closing of the database
conn.close()
def add (self, message):
MessageManager.add(self,message)
self.synchronize()
def get_new_messages (self):
retval = MessageManager.get_new_messages(self)
self.synchronize()
return retval
- abs(self.__translate(self.controlData['leftRight']))
right_speed = abs(self.__translate(self.controlData['forBack']))
if left_speed < 0:
left_speed = 0
if right_speed < 0:
right_speed = 0
message.set_left_speed(int(left_speed))
message.set_right_speed(int(right_speed))
return message
message_in_queue = Queue.LifoQueue()
messenger_thread = MessageManager(message_in_queue, '/dev/ttyACM1', 115200)
if __name__ == "__main__":
messenger_thread.start()
robots = [
#RobotControl(1, 66),
RobotControl(0, 65)
]
while True:
ev = pygame.event.wait()
for r in robots:
if r.update_control():
message_in_queue.put(r.generate_message())
def loadModel(self):
MessageManager.debugMessage("DataClassifier : Load Model")
self.model = pickle.load(open(self.modelpath, 'rb'))
def saveModel(self):
MessageManager.debugMessage("DataClassifier : Save Model")
pickle.dump(self.model, open(self.modelpath, 'wb'))
return News(pubDate=date, symbole='NASDAQ:GOOGL', publication=txt, pubSource='Reuteurs', marketStatus=[m1,m2,m3])
if __name__ == "__main__":
allNews = [createRandomNews() for x in range(30)]
sc = SparkContext()
newsRDD = sc.parallelize(allNews).distinct()
dataSetMaker = DataSetMakerV2()
fullDataSet = dataSetMaker.process(newsRDD)
fullDataSet.cache()
myClassifier = ClassifiersWrapper()
myClassifier.addClassifier(classifier=SVMWithSGD, trainParameters={}, weight=0.3)
myClassifier.addClassifier(classifier=LogisticRegressionWithSGD, trainParameters={}, weight=0.3)
myClassifier.addClassifier(classifier=LogisticRegressionWithLBFGS, trainParameters={}, weight=0.3)
dc = DataClassifier(fullDataSet, myClassifier)
MessageManager.debugMessage("main : start crossvalidation")
precMin, precMax, prec = dc.crossvalidation(5)
print('min : %f, max : %f, mean : %f' % (precMin, precMax, prec))
'''
featuresRDD = newsRDD.map(lambda x: FeaturesV2(x))
allBg2 = featuresRDD.map(lambda x: list(x.bg2)).reduce(lambda a,b : a+b)
allBg3 = featuresRDD.map(lambda x: list(x.bg3)).reduce(lambda a,b : a+b)
setAllBg2 = set(allBg2)
setAllBg3 = set(allBg3)
print('size of setAllBg2 : %d' % len(setAllBg2))
print('size of setAllBg3 : %d' % len(setAllBg3))
allBg2Flat = featuresRDD.flatMap(lambda x: list(x.bg2))
allBg2FlatUnique = allBg2Flat.intersection(allBg2Flat).collect()
print('size of allBg2FlatUnique %d' % len(allBg2FlatUnique))
'''
class Server():
def __init__(self, serial_port):
'''Constructor with default values only
TODO have default values and user values as input
'''
self.serial_port = serial_port
self.serial_baudrate = 115200
self.serial_parity = serial.PARITY_ODD
self.serial_timeout = None
def readlineCR(self, port):
'''Listen on the serial port and construct a string until \r\n is met
'''
print("--- Reading line")
rline = ""
prevch = ''
while True:
if (sys.version_info.major==3):
ch = str(port.read(), encoding="utf-7") # for Python 3
elif (sys.version_info.major==2):
ch = str(port.read()) # for python 2
else:
print("The used version of Python is too old : version "+str(sys.version_info.major))
return rline
rline += ch
if (prevch=='\r' or ch=='\n') or ch=='':
return rline
prevch=ch
def run(self):
port = serial.Serial(self.serial_port,
baudrate = self.serial_baudrate,
timeout = self.serial_timeout,
parity = self.serial_parity,
stopbits=serial.STOPBITS_TWO,
bytesize=serial.SEVENBITS)
# database creation
conn = DatabaseConnector.connectSQLiteDB('meshDB.db')
# MessageManager creation
self.messageManager = MessageManager(conn)
# reading loop
print("Python version detected : "+str(sys.version_info.major))
while True:
print("--- Listening...")
rline = self.readlineCR(port)
currentDate = datetime.datetime.utcnow()
try:
print("Parsing line: "+rline)
message = self.messageManager.parse_line(currentDate, rline)
# CSV writing
#self.df.loc[self.row_iter] = [datetime.datetime.utcnow(), node_number, sensor_type, float(sensor_value)]
#self.df.to_csv('./test.csv',index = False)
# DB writing
self.messageManager.postMessage(message)
except ValueError:
print("ValueError Exception")
print(str(len(rline))+" "+rline)
except Exception as e:
print("Erreur"+e)
conn.rollback()
# raise e
print("--- Received: "+rline+"\n")
# closing of the database
conn.close()
class AutoFieldsTests( unittest.TestCase ):
@classmethod
def setUpClass( self ):
self.msg = MessageManager( 'debug', None )
self.msg.show( "Info! SetUp started", 'info', True )
#Initialize QGIS app
app = QgsApplication([], True)
QgsApplication.setPrefixPath("/usr", True)
QgsApplication.initQgis()
#Configure QSettings (organization and application name)
self.settings = QSettings("GeoTux", "QGIS-Plugin-Test")
#Load layer, add field f1, and add layer to Registry
baseDir = os.path.dirname( os.path.realpath( __file__ ) )
self.layerPath = os.path.join( baseDir, 'test_data', 'test_points.shp' )
self.layer = QgsVectorLayer( self.layerPath, 'puntos', 'ogr' )
self.layer.dataProvider().addAttributes([QgsField('f1', QVariant.Double)])
self.layer.updateFields()
QgsMapLayerRegistry.instance().addMapLayer( self.layer )
#Instantiate AutoFieldManager
self.autoFieldManager = AutoFieldManager( self.msg, None, '/AutoFieldsTest', 'GeoTux', 'QGIS-Plugin-Test' )
def readStoredSettings( self, layer, fieldName ):
""" Helper function to get a dictionary of stored QSettings for an AutoField """
dictTmpProperties = {}
autoFieldId = self.autoFieldManager.buildAutoFieldId( layer, fieldName )
self.settings.beginGroup('/AutoFieldsTest/data/' + autoFieldId)
dictTmpProperties['layer'] = self.settings.value( "layer", "", type=str )
dictTmpProperties['field'] = self.settings.value( "field", u"", type=unicode )
dictTmpProperties['expression'] = self.settings.value( "expression", u"", type=unicode )
dictTmpProperties['layer2'] = self.settings.value( "layer2", "", type=str )
dictTmpProperties['field2'] = self.settings.value( "field2", "", type=str )
dictTmpProperties['enabled'] = self.settings.value( "enabled", False, type=bool )
self.settings.endGroup()
return dictTmpProperties
def test01CreateEnabledAutoField( self ):
""" QSettings should be properly stored, AutoField should be enabled """
self.msg.show( "Info! Test 1 started", 'info', True )
self.autoFieldManager.createAutoField(
layer=self.layer,
fieldName=u'f1',
expression=u'$x'
)
dictTmpProperties = self.readStoredSettings( self.layer, u'f1' )
dictExpectedProperties = {
'layer':self.layerPath,
'field':u'f1',
'expression':u'$x',
'layer2':"",
'field2':"",
'enabled':True
}
self.assertEqual( dictTmpProperties, dictExpectedProperties )
def test02AvoidTwoAutoFieldsOnSameField( self ):
""" AutoField should not be created if another one already exists on the same field."""
self.msg.show( "Info! Test 2 started", 'info', True )
res = self.autoFieldManager.createAutoField(
layer=self.layer,
fieldName=u'f1',
expression=u'$y'
)
self.assertFalse( res )
def test03EditAutoFieldLayer( self ):
""" AutoField value should be updated if a feature is added.
Note: It cannot handle the case when writing directly to the provider,
as QGIS doesn't have a SIGNAL for that.
self.layer.dataProvider().addFeatures( [ tmpFeature ] )
"""
self.msg.show( "Info! Test 3 started", 'info', True )
tmpFeature = QgsFeature( self.layer.pendingFields() )
tmpFeature.setGeometry( QgsGeometry.fromPoint( QgsPoint(-74.4, 4.5) ) )
# Either 1:
self.layer.startEditing()
self.layer.addFeature( tmpFeature )
self.layer.commitChanges()
# Or 2:
#with edit( self.layer ):
# self.layer.addFeature( tmpFeature )
addedFeature = self.layer.getFeatures().next()
self.assertEquals( addedFeature['f1'], -74.4 )
def test04ChangeAttributeValue( self ):
""" AutoField value should be updated if another AutoField value is changed """
self.msg.show( "Info! Test 4 started", 'info', True )
self.autoFieldManager.createAutoField(
layer=self.layer,
fieldName=u'modified',
expression=u'\'now: \' + to_string("f1")'
)
self.layer.startEditing()
self.layer.changeAttributeValue( 0, self.layer.fieldNameIndex( u'id' ), 1 )
self.layer.commitChanges()
feature = self.layer.getFeatures().next()
self.assertEquals( feature['modified'], 'now: -74.4' )
def test05FieldRemovedThenDisableAutoField( self ):
""" AutoField should be disabled if its base field is removed """
self.msg.show( "Info! Test 5 started", 'info', True )
fieldIndex = self.layer.fieldNameIndex( u'f1' )
self.layer.startEditing()
self.layer.deleteAttribute( fieldIndex )
self.layer.commitChanges()
dictTmpProperties = self.readStoredSettings( self.layer, u'f1' )
self.assertFalse( dictTmpProperties['enabled'] )
def test06MissingFieldAddedThenEnableAutoField( self ):
""" AutoField should be enabled if missing field is added """
self.msg.show( "Info! Test 6 started", 'info', True )
self.layer.startEditing()
self.layer.addAttribute( QgsField( 'f1', QVariant.Double, len=10, prec=2 ) )
self.layer.commitChanges()
dictTmpProperties = self.readStoredSettings( self.layer, u'f1' )
self.assertTrue( dictTmpProperties['enabled'] )
def test07LayerRemovedThenDisableAutoField( self ):
""" AutoField should be disabled if its base layer is removed """
self.msg.show( "Info! Test 7 started", 'info', True )
QgsMapLayerRegistry.instance().removeMapLayer( self.layer.id() )
# removeMapLayer deletes the underlying object, so create it again
self.layer = QgsVectorLayer( self.layerPath, 'puntos', 'ogr' )
dictTmpProperties = self.readStoredSettings( self.layer, u'f1' )
self.assertFalse( dictTmpProperties['enabled'] )
def test08MissingLayerAddedThenEnableAutoField( self ):
""" AutoField should be enabled if missing layer is added """
self.msg.show( "Info! Test 8 started", 'info', True )
# test07 deletes the layer object, so create it again
self.layer = QgsVectorLayer( self.layerPath, 'puntos', 'ogr' )
QgsMapLayerRegistry.instance().addMapLayer( self.layer )
dictTmpProperties = self.readStoredSettings( self.layer, u'f1' )
self.assertTrue( dictTmpProperties['enabled'] )
def test09RemoveAutoField( self ):
""" QSettings should be deleted for the removed AutoField """
self.msg.show( "Info! Test 9 started", 'info', True )
# test07 deletes the layer object, so create it again
self.layer = QgsVectorLayer( self.layerPath, 'puntos', 'ogr' )
autoFieldId = self.autoFieldManager.buildAutoFieldId( self.layer, u'f1')
self.autoFieldManager.removeAutoField( autoFieldId )
dictTmpProperties = self.readStoredSettings( self.layer, u'f1' )
self.assertEqual( dictTmpProperties, {'layer':"",'field':"",'expression':"",'layer2':"",'field2':"",'enabled':False} )
@classmethod
def tearDownClass( self ):
self.msg.show( "Info! TearDown started", 'info', True )
# test07 deletes the layer object, so create it again
self.layer = QgsVectorLayer( self.layerPath, 'puntos', 'ogr' )
#Remove AutoField modified
autoFieldId = self.autoFieldManager.buildAutoFieldId( self.layer, u'modified' )
self.autoFieldManager.removeAutoField( autoFieldId )
#Delete field f1
fieldIndex = self.layer.fieldNameIndex('f1')
self.layer.dataProvider().deleteAttributes( [fieldIndex] )
self.layer.updateFields()
#Delete features from test layer
fIds = self.layer.allFeatureIds()
self.layer.dataProvider().deleteFeatures( fIds )
#Remove layer from Registry
QgsMapLayerRegistry.instance().removeMapLayer( self.layer.id() )
self.msg.show( "Info! TearDown finished", 'info', True )
QgsApplication.exitQgis()
def initGui( self ):
self.messageManager = MessageManager(self.messageMode, self.iface)
self.autoFieldManager = AutoFieldManager(self.messageManager, self.iface)
self.autoFieldManager.readAutoFields()
''' if the widget is loaded .... then unlosad'''
# Get list of all Dockwidgets
existe = False
for x in self.iface.mainWindow().findChildren(QDockWidget):
name = x.objectName()
if (name == "SeismicRiskDockWidget"):
existe = True
#self.iface.removeDockWidget(x)
if (not existe):
self.dockWidget = SeismicRiskDockWidget(self.iface.mainWindow(), self.iface, self.autoFieldManager,
self.messageManager, self.language)
self.iface.addDockWidget(Qt.RightDockWidgetArea, self.dockWidget)
self.actionDock = QAction(QIcon( ":/plugins/SeismicRisk/icon.png"), \
"Seismic Risk plugin...", self.iface.mainWindow() )
self.actionDock.triggered.connect( self.toggleDockWidget )
# Add custom submenu to Vector menu
self.iface.addPluginToVectorMenu( "&Seismic Risk ", self.actionDock )
''' section pick layers '''
# icon_path = ':/plugins/pickLayer/icon.png'
#icon_path = ':/plugins/PruebaAutoFields/icons/info.png'
#icon_path = os.path.join(self.plugin_dir, "icons", "pickLayer.png")
icon_path = os.path.join(self.plugin_dir, "icons", "points.png")
# map tool action
print " 66 iconpath " + icon_path
self.mapToolAction = QAction(QIcon(icon_path), "Get seismic risk", self.iface.mainWindow())
self.mapToolAction.setCheckable(True)
self.mapTool = IdentifyGeometry(self.mapCanvas)
self.mapTool.geomIdentified.connect(self.editFeature)
self.mapTool.setAction(self.mapToolAction)
self.mapToolAction.triggered.connect(self.setMapTool)
#self.iface.addToolBarIcon(self.mapToolAction)
self.iface.addPluginToMenu("&Pick to Layer in autofields", self.mapToolAction)
''' fin section pick layers'''
# Remove Redo buttons from menus and toolbars, they can lead to crashes due
# to a corrupted undo stack.
redoActionList = [action for action in self.iface.advancedDigitizeToolBar().actions() if action.objectName() == u'mActionRedo']
if redoActionList:
self.iface.advancedDigitizeToolBar().removeAction( redoActionList[0] )
self.iface.editMenu().removeAction( redoActionList[0] )
QSettings().setValue( "/shortcuts/Redo", "" ) # Override Redo shortcut
# This block (2 options for disabling the Undo panel) didn't work
#QSettings().setValue( '/UI/Customization/enabled', True )
#QSettings( "QGIS", "QGISCUSTOMIZATION2" ).setValue( '/Customization/Panels/Undo', False )
#undoDock = self.iface.mainWindow().findChild( QDockWidget, u'Undo' )
#self.iface.removeDockWidget( undoDock )
# Create action that will start plugin configuration
#self.action = QAction(QIcon( ":/plugins/AutoFields/icon.png"), "AutoFields plugin...", self.iface.mainWindow() )
icon_path = os.path.join(self.plugin_dir, "icons", "lines.png")
print " path " + icon_path
self.action = QAction(QIcon(icon_path), "Seismic Risk plugin...", self.iface.mainWindow())
#self.iface.addToolBarIcon(self.action)
self.iface.addPluginToMenu("&Show seismic Risk Calculator", self.mapToolAction)
# connect the action to the run method
self.action.triggered.connect( self.show )
# Add a custom toolbar
self.toolbar = self.iface.addToolBar("SeismicRisk")
self.toolbar.setObjectName("SeismicRisk")
self.toolbar.addAction(self.action)
self.toolbar.addAction(self.mapToolAction)
# Add custom submenu to Vector menu
self.iface.addPluginToVectorMenu( "&AutoFields for calculating resilience", self.action )
# Add a custom toolbar
#self.toolbar = self.iface.addToolBar( "SeismicRisk" )
#self.toolbar.setObjectName("SeismicRisk")
self.messageManager = MessageManager( self.messageMode, self.iface )
self.autoFieldManager = AutoFieldManager( self.messageManager, self.iface )
self.autoFieldManager.readAutoFields()
self.dockWidget = SeismicRiskDockWidget( self.iface.mainWindow(), self.iface, self.autoFieldManager, self.messageManager, self.language )
self.iface.addDockWidget( Qt.RightDockWidgetArea, self.dockWidget )
def requestForMarkets(self, symbole):
params = {'q' : symbole, 'startdate' : '2000-01-01', 'enddate' : time.strftime('%Y-%m-%d'), 'num' : 30, 'output' : 'csv'}
r = requests.get(self.url, params=params)
MessageManager.debugMessage("GoogleFinanceMarketSource : request")
return r.text.encode('utf-8').split('\n')
def saveModel(self):
MessageManager.debugMessage("DataClassifier : Save Model")
pickle.dump(self.model, open(self.modelpath, 'wb'))
def doIt(self):
lookingList = self.lookingList
newsRdd = self.filenameRdd.flatMap(lambda x: _fct(lookingList, x)).filter(lambda x: x != [])
MessageManager.debugMessage("ReutersNewsSourceHDFS : stop reading Reuters corpus")
return newsRdd
def main(argv):
print('Kamonohashi Slackbot')
# KQI実行のための環境変数追加
if os.name == 'posix':
# linux: kqiが格納されている場所 (デフォルトの想定場所)
# 本当は外だしすべき
kqicli = '/home/(username)/.local/bin'
os.environ['PATH'] = os.environ['PATH'] + ':' + kqicli
# 設定ファイルの読込み
sec = JsonManager(filepath=os.path.join(FLAGS.config_root, 'secure.json'))
cfg = JsonManager(filepath=os.path.join(FLAGS.config_root, 'config.json'))
msg = MessageManager(filepath=os.path.join(FLAGS.config_root, 'message.json'), kamonohashi_uri=sec['KAMONOHASHI']['Server'], postmessage_uri=sec['SlackURI'])
# 監視状態変数
watch_status = {
'dailyjob' : False,
'watching' : False,
'waketime' : datetime.now()
}
# テナント状態変数
tenant_status = {
'count' : {},
'state' : {}
}
tenant_active_job = {}
# 監視開始
print('Watch Loop Start')
while True:
sec.Load()
kqi.Login(sec['KAMONOHASHI']['Application']['Username'], sec['KAMONOHASHI']['Application']['Password'])
tenants = kqi.GetTenantDict()
print('Daily Loop Start')
while datetime.now().day == watch_status['waketime'].day:
# JSONファイルの更新確認
msg.Load()
if cfg.Load():
# 設定ファイルの更新があればテナント情報も再取得する
tenants = kqi.GetTenantDict()
# 監視状態更新
watch_status = UpdateWatchStatus(watch_status, GetWatchCondition(cfg), msg)
# 監視状態変数のクリア
tenant_status['count'] = {}
# 監視状態がアクティブの場合はテナント毎に監視処理を行う
for tenant in tenants:
tenant_condition = {}
if str(tenant) in cfg['Tenant']:
tenant_condition = cfg['Tenant'][str(tenant)]
kqi.SwitchTenant(tenant)
# 監視状態かどうか
tenant_status = UpdateTenantTrainStatus(tenant_status, tenant_condition, tenant, tenants, watch_status, msg, kqi)
tenant_status = UpdateTenantInferStatus(tenant_status, tenant_condition, tenant, tenants, watch_status, msg, kqi)
# サーバーヘルス監視(特に監視したいのはGPU温度)
# T.B.D
# すべての監視対象テナントでしばらくメッセージ送信していない場合は実行中に応じてメッセージを投げる
if watch_status['watching'] and msg.IsPassedTime(seconds=120 * 60):
msg.SendMessage(msg.CreateTenantsRunningInfo(tenants, tenant_status['count']))
msg.UpdateTimestamp()
# 連続で処理すると負荷がかかるため、指定したポーリング間隔で待機
time.sleep(cfg['Polling'])
# 日の処理完了を抜けた場合はリセットする
watch_status['dailyjob'] = False
watch_status['waketime'] = datetime.now()
import os
from MessageManager import MessageManager
from AdvancedScheduleManager import AdvancedScheduleManager
from redis_manager import RedisManager
with open("./sources/line_bot_token.json", 'r') as f:
json_data = json.load(f)
access_token = json_data["access_token"]
channel_secret = json_data["channel_secret"]
line_bot_api = LineBotApi(access_token)
handler = WebhookHandler(channel_secret)
messageManager = MessageManager(line_bot_api)
# redisManager = RedisManager()
advancedScheduleManager = AdvancedScheduleManager(bot=messageManager)
@csrf_exempt
def callback(request: HttpRequest) -> HttpResponse:
if request.method == "POST":
# get X-Line-Signature header value
signature = request.META['HTTP_X_LINE_SIGNATURE']
# get request body as text
from MessageManager import MessageManager
from DatabaseConnector import DatabaseConnector
if __name__ =='__main__':
if len(sys.argv) > 1: # /dev/ttyACM0
A = Server(sys.argv[1])
A.run()
else:
print("Missing serial port pathname")
# database creation
conn = DatabaseConnector.connectSQLiteDB('meshDB.db')
# MessageManager creation
messageManager = MessageManager(conn)
# DB reading
messageManager.getAllMessages()
# CSV writing
df = messageManager.getAllMessagesintoPandaDataframe()
df.to_csv('csvFile.csv', index = False)
# DB update
newMessage = Message('3', datetime.datetime.utcnow(), '0000', '0000', '0000')
messageManager.putMessage(newMessage)
# DB post
newMessage = Message('4', datetime.datetime.utcnow(), '0000', '0000', '0000')
messageManager.postMessage(newMessage)
def loadModel(self):
MessageManager.debugMessage("DataClassifier : Load Model")
self.model = pickle.load(open(self.modelpath, 'rb'))
