def doIt(self): lookingList = self.lookingList newsRdd = self.filenameRdd.flatMap( lambda x: _fct(lookingList, x)).filter(lambda x: x != []) MessageManager.debugMessage( "ReutersNewsSourceHDFS : stop reading Reuters corpus") return newsRdd
def lookingAt(self, symbole, startDate, endDate, keywords): upperKeywords = [x.upper() for x in keywords] MessageManager.debugMessage("ReutersNewsSourceHDFS : start reading Reuters corpus") def hasAnyofTheresKeywords(keywords, text): for word in keywords: if(word in text): return True return False def fct(line): try: lines = line.split(',') date = datetime.datetime.strptime(lines[0], "%Y-%m-%d %H:%M:%S") if(date >= startDate and date <= endDate): head = lines[1] msg = ''.join(lines[2:]) if(hasAnyofTheresKeywords(upperKeywords, head.upper()) or hasAnyofTheresKeywords(upperKeywords, msg.upper())): #MessageManager.debugMessage("ReutersNewsSource : head or msg has keywords") return News(pubDate=date, symbole=symbole, publication=head+msg, pubSource="Reuters") except: pass # explicative line or empty return None newsRdd = self.filenameRdd.map(fct).filter(lambda x: x != None) MessageManager.debugMessage("ReutersNewsSourceHDFS : stop reading Reuters corpus") return newsRdd
def crossvalidation(self, nbSplits=5): ''' return precision min, precision max, moyenne des precisions ''' precMin = 1.0 precMax = 0.0 prec = 0.0 listPrec = [] aSplit = [1 for x in range(0,nbSplits)] rdds = self.dataset.randomSplit(aSplit,random.randint(1,100000)) for cpt in range(0, nbSplits): MessageManager.debugMessage("DataClassifier : start new cross-validation iteration %d/%d" % (cpt, nbSplits)) trainSet ,testSet = self._giveTrainAndtest(rdds, cpt) #one = trainSet.take(1)[0] #print('size of vect : %d' % len(one.features)) print('trainset size : %d' % trainSet.count()) print('testset size : %d' % testSet.count()) # cheat because we can't use self.predict in a map here... toto = DataClassifier(None,self.classifier) toto.train(trainSet) #TODO test here evaluatorRdd = testSet.map(lambda p: (p.label, toto.predict(p.features))) #evaluatorRdd = testSet.map(lambda p: (p.label, 1)) print('evaluatorRdd size : %d' % evaluatorRdd.count()) rddOK = evaluatorRdd.filter(lambda (a, b): a == b) nbOK = rddOK.count() nbTOT = testSet.count() prec = nbOK/float(nbTOT) if(prec < precMin): precMin = prec if(prec > precMax): precMax = prec listPrec.append(prec) print('iteration precision : %f' % prec) return precMin, precMax, mean(listPrec)
def lookingAt(self, symbole, startDate, endDate, keywords): upperKeywords = [x.upper() for x in keywords] MessageManager.debugMessage( "ReutersNewsSourceHDFS : start reading Reuters corpus") def hasAnyofTheresKeywords(keywords, text): for word in keywords: if (word in text): return True return False def fct(line): try: lines = line.split(',') date = datetime.datetime.strptime(lines[0], "%Y-%m-%d %H:%M:%S") if (date >= startDate and date <= endDate): head = lines[1] msg = ''.join(lines[2:]) if (hasAnyofTheresKeywords(upperKeywords, head.upper()) or hasAnyofTheresKeywords(upperKeywords, msg.upper())): #MessageManager.debugMessage("ReutersNewsSource : head or msg has keywords") return News(pubDate=date, symbole=symbole, publication=head + msg, pubSource="Reuters") except: pass # explicative line or empty return None newsRdd = self.filenameRdd.map(fct).filter(lambda x: x != None) MessageManager.debugMessage( "ReutersNewsSourceHDFS : stop reading Reuters corpus") return newsRdd
def requestForMarkets(self, symbole): params = { 'q': symbole, 'startdate': '2000-01-01', 'enddate': time.strftime('%Y-%m-%d'), 'num': 30, 'output': 'csv' } r = requests.get(self.url, params=params) MessageManager.debugMessage("GoogleFinanceMarketSource : request") return r.text.encode('utf-8').split('\n')
def __init__(self,dir): MessageManager.__init__(self) import os path = os.path.join(dir,"messages.shelf") # print "Will open shelf at " + path from shelve import open self.shelf = open(path) for k in self.__dict__: if k != "shelf": self.__dict__[k] = self.shelf[k] # print " loaded " + k + " = " + str(self.shelf[k]) self.synchronize()
def addMarketStatusToNews(self, news): for new in news: self.addIfNotExist(new.symbole) enddate = new.pubDate + datetime.timedelta(days=7) isFirstLline = True new.marketStatus = [] for line in self.symboles[new.symbole]: if (isFirstLline): isFirstLline = False else: try: date_m, open_m, high_m, low_m, close_m, volume_m = line.split( ',') date_m = datetime.datetime.strptime(date_m, "%d-%b-%y") if (date_m >= new.pubDate and date_m <= enddate): MessageManager.debugMessage( "GoogleFinanceMarketSource : add marketStatus") for machin in [ date_m, open_m, high_m, low_m, close_m, volume_m ]: MessageManager.debugMessage(str(machin)) new.marketStatus.append( MarketStatus(date_m, open_m, high_m, low_m, close_m, volume_m)) MessageManager.debugMessage( "GoogleFinanceMarketSource : marketStatus added" ) except: pass # empty line MessageManager.debugMessage( "GoogleFinanceMarketSource : exception") new.marketStatus = sorted(new.marketStatus, key=lambda x: x.market_date)[:3]
def lookingAt(self, symbole, startDate, endDate, keywords): upperKeywords = [x.upper() for x in keywords] MessageManager.debugMessage( "ReutersNewsSource : start reading Reuters corpus") f = open(self.filename, 'r') for line in f: try: lines = line.split(',') date = datetime.datetime.strptime(lines[0], "%Y-%m-%d %H:%M:%S") if (date >= startDate and date <= endDate): head = lines[1] msg = ''.join(lines[2:]) if (self.hasAnyofTheresKeywords(upperKeywords, head.upper()) or self.hasAnyofTheresKeywords( upperKeywords, msg.upper())): MessageManager.debugMessage( "ReutersNewsSource : head or msg has keywords") self.news.append( News(pubDate=date, symbole=symbole, publication=head, pubSource="Reuters")) except: pass # explicative line or empty f.close() MessageManager.debugMessage( "ReutersNewsSource : stop reading Reuters corpus") MessageManager.debugMessage("ReutersNewsSource : %d news found" % len(self.news))
def classification(filepath='/media/droz/KIKOOLOL HDD/Corpus/dataset/dataset.txt', sc=None): MessageManager.debugMessage("classification : start open file %s" % filepath) lines = sc.textFile(filepath) fullDataSet = lines.map(lambda line: literal_eval(line)).map(lambda (data,label): LabeledPoint((1 if label else 0), data)).cache() MessageManager.debugMessage("classification : start split dataset") trainRdd, testRdd = fullDataSet.randomSplit([80,20], 17) dc = DataClassifier() MessageManager.debugMessage("classification : start training") dc.train(trainRdd) MessageManager.debugMessage("classification : stop training") MessageManager.debugMessage("classification : start prediction") evaluatorRdd = testRdd.map(lambda p: (p.label, dc.predict(p.features))) nbOK = evaluatorRdd.filter(lambda (a, b): a == b).count() nbTOT = testRdd.count() precision = nbOK/float(nbTOT) print('precision : %f' % precision)
def crossvalidation(self, nbSplits=5): ''' Cross validate the algorithmes for show/select the best return best algorithme name and mean precision ''' dicoPrec = {} for (classifier, name) in self.classifier: dicoPrec[name] = {'min': 1.0, 'max': 0.0, 'mean': []} aSplit = [1 for x in range(0, nbSplits)] rdds = self.dataset.randomSplit(aSplit, random.randint(1, 100000)) matrixList = [] for cpt in range(0, nbSplits): MessageManager.debugMessage( "DataClassifierEvaluator : start new cross-validation iteration %d/%d" % (cpt + 1, nbSplits)) trainSet, testSet = self._giveTrainAndtest(rdds, cpt) #one = trainSet.take(1)[0] #print('size of vect : %d' % len(one.features)) print('trainset size : %d' % trainSet.count()) print('testset size : %d' % testSet.count()) for (classifier, name) in self.classifier: currentModel = DataClassifier(None, classifier) currentModel.train(trainSet) def f(iterator): yield ((p.label, currentModel.predict(p.features)) for p in iterator) #evaluatorRdd = testSet.mapPartitions(lambda p: (p.label, currentModel.predict(p.features))) evaluatorRdd = testSet.mapPartitions(f).flatMap(lambda x: x) matrix = self._createConfusionMatrix(evaluatorRdd) matrixList.append(matrix) self._showMatrix(matrix) prec = self._results(evaluatorRdd) if (prec < dicoPrec[name]['min']): dicoPrec[name]['min'] = prec if (prec > dicoPrec[name]['max']): dicoPrec[name]['max'] = prec dicoPrec[name]['mean'].append(prec) print('=== Result of iteration ===') self.showResultConsole(dicoPrec) print('+++=== mean confusion matrix ===+++') self.matrixConfusion = self._meanMatrix(matrixList) self._showMatrix(self.matrixConfusion) print('+++=== Final Result ===+++') return self.showResultConsole(dicoPrec)
def run(self): port = serial.Serial(self.serial_port, baudrate=self.serial_baudrate, timeout=self.serial_timeout, parity=self.serial_parity, stopbits=serial.STOPBITS_TWO, bytesize=serial.SEVENBITS) # database creation conn = DatabaseConnector.connectSQLiteDB('meshDB.db') # MessageManager creation self.messageManager = MessageManager(conn) # reading loop print("Python version detected : " + str(sys.version_info.major)) while True: print("--- Listening...") rline = self.readlineCR(port) currentDate = datetime.datetime.utcnow() try: print("Parsing line: " + rline) message = self.messageManager.parse_line(currentDate, rline) # CSV writing #self.df.loc[self.row_iter] = [datetime.datetime.utcnow(), node_number, sensor_type, float(sensor_value)] #self.df.to_csv('./test.csv',index = False) # DB writing self.messageManager.postMessage(message) except ValueError: print("ValueError Exception") print(str(len(rline)) + " " + rline) except Exception as e: print("Erreur" + e) conn.rollback() # raise e print("--- Received: " + rline + "\n") # closing of the database conn.close()
def crossvalidation(self, nbSplits=5): ''' Cross validate the algorithmes for show/select the best return best algorithme name and mean precision ''' dicoPrec = {} for (classifier, name) in self.classifier: dicoPrec[name] = {'min' : 1.0, 'max' : 0.0, 'mean' : []} aSplit = [1 for x in range(0,nbSplits)] rdds = self.dataset.randomSplit(aSplit,random.randint(1,100000)) matrixList = [] for cpt in range(0, nbSplits): MessageManager.debugMessage("DataClassifierEvaluator : start new cross-validation iteration %d/%d" % (cpt+1, nbSplits)) trainSet ,testSet = self._giveTrainAndtest(rdds, cpt) #one = trainSet.take(1)[0] #print('size of vect : %d' % len(one.features)) print('trainset size : %d' % trainSet.count()) print('testset size : %d' % testSet.count()) for (classifier, name) in self.classifier: currentModel = DataClassifier(None, classifier) currentModel.train(trainSet) def f(iterator): yield ((p.label, currentModel.predict(p.features)) for p in iterator) #evaluatorRdd = testSet.mapPartitions(lambda p: (p.label, currentModel.predict(p.features))) evaluatorRdd = testSet.mapPartitions(f).flatMap(lambda x: x) matrix = self._createConfusionMatrix(evaluatorRdd) matrixList.append(matrix) self._showMatrix(matrix) prec = self._results(evaluatorRdd) if(prec < dicoPrec[name]['min']): dicoPrec[name]['min'] = prec if(prec > dicoPrec[name]['max']): dicoPrec[name]['max'] = prec dicoPrec[name]['mean'].append(prec) print('=== Result of iteration ===') self.showResultConsole(dicoPrec) print('+++=== mean confusion matrix ===+++') self.matrixConfusion = self._meanMatrix(matrixList) self._showMatrix(self.matrixConfusion) print('+++=== Final Result ===+++') return self.showResultConsole(dicoPrec)
def lookingAt(self, symbole, startDate, endDate, keywords): ''' méthode pour rechercher des news -symbole ce qu'on cherche depuis startDate jusqu'à endDate avec les keywords dans le contenu ''' hasMoreQuote = True params = { 'q': symbole, 'startdate': str(startDate.strftime('%Y-%m-%d')), 'enddate': str(endDate.strftime('%Y-%m-%d')), 'start': 0, 'num': self.num } while (hasMoreQuote): r = requests.get(self.url, params=params) MessageManager.debugMessage("GoogleFinanceNewsSource : request") text = self.h.unescape(r.text).encode('utf-8') quotes = re.findall(self.expNews, text) dates = re.findall(self.expDate, text) sources = re.findall(self.expPubSource, text) if (len(quotes) < self.num): hasMoreQuote = False for cpt in xrange(len(quotes)): try: #Feb 26, 2015 date = datetime.datetime.strptime(dates[cpt], "%b %d, %Y") self.news.append( News(pubDate=date, symbole=symbole, publication=quotes[cpt], pubSource=sources[cpt])) except: MessageManager.debugMessage( "new recent, ... set for today") self.news.append( News(pubDate=datetime.datetime.now(), symbole=symbole, publication=quotes[cpt], pubSource=sources[cpt])) params['start'] += self.num
def setUpClass( self ): self.msg = MessageManager( 'debug', None ) self.msg.show( "Info! SetUp started", 'info', True ) #Initialize QGIS app app = QgsApplication([], True) QgsApplication.setPrefixPath("/usr", True) QgsApplication.initQgis() #Configure QSettings (organization and application name) self.settings = QSettings("GeoTux", "QGIS-Plugin-Test") #Load layer, add field f1, and add layer to Registry baseDir = os.path.dirname( os.path.realpath( __file__ ) ) self.layerPath = os.path.join( baseDir, 'test_data', 'test_points.shp' ) self.layer = QgsVectorLayer( self.layerPath, 'puntos', 'ogr' ) self.layer.dataProvider().addAttributes([QgsField('f1', QVariant.Double)]) self.layer.updateFields() QgsMapLayerRegistry.instance().addMapLayer( self.layer ) #Instantiate AutoFieldManager self.autoFieldManager = AutoFieldManager( self.msg, None, '/AutoFieldsTest', 'GeoTux', 'QGIS-Plugin-Test' )
def lookingAt(self, symbole, startDate, endDate, keywords): hasMoreQuote=True params = {'q' : symbole, 'startdate' : str(startDate.strftime('%Y-%m-%d')), 'enddate' : str(endDate.strftime('%Y-%m-%d')), 'start' : 0, 'num' : self.num} while(hasMoreQuote): r = requests.get(self.url, params=params) MessageManager.debugMessage("GoogleFinanceNewsSource : request") text = self.h.unescape(r.text).encode('utf-8') quotes = re.findall(self.expNews, text) dates = re.findall(self.expDate, text) sources = re.findall(self.expPubSource, text) if(len(quotes) < self.num): hasMoreQuote=False for cpt in xrange(len(quotes)): try: #Feb 26, 2015 date = datetime.datetime.strptime(dates[cpt], "%b %d, %Y") self.news.append(News(pubDate=date, symbole=symbole, publication=quotes[cpt], pubSource=sources[cpt])) except: MessageManager.debugMessage("new recent, ... set for today") self.news.append(News(pubDate=datetime.datetime.now(), symbole=symbole, publication=quotes[cpt], pubSource=sources[cpt])) params['start'] += self.num
def message_actions(): # Parse the request payload payload = json.loads(request.form["payload"]) selection_item = payload["actions"][0]["name"] selection_value = payload["actions"][0]["value"] reply_action = MessageManager(payload) # get tag using trained model if selection_item == "isQuestion": if selection_value == "yes": question_content = payload["original_message"]["attachments"][0][ "fallback"] # print(question_content) tag = prediction_model.get_Question_tags(question_content) print(type(tag)) linklist, titlelist = stackOverFlowApi.get_Frequent_Questions_of_a_tag( tag) taglist = stackOverFlowApi.get_related_tag(tag) message = reply_action.selectIsQuestion(tag, linklist, titlelist, taglist) response = slack_client.api_call( "chat.postMessage", **message, attachments=reply_action.getSearchingBlock(question_content)) return "" elif selection_value == 'no': #reply_is_question = MessageActions(payload) message = reply_action.selectIsnotQuestion() response = slack_client.api_call("chat.update", **message, attachments=[]) return "" else: return "" if selection_item == "search": if selection_value == "yes": # reply_is_selection = MessageActions(payload) question_content = payload["original_message"]["attachments"][0][ "fallback"] # get search link link = stackOverFlowApi.search_question_on_stackoverflow( question_content) message = reply_action.selectIsSearch(link) response = slack_client.api_call("chat.postMessage", **message) else: response = slack_client.api_call( "chat.postMessage", channel=payload["channel"]["id"], ts=payload["message_ts"], text="It's great to help you! :smile:") return "" return ""
def crossvalidation(self, nbSplits=5): ''' return precision min, precision max, moyenne des precisions ''' precMin = 1.0 precMax = 0.0 prec = 0.0 listPrec = [] aSplit = [1 for x in range(0, nbSplits)] rdds = self.dataset.randomSplit(aSplit, random.randint(1, 100000)) for cpt in range(0, nbSplits): MessageManager.debugMessage( "DataClassifier : start new cross-validation iteration %d/%d" % (cpt, nbSplits)) trainSet, testSet = self._giveTrainAndtest(rdds, cpt) #one = trainSet.take(1)[0] #print('size of vect : %d' % len(one.features)) print('trainset size : %d' % trainSet.count()) print('testset size : %d' % testSet.count()) # cheat because we can't use self.predict in a map here... toto = DataClassifier(None, self.classifier) toto.train(trainSet) #TODO test here evaluatorRdd = testSet.map(lambda p: (p.label, toto.predict(p.features))) #evaluatorRdd = testSet.map(lambda p: (p.label, 1)) print('evaluatorRdd size : %d' % evaluatorRdd.count()) rddOK = evaluatorRdd.filter(lambda (a, b): a == b) nbOK = rddOK.count() nbTOT = testSet.count() prec = nbOK / float(nbTOT) if (prec < precMin): precMin = prec if (prec > precMax): precMax = prec listPrec.append(prec) print('iteration precision : %f' % prec) return precMin, precMax, mean(listPrec)
def __init__(self, sourlient): AFTNPaths.normalPaths(sourlient.name) PXPaths.normalPaths() self.sysman = SystemManager() # General system manager self.sourlient = sourlient # Sourlient (Source/Client) object containing configuration infos. self.logger = sourlient.logger # Logger object self.subscriber = sourlient.subscriber # Determine if it will act like a subscriber or a provider(MHS) self.host = sourlient.host # Remote host (name or ip) self.portR = sourlient.portR # Receiving port self.portS = sourlient.portS # Sending port self.batch = sourlient.batch # Number of files we read in a pass (20) self.timeout = sourlient.timeout # Timeout time in seconds (default = 10 seconds) self.sleepBetweenConnect = int('10') # Time (in seconds) between connection trials self.slow = sourlient.slow # Sleeps are added when we want to be able to decrypt log entries self.igniter = None # Igniter object (link to pid) self.writePath = AFTNPaths.RECEIVED # Where we write messages we receive self.archivePath = AFTNPaths.SENT # Where we put sent messages self.specialOrdersPath = AFTNPaths.SPECIAL_ORDERS # Where we put special orders # Paths creation self.sysman.createDir(PXPaths.TXQ + self.sourlient.name) self.sysman.createDir(self.writePath) self.sysman.createDir(self.archivePath) self.sysman.createDir(self.specialOrdersPath) self.mm = MessageManager(self.logger, self.sourlient) # AFTN Protocol is implemented in MessageManager Object self.remoteAddress = None # Remote address (where we will connect()) self.socket = None # Socket object self.dataFromFiles = [] # A list of tuples (content, filename) obtained from a DiskReader self.reader = DiskReader(PXPaths.TXQ + self.sourlient.name, self.sourlient.batch, self.sourlient.validation, self.sourlient.diskReaderPatternMatching, self.sourlient.mtime, True, self.logger, eval(self.sourlient.sorter), self.sourlient) self.debug = True # Debugging switch self.justConnect = False # Boolean that indicates when a connexion just occur self.totBytes = 0 #self.printInitInfos() self.makeConnection()
def lookingAt(self, symbole, startDate, endDate, keywords): upperKeywords = [x.upper() for x in keywords] MessageManager.debugMessage("ReutersNewsSource : start reading Reuters corpus") f = open(self.filename, 'r') for line in f: try: lines = line.split(',') date = datetime.datetime.strptime(lines[0], "%Y-%m-%d %H:%M:%S") if(date >= startDate and date <= endDate): head = lines[1] msg = ''.join(lines[2:]) if(self.hasAnyofTheresKeywords(upperKeywords, head.upper()) or self.hasAnyofTheresKeywords(upperKeywords, msg.upper())): MessageManager.debugMessage("ReutersNewsSource : head or msg has keywords") self.news.append(News(pubDate=date, symbole=symbole, publication=head, pubSource="Reuters")) except: pass # explicative line or empty f.close() MessageManager.debugMessage("ReutersNewsSource : stop reading Reuters corpus") MessageManager.debugMessage("ReutersNewsSource : %d news found" % len(self.news))
def run(self): port = serial.Serial(self.serial_port, baudrate = self.serial_baudrate, timeout = self.serial_timeout, parity = self.serial_parity, stopbits=serial.STOPBITS_TWO, bytesize=serial.SEVENBITS) # database creation conn = DatabaseConnector.connectSQLiteDB('meshDB.db') # MessageManager creation self.messageManager = MessageManager(conn) # reading loop print("Python version detected : "+str(sys.version_info.major)) while True: print("--- Listening...") rline = self.readlineCR(port) currentDate = datetime.datetime.utcnow() try: print("Parsing line: "+rline) message = self.messageManager.parse_line(currentDate, rline) # CSV writing #self.df.loc[self.row_iter] = [datetime.datetime.utcnow(), node_number, sensor_type, float(sensor_value)] #self.df.to_csv('./test.csv',index = False) # DB writing self.messageManager.postMessage(message) except ValueError: print("ValueError Exception") print(str(len(rline))+" "+rline) except Exception as e: print("Erreur"+e) conn.rollback() # raise e print("--- Received: "+rline+"\n") # closing of the database conn.close()
def useDataClassifier(filepath='/media/droz/KIKOOLOL HDD/Corpus/dataset/dataset.txt', sc=None): MessageManager.debugMessage("useDataClassifier : start open file %s" % filepath) lines = sc.textFile(filepath) fullDataSet = lines.map(lambda line: literal_eval(line)).map(lambda (data,label): LabeledPoint((1 if label else 0), data)) fullDataSet.cache() #fullDataSet = sc.parallelize(fullDataSet) dc = DataClassifier(fullDataSet, SVMWithSGD) MessageManager.debugMessage("useDataClassifier : start crossvalidation") precMin, precMax, prec = dc.crossvalidation(5) MessageManager.debugMessage("useDataClassifier : train full dataset") dc.train(fullDataSet) dc.saveModel() print('min : %f, max : %f, mean : %f' % (precMin, precMax, prec))
def selectBestModel(self): nameBest = '' bestPrec = 0.0 bestClassifier = None for (classifier, name) in self.classifier: MessageManager.debugMessage('DataClassifierEvaluator : Start evaluation of %s' % name) dc = DataClassifier(self.dataset, classifier) precMin, precMax, precMean = dc.crossvalidation() MessageManager.debugMessage('DataClassifierEvaluator : Results for %s : \n\tPrecMin : %f\n\tPrecMax : %f\n\tPrecMean : %f' % (name, precMin, precMax, precMean)) if(precMean > bestPrec): bestPrec = precMean nameBest = name bestClassifier = classifier MessageManager.debugMessage('DataClassifierEvaluator : best classifier is %s with precision of %f' % (nameBest, bestPrec)) return bestClassifier, nameBest
def addMarketStatusToNews(self, news): for new in news: self.addIfNotExist(new.symbole) enddate = new.pubDate + datetime.timedelta(days=7) isFirstLline=True new.marketStatus = [] for line in self.symboles[new.symbole]: if(isFirstLline): isFirstLline=False else: try: date_m,open_m,high_m,low_m,close_m,volume_m = line.split(',') date_m = datetime.datetime.strptime(date_m, "%d-%b-%y") if(date_m >= new.pubDate and date_m <= enddate): MessageManager.debugMessage("GoogleFinanceMarketSource : add marketStatus") for machin in [date_m,open_m,high_m,low_m,close_m,volume_m]: MessageManager.debugMessage(str(machin)) new.marketStatus.append(MarketStatus(date_m,open_m,high_m,low_m,close_m,volume_m)) MessageManager.debugMessage("GoogleFinanceMarketSource : marketStatus added") except: pass # empty line MessageManager.debugMessage("GoogleFinanceMarketSource : exception") new.marketStatus = sorted(new.marketStatus, key=lambda x:x.market_date)[:3]
def selectBestModel(self): nameBest = '' bestPrec = 0.0 bestClassifier = None for (classifier, name) in self.classifier: MessageManager.debugMessage( 'DataClassifierEvaluator : Start evaluation of %s' % name) dc = DataClassifier(self.dataset, classifier) precMin, precMax, precMean = dc.crossvalidation() MessageManager.debugMessage( 'DataClassifierEvaluator : Results for %s : \n\tPrecMin : %f\n\tPrecMax : %f\n\tPrecMean : %f' % (name, precMin, precMax, precMean)) if (precMean > bestPrec): bestPrec = precMean nameBest = name bestClassifier = classifier MessageManager.debugMessage( 'DataClassifierEvaluator : best classifier is %s with precision of %f' % (nameBest, bestPrec)) return bestClassifier, nameBest
return News(pubDate=date, symbole='NASDAQ:GOOGL', publication=txt, pubSource='Reuteurs', marketStatus=[m1,m2,m3]) if __name__ == "__main__": allNews = [createRandomNews() for x in range(30)] sc = SparkContext() newsRDD = sc.parallelize(allNews).distinct() dataSetMaker = DataSetMakerV2() fullDataSet = dataSetMaker.process(newsRDD) fullDataSet.cache() myClassifier = ClassifiersWrapper() myClassifier.addClassifier(classifier=SVMWithSGD, trainParameters={}, weight=0.3) myClassifier.addClassifier(classifier=LogisticRegressionWithSGD, trainParameters={}, weight=0.3) myClassifier.addClassifier(classifier=LogisticRegressionWithLBFGS, trainParameters={}, weight=0.3) dc = DataClassifier(fullDataSet, myClassifier) MessageManager.debugMessage("main : start crossvalidation") precMin, precMax, prec = dc.crossvalidation(5) print('min : %f, max : %f, mean : %f' % (precMin, precMax, prec)) ''' featuresRDD = newsRDD.map(lambda x: FeaturesV2(x)) allBg2 = featuresRDD.map(lambda x: list(x.bg2)).reduce(lambda a,b : a+b) allBg3 = featuresRDD.map(lambda x: list(x.bg3)).reduce(lambda a,b : a+b) setAllBg2 = set(allBg2) setAllBg3 = set(allBg3) print('size of setAllBg2 : %d' % len(setAllBg2)) print('size of setAllBg3 : %d' % len(setAllBg3)) allBg2Flat = featuresRDD.flatMap(lambda x: list(x.bg2)) allBg2FlatUnique = allBg2Flat.intersection(allBg2Flat).collect() print('size of allBg2FlatUnique %d' % len(allBg2FlatUnique)) '''
class Server(): def __init__(self, serial_port): '''Constructor with default values only TODO have default values and user values as input ''' self.serial_port = serial_port self.serial_baudrate = 115200 self.serial_parity = serial.PARITY_ODD self.serial_timeout = None def readlineCR(self, port): '''Listen on the serial port and construct a string until \r\n is met ''' print("--- Reading line") rline = "" prevch = '' while True: if (sys.version_info.major == 3): ch = str(port.read(), encoding="utf-7") # for Python 3 elif (sys.version_info.major == 2): ch = str(port.read()) # for python 2 else: print("The used version of Python is too old : version " + str(sys.version_info.major)) return rline rline += ch if (prevch == '\r' or ch == '\n') or ch == '': return rline prevch = ch def run(self): port = serial.Serial(self.serial_port, baudrate=self.serial_baudrate, timeout=self.serial_timeout, parity=self.serial_parity, stopbits=serial.STOPBITS_TWO, bytesize=serial.SEVENBITS) # database creation conn = DatabaseConnector.connectSQLiteDB('meshDB.db') # MessageManager creation self.messageManager = MessageManager(conn) # reading loop print("Python version detected : " + str(sys.version_info.major)) while True: print("--- Listening...") rline = self.readlineCR(port) currentDate = datetime.datetime.utcnow() try: print("Parsing line: " + rline) message = self.messageManager.parse_line(currentDate, rline) # CSV writing #self.df.loc[self.row_iter] = [datetime.datetime.utcnow(), node_number, sensor_type, float(sensor_value)] #self.df.to_csv('./test.csv',index = False) # DB writing self.messageManager.postMessage(message) except ValueError: print("ValueError Exception") print(str(len(rline)) + " " + rline) except Exception as e: print("Erreur" + e) conn.rollback() # raise e print("--- Received: " + rline + "\n") # closing of the database conn.close()
def add (self, message): MessageManager.add(self,message) self.synchronize()
def get_new_messages (self): retval = MessageManager.get_new_messages(self) self.synchronize() return retval
- abs(self.__translate(self.controlData['leftRight'])) right_speed = abs(self.__translate(self.controlData['forBack'])) if left_speed < 0: left_speed = 0 if right_speed < 0: right_speed = 0 message.set_left_speed(int(left_speed)) message.set_right_speed(int(right_speed)) return message message_in_queue = Queue.LifoQueue() messenger_thread = MessageManager(message_in_queue, '/dev/ttyACM1', 115200) if __name__ == "__main__": messenger_thread.start() robots = [ #RobotControl(1, 66), RobotControl(0, 65) ] while True: ev = pygame.event.wait() for r in robots: if r.update_control(): message_in_queue.put(r.generate_message())
def loadModel(self): MessageManager.debugMessage("DataClassifier : Load Model") self.model = pickle.load(open(self.modelpath, 'rb'))
def saveModel(self): MessageManager.debugMessage("DataClassifier : Save Model") pickle.dump(self.model, open(self.modelpath, 'wb'))
class Server(): def __init__(self, serial_port): '''Constructor with default values only TODO have default values and user values as input ''' self.serial_port = serial_port self.serial_baudrate = 115200 self.serial_parity = serial.PARITY_ODD self.serial_timeout = None def readlineCR(self, port): '''Listen on the serial port and construct a string until \r\n is met ''' print("--- Reading line") rline = "" prevch = '' while True: if (sys.version_info.major==3): ch = str(port.read(), encoding="utf-7") # for Python 3 elif (sys.version_info.major==2): ch = str(port.read()) # for python 2 else: print("The used version of Python is too old : version "+str(sys.version_info.major)) return rline rline += ch if (prevch=='\r' or ch=='\n') or ch=='': return rline prevch=ch def run(self): port = serial.Serial(self.serial_port, baudrate = self.serial_baudrate, timeout = self.serial_timeout, parity = self.serial_parity, stopbits=serial.STOPBITS_TWO, bytesize=serial.SEVENBITS) # database creation conn = DatabaseConnector.connectSQLiteDB('meshDB.db') # MessageManager creation self.messageManager = MessageManager(conn) # reading loop print("Python version detected : "+str(sys.version_info.major)) while True: print("--- Listening...") rline = self.readlineCR(port) currentDate = datetime.datetime.utcnow() try: print("Parsing line: "+rline) message = self.messageManager.parse_line(currentDate, rline) # CSV writing #self.df.loc[self.row_iter] = [datetime.datetime.utcnow(), node_number, sensor_type, float(sensor_value)] #self.df.to_csv('./test.csv',index = False) # DB writing self.messageManager.postMessage(message) except ValueError: print("ValueError Exception") print(str(len(rline))+" "+rline) except Exception as e: print("Erreur"+e) conn.rollback() # raise e print("--- Received: "+rline+"\n") # closing of the database conn.close()
class AutoFieldsTests( unittest.TestCase ): @classmethod def setUpClass( self ): self.msg = MessageManager( 'debug', None ) self.msg.show( "Info! SetUp started", 'info', True ) #Initialize QGIS app app = QgsApplication([], True) QgsApplication.setPrefixPath("/usr", True) QgsApplication.initQgis() #Configure QSettings (organization and application name) self.settings = QSettings("GeoTux", "QGIS-Plugin-Test") #Load layer, add field f1, and add layer to Registry baseDir = os.path.dirname( os.path.realpath( __file__ ) ) self.layerPath = os.path.join( baseDir, 'test_data', 'test_points.shp' ) self.layer = QgsVectorLayer( self.layerPath, 'puntos', 'ogr' ) self.layer.dataProvider().addAttributes([QgsField('f1', QVariant.Double)]) self.layer.updateFields() QgsMapLayerRegistry.instance().addMapLayer( self.layer ) #Instantiate AutoFieldManager self.autoFieldManager = AutoFieldManager( self.msg, None, '/AutoFieldsTest', 'GeoTux', 'QGIS-Plugin-Test' ) def readStoredSettings( self, layer, fieldName ): """ Helper function to get a dictionary of stored QSettings for an AutoField """ dictTmpProperties = {} autoFieldId = self.autoFieldManager.buildAutoFieldId( layer, fieldName ) self.settings.beginGroup('/AutoFieldsTest/data/' + autoFieldId) dictTmpProperties['layer'] = self.settings.value( "layer", "", type=str ) dictTmpProperties['field'] = self.settings.value( "field", u"", type=unicode ) dictTmpProperties['expression'] = self.settings.value( "expression", u"", type=unicode ) dictTmpProperties['layer2'] = self.settings.value( "layer2", "", type=str ) dictTmpProperties['field2'] = self.settings.value( "field2", "", type=str ) dictTmpProperties['enabled'] = self.settings.value( "enabled", False, type=bool ) self.settings.endGroup() return dictTmpProperties def test01CreateEnabledAutoField( self ): """ QSettings should be properly stored, AutoField should be enabled """ self.msg.show( "Info! Test 1 started", 'info', True ) self.autoFieldManager.createAutoField( layer=self.layer, fieldName=u'f1', expression=u'$x' ) dictTmpProperties = self.readStoredSettings( self.layer, u'f1' ) dictExpectedProperties = { 'layer':self.layerPath, 'field':u'f1', 'expression':u'$x', 'layer2':"", 'field2':"", 'enabled':True } self.assertEqual( dictTmpProperties, dictExpectedProperties ) def test02AvoidTwoAutoFieldsOnSameField( self ): """ AutoField should not be created if another one already exists on the same field.""" self.msg.show( "Info! Test 2 started", 'info', True ) res = self.autoFieldManager.createAutoField( layer=self.layer, fieldName=u'f1', expression=u'$y' ) self.assertFalse( res ) def test03EditAutoFieldLayer( self ): """ AutoField value should be updated if a feature is added. Note: It cannot handle the case when writing directly to the provider, as QGIS doesn't have a SIGNAL for that. self.layer.dataProvider().addFeatures( [ tmpFeature ] ) """ self.msg.show( "Info! Test 3 started", 'info', True ) tmpFeature = QgsFeature( self.layer.pendingFields() ) tmpFeature.setGeometry( QgsGeometry.fromPoint( QgsPoint(-74.4, 4.5) ) ) # Either 1: self.layer.startEditing() self.layer.addFeature( tmpFeature ) self.layer.commitChanges() # Or 2: #with edit( self.layer ): # self.layer.addFeature( tmpFeature ) addedFeature = self.layer.getFeatures().next() self.assertEquals( addedFeature['f1'], -74.4 ) def test04ChangeAttributeValue( self ): """ AutoField value should be updated if another AutoField value is changed """ self.msg.show( "Info! Test 4 started", 'info', True ) self.autoFieldManager.createAutoField( layer=self.layer, fieldName=u'modified', expression=u'\'now: \' + to_string("f1")' ) self.layer.startEditing() self.layer.changeAttributeValue( 0, self.layer.fieldNameIndex( u'id' ), 1 ) self.layer.commitChanges() feature = self.layer.getFeatures().next() self.assertEquals( feature['modified'], 'now: -74.4' ) def test05FieldRemovedThenDisableAutoField( self ): """ AutoField should be disabled if its base field is removed """ self.msg.show( "Info! Test 5 started", 'info', True ) fieldIndex = self.layer.fieldNameIndex( u'f1' ) self.layer.startEditing() self.layer.deleteAttribute( fieldIndex ) self.layer.commitChanges() dictTmpProperties = self.readStoredSettings( self.layer, u'f1' ) self.assertFalse( dictTmpProperties['enabled'] ) def test06MissingFieldAddedThenEnableAutoField( self ): """ AutoField should be enabled if missing field is added """ self.msg.show( "Info! Test 6 started", 'info', True ) self.layer.startEditing() self.layer.addAttribute( QgsField( 'f1', QVariant.Double, len=10, prec=2 ) ) self.layer.commitChanges() dictTmpProperties = self.readStoredSettings( self.layer, u'f1' ) self.assertTrue( dictTmpProperties['enabled'] ) def test07LayerRemovedThenDisableAutoField( self ): """ AutoField should be disabled if its base layer is removed """ self.msg.show( "Info! Test 7 started", 'info', True ) QgsMapLayerRegistry.instance().removeMapLayer( self.layer.id() ) # removeMapLayer deletes the underlying object, so create it again self.layer = QgsVectorLayer( self.layerPath, 'puntos', 'ogr' ) dictTmpProperties = self.readStoredSettings( self.layer, u'f1' ) self.assertFalse( dictTmpProperties['enabled'] ) def test08MissingLayerAddedThenEnableAutoField( self ): """ AutoField should be enabled if missing layer is added """ self.msg.show( "Info! Test 8 started", 'info', True ) # test07 deletes the layer object, so create it again self.layer = QgsVectorLayer( self.layerPath, 'puntos', 'ogr' ) QgsMapLayerRegistry.instance().addMapLayer( self.layer ) dictTmpProperties = self.readStoredSettings( self.layer, u'f1' ) self.assertTrue( dictTmpProperties['enabled'] ) def test09RemoveAutoField( self ): """ QSettings should be deleted for the removed AutoField """ self.msg.show( "Info! Test 9 started", 'info', True ) # test07 deletes the layer object, so create it again self.layer = QgsVectorLayer( self.layerPath, 'puntos', 'ogr' ) autoFieldId = self.autoFieldManager.buildAutoFieldId( self.layer, u'f1') self.autoFieldManager.removeAutoField( autoFieldId ) dictTmpProperties = self.readStoredSettings( self.layer, u'f1' ) self.assertEqual( dictTmpProperties, {'layer':"",'field':"",'expression':"",'layer2':"",'field2':"",'enabled':False} ) @classmethod def tearDownClass( self ): self.msg.show( "Info! TearDown started", 'info', True ) # test07 deletes the layer object, so create it again self.layer = QgsVectorLayer( self.layerPath, 'puntos', 'ogr' ) #Remove AutoField modified autoFieldId = self.autoFieldManager.buildAutoFieldId( self.layer, u'modified' ) self.autoFieldManager.removeAutoField( autoFieldId ) #Delete field f1 fieldIndex = self.layer.fieldNameIndex('f1') self.layer.dataProvider().deleteAttributes( [fieldIndex] ) self.layer.updateFields() #Delete features from test layer fIds = self.layer.allFeatureIds() self.layer.dataProvider().deleteFeatures( fIds ) #Remove layer from Registry QgsMapLayerRegistry.instance().removeMapLayer( self.layer.id() ) self.msg.show( "Info! TearDown finished", 'info', True ) QgsApplication.exitQgis()
def initGui( self ): self.messageManager = MessageManager(self.messageMode, self.iface) self.autoFieldManager = AutoFieldManager(self.messageManager, self.iface) self.autoFieldManager.readAutoFields() ''' if the widget is loaded .... then unlosad''' # Get list of all Dockwidgets existe = False for x in self.iface.mainWindow().findChildren(QDockWidget): name = x.objectName() if (name == "SeismicRiskDockWidget"): existe = True #self.iface.removeDockWidget(x) if (not existe): self.dockWidget = SeismicRiskDockWidget(self.iface.mainWindow(), self.iface, self.autoFieldManager, self.messageManager, self.language) self.iface.addDockWidget(Qt.RightDockWidgetArea, self.dockWidget) self.actionDock = QAction(QIcon( ":/plugins/SeismicRisk/icon.png"), \ "Seismic Risk plugin...", self.iface.mainWindow() ) self.actionDock.triggered.connect( self.toggleDockWidget ) # Add custom submenu to Vector menu self.iface.addPluginToVectorMenu( "&Seismic Risk ", self.actionDock ) ''' section pick layers ''' # icon_path = ':/plugins/pickLayer/icon.png' #icon_path = ':/plugins/PruebaAutoFields/icons/info.png' #icon_path = os.path.join(self.plugin_dir, "icons", "pickLayer.png") icon_path = os.path.join(self.plugin_dir, "icons", "points.png") # map tool action print " 66 iconpath " + icon_path self.mapToolAction = QAction(QIcon(icon_path), "Get seismic risk", self.iface.mainWindow()) self.mapToolAction.setCheckable(True) self.mapTool = IdentifyGeometry(self.mapCanvas) self.mapTool.geomIdentified.connect(self.editFeature) self.mapTool.setAction(self.mapToolAction) self.mapToolAction.triggered.connect(self.setMapTool) #self.iface.addToolBarIcon(self.mapToolAction) self.iface.addPluginToMenu("&Pick to Layer in autofields", self.mapToolAction) ''' fin section pick layers''' # Remove Redo buttons from menus and toolbars, they can lead to crashes due # to a corrupted undo stack. redoActionList = [action for action in self.iface.advancedDigitizeToolBar().actions() if action.objectName() == u'mActionRedo'] if redoActionList: self.iface.advancedDigitizeToolBar().removeAction( redoActionList[0] ) self.iface.editMenu().removeAction( redoActionList[0] ) QSettings().setValue( "/shortcuts/Redo", "" ) # Override Redo shortcut # This block (2 options for disabling the Undo panel) didn't work #QSettings().setValue( '/UI/Customization/enabled', True ) #QSettings( "QGIS", "QGISCUSTOMIZATION2" ).setValue( '/Customization/Panels/Undo', False ) #undoDock = self.iface.mainWindow().findChild( QDockWidget, u'Undo' ) #self.iface.removeDockWidget( undoDock ) # Create action that will start plugin configuration #self.action = QAction(QIcon( ":/plugins/AutoFields/icon.png"), "AutoFields plugin...", self.iface.mainWindow() ) icon_path = os.path.join(self.plugin_dir, "icons", "lines.png") print " path " + icon_path self.action = QAction(QIcon(icon_path), "Seismic Risk plugin...", self.iface.mainWindow()) #self.iface.addToolBarIcon(self.action) self.iface.addPluginToMenu("&Show seismic Risk Calculator", self.mapToolAction) # connect the action to the run method self.action.triggered.connect( self.show ) # Add a custom toolbar self.toolbar = self.iface.addToolBar("SeismicRisk") self.toolbar.setObjectName("SeismicRisk") self.toolbar.addAction(self.action) self.toolbar.addAction(self.mapToolAction) # Add custom submenu to Vector menu self.iface.addPluginToVectorMenu( "&AutoFields for calculating resilience", self.action ) # Add a custom toolbar #self.toolbar = self.iface.addToolBar( "SeismicRisk" ) #self.toolbar.setObjectName("SeismicRisk") self.messageManager = MessageManager( self.messageMode, self.iface ) self.autoFieldManager = AutoFieldManager( self.messageManager, self.iface ) self.autoFieldManager.readAutoFields() self.dockWidget = SeismicRiskDockWidget( self.iface.mainWindow(), self.iface, self.autoFieldManager, self.messageManager, self.language ) self.iface.addDockWidget( Qt.RightDockWidgetArea, self.dockWidget )
def requestForMarkets(self, symbole): params = {'q' : symbole, 'startdate' : '2000-01-01', 'enddate' : time.strftime('%Y-%m-%d'), 'num' : 30, 'output' : 'csv'} r = requests.get(self.url, params=params) MessageManager.debugMessage("GoogleFinanceMarketSource : request") return r.text.encode('utf-8').split('\n')
def doIt(self): lookingList = self.lookingList newsRdd = self.filenameRdd.flatMap(lambda x: _fct(lookingList, x)).filter(lambda x: x != []) MessageManager.debugMessage("ReutersNewsSourceHDFS : stop reading Reuters corpus") return newsRdd
def main(argv): print('Kamonohashi Slackbot') # KQI実行のための環境変数追加 if os.name == 'posix': # linux: kqiが格納されている場所 (デフォルトの想定場所) # 本当は外だしすべき kqicli = '/home/(username)/.local/bin' os.environ['PATH'] = os.environ['PATH'] + ':' + kqicli # 設定ファイルの読込み sec = JsonManager(filepath=os.path.join(FLAGS.config_root, 'secure.json')) cfg = JsonManager(filepath=os.path.join(FLAGS.config_root, 'config.json')) msg = MessageManager(filepath=os.path.join(FLAGS.config_root, 'message.json'), kamonohashi_uri=sec['KAMONOHASHI']['Server'], postmessage_uri=sec['SlackURI']) # 監視状態変数 watch_status = { 'dailyjob' : False, 'watching' : False, 'waketime' : datetime.now() } # テナント状態変数 tenant_status = { 'count' : {}, 'state' : {} } tenant_active_job = {} # 監視開始 print('Watch Loop Start') while True: sec.Load() kqi.Login(sec['KAMONOHASHI']['Application']['Username'], sec['KAMONOHASHI']['Application']['Password']) tenants = kqi.GetTenantDict() print('Daily Loop Start') while datetime.now().day == watch_status['waketime'].day: # JSONファイルの更新確認 msg.Load() if cfg.Load(): # 設定ファイルの更新があればテナント情報も再取得する tenants = kqi.GetTenantDict() # 監視状態更新 watch_status = UpdateWatchStatus(watch_status, GetWatchCondition(cfg), msg) # 監視状態変数のクリア tenant_status['count'] = {} # 監視状態がアクティブの場合はテナント毎に監視処理を行う for tenant in tenants: tenant_condition = {} if str(tenant) in cfg['Tenant']: tenant_condition = cfg['Tenant'][str(tenant)] kqi.SwitchTenant(tenant) # 監視状態かどうか tenant_status = UpdateTenantTrainStatus(tenant_status, tenant_condition, tenant, tenants, watch_status, msg, kqi) tenant_status = UpdateTenantInferStatus(tenant_status, tenant_condition, tenant, tenants, watch_status, msg, kqi) # サーバーヘルス監視(特に監視したいのはGPU温度) # T.B.D # すべての監視対象テナントでしばらくメッセージ送信していない場合は実行中に応じてメッセージを投げる if watch_status['watching'] and msg.IsPassedTime(seconds=120 * 60): msg.SendMessage(msg.CreateTenantsRunningInfo(tenants, tenant_status['count'])) msg.UpdateTimestamp() # 連続で処理すると負荷がかかるため、指定したポーリング間隔で待機 time.sleep(cfg['Polling']) # 日の処理完了を抜けた場合はリセットする watch_status['dailyjob'] = False watch_status['waketime'] = datetime.now()
import os from MessageManager import MessageManager from AdvancedScheduleManager import AdvancedScheduleManager from redis_manager import RedisManager with open("./sources/line_bot_token.json", 'r') as f: json_data = json.load(f) access_token = json_data["access_token"] channel_secret = json_data["channel_secret"] line_bot_api = LineBotApi(access_token) handler = WebhookHandler(channel_secret) messageManager = MessageManager(line_bot_api) # redisManager = RedisManager() advancedScheduleManager = AdvancedScheduleManager(bot=messageManager) @csrf_exempt def callback(request: HttpRequest) -> HttpResponse: if request.method == "POST": # get X-Line-Signature header value signature = request.META['HTTP_X_LINE_SIGNATURE'] # get request body as text
from MessageManager import MessageManager from DatabaseConnector import DatabaseConnector if __name__ =='__main__': if len(sys.argv) > 1: # /dev/ttyACM0 A = Server(sys.argv[1]) A.run() else: print("Missing serial port pathname") # database creation conn = DatabaseConnector.connectSQLiteDB('meshDB.db') # MessageManager creation messageManager = MessageManager(conn) # DB reading messageManager.getAllMessages() # CSV writing df = messageManager.getAllMessagesintoPandaDataframe() df.to_csv('csvFile.csv', index = False) # DB update newMessage = Message('3', datetime.datetime.utcnow(), '0000', '0000', '0000') messageManager.putMessage(newMessage) # DB post newMessage = Message('4', datetime.datetime.utcnow(), '0000', '0000', '0000') messageManager.postMessage(newMessage)