def _getDbManager(): "按需获取数据库对象" db = getattr(g, "__dataManager", None); if db is None: db = DataManager() g.__dataManager = db return db
def setUp(self): self.d1 = DataManager() self.wd1 = Word('software', '소프트웨어', (True, self.d1)) self.wd2 = Word('project', '프로젝트', (True, self.d1)) self.d1.words = [self.wd1, self.wd2]
def __init__(self, root): Frame.__init__(self, root) self.__root = root self.__data_manager = DataManager() self.__check_button_type = namedtuple('CheckButtonType', 'widget var') self.__natures = [ "Single Carriageway", "Traffic Island Link", "Dual Carriageway", "Roundabout", "Traffic Island Link At Junction", "Slip Road" ] self.__roads = [ "M3", "M40", "M4", "A1(M)", "M11", "M23", "M20", "M25", "M1", "HIGH STREET", "LONDON ROAD", "HIGH ROAD", "UXBRIDGE ROAD", "STATION ROAD", "BRIGHTON ROAD", "GREEN LANES", "FINCHLEY ROAD", "HARROW ROAD", "NORTH CIRCULAR ROAD", "KINGSTON ROAD", "PORTSMOUTH ROAD", "HERTFORD ROAD", "STAINES ROAD", "CROYDON ROAD", "MAIN ROAD", "CHURCH ROAD", "PARK ROAD" ] self.__motorways = [ "M3", "M40", "M4", "A1(M)", "M11", "M23", "M20", "M25", "M1" ] self.__init_grid() self.__draw_grid()
def __init__(self, ArduinoHoistingData, ArduinoRotationData, ArduinoCirculationData, Hoistingsystem, CirculationSystem, RotaitonSystem): Thread.__init__(self,daemon=True) self.ArduinoHoistingData = ArduinoHoistingData self.ArduinoRotationData = ArduinoRotationData self.ArduinoCirculationData = ArduinoCirculationData self.HoistingSystem = Hoistingsystem self.CirculationSystem = CirculationSystem self.RotaitonSystem = RotaitonSystem self.dataManager = DataManager() self.dataBaseBuffer = { "RPM" :0, "Top_Drive_Torque" :0, "Pressure" : 0, "Loadcell_z1" : 0, "Loadcell_z2" : 0, "Loadcell_z3" :0, "ROP_15s_avg" : 0, "ROP_3m_avg" : 0, "Flow_Rate" : 0, "MSE" : 0, "UCS" : 0, "TVD" : 0, "Bit_Torque" : 0, "WOB" : 0, "d_exponent" : 0, "act1" : 0, "act2" : 0, "act3" : 0, "Velocity" : 0, "Height_Sensor": 0 }
def saveOrUpdateMotion(self, motion): dm = DataManager() #Prüfen ob die Motion existiert #Sie existiert wenn es eine MotionFile mit passendem #Namen gibt path = None if motion.getName() in self.getAllMotionFiles(): path = self.TEMPLATES_PATH + self.getMotionFile(motion.getName()) #print(path) else: i = 0 plainPath = self.TEMPLATES_PATH + 'template' while os.path.exists(plainPath + "%s.txt" % i): i = i + 1 else: plainPath = plainPath + "%s.txt" % i if not os.path.exists(os.path.dirname(self.TEMPLATES_PATH)): os.makedirs(os.path.dirname(self.TEMPLATES_PATH)) path = plainPath dm.saveMotion(motion, path) if motion.getName() not in self.__motions: self.__motions[motion.getName()] = motion self.__motionFiles[motion.getName()] = path print('Motion Saved or Updated')
def beginScan(aPaths, aUsers, aDM=None, aMergeRootPath=True): if aDM is None: aDM = DataManager() users = makeUsers(aDM, aUsers) sd = ScanDisk(users, aPaths, aDM) if aMergeRootPath is not None: sd.mergeRootPaths = aMergeRootPath sd.startScan() aDM.save()
def saveMotion(self, template): dm = DataManager() i = 0 plainPath = self.TEMPLATES_PATH + 'template' while os.path.exists(plainPath + "%s.txt" % i): i = i + 1 else: plainPath = plainPath + "%s.txt" % i if not os.path.exists(os.path.dirname(self.TEMPLATES_PATH)): os.makedirs(os.path.dirname(self.TEMPLATES_PATH)) dm.saveMotion(template, plainPath) print('Motion Saved')
def run(): #数据爬取器 factory_spider = Spiser() #数据解析器 json_parser = dataParser.DataParser() data_manager = DataManager() #总页数:250页 total_page = 251 #获取数据 for i in range(total_page): print i pagedata = factory_spider.get_pageData(i) if pagedata is not None: factory_list = json_parser.json_parser(pagedata) data_manager.save_local(factory_list) time.sleep(1.5)
def run(): #数据爬取器 factory_spider = Spiser() #数据解析器 json_parser = dataParser.DataParser() data_manager = DataManager() # #总页数:250页 # total_page = 251 # #获取数据 # for i in range(total_page): # print i # pagedata = factory_spider.get_pageData(i) # if pagedata is not None: # factory_list = json_parser.json_parser(pagedata) # data_manager.save_local(factory_list) # time.sleep(1.5) memberId_list = data_manager.read_local() for i in memberId_list: print i
def test1(): data = DataManager().query_e2019('%', '7,1') for d in data: print(d) data = [{'content': d[1]} for d in data] # data=[{'content':'在华盛顿期间,习近平还先后会见了前来参加本届核安全峰会的丹麦首相拉斯穆森、韩国总统朴槿惠和阿根廷总统马克里,并出席了伊核问题六国机制领导人会议。'}] data = [{ 'content': '谈及美国以所谓“安全威胁”为由打压中国高科技企业,马尔科表示,美国必须提供有说服力的证据,否则指控只是猜测而已。马尔科希望,中美两国能以建设性的方式解决贸易分歧,“贸易战没有赢家,应尽快找到和平解决方案”。\ 摩尔多瓦共产党人党意识形态秘书康斯坦丁·斯塔里什说,近几十年,全球经济体系已被证实行之有效,并推动了各国经济和贸易发展。如今,美国借保护自身经济利益为由,破坏这一体系基础,这好比“大象闯进瓷器店”,破坏了现有模式,却又不提供替代方案。\ 斯塔里什认为,美国盲目挑起经贸摩擦,是为保持自身“世界经济霸主地位”,此举严重违背市场规律,表明美国不愿对世界经济发展负责,同时也将影响美国自身经济发展。\ 巴勒斯坦法塔赫革命委员会委员巴萨姆说,美国挑起对华贸易摩擦的行为是“霸凌逻辑”,贸易战对中美双方都会造成损失。\ 巴勒斯坦人民党总书记萨利希指出,美国此举违背市场规律和国际贸易规则,不仅对美国和中国,乃至对世界经济都造成威胁。此外,美国对华为等中国企业进行打压,是因为相关企业具有强劲的全球竞争力。(执笔记者:马湛;参与记者:张修智、林惠芬、陈进、杨媛媛、赵悦、周天翮)' }] data = extract_all(data) + extra_info for d in data: print(d) db_manager = NeoManager() db_manager.clear() db_manager.write2db(data)
def scanByJsonFile(aJsonFileName): "根据配置文件扫描数据" try: with open(aJsonFileName) as f: config = json.load(f) except Exception as e: print(e) return if not isinstance(config, list): print("文件格式有误") return dm = DataManager() for item in config: try: beginScan(item.get("paths"), item.get("users"), dm, item.get("merge")) except Exception as e: print(e)
def __init__(self, pushsendingstart): self.dataManager = DataManager() topic, intervalStart, intervalEnd = self.dataManager.readAllSettings() self.webCrawler = WebCrawler() self.pushSender = PushSender(self.dataManager, "default") if pushsendingstart: self.pushSender.pushSendingThreadStart() self.nowListedWords = self.getAllWords() self.mainWindow = MainWindow(self) self.memorizeWindow = MemorizeWindow(self) self.settingWindow = SettingWindow(self) self.settingWindow.settingLoad(topic, intervalStart, intervalEnd) self.mainWindow.showWindow() self.memorizeWindow.hideWindow() self.settingWindow.hide()
def __init__(self, pushsendingstart): self.dataManager = DataManager() sts = self.dataManager.realAllSettings() self.webCrawler = WebCrawler() self.pushSender = PushSender(self.dataManager, "wowawesome") if pushsendingstart: self.pushSender.pushSendingThreadStart() self.nowListedWords = self.getAllWords() self.mainWindow = MainWindow(self) self.memorizeWindow = MemorizeWindow(self) self.settingWindow = SettingWindow(self) self.settingWindow.settingLoad(sts) self.mainWindow.showWindow() self.memorizeWindow.hideWindow() self.settingWindow.hide()
def initMotions(self): dm = DataManager() #Hier alle Devices intitialisieren oldPath = os.getcwd() try: os.chdir(self.TEMPLATES_PATH) except FileNotFoundError: print("Could not load Motions") return #For jede Geste for file in sorted(os.listdir()): filePath = os.getcwd() + "/" + file if os.path.isdir(filePath): continue motion = dm.getMotion(filePath) self.__motions[motion.getName()] = motion self.__motionFiles[motion.getName()] = file os.chdir(oldPath)
def __init__(self): self.tp=TextProcess() #数据库管理,加载政要数据 self.dataManager=DataManager() # self.political_person_dict=list() #改用aho形式进行存储,方便进行多模匹配。 self.aho_policical_person=ahocorasick.Automaton() try: # load_file = open('./mod/political_person_dict.bin', 'rb') # self.political_person_dict = pickle.load(load_file) # logging.info('political_person_dict count %d' % (len(self.political_person_dict))) file = open('./mod/aho_policical_person.aho', 'rb') self.aho_policical_person = pickle.load(file) logging.info('aho_policical_person count %d' % (len(self.aho_policical_person))) except: pass self.detector=MultiSenDetect() #加载地名数据索引,用于判断词性为hs的是否是地名 load_file = open('./mod/place_dict.bin', 'rb') self.place_dict = pickle.load(load_file) logging.info('place_dict count %d' % (len(self.aho_policical_person))) return
def run(): #数据爬取器 factory_spider = Spiser() #数据解析器 data_parser = dataParser.DataParser() data_manager = DataManager() url_manager = UrlManager() #获取到三个url列表,均为需要爬取的数据 url_list, shopurl1_list, shopurl2_list = url_manager.tel_url() total_num = len(url_list) crawred_url = url_manager.crawred_url() company_dataList = [] for i in range(total_num): url = url_list[i] shopurl1 = shopurl1_list[i] shopurl2 = shopurl2_list[i] if url not in crawred_url: page_data = factory_spider.get_urlpage(url) page_shop1 = factory_spider.get_urlpage(shopurl1) page_shop2 = factory_spider.get_urlpage(shopurl2) #使用解析器,解析三个页面的数据 companydata = data_parser.get_company_data(page_data, page_shop1, page_shop2, url) #将解析后的数据元组保存至列表 company_dataList.append(companydata) time.sleep(1.1) # elif url in crawred_url: # print '已经爬取过了',url # 将爬取结果保存至本地csv文件,爬5家店铺保存一次 print '=========', i, '==========' if i % 10 == 0 and len(company_dataList) > 0: data_manager.save_local_tel(company_dataList) company_dataList = [] time.sleep(10)
cur.execute( "select distinct classification from itn_link where description = 'Motorway';" ) motorways = [("classification", m[0].replace("'", "''")) for m in cur.fetchall()] cur.execute('select distinct street from itn_link;') streets = [("street", s[0].replace("'", "''")) for s in cur.fetchall()] natures = [ "Single Carriageway", "Traffic Island Link", "Dual Carriageway", "Roundabout", "Traffic Island Link At Junction", "Slip Road" ] dM = DataManager() def days_to_binary(day): if day == 0 or day == 6: return 0 return 1 def func0(params, speed, rainfall_depth, day, hour): p0, e0, p1, e1, p2, e2, c = params return p0 * rainfall_depth**e0 + p1 * day**e1 + p2 * hour**e2 + c - speed def plot_func0(params, rainfall_depth, day, hour): p0, e0, p1, e1, p2, e2, c = params
import traceback from flask import Flask, render_template, request, redirect, url_for from dataManager import DataManager import datetime from dbManager import DashboardInfo OPTIONS = [ "ALL_TEST_CASES", "FAILED_IN_CURRENT_RUN", "FAILING_FROM_LAST_10_RUNS", "FAILING_FROM_LAST_3_RUNS", "UNSTABLE_TEST_CASES", "PASS_STATUS_SWITCHED" ] VIEWS = ['TABLE_VIEW', 'GRAPH_VIEW'] dataManager = DataManager() ####################### HOME ###################### app = Flask(__name__) @app.errorhandler(Exception) def default_error_handler(error): '''Default error handler''' original = getattr(error, "original_exception", error) traceback.print_tb(error.__traceback__) print("ERROR occurded during handling message:", original) return render_template("view_error.html") @app.route('/', methods=["GET", "POST"]) def hello(): all_dashboards = dataManager.get_all_dashboards()
def getFeaturesFavorAgainst(self, mode, listOfFeats): #only tweets with favor or against X, y = self.getFeaturesMatrix(mode, listOfFeats, 'stance') # print X,y nonerows = np.where(y == self.labelenc.transform('NONE'))[0] # print y # print nonerows X = np.delete(X, nonerows, axis=0) y = np.delete(y, nonerows) return X, y def getFeaturesStanceNone(self, mode, listOfFeats): X, y = self.getFeaturesMatrix(mode, listOfFeats, 'stance') y[y == self.labelenc.transform('FAVOR')] = 3 y[y == self.labelenc.transform('AGAINST')] = 3 return X, y if __name__ == '__main__': dp = DataManager('../data/train.csv', '../data/test.csv') fe = FeatureExtractor(dp) # fe.getYStanceNone('train') # fe.getFeaturesFavorAgainst('train',['words2vec']) # fe.getFeaturesStanceNone('train',['words2vec']) # X,y = fe.getFeaturesFavorAgainst('train',['words2vec']) # print fe.getFeaturesMatrix('train',['words'],'topic','Hillary Clinton')[0].shape # print fe.getFeaturesTopicNontopic('train',['words'],'topic', 'Hillary Clinton')[0].shape # print fe.getX('train',fe.data.trainTweets, ['words2vec']).shape
def active_learning(network: str, dataset: str, pool_length: int, method: str, k: str, num_trainings: int, batch_size: int, num_epochs: int, learning_rate: float, use_cuda: bool): """ Function that execute an active learning depending on several arguments Args: network: the model that will be trained and tested dataset: the data used for the active learning method: method used to select the k samples to add to the training set at each active learning loop k: number of samples to add num_trainings: number of active learning loops batch_size: number of samples in a batch num_epochs: number of loops during one training learning_rate: learning rate of the optimizer use_cuda: boolean to use the gpu for training Returns: The list of accuracies of each test phase """ print("Starting active learning with" "\n\tmodel: " + network + "\n\tdataset: " + dataset + "\n\tselection method: " + method + "\n\tk: " + k + "\n\tnum trainings: " + str(num_trainings) + "\n\tbatch size: " + str(batch_size) + "\n\tnum epochs: " + str(num_epochs) + "\n\tlearning rate: " + str(learning_rate) + "\n\tuse cuda: " + str(use_cuda)) model = getModel(network) data = getData(dataset, pool_length) selection_method = getSelectionMethod(method) k = int(k) if len(data.get_pool_data()[0]) < k * num_trainings: raise ValueError( "'k' or 'num-trainings' is too big, " "the program will not be able to extract the training " "samples from the pool at some point") # Set the optimizer factory function optimizer = optimizer_setup(SGD, lr=learning_rate, momentum=0.9) # Create the network depending on the number of classes model = model(num_classes=len(data.get_label_names())) # First index samples to train train_idx = np.arange(k) # List that will contain the test accuracy of each training accuracies = [] for num_training in range(num_trainings): print("\nActive learning loop " + str(num_training + 1) + "/" + str(num_trainings)) # Set data loaders depending on training samples dataManager = DataManager(data=data, \ train_idx=train_idx, \ batch_size=batch_size) # Set the network trainer and launch the training netTrainer = NetTrainer(model=model, \ data_manager=dataManager, \ selection_method=selection_method, \ loss_fn=nn.CrossEntropyLoss() , \ optimizer_factory=optimizer, \ use_cuda=use_cuda) netTrainer.train(num_epochs) # Select k samples depending on the selection method # and add them to the training samples add_to_train_idx = netTrainer.evaluate_on_validation_set(k) train_idx = np.concatenate((train_idx, add_to_train_idx)) print("Selected next training set indexes") # Compute the accuracy on the test set and save it accuracy = netTrainer.evaluate_on_test_set() accuracies.append(accuracy) return accuracies
from __future__ import absolute_import, unicode_literals from .celery import app from configs import configs, locations from dataManager import DataManager import time from datetime import datetime import requests import json # import redis dm = DataManager() def get_utc_time(date_t: datetime): """获取指定的北京时间的utc时间""" t = date_t.timestamp() # 目标时间的unix时间戳 return datetime.utcfromtimestamp(t) def send(url, method='get', data=None) -> dict: if method == 'get' or method == 'GET': headers = {'Authorization': configs['Authorization']} r = requests.get(url, headers=headers) # request elif method == 'post' or method == 'POST': headers = { 'Authorization': configs['Authorization'], 'Content-Type': 'application/json;charset=UTF-8' } if data: r = requests.post(url, data=data, headers=headers)
# -*- coding: utf-8 -*- """ Created on Mon May 14 17:12:19 2018 @author: greg """ import requests from bs4 import BeautifulSoup import codecs import time import pandas as pd import numpy as np from competition import Competition from competitor import Competitor from dataManager import DataManager competition_manager = Competition() competitor_manager = Competitor() data_manager = DataManager()
def updateMotion(self, motion): dm = DataManager() path = self.TEMPLATES_PATH + self.getMotionFile(motion.getName()) dm.saveMotion(motion, path) print('Motion Updated')
def __init__(self): self.data_manager = DataManager()
def extractSPDHandSave(modelPath='', inputScheme='cropped'): num_classes = 6 batchSize = 32 # placeholder for input image data saver = tf.train.import_meta_graph(modelPath + '.meta') if (inputScheme == 'cropped'): trainImg_path = '../dataNpyFiles/bbox_cropped_train_img.npy' trainLabel_path = '../dataNpyFiles/bbox_cropped_train_label.npy' testImg_path = '../dataNpyFiles/bbox_cropped_test_img.npy' testLabel_path = '../dataNpyFiles/bbox_cropped_test_label.npy' elif (inputScheme == 'activated'): trainImg_path = '../dataNpyFiles/bbox_activated_train_img.npy' trainLabel_path = '../dataNpyFiles/bbox_activated_train_label.npy' testImg_path = '../dataNpyFiles/bbox_activated_test_img.npy' testLabel_path = '../dataNpyFiles/bbox_activated_test_label.npy' elif (inputScheme == 'origin'): trainImg_path = '../dataNpyFiles/origin_train_img.npy' trainLabel_path = '../dataNpyFiles/origin_train_label.npy' testImg_path = '../dataNpyFiles/origin_test_img.npy' testLabel_path = '../dataNpyFiles/origin_test_label.npy' else: print('Wrong with param...') return print('load train data...') trainData = DataManager(imgNpyPath=trainImg_path, labelNpyPath=trainLabel_path, batchSize=batchSize, classNumber=num_classes) print('load test data...') testData = DataManager(imgNpyPath=testImg_path, labelNpyPath=testLabel_path, batchSize=batchSize, classNumber=num_classes) trainDataSize = np.shape(trainData.label_data)[0] testDataSize = np.shape(testData.label_data)[0] train_batches_per_epoch = int(np.floor(trainDataSize / batchSize)) val_batches_per_epoch = int(np.floor(testDataSize / batchSize)) train_bit_representation = [] train_label_representation = [] test_bit_representation = [] test_label_representation = [] graph = tf.get_default_graph() # load for all operation restored latent_op = graph.get_tensor_by_name('latentLayer/latentLayer_sig:0') # x = tf.placeholder(tf.float32, [32, 227, 227, 3], name='Placeholder_4') for op in graph.get_operations(): print(op.name) with tf.Session() as sess: saver.restore(sess, modelPath) print('bit generation for train...') # for extract bit of training data print('trainData initializer on GPU') q = trainData.getTrueData() sess.run(trainData.getInitializer(), feed_dict={ trainData.x: q[0], trainData.y: q[1] }) for step in range(train_batches_per_epoch): if (step % 20 == 0): print('For ' + str(step)) img_batch, label_batch = sess.run( trainData.getNextBatchPlaceholder()) # bitRepresentation = sess.run('latentLayer/Sigmoid:0', feed_dict={'Placeholder_4:0': img_batch, 'Placeholder_6:0': 1.0}) bitRepresentation = sess.run(latent_op, feed_dict={ 'input_network:0': img_batch, 'keep_prob:0': 1.0 }) train_label_representation.append(label_batch) train_bit_representation.append(bitRepresentation) # for extract bit of testing data print('bit generation for test...') print('testData initializer on GPU') p = testData.getTrueData() sess.run(testData.getInitializer(), feed_dict={ testData.x: p[0], testData.y: p[1] }) for step in range(val_batches_per_epoch): if (step % 20 == 0): print('For ' + str(step)) # Is it right?? img_batch, label_batch = sess.run( testData.getNextBatchPlaceholder()) bitRepresentation = sess.run(latent_op, feed_dict={ 'input_network:0': img_batch, 'keep_prob:0': 1.0 }) test_label_representation.append(label_batch) test_bit_representation.append(bitRepresentation) train_bit_representation_npy = np.asarray(train_bit_representation) test_bit_representation_npy = np.asarray(test_bit_representation) train_label_representation_npy = np.asarray(train_label_representation) test_label_representation_npy = np.asarray(test_label_representation) train_save_dic = { 'bit': train_bit_representation_npy, 'label': train_label_representation_npy } test_save_dic = { 'bit': test_bit_representation_npy, 'label': test_label_representation_npy } print('save train bit...') np.save(modelPath.split('/')[-1] + '_train_bit.npy', train_save_dic) print('save test bit...') np.save(modelPath.split('/')[-1] + '_test_bit_npy', test_save_dic) print('done')
import numpy as np from event import Event from fiducialCuts import FiducialCuts import time import matplotlib.pyplot as plt from dataManager import DataManager import math # Read in data here start = time.time() # data_file = '/media/tylerviducic/Elements/aidapt/data/synthetic/clasfilter2_5M780.npy' # change to your path, obviously data_file = '/media/tylerviducic/Elements/aidapt/data/recon/twopi_ppip.10.zzz' data_manager = DataManager(data_file) input_array = data_manager.get_numpy_array() output_list = [] phi_list = [] num_bins = 180 num_rows, num_columns = input_array.shape for n in range(num_rows): row = input_array[n] event = Event(row) phi = math.degrees(abs(event.get_proton_phi())) phi_list.append(phi)
def __init__(self, n): self.data = DataManager('../data/train.csv','../data/test.csv', n) self.fe = FeatureExtractor(self.data) self.eval = Evaluate()