def CsvtoMongoDB(self, host, database, collection, rowCount, chunkSize, domain_index, domains): self.domain_index = domain_index self.domains = domains self.rowCount = rowCount self.chunkSize = chunkSize self.host = host self.database = database self.collection = collection self.mongoDb = MongoDB.MongoDB(self.host, self.database, self.collection) durum = 0 try: if len(domain_index) != 0: for chunk in pd.read_csv(self.filepath, skiprows=1 + self.rowCount, chunksize=self.chunkSize, usecols=self.domain_index, names=self.domains, low_memory=False, encoding='utf'): try: data_json = json.loads(chunk.to_json(orient='records')) durum = 1 except: print("Csv to json coverting error") error = "Csv to json coverting error ---> user name " writeToErrorsInLogFiles.writeErrorLogFileMongo(error) durum = 0 break elif len(domain_index) == 0: for chunk in pd.read_csv(self.filepath, skiprows=1 + self.rowCount, chunksize=self.chunkSize, names=self.domains, low_memory=False, encoding='utf'): try: data_json = json.loads(chunk.to_json(orient='records')) durum = 1 except: print("Csv to json coverting error") error = "Csv to json coverting error ---> user name " writeToErrorsInLogFiles.writeErrorLogFileMongo(error) durum = 0 break except: print("Csv reading error") error = "Csv reading error ---> user name " writeToErrorsInLogFiles.writeErrorLogFileMongo(error) durum = 0 if durum == 1: self.mongoDb.InsertMany(data_json) durum = 1 else: pass self.mongoDb.ConnectionClose()
def __init__(self): # 필요한 객체 생성 self.otp_client = MQTT_OTP_Subscriber("otp_result") self.yolo_checker = Yolo_checker() self.mongo = MongoDB() # 사람을 확인했는지 확인하는 변수 self.flag = 0 # 터틀봇 영상 가져오기 self.cap = cv2.VideoCapture( 'http://192.168.0.32:8080/stream?topic=/usb_cam/image_raw')
def __init__(self, topic): self.limit = 0 # 최대 인증 시도 5회까지 check self.result_msg = None # otp 인증 결과 self.topic = topic # mongoDB 객체 생성해서 OTP 결과값 저장 self.mongo = MongoDB() client = mqtt.Client() client.connect("localhost", 1883, 60) client.on_connect = self.on_connect client.on_message = self.on_message client.loop_start()
def __init__(self, api_key, secret_key, uri): self.api_key = api_key self.secret_key = secret_key self.uri = uri # Create websocket to add and cancel orders self.ws = create_connection(self.uri) # Duplicate object for the threading channel WebSocket.Okcoin.instance = self # Threading to run separetely the run_forever function of the websocket aimed to retrieve market data thread = threading.Thread(target=self.run) thread.daemon = True # Daemonize thread - it allows the main application to exit even if the thread is still running thread.start() # Instance of mongoDb database to log trades and candlesticks self.mongo = MongoDB.MongoDB() self.initObject()
def __init__(self, raceNo, eventTimeStamp): ################################################################ # Analyzer constructor initiate all the modulus of the program # # This is the main module of the project - the Brain which # # Main module == in charge of all the other modules # ################################################################ ########################################################################################### # If folder does not exist on user local PC, then connect to remote server and copy files # ########################################################################################### if os.path.isdir(LOCAL_DIR_PATH + str(raceNo)) == False: client = ClientOps.ClientOps(raceNo) ############################################################ # Start the analyzing process - execute Modules functions # ############################################################ self.mongo = MongoDB.MongoDB() self.fileObj = FilesOps.FilesOps(raceNo, eventTimeStamp) #self.fileObj.find_SVC_errors() svcErrorCode = self.fileObj.find_SVC_errors() if svcErrorCode == -1: print "SVC File error! -- File does not exist! or SVC Error not found" return self.fileObj.collect_issues() self.fileObj.sotrtedAssertArr print "#" * 20 print "RACE VERSION:", self.fileObj.getVers() print "#" * 20 ####################################################################### # Find in SVC file the relevant error code which related to the issue # ####################################################################### self.mongo.find_apar(svcErrorCode, self.fileObj.raceVers) self.raceObjects = [] # List of RACE objects return
# Unit tests for MongoDB class import MongoDB import unittest db1 = MongoDB.MongoDB() # clear the database to remove duplicates db1.clear_db() test_post1 = {"Author": "Stephen King", "Name": "It"} db1.insert(test_post1) print(db1.find({"Author": "Stephen King"})) db1.change_stock({"Name": "It"}, 6) print('test find') print(db1.find({"Name": "It", "Author": "Stephen King"})) test_post2 = {"Author": "Herman Melville", "Name": "Moby Dick"} res = db1.insert(test_post2) print(res) print(db1.find({"Name": "Moby Dick"})) print("First list all") temp_list = db1.list_all() print(*temp_list, sep='\n')
def __init__(self): self.mongo = MongoDB.MongoDB() pass
from DHT import * from HCR import * from PIR import * from DataList import DataList from File import File from MySQL import * from MongoDB import * import sys try: file = File.readData() except Exception as e: file = [] newSQL = MySQL() newMongo = MongoDB() newSQL.Conexion() newMongo.mongoConexion() sensorList = newSQL.getSensors() class Sensors: def __init__(self): self.instancesList = DataList() self.createInstances() def createInstances(self): for o in sensorList: if self.instancesList.getData(None, o['name']) is None:
print("_________ SERVER get Data _________") #String형의 이미지를 수신받아서 이미지로 변환 하고 화면에 출력 # length = recvall(conn, 16) #길이 16의 데이터를 먼저 수신하는 것은 여기에 이미지의 길이를 먼저 받아서 이미지를 받을 때 편리하려고 하는 것이다. # stringData = recvall(conn, int(length)) # data = numpy.fromstring(stringData, dtype='uint8') # s.close() #................이부분 닫지 말까 # decimg = cv2.imdecode(data, 1) # 이미지 확인하기 #cv2.imshow('SERVER', decimg) #cv2.waitKey(0) #cv2.destroyAllWindows() # ...... 0. mongoDB에 저장할 객체 생성 mongo = MongoDB() # mongoDB 객체 생성 # ...... 1. save origin image from drone # cv2.imwrite('./container/origin.jpg', decimg) # img = open('./container/origin.jpg', 'rb') # mongo.storeImg_map(img, 'map_origin.jpg') # print("Drone image save!!") # ...... 2. 길찾기 시작 및 맵 DB에 저장 print("____________find_path_____________") # drone이 넘겨준 이미지를 이용하여 길찾기 알고리즘 decimg = cv2.imread('./container/origin.jpg') find_path = Find_path(decimg) find_path.bfs() find_path.real_path()
warnings.simplefilter("ignore", DeprecationWarning) #socket 수신 버퍼를 읽어서 반환하는 함수 def recvall(sock, count): buf = b'' while count: newbuf = sock.recv(count) if not newbuf: return None buf += newbuf count -= len(newbuf) return buf # .......... 0-2. mongoDB 객체 생성 mongo = MongoDB() # .......... 0-1. 서버의 IP와 port 번호 # .......... 4-2. TurtleBot 에게 MQTT 로 경로 정보 넘기기 mqtt = mqtt.Client("loadFinder") # MQTT client 생성, 이름 "" mqtt.connect("localhost", 1883) # 로컬호스트에 있는 MQTT서버에 접속 # .......... 1. TCP 소켓 열고 수신 대기 TCP_IP = '192.168.0.15' TCP_PORT = int(sys.argv[1]) s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.bind((TCP_IP, TCP_PORT)) s.listen(True) print( "###############################################################################" )
918165914914304003: 1, 918169282093121539: 2, 918133272084959232: 3, 918078494000721922: 4, 917879954712465414: 5, 917335939478949888: 6, 917783890231791617: 7, 918084964180201472: 8, 918056753287311360: 9 } listMetric = ["RS", "RSA", "visibility_value", "MD", "PD"] connectMongoDB = Connect2MongoDB('localhost', 27017) connectMongoDB.setDB('test1') db = MongoDB(connectMongoDB) data = dict() print("\nStart SORT METRIC: {0}".format(datetime.datetime.now())) for key in listMetric: tweets = db.getSort(MongoDB.CLEAR_TWEETS_COLLECTION, (key, -1)) count = 0 for t in tweets: tweetId = t["tweetId"] if tweetId in listRS and key != "RS": if "RS" in data: if tweetId in data["RS"]: data["RS"][tweetId] += "\t" + key + ":" + str( t[key]) + ",\tpos" + key + ":" + str(count)
server_sock.close() def LEDShow(): while(led.stop == False): # db_result = db.list_all() # led.size = len(db_result) led.size = 254 led.showVarieties() # Get command line arguments args = storage_parser().parse_args() # Create clean exit signal signal.signal(signal.SIGINT, signal_handler) db = MongoDB.MongoDB() led = LED.LED(0, False) backlog = int(args.backlog) port = int(args.port_num) s_size = int(args.socket_size) # Start the two threads threadLED = threading.Thread(target=LEDShow, args=()) threadMongo = threading.Thread(target=MongoControl, args=()) threadLED.daemon = True threadMongo.daemon = True threadLED.start() threadMongo.start()
def main(): filepath = "" host = "" database = "" collection = "" core_count = multiprocessing.cpu_count() ram_available = (psutil.virtual_memory().available / (1024 * 1024 * 1024)) calculate_chunk_size = int(ram_available / (core_count * (15 / 10)) * 1000000) performance = PerformanceOfDatabase.PerformanceOfDatabase() print("*************************************") while True: filepath = input("file path --") if (os.path.exists(filepath)): break else: print("File not exists. Please enter again file name!!!\n") nesne = DataImportMongoDB.DataImportMongoDB(filepath) print("*************************************") #size calculate manager = multiprocessing.Manager() size_of_csv = manager.dict() domains_of_csv = manager.dict() proc = multiprocessing.Process(target=size_calculate, args=(filepath, size_of_csv, (domains_of_csv))) proc.start() #Mongo connect condition while True: host = input("host --") database = input("database --") collection = input("collection --") mongo_connect = MongoDB.MongoDB(host, database, collection) if mongo_connect.connect == True: break else: print("Database not founded !!! \n") os.system("pause") os.system("cls") print("*************************************") print("*************************************") toMail = input("Enter the Gmail address --") sendToMailPsql.sendMailStartMongo(toMail) print("*************************************") #header info while True: header_info = input("Header(0)/ No Header(1) --") if header_info == "1" or header_info == "0": if header_info == "0": break else: print("if no header csv is not convert json") else: print("please correct value !!! \n") os.system("pause") os.system("cls") print("*************************************") file_size = os.stat(filepath).st_size / (1024 * 1024) proc.join() for alfa in size_of_csv.values(): limit = int(alfa) domains = domains_of_csv.values() domains = domains[0] limit -= 1 selected_columns = [] selected_columns_index = [] print("*************************************") while True: column_state = input("Required Columns(0)/ All Columns(1) --") if column_state == "1": print("selected all columns") break elif column_state == "0": print(domains) while True: column_count = input("column count: --") try: column_count = int(column_count) if column_count > len(domains): print( "columns count is not bigger than all columns count" ) else: break except: print("please enter integer value \n") try: column_count = int(column_count) counter = 0 while counter < column_count: enter_column_name = input( "enter column name --") if enter_column_name not in domains: print("please right column name \n") else: find_index = domains.index(enter_column_name) selected_columns.append(enter_column_name) selected_columns_index.append(find_index) counter += 1 domains = selected_columns break except: print("SYSTEM ERROR") else: print("please correct value !!! \n") os.system("pause") os.system("cls") print("*************************************") print("*************************************") if (limit / core_count) < calculate_chunk_size: chunk_size = int(limit / core_count) chunk_size = max(chunk_size, 1) else: chunk_size = calculate_chunk_size chunk_size = max(chunk_size, 1) init_size = 0 procs = [] #procesing list working_proc_count = 0 performance.TimePerformanceStart() while True: try: if working_proc_count < core_count: if init_size + 1 > limit: break proc = multiprocessing.Process(target=nesne.CsvtoMongoDB, args=( host, database, collection, init_size, chunk_size, selected_columns_index, domains, )) procs.append(proc) proc.start() init_size += chunk_size working_proc_count += 1 for one_proc in procs: if one_proc.is_alive() == False: procs.remove(one_proc) working_proc_count -= 1 if init_size + 1 > limit: break except: break for one_proc in procs: if one_proc.is_alive() == False: procs.remove(one_proc) working_proc_count -= 1 while True: if len(procs) == 0: break for one_proc in procs: if one_proc.is_alive() == False: procs.remove(one_proc) working_proc_count -= 1 if len(procs) == 0: break pipeline, time_difference = performance.TimePerformanceEnd( "Importing Finish. MongoDB", file_size) sendToMailPsql.sendMailFinishMongo(toMail, str(time_difference), limit, pipeline) print("Importing data rows count : ", limit)