Beispiel #1
0
    def CsvtoMongoDB(self, host, database, collection, rowCount, chunkSize,
                     domain_index, domains):
        self.domain_index = domain_index
        self.domains = domains
        self.rowCount = rowCount
        self.chunkSize = chunkSize
        self.host = host
        self.database = database
        self.collection = collection
        self.mongoDb = MongoDB.MongoDB(self.host, self.database,
                                       self.collection)
        durum = 0

        try:
            if len(domain_index) != 0:
                for chunk in pd.read_csv(self.filepath,
                                         skiprows=1 + self.rowCount,
                                         chunksize=self.chunkSize,
                                         usecols=self.domain_index,
                                         names=self.domains,
                                         low_memory=False,
                                         encoding='utf'):
                    try:
                        data_json = json.loads(chunk.to_json(orient='records'))
                        durum = 1
                    except:
                        print("Csv to json coverting error")
                        error = "Csv to json coverting error ---> user name "
                        writeToErrorsInLogFiles.writeErrorLogFileMongo(error)
                        durum = 0
                    break
            elif len(domain_index) == 0:
                for chunk in pd.read_csv(self.filepath,
                                         skiprows=1 + self.rowCount,
                                         chunksize=self.chunkSize,
                                         names=self.domains,
                                         low_memory=False,
                                         encoding='utf'):
                    try:
                        data_json = json.loads(chunk.to_json(orient='records'))
                        durum = 1
                    except:
                        print("Csv to json coverting error")
                        error = "Csv to json coverting error ---> user name "
                        writeToErrorsInLogFiles.writeErrorLogFileMongo(error)
                        durum = 0
                    break
        except:
            print("Csv reading error")
            error = "Csv reading error ---> user name "
            writeToErrorsInLogFiles.writeErrorLogFileMongo(error)
            durum = 0
        if durum == 1:
            self.mongoDb.InsertMany(data_json)
            durum = 1
        else:
            pass
        self.mongoDb.ConnectionClose()
    def __init__(self):
        # 필요한 객체 생성
        self.otp_client = MQTT_OTP_Subscriber("otp_result")
        self.yolo_checker = Yolo_checker()
        self.mongo = MongoDB()

        # 사람을 확인했는지 확인하는 변수
        self.flag = 0
        # 터틀봇 영상 가져오기
        self.cap = cv2.VideoCapture(
            'http://192.168.0.32:8080/stream?topic=/usb_cam/image_raw')
    def __init__(self, topic):
        self.limit = 0  # 최대 인증 시도 5회까지 check
        self.result_msg = None  # otp 인증 결과
        self.topic = topic

        # mongoDB 객체 생성해서 OTP 결과값 저장
        self.mongo = MongoDB()

        client = mqtt.Client()
        client.connect("localhost", 1883, 60)
        client.on_connect = self.on_connect
        client.on_message = self.on_message
        client.loop_start()
Beispiel #4
0
    def __init__(self, api_key, secret_key, uri):
        self.api_key = api_key
        self.secret_key = secret_key
        self.uri = uri
        # Create websocket to add and cancel orders
        self.ws = create_connection(self.uri)

        # Duplicate object for the threading channel
        WebSocket.Okcoin.instance = self

        # Threading to run separetely the run_forever function of the websocket aimed to retrieve market data
        thread = threading.Thread(target=self.run)
        thread.daemon = True  # Daemonize thread - it allows the main application to exit even if the thread is still running
        thread.start()

        # Instance of mongoDb database to log trades and candlesticks
        self.mongo = MongoDB.MongoDB()
        self.initObject()
Beispiel #5
0
    def __init__(self, raceNo, eventTimeStamp):

        ################################################################
        # Analyzer constructor initiate all the modulus of the program #
        # This is the main module of the project - the Brain which     #
        # Main module == in charge of all the other modules            #
        ################################################################

        ###########################################################################################
        # If folder does not exist on user local PC, then connect to remote server and copy files #
        ###########################################################################################
        if os.path.isdir(LOCAL_DIR_PATH + str(raceNo)) == False:
            client = ClientOps.ClientOps(raceNo)

        ############################################################
        # Start the analyzing process - execute Modules functions  #
        ############################################################
        self.mongo = MongoDB.MongoDB()
        self.fileObj = FilesOps.FilesOps(raceNo, eventTimeStamp)
        #self.fileObj.find_SVC_errors()
        svcErrorCode = self.fileObj.find_SVC_errors()
        if svcErrorCode == -1:
            print "SVC File error! -- File does not exist! or SVC Error not found"
            return

        self.fileObj.collect_issues()
        self.fileObj.sotrtedAssertArr
        print "#" * 20
        print "RACE VERSION:", self.fileObj.getVers()
        print "#" * 20

        #######################################################################
        # Find in SVC file the relevant error code which related to the issue #
        #######################################################################

        self.mongo.find_apar(svcErrorCode, self.fileObj.raceVers)
        self.raceObjects = []  # List of RACE objects
        return
Beispiel #6
0
# Unit tests for MongoDB class

import MongoDB
import unittest

db1 = MongoDB.MongoDB()

# clear the database to remove duplicates
db1.clear_db()

test_post1 = {"Author": "Stephen King", "Name": "It"}

db1.insert(test_post1)

print(db1.find({"Author": "Stephen King"}))

db1.change_stock({"Name": "It"}, 6)

print('test find')
print(db1.find({"Name": "It", "Author": "Stephen King"}))

test_post2 = {"Author": "Herman Melville", "Name": "Moby Dick"}

res = db1.insert(test_post2)
print(res)
print(db1.find({"Name": "Moby Dick"}))

print("First list all")
temp_list = db1.list_all()
print(*temp_list, sep='\n')
Beispiel #7
0
    def __init__(self):

        self.mongo = MongoDB.MongoDB()

        pass
from DHT import *
from HCR import *
from PIR import *
from DataList import DataList
from File import File
from MySQL import *
from MongoDB import *
import sys

try:
    file = File.readData()
except Exception as e:
    file = []

newSQL = MySQL()
newMongo = MongoDB()
newSQL.Conexion()
newMongo.mongoConexion()

sensorList = newSQL.getSensors()


class Sensors:

    def __init__(self):
        self.instancesList = DataList()
        self.createInstances()

    def createInstances(self):
        for o in sensorList:
            if self.instancesList.getData(None, o['name']) is None:
Beispiel #9
0
print("_________ SERVER get Data _________")

#String형의 이미지를 수신받아서 이미지로 변환 하고 화면에 출력
# length = recvall(conn, 16) #길이 16의 데이터를 먼저 수신하는 것은 여기에 이미지의 길이를 먼저 받아서 이미지를 받을 때 편리하려고 하는 것이다.
# stringData = recvall(conn, int(length))
# data = numpy.fromstring(stringData, dtype='uint8')
# s.close()  #................이부분 닫지 말까
# decimg = cv2.imdecode(data, 1)

# 이미지 확인하기
#cv2.imshow('SERVER', decimg)
#cv2.waitKey(0)
#cv2.destroyAllWindows()

# ...... 0. mongoDB에 저장할 객체 생성
mongo = MongoDB()  # mongoDB 객체 생성

# ...... 1. save origin image from drone
# cv2.imwrite('./container/origin.jpg', decimg)
# img = open('./container/origin.jpg', 'rb')
# mongo.storeImg_map(img, 'map_origin.jpg')
# print("Drone image save!!")

# ...... 2. 길찾기 시작 및 맵 DB에 저장
print("____________find_path_____________")
# drone이 넘겨준 이미지를 이용하여 길찾기 알고리즘
decimg = cv2.imread('./container/origin.jpg')
find_path = Find_path(decimg)
find_path.bfs()
find_path.real_path()
Beispiel #10
0
warnings.simplefilter("ignore", DeprecationWarning)


#socket 수신 버퍼를 읽어서 반환하는 함수
def recvall(sock, count):
    buf = b''
    while count:
        newbuf = sock.recv(count)
        if not newbuf: return None
        buf += newbuf
        count -= len(newbuf)
    return buf


# .......... 0-2. mongoDB 객체 생성
mongo = MongoDB()

# .......... 0-1. 서버의 IP와 port 번호
# .......... 4-2. TurtleBot 에게 MQTT 로 경로 정보 넘기기
mqtt = mqtt.Client("loadFinder")  # MQTT client 생성, 이름 ""
mqtt.connect("localhost", 1883)  # 로컬호스트에 있는 MQTT서버에 접속

# .......... 1. TCP 소켓 열고 수신 대기
TCP_IP = '192.168.0.15'
TCP_PORT = int(sys.argv[1])
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((TCP_IP, TCP_PORT))
s.listen(True)
print(
    "###############################################################################"
)
    918165914914304003: 1,
    918169282093121539: 2,
    918133272084959232: 3,
    918078494000721922: 4,
    917879954712465414: 5,
    917335939478949888: 6,
    917783890231791617: 7,
    918084964180201472: 8,
    918056753287311360: 9
}

listMetric = ["RS", "RSA", "visibility_value", "MD", "PD"]

connectMongoDB = Connect2MongoDB('localhost', 27017)
connectMongoDB.setDB('test1')
db = MongoDB(connectMongoDB)

data = dict()

print("\nStart SORT METRIC: {0}".format(datetime.datetime.now()))

for key in listMetric:
    tweets = db.getSort(MongoDB.CLEAR_TWEETS_COLLECTION, (key, -1))
    count = 0
    for t in tweets:
        tweetId = t["tweetId"]
        if tweetId in listRS and key != "RS":
            if "RS" in data:
                if tweetId in data["RS"]:
                    data["RS"][tweetId] += "\t" + key + ":" + str(
                        t[key]) + ",\tpos" + key + ":" + str(count)
Beispiel #12
0
        server_sock.close()

def LEDShow():
    while(led.stop == False):
       # db_result = db.list_all()
       # led.size = len(db_result)
        led.size = 254
        led.showVarieties()

# Get command line arguments
args = storage_parser().parse_args()

# Create clean exit signal
signal.signal(signal.SIGINT, signal_handler)

db = MongoDB.MongoDB()
led = LED.LED(0, False)

backlog = int(args.backlog)
port = int(args.port_num)
s_size = int(args.socket_size)

# Start the two threads
threadLED = threading.Thread(target=LEDShow, args=())
threadMongo = threading.Thread(target=MongoControl, args=())

threadLED.daemon = True
threadMongo.daemon = True

threadLED.start()
threadMongo.start()
Beispiel #13
0
def main():
    filepath = ""
    host = ""
    database = ""
    collection = ""
    core_count = multiprocessing.cpu_count()
    ram_available = (psutil.virtual_memory().available / (1024 * 1024 * 1024))
    calculate_chunk_size = int(ram_available / (core_count * (15 / 10)) *
                               1000000)

    performance = PerformanceOfDatabase.PerformanceOfDatabase()
    print("*************************************")
    while True:
        filepath = input("file path                           --")
        if (os.path.exists(filepath)):
            break
        else:
            print("File not exists. Please enter again file name!!!\n")

    nesne = DataImportMongoDB.DataImportMongoDB(filepath)

    print("*************************************")

    #size calculate
    manager = multiprocessing.Manager()
    size_of_csv = manager.dict()
    domains_of_csv = manager.dict()
    proc = multiprocessing.Process(target=size_calculate,
                                   args=(filepath, size_of_csv,
                                         (domains_of_csv)))
    proc.start()

    #Mongo connect condition
    while True:
        host = input("host                                --")
        database = input("database                            --")
        collection = input("collection                          --")
        mongo_connect = MongoDB.MongoDB(host, database, collection)
        if mongo_connect.connect == True:
            break
        else:
            print("Database not founded !!! \n")
            os.system("pause")
            os.system("cls")
            print("*************************************")

    print("*************************************")
    toMail = input("Enter the Gmail address             --")

    sendToMailPsql.sendMailStartMongo(toMail)

    print("*************************************")
    #header info
    while True:
        header_info = input("Header(0)/ No Header(1)             --")
        if header_info == "1" or header_info == "0":
            if header_info == "0":
                break
            else:
                print("if no header csv is not convert json")

        else:
            print("please correct value !!! \n")
            os.system("pause")
            os.system("cls")
            print("*************************************")

    file_size = os.stat(filepath).st_size / (1024 * 1024)
    proc.join()
    for alfa in size_of_csv.values():
        limit = int(alfa)
    domains = domains_of_csv.values()
    domains = domains[0]
    limit -= 1
    selected_columns = []
    selected_columns_index = []

    print("*************************************")
    while True:
        column_state = input("Required Columns(0)/ All Columns(1) --")
        if column_state == "1":
            print("selected all columns")
            break
        elif column_state == "0":
            print(domains)
            while True:
                column_count = input("column count:                       --")
                try:
                    column_count = int(column_count)
                    if column_count > len(domains):
                        print(
                            "columns count is not bigger than all columns count"
                        )
                    else:
                        break
                except:
                    print("please enter integer value \n")
            try:
                column_count = int(column_count)
                counter = 0
                while counter < column_count:
                    enter_column_name = input(
                        "enter column name            --")
                    if enter_column_name not in domains:
                        print("please right column name \n")
                    else:
                        find_index = domains.index(enter_column_name)
                        selected_columns.append(enter_column_name)
                        selected_columns_index.append(find_index)
                        counter += 1
                domains = selected_columns
                break
            except:
                print("SYSTEM ERROR")
        else:
            print("please correct value !!! \n")
            os.system("pause")
            os.system("cls")
            print("*************************************")

    print("*************************************")
    if (limit / core_count) < calculate_chunk_size:
        chunk_size = int(limit / core_count)
        chunk_size = max(chunk_size, 1)
    else:
        chunk_size = calculate_chunk_size
        chunk_size = max(chunk_size, 1)
    init_size = 0
    procs = []  #procesing list
    working_proc_count = 0
    performance.TimePerformanceStart()
    while True:
        try:
            if working_proc_count < core_count:
                if init_size + 1 > limit:
                    break
                proc = multiprocessing.Process(target=nesne.CsvtoMongoDB,
                                               args=(
                                                   host,
                                                   database,
                                                   collection,
                                                   init_size,
                                                   chunk_size,
                                                   selected_columns_index,
                                                   domains,
                                               ))
                procs.append(proc)
                proc.start()
                init_size += chunk_size
                working_proc_count += 1
                for one_proc in procs:
                    if one_proc.is_alive() == False:
                        procs.remove(one_proc)
                        working_proc_count -= 1
            if init_size + 1 > limit:
                break
        except:
            break
        for one_proc in procs:
            if one_proc.is_alive() == False:
                procs.remove(one_proc)
                working_proc_count -= 1

    while True:
        if len(procs) == 0:
            break
        for one_proc in procs:
            if one_proc.is_alive() == False:
                procs.remove(one_proc)
                working_proc_count -= 1
            if len(procs) == 0:
                break

    pipeline, time_difference = performance.TimePerformanceEnd(
        "Importing Finish. MongoDB", file_size)
    sendToMailPsql.sendMailFinishMongo(toMail, str(time_difference), limit,
                                       pipeline)
    print("Importing data rows count : ", limit)