def CsvtoMongoDB(self, host, database, collection, rowCount, chunkSize,
domain_index, domains):
self.domain_index = domain_index
self.domains = domains
self.rowCount = rowCount
self.chunkSize = chunkSize
self.host = host
self.database = database
self.collection = collection
self.mongoDb = MongoDB.MongoDB(self.host, self.database,
self.collection)
durum = 0
try:
if len(domain_index) != 0:
for chunk in pd.read_csv(self.filepath,
skiprows=1 + self.rowCount,
chunksize=self.chunkSize,
usecols=self.domain_index,
names=self.domains,
low_memory=False,
encoding='utf'):
try:
data_json = json.loads(chunk.to_json(orient='records'))
durum = 1
except:
print("Csv to json coverting error")
error = "Csv to json coverting error ---> user name "
writeToErrorsInLogFiles.writeErrorLogFileMongo(error)
durum = 0
break
elif len(domain_index) == 0:
for chunk in pd.read_csv(self.filepath,
skiprows=1 + self.rowCount,
chunksize=self.chunkSize,
names=self.domains,
low_memory=False,
encoding='utf'):
try:
data_json = json.loads(chunk.to_json(orient='records'))
durum = 1
except:
print("Csv to json coverting error")
error = "Csv to json coverting error ---> user name "
writeToErrorsInLogFiles.writeErrorLogFileMongo(error)
durum = 0
break
except:
print("Csv reading error")
error = "Csv reading error ---> user name "
writeToErrorsInLogFiles.writeErrorLogFileMongo(error)
durum = 0
if durum == 1:
self.mongoDb.InsertMany(data_json)
durum = 1
else:
pass
self.mongoDb.ConnectionClose()
def __init__(self):
# 필요한 객체 생성
self.otp_client = MQTT_OTP_Subscriber("otp_result")
self.yolo_checker = Yolo_checker()
self.mongo = MongoDB()
# 사람을 확인했는지 확인하는 변수
self.flag = 0
# 터틀봇 영상 가져오기
self.cap = cv2.VideoCapture(
'http://192.168.0.32:8080/stream?topic=/usb_cam/image_raw')
def __init__(self, topic):
self.limit = 0 # 최대 인증 시도 5회까지 check
self.result_msg = None # otp 인증 결과
self.topic = topic
# mongoDB 객체 생성해서 OTP 결과값 저장
self.mongo = MongoDB()
client = mqtt.Client()
client.connect("localhost", 1883, 60)
client.on_connect = self.on_connect
client.on_message = self.on_message
client.loop_start()
def __init__(self, api_key, secret_key, uri):
self.api_key = api_key
self.secret_key = secret_key
self.uri = uri
# Create websocket to add and cancel orders
self.ws = create_connection(self.uri)
# Duplicate object for the threading channel
WebSocket.Okcoin.instance = self
# Threading to run separetely the run_forever function of the websocket aimed to retrieve market data
thread = threading.Thread(target=self.run)
thread.daemon = True # Daemonize thread - it allows the main application to exit even if the thread is still running
thread.start()
# Instance of mongoDb database to log trades and candlesticks
self.mongo = MongoDB.MongoDB()
self.initObject()
def __init__(self, raceNo, eventTimeStamp):
################################################################
# Analyzer constructor initiate all the modulus of the program #
# This is the main module of the project - the Brain which #
# Main module == in charge of all the other modules #
################################################################
###########################################################################################
# If folder does not exist on user local PC, then connect to remote server and copy files #
###########################################################################################
if os.path.isdir(LOCAL_DIR_PATH + str(raceNo)) == False:
client = ClientOps.ClientOps(raceNo)
############################################################
# Start the analyzing process - execute Modules functions #
############################################################
self.mongo = MongoDB.MongoDB()
self.fileObj = FilesOps.FilesOps(raceNo, eventTimeStamp)
#self.fileObj.find_SVC_errors()
svcErrorCode = self.fileObj.find_SVC_errors()
if svcErrorCode == -1:
print "SVC File error! -- File does not exist! or SVC Error not found"
return
self.fileObj.collect_issues()
self.fileObj.sotrtedAssertArr
print "#" * 20
print "RACE VERSION:", self.fileObj.getVers()
print "#" * 20
#######################################################################
# Find in SVC file the relevant error code which related to the issue #
#######################################################################
self.mongo.find_apar(svcErrorCode, self.fileObj.raceVers)
self.raceObjects = [] # List of RACE objects
return
# Unit tests for MongoDB class
import MongoDB
import unittest
db1 = MongoDB.MongoDB()
# clear the database to remove duplicates
db1.clear_db()
test_post1 = {"Author": "Stephen King", "Name": "It"}
db1.insert(test_post1)
print(db1.find({"Author": "Stephen King"}))
db1.change_stock({"Name": "It"}, 6)
print('test find')
print(db1.find({"Name": "It", "Author": "Stephen King"}))
test_post2 = {"Author": "Herman Melville", "Name": "Moby Dick"}
res = db1.insert(test_post2)
print(res)
print(db1.find({"Name": "Moby Dick"}))
print("First list all")
temp_list = db1.list_all()
print(*temp_list, sep='\n')
def __init__(self):
self.mongo = MongoDB.MongoDB()
pass
from DHT import *
from HCR import *
from PIR import *
from DataList import DataList
from File import File
from MySQL import *
from MongoDB import *
import sys
try:
file = File.readData()
except Exception as e:
file = []
newSQL = MySQL()
newMongo = MongoDB()
newSQL.Conexion()
newMongo.mongoConexion()
sensorList = newSQL.getSensors()
class Sensors:
def __init__(self):
self.instancesList = DataList()
self.createInstances()
def createInstances(self):
for o in sensorList:
if self.instancesList.getData(None, o['name']) is None:
print("_________ SERVER get Data _________")
#String형의 이미지를 수신받아서 이미지로 변환 하고 화면에 출력
# length = recvall(conn, 16) #길이 16의 데이터를 먼저 수신하는 것은 여기에 이미지의 길이를 먼저 받아서 이미지를 받을 때 편리하려고 하는 것이다.
# stringData = recvall(conn, int(length))
# data = numpy.fromstring(stringData, dtype='uint8')
# s.close() #................이부분 닫지 말까
# decimg = cv2.imdecode(data, 1)
# 이미지 확인하기
#cv2.imshow('SERVER', decimg)
#cv2.waitKey(0)
#cv2.destroyAllWindows()
# ...... 0. mongoDB에 저장할 객체 생성
mongo = MongoDB() # mongoDB 객체 생성
# ...... 1. save origin image from drone
# cv2.imwrite('./container/origin.jpg', decimg)
# img = open('./container/origin.jpg', 'rb')
# mongo.storeImg_map(img, 'map_origin.jpg')
# print("Drone image save!!")
# ...... 2. 길찾기 시작 및 맵 DB에 저장
print("____________find_path_____________")
# drone이 넘겨준 이미지를 이용하여 길찾기 알고리즘
decimg = cv2.imread('./container/origin.jpg')
find_path = Find_path(decimg)
find_path.bfs()
find_path.real_path()
warnings.simplefilter("ignore", DeprecationWarning)
#socket 수신 버퍼를 읽어서 반환하는 함수
def recvall(sock, count):
buf = b''
while count:
newbuf = sock.recv(count)
if not newbuf: return None
buf += newbuf
count -= len(newbuf)
return buf
# .......... 0-2. mongoDB 객체 생성
mongo = MongoDB()
# .......... 0-1. 서버의 IP와 port 번호
# .......... 4-2. TurtleBot 에게 MQTT 로 경로 정보 넘기기
mqtt = mqtt.Client("loadFinder") # MQTT client 생성, 이름 ""
mqtt.connect("localhost", 1883) # 로컬호스트에 있는 MQTT서버에 접속
# .......... 1. TCP 소켓 열고 수신 대기
TCP_IP = '192.168.0.15'
TCP_PORT = int(sys.argv[1])
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((TCP_IP, TCP_PORT))
s.listen(True)
print(
"###############################################################################"
)
918165914914304003: 1,
918169282093121539: 2,
918133272084959232: 3,
918078494000721922: 4,
917879954712465414: 5,
917335939478949888: 6,
917783890231791617: 7,
918084964180201472: 8,
918056753287311360: 9
}
listMetric = ["RS", "RSA", "visibility_value", "MD", "PD"]
connectMongoDB = Connect2MongoDB('localhost', 27017)
connectMongoDB.setDB('test1')
db = MongoDB(connectMongoDB)
data = dict()
print("\nStart SORT METRIC: {0}".format(datetime.datetime.now()))
for key in listMetric:
tweets = db.getSort(MongoDB.CLEAR_TWEETS_COLLECTION, (key, -1))
count = 0
for t in tweets:
tweetId = t["tweetId"]
if tweetId in listRS and key != "RS":
if "RS" in data:
if tweetId in data["RS"]:
data["RS"][tweetId] += "\t" + key + ":" + str(
t[key]) + ",\tpos" + key + ":" + str(count)
server_sock.close()
def LEDShow():
while(led.stop == False):
# db_result = db.list_all()
# led.size = len(db_result)
led.size = 254
led.showVarieties()
# Get command line arguments
args = storage_parser().parse_args()
# Create clean exit signal
signal.signal(signal.SIGINT, signal_handler)
db = MongoDB.MongoDB()
led = LED.LED(0, False)
backlog = int(args.backlog)
port = int(args.port_num)
s_size = int(args.socket_size)
# Start the two threads
threadLED = threading.Thread(target=LEDShow, args=())
threadMongo = threading.Thread(target=MongoControl, args=())
threadLED.daemon = True
threadMongo.daemon = True
threadLED.start()
threadMongo.start()
def main():
filepath = ""
host = ""
database = ""
collection = ""
core_count = multiprocessing.cpu_count()
ram_available = (psutil.virtual_memory().available / (1024 * 1024 * 1024))
calculate_chunk_size = int(ram_available / (core_count * (15 / 10)) *
1000000)
performance = PerformanceOfDatabase.PerformanceOfDatabase()
print("*************************************")
while True:
filepath = input("file path --")
if (os.path.exists(filepath)):
break
else:
print("File not exists. Please enter again file name!!!\n")
nesne = DataImportMongoDB.DataImportMongoDB(filepath)
print("*************************************")
#size calculate
manager = multiprocessing.Manager()
size_of_csv = manager.dict()
domains_of_csv = manager.dict()
proc = multiprocessing.Process(target=size_calculate,
args=(filepath, size_of_csv,
(domains_of_csv)))
proc.start()
#Mongo connect condition
while True:
host = input("host --")
database = input("database --")
collection = input("collection --")
mongo_connect = MongoDB.MongoDB(host, database, collection)
if mongo_connect.connect == True:
break
else:
print("Database not founded !!! \n")
os.system("pause")
os.system("cls")
print("*************************************")
print("*************************************")
toMail = input("Enter the Gmail address --")
sendToMailPsql.sendMailStartMongo(toMail)
print("*************************************")
#header info
while True:
header_info = input("Header(0)/ No Header(1) --")
if header_info == "1" or header_info == "0":
if header_info == "0":
break
else:
print("if no header csv is not convert json")
else:
print("please correct value !!! \n")
os.system("pause")
os.system("cls")
print("*************************************")
file_size = os.stat(filepath).st_size / (1024 * 1024)
proc.join()
for alfa in size_of_csv.values():
limit = int(alfa)
domains = domains_of_csv.values()
domains = domains[0]
limit -= 1
selected_columns = []
selected_columns_index = []
print("*************************************")
while True:
column_state = input("Required Columns(0)/ All Columns(1) --")
if column_state == "1":
print("selected all columns")
break
elif column_state == "0":
print(domains)
while True:
column_count = input("column count: --")
try:
column_count = int(column_count)
if column_count > len(domains):
print(
"columns count is not bigger than all columns count"
)
else:
break
except:
print("please enter integer value \n")
try:
column_count = int(column_count)
counter = 0
while counter < column_count:
enter_column_name = input(
"enter column name --")
if enter_column_name not in domains:
print("please right column name \n")
else:
find_index = domains.index(enter_column_name)
selected_columns.append(enter_column_name)
selected_columns_index.append(find_index)
counter += 1
domains = selected_columns
break
except:
print("SYSTEM ERROR")
else:
print("please correct value !!! \n")
os.system("pause")
os.system("cls")
print("*************************************")
print("*************************************")
if (limit / core_count) < calculate_chunk_size:
chunk_size = int(limit / core_count)
chunk_size = max(chunk_size, 1)
else:
chunk_size = calculate_chunk_size
chunk_size = max(chunk_size, 1)
init_size = 0
procs = [] #procesing list
working_proc_count = 0
performance.TimePerformanceStart()
while True:
try:
if working_proc_count < core_count:
if init_size + 1 > limit:
break
proc = multiprocessing.Process(target=nesne.CsvtoMongoDB,
args=(
host,
database,
collection,
init_size,
chunk_size,
selected_columns_index,
domains,
))
procs.append(proc)
proc.start()
init_size += chunk_size
working_proc_count += 1
for one_proc in procs:
if one_proc.is_alive() == False:
procs.remove(one_proc)
working_proc_count -= 1
if init_size + 1 > limit:
break
except:
break
for one_proc in procs:
if one_proc.is_alive() == False:
procs.remove(one_proc)
working_proc_count -= 1
while True:
if len(procs) == 0:
break
for one_proc in procs:
if one_proc.is_alive() == False:
procs.remove(one_proc)
working_proc_count -= 1
if len(procs) == 0:
break
pipeline, time_difference = performance.TimePerformanceEnd(
"Importing Finish. MongoDB", file_size)
sendToMailPsql.sendMailFinishMongo(toMail, str(time_difference), limit,
pipeline)
print("Importing data rows count : ", limit)
