Esempio n. 1
0
def update_last_check(db, alertType, url, webId, statusCode):
	try:
		cur   = db.alert.find({
						"_type"	: url,
						"_index"	: setting.agentName,
						"type"		: alertType,
						"status_code" 	: statusCode,
						"web_id"	: webId
			}).sort('update_at', -1)
		try:
			res = list(cur)[0]
			if (time.time() - int(res['insert_time'])) <= 60*60*24:
				db.alert.update({
						"_type"	: url,
						"_index"	: setting.agentName,
						"type"		: alertType,
						"status_code" 	: statusCode,
						"web_id"	: webId,
						"update_at"	: res['update_at']
					}, {"$set": {
						"update_at" 		: time.time()
						}}, upsert = False, multi = False)
		except:
			pass
	except Exception:
		error.catchError(traceback.format_exc())
		return
Esempio n. 2
0
def take_shot(url, name, statusCode, id):
	try:
		driver 		= setting.webdriver.PhantomJS(service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any', '--web-security=false'])
		driver.set_window_size(1280, 720)
		driver.set_page_load_timeout(150)
		try:
			#setting.threadLock.acquire()
			if isinstance(id, int):
				if not os.path.isfile("/var/www/html/web/app/alert_image/" + name + "_" +str(id)+"_"+ str(statusCode) + "_" +str(setting.date.day)+"-"+str(setting.date.month)+ "-" + str(setting.date.year) +".jpg"):
					driver.get(url)
					#setting.driver.execute_script('document.body.style.background = "white"')
					name = "/var/www/html/web/app/alert_image/" + name + "_" +str(id)+"_"+ str(statusCode) + "_" +str(setting.date.day)+"-"+str(setting.date.month)+ "-" + str(setting.date.year) +".jpg"
					driver.save_screenshot(name)
			else:
				if not os.path.isfile("/var/www/html/web/app/alert_image/" + name + "_" +str(id[0][0])+"_"+ str(statusCode) + "_" +str(setting.date.day)+"-"+str(setting.date.month)+ "-" + str(setting.date.year) +".jpg"):
					driver.get(url)
					for each in id:
						#setting.driver.execute_script('document.body.style.background = "white"')
						name = "/var/www/html/web/app/alert_image/" + name + "_" +str(each[0])+"_"+ str(statusCode) + "_" +str(setting.date.day)+"-"+str(setting.date.month)+ "-" + str(setting.date.year) +".jpg"
						driver.save_screenshot(name)
			driver.close()
			driver.quit()
			#setting.threadLock.release()
			return True
		except Exception:
			driver.close()
			driver.quit()
			#setting.threadLock.release()
			error.catchError(traceback.format_exc())
			return False
	except:
		for proc in psutil.process_iter():
			if proc.as_dict(attrs=['pid', 'name'])['name'] == 'phantomjs':
				proc.kill()
		return False
Esempio n. 3
0
def check_status(db, alertType, url, webId):
	try:
		cur   = db.alert.find({
					"_type"            : url,
					"_index"          : setting.agentName,
					"type"              : alertType,
					"web_id"         : webId
			}).sort('update_at', -1)
		try:
			res = list(cur)[0]
			if res['events'][len(res['events']) - 1]['end_at'] == u"thời điểm hiện tại":
				res['events'][len(res['events']) - 1]['end_at'] = str(datetime.now().replace(microsecond=0)).replace(' ','T')
				db.alert.update({
						"_type"     	: url,
						"_index"    	: setting.agentName,
						"type"      	: alertType,
						"web_id"    	: webId,
						"update_at"  	: res['update_at']
					}, {"$set": {
						"events" 	: res['events'], 
						"update_at"	: time.time()
						}}, upsert = False)
		except:
			pass
	except Exception:
		error.catchError(traceback.format_exc())
		return
Esempio n. 4
0
def ip_alert(oldIp, newIp, hostName, webId, db):
	setting.listHostIp[hostName] = 1
	try:
		cur   = db.alert.find({
						"_type"     		: hostName, 
						"_index"    		: setting.agentName,
						"type"      		: "ipchanged",
						"web_id"    		: webId,
						"new_ip"		: newIp
					}).sort('update_at',-1)
		try:
			res = list(cur)[0]
		except:
			alert = {
				'_type' 		: hostName,
				'_index'	: setting.agentName,
				'type'		: "ipchanged",
				'name'		: "Địa chỉ IP đã bị thay đổi",
				'is_alerted'	: 0,
				'false_positive'	: 0,
				'severity'	: 2,
				'old_ip'		: oldIp,
				'web_id'	: webId,
				'events'		: [],
				'insert_time' 	: int(time.time()),
				'new_ip'	: newIp,
				'host_name'	: hostName
			}
			temp = {}
			temp['start_at'] 	= str(datetime.now().replace(microsecond=0)).replace(' ', 'T')
			temp['end_at']		= 'thời điểm hiện tại'
			alert['events'].append(temp)
			alert['update_at']	= time.time()
			setting.db.alert.insert_one(alert)
		else:
			if res['false_positive'] == 1:
				print "DETECTED wrong ip alert"
				if newIp not in oldIp:
					oldIp = oldIp + ';' + newIp
					tmp = "UPDATE  trackWebsite SET ip = '%s' WHERE id = '%d'" % (oldIp, webId)
					setting.MySQLUpdate.append(tmp)
				return True
			else:
				setting.db.alert.update({
						"_type"     		: hostName,
						"_index"    		: setting.agentName,
						"type"      		: "ipchanged",
						"insert_time" 		: res['insert_time'],
						"web_id"    		: webId
					}, {"$set": {
						"update_at"		: time.time()
						}}, upsert = False)


		return False
			
	except Exception:
		error.catchError(traceback.format_exc())
		return
Esempio n. 5
0
def insert_event(statusCode, hostName, timecheck, cursor, timeResponse,
                 ipAddress, userId, webId):
    try:
        if statusCode < 400:
            status = "OK (%d)" % statusCode
            event = "Up"
        else:
            status = "FALSE (%d)" % statusCode
            event = "Down"

        if timeResponse > 10000:
            status = "OK (Slow Connection)"

        sql = "SELECT * FROM events WHERE  webId = %d ORDER BY time DESC" % (
            webId)
        cursor.execute(sql)
        result = cursor.fetchall()
        if len(result) == 0:
            sql = "INSERT INTO events (event, hostName, reason, time, duration, userId, webId) VALUES ('%s', '%s' , '%s', '%d', '%s', '%d', '%d')" % (
                "Started", hostName, "OK", timecheck, "0:0:0", userId, webId)
            cursor.execute(sql)

            sql = "INSERT INTO events (event, hostName, reason, time, duration, lastCheck, userId, webId) VALUES ('%s', '%s' , '%s', '%d', '%s', '%d', '%d', '%d')" % (
                event, hostName, status, timecheck + 1, "0:0:0", timecheck + 1,
                userId, webId)
            cursor.execute(sql)

            statistics(hostName, statusCode, 0, cursor, timeResponse,
                       ipAddress, userId, webId)
        else:
            triggerEvent = result[0][1]
            triggerHostName = result[0][2]
            triggerReason = result[0][3]
            triggerTime = result[0][4]
            triggerLast = result[0][6]
            statistics(hostName, statusCode, int(timecheck - triggerLast),
                       cursor, timeResponse, ipAddress, userId, webId)
            flag = 1
            if statusCode == 408:
                if time.time() - setting.alert408[webId] < 300:
                    flag = 0
                elif status != triggerReason:
                    timecheck = timecheck - 12 * 60 * 500
            if flag == 1:
                duration = calculate_duration(triggerTime, timecheck)
                sql = " UPDATE events SET duration = '%s', lastCheck = %d WHERE time = %d AND  webId = %d" % (
                    duration, timecheck, triggerTime, webId)
                cursor.execute(sql)

                if status != triggerReason:
                    sql = "INSERT INTO events (event, hostName, reason, time, duration, lastCheck, userId, webId) VALUES ('%s', '%s' , '%s', '%d', '%s', '%d', '%d', '%d')" % (
                        event, hostName, status, timecheck, "0:0:0", timecheck,
                        userId, webId)
                    cursor.execute(sql)
    except Exception:
        error.catchError(traceback.format_exc(), sql)
        return
Esempio n. 6
0
def thread_insert_data_MySQL():
    # global actions, MySQLEvent, MySQLQuery, MongoQuery
    # client     	= MongoClient()
    # db          	= client['webix']
    openData = MySQLdb.connect('localhost', 'root', 'vnistadmin', 'webix')
    MySQLCur = openData.cursor()
    cnt = 0
    while 1:
        print str(time.ctime()) + " Checking Data MySQL"

        try:
            if len(setting.MySQLUpdate) > 0 or len(setting.MySQLEvent) > 0:
                print str(time.ctime()) + " Inserting Data MySQL"
            while len(setting.MySQLUpdate) > 0:
                query = setting.MySQLUpdate[0]
                MySQLCur.execute(query)
                setting.MySQLUpdate.pop(0)
                cnt += 1
            while len(setting.MySQLEvent) > 0:
                data = setting.MySQLEvent[0]
                events.insert_event(data[0], data[1], data[2], MySQLCur,
                                    data[3], data[4], data[5], data[6])
                setting.MySQLEvent.pop(0)
                cnt += 1
            openData.commit()
            if cnt > 0:
                print str(time.ctime()) + " Inserted Data MYSQL"
                cnt = 0
            if setting.helpstat == 1:
                f = open("help.txt", "a")
                f.write("Done MySQL\n")
                f.close()
            time.sleep(60)

        except KeyboardInterrupt:
            print "Cancel Inserting Data"
            return

        except:
            error.catchError(traceback.format_exc(), query)
            print "Error while inserting MySQL Data"
            openData = MySQLdb.connect('localhost', 'root', 'vnistadmin',
                                       'webix')
            MySQLCur = openData.cursor()
            time.sleep(60)
            pass
Esempio n. 7
0
def thread_insert_data_ElasticSearch():
    times = 0
    while 1:
        print str(time.ctime()) + " Checking Data ES"

        try:
            if len(setting.ESData) > 0:
                print str(time.ctime()) + " Inserting Data ES"
            while len(setting.ESData) > 0:
                try:
                    #print setting.ESData
                    while len(setting.ESData) > 100:
                        tmp = setting.ESData[0:100]
                        helpers.bulk(setting.es, tmp)
                        for i in range(0, 100):
                            setting.ESData.pop(0)
                    helpers.bulk(setting.es, setting.ESData)

                except:
                    times += 1
                    if times > 3:
                        setting.es = Elasticsearch()
                        break
                    print traceback.format_exc()
                    print "Failed. Trying Again"
                    pass
                else:
                    del setting.ESData[:]
                    print str(time.ctime()) + " Inserted Data ElasticSearch"
                    times = 0
            if setting.helpstat == 1:
                f = open("help.txt", "a")
                f.write("Done ES\n")
                f.close()
            time.sleep(60)

        except KeyboardInterrupt:
            print "Cancel Inserting Data ElasticSearch"
            return

        except:
            error.catchError(traceback.format_exc())
            print "Error while inserting ElasticSearch"
            return
Esempio n. 8
0
def thread_insert_data_MongoDb():
    cnt = 0
    while 1:
        print str(time.ctime()) + " Checking Data MongoDb"

        try:
            if len(setting.MongoData) > 0:
                print str(time.ctime()) + " Inserting Data MongoDb"
            while len(setting.MongoData) > 0:
                data = setting.MongoData[0]
                POT = alert.insert_alert(setting.db, data[0], data[1], data[2],
                                         data[3], data[4], data[5], data[6],
                                         data[7], data[8], data[9], data[10])
                if POT == False:
                    alert.check_status(setting.db, 'defaced', data[3], data[6])
                    sql = "UPDATE webStruct SET isStructed = 0 where webId = '%d'" % data[
                        6]
                    setting.MySQLUpdate.append(sql)
                setting.MongoData.pop(0)
                cnt += 1
            if cnt > 0:
                print str(time.ctime()) + " Inserted Data MongoDb"
                cnt = 0
            if setting.helpstat == 1:
                f = open("help.txt", "a")
                f.write("Done Mongo\n")
                f.close()
            time.sleep(60)

        except KeyboardInterrupt:
            print "Cancel Inserting Data MongoDb"
            return

        except:
            error.catchError(traceback.format_exc())
            print "Error while inserting MongoDb"
            setting.client = MongoClient()
            setting.db = setting.client['webix']
            time.sleep(60)
            pass
Esempio n. 9
0
def statistics(hostName,
               statusCode,
               time,
               cursor,
               timeResponse,
               ipAddress,
               userId,
               webId,
               timeOK=0):
    try:
        if statusCode < 400:
            timeOK = time

        sql = "SELECT timeOK, timeTOTAL, averageTime, times FROM statistic WHERE webId = %d" % (
            webId)
        cursor.execute(sql)
        result = cursor.fetchall()

        if len(result) == 0:
            sql = "INSERT INTO statistic (hostName, timeOK, timeTOTAL, averageTime, ip, userId, webId, times) VALUES ('%s','%d', '%d', '%f', '%s', '%d', '%d', 1)" % (
                hostName, 0, 0, timeResponse, ipAddress, userId, webId)
            cursor.execute(sql)

        else:
            timeOK = result[0][0] + timeOK
            averageTime = float(result[0][2])
            timeTOTAL = result[0][1] + time
            times = int(result[0][3])
            if statusCode < 400:
                averageTime = (result[0][2] * int(times) +
                               timeResponse) / (int(times) + 1)
                times += 1
            sql = "UPDATE statistic SET timeOK = %d, timeTOTAL = %d, averageTime = '%f', times = %d WHERE webId = %d" % (
                timeOK, timeTOTAL, averageTime, times, webId)
            cursor.execute(sql)
    except Exception:
        error.catchError(traceback.format_exc(), sql)
        return
Esempio n. 10
0
    def check(self, url, name, ipAddress, userId, webId, prevStatus):
        sql = ""
        statusCode = 0
        state = 0
        timeAVG = 0
        warning = ""
        severity = 0
        timecheck = (time.time()) * 1000
        flag = 1
        fail = 0
        lastTime = time.time()
        try:
            # print "%d - dong 88: - %s" %(self.id, hostName)
            for loop in range(2):
                try:
                    r = requests.get(url,
                                     allow_redirects=True,
                                     headers=setting.headers,
                                     verify=False,
                                     timeout=20)

                except:
                    # print "%d - dong 94: - %s" %(self.id, hostName)
                    if fail == 0:
                        fail += 1
                    else:
                        # print "%s : Fail to check " % datetime.now(), link
                        warning = "Request Timeout"
                        timeAVG = 0.0
                        statusCode = 408
                        state = 0
                        link = None
                        severity = 2
                        for web in webId:
                            try:
                                lastTime = setting.alert408[web[0]]
                            except KeyError:
                                setting.alert408[web[0]] = time.time()
                                continue
                    # print "%d - dong 111: - %s" %(self.id, hostName)
                else:
                    # print "%d - dong 113: - %s" %(self.id, hostName)
                    for web in webId:
                        try:
                            del setting.alert408[web[0]]
                            del setting.tmpMongoData[web[0]]
                        except KeyError:
                            # for web in webId:
                            # 	print web[0]
                            # print setting.alert408
                            continue
                    # print "%d - dong 119: - %s" %(self.id, hostName)
                    timeAVG = r.elapsed.total_seconds() * 1000
                    if r.status_code == 200:
                        state = 1

                        if timeAVG > 10000:
                            warning = "Slow Connection"
                            severity = 1
                        else:
                            # print "%d - dong 128: - %s" %(self.id, hostName)
                            for web in webId:
                                alert.check_status(setting.db, "performance",
                                                   url, web[0])
                            # print "%d - dong 131: - %s" %(self.id, hostName)
                            warning = "Normal"
                            i = 0
                            # print str(time.ctime()) + " : Checking deface " + hostName
                            # print "%d - dong 135: - %s" %(self.id, hostName)
                            for user in userId:
                                deface.check_deface(r.content, url, name,
                                                    ipAddress, webId[i][0],
                                                    user[0])
                                i += 1
                            # print "%d - dong 139: - %s" %(self.id, hostName)
                    elif r.status_code > 399 and r.status_code < 500:
                        warning = "Client Error"
                        timeAVG = 0.0
                        severity = 2
                        # print str(time.ctime()) + " : Taking pic " + hostName
                        # print "%d - dong 145: - %s" %(self.id, hostName)
                        alert.take_shot(url, name, r.status_code, webId)
                        # print "%d - dong 148: - %s" %(self.id, hostName)
                        state = 0
                        flag = 0
                        # print str(time.ctime()) + " : Done " + hostName

                    elif r.status_code >= 500:
                        warning = "Internal Server Error"
                        timeAVG = 0.0
                        severity = 3
                        # print str(time.ctime()) + " : Taking pic " + hostName
                        # print "%d - dong 158: - %s" %(self.id, hostName)
                        alert.take_shot(url, name, r.status_code, webId)
                        # print "%d - dong 161: - %s" %(self.id, hostName)
                        state = 0
                        # print str(time.ctime()) + " : Done " + hostName
                        flag = 0
                    statusCode = r.status_code
                    break
            i = 0
            # print "%d - dong 168: - %s" %(self.id, hostName)
            for user in userId:
                action = {
                    "_index": "user-%d" % user[0],
                    "_type": "web-%d" % webId[i][0],
                    "_id": timecheck,
                    "_source": {
                        'name': name,
                        'ip': ipAddress,
                        'state': state,
                        'time_res': timeAVG,
                        'status_code': statusCode,
                        'warning': warning,
                        'time': timecheck,
                        'user_id': user[0],
                        'web_id': webId[i][0]
                    }
                }
                i += 1
                setting.ESData.append(action)
            # print "%d - dong 189: - %s" %(self.id, hostName)
            i = 0
            if statusCode == 408:
                if time.time() - lastTime > 300:
                    flag = 0
                    state = 0
                    print time.time() - lastTime

                else:
                    for web in webId:
                        try:
                            setting.tmpMongoData[web[0]]
                            print "Adding more Query of id: %d" % (web[0])
                        except KeyError:
                            print "Creating TMP Query for 408 Error of %s" % (
                                name)
                            setting.tmpMongoData[web[0]] = []
                            continue
                    i = 0
                    for web in webId:
                        tmpMongo = ('performance', severity, ipAddress, name,
                                    timeAVG, statusCode, web[0], url, None,
                                    None,
                                    str(datetime.now().replace(
                                        microsecond=0)).replace(' ', 'T'))
                        setting.tmpMongoData[web[0]].append(tmpMongo)
                # print "%d - dong 226: - %s" %(self.id, hostName)
            # print "%d - dong 232: - %s" %(self.id, hostName)
            for user in userId:
                tmpSQL = (statusCode, url, timecheck, timeAVG, ipAddress,
                          user[0], webId[i][0])
                setting.MySQLEvent.append(tmpSQL)
                i += 1
            if flag == 0:
                for web in webId:
                    try:
                        setting.MongoData += setting.tmpMongoData[web[0]]
                        del setting.tmpMongoData[web[0]]
                    except KeyError:
                        continue

                for web in webId:
                    tmpMongo = ('performance', severity, ipAddress, name,
                                timeAVG, statusCode, web[0], url, None, None,
                                str(datetime.now().replace(
                                    microsecond=0)).replace(' ', 'T'))
                    setting.MongoData.append(tmpMongo)
            # print "%d - dong 244: - %s" %(self.id, hostName)
            if prevStatus != state:
                sql = "UPDATE trackWebsite SET status = %d WHERE url = '%s'" % (
                    state, url)
                setting.MySQLUpdate.append(sql)
            # print "End: %d - dong 248: - %s" %(self.id, hostName)
        except Exception:
            error.catchError(traceback.format_exc())
            return
Esempio n. 11
0
def get_mysql_data():
    connect = MySQLdb.connect('localhost', 'root', 'vnistadmin', 'webix')
    cur = connect.cursor()
    threadMySQL = Thread(target=thread_insert_data_MySQL, args=())
    threadMySQL.start()
    threadMongoDb = Thread(target=thread_insert_data_MongoDb, args=())
    threadMongoDb.start()
    threadElasticSearch = Thread(target=thread_insert_data_ElasticSearch,
                                 args=())
    threadElasticSearch.start()
    threadRestart = Thread(target=force_to_restart, args=())
    threadRestart.start()
    print threading.enumerate()
    cur.execute("SELECT object, struct, webId FROM webStruct")
    res_obj = cur.fetchall()
    for each in res_obj:
        setting.listObject[each[2]] = each
    while 1:
        setting.date = datetime.now()
        setting.exitFlag = 0
        try:
            threadworker = []

            with connect:
                try:
                    cur.execute(
                        "SELECT limitTime, isStructed, webId FROM webStruct")
                    res_web = cur.fetchall()
                    for each in res_web:
                        setting.listWebData[each[2]] = each
                    cur.execute("SELECT learningTime, id FROM trackWebsite")
                    res_id = cur.fetchall()
                    for each in res_id:
                        setting.listLearnTime[each[1]] = each[0]
                    cur.execute(
                        "SELECT DISTINCT  name,url,status, ip FROM trackWebsite"
                    )
                    result_Host = cur.fetchall()
                    print "Processing " + str(len(result_Host)) + " website"
                    index = 1
                    for each in result_Host:
                        #print "Index: %d 	- Domain: %s" %( index, each[1])
                        index += 1
                        hostName = each[1]
                        temp = {}
                        temp['status'] = each[2]
                        temp['ip'] = each[3]
                        temp['url'] = hostName
                        res = urlparse(hostName)
                        # if "http://" in hostName:
                        # 	hostName 	= hostName.replace("http://","")
                        # if "https://" in hostName:
                        # 	hostName 	= hostName.replace("https://","")
                        temp['name'] = each[0]
                        temp['hostName'] = res.netloc
                        sql = "SELECT userId from trackWebsite WHERE url = '%s'" % (
                            each[1])
                        cur.execute(sql)
                        res = cur.fetchall()
                        temp['userId'] = res
                        sql = "SELECT id from trackWebsite WHERE url = '%s'" % (
                            each[1])
                        cur.execute(sql)
                        res = cur.fetchall()
                        temp['id'] = res
                        setting.queue.put(temp)
                        #print temp
                    connect.commit()
                except:
                    error.catchError(traceback.format_exc())
                    print "QUERYING DATABASE ERROR"
                    sys.exit()
            startTime = time.time()
            qsize = setting.queue.qsize()
            print "Creating 100 Thread"
            for i in range(0, 100):
                threadworker.append(worker.worker(i))
                threadworker[i].start()
            while not setting.queue.empty():
                time.sleep(3)
                print "%f s -------- %d s ------ queue size: %d -------- active thread: %d" % (
                    time.time() - startTime, 20 * qsize, setting.queue.qsize(),
                    threading.active_count())
                if time.time() - startTime > 20 * qsize:
                    while not setting.queue.empty():
                        setting.queue.get()
                    break
                pass
            setting.exitFlag = 1
            time.sleep(30)
            print "Killing alll"
            setting.startTime = time.time()
            while threading.active_count() > 7:
                for x in threadworker:
                    x.join(0.5)
                    Thread._Thread__stop(x)
            setting.startTime = 0
            print "All thread are closed!"
            print "active thread: %d" % (threading.active_count())
            print threading.enumerate()
            time.sleep(150)

        except KeyboardInterrupt:
            print "\nYou press Ctrl + C\nProgram terminated"
            setting.exitFlag = 1
            connect.close()
            for t in threadworker:
                t.join(10)
            Thread._Thread__stop(threadMySQL)
            Thread._Thread__stop(threadMongoDb)
            Thread._Thread__stop(threadElasticSearch)
            Thread._Thread__stop(threadRestart)
            sys.exit()

        except Exception:
            error.catchError(traceback.format_exc())
            connect.close()
            sys.exit()
Esempio n. 12
0
def insert_alert(db, alertType, severity, des_ip, hostName, timeResponse, statusCode, webId, url, diff, suspicious, startTime):

	try:
		image_path = ""
		cur   = db.alert.find({
					"_type"     		: url, 
					"_index"    		: setting.agentName,
					"type"      		: alertType,
					"web_id"    		: webId
				}).sort('update_at',-1)
		flag = 0
		try:
			res   = list(cur)[0]
		except:
			flag = 1
		else:
			if (res['status_code'] != statusCode) or (res['type'] == 'defaced' and check_diff(diff, res['deface_at'])):
				if res['events'][len(res["events"]) -1]['end_at'] == u'thời điểm hiện tại':
					res['events'][len(res['events']) - 1]['end_at'] = str(datetime.now().replace(microsecond=0)).replace(' ','T')
					db.alert.update({
							"_type" 		: url,
							"_index"		: setting.agentName,
							"status_code"		: res['status_code'],
							"insert_time"		: res['insert_time'],
							"severity"		: res['severity'],
							"web_id"		: webId,
						},{"$set": {
							"events" 		: res['events'], 
							"update_at"		: time.time()
							}}, upsert = False)
					time.sleep(1)
				cur   = db.alert.find({
						"_type"     		: url, 
						"_index"    		: setting.agentName,
						"type"      		: alertType,
						"status_code"		: statusCode,
						"severity"		: severity,
						"web_id"    		: webId,
						'deface_at'		: diff,
						'dangerous_word'	: suspicious
					}).sort('update_at',-1)
				try:
					res 	= list(cur)[0]
				except: 
					flag 	= 1
			if flag == 0:
				if res['false_positive'] == 1 and res['type'] == "defaced" and not check_diff(res['deface_at'], diff):
					print "Detect FALSE ALERT"
					return False
	 			elif (time.time() - int(res['insert_time'])) > 60*60*24 :
					flag = 1
				elif  res['events'][len(res['events']) - 1]['end_at'] != u"thời điểm hiện tại" and res['false_positive'] == 0:
					temp = {"start_at": startTime, "end_at": "thời điểm hiện tại"}
					res['events'].append(temp)
					db.alert.update({
							"_type"     		: url,
							"_index"    		: setting.agentName,	
							"type"      		: alertType,
							"status_code" 		: statusCode,
							"insert_time" 		: res['insert_time'],
							"severity"		: res['severity'],
							"web_id"    		: webId
						}, {"$set": {
							"events" 		: res['events'], 
							"update_at"		: time.time()
							}}, upsert = False)
				else:
					update_last_check(db, alertType, url,  webId, statusCode)
		if flag == 1:
			if statusCode != 408 and statusCode != 200:
				image_path = hostName + "_" + str(webId)+"_"+ str(statusCode) + "_" +str(setting.date.day)+"-"+str(setting.date.month)+ "-" + str(setting.date.year) +".jpg"
				flag_image = os.path.isfile("/var/www/html/web/app/alert_image/" + hostName + "_" +str(webId)+"_"+ str(statusCode) + "_" +str(setting.date.day)+"-"+str(setting.date.month)+ "-" + str(setting.date.year) +".jpg")
				if flag_image == False:
					flag_image = take_shot(url, hostName, statusCode, int(webId))
					if flag_image == False:
						image_path = ""
			cur   = db.alert.find({
					"_type"    	: url, 
					"_index"    	: setting.agentName,
					"type"      	: alertType,
					"severity"	: severity,
					"web_id"    	: webId
				}).sort('update_at',-1)
			try:
				res   = list(cur)[0]
				if res['events'][len(res['events']) - 1]['end_at'] == u'thời điểm hiện tại':
					res['events'][len(res['events']) - 1]['end_at'] = str(datetime.now().replace(microsecond=0)).replace(' ','T')
					db.alert.update({
						"_type"     		: url,
						"_index"    		: setting.agentName,
						"type"      		: alertType,
						"insert_time" 	: res['insert_time'],
						"severity"		: res['severity'],
						"web_id"    		: webId
					}, {"$set": {
						"events" 		: res['events'], 
						"update_at"		: time.time()
						}}, upsert = False)
				time.sleep(1)
			except:
				pass
			describe, referer, alertName = describe_referer(alertType, statusCode, severity)
			# if severity == 3 and alertType == 'defaced': 
			# 	alertName = "Lỗi DEFACED - Có sự thay đổi trong cấu trúc website"
			# if severity == 4 and alertType == 'defaced':
			# 	alertName = "Lỗi DEFACED - Phát hiện từ nghi vấn trong cấu trúc website"
			alert = {
					'_type'         		: url,
					'_index'        		: setting.agentName,
					'type'          		: alertType,
					'name'          		: alertName,
					'is_alerted'    		: 0,
					'false_positive'	: 0,
					'severity'     	 	: severity,
					'dst_ip'        		: des_ip,
					'time_response' 	: timeResponse,
					'status_code'   	: statusCode,
					'description'   	: describe,
					'reference'     	: referer,
					'web_id'        		: webId,
					'host_name'		: hostName,
					'image_name'	: image_path,
					'events'		: [],
					'insert_time'		: int(time.time()),
					'deface_at'		: diff,
					'dangerous_word'	: suspicious
				}
			temp 			= {}
			temp['start_at'] 	= startTime
			temp['end_at']	= 'thời điểm hiện tại'
			alert['events'].append(temp)
			alert['update_at'] 	= time.time()
			db.alert.insert_one(alert)
		return True
	except Exception:
		error.catchError(traceback.format_exc())
		return
Esempio n. 13
0
	def check(self,link,hostName,ipAddress,port, userId, webId, protocol, prevStatus):
		sql 		= ""
		statusCode  	= 0
		state       	= 0
		timeAVG     	= 0
		warning     	= ""
		severity 	= ""
		timecheck   	= (time.time())*1000
		flag 		= 1
		fail 		= 0
		lastTime 	= time.time()
		try:
			for loop in range(2):
				try:
					# print str(time.ctime()) + " : Requesting to " + hostName
					r = requests.get(link, allow_redirects = True, headers = setting.headers, verify = False, timeout = 20)
					# print str(time.ctime()) + " : Done " + hostName

				except:
					if fail == 0:
						fail += 1
					else:
						print "%s : Fail to check " % datetime.now(), link
						warning     	= "Request Timeout"
						timeAVG     	= 0.0
						statusCode  	= 408
						state 		= 0
						link 		= None
						severity 	= "medium"
						try:
							lastTime = setting.alert408[hostName]
						except KeyError:
							setting.alert408[hostName] = time.time()
				else:
					try:
						del setting.alert408[hostName]
					except KeyError:
						pass

					timeAVG     = r.elapsed.total_seconds()*1000
					if r.status_code == 200:
						state       	= 1

						if timeAVG > 10000:
							warning     	= "Long Initial Connection Time"
							severity 	= "low"
						else:
							for web in webId:
								alert.check_status(setting.db, "performance", hostName, web[0])
							warning     	= "Normal"
							i 		= 0
							# print str(time.ctime()) + " : Checking deface " + hostName
							for user in userId:
								deface.check_deface(r.content,hostName, ipAddress, webId[i][0], user[0], link)
								i += 1
							# print str(time.ctime()) + " : Done " + hostName
						i = 0
						for user in userId:
							action = {
								"_index" 		: "user-%d" % user[0],
								"_type"			: "web-%d" % webId[i][0],
								"_id"			: timecheck,
								"_source" 		:{
									'host_name' 	: hostName,
									'ip'		: ipAddress,
									'port'		: int(port),
									'state'		: state,
									'time_res'	: timeAVG,
									'status_code' 	: r.status_code,
									'warning'	: warning,
									'time'		: timecheck,
									'user_id'	: user[0],
									'web_id'	: webId[i][0]
									}
							}
							i += 1
							setting.ESData.append(action)
					elif r.status_code > 399 and r.status_code < 500:
						warning         	= "Client Error"
						timeAVG         	= 0.0
						severity 	= "medium"
						# print str(time.ctime()) + " : Taking pic " + hostName
						for eachid in webId:
							alert.take_shot(hostName, link, r.status_code, eachid[0])
						state = 0
						flag = 0
						# print str(time.ctime()) + " : Done " + hostName
						
					elif r.status_code >= 500:
						warning         	= "Internal Server Error"
						timeAVG         = 0.0
						severity 	= "high"
						# print str(time.ctime()) + " : Taking pic " + hostName
						for eachid in webId:
							alert.take_shot(hostName, link, r.status_code, eachid[0])
						state = 0
						# print str(time.ctime()) + " : Done " + hostName
						flag = 0
					statusCode  = r.status_code
					break
			i = 0
			if statusCode == 408 :
				if time.time() - lastTime > 600:
					flag = 0
					state = 0
					setting.MySQLEvent += setting.tmpMySQLEvent[hostName]
					del setting.tmpMySQLEvent[hostName]
					for user in userId:
						tmpSQL = (statusCode, hostName, timecheck, timeAVG, ipAddress, user[0], webId[i][0])
						setting.MySQLEvent.append(tmpSQL)
						i += 1

				else:
					try:
						setting.tmpMySQLEvent[hostName]
					except KeyError:
						setting.tmpMySQLEvent[hostName] = []
					for user in userId:
						tmpSQL = (statusCode, hostName, timecheck, timeAVG, ipAddress, user[0], webId[i][0])
						setting.tmpMySQLEvent[hostName].append(tmpSQL)
						i += 1
			else:
				for user in userId:
					tmpSQL = (statusCode, hostName, timecheck, timeAVG, ipAddress, user[0], webId[i][0])
					setting.MySQLEvent.append(tmpSQL)
					i += 1

			if flag == 0:
				for web in webId:
					tmpMongo = ('performance', severity, ipAddress, hostName, timeAVG, statusCode, web[0], link, None, None)
					setting.MongoData.append(tmpMongo)
			
			if prevStatus != state:
				sql = "UPDATE trackWebsite SET status = %d WHERE hostName = '%s' AND protocol = '%s' AND port = '%s'" % (state,hostName, protocol, port)
				setting.MySQLUpdate.append(sql)
		except Exception:
			error.catchError(traceback.format_exc())
			return
Esempio n. 14
0
def check_deface(newContent, url, name, ipAddress, webId, userId):
    try:
        flag = 0
        rawContent = base64.b64encode(newContent)

        newContent = newContent.strip()
        newContent = re.sub('</', '\n</', newContent)
        newContent = re.sub('>\s*<', '>\n<', newContent)
        newContent = re.sub('><', '>\n<', newContent)
        newContent = newContent.split('\n')

        newNode = Node.Node()
        newNode.import_object(newContent)
        newNode.importContent(newContent)
        try:
            result = setting.listWebData[webId]
        except:
            newNode = base64.b64encode(cPickle.dumps(newNode))
            update = "INSERT INTO webStruct (url, limitTime, isStructed, struct, webId, userId, object) VALUES ('%s' , '%d', '%d', '%s',  '%d', '%d', '%s')" % (
                url, 1, 0, rawContent, webId, userId, newNode)
            setting.listObject[webId] = (newNode, rawContent, webId)
        else:
            limitTime = result[0]
            isStructed = int(result[1])
            struct = setting.listObject[webId][1]
            learningTime = setting.listLearnTime[webId]
            if isStructed == 1:
                # confirmstruct 		= base64.b64decode(struct)
                # confirmstruct 		= confirmstruct.strip()
                # confirmstruct 		= re.sub('</', '\n</', confirmstruct)
                # confirmstruct		= re.sub('>\s*<', '>\n<', confirmstruct)
                # confirmstruct 		= re.sub('><', '>\n<', confirmstruct)
                # confirmstruct 		= confirmstruct.split('\n')
                # oldNode 		= Node.Node()
                # oldNode.import_object(confirmstruct, hostName)
                # oldNode.importContent(confirmstruct)
                oldNode = cPickle.loads(
                    base64.b64decode(setting.listObject[webId][0]))
                oldNode, tmp = find_struct(oldNode, newNode, 'C', ipAddress,
                                           name, webId, url)
                if tmp == 0:
                    alert.check_status(setting.db, 'defaced', url, webId)
                else:
                    newContent = oldNode.boderDiffHTML(newContent)
            else:
                oldNode = cPickle.loads(
                    base64.b64decode(setting.listObject[webId][0]))
                oldNode, tmp = find_struct(oldNode, newNode, 'F', ipAddress,
                                           name, webId, url)

                newContent = oldNode.render_html(newContent)
                struct = ""
                for each in newContent:
                    if each != None:
                        struct += each + '\n'
                struct = base64.b64encode(struct)
                if limitTime <= 480 * learningTime:
                    limitTime += 1

                else:
                    isStructed = 1
                    if tmp == 0:
                        alert.check_status(setting.db, 'defaced', url, webId)
                    else:
                        newContent = oldNode.boderDiffHTML(newContent)

            oldNode = base64.b64encode(cPickle.dumps(oldNode))
            update = "UPDATE webStruct SET isStructed = '%d', struct = '%s', limitTime = '%d', object = '%s' WHERE webId = '%d'" % (
                isStructed, struct, limitTime, oldNode, webId)
            setting.listObject[webId] = (oldNode, struct, webId)

        setting.MySQLUpdate.append(update)

    except RuntimeError:
        print "Hitted maximum recursion depth! Cannot save this object"
    except Exception:
        error.catchError(traceback.format_exc())
        return