def __init__(self,transport,source_address,source_port, destination_address,destination_port,app=None,window=1000): Connection.__init__(self,transport,source_address,source_port, destination_address,destination_port,app) ### Sender functionality # send window; represents the total number of bytes that may # be outstanding at one time self.window = window # send buffer self.send_buffer = SendBuffer() # maximum segment size, in bytes self.mss = 1000 # largest sequence number that has been ACKed so far; represents # the next sequence number the client expects to receive self.sequence = 0 # retransmission timer self.timer = None # timeout duration in seconds self.timeout = 1 ### Receiver functionality # receive buffer self.receive_buffer = ReceiveBuffer() # ack number to send; represents the largest in-order sequence # number not yet received self.ack = 0
def __init__(self, transport, source_address, source_port, destination_address, destination_port, app=None, window=1000): Connection.__init__(self, transport, source_address, source_port, destination_address, destination_port, app) ### Sender functionality # send window; represents the total number of bytes that may # be outstanding at one time self.window = window # send buffer self.send_buffer = SendBuffer() # maximum segment size, in bytes self.mss = 1000 # largest sequence number that has been ACKed so far; represents # the next sequence number the client expects to receive self.sequence = 0 # retransmission timer self.timer = None # timeout duration in seconds self.timeout = 1 ### Receiver functionality # receive buffer self.receive_buffer = ReceiveBuffer() # ack number to send; represents the largest in-order sequence # number not yet received self.ack = 0
def test_connection_doesnt_keep_ignored_messages_sent_from_twitch(self): bot = "nick_BOTtom" user = "******" body = "Your shoes are made of peanut butter. Woot!" last_user_message = (user, body) message = f':{user}!{user}@{user}.tmi.twitch.tv PRIVMSG #t_tv :{body}' bot_message = f':{bot}!{bot}@{bot}.tmi.twitch.tv PRIVMSG #t_tv :{body}' connect_to = { 'irc_url':'some_twitch_url', 'irc_port': 1701, 'bot_name': 'nick_BOTtom', 'oauth_token': 'oauth:1337_P@SSw0rd123', 'channel': "t_tv" } dont_sleep = Time(dont_print).sleep mock_socket = socket.socket(dont_print, message) s = Subject(connect_to, mock_socket, dont_print, dont_sleep) s.scan() self.assertEqual(mock_socket.message, message) self.assertEqual(s.last_response, last_user_message) mock_socket.message = bot_message s.scan() self.assertEqual(mock_socket.message, bot_message) self.assertEqual(s.last_response, last_user_message) self.assertNotEqual(s.last_response, (bot, body)) self.assertNotEqual(s.last_response, ('bot', 'No Messages Recieved'))
def test_connection_doesnt_log_ignored_messages_sent_from_twitch(self): user = "******" bot = "nick_BOTtom" body = "Woot!" message = f':{user}!{user}@{user}.tmi.twitch.tv PRIVMSG #t_tv :{body}' bot_message = f':{bot}!{bot}@{bot}.tmi.twitch.tv PRIVMSG #t_tv :{body}' expected_print = "Chat Message From: happy_lass : Woot!" connect_to = { 'irc_url':'some_twitch_url', 'irc_port': 1701, 'bot_name': bot, 'oauth_token': 'oauth:1337_P@SSw0rd123', 'channel': "t_tv" } dont_sleep = Time(dont_print).sleep spy_log = Spy_Log() mock_socket = socket.socket(dont_print, message) s = Subject(connect_to, mock_socket, spy_log.log, dont_sleep) s.scan() self.assertEqual(spy_log._history[-1], expected_print) mock_socket.message = bot_message s.scan() self.assertEqual(spy_log._history[-1], expected_print) self.assertTrue(bot not in spy_log._history[-1])
def __init__(self, port, baud_rate, terminal=None, reset=False): Connection.__init__(self, terminal) self._port = port self._baud_rate = baud_rate try: # These timeouts should be large enough so that any continuous transmission is fully received self._serial = serial.Serial(None, self._baud_rate, timeout=0.2, write_timeout=0.2) self._serial.dtr = False self._serial.rts = False self._serial.port = port self._serial.open() if reset: self._serial.rts = True time.sleep(0.1) self._serial.rts = False x = "" while not x.endswith(">>>"): x += self._serial.read().decode('utf-8', errors="ignore") self.send_kill() except (OSError, serial.SerialException) as e: self._serial = None return except Exception as e: return self._reader_thread = Thread(target=self._reader_thread_routine) self._reader_thread.start()
def __init__(self, transport, source_address, source_port, destination_address, destination_port, app=None, drop=[], fast_retransmit=False, measure=False): Connection.__init__(self, transport, source_address, source_port, destination_address, destination_port, app) # -- Sender functionality # send buffer self.send_buffer = SendBuffer() # maximum segment size, in bytes self.mss = 1000 # initial cwnd is equal to 1mss self.cwnd = float(self.mss) # boolean flag that indicates if in slow start or additive increase self.slow_start = True # When cwnd >= ss_thresh, slow start ends self.ss_thresh = 100000 if self.node.hostname == 'n1': self.trace("Entering Slow Start: SS Thresh = %s" % (self.ss_thresh)) # Tracks additive increase - determines when to add another MSS to cwnd self.cwnd_inc = 0 # largest sequence number that has been ACKed so far; represents # the next sequence number the client expects to receive self.sequence = 0 # plot sequence numbers self.plot_sequence_header() self.plot_cwnd_header() self.plot_cwnd() # packets to drop self.drop = drop self.dropped = [] # retransmission timer self.timer = None # timeout duration in seconds self.timeout = 1 # -- Receiver functionality # receive buffer self.receive_buffer = ReceiveBuffer() # ack number to send; represents the largest in-order sequence # number not yet received self.ack = 0 self.fast_retransmit = fast_retransmit self.last_ack = 0 self.last_ack_count = 0
def __init__( self, transport, source_address, source_port, destination_address, destination_port, app=None, window=1000, dynamic_rto=True, type="Tahoe", ): Connection.__init__(self, transport, source_address, source_port, destination_address, destination_port, app) ### Sender functionality # send window; represents the total number of bytes that may # be outstanding at one time self.window = default_mss # send buffer self.send_buffer = SendBuffer() # maximum segment size, in bytes self.mss = default_mss # largest sequence number that has been ACKed so far; represents # the next sequence number the client expects to receive self.sequence = 0 # retransmission timer self.timer = None # timeout duration in seconds self.timeout = 1 self.threshold = 100000 self.ack_received_count = {} self.wait_for_timeout = False self.reno_fast_recovery = type == "Reno" # constants self.k = 4 self.g = 0.1 self.alpha = 1 / 8 self.beta = 1 / 4 self.dynamic_rto = dynamic_rto self.rto = 3 self.srtt = None self.rttvar = None ### Receiver functionality # receive buffer self.receive_buffer = ReceiveBuffer() # ack number to send; represents the largest in-order sequence # number not yet received self.ack = 0
def __init__(self, transport, source_address, source_port, destination_address, destination_port, app=None, window=1000, drop=[]): Connection.__init__(self, transport, source_address, source_port, destination_address, destination_port, app) # -- Sender functionality # send window; represents the total number of bytes that may # be outstanding at one time self.window = window # send buffer self.send_buffer = SendBuffer() # maximum segment size, in bytes self.mss = 1000 # largest sequence number that has been ACKed so far; represents # the next sequence number the client expects to receive self.sequence = 0 # plot sequence numbers self.plot_sequence_header() # packets to drop self.drop = drop self.dropped = [] # retransmission timer self.timer = None # timeout duration in seconds self.timeout = 1 # RTO calculation variables self.rto = 1 self.srtt = 0 self.rttvar = 0 # Variables for handling fast retransmit self.fast_enable = False self.last_ack = 0 self.same_ack_count = 0 self.fast_retransmitted = False # -- Receiver functionality # receive buffer self.receive_buffer = ReceiveBuffer() # ack number to send; represents the largest in-order sequence # number not yet received self.ack = 0
def __init__(self, host, port, terminal, password_prompt): Connection.__init__(self, terminal) self._host = host self._port = port self.s = None self.ws = None if not self._start_connection(): return if not self.handle_password(password_prompt): self._clear() raise PasswordException() self._reader_thread = Thread(target=self._reader_thread_routine) self._reader_thread.start()
def test_connection_doesnt_report_pings(self): message = 'PING :tmi.twitch.tv\r\n' connect_to = { 'irc_url':'some_twitch_url', 'irc_port': 1701, 'bot_name': 'nick_BOTtom', 'oauth_token': 'oauth:1337_P@SSw0rd123', 'channel': "t_tv" } dont_sleep = Time(dont_print).sleep mock_socket = socket.socket(dont_print, message) s = Subject(connect_to, mock_socket, dont_print, dont_sleep) s.scan() self.assertEqual(s.last_response, ('bot', 'No Messages Recieved'))
def __init__(self,transport,source_address,source_port, destination_address,destination_port,window_size,app=None): Connection.__init__(self,transport,source_address,source_port, destination_address,destination_port,app) self.send_buffer = '' self.next_sequence_num = 0 self.received_ack_number = 0 self.unacked_packet_count = 0 self.mss = 1000 self.sequence = 0 self.received_sequences = set([]) self.receive_buffer = [] self.ack = 0 self.timer = None self.timeout = 1 self.max_sequence = math.pow(2,64) self.window_size = window_size
def __init__(self, transport, source_address, source_port, destination_address, destination_port, app=None, window=1000): Connection.__init__(self, transport, source_address, source_port, destination_address, destination_port, app) ### Sender functionality # send window; represents the total number of bytes that may # be outstanding at one time # Step 2 self.window = window # send buffer self.send_buffer = SendBuffer() # maximum segment size, in bytes # self.mss = 1000 self.mss = min(1000, window) # largest sequence number that has been ACKed so far; represents # the next sequence number the client expects to receive self.sequence = 0 # retransmission timer self.timer = None # timeout duration in seconds self.timeout = 3.0 # estimated rtt self.est_rtt = None # alpha self.alpha = 0.125 # variation rtt self.var_rtt = None ## TODO: revisit this later # beta self.beta = 0.25 # timer start self.start_time = 0.0 ### Receiver functionality # receive buffer self.receive_buffer = ReceiveBuffer() # ack number to send; represents the largest in-order sequence # number not yet received self.ack = 0
def register_link(self, reader, writer): connection = Connection(reader, writer, notify_queue=asyncio.Queue()) self._connections.append(connection) try: yield connection finally: self._connections.remove(connection)
def test_connection_doesnt_recieve_messages_too_fast(self): connect_to = { 'irc_url':'some_twitch_url', 'irc_port': 1701, 'bot_name': 'nick_BOTtom', 'oauth_token': 'oauth:1337_P@SSw0rd123', 'channel': "home_shopping_network" } spy_log = Spy_Log() spy_on_sleep = Time(spy_log.log).sleep mock_socket = socket.socket(dont_print) s = Subject(connect_to, mock_socket, dont_print, spy_on_sleep) scan_rate = s.seconds_per_message s.scan() self.assertEqual(scan_rate, 1 / 120 ) self.assertEqual(spy_log._history[-1], f'slept for {scan_rate} second(s)')
def test_connection_logs_its_pongs(self): ping = 'PING :tmi.twitch.tv\r\n' pong_log = Log.connect_pong connect_to = { 'irc_url':'some_twitch_url', 'irc_port': 1701, 'bot_name': 'nick_BOTtom', 'oauth_token': 'oauth:1337_P@SSw0rd123', 'channel': "t_tv" } dont_sleep = Time(dont_print).sleep spy_log = Spy_Log() mock_socket = socket.socket(dont_print, ping) s = Subject(connect_to, mock_socket, spy_log.log, dont_sleep) s.scan() self.assertEqual(spy_log._history[-1], pong_log)
def test_connection_ignores_messages_from_itself_sent_from_twitch(self): user = "******" body = "Your shoes are made of peanut butter. Woot!" message = f':{user}!{user}@{user}.tmi.twitch.tv PRIVMSG #t_tv :{body}' connect_to = { 'irc_url':'some_twitch_url', 'irc_port': 1701, 'bot_name': user, 'oauth_token': 'oauth:1337_P@SSw0rd123', 'channel': "t_tv" } dont_sleep = Time(dont_print).sleep mock_socket = socket.socket(dont_print, message) s = Subject(connect_to, mock_socket, dont_print, dont_sleep) s.scan() self.assertEqual(s.last_response, ('bot', 'No Messages Recieved'))
def test_connection_returns_poorly_formatted_messages_from_twitch(self): first_word = "Badabing" whole_bad_message = f"{first_word} BAD FORMAT: U shoes r peanut butter." connect_to = { 'irc_url':'some_twitch_url', 'irc_port': 1701, 'bot_name': 'nick_BOTtom', 'oauth_token': 'oauth:1337_P@SSw0rd123', 'channel': "home_shopping_network" } dont_sleep = Time(dont_print).sleep mock_socket = socket.socket(dont_print, whole_bad_message) s = Subject(connect_to, mock_socket, dont_print, dont_sleep) s.scan() self.assertEqual(s.last_response[0], first_word) self.assertEqual(s.last_response[1], whole_bad_message)
def __init__(self,transport,source_address,source_port, destination_address,destination_port,app=None): Connection.__init__(self,transport,source_address,source_port, destination_address,destination_port,app) self.send_buffer = '' self.receive_buffer = [] self.packets_outstanding = 0 self.mss = 1000 self.sequence = 0 self.ack = 0 self.timer = None self.timeout = 1 self.max_sequence = math.pow(2,64) self.window_size = 1 self.window_start = 0 self.timer_set = False self.queueing_delay = 0 self.pkt_q_delay_threshold = 0.0000000001 self.pkts_rcvd = 0
def main(user_input): worker = Option_Worker(user_input) config_file, socket = worker.setWithOptions() config = Bot_Configuration(config_file) connect_to = config.get_connection_constants() twitch_connection = Connection(connect_to, socket) route_commander = Commander(All().commands(), config.get_admins(), twitch_connection) app = Trivvy(twitch_connection, route_commander) app.run()
def test_connection_logs_messages_recieved(self): user = "******" body = "Woot!" message = f':{user}!{user}@{user}.tmi.twitch.tv PRIVMSG #t_tv :{body}' expected_print = Log.connect_response(user, body) connect_to = { 'irc_url':'some_twitch_url', 'irc_port': 1701, 'bot_name': 'nick_BOTtom', 'oauth_token': 'oauth:1337_P@SSw0rd123', 'channel': "t_tv" } dont_sleep = Time(dont_print).sleep spy_log = Spy_Log() mock_socket = socket.socket(dont_print, message) s = Subject(connect_to, mock_socket, spy_log.log, dont_sleep) s.scan() self.assertEqual(spy_log._history[-1], expected_print)
def test_connection_sends_twitch_arbitrary_messages(self): my_id = b':nick_BOTtom!nick_BOTtom@nick_BOTtom.tmi.twitch.tv ' address = b'PRIVMSG #home_shopping_network :' body = "My cat's breath smells like cat food." crlf = b'\r\n' expected_message = my_id + address + body.encode('utf-8') + crlf connect_to = { 'irc_url':'some_twitch_url', 'irc_port': 1701, 'bot_name': 'nick_BOTtom', 'oauth_token': 'oauth:1337_P@SSw0rd123', 'channel': "home_shopping_network" } spy_log = Spy_Log() dont_sleep = Time(dont_print).sleep s = Subject(connect_to, socket.socket(spy_log.log), dont_print, dont_sleep) s.send(body) self.assertEqual(spy_log._history[-1], expected_message)
def __init__(self,transport,source_address,source_port, destination_address,destination_port,app=None): Connection.__init__(self,transport,source_address,source_port, destination_address,destination_port,app) self.send_buffer = '' self.received_ack_number = 0 self.unacked_packet_count = 0 self.mss = 1000 self.sequence = 0 self.received_sequences = set([]) self.receive_buffer = [] self.ack = 0 self.cwnd = 0 self.threshold = 100000 self.timer = None self.timeout = 0.1 self.max_sequence = math.pow(2,64) self.rate_file = open ('output/rate%d.txt' % source_port, 'w+') self.sequence_file = open('output/sequence%d.txt' % source_port, 'w+') self.window_file = open('output/window%d.txt' % source_port, 'w+') self.ack_file = open('output/acks%d.txt' % source_port, 'w+')
def __init__(self,transport,source_address,source_port, destination_address,destination_port,window,app=None): Connection.__init__(self,transport,source_address,source_port, destination_address,destination_port,app) #sender self.send_buffer = '' self.next_sequence_num = 0 self.send_base = self.next_sequence_num self.not_yet_acknowledged_segments = 0 #receiver self.receive_temp_buffer = set([]) self.receive_packet_buffer = [] self.ack = 0 self.timer_counter = 0 self.triple_dup_ack_counter = 0 self.mss = 1000 self.window = window self.timer = None self.timeout = 1 # 1 second self.max_sequence = math.pow(2,64)
def __init__(self, low=0.5, high=0.5): # classification thresholds self._low = float(low) self._high = float(high) # add and setup logger self._logger = logging.getLogger() logging.basicConfig(level=logging.DEBUG) # db connection self.db = Connection() # load info about allready classified entries self._logger.info('Loading Allready classified entries...') self.human = HumanClassification( '/mnt/minerva1/nlp/projects/twitter_classification/TwitterClassifier/pickles/HumanClassification' ) self.human.load() # load database of words self._logger.info('Loading word dictionary...') self.word_dict = WordDictionary( '/mnt/minerva1/nlp/projects/twitter_classification/TwitterClassifier/pickles/WordDictionary' ) self.word_dict.load() # timer self._timer = timeit.Timer()
def __init__(self, host, port, terminal, ask_for_password): Connection.__init__(self, terminal) self.s = socket.socket() self.s.settimeout(3) errno = self.s.connect_ex((host, port)) if errno != 0: self._clear() return self.s.settimeout(None) websocket_helper.client_handshake(self.s) self.ws = WebSocket(self.s) self.login(ask_for_password()) try: self.read_all() except: self._clear() return self._reader_thread = Thread(target=self._reader_thread_routine) self._reader_thread.start()
def __init__(self, low=0.5, high=0.5): # classification thresholds self._low = float(low) self._high = float(high) # add and setup logger self._logger = logging.getLogger() logging.basicConfig(level=logging.DEBUG) # db connection self.db = Connection() # load info about allready classified entries self._logger.info('Loading Allready classified entries...') self.human = HumanClassification('/mnt/minerva1/nlp/projects/twitter_classification/TwitterClassifier/pickles/HumanClassification') self.human.load() # load database of words self._logger.info('Loading word dictionary...') self.word_dict = WordDictionary('/mnt/minerva1/nlp/projects/twitter_classification/TwitterClassifier/pickles/WordDictionary') self.word_dict.load() # timer self._timer = timeit.Timer()
class Views(object): engine=Connection().create_connection() logger=Log().logger() fomatter=Fomatter() Session=sessionmaker(bind=engine) session=Session() def __init__(self): self.auth_dict={} @classmethod def syncdb(self,argvs): """ 创建表结构 :param argvs: :return: """ from src.models import BASE BASE.metadata.create_all(bind=self.engine) def auth(self): """ 用户认证 :return: """ username=self.fomatter.input_str_format("请输入用户名") password=self.fomatter.input_str_format("请输入密码") user_obj=self.session.query(models.User).filter(models.User.username==username,models.User.password==password).first() if user_obj is None: self.auth_dict["username"]=username self.auth_dict["is_auth"]=False self.logger.debug("用户认证失败") else: self.auth_dict["username"]=username self.auth_dict["is_auth"]=True self.logger.debug("用户认证成功") return self.auth_dict def host_add(self,*args): """ 增加主机 :param args: :return: """ name=self.fomatter.input_str_format("请输入主机名") host=self.fomatter.input_str_format("请输入主机地址") port=self.fomatter.input_str_format("请输入主机端口") host_obj=models.Hosts(name=name,host=host,port=port) try: self.session_add(host_obj) except Exception as e: print(e) def host_list(self,*args): """ 查看所有主机 :param args: :return: """ host_objs=self.session.query(models.Hosts).all() if len(host_objs)==0: self.logger.info("主机表为空") return None else: self.fomatter.show_color("序号\t主机名\t主机地址\t端口") for host in host_objs: self.fomatter.show_color("%s\t%s\t%s\t%s" %(host.id,host.name,host.host,host.port)) return True def group_add(self,*args): """ 增加主机组 :param args: :return: """ name=self.fomatter.input_str_format("请输入主机名") group_obj=models.Groups(name=name) try: self.session_add(group_obj) except Exception as e: print(e) def group_list(self,*args): """ 查看所有主机 :param args: :return: """ group_objs=self.session.query(models.Groups).all() if len(group_objs)==0: self.logger.info("主机表组为空") return None else: self.fomatter.show_color("序号\t主机组名") for group_obj in group_objs: self.fomatter.show_color("%s\t%s" %(group_obj.id,group_obj.name)) return True def remoteuser_add(self,*args): """ 增加主机用户 :param args: :return: """ name=self.fomatter.input_str_format("请输入主机用户名") password=self.fomatter.show_color(input("请输入密码>>")) while True: type=self.fomatter.input_str_format("请选择类型1:密码;2秘钥") if type not in ("1","2"): continue else:break if type=="1": remoteuser_obj=models.RemoteUser(username=name,password=password,type="auth-password") elif type=="2": remoteuser_obj=models.RemoteUser(username=name,password=password,type="auth-key") try: self.session_add(remoteuser_obj) except Exception as e: print(e) def remoteuser_list(self,*args): """ 查看所有主机用户 :param args: :return: """ remoteuser_objs=self.session.query(models.RemoteUser).all() if len(remoteuser_objs)==0: self.logger.info("主机用户表为空") return None else: self.fomatter.show_color("序号\t用户名\t密码\t类型") for remoteuser_obj in remoteuser_objs: self.fomatter.show_color("%s\t%s\t%s\t%s" %(remoteuser_obj.id,remoteuser_obj.username,remoteuser_obj.password,remoteuser_obj.type.value)) return True def user_add(self,*args): """ 增加系统用户 :param args: :return: """ name=self.fomatter.input_str_format("请输入系统用户名") password=self.fomatter.input_str_format("请输入系统用户密码") user_obj=models.User(username=name,password=password) try: self.session_add(user_obj) except Exception as e: print(e) def user_list(self,*args): """ 查看所有主机 :param args: :return: """ user_objs=self.session.query(models.User).all() if len(user_objs)==0: self.logger.info("系统用户表为空") return None else: self.fomatter.show_color("序号\t主机组名") for user_obj in user_objs: self.fomatter.show_color("%s\t%s" %(user_obj.id,user_obj.username)) return True def hostuser_add(self,*args): """ 关联主机和主机用户 :param args: :return: """ if self.host_list() is None: self.logger.info("主机为空,请先添加主机") return host_id=self.fomatter.input_str_format("请输入主机序号") host_obj=self.session.query(models.Hosts).filter_by(id=host_id).first() if host_obj is None: self.logger.error("输入主机序号有误,请重新输入") return if self.remoteuser_list() is None: self.logger.info("主机用户为空,请先增加主机用户") return remoteuser_id=self.fomatter.input_str_format("请输入主机用户号") remoteuser_obj=self.session.query(models.RemoteUser).filter_by(id=remoteuser_id).first() if remoteuser_obj is None: self.logger.error("输入主机用户序号有误,请重新输入") return host_user_obj=models.HostUser(host_id=host_id,remoteuser_id=remoteuser_id) try: self.session_add(host_user_obj) except Exception as e: self.session.rollback() self.logger.info("记录已存在") def hostuser_list(self,*args): """ 查看所有主机和主机用户关联 :param args: :return: """ host_user_objs=self.session.query(models.HostUser).all() if len(host_user_objs)==0: self.logger.info("没有主机和主机用户关系") return None else: self.fomatter.show_color("序号\t主机名\t主机用户\t主机类型") for host_user_obj in host_user_objs: self.fomatter.show_color("%s\t%s\t%s\t%s" %(host_user_obj.id, host_user_obj.host.name,host_user_obj.remoteuser.username,host_user_obj.remoteuser.type.value)) return True def user_hostuser_add(self,*args): """ 增加系统用户和主机及用户关系统 :param args: :return: """ if self.user_list() is None: self.logger.info("用户表为空,请先添加") return user_id=self.fomatter.input_str_format("请输入用户序号") user_obj=self.session.query(models.User).filter_by(id=user_id).first() if user_obj is None: self.logger.error("输入用户序号有误,请重新输入") return if self.hostuser_list() is None: self.logger.info("请先增加主机和主机用户关系") return hostuser_id=self.fomatter.input_str_format("请输入主机与用户关系序号") hostuser_obj=self.session.query(models.HostUser).filter_by(id=hostuser_id).first() if hostuser_obj is None: self.logger.error("输入主机和用户关系序号有误,请重新输入") return if hostuser_obj not in user_obj.hostuser: user_obj.hostuser.append(hostuser_obj) try: self.session_add(user_obj) except Exception as e: self.session.rollback() self.logger.info("记录已存在") def user_hostuser_list(self,*args): """ 查看系统用户和主机及用户关系统 :param args: :return: """ user_objs=self.session.query(models.User).all() if user_objs is None: self.logger.error("输入主机和用户关系序号有误,请重新输入") self.fomatter.show_color("序号\t用户名\t主机名\t主机用户名\t主机用户类型") for user_obj in user_objs: for hostuser_obj in user_obj.hostuser: self.fomatter.show_color("%s\t%s\t%s\t%s\t%s" %(user_obj.id, user_obj.username,hostuser_obj.host.name,hostuser_obj.remoteuser.username,hostuser_obj.remoteuser.type.value)) def group_hostuser_add(self,*args): """ 增加主机组和主机及用户关系统 :param args: :return: """ if self.group_list() is None: self.logger.info("用户组表为空,请先添加") return group_id=self.fomatter.input_str_format("请输入用户序号") group_obj=self.session.query(models.Groups).filter_by(id=group_id).first() if group_obj is None: self.logger.error("输入用户组序号有误,请重新输入") return if self.hostuser_list() is None: self.logger.info("请先增加主机和主机用户关系") return hostuser_id=self.fomatter.input_str_format("请输入主机与用户关系序号") hostuser_obj=self.session.query(models.HostUser).filter_by(id=hostuser_id).first() if hostuser_obj is None: self.logger.error("输入主机和用户关系序号有误,请重新输入") return if hostuser_obj not in group_obj.hostuser: group_obj.hostuser.append(hostuser_obj) try: self.session_add(group_obj) except Exception as e: self.session.rollback() self.logger.info("记录已存在") def group_hostuser_list(self,*args): """ 查看主机组和主机及用户关系统 :param args: :return: """ group_objs=self.session.query(models.Groups).all() if group_objs is None: self.logger.error("输入主机和用户关系序号有误,请重新输入") self.fomatter.show_color("序号\t主机组名\t主机名\t主机用户名") for group_obj in group_objs: for hostuser_obj in group_obj.hostuser: self.fomatter.show_color("%s\t%s\t%s\t%s" %(group_obj.id, group_obj.name,hostuser_obj.host.name,hostuser_obj.remoteuser.username)) def user_group_add(self,*args): """ 增加系统用户和主机组关系统 :param args: :return: """ if self.user_list() is None: self.logger.info("用户表为空,请先添加") return user_id=self.fomatter.input_str_format("请输入用户序号") user_obj=self.session.query(models.User).filter_by(id=user_id).first() if user_obj is None: self.logger.error("输入用户序号有误,请重新输入") return if self.group_list() is None: self.logger.info("请先增加主机组") return group_id=self.fomatter.input_str_format("请输入主机组序号") group_obj=self.session.query(models.Groups).filter_by(id=group_id).first() if group_obj is None: self.logger.error("输入主机和用户关系序号有误,请重新输入") return if group_obj not in user_obj.groups: user_obj.groups.append(group_obj) try: self.session_add(user_obj) except Exception as e: self.logger.info("系统用户与主机组记录已存在") self.session.rollback() def user_group_list(self,*args): """ 查看系统用户和主机组关系统 :param args: :return: """ user_objs=self.session.query(models.User).all() if user_objs is None: self.logger.error("输入主机和用户关系序号有误,请重新输入") self.fomatter.show_color("序号\t用户\t用户组") for user_obj in user_objs: for group_obj in user_obj.groups: self.fomatter.show_color("%s\t%s\t%s" %(user_obj.id,user_obj.username,group_obj.name)) def session_add(self,data): """ 数据提交 :param data: :return: """ if type(data)=="list": self.session.add_all(data) else: self.session.add(data) self.session.commit() @classmethod def start_session(self): """ 用户会话 :return: """ print('欢迎使用保垒机 ') username=self.fomatter.input_str_format("请输入用户名") password=self.fomatter.input_str_format("请输入密码") user_obj=self.session.query(models.User).filter(models.User.username==username,models.User.password==password).first() if user_obj: self.fomatter.show_color(user_obj) # print(user_obj.hostuser) # print(user_obj.groups) exit_flag = False while not exit_flag: if user_obj.hostuser: print('\033[32;1mz.\tungroupped hosts (%s)\033[0m' %len(user_obj.hostuser) ) for index,group in enumerate(user_obj.groups): print('\033[32;1m%s.\t%s (%s)\033[0m' %(index,group.name, len(group.hostuser)) ) choice = input("[%s]:" % user_obj.username).strip() if len(choice) == 0:continue if choice == 'z': print("------ 主机组: 分未配的主机 ------" ) for index,hostuser in enumerate(user_obj.hostuser): print(" %s.\t%s@%s(%s)"%(index, hostuser.remoteuser.username, hostuser.host.name, hostuser.host.host, )) print("----------- END -----------" ) elif choice.isdigit(): choice = int(choice) if choice < len(user_obj.groups): print("------ 主机组: %s ------" % user_obj.groups[choice].name ) for index,bind_host in enumerate(user_obj.groups[choice].hostuser): print(" %s.\t%s@%s(%s)"%(index, bind_host.remoteuser.username, bind_host.host.name, bind_host.host.host, )) print("----------- END -----------" ) #host selection while not exit_flag: user_option = input("[(b)返回, (q)退出, 选择主机登陆]:").strip() if len(user_option)==0:continue if user_option == 'b':break if user_option == 'q': exit_flag=True if user_option.isdigit(): user_option = int(user_option) if user_option < len(user_obj.groups[choice].hostuser) : # print('host:',user_obj.groups[choice].hostuser[user_option]) # print('audit log:',user_obj.groups[choice].hostuser[user_option].audit_logs) ssh_login(user_obj, user_obj.groups[choice].hostuser[user_option], self.session, self.log_recording) else: print("no this option..") @classmethod def log_recording(self,user_obj,bind_host_obj,logs): ''' 日志记录 ''' # print("\033[41;1m--logs:\033[0m",logs) self.session.add_all(logs) self.session.commit()
class Function(object): logger = Log().logger() conn = Connection().create_connection() engine = sqlalchemy.create_engine(conn, encoding="utf-8", echo=False) auth_dict = {} def __init__(self): session_class = sessionmaker(bind=self.engine) self.session = session_class() self.format = Fomatter() def auth(self): """ 用户认证 :return: """ username = self.format.input_str_format("请输入用户名") password = self.format.input_str_format("请输入密码") user_obj = self.session.query(User).filter( User.name == username, User.password == password).first() if user_obj is None: self.auth_dict["username"] = username self.auth_dict["is_auth"] = False self.auth_dict["role"] = None self.auth_dict["user_id"] = None self.logger.debug("用户认证失败") else: self.auth_dict["username"] = username self.auth_dict["is_auth"] = True self.auth_dict["role"] = user_obj.role.name self.auth_dict["user_id"] = user_obj.id self.logger.debug("用户认证成功") return self.auth_dict def role_add(self, *args): """ 增加角色 :param args: :return: """ name = self.format.input_str_format("请输入角色名") role_obj = Role(name=name) try: self.session_add(role_obj) self.logger.debug("角色[%s]添加成功" % name) except: self.session.rollback() self.logger.error("角色[%s]已存在" % name) def role_update(self, *args): """ 角色更新 :param args: :return: """ name_old = self.format.input_str_format("请输入需要修改的角色名") role_obj = self.session.query(Role).filter_by(name=name_old).first() if role_obj is None: self.logger.info("角色名[%s]不存在!" % name_old) else: name_new = self.format.input_str_format("请输入需要修改新的角色名") role_obj.name = name_new self.session.commit() def role_list(self, *args): """ 所有角色列表 :param args: :return: """ role_objs = self.session.query(Role).all() if len(role_objs) == 0: self.logger.info("角色表为空") return None else: self.format.show_color("序号\t角色名") for role_obj in role_objs: self.format.show_color("%s\t%s" % (role_obj.id, role_obj.name)) return True def role_query(self, *args): """ 角色更新 :param args: :return: """ name = self.format.input_str_format("请输入查询角色名") role_objs = self.session.query(Role).filter( Role.name.like("%" + "%s" % name + "%")).all() if len(role_objs) == 0: self.logger.info("角色表为空") else: self.format.show_color("序号\t角色名") for role_obj in role_objs: self.format.show_color("%s\t%s" % (role_obj.id, role_obj.name)) def classes_add(self, *args): """ 增加班级 :param args: :return: """ name = self.format.input_str_format("请输入班级名") role_obj = Classes(name=name) try: self.session_add(role_obj) self.logger.debug("班级[%s]添加成功" % name) except: self.session.rollback() self.logger.error("班级[%s]已存在" % name) def classes_list(self, *args): """ 班级列表 :param args: :return: """ classes_objs = self.session.query(Classes).all() if len(classes_objs) == 0: self.logger.info("班级表为空") return None else: self.format.show_color("序号\t班级名") for classes_obj in classes_objs: self.format.show_color("%s\t%s" % (classes_obj.id, classes_obj.name)) return True def user_add(self, *args): """ 增加用户 :param args: :return: """ name = self.format.input_str_format("请输入用户名") password = self.format.input_str_format("请输入密码") qq = self.format.input_str_format("请输入QQ号") if self.role_list(): role_id = self.format.input_str_format("请输入角色ID号") user_obj = User(name=name, password=password, qq=qq, role_id=role_id) try: self.session_add(user_obj) self.logger.debug("用户[%s]添加成功" % name) except: self.session.rollback() self.logger.error("用户名或者QQ已存在") def user_add_teacher(self, *args): """ 增加学员 :param args: :return: """ name = self.format.input_str_format("请输入用户名") password = self.format.input_str_format("请输入密码") qq = self.format.input_str_format("请输入QQ号") role_obj = self.session.query(Role).filter_by(name="student").first() user_obj = User(name=name, password=password, qq=qq, role_id=role_obj.id) try: self.session_add(user_obj) self.logger.debug("学员[%s]添加成功" % name) except: self.session.rollback() self.logger.error("学员名或者QQ已存在") def user_list(self, *args): """ 查看所有用户 :param args: :return: """ user_objs = self.session.query(User).all() if len(user_objs) == 0: self.logger.info("用户表为空") return None else: self.format.show_color("序号\t姓名\tqq号\t角色") for user_obj in user_objs: self.format.show_color("%s\t%s\t%s\t\t%s" % (user_obj.id, user_obj.name, user_obj.qq, user_obj.role.name)) return True def user_student_list(self, *args): """ 查看所有学员 :param args: :return: """ user_objs = self.session.query(User).filter( User.role_id.in_( self.session.query( Role.id).filter_by(name="student").first())).all() if len(user_objs) == 0: self.logger.info("用户表为空") return None else: self.format.show_color("序号\t姓名\tqq号\t角色") for user_obj in user_objs: self.format.show_color("%s\t%s\t%s\t\t%s" % (user_obj.id, user_obj.name, user_obj.qq, user_obj.role.name)) return True def class_user(self, *args): """ 学员分配班级 :param args: :return: """ if self.classes_list(): class_id = self.format.input_str_format("请输入班级ID号") class_obj = self.session.query(Classes).filter_by( id=class_id).first() if self.user_list(): qq = self.format.input_str_format("请输入学号QQ号") user_obj = self.session.query(User).filter_by(qq=qq).first() user_obj.classes = [class_obj] try: self.session.commit() self.logger.debug("学员QQ[%s]分配班级成功" % qq) except: self.logger.error("学员已分配到此班级") def course_add(self, *args): """ 增加课程 :param args: :return: """ if self.classes_list(): class_id = self.format.input_str_format("请输入班级ID号") try: name = self.format.input_str_format("请输入课程名") course_oj = Course(name=name, classes_id=class_id) self.session_add(course_oj) self.logger.debug("课程[%s]增加成功" % name) except: self.logger.error("你输入的班级号不存在!") def course_list(self, *args): """ 查看所有课程 :param args: :return: """ course_objs = self.session.query(Course).all() if len(course_objs) == 0: self.logger.info("课程表为空") return None else: self.format.show_color("序号\t课程名\t班级名") for course_obj in course_objs: self.format.show_color( "%s\t%s\t%s" % (course_obj.id, course_obj.name, course_obj.classes.name)) return True def record_add(self, *args): """ 增加上课记录 :param args: :return: """ if self.course_list(): course_id = self.format.input_str_format("请输入课程ID号") try: name = self.format.input_str_format("请输入上课记录名") taskname = self.format.input_str_format("请输入上课作业要求") record_oj = Record(name=name, task=taskname, course_id=course_id) self.session_add(record_oj) self.logger.debug("上课记录[%s]增加成功" % name) except: self.logger.error("你输入的课程号不存在!") def record_list(self, *args): """ 所有课程记录 :param args: :return: """ record_objs = self.session.query(Record).all() if len(record_objs) == 0: self.logger.info("上课记录表为空") return None else: self.format.show_color("序号\t\t上课记录\t\t上课作业\t课程名\t\t班级名") for record_obj in record_objs: self.format.show_color("%s\t%s\t%s\t%s\t%s" \ %(record_obj.id,record_obj.name,record_obj.task,record_obj.course.name,record_obj.course.classes.name)) return True def record_user_add(self, *args): """ 学员上课记录 :param args: :return: """ if self.record_list(): record_id = self.format.input_str_format("请输入上课程记录ID号") record_obj = self.session.query(Record).filter_by( id=record_id).first() if self.user_list(): user_id = self.format.input_str_format("请输入学员ID号") user_obj = self.session.query(User).filter_by( id=user_id).first() record_user_obj = RecordUser(user_id=user_id, record_id=record_id) self.session_add(record_user_obj) def record_user_list(self, *args): """ 查看所有学员上课记录 :param args: :return: """ record_user_objs = self.session.query(RecordUser).all() if len(record_user_objs) == 0: self.logger.info("上课记录和用户关系表为空") return None else: self.format.show_color("序号\t课程\t上课记录\t作业\t上课学员") for record_user_obj in record_user_objs: user_obj = self.session.query(User).filter_by( id=record_user_obj.user_id).first() record_obj = self.session.query(Record).filter_by( id=record_user_obj.record_id).first() self.format.show_color("%s\t%s\t%s\t%s\t%s" \ %(record_user_obj.id,record_obj.course.name,record_obj.name,record_obj.task,user_obj.name)) return True def sumbit_user_record(self, *args): """ 提交作业 :param args: :return: """ userid = args[0] record_user_objs = self.session.query(RecordUser).filter( RecordUser.user_id == userid, RecordUser.record_id.in_( self.session.query(Record.id).filter_by(task_status=0))).all() if len(record_user_objs) == 0: self.logger.error("用户没有需要上交的作业") return record_user_ids = [] for record_user_obj in record_user_objs: record_user_ids.append("%s" % record_user_obj.id) self.format.show_color("%s\t%s\t%s\t%s\t%s" \ %(record_user_obj.id,record_user_obj.record.course.classes.name,\ record_user_obj.record.course.name,record_user_obj.record.name, record_user_obj.record.task )) while True: record_user_id = self.format.input_str_format("请输入作业ID号") if record_user_id not in record_user_ids: continue else: break is_ok = self.format.input_str_format("是否提交作业Y/N") if is_ok == "Y": record_obj = self.session.query(Record).filter( Record.id == self.session.query(RecordUser.record_id).filter( RecordUser.id == record_user_id)).first() record_obj.task_status = 1 self.logger.debug("[%s]提交作业成功" % userid) def score_add(self, *args): """ 审批作业 :param args: :return: """ if self.record_user_list(): record_user_id = self.format.input_str_format("请输入作业ID号") score_num = self.format.input_str_format("请输入成绩") try: score_obj = Score(number=score_num, record_user_id=record_user_id) self.session_add(score_obj) self.logger.debug("审批作业成功") except: self.logger.error("你输入的作业号不存在!") def score_query(self, *args): """ 查询成绩 :param args: :return: """ userid = args[0] score_objs = self.session.query(RecordUser).filter_by( user_id=userid).all() if len(score_objs) == 0: self.logger.error("用户成绩不存在!") return try: for score_obj in score_objs: self.format.show_color("%s\t%s\t%s\t%s\t%s\t%s" \ %(score_obj.id,score_obj.record.course.classes.name,\ score_obj.record.course.name,score_obj.record.name, score_obj.record.task,score_obj.score[0].number )) except: self.logger.error("用户成绩不存在!") def score_list(self, *args): """ 所有成绩列表 :param args: :return: """ score_objs = self.session.query(RecordUser).all() if len(score_objs) == 0: self.logger.error("用户[%s]成绩不存在!" % (score_objs.user.name)) return self.format.show_color("班级\t学员\t课程名\t上课记录\t作业名\t成绩") for score_obj in score_objs: self.format.show_color("%s\t%s\t%s\t%s\t%s\t%s" \ %(score_obj.user.classes[0].name,score_obj.user.name,score_obj.record.course.name,score_obj.record.name,score_obj.record.task,score_obj.score[0].number)) def sum_score_list(self, *args): """ 成绩排名 :param args: :return: """ userid = args[0] user_objs = (self.session.query(User).filter_by(id=userid).first()) self.format.show_color("班级\t用户\t总成绩\t排名") for class_obj in user_objs.classes: user_objs = self.session.query(Classes).filter( Classes.id == class_obj.id).first() if len(user_objs.user) == 0: return score_sorted = [] for user_obj in user_objs.user: record_user_objs = self.session.query(RecordUser).filter( RecordUser.user_id == user_obj.id).all() nums = [] for record_user_obj in record_user_objs: for score_obj in record_user_obj.score: nums.append(copy.deepcopy(score_obj.number)) score_sorted.append((user_obj.name, sum(nums))) new_scored = sorted(score_sorted, key=lambda score: score[1]) new_scored.reverse() for score_tmp in range(len(new_scored)): self.format.show_color( "%s\t%s\t%s\t%s" % (class_obj.name, new_scored[score_tmp][0], new_scored[score_tmp][1], score_tmp + 1)) def session_add(self, data): if type(data) == "list": self.session.add_all(data) else: self.session.add(data) self.session.commit()
class Classifier: '''Class using for classification of tweets. Use classify() method for classification, train() method for training of bayesian filter.''' MAX_TOKEN_SIZE = 6 # defines word count in dictionary tuples HUMAN_RATING_PROBABILITY = 0.99 def __init__(self, low=0.5, high=0.5): # classification thresholds self._low = float(low) self._high = float(high) # add and setup logger self._logger = logging.getLogger() logging.basicConfig(level=logging.DEBUG) # db connection self.db = Connection() # load info about allready classified entries self._logger.info('Loading Allready classified entries...') self.human = HumanClassification('/mnt/minerva1/nlp/projects/twitter_classification/TwitterClassifier/pickles/HumanClassification') self.human.load() # load database of words self._logger.info('Loading word dictionary...') self.word_dict = WordDictionary('/mnt/minerva1/nlp/projects/twitter_classification/TwitterClassifier/pickles/WordDictionary') self.word_dict.load() # timer self._timer = timeit.Timer() def _add_classification(self, entry, classification): 'Add each token to word dictionary for futher classification.' language = entry.get_language() for i in xrange(1, self.MAX_TOKEN_SIZE + 1): # for each token add to word dictionary for token in entry.get_token(i): self.word_dict.words.setdefault(language, {}).setdefault(token, {'count':0, 'weight':0})['count'] += 1 if classification: self.word_dict.words[language][token]['weight'] += self.HUMAN_RATING_PROBABILITY else: self.word_dict.words[language][token]['weight'] += (1 - self.HUMAN_RATING_PROBABILITY) def _add_to_human_classification(self, entry, classification): 'Adds classified text to human classification. Stores classification, text and language of each text.' self.human.classification[entry.get_id()] = (classification, entry.get_guid(), entry.get_original_entry(), entry.get_language()) if classification is None: return self._add_classification(entry, classification) def train(self, language, count=None, offset=0): 'Given the language and optionaly count or offset shows dialog for realning' self.db.connect(user='******', database='meco', host='localhost', port=5432) try: for entry in self.db.entries(language=language, entry_count=10000, entry_offset=offset): # when entry was allready processed apply and skip if entry.id in self.human.classification: continue # ask whether entry is relevant automatic_classification = self.classify(entry.original_entry, language) print 'Original entry(' + str(entry.id) + '): \n"'+ entry.original_entry + '"\n automatic classification = ' + str(automatic_classification) if automatic_classification < self._low: auto = 'n' continue # TODO:odstranit elif automatic_classification >= self._high: auto = 'y' else: auto = '?' answer = raw_input('Is this entry relevant? (y/n/?/END))[' + auto + ']: ') if answer == 'y': self._add_to_human_classification(entry, True) elif answer == 'n': self._add_to_human_classification(entry, False) elif answer == '?': continue elif answer == 'END': break else: if automatic_classification < self._low: self._add_to_human_classification(entry, False) elif automatic_classification >= self._high: self._add_to_human_classification(entry, True) else: continue print 'after classification: ' + str(self.classify(entry.original_entry, language)) except KeyboardInterrupt: pass # store human input and word_dictionary self.human.store() self.word_dict.store() def manual_train(self, text, language, classification): 'Method for manual training of bayesian filter.' e = Entry(None, text, language) if classification is True: self._add_classification(e, True) if classification is False: self._add_classification(e, False) self.word_dict.store() def train_from_human_classification(self, filename, language): 'Method for training current bayesian filter from external human classification file' filehandler = open(filename, 'rb') content = pickle.load(filehandler) for entry_id in content: e = Entry(entry_id, list(content[entry_id])[1], list(content[entry_id])[2]) if e.get_language() == language: self._add_to_human_classification(e, list(content[entry_id])[0]) self.human.store() self.word_dict.store() def regenerate_word_dict(self): 'regenerate word dictionary according to human_input.' print self.human.classification self.word_dict.words = {} # go through human classification and create new word dictionary using classification for entry_id in self.human.classification: e = Entry(entry_id, list(self.human.classification[entry_id])[1], list(self.human.classification[entry_id])[2]) if list(self.human.classification[entry_id])[0] == True: self._add_classification(e, True) if list(self.human.classification[entry_id])[0] == False: self._add_classification(e, False) self.word_dict.store() def human_classify(self, output_pickle, language): 'This method creates output_pickle file containing user defined classifications of entries. May be used for creating test data.' self.db.connect(user='******', database='meco', host='localhost', port=5432) new_human_classify = HumanClassification(output_pickle) new_human_classify.load() try: for entry in self.db.entries(language=language, entry_count=None, entry_offset=0): # when entry was allready processed skip if entry.id in new_human_classify.classification: continue print 'Original entry: \n"'+ entry.original_entry + '"\n automatic classification = ' + str(self.classify(entry.original_entry, language)) automatic_classification = self.classify(entry.original_entry, language) if automatic_classification < self._low: auto = 'n' continue # TODO: odstranit elif automatic_classification >= self._high: auto = 'y' else: auto = '?' answer = raw_input('Is this entry relevant? (y/n/?/END))['+ auto +']: ') if answer == 'y': new_human_classify.classification[entry.id] = True elif answer == 'n': new_human_classify.classification[entry.id] = False elif answer == 'END': break else: if automatic_classification < self._low: new_human_classify.classification[entry.id] = False elif automatic_classification >= self._high: new_human_classify.classification[entry.id] = True else: new_human_classify.classification[entry.id] = None print 'Cassified count = ' + str(len(new_human_classify.classification)) except KeyboardInterrupt: pass new_human_classify.store() def classify(self, text, language): '''Given input text and language, method calculates probability of text being relevant to topic. @result probability that text is relevant''' input_entry = Entry(id=None, guid=None, entry=text, language=language) self.word_dict.words.setdefault(language, {}) # for each token claculate probability of being relevant to topic # and calculate according to bayes theorem # # p1p2p3........pn a # P = ------------------------------------------ = ------- # p1p2p3........pn + (1-p1)(1-p2)...(1-pn) a + b # a = 1.0 b = 1.0 for i in xrange(1, self.MAX_TOKEN_SIZE + 1): for token in input_entry.get_token(i): if not token in self.word_dict.words[language]: probability = 0.5 else: token_stats = self.word_dict.words[language][token] probability = token_stats['weight'] / token_stats['count'] a *= probability b *= 1 - probability if a + b == 0: return 0 else: result = a / (a + b) if result == 0.5: return -1 else: return a / (a + b) def _test_corelation(self, human_classified_pickle, language): 'This method prints corelation between user defined input in human_classified_pickle and automatic classification.' # # covariance # | # C(X,Y) E(XY) - E(X)E(Y) # corelation = ------------------ = ------------------------------------------- , a = E(XY), b = E(X), c = E(Y), d,= E(X^2), e = E(Y^2) # d(X)d(Y) sqrt(E(X^2) - E(X)^2) sqrt(E(Y^2) - E(Y)^2) # | # standard deviations # # X - automatically calculated probabilities # Y - human input probabilities # human_classified = HumanClassification(human_classified_pickle) human_classified.load() entry_count = len(human_classified.classification) a = 0.0 b = 0.0 c = 0.0 d = 0.0 e = 0.0 for entry_id in human_classified.classification: processed_entry = self.db.get_entry_by_id(entry_id) probability_auto = self.classify(processed_entry.original_entry, language) if human_classified.classification[entry_id]: probability_human = self.HUMAN_RATING_PROBABILITY else: probability_human = (1 - self.HUMAN_RATING_PROBABILITY) a += probability_human * probability_auto b += probability_auto c += probability_human d += probability_auto * probability_auto e += probability_human * probability_human # E() values a /= entry_count b /= entry_count c /= entry_count d /= entry_count e /= entry_count return (a - (b * c)) / (sqrt(d - (b * b)) * sqrt(e - (c * c))) def _test_percents(self, human_classified_pickle, language): 'This method returns ntuple containing (matches, false_positive, false_negative, unknown)' human_classified = HumanClassification(human_classified_pickle) human_classified.load() entry_count = len(human_classified.classification) true_positive = 0.0 true_negative = 0.0 matches = 0.0 false_positive = 0.0 false_negative = 0.0 unknown = 0.0 for entry_id in human_classified.classification: processed_entry = self.db.get_entry_by_id(entry_id) probability = self.classify(processed_entry.original_entry, language) if probability < self._low: if not human_classified.classification[entry_id]: matches += 1 true_negative += 1 else: false_negative += 1 elif probability >= self._high: if human_classified.classification[entry_id]: matches += 1 true_positive += 1 else: false_positive += 1 else: unknown += 1 return (matches, true_positive, true_negative, false_positive, false_negative, unknown, entry_count) def run_tests(self, input_file, language): 'Method for running tests on input file and get time elapsed for classification of one entry' self.db.connect(user='******', database='meco', host='localhost', port=5432) tmp = HumanClassification(input_file) tmp.load() self._logger.info('Running tests...') tests = Tests() tests.set_test_len(len(tmp.classification)) tests.set_train_len(len(self.human.classification)) tests.set_train_positive_len(self.human.get_positively_classified_count(language)) tests.set_train_negative_len(self.human.get_negatively_classified_count(language)) self._logger.info('Calculating corelation...') tests.set_corelation(self._test_corelation(input_file, language)) self._logger.info('Calculating percentage of classification accuracy...') tests.set_percents(self._test_percents(input_file, language)) print tests def get_time(self): 'Method for calculating time needed for one entry classification' self._logger.info('Downloading entries to run tests on...') i = 0 imax = 1000 entries = [] for entry in self.db.entries(language='en', entry_count=imax): i += 1 if i >= imax: break entries.append(entry.original_entry) self._logger.info('Masuring amount of entries to be calculated in 1sec') repetitions = 100 result_avg = 0.0 for i in xrange(0, repetitions): average = 0 for j in xrange(0, imax - 1): start = time.time() self.classify(entries[j], 'en') average += time.time() - start average /= imax result_avg += average result_avg /= repetitions return 'Classifier is able to classify ' + str(round(1/result_avg, 2)) + ' entries in one second.' def export_to_xml(self, language, specification): 'Method exports all data to xml files' self.word_dict.to_xml(filename='XML/word_dict', specification=specification) self.human.to_xml(self.db, 'XML/human_classification', language=language) def fix_old_human_classification(self, filename): 'method converts old human classification file to new one inlcluding text of tweets' self.db.connect(user='******', database='meco', host='localhost', port=5432) file = open(filename, 'rb') content = pickle.load(file) new_content = {} for entry_id in content: e = self.db.get_entry_by_id(entry_id) if e: new_content[entry_id] = (content[entry_id], e.original_entry, e.get_language()) new_file = open(filename +'new', 'wb') pickle.dump(new_content, new_file) def fix_old_human_classification2(self, filename): 'method converts old human classification file to new one inlcluding text of tweets' self.db.connect(user='******', database='meco', host='localhost', port=5432) file = open(filename, 'rb') content = pickle.load(file) new_content = {} for entry_id in content: e = self.db.get_entry_by_id(entry_id) if e: new_content[entry_id] = (list(content[entry_id])[0], e.get_guid(), e.get_original_entry(), e.get_language()) new_file = open(filename +'_new', 'wb') pickle.dump(new_content, new_file) def train_from_file(self, filename, language, classification): 'method trains classifier from some file' file = open(filename, 'r') for line in file: e = Entry(None, None, line, language) self._add_to_human_classification(e, classification) self.human.store() self.word_dict.store()
from src.teste import Teste from src.connection import Connection import datetime import configparser #teste = Teste(10,20) #print(teste.soma()) cfg = configparser.ConfigParser() cfg.read('config.ini') url = cfg.get('development', 'url') port = cfg.getint('development', 'port') bd = cfg.get('development', 'bd') print(url + ' ' + str(port)) c = Connection(url, port, bd) db = c.getDB() json = {'nome': 'teste', 'date': datetime.datetime.now()} db.tickets.replace_one({'nome': 'teste'}, json, True)
class Classifier: '''Class using for classification of tweets. Use classify() method for classification, train() method for training of bayesian filter.''' MAX_TOKEN_SIZE = 6 # defines word count in dictionary tuples HUMAN_RATING_PROBABILITY = 0.99 def __init__(self, low=0.5, high=0.5): # classification thresholds self._low = float(low) self._high = float(high) # add and setup logger self._logger = logging.getLogger() logging.basicConfig(level=logging.DEBUG) # db connection self.db = Connection() # load info about allready classified entries self._logger.info('Loading Allready classified entries...') self.human = HumanClassification( '/mnt/minerva1/nlp/projects/twitter_classification/TwitterClassifier/pickles/HumanClassification' ) self.human.load() # load database of words self._logger.info('Loading word dictionary...') self.word_dict = WordDictionary( '/mnt/minerva1/nlp/projects/twitter_classification/TwitterClassifier/pickles/WordDictionary' ) self.word_dict.load() # timer self._timer = timeit.Timer() def _add_classification(self, entry, classification): 'Add each token to word dictionary for futher classification.' language = entry.get_language() for i in xrange(1, self.MAX_TOKEN_SIZE + 1): # for each token add to word dictionary for token in entry.get_token(i): self.word_dict.words.setdefault(language, {}).setdefault( token, { 'count': 0, 'weight': 0 })['count'] += 1 if classification: self.word_dict.words[language][token][ 'weight'] += self.HUMAN_RATING_PROBABILITY else: self.word_dict.words[language][token]['weight'] += ( 1 - self.HUMAN_RATING_PROBABILITY) def _add_to_human_classification(self, entry, classification): 'Adds classified text to human classification. Stores classification, text and language of each text.' self.human.classification[entry.get_id()] = ( classification, entry.get_guid(), entry.get_original_entry(), entry.get_language()) if classification is None: return self._add_classification(entry, classification) def train(self, language, count=None, offset=0): 'Given the language and optionaly count or offset shows dialog for realning' self.db.connect(user='******', database='meco', host='localhost', port=5432) try: for entry in self.db.entries(language=language, entry_count=10000, entry_offset=offset): # when entry was allready processed apply and skip if entry.id in self.human.classification: continue # ask whether entry is relevant automatic_classification = self.classify( entry.original_entry, language) print 'Original entry(' + str( entry.id ) + '): \n"' + entry.original_entry + '"\n automatic classification = ' + str( automatic_classification) if automatic_classification < self._low: auto = 'n' continue # TODO:odstranit elif automatic_classification >= self._high: auto = 'y' else: auto = '?' answer = raw_input('Is this entry relevant? (y/n/?/END))[' + auto + ']: ') if answer == 'y': self._add_to_human_classification(entry, True) elif answer == 'n': self._add_to_human_classification(entry, False) elif answer == '?': continue elif answer == 'END': break else: if automatic_classification < self._low: self._add_to_human_classification(entry, False) elif automatic_classification >= self._high: self._add_to_human_classification(entry, True) else: continue print 'after classification: ' + str( self.classify(entry.original_entry, language)) except KeyboardInterrupt: pass # store human input and word_dictionary self.human.store() self.word_dict.store() def manual_train(self, text, language, classification): 'Method for manual training of bayesian filter.' e = Entry(None, text, language) if classification is True: self._add_classification(e, True) if classification is False: self._add_classification(e, False) self.word_dict.store() def train_from_human_classification(self, filename, language): 'Method for training current bayesian filter from external human classification file' filehandler = open(filename, 'rb') content = pickle.load(filehandler) for entry_id in content: e = Entry(entry_id, list(content[entry_id])[1], list(content[entry_id])[2]) if e.get_language() == language: self._add_to_human_classification(e, list(content[entry_id])[0]) self.human.store() self.word_dict.store() def regenerate_word_dict(self): 'regenerate word dictionary according to human_input.' print self.human.classification self.word_dict.words = {} # go through human classification and create new word dictionary using classification for entry_id in self.human.classification: e = Entry(entry_id, list(self.human.classification[entry_id])[1], list(self.human.classification[entry_id])[2]) if list(self.human.classification[entry_id])[0] == True: self._add_classification(e, True) if list(self.human.classification[entry_id])[0] == False: self._add_classification(e, False) self.word_dict.store() def human_classify(self, output_pickle, language): 'This method creates output_pickle file containing user defined classifications of entries. May be used for creating test data.' self.db.connect(user='******', database='meco', host='localhost', port=5432) new_human_classify = HumanClassification(output_pickle) new_human_classify.load() try: for entry in self.db.entries(language=language, entry_count=None, entry_offset=0): # when entry was allready processed skip if entry.id in new_human_classify.classification: continue print 'Original entry: \n"' + entry.original_entry + '"\n automatic classification = ' + str( self.classify(entry.original_entry, language)) automatic_classification = self.classify( entry.original_entry, language) if automatic_classification < self._low: auto = 'n' continue # TODO: odstranit elif automatic_classification >= self._high: auto = 'y' else: auto = '?' answer = raw_input('Is this entry relevant? (y/n/?/END))[' + auto + ']: ') if answer == 'y': new_human_classify.classification[entry.id] = True elif answer == 'n': new_human_classify.classification[entry.id] = False elif answer == 'END': break else: if automatic_classification < self._low: new_human_classify.classification[entry.id] = False elif automatic_classification >= self._high: new_human_classify.classification[entry.id] = True else: new_human_classify.classification[entry.id] = None print 'Cassified count = ' + str( len(new_human_classify.classification)) except KeyboardInterrupt: pass new_human_classify.store() def classify(self, text, language): '''Given input text and language, method calculates probability of text being relevant to topic. @result probability that text is relevant''' input_entry = Entry(id=None, guid=None, entry=text, language=language) self.word_dict.words.setdefault(language, {}) # for each token claculate probability of being relevant to topic # and calculate according to bayes theorem # # p1p2p3........pn a # P = ------------------------------------------ = ------- # p1p2p3........pn + (1-p1)(1-p2)...(1-pn) a + b # a = 1.0 b = 1.0 for i in xrange(1, self.MAX_TOKEN_SIZE + 1): for token in input_entry.get_token(i): if not token in self.word_dict.words[language]: probability = 0.5 else: token_stats = self.word_dict.words[language][token] probability = token_stats['weight'] / token_stats['count'] a *= probability b *= 1 - probability if a + b == 0: return 0 else: result = a / (a + b) if result == 0.5: return -1 else: return a / (a + b) def _test_corelation(self, human_classified_pickle, language): 'This method prints corelation between user defined input in human_classified_pickle and automatic classification.' # # covariance # | # C(X,Y) E(XY) - E(X)E(Y) # corelation = ------------------ = ------------------------------------------- , a = E(XY), b = E(X), c = E(Y), d,= E(X^2), e = E(Y^2) # d(X)d(Y) sqrt(E(X^2) - E(X)^2) sqrt(E(Y^2) - E(Y)^2) # | # standard deviations # # X - automatically calculated probabilities # Y - human input probabilities # human_classified = HumanClassification(human_classified_pickle) human_classified.load() entry_count = len(human_classified.classification) a = 0.0 b = 0.0 c = 0.0 d = 0.0 e = 0.0 for entry_id in human_classified.classification: processed_entry = self.db.get_entry_by_id(entry_id) probability_auto = self.classify(processed_entry.original_entry, language) if human_classified.classification[entry_id]: probability_human = self.HUMAN_RATING_PROBABILITY else: probability_human = (1 - self.HUMAN_RATING_PROBABILITY) a += probability_human * probability_auto b += probability_auto c += probability_human d += probability_auto * probability_auto e += probability_human * probability_human # E() values a /= entry_count b /= entry_count c /= entry_count d /= entry_count e /= entry_count return (a - (b * c)) / (sqrt(d - (b * b)) * sqrt(e - (c * c))) def _test_percents(self, human_classified_pickle, language): 'This method returns ntuple containing (matches, false_positive, false_negative, unknown)' human_classified = HumanClassification(human_classified_pickle) human_classified.load() entry_count = len(human_classified.classification) true_positive = 0.0 true_negative = 0.0 matches = 0.0 false_positive = 0.0 false_negative = 0.0 unknown = 0.0 for entry_id in human_classified.classification: processed_entry = self.db.get_entry_by_id(entry_id) probability = self.classify(processed_entry.original_entry, language) if probability < self._low: if not human_classified.classification[entry_id]: matches += 1 true_negative += 1 else: false_negative += 1 elif probability >= self._high: if human_classified.classification[entry_id]: matches += 1 true_positive += 1 else: false_positive += 1 else: unknown += 1 return (matches, true_positive, true_negative, false_positive, false_negative, unknown, entry_count) def run_tests(self, input_file, language): 'Method for running tests on input file and get time elapsed for classification of one entry' self.db.connect(user='******', database='meco', host='localhost', port=5432) tmp = HumanClassification(input_file) tmp.load() self._logger.info('Running tests...') tests = Tests() tests.set_test_len(len(tmp.classification)) tests.set_train_len(len(self.human.classification)) tests.set_train_positive_len( self.human.get_positively_classified_count(language)) tests.set_train_negative_len( self.human.get_negatively_classified_count(language)) self._logger.info('Calculating corelation...') tests.set_corelation(self._test_corelation(input_file, language)) self._logger.info( 'Calculating percentage of classification accuracy...') tests.set_percents(self._test_percents(input_file, language)) print tests def get_time(self): 'Method for calculating time needed for one entry classification' self._logger.info('Downloading entries to run tests on...') i = 0 imax = 1000 entries = [] for entry in self.db.entries(language='en', entry_count=imax): i += 1 if i >= imax: break entries.append(entry.original_entry) self._logger.info( 'Masuring amount of entries to be calculated in 1sec') repetitions = 100 result_avg = 0.0 for i in xrange(0, repetitions): average = 0 for j in xrange(0, imax - 1): start = time.time() self.classify(entries[j], 'en') average += time.time() - start average /= imax result_avg += average result_avg /= repetitions return 'Classifier is able to classify ' + str(round( 1 / result_avg, 2)) + ' entries in one second.' def export_to_xml(self, language, specification): 'Method exports all data to xml files' self.word_dict.to_xml(filename='XML/word_dict', specification=specification) self.human.to_xml(self.db, 'XML/human_classification', language=language) def fix_old_human_classification(self, filename): 'method converts old human classification file to new one inlcluding text of tweets' self.db.connect(user='******', database='meco', host='localhost', port=5432) file = open(filename, 'rb') content = pickle.load(file) new_content = {} for entry_id in content: e = self.db.get_entry_by_id(entry_id) if e: new_content[entry_id] = (content[entry_id], e.original_entry, e.get_language()) new_file = open(filename + 'new', 'wb') pickle.dump(new_content, new_file) def fix_old_human_classification2(self, filename): 'method converts old human classification file to new one inlcluding text of tweets' self.db.connect(user='******', database='meco', host='localhost', port=5432) file = open(filename, 'rb') content = pickle.load(file) new_content = {} for entry_id in content: e = self.db.get_entry_by_id(entry_id) if e: new_content[entry_id] = (list(content[entry_id])[0], e.get_guid(), e.get_original_entry(), e.get_language()) new_file = open(filename + '_new', 'wb') pickle.dump(new_content, new_file) def train_from_file(self, filename, language, classification): 'method trains classifier from some file' file = open(filename, 'r') for line in file: e = Entry(None, None, line, language) self._add_to_human_classification(e, classification) self.human.store() self.word_dict.store()
def __init__(self, transport, source_address, source_port, destination_address, destination_port, app=None, window=1000, threshold=100000, fast_recovery=False, aiad=False, aimdc=0.5): Connection.__init__(self, transport, source_address, source_port, destination_address, destination_port, app) ### Sender functionality # send window; represents the total number of bytes that may # be outstanding at one time (cwnd) self.window = window # send buffer self.send_buffer = SendBuffer() # maximum segment size, in bytes self.mss = 1000 # largest sequence number that has been ACKed so far; represents # the next sequence number the client expects to receive self.sequence = 0 # retransmission timer self.timer = None # timeout duration in seconds self.timeout = 1 # Round trip time in seconds self.estimated_rtt = None # Deviation of sample rtt from estimated rtt self.deviation_rtt = None # State determines slow start vs. additive increase # 0 = slow start # 1 = additive increase self.state = 0 # Threshold for slow start self.threshold = threshold self.trace("Threshold starting at %d" % self.threshold) # Same ack count (used for calculating three duplicate acks) self.same_ack_count = 0 # Most recent ack (used for calculating three duplicate acks) self.last_ack = None # Make sure settings start with slow start self.window = self.mss # Used when dropping certain packets self.dropped_count = 0 # Fast Recovery (Reno) self.fast_recovery = fast_recovery # AIAD self.aiad = aiad # AIMD Constant self.aimdc = aimdc ### Used to make TCP Sequence Graphs self.x = [] self.y = [] self.dropX = [] self.dropY = [] self.ackX = [] self.ackY = [] ### Used to make window size graphs self.window_x = [] self.window_y = [] ### Receiver functionality # receive buffer self.receive_buffer = ReceiveBuffer() # ack number to send; represents the largest in-order sequence # number not yet received self.ack = 0 # a list of all the queuing delays from the sender to receiver self.queueing_delay_list = [] # keep track of when packets were received self.packets_received = {}
def __init__(self,transport,source_address,source_port, destination_address,destination_port,app=None,window=1000, threshold=100000,fast_recovery=False): Connection.__init__(self,transport,source_address,source_port, destination_address,destination_port,app) ### Sender functionality # send window; represents the total number of bytes that may # be outstanding at one time (cwnd) self.window = window # send buffer self.send_buffer = SendBuffer() # maximum segment size, in bytes self.mss = 1000 # largest sequence number that has been ACKed so far; represents # the next sequence number the client expects to receive self.sequence = 0 # retransmission timer self.timer = None # timeout duration in seconds self.timeout = 1 # Round trip time in seconds self.estimated_rtt = None # Deviation of sample rtt from estimated rtt self.deviation_rtt = None # State determines slow start vs. additive increase # 0 = slow start # 1 = additive increase self.state = 0 # Threshold for slow start self.threshold = threshold self.trace("Threshold starting at %d" % self.threshold) # Same ack count (used for calculating three duplicate acks) self.same_ack_count = 0 # Most recent ack (used for calculating three duplicate acks) self.last_ack = None # Make sure settings start with slow start self.window = self.mss # Used when dropping certain packets self.dropped_count = 0 # Fast Recovery (Reno) self.fast_recovery=fast_recovery ### Used to make TCP Sequence Graphs self.x = [] self.y = [] self.dropX = [] self.dropY = [] self.ackX = [] self.ackY = [] ### Receiver functionality # receive buffer self.receive_buffer = ReceiveBuffer() # ack number to send; represents the largest in-order sequence # number not yet received self.ack = 0 # a list of all the queuing delays from the sender to receiver self.queueing_delay_list = []
def main(): folder = FileHandler() folder.create_folder() folder.move_file() for file in folder.fb2_books(folder.path_input): Connection().for_main(Model(folder.path_input, file))
import os, sys import copy import sqlalchemy from sqlalchemy.orm import sessionmaker #获取父级目录 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) #父级目录加入系统环境 sys.path.append(BASE_DIR) from src.connection import Connection from src.log import Log from src.models import Role, User, Classes, Course, Record, Score, RecordUser from src.fomatter import Fomatter conn = Connection().create_connection() engine = sqlalchemy.create_engine(conn, encoding="utf-8", echo=False) session_class = sessionmaker(bind=engine) session = session_class() r1 = Role(name="admin") r2 = Role(name="teacher") r3 = Role(name="student") session.add_all([r1, r2, r3]) u1 = User(name="admin", password="******", qq="1", role_id=1) u2 = User(name="teacher", password="******", qq="2", role_id=2) u3 = User(name="shisanjun", password="******", qq="3", role_id=3)