def unite_forest(self, msg): """This fuunction is called by leader process to update the component of all process in the current forest.""" if msg.component_id > self.component_id: logging.info("Process %s is setting as non leader.", str(self.process_id)) self.update_leader(False) else: if self.level == msg.level: self.level += 1 elif self.level < msg.level: self.level = msg.level msg = Message() msg = msg.update_component(self.level, self.component_id) for edge in self.mst_edges: edge.send_message(msg, self.process_id) seen_edges = [] self.log_mst_edges() while len(seen_edges) != len(self.mst_edges): for edge in [e for e in self.mst_edges if e not in seen_edges]: self.messages_lock.acquire() if "ack_leader" in self.messages[edge.id] and\ len(self.messages[edge.id]["ack_leader"]): logging.info("acknowledge received from edge: %s", edge.id) msg = self.messages[edge.id]["ack_leader"].pop() seen_edges.append(edge) self.messages_lock.release() self.round_completed = True
def handle(self, message): if message.command == "PING": # respond quick before we get booted! m = Message() m.command = "PONG" m.params = message.params self.connection.send(m) elif message.command == "QUIT": # user left. Log them out if need be u = db.User(nick=message.sender) u.logout() elif message.command == "NICK": # nick updated. Chase them! u = db.User(nick=message.sender) u.change_nick(message.params[0]) elif message.command == "PRIVMSG" and message.params[1].startswith(self.settings["command_prefix"]): # palm it off to the plugins for plugin in self.plugins: if plugin.handle(message): break channel = None if message.params[0].startswith("#"): channel = message.params[0] db.log_message(message.command, message.sender, channel, " ".join(message.params))
def xsend(self): ''' set appid and appkey 使用新的线程 发送短信 ''' message_configs = {} message_configs['appid'] = self.appid message_configs['appkey'] = self.appkey ''' set sign_type,if is set ''' if self.sign_type != '': message_configs['sign_type'] = self.sign_type ''' init mail class ''' message = Message(message_configs) ''' build request and send email and return the result ''' result = message.xsend(self.build_request()) # MessageThread(message).start() return result
def t_one(self, dest_x, dest_y, self_x, self_y, self_dir, expected): s = "message {} {} {} {} {} aaa\\".format( dest_x, dest_y, self_x, self_y, self_dir.value ) m = Message(s) action = m.action_to_join_sender() self.assertTrue(expected == action)
def get_messages(self): if not self.initialized: self.error_message = 'Cannot get messages for non initialized Bot' return [] messages = [] updates = self.bot.get_updates(offset=self.last_update_id).wait() for update in updates: # always increase last_update_id to get only new messages next time self.last_update_id = update.update_id + 1 self.chat_id = update.message.chat.id # skip all types of message except Message if not update.message or not isinstance(update.message, Message): continue m = Msg() ok = m.init(update) if not ok: self.errors[update.message.text] = m.get_error_message() continue messages.append(m) return messages
def wiki(self, nick, channel, params): if len(params) < 1: return; replyto = nick if channel == None else channel qs = { 'action': 'query', 'prop': 'revisions', 'rvprop': 'content', 'format': 'xml', 'titles': ' '.join(params) } url = 'http://en.wikipedia.org/w/api.php?' + urllib.urlencode(qs) xml = ElementTree.parse(urllib.urlopen(url)) result = xml.findtext('.//rev') if result is not None: result = self.deugly_wikimedia(result) else: result = 'Couldn\'t find it :(' m = Message() m.command = 'PRIVMSG' m.params = [replyto, result] self.connection.send(m)
def run_round(self): """Runs the current round before rporting back.""" self.round_completed = False self.merge_mwoe_received = None self.clear_ret_messages() self.get_non_mst_edges() if self.leader: logging.info("finding mwoe") mwoe_msg = self.find_MWOE() logging.info("MWOE details: edge id: %s, weight: %f", mwoe_msg.edge_id, mwoe_msg.msg) if mwoe_msg.edge_id is None: self.completed = True self.round_completed = True completed_msg = Message() completed_msg = completed_msg.get_completed_msg() for edge in self.mst_edges: edge.send_message(completed_msg, self.process_id) return self.merge_mwoe_received_lock.acquire() self.merge_mwoe_received = mwoe_msg self.merge_mwoe_received_lock.release() # code for leader to send the mwoe request to its child merge_mwoe_msg = Message() merge_mwoe_msg = merge_mwoe_msg.merge_MWOE(mwoe_msg.edge_id, mwoe_msg.process) self.log_mst_edges() for edge in self.mst_edges: edge.send_message(merge_mwoe_msg, self.process_id) if mwoe_msg.process == self.process_id: self.send_join_request(self.get_edge(mwoe_msg.edge_id)) self.merge_mwoe(mwoe_msg)
def __init__(self, msg=None, data=None, filename=None, password=None, vals=None, file_obj=None): self.p = None self.q = None self.g = None self.y = None self.x = None if file_obj is not None: self._from_private_key(file_obj, password) return if filename is not None: self._from_private_key_file(filename, password) return if (msg is None) and (data is not None): msg = Message(data) if vals is not None: self.p, self.q, self.g, self.y = vals else: if msg is None: raise SSHException('Key object may not be empty') if msg.get_string() != 'ssh-dss': raise SSHException('Invalid key') self.p = msg.get_mpint() self.q = msg.get_mpint() self.g = msg.get_mpint() self.y = msg.get_mpint() self.size = util.bit_length(self.p)
class GoodMessage(unittest.TestCase): def setUp(self): """ initial setup for testing """ self.msg = Message('Sender', 'Recepient', 'Content') def testGoodSender(self): """ tests for good sender. It should pass. """ self.assertTrue(self.msg.isSender()) def testGoodRecipient(self): """ tests for good recipient. It should pass. """ self.assertTrue(self.msg.isRecipient()) def testGoodContent(self): """ tests for good content. It should pass. """ self.assertTrue(self.msg.isContent()) def testRealMsg(self): """ tests for an existed, 'populated' message. It should pass. """ self.assertFalse(self.msg.isEmpty()) @unittest.skip("no use for this test") def testLargeMsg(self): """ tests for a large message. It should pass. """ self.assertRaises(Message.OutOfRangeError, self.msg.size, 999999999999) def testComputeMsgSize(self): """ tests for size computation. It should pass. """ self.assertTrue(self.msg.computeMsgSize())
def data(self): msg = Message() for carrier in self.carriers: msg.append(Record('urn:nfc:wkt:ac', data=str(carrier))) if self.error.reason is not None: msg.append(Record("urn:nfc:wkt:err", data=str(self.error))) return str(self.version) + str(msg)
def delete_messages(self, mtype = None): """ Delete messages of a specified kind. Args: type: A string of the message type to delete. Due to timeout issues with App Engine, this method will currently only succeed when running on App Engine if the number of messages being deleted is relatively small (~hundreds). It will attempt to delete up to 1000. The timeout retry wrapper (see game_server/autoretry_datastore.py) and using keys only search drastically increases the chances of success, but this method is still not guaranteed to complete. For more information see: http://groups.google.com/group/google-appengine/ browse_thread/thread/ec0800a3ca92fe69?pli=1 http://stackoverflow.com/questions/108822/ delete-all-data-for-a-kind-in-google-app-engine """ if mtype: db.delete(Message.all(keys_only = True).filter('msg_type =', mtype) .ancestor(self.key()).order('date').fetch(1000)) db.delete(Message.all(keys_only = True).ancestor(self.key()).order('date') .fetch(1000))
def receiveData(self): data, addr = self.sock.recvfrom( 1024 ) #prints a string but crashes on OSC msg #print("received: " + data.decode("utf-8")) m = Message() m.parse(data) return m
def _keepalive_conversation(self): if self.state != ProtocolState.OPEN: return True message = Message() message.add_header('Content-Type', 'text/x-keep-alive') self._send_command('MSG', 'N', message, True) return True
def unicast(self, message, destination): delay_time = random.uniform(self.min_delay, self.max_delay) message = Message(self.pid, destination, message) print(message.send_str()) print('delay unicast with {0:.2f}s '.format(delay_time)) delayed_t = threading.Timer(delay_time, self.__unicast, (message, destination,)) delayed_t.start()
def publish(self): """Publish a mbox""" mbox = Mbox(self.config.get('mbox')) messages = 0 if not (os.path.exists(self.config.get('dir'))): os.mkdir(self.config.get('dir')) message = mbox.nextMessage() while(message != None): msg = Message(message, self.config) messages += 1 self.index.add(msg) self.subscribers.add(msg) msg.toRDF() #msg.toHTML() message = mbox.nextMessage() self.index.toRDF() self.subscribers.toRDF() return messages
def do_call(*args, **kwargs): data = { 'name': name, 'args': args, 'kwargs': kwargs, } data.update(self.env) request = Message(data) # Send request self.send(request.serialize()) # Receive response bufs = [] while True: try: buf = self.recv() except ClientException, e: raise ClientCommunicationError(e) if not buf: raise ClientCommunicationError('Server hang up.') bufs.append(buf) try: response, _ = Message.parse("".join(bufs)) except MessageIncomplete: continue except MessageInvalid, e: raise ClientCommunicationError(e)
def handle_message(self, data): """ message handler. """ m = Message(data) m.parse(self) if data.type == 'groupchat' and data.subject: logging.debug("%s - checking topic" % self.name) self.topiccheck(m) nm = Message(m) callbacks.check(self, nm) return if data.get('x').xmlns == 'jabber:x:delay': logging.warn("%s - ignoring delayed message" % self.name) return self.privwait.check(m) if m.isresponse: logging.debug("%s - message is a response" % self.name) return jid = None m.origjid = m.jid for node in m.subelements: try: m.jid = node.x.item.jid except (AttributeError, TypeError): continue if self.me in m.fromm: logging.debug("%s - message to self .. ignoring" % self.name) return 0 try: if m.type == 'error': if m.code: logging.error('%s - error - %s' % (self.name, str(m))) self.errorwait.check(m) self.errorHandler(m) except Exception, ex: handle_exception()
def move(self, state): """Use the game state to decide what action to take, and then output the direction to move.""" self.game.update(state) message = Message() message.game_message(self) self.input_interface.append( message ) self.action = self.decide() retval = 'Stay' if ( self.action == 'Heal' ): if ( None != self.game.board.taverns_list[0].path ): retval = self.game.board.taverns_list[0].path[0] elif ( self.action == 'Mine' ): if ( None != self.game.board.mines_list[0].path ): retval = self.game.board.mines_list[0].path[0] elif ( self.action == 'Attack' ): if ( None != self.game.enemies_list[0].path ): retval = self.game.enemies_list[0].path[0] elif ( self.action == 'Wait' ): retval = 'Stay' self.expected_pos = self.game.board.to( self.game.hero.pos, retval ) self.prev_mines = self.game.hero.mines self.prev_gold = self.game.hero.gold self.prev_life = self.game.hero.life return retval
def __init__(self): super(EventProcessor, self).__init__() self.message_handler = MessageHandler() #self.gps = GPS() Message.init(self.message_handler.plugin.address()) Detector.start_plugins() Reactor.add_plugin_events()
def _keepalive_conversation(self): if self.state != ProtocolState.OPEN: return True message = Message() message.add_header("Content-Type", "text/x-keep-alive") self._send_command("MSG", "N", message, True) return True
def read(self): try: message = self.socket.recv(4096) except Exception as e: Log.e(getExceptionInfo(e)) return Client.ReadError if len(message) == 0: return Client.NoMessage self.buf += message while len(self.buf) > 4: length = int(self.buf[0:4]) if not len(self.buf) >= length + 4: break msg = self.buf[4: length + 4] self.buf = self.buf[length + 4:] message = Message() if not message.loads(msg): Log.w(u'Unknown Message') else: self.requests.put(message) return Client.NewMessage
def _receive_ack(self, msg): """Receives an ACK and transitions message's state based on that. """ self._remove_message(msg) idx, parent_msg = self._find_message(token=None, message_id=msg.message_id, state=MessageState.wait_for_ack) if parent_msg is None: coap_log.warning('ACK received but no matching message found - Sending RESET') reset_msg = Message(message_id=msg.message_id, message_type=MessageType.reset) self._socket.send(reset_msg.build()) return if parent_msg.type != MessageType.confirmable: coap_log.error('ACK received for NON-CONFIRMABLE message - Ignoring') return self._transition_message(parent_msg, MessageState.wait_for_response) if msg.class_code == 0 and msg.class_detail == 0: # if this is empty message send just for ACK we are already done. coap_log.debug('Separate ACK received') return #coap_log.debug('Piggybacked RESPONSE {0}'.format(str(msg))) if msg.has_observe_option(): self._receive_observe(parent_msg, msg) return # This message has a piggybacked response, so receive it. self._receive_response(parent_msg, msg)
def startRound(self, connection, *args): game = connection.player.game if game.state != Game.STATE_FINISHED_ROUND: return # if game.master is connection.player: # game.start() game.startNewRound() # message = Message.encode(Message.SEND_GAME_MESSAGE_STARTED, '') # self.server.sendMessageMultiple(game.players, message) scores = {} connections = [] for player in game.players: connections.append(player.connection) scores[player.name] = len(player.green_apples) red_apples = [] for red_apple in player.red_apples: red_apples.append(red_apple.dictionary) message = Message.encode(Message.SEND_GAME_MESSAGE_PLAYER_RED_APPLES, red_apples) self.server.sendMessageSingle(player.connection, message) round_details = {} round_details['JUDGE'] = game.players[game.judge].name round_details['SCORES'] = scores round_details['GREEN_APPLE'] = game.active_green_apple.dictionary message = Message.encode(Message.SEND_GAME_MESSAGE_ROUND_DETAILS, round_details) self.server.sendMessageMultiple(connections, message)
def test_multiple_packet_message(self): body = "1a0eeb5701b202010a0101a000340034170d000208000186a0000000000000".decode('hex') msg = Message("1f07b1ee", 14, 0x18, body) packets = msg.packets() self.assertEqual(len(packets), 2) self.assertEqual(packets[0].raw_hex(), "1f07b1eeae1f07b1ee181f1a0eeb5701b202010a0101a000340034170d000208000186a019") self.assertEqual(packets[1].raw_hex(), "1f07b1ee900000000000000251e2")
def _parse_kexdh_gex_request(self, m): minbits = m.get_int() preferredbits = m.get_int() maxbits = m.get_int() # smoosh the user's preferred size into our own limits if preferredbits > self.max_bits: preferredbits = self.max_bits if preferredbits < self.min_bits: preferredbits = self.min_bits # fix min/max if they're inconsistent. technically, we could just pout # and hang up, but there's no harm in giving them the benefit of the # doubt and just picking a bitsize for them. if minbits > preferredbits: minbits = preferredbits if maxbits < preferredbits: maxbits = preferredbits # now save a copy self.min_bits = minbits self.preferred_bits = preferredbits self.max_bits = maxbits # generate prime pack = self.transport._get_modulus_pack() if pack is None: raise SSHException('Can\'t do server-side gex with no modulus pack') self.transport._log(DEBUG, 'Picking p (%d <= %d <= %d bits)' % (minbits, preferredbits, maxbits)) self.g, self.p = pack.get_modulus(minbits, preferredbits, maxbits) m = Message() m.add_byte(chr(_MSG_KEXDH_GEX_GROUP)) m.add_mpint(self.p) m.add_mpint(self.g) self.transport._send_message(m) self.transport._expect_packet(_MSG_KEXDH_GEX_INIT)
def unicastTCP(self, serverID, message): delay_time = random.uniform(self.min_delay, self.max_delay) m = Message(self.pid, serverID, message) print(m.send_str()) print('delay unicastTCP with {0:.2f}s '.format(delay_time)) delayed_t = threading.Timer(delay_time, self.__unicastTCP, (message,)) delayed_t.start()
def handle_request(self, request): args = {} for k, v in request.args.items(): args.update({k: v[0]},) if len(request.postpath[0]) and not args.get('callback'): request.setResponseCode(400) return HTML if 'register' in request.postpath: register = Register(request, args) return register.response() if 'login' in request.postpath: login = Login(request, args) return login.response() if 'poll' in request.postpath: poll = Poll(request, args) return poll.response() if 'message' in request.postpath: message = Message(request, args) return message.response() if 'join' in request.postpath: join = Join(request, args) return join.response() else: return """
def candidate_receive_message(self, msg): """ All Candidate Message Receiving @:param msg - the JSON message received @:return: Void """ if msg['type'] == 'vote': #print str(self.id) + ": Got Vote Message----------" message = Message.create_message_from_json(msg) self.receive_vote(message) if msg['type'] == 'heartbeat': #print str(self.id) + "got ~~~HEARTBEAT~~~" heart_beat = Message.create_message_from_json(msg) if heart_beat.term >= self.current_term: self.current_term = heart_beat.term self.become_follower(heart_beat.leader) if msg['type'] == 'voteRequest': message = Message.create_message_from_json(msg) if message.term > self.current_term: self.become_follower(msg['leader']) if msg['type'] in ['get', 'put']: message = Message.create_message_from_json(msg) redirect_message = message.create_redirect_message("FFFF") self.send(redirect_message)
def create_msg_adiumLog(self): # Logs of the format: # (00:07:25)fetchgreebledonx:Do you know how you replace spaces in a string with nothing? # Logs are passed to TXTMessageParser, which returns a list of dicts with # timestamp, message_type, message, and sender # message_type will be either message or sender, and for now we ignore status date = self.filename.split("(")[1].split(")")[0].replace("|", "-") try: f = open(self.filename, "r") data = f.read() f.close() p = TXTMessageParser() p.parse(data) for msg in [f for f in p.messages if f["message_type"] in ["message"]]: m = Message( msg["message"], self.account, self.contact, msg["sender"], self.service, date + " " + msg["timestamp"], ) m.insert(self.connection) except IOError: print "Error: %s" % self.filename
def exchange(self, vault=None): ''' First thing first, the initiator issues a "exchange request". In Cyclon protocol, the exchange request consists in generating a subset of peers to exchange with the oldest peer. The "age" of a peer is based on the how many exchanges have passed since I last updated the state of that peer of my neighboring list. ''' if self.itsExchanging(): return True Cyclon.peerList.increaseAges() try: oldestPeer = Cyclon.peerList.getOldest() except IndexError: print ' * No peer available for exchanging' return True print '''Starting an exchange with ''' + oldestPeer.getName() self.exchangeStarted() print >> stderr, ''' * computing subset ''' shuffleSize = randint(1, self.getShuffleLength()) - 1 randomSubset = Cyclon.peerList.drop(oldestPeer).getSubset(shuffleSize) sentSet = CyclonPeerList([ oldestPeer ]) + randomSubset message = Message(kind='exchange', source=Cyclon.myself.getName(), destination=oldestPeer.getName()) randomSubset.insert(0, Cyclon.myself) body = message.getBody() print >> stderr, ''' * building message ''' for p in randomSubset: node = body.appendChild(message.createElement('node')) node.setAttribute('name', p.getName()) node.childNodes = p.getConnectionsXML() print >> stderr, ''' * Sending request ''' message.send( oldestPeer.transport, 3, 3 ).addCallback( self.exchangeReplyReceived, sentSet=sentSet ).addErrback( self.removePeerErrback, peer=oldestPeer.getName(), success=False ).addErrback( self.exchangeFinished, success=False ).addErrback( self.exchange) return True
sys.exit(0) serverIp = sys.argv[1] serverPort = 5425 filepath = sys.argv[2] sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # TCP 소켓을 생성한다. try: print("서버:{0}/{1}".format(serverIp, serverPort)) sock.connect((serverIp, serverPort)) # 접속 요청을 수락한다. msgId = 0 reqMsg = Message() filesize = os.path.getsize(filepath) reqMsg.Body = BodyRequest(None) reqMsg.Body.FILESIZE = filesize reqMsg.Body.FILENAME = os.path.basename(filepath) msgId += 1 reqMsg.Header = Header(None) reqMsg.Header.MSGID = msgId reqMsg.Header.MSGTYPE = message.REQ_FILE_SEND reqMsg.Header.BODYLEN = reqMsg.Body.GetSize() reqMsg.Header.FRAGMENTED = message.NOT_FRAGMENTED reqMsg.Header.LASTMSG = message.LASTMSG reqMsg.Header.SEQ = 0 MessageUtil.send(sock, reqMsg) # 클라이언트는 서버에 접속하자마자 파일 전송 요청 메세지를 보낸다.
data = self.sock.recv(16) amount_received += len(data) print('received "%s"' % data) finally: if (close_connection): print("Closing socket") self.sock.close() print("Socket closed") if __name__ == '__main__': while True: MESSAGE_SENDER = MessageSender("config.xml") MESSAGE = Message("Test", \ "1", \ MESSAGE_SENDER.config.sensors[0].sensor_id, \ MESSAGE_SENDER.config.sensors[0].sensor_type, \ MESSAGE_SENDER.config.sensors[0].sensor_data_type, \ MESSAGE_SENDER.config.sensors[0].sensor_interval) MESSAGE_SENDER.send_message(MESSAGE) time.sleep(0.500) # # Set up device readings # tempReading = {'machineId':'001','sensorId':'001','sensorType':'Temp','Values':'30.1'} # message = json.dumps(tempReading) # message = message + '<EOF>' # print ('sending "%s"' % message) # # Create a TCP/)IP socket # sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # # Connect the socket to the port where the server is listening # server_address = ('localhost', 8010)
def nospace_wrapper(_): """Wrapper for nospace function.""" return Message(nospace(_['T']))
@command('nospace', __name__, help='Remove whitespaces from a string.', usage='<text>') def nospace_wrapper(_): """Wrapper for nospace function.""" return Message(nospace(_['T'])) @command('unspace', __name__, help='Remove extra whitespaces from a string.', usage='<text>') def unspace_wrapper(_): """Wrapper for unspace function.""" return Message(unspace(_['T'])) command('len', __name__, help='Return the length of a string.', usage='<text>')(lambda _: Message(len(_['T']))) def nospace(s): """Remove whitespaces from a string.""" return s.replace(' ', '') def unspace(s): """Remove extra whitespaces from a string.""" # 'a b _ c d' to 'ab cd' if max([len(x) for x in s.split()]) > 1: return s n = [] m = 0 for c in s:
def scrapeCommentsBot(url, channel_users, messageId): """ Scrapes https://comments.bot/ website for comment feature extension.""" hdr = { 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Referer': 'https://cssspritegenerator.com', 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3', 'Accept-Encoding': 'none', 'Accept-Language': 'en-US,en;q=0.8', 'Connection': 'keep-alive' } soup = Bots.javaScriptLoadMoreHack(url, "bot") if soup is None: try: req = Request(url, headers=hdr) page = urlopen(req) soup = BeautifulSoup(page, 'html.parser') except: logging.info("Could not open comments app.") commentList = [] if soup is not None: comments = soup.find_all('div', class_='comment-content') count = 1 for comment in comments: # Save comment Message new_comment = Message() new_comment.isComment = True new_comment.parent = messageId # Save Id new_comment.id = str(messageId) + "." + str(count) # Save Text text_list = comment.find('div', class_='comment-text').contents text_list = [ elem for elem in text_list if str(elem) != "<br/>" ] # Eliminate Html new_comment.text = eliminateWhitespaces(' '.join( [str(elem) for elem in text_list])) # Save replyId reply_text_list = comment.find('span', class_='comment-reply-text') if reply_text_list is not None: reply_text = eliminateWhitespaces(' '.join( [str(elem) for elem in reply_text_list])) try: new_comment.replyToMessageId = next( (new_comment for new_comment in commentList if new_comment.text == reply_text), None).id except: logging.info("Could not find quoted comment in Bot: " + url) count = count + 1 # Find not identified User new_comment.sender_name = comment.find( 'div', class_='name-row').findChildren()[0].contents[0] commentUser = telethon.types.User(0) commentUser.first_name = new_comment.sender_name channel_users.append(commentUser) new_comment.timestamp = comment.find( 'div', class_='comment-date').findChildren()[0].contents[0] # No identified Users commentList.append(new_comment) return commentList
def _update_cached_message_flags(self, highest_mod_seq, event_data_list): # type: (int, t.List[AbstractEventData]) -> None """Update the flags on any cached messages. """ # we just check the highestmodseq and revert to full sync if they don't match # this is kind of what thunderbird does https://wiki.mozilla.org/Thunderbird:IMAP_RFC_4551_Implementation if highest_mod_seq is not None: if self._highest_mod_seq == highest_mod_seq: logger.debug("%s matching highest mod seq no flag update" % self) return logger.debug("%s started updating flags" % self) # get all the flags for the old messages fetch_data = self._imap_client.fetch('1:%d' % (self._last_seen_uid), [ 'FLAGS']) # type: t.Dict[int, t.Dict[str, t.Any]] # update flags in the cache for message_schema in MessageSchema.objects.filter(folder_schema=self._schema): assert isinstance(message_schema, MessageSchema) # ignore cached messages that we just fetched if message_schema.uid > self._last_seen_uid: continue # if we don't get any information about the message we have to remove it from the cache if message_schema.uid not in fetch_data: message_schema.delete() logger.debug("%s deleted message with uid %d" % (self, message_schema.uid)) continue message_data = fetch_data[message_schema.uid] # TODO make this more DRY if 'SEQ' not in message_data: logger.critical('Missing SEQ in message data') logger.critical('Message data %s' % message_data) continue if 'FLAGS' not in message_data: logger.critical('Missing FLAGS in message data') logger.critical('Message data %s' % message_data) continue old_flags = set(message_schema.flags) new_flags = set(message_data['FLAGS']) if old_flags - new_flags: # flag removed old flag exists which does not exist in new flags event_data_list.append(RemovedFlagsData(Message(message_schema, self._imap_client), list(old_flags - new_flags))) elif new_flags - old_flags: # flag added, new flags exists which does not exist in old flags event_data_list.append(NewFlagsData(Message(message_schema, self._imap_client), list(new_flags - old_flags))) message_schema.flags = list(new_flags) message_schema.msn = message_data['SEQ'] message_schema.save() # TODO maybe trigger the user logger.debug("%s updated flags" % self) if highest_mod_seq is not None: self._highest_mod_seq = highest_mod_seq logger.debug("%s updated highest mod seq to %d" % (self, highest_mod_seq))
return True if __name__ == '__main__': server = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) server.bind(("", 0)) print("Server started at 0.0.0.0 port " + str(server.getsockname()[1])) ctx = {} plant = Plant(ctx) workshop = Workshop(ctx) designer = Designer(ctx) plant.launch() workshop.launch() designer.launch() # On dit à notre Logistics qu'on se connecte sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) sock.sendto(pickle.dumps(Message(Message.CONNECTED, server.getsockname()[1])), ('localhost', 33333)) print(f"ctx:{ctx}") while True: data, addr = server.recvfrom(1024) print(f"Data recv: {data}") msg = pickle.loads(data) plant.parent_pipe.send(msg)
import socket import select import sys from message import Message import pickle host = '127.0.0.1' port = 12345 mySocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) mySocket.connect((host, port)) _name = str(sys.argv[1]) m = Message(_name, "") while True: sockets_list = [sys.stdin, mySocket] read_sockets, write_socket, error_socket = select.select( sockets_list, [], []) """ There are two possible input situations. Either the # user wants to give manual input to send to other people, # or the server is sending a message to be printed on the # screen. Select returns from sockets_list, the stream that # is reader for input. So for example, if the server wants # to send a message, then the if condition will hold true # below.If the user wants to send a message, the else # condition will evaluate as true
def run(message, matches, chat_id, step): return [ Message(chat_id).set_text( "🎗*Butely Bot*🎗\nbased on _telepot_\nfeedback:@Amrrezaebrahimi\nMade whit ❤️ in 🇮🇷", parse_mode="markdown") ]
def _save_new_messages(self, last_seen_uid, event_data_list = None, urgent=False): # type: (int, t.List[AbstractEventData]) -> None """Save any messages we haven't seen before Args: last_seen_uid (int): the max uid we have stored, should be 0 if there are no messages stored. urgent (bool): if True, save only one email """ # add thread id to the descriptors if there is a thread id descriptors = list(Message._descriptors) + ['X-GM-THRID'] if self._imap_account.is_gmail \ else list(Message._descriptors) uid_criteria = "" if urgent: uid_criteria = '%d' % (last_seen_uid + 1) else: uid_criteria = '%d:*' % (last_seen_uid + 1) fetch_data = self._imap_client.fetch( uid_criteria, descriptors) # seperate fetch in order to detect if the message is already read or not header_data = self._imap_client.fetch( uid_criteria, list(Message._header_descriptors)) # if there is only one item in the return field # and we already have it in our database # delete it to be safe and save it again # TODO not sure why this happens maybe the folder._uid_next isn't getting updated properly if len(fetch_data) == 1 and last_seen_uid in fetch_data: already_saved = MessageSchema.objects.filter(folder_schema=self._schema, uid=last_seen_uid) if already_saved: logger.critical("%s found already saved message, deleting it" % self) already_saved[0].delete() logger.info("%s saving new messages" % (self)) for uid in fetch_data: message_data = fetch_data[uid] header = header_data[uid] logger.debug("Message %d data: %s" % (uid, message_data)) if 'SEQ' not in message_data: logger.critical('Missing SEQ in message data') logger.critical('Message data %s' % message_data) continue if 'FLAGS' not in message_data: logger.critical('Missing FLAGS in message data') logger.critical('Message data %s' % message_data) continue if 'INTERNALDATE' not in message_data: logger.critical('Missing INTERNALDATE in message data') logger.critical('Message data %s' % message_data) continue if 'RFC822.SIZE' not in message_data: logger.critical('Missing RFC822.SIZE in message data') logger.critical('Message data %s' % message_data) continue # if 'ENVELOPE' not in message_data: # logger.critical('Missing ENVELOPE in message data') # logger.critical('Message data %s' % message_data) # continue if self._imap_account.is_gmail and 'X-GM-THRID' not in message_data: logger.critical('Missing X-GM-THRID in message data') logger.critical('Message data %s' % message_data) continue # check for supported thread algorithms here if not self._imap_account.is_gmail: capabilities = self._imap_client.capabilities() capabilities = list(capabilities) capabilities = filter(lambda cap: 'THREAD=' in cap, capabilities) capabilities = [cap.replace('THREAD=', '') for cap in capabilities] logger.critical("Add support for one of the following threading algorithms %s" % capabilities) raise NotImplementedError("Unsupported threading algorithm") # this is the date the message was received by the server internal_date = message_data['INTERNALDATE'] # type: datetime msn = message_data['SEQ'] flags = message_data['FLAGS'] if "\\Seen" not in flags: self._imap_client.remove_flags(uid, ['\\Seen']) # header = [h.replace('\r\n\t', ' ') for h in header] header = header[list(Message._header_descriptors)[0]] header = header.replace('\r\n\t', ' ') header = header.replace('\r\n', ' ') meta_data = {} # figure out text encoding issue here # logger.info(header[uid][list(Message._header_descriptors)[0]]) try: header = self._parse_email_header(header) except Exception as e: logger.critical("header parsing problem %s %s, skip this message" % (header, e)) continue try: f_tmp = "" header_field = ['Subject:', 'From:', 'To:', 'Cc:', 'CC:', 'Bcc:', 'Date:', 'In-Reply-To:', 'Message-Id:', 'Message-ID:', 'Message-id:'] for v in re.split('('+ "|".join(header_field) +')', header): if not v: continue if v.strip() in header_field: # Remove a colon and add to a dict f_tmp = v[:-1].lower().strip() else: meta_data[f_tmp] = v.strip() except Exception as e: logger.critical("header parsing problem %s, skip this message" % e) continue # if we have a gm_thread_id set thread_schema to the proper thread gm_thread_id = message_data.get('X-GM-THRID') thread_schema = None if gm_thread_id is not None: result = self._imap_client.search(['X-GM-THRID', gm_thread_id]) logger.debug("thread messages %s, current message %d" % (result, uid)) thread_schema = self._find_or_create_thread(gm_thread_id) logger.debug("message %d envelope %s" % (uid, meta_data)) try: if internal_date.tzinfo is None or internal_date.tzinfo.utcoffset(internal_date) is None: internal_date = timezone('US/Eastern').localize(internal_date) # logger.critical("convert navie %s " % internal_date) except Exception: logger.critical("Internal date parsing error %s" % internal_date) continue try: date = parser.parse(meta_data["date"]) # if date is naive then reinforce timezone if date.tzinfo is None or date.tzinfo.utcoffset(date) is None: date = timezone('US/Eastern').localize(date) except Exception: if "date" in meta_data: logger.critical("Can't parse date %s, skip this message" % meta_data["date"]) continue else: date = internal_date logger.info("Date not exist, put internal date instead") # TODO seems like bulk email often not have a message-id if "message-id" not in meta_data: logger.critical("message-id not exist, skil this message %s" % meta_data) continue # create and save the message schema message_schema = MessageSchema(imap_account=self._schema.imap_account, folder_schema=self._schema, uid=uid, msn=msn, flags=flags, date=date, subject="" if "subject" not in meta_data else meta_data['subject'], message_id=meta_data["message-id"], internal_date=internal_date, _thread=thread_schema ) if "from" in meta_data: message_schema.from_m = self._find_or_create_contacts(meta_data['from'])[0] # if envelope.from_ is not None: # message_schema.from_m = self._find_or_create_contacts(envelope.from_)[0] try: message_schema.save() except Exception as e: logger.critical("%s failed to save message %d" % (self, uid)) logger.critical("%s stored last_seen_uid %d, passed last_seen_uid %d" % (self, self._last_seen_uid, last_seen_uid)) logger.critical("number of messages returned %d" % (len(fetch_data))) # to prevent dup saved email continue if last_seen_uid != 0 and event_data_list is not None: logger.critical(internal_date) if tz.now() - internal_date < timedelta(seconds=5*60): event_data_list.append(NewMessageData(Message(message_schema, self._imap_client))) logger.debug("%s finished saving new messages..:" % self) # create and save the message contacts if "reply-to" in meta_data: message_schema.reply_to.add(*self._find_or_create_contacts(meta_data["reply-to"])) if "to" in meta_data: message_schema.to.add(*self._find_or_create_contacts(meta_data["to"])) if "cc" in meta_data: message_schema.cc.add(*self._find_or_create_contacts(meta_data["cc"])) if "bcc" in meta_data: message_schema.bcc.add(*self._find_or_create_contacts(meta_data["bcc"])) logger.debug("%s saved new message with uid %d" % (self, uid))
def sign_ssh_data(self, data): m = Message() m.add_string("ssh-ed25519") m.add_string(self._signing_key.sign(data).signature) return m
def _parse_signing_key_data(self, data, password): from transport import Transport # We may eventually want this to be usable for other key types, as # OpenSSH moves to it, but for now this is just for Ed25519 keys. # This format is described here: # https://github.com/openssh/openssh-portable/blob/master/PROTOCOL.key # The description isn't totally complete, and I had to refer to the # source for a full implementation. message = Message(data) if message.get_bytes(len(OPENSSH_AUTH_MAGIC)) != OPENSSH_AUTH_MAGIC: raise SSHException("Invalid key") ciphername = message.get_text() kdfname = message.get_text() kdfoptions = message.get_binary() num_keys = message.get_int() if kdfname == "none": # kdfname of "none" must have an empty kdfoptions, the ciphername # must be "none" if kdfoptions or ciphername != "none": raise SSHException("Invalid key") elif kdfname == "bcrypt": if not password: raise PasswordRequiredException( "Private key file is encrypted") kdf = Message(kdfoptions) bcrypt_salt = kdf.get_binary() bcrypt_rounds = kdf.get_int() else: raise SSHException("Invalid key") if ciphername != "none" and ciphername not in Transport._cipher_info: raise SSHException("Invalid key") public_keys = [] for _ in range(num_keys): pubkey = Message(message.get_binary()) if pubkey.get_text() != "ssh-ed25519": raise SSHException("Invalid key") public_keys.append(pubkey.get_binary()) private_ciphertext = message.get_binary() if ciphername == "none": private_data = private_ciphertext else: cipher = Transport._cipher_info[ciphername] key = bcrypt.kdf( password=b(password), salt=bcrypt_salt, desired_key_bytes=cipher["key-size"] + cipher["block-size"], rounds=bcrypt_rounds, # We can't control how many rounds are on disk, so no sense # warning about it. ignore_few_rounds=True, ) decryptor = Cipher(cipher["class"](key[:cipher["key-size"]]), cipher["mode"](key[cipher["key-size"]:]), backend=default_backend()).decryptor() private_data = (decryptor.update(private_ciphertext) + decryptor.finalize()) message = Message(unpad(private_data)) if message.get_int() != message.get_int(): raise SSHException("Invalid key") signing_keys = [] for i in range(num_keys): if message.get_text() != "ssh-ed25519": raise SSHException("Invalid key") # A copy of the public key, again, ignore. public = message.get_binary() key_data = message.get_binary() # The second half of the key data is yet another copy of the public # key... signing_key = nacl.signing.SigningKey(key_data[:32]) # Verify that all the public keys are the same... assert (signing_key.verify_key.encode() == public == public_keys[i] == key_data[32:]) signing_keys.append(signing_key) # Comment, ignore. message.get_binary() if len(signing_keys) != 1: raise SSHException("Invalid key") return signing_keys[0]
def _valid_message(msg): return Message.addrs(msg.id) is None and not msg.has_expired()
async def run(message, matches, chat_id, step): if not is_group(message): from_id = message['from']['id'] if step == 0: user_steps[from_id] = {"name": "Soundcloud", "step": 1, "data": {}} hide_keyboard = {'hide_keyboard': True, "selective": True} return [ Message(chat_id).set_text( "لطفا اسم موزیک یا نام خواننده را وارد کنید🤔", reply_to_message_id=message['message_id'], reply_markup=hide_keyboard) ] if step == 1: await sender( Message(chat_id).set_text( "اومم یکم صبر کن برم جستجو کنم ببینم چی گیرم میاد 😘😬", parse_mode="markdown")) user_steps[from_id] = {"name": "Soundcloud", "step": 2, "data": {}} i = 0 show_keyboard = {'keyboard': [], "selective": True} ttt = message['text'].replace(" ", "+") for song in await search(ttt): title, link, cover = song[0], song[1], song[2] user_steps[from_id]['data'][title] = link show_keyboard['keyboard'].append([title]) i += 1 if i == 20: break if len(show_keyboard['keyboard']) in [0, 1]: hide_keyboard = {'hide_keyboard': True, 'selective': True} del user_steps[from_id] return [ Message(chat_id).set_text( "چیزی نیافتم 😣", reply_to_message_id=message['message_id'], reply_markup=hide_keyboard, parse_mode="markdown") ] return [ Message(chat_id).set_text( "یکی از اینارو انتخاب کن 🙃👇🏻", reply_to_message_id=message['message_id'], reply_markup=show_keyboard) ] elif step == 2: try: await sender( Message(chat_id).set_text( "خب یکم صبر کن تا موزیکو دانلود کنم و برات بفرستم 🤓", reply_to_message_id=message['message_id'], reply_markup=key, parse_mode="markdown")) await downloader( await getfile(user_steps[from_id]['data'][message['text']]), "tmp/{}.mp3".format(message['text'])) del user_steps[from_id] bot.sendAudio(chat_id, open("tmp/{}.mp3".format(message['text']), 'rb'), title=message['text'], performer="@SpntaBot") os.remove("tmp/{}.mp3".format(message['text'])) except Exception as e: del user_steps[from_id] return [ Message(chat_id).set_text("داری اشتباه میزنی 😕", parse_mode="markdown") ]
def build(): return Message({'@type': TrustPing.PING, '@id': str(uuid.uuid4())})
def broadcast(self, msg): Message.register(msg.id, {msg.event_creator(), msg.sender}) self.plugin.broadcast(msg)
def _handle_UBM(self, command): idx, network_id, account = self.__parse_network_and_account(command) contact = self.__search_account(account, network_id) if contact is not None: message = Message(contact, command.payload) self.emit("unmanaged-message-received", contact, message)
def setUp(self) -> None: self.message_row = ['0.000224', 809, 0, 8, "07 a7 7f 8c 11 2f 00 10"] self.message_object = Message(0.000224, 809, 0, 8, bytearray(b'\x07\xa7\x7f\x8c\x11/\x00\x10'))
def default(self, message): response = Message(message) response.status = 401 response.params['res'] = 'Invalid operation for state!' self.context.socket.send(pickle.dumps(response))
def _handle_MSG(self, command): message = Message(None, command.payload) content_type = message.content_type if content_type[0] == 'text/x-msmsgsprofile': profile = {} lines = command.payload.split("\r\n") for line in lines: line = line.strip() if line: name, value = line.split(":", 1) profile[name] = value.strip() self._client.profile._server_property_changed("profile", profile) self.set_privacy(self._client.profile.privacy) self._state = ProtocolState.SYNCHRONIZING self._client.address_book.sync() elif content_type[0] in \ ('text/x-msmsgsinitialmdatanotification', \ 'text/x-msmsgsoimnotification'): if self._client.oim_box is not None: self._client.oim_box._state = \ OIM.OfflineMessagesBoxState.NOT_SYNCHRONIZED m = HTTPMessage() m.parse(message.body) mail_data = m.get_header('Mail-Data').strip() if mail_data == 'too-large': mail_data = None self._client.oim_box.sync(mail_data) if mail_data and \ content_type[0] == 'text/x-msmsgsinitialmdatanotification': #Initial mail start = mail_data.find('<IU>') + 4 end = mail_data.find('</IU>') if start < end: mailbox_unread = int(mail_data[start:end]) self._client.mailbox._initial_set(mailbox_unread) elif content_type[0] == 'text/x-msmsgsinitialemailnotification': #Initial mail (obsolete by MSNP11) pass elif content_type[0] == 'text/x-msmsgsemailnotification': #New mail m = HTTPMessage() m.parse(message.body) name = decode_rfc2047_string(m.get_header('From')) address = m.get_header('From-Addr') subject = decode_rfc2047_string(m.get_header('Subject')) message_url = m.get_header('Message-URL') post_url = m.get_header('Post-URL') post_id = m.get_header('id') dest = m.get_header('Dest-Folder') if dest == 'ACTIVE': self._client.mailbox._unread_mail_increased(1) build = self._build_url_post_data post_url, form_data = build(message_url, post_url, post_id) self._client.mailbox._new_mail(name, address, subject, post_url, form_data) elif content_type[0] == 'text/x-msmsgsactivemailnotification': #Movement of unread mail m = HTTPMessage() m.parse(message.body) src = m.get_header('Src-Folder') dest = m.get_header('Dest-Folder') delta = int(m.get_header('Message-Delta')) if src == 'ACTIVE': self._client.mailbox._unread_mail_decreased(delta) elif dest == 'ACTIVE': self._client.mailbox._unread_mail_increased(delta)
def conf(self, message): response = Message('CONFREPLY') try: transf_id = int(message.params['idtransf']) status = message.params['status'] transf = self.context.server.transactions[transf_id] if transf: if status.upper() == 'APPROVED' or status.upper( ) == 'REPROVED': if self.context.client.username == transf[ 'destiny'].username: if transf['status'] == 'pending': response.status = 200 if status.upper() == 'APPROVED': response.params[ 'res'] = 'Transaction approved!' elif status.upper() == 'REPROVED': response.params[ 'res'] = 'Transaction reproved!' transf['status'] = status.lower() transf['origin'].semaphore.release() else: response.status = 400 response.params[ 'res'] = 'This transaction is already confirmed!' else: response.status = 400 response.params[ 'res'] = 'This transaction is not for you!' else: response.status = 400 response.params['res'] = 'Invalid status!' else: response.status = 400 response.params['res'] = 'This transaction do not exists!' except Exception as e: response.status = 500 response.params['res'] = 'Internal Server Error!' self.context.socket.send(pickle.dumps(response))
def sair(self, message): response = Message('SAIRREPLY') response.status = 200 response.params['res'] = 'Connection is going to close!' self.context.socket.send(pickle.dumps(response))
def saq(self, message): response = Message('SAQREPLY') try: quantity = float(message.params['qtd']) if not quantity: response.status = 400 response.params['res'] = 'Param qtd invalid!' elif self.context.client.wallet.balance < quantity: response.status = 400 response.params['res'] = 'Balance insufficient!' else: balance_before = self.context.client.wallet.balance self.context.client.wallet.balance -= quantity response.status = 200 response.params['balancebefore'] = balance_before response.params[ 'balanceafter'] = self.context.client.wallet.balance except Exception as e: response.status = 500 response.params['res'] = 'Internal Server Error!' self.context.socket.send(pickle.dumps(response))
def transf(self, message): response = Message('TRANSFREPLY') try: quantity = float(message.params['amount']) client = self.context.server.find_user(message.params['destiny']) if not quantity or not client: response.status = 400 response.params['res'] = 'Param amount or destiny invalid!' elif self.context.client.wallet.balance < quantity: response.status = 400 response.params['res'] = 'Balance insufficient!' else: id = len(self.context.server.transactions) self.context.server.transactions[id] = \ { 'origin': self.context.client, 'destiny': client, 'amount': quantity, 'status': 'pending' } self.context.client.semaphore.acquire() self.context.client.semaphore.acquire() response.status = 200 transf = self.context.server.transactions[id] if transf['status'] == 'approved': self.context.client.wallet.balance -= quantity client.wallet.balance += quantity response.params['res'] = 'Transaction approved!' response.params[ 'balanceafter'] = self.context.client.wallet.balance elif transf['status'] == 'reproved': response.params['res'] = 'Transaction reproved!' response.params[ 'balanceafter'] = self.context.client.wallet.balance self.context.client.semaphore.release() except Exception as e: response.status = 500 response.params['res'] = 'Internal Server Error!' self.context.socket.send(pickle.dumps(response))
def mail(self, prefetch=False, **kwargs): search = ['ALL'] kwargs.get('read') and search.append('SEEN') kwargs.get('unread') and search.append('UNSEEN') kwargs.get('starred') and search.append('FLAGGED') kwargs.get('unstarred') and search.append('UNFLAGGED') kwargs.get('deleted') and search.append('DELETED') kwargs.get('undeleted') and search.append('UNDELETED') kwargs.get('draft') and search.append('DRAFT') kwargs.get('undraft') and search.append('UNDRAFT') kwargs.get('before') and search.extend( ['BEFORE', kwargs.get('before').strftime(self.date_format)]) kwargs.get('after') and search.extend( ['SINCE', kwargs.get('after').strftime(self.date_format)]) kwargs.get('on') and search.extend( ['ON', kwargs.get('on').strftime(self.date_format)]) kwargs.get('header') and search.extend( ['HEADER', kwargs.get('header')[0], kwargs.get('header')[1]]) kwargs.get('sender') and search.extend(['FROM', kwargs.get('sender')]) kwargs.get('fr') and search.extend(['FROM', kwargs.get('fr')]) kwargs.get('to') and search.extend(['TO', kwargs.get('to')]) kwargs.get('cc') and search.extend(['CC', kwargs.get('cc')]) kwargs.get('subject') and search.extend( ['SUBJECT', kwargs.get('subject')]) kwargs.get('body') and search.extend(['BODY', kwargs.get('body')]) kwargs.get('label') and search.extend( ['X-GM-LABELS', kwargs.get('label')]) kwargs.get('attachment') and search.extend(['HAS', 'attachment']) kwargs.get('query') and search.extend([kwargs.get('query')]) emails = [] # print search response, data = self.gmail.imap.uid('SEARCH', *search) if response == 'OK': uids = filter(None, data[0].split(' ')) # filter out empty strings for uid in uids: if not self.messages.get(uid): self.messages[uid] = Message(self, uid) emails.append(self.messages[uid]) if prefetch and emails: messages_dict = {} for email in emails: messages_dict[email.uid] = email self.messages.update( self.gmail.fetch_multiple_messages(messages_dict)) return emails
def __init__(self, src, req_id): Message.__init__(self, src, None) # The id that identifies the request itself self.req_id = req_id
def test_extract_attachments(): message = Message("test_message") task_common.extract_attachments(attachments_required, message)
def run(self): c = Connection(self.config) logging.config.dictConfig(self.config) self.logger = logging.getLogger(__name__) self.logger.info('Process {} Running'.format(self.id)) self.msgQ.put(Message('Process', self.id, 'Running')) while True: parkrun = self.inQ.get() self.logger.debug(parkrun) if parkrun is None: self.logger.info('Process {} Exiting'.format(self.id)) self.msgQ.put(Message('Process', self.id, 'Exiting')) break self.logger.debug('Process {} got record {}'.format( self.id, parkrun['EventURL'])) if parkrun['lastEvent'] is None: parkrun['lastEvent'] = 0 if self.mode == Mode.CHECKURLS: if self.getURL(parkrun['URL']) is not None: c.updateParkrunURL(parkrun['Name'], True, True) self.msgQ.put( Message('Process', self.id, 'Verified ' + parkrun['Name'] + ' valid')) else: c.updateParkrunURL(parkrun['Name'], True, False) self.msgQ.put( Message( 'Error', self.id, 'Could not verify ' + parkrun['Name'] + ' as valid')) if self.mode == Mode.NEWEVENTS: self.logger.info( 'Process {} checking for new results for {}'.format( self.id, parkrun['EventURL'])) self.msgQ.put( Message('Process', self.id, 'Checking for new results for ' + parkrun['Name'])) parkrun['EventNumber'], parkrun[ 'EventDate'], data = self.getLatestEvent( parkrun['URL'] + parkrun['LatestResultsURL']) if data is not None: self.logger.debug( 'Event {} got {} events in history'.format( parkrun['EventURL'], len(data))) parkrun['Runners'] = len(data) # Add the event if it's a new event # Check the event has the correct number of runners if not c.checkParkrunEvent(parkrun): self.logger.info( 'Parkrun {} event {}: runners did not match - reimporting.' .format(parkrun['Name'], parkrun['EventNumber'])) #if not, delete the old event record and re-import the data self.msgQ.put( Message( 'Process', self.id, 'Updating ' + parkrun['Name'] + ' event ' + xstr(parkrun['EventNumber']))) eventID = c.replaceParkrunEvent(parkrun) self.logger.debug( 'getLastEvent found {} runners'.format(len(data))) for row in data: row['EventID'] = eventID c.addParkrunEventPosition(row) sleep(self.delay) if self.mode == Mode.NORMAL: data = self.getEventHistory(parkrun['URL'] + parkrun['EventHistoryURL']) if data is not None: self.logger.debug( 'Event {} got {} events in history'.format( parkrun['URL'], len(data))) for row in data: row['Name'] = parkrun['Name'] row['EventURL'] = parkrun['EventURL'] # Add the event if it's a new event self.msgQ.put( Message( 'Process', self.id, 'Checking ' + row['Name'] + ' event ' + xstr(row['EventNumber']))) self.logger.debug(row) self.logger.debug( 'Process {} Checking {} event {}'.format( self.id, row['EventURL'], xstr(row['EventNumber']))) # Check the event has the correct number of runners if not c.checkParkrunEvent(row): #if not, delete the old event record and re-import the data self.logger.info( 'Parkrun {} event {}: runners did not match - reimporting.' .format(parkrun['EventURL'], row['EventNumber'])) self.msgQ.put( Message( 'Process', self.id, 'Updating ' + row['Name'] + ' event ' + xstr(row['EventNumber']))) eventID = c.replaceParkrunEvent(row) eData = self.getEvent( parkrun['URL'] + parkrun['EventNumberURL'], row['EventNumber']) if eData is not None: self.logger.debug( 'getEvent found {} runners'.format( len(eData))) for eRow in eData: eRow['EventID'] = eventID c.addParkrunEventPosition(eRow) sleep(self.delay) else: self.logger.debug('getEvent found no runners') else: self.logger.warning( 'Parkrun {} returns no history page.'.format( parkrun['Name'])) c.execute( "update p set p.LastUpdated = e.LastEvent from parkruns as p inner join (select ParkrunID, max(EventDate) as LastEvent from events group by ParkrunID) as e on p.ParkrunID = e.ParkrunID" ) self.logger.debug('Sleeping for {} seconds'.format(self.delay)) sleep(self.delay) c.close()
def request_values(self, num_iterations): """ Method invoked to start simulation. Prints out what clients have converged on what iteration. Also prints out accuracy for each client on each iteration (what weights would be if not for the simulation) and federated accuaracy. :param iters: number of iterations to run """ converged = { } # maps client names to iteration of convergence. Contains all inactive clients active_clients = set(self.directory.clients.keys()) for i in range(1, num_iterations + 1): weights = {} intercepts = {} m = multiprocessing.Manager() lock = m.Lock() with ThreadPool(len(active_clients)) as calling_pool: args = [] for client_name in active_clients: client_instance = self.directory.clients[client_name] body = { 'iteration': i, 'lock': lock, 'simulated_time': config.LATENCY_DICT[self.name][client_name] } arg = Message(sender_name=self.name, recipient_name=client_name, body=body) args.append((client_instance, arg)) messages = calling_pool.map(client_computation_caller, args) server_logic_start = datetime.now() vals = { message.sender: (message.body['weights'], message.body['intercepts']) for message in messages } simulated_time = find_slowest_time(messages) # add them to the weights_dictionary for client_name, return_vals in vals.items(): client_weights, client_intercepts = return_vals weights[client_name] = np.array(client_weights) intercepts[client_name] = np.array(client_intercepts) weights_np = list( weights.values()) # the weights for this iteration! intercepts_np = list(intercepts.values()) try: averaged_weights = np.average( weights_np, axis=0) # gets rid of security offsets except: raise ValueError( '''DATA INSUFFICIENT: Some client does not have a sample from each class so dimension of weights is incorrect. Make train length per iteration larger for each client to avoid this issue''' ) averaged_intercepts = np.average(intercepts_np, axis=0) self.averaged_weights[ i] = averaged_weights ## averaged weights for this iteration!! self.averaged_intercepts[i] = averaged_intercepts # add time server logic takes server_logic_end = datetime.now() server_logic_time = server_logic_end - server_logic_start simulated_time += server_logic_time with ThreadPool(len(active_clients)) as returning_pool: args = [] for client_name in active_clients: client_instance = self.directory.clients[client_name] body = { 'iteration': i, 'return_weights': averaged_weights, 'return_intercepts': averaged_intercepts, 'simulated_time': simulated_time + config.LATENCY_DICT[self.name][client_name] } message = Message(sender_name=self.name, recipient_name=client_name, body=body) args.append((client_instance, message)) return_messages = returning_pool.map(client_weights_returner, args) simulated_time = find_slowest_time(return_messages) server_logic_start = datetime.now() clients_to_remove = set() for message in return_messages: if message.body[ 'converged'] == True and message.sender not in converged: # converging converged[message.sender] = i # iteration of convergence clients_to_remove.add(message.sender) server_logic_end = datetime.now() server_logic_time = server_logic_end - server_logic_start simulated_time += server_logic_time if config.CLIENT_DROPOUT: # tell the clients which other clients have dropped out active_clients -= clients_to_remove if len( active_clients ) < 2: # no point in continuing if don't have at least 2 clients self.print_convergences(converged) return with ThreadPool(len(active_clients)) as calling_pool: args = [] for client_name in active_clients: client_instance = self.directory.clients[client_name] body = { 'clients_to_remove': clients_to_remove, 'simulated_time': simulated_time + config.LATENCY_DICT[self.name][client_name], 'iteration': i } message = Message(sender_name=self.name, recipient_name=client_name, body=body) args.append((client_instance, message)) __ = calling_pool.map(client_agent_dropout_caller, args) # at end of all iterations self.print_convergences(converged)
def __parseCommentFromApp(comment, commentList, userList, count, messageId, queryUser): # Save comment Message new_comment = Message() new_comment.isComment = True new_comment.parent = messageId new_comment.id = str(messageId) + "." + str(count) # Get all text elements of a comment textList = None try: textList = comment.find_all('div', class_='bc-comment-text') except: logging.info("This thread has no comments") # Save comment text and reply id try: # Find comment text and quoted comment id. if len(textList) > 1: try: new_comment.text = textList[1].text new_comment.replyToMessageId = next( (new_comment for new_comment in commentList if new_comment.text == textList[0].text), None).id except AttributeError: logging.info("Could not find quoted comment" + ", MessageId: " + str(new_comment.id)) # Find only comment text else: try: new_comment.text = textList[0].text except IndexError: logging.info("Could not find comment text" + ", MessageId: " + str(messageId)) except: traceback.print_exc() logging.info("An error occurred reading comment text" + ", MessageId: " + str(messageId)) # Find not identified User new_comment.sender_name = comment.find( 'span', class_='bc-comment-author-name').contents[0].contents[0] # Find identified User and save User ich channel.users and save message.sendername try: identifierName = comment.find( 'span', class_='bc-comment-author-name').contents[0]['href'].rsplit( '/', 1)[-1] except: identifierName = "" user = None # Query user if identifier name was found if not identifierName == "" and queryUser: try: user = Bots.getEntity(identifierName) userList.append(user) except ValueError: # User for some reason not found. pass new_comment.username = identifierName if user is not None: new_comment.sender_name = concat(user.first_name, user.last_name) new_comment.sender = user.id else: commentUser = telethon.types.User(0) commentUser.first_name = new_comment.sender_name new_comment.timestamp = comment.find('time')['datetime'] commentList.append(new_comment)