def unite_forest(self, msg):
"""This fuunction is called by leader process to update the component
of all process in the current forest."""
if msg.component_id > self.component_id:
logging.info("Process %s is setting as non leader.",
str(self.process_id))
self.update_leader(False)
else:
if self.level == msg.level:
self.level += 1
elif self.level < msg.level:
self.level = msg.level
msg = Message()
msg = msg.update_component(self.level, self.component_id)
for edge in self.mst_edges:
edge.send_message(msg, self.process_id)
seen_edges = []
self.log_mst_edges()
while len(seen_edges) != len(self.mst_edges):
for edge in [e for e in self.mst_edges if e not in seen_edges]:
self.messages_lock.acquire()
if "ack_leader" in self.messages[edge.id] and\
len(self.messages[edge.id]["ack_leader"]):
logging.info("acknowledge received from edge: %s", edge.id)
msg = self.messages[edge.id]["ack_leader"].pop()
seen_edges.append(edge)
self.messages_lock.release()
self.round_completed = True
def handle(self, message):
if message.command == "PING":
# respond quick before we get booted!
m = Message()
m.command = "PONG"
m.params = message.params
self.connection.send(m)
elif message.command == "QUIT":
# user left. Log them out if need be
u = db.User(nick=message.sender)
u.logout()
elif message.command == "NICK":
# nick updated. Chase them!
u = db.User(nick=message.sender)
u.change_nick(message.params[0])
elif message.command == "PRIVMSG" and message.params[1].startswith(self.settings["command_prefix"]):
# palm it off to the plugins
for plugin in self.plugins:
if plugin.handle(message):
break
channel = None
if message.params[0].startswith("#"):
channel = message.params[0]
db.log_message(message.command, message.sender, channel, " ".join(message.params))
def xsend(self):
'''
set appid and appkey
使用新的线程 发送短信
'''
message_configs = {}
message_configs['appid'] = self.appid
message_configs['appkey'] = self.appkey
'''
set sign_type,if is set
'''
if self.sign_type != '':
message_configs['sign_type'] = self.sign_type
'''
init mail class
'''
message = Message(message_configs)
'''
build request and send email and return the result
'''
result = message.xsend(self.build_request())
# MessageThread(message).start()
return result
def t_one(self, dest_x, dest_y, self_x, self_y, self_dir, expected):
s = "message {} {} {} {} {} aaa\\".format(
dest_x, dest_y, self_x, self_y, self_dir.value
)
m = Message(s)
action = m.action_to_join_sender()
self.assertTrue(expected == action)
def get_messages(self):
if not self.initialized:
self.error_message = 'Cannot get messages for non initialized Bot'
return []
messages = []
updates = self.bot.get_updates(offset=self.last_update_id).wait()
for update in updates:
# always increase last_update_id to get only new messages next time
self.last_update_id = update.update_id + 1
self.chat_id = update.message.chat.id
# skip all types of message except Message
if not update.message or not isinstance(update.message, Message):
continue
m = Msg()
ok = m.init(update)
if not ok:
self.errors[update.message.text] = m.get_error_message()
continue
messages.append(m)
return messages
def wiki(self, nick, channel, params):
if len(params) < 1:
return;
replyto = nick if channel == None else channel
qs = {
'action': 'query',
'prop': 'revisions',
'rvprop': 'content',
'format': 'xml',
'titles': ' '.join(params)
}
url = 'http://en.wikipedia.org/w/api.php?' + urllib.urlencode(qs)
xml = ElementTree.parse(urllib.urlopen(url))
result = xml.findtext('.//rev')
if result is not None:
result = self.deugly_wikimedia(result)
else:
result = 'Couldn\'t find it :('
m = Message()
m.command = 'PRIVMSG'
m.params = [replyto, result]
self.connection.send(m)
def run_round(self):
"""Runs the current round before rporting back."""
self.round_completed = False
self.merge_mwoe_received = None
self.clear_ret_messages()
self.get_non_mst_edges()
if self.leader:
logging.info("finding mwoe")
mwoe_msg = self.find_MWOE()
logging.info("MWOE details: edge id: %s, weight: %f",
mwoe_msg.edge_id, mwoe_msg.msg)
if mwoe_msg.edge_id is None:
self.completed = True
self.round_completed = True
completed_msg = Message()
completed_msg = completed_msg.get_completed_msg()
for edge in self.mst_edges:
edge.send_message(completed_msg, self.process_id)
return
self.merge_mwoe_received_lock.acquire()
self.merge_mwoe_received = mwoe_msg
self.merge_mwoe_received_lock.release()
# code for leader to send the mwoe request to its child
merge_mwoe_msg = Message()
merge_mwoe_msg = merge_mwoe_msg.merge_MWOE(mwoe_msg.edge_id,
mwoe_msg.process)
self.log_mst_edges()
for edge in self.mst_edges:
edge.send_message(merge_mwoe_msg, self.process_id)
if mwoe_msg.process == self.process_id:
self.send_join_request(self.get_edge(mwoe_msg.edge_id))
self.merge_mwoe(mwoe_msg)
def __init__(self, msg=None, data=None, filename=None, password=None, vals=None, file_obj=None):
self.p = None
self.q = None
self.g = None
self.y = None
self.x = None
if file_obj is not None:
self._from_private_key(file_obj, password)
return
if filename is not None:
self._from_private_key_file(filename, password)
return
if (msg is None) and (data is not None):
msg = Message(data)
if vals is not None:
self.p, self.q, self.g, self.y = vals
else:
if msg is None:
raise SSHException('Key object may not be empty')
if msg.get_string() != 'ssh-dss':
raise SSHException('Invalid key')
self.p = msg.get_mpint()
self.q = msg.get_mpint()
self.g = msg.get_mpint()
self.y = msg.get_mpint()
self.size = util.bit_length(self.p)
class GoodMessage(unittest.TestCase):
def setUp(self):
""" initial setup for testing """
self.msg = Message('Sender', 'Recepient', 'Content')
def testGoodSender(self):
""" tests for good sender. It should pass. """
self.assertTrue(self.msg.isSender())
def testGoodRecipient(self):
""" tests for good recipient. It should pass. """
self.assertTrue(self.msg.isRecipient())
def testGoodContent(self):
""" tests for good content. It should pass. """
self.assertTrue(self.msg.isContent())
def testRealMsg(self):
""" tests for an existed, 'populated' message. It should pass. """
self.assertFalse(self.msg.isEmpty())
@unittest.skip("no use for this test")
def testLargeMsg(self):
""" tests for a large message. It should pass. """
self.assertRaises(Message.OutOfRangeError, self.msg.size, 999999999999)
def testComputeMsgSize(self):
""" tests for size computation. It should pass. """
self.assertTrue(self.msg.computeMsgSize())
def data(self):
msg = Message()
for carrier in self.carriers:
msg.append(Record('urn:nfc:wkt:ac', data=str(carrier)))
if self.error.reason is not None:
msg.append(Record("urn:nfc:wkt:err", data=str(self.error)))
return str(self.version) + str(msg)
def delete_messages(self, mtype = None):
""" Delete messages of a specified kind.
Args:
type: A string of the message type to delete.
Due to timeout issues with App Engine, this method will currently
only succeed when running on App Engine if the number of messages
being deleted is relatively small (~hundreds). It will attempt to
delete up to 1000. The timeout retry wrapper (see
game_server/autoretry_datastore.py) and using keys only search
drastically increases the chances of success, but this method is
still not guaranteed to complete.
For more information see:
http://groups.google.com/group/google-appengine/
browse_thread/thread/ec0800a3ca92fe69?pli=1
http://stackoverflow.com/questions/108822/
delete-all-data-for-a-kind-in-google-app-engine
"""
if mtype:
db.delete(Message.all(keys_only = True).filter('msg_type =', mtype)
.ancestor(self.key()).order('date').fetch(1000))
db.delete(Message.all(keys_only = True).ancestor(self.key()).order('date')
.fetch(1000))
def receiveData(self):
data, addr = self.sock.recvfrom( 1024 )
#prints a string but crashes on OSC msg
#print("received: " + data.decode("utf-8"))
m = Message()
m.parse(data)
return m
def _keepalive_conversation(self):
if self.state != ProtocolState.OPEN:
return True
message = Message()
message.add_header('Content-Type', 'text/x-keep-alive')
self._send_command('MSG', 'N', message, True)
return True
def unicast(self, message, destination):
delay_time = random.uniform(self.min_delay, self.max_delay)
message = Message(self.pid, destination, message)
print(message.send_str())
print('delay unicast with {0:.2f}s '.format(delay_time))
delayed_t = threading.Timer(delay_time, self.__unicast, (message, destination,))
delayed_t.start()
def publish(self):
"""Publish a mbox"""
mbox = Mbox(self.config.get('mbox'))
messages = 0
if not (os.path.exists(self.config.get('dir'))):
os.mkdir(self.config.get('dir'))
message = mbox.nextMessage()
while(message != None):
msg = Message(message, self.config)
messages += 1
self.index.add(msg)
self.subscribers.add(msg)
msg.toRDF()
#msg.toHTML()
message = mbox.nextMessage()
self.index.toRDF()
self.subscribers.toRDF()
return messages
def do_call(*args, **kwargs):
data = {
'name': name,
'args': args,
'kwargs': kwargs,
}
data.update(self.env)
request = Message(data)
# Send request
self.send(request.serialize())
# Receive response
bufs = []
while True:
try:
buf = self.recv()
except ClientException, e:
raise ClientCommunicationError(e)
if not buf:
raise ClientCommunicationError('Server hang up.')
bufs.append(buf)
try:
response, _ = Message.parse("".join(bufs))
except MessageIncomplete:
continue
except MessageInvalid, e:
raise ClientCommunicationError(e)
def handle_message(self, data):
""" message handler. """
m = Message(data)
m.parse(self)
if data.type == 'groupchat' and data.subject:
logging.debug("%s - checking topic" % self.name)
self.topiccheck(m)
nm = Message(m)
callbacks.check(self, nm)
return
if data.get('x').xmlns == 'jabber:x:delay':
logging.warn("%s - ignoring delayed message" % self.name)
return
self.privwait.check(m)
if m.isresponse:
logging.debug("%s - message is a response" % self.name)
return
jid = None
m.origjid = m.jid
for node in m.subelements:
try: m.jid = node.x.item.jid
except (AttributeError, TypeError): continue
if self.me in m.fromm:
logging.debug("%s - message to self .. ignoring" % self.name)
return 0
try:
if m.type == 'error':
if m.code:
logging.error('%s - error - %s' % (self.name, str(m)))
self.errorwait.check(m)
self.errorHandler(m)
except Exception, ex:
handle_exception()
def move(self, state):
"""Use the game state to decide what action to take, and then output
the direction to move."""
self.game.update(state)
message = Message()
message.game_message(self)
self.input_interface.append( message )
self.action = self.decide()
retval = 'Stay'
if ( self.action == 'Heal' ):
if ( None != self.game.board.taverns_list[0].path ):
retval = self.game.board.taverns_list[0].path[0]
elif ( self.action == 'Mine' ):
if ( None != self.game.board.mines_list[0].path ):
retval = self.game.board.mines_list[0].path[0]
elif ( self.action == 'Attack' ):
if ( None != self.game.enemies_list[0].path ):
retval = self.game.enemies_list[0].path[0]
elif ( self.action == 'Wait' ):
retval = 'Stay'
self.expected_pos = self.game.board.to( self.game.hero.pos, retval )
self.prev_mines = self.game.hero.mines
self.prev_gold = self.game.hero.gold
self.prev_life = self.game.hero.life
return retval
def __init__(self):
super(EventProcessor, self).__init__()
self.message_handler = MessageHandler()
#self.gps = GPS()
Message.init(self.message_handler.plugin.address())
Detector.start_plugins()
Reactor.add_plugin_events()
def _keepalive_conversation(self):
if self.state != ProtocolState.OPEN:
return True
message = Message()
message.add_header("Content-Type", "text/x-keep-alive")
self._send_command("MSG", "N", message, True)
return True
def read(self):
try:
message = self.socket.recv(4096)
except Exception as e:
Log.e(getExceptionInfo(e))
return Client.ReadError
if len(message) == 0:
return Client.NoMessage
self.buf += message
while len(self.buf) > 4:
length = int(self.buf[0:4])
if not len(self.buf) >= length + 4:
break
msg = self.buf[4: length + 4]
self.buf = self.buf[length + 4:]
message = Message()
if not message.loads(msg):
Log.w(u'Unknown Message')
else:
self.requests.put(message)
return Client.NewMessage
def _receive_ack(self, msg):
"""Receives an ACK and transitions message's state based on that.
"""
self._remove_message(msg)
idx, parent_msg = self._find_message(token=None, message_id=msg.message_id, state=MessageState.wait_for_ack)
if parent_msg is None:
coap_log.warning('ACK received but no matching message found - Sending RESET')
reset_msg = Message(message_id=msg.message_id, message_type=MessageType.reset)
self._socket.send(reset_msg.build())
return
if parent_msg.type != MessageType.confirmable:
coap_log.error('ACK received for NON-CONFIRMABLE message - Ignoring')
return
self._transition_message(parent_msg, MessageState.wait_for_response)
if msg.class_code == 0 and msg.class_detail == 0:
# if this is empty message send just for ACK we are already done.
coap_log.debug('Separate ACK received')
return
#coap_log.debug('Piggybacked RESPONSE {0}'.format(str(msg)))
if msg.has_observe_option():
self._receive_observe(parent_msg, msg)
return
# This message has a piggybacked response, so receive it.
self._receive_response(parent_msg, msg)
def startRound(self, connection, *args):
game = connection.player.game
if game.state != Game.STATE_FINISHED_ROUND:
return
# if game.master is connection.player:
# game.start()
game.startNewRound()
# message = Message.encode(Message.SEND_GAME_MESSAGE_STARTED, '')
# self.server.sendMessageMultiple(game.players, message)
scores = {}
connections = []
for player in game.players:
connections.append(player.connection)
scores[player.name] = len(player.green_apples)
red_apples = []
for red_apple in player.red_apples:
red_apples.append(red_apple.dictionary)
message = Message.encode(Message.SEND_GAME_MESSAGE_PLAYER_RED_APPLES, red_apples)
self.server.sendMessageSingle(player.connection, message)
round_details = {}
round_details['JUDGE'] = game.players[game.judge].name
round_details['SCORES'] = scores
round_details['GREEN_APPLE'] = game.active_green_apple.dictionary
message = Message.encode(Message.SEND_GAME_MESSAGE_ROUND_DETAILS, round_details)
self.server.sendMessageMultiple(connections, message)
def test_multiple_packet_message(self):
body = "1a0eeb5701b202010a0101a000340034170d000208000186a0000000000000".decode('hex')
msg = Message("1f07b1ee", 14, 0x18, body)
packets = msg.packets()
self.assertEqual(len(packets), 2)
self.assertEqual(packets[0].raw_hex(), "1f07b1eeae1f07b1ee181f1a0eeb5701b202010a0101a000340034170d000208000186a019")
self.assertEqual(packets[1].raw_hex(), "1f07b1ee900000000000000251e2")
def _parse_kexdh_gex_request(self, m):
minbits = m.get_int()
preferredbits = m.get_int()
maxbits = m.get_int()
# smoosh the user's preferred size into our own limits
if preferredbits > self.max_bits:
preferredbits = self.max_bits
if preferredbits < self.min_bits:
preferredbits = self.min_bits
# fix min/max if they're inconsistent. technically, we could just pout
# and hang up, but there's no harm in giving them the benefit of the
# doubt and just picking a bitsize for them.
if minbits > preferredbits:
minbits = preferredbits
if maxbits < preferredbits:
maxbits = preferredbits
# now save a copy
self.min_bits = minbits
self.preferred_bits = preferredbits
self.max_bits = maxbits
# generate prime
pack = self.transport._get_modulus_pack()
if pack is None:
raise SSHException('Can\'t do server-side gex with no modulus pack')
self.transport._log(DEBUG, 'Picking p (%d <= %d <= %d bits)' % (minbits, preferredbits, maxbits))
self.g, self.p = pack.get_modulus(minbits, preferredbits, maxbits)
m = Message()
m.add_byte(chr(_MSG_KEXDH_GEX_GROUP))
m.add_mpint(self.p)
m.add_mpint(self.g)
self.transport._send_message(m)
self.transport._expect_packet(_MSG_KEXDH_GEX_INIT)
def unicastTCP(self, serverID, message):
delay_time = random.uniform(self.min_delay, self.max_delay)
m = Message(self.pid, serverID, message)
print(m.send_str())
print('delay unicastTCP with {0:.2f}s '.format(delay_time))
delayed_t = threading.Timer(delay_time, self.__unicastTCP, (message,))
delayed_t.start()
def handle_request(self, request):
args = {}
for k, v in request.args.items():
args.update({k: v[0]},)
if len(request.postpath[0]) and not args.get('callback'):
request.setResponseCode(400)
return HTML
if 'register' in request.postpath:
register = Register(request, args)
return register.response()
if 'login' in request.postpath:
login = Login(request, args)
return login.response()
if 'poll' in request.postpath:
poll = Poll(request, args)
return poll.response()
if 'message' in request.postpath:
message = Message(request, args)
return message.response()
if 'join' in request.postpath:
join = Join(request, args)
return join.response()
else:
return """
def candidate_receive_message(self, msg):
"""
All Candidate Message Receiving
@:param msg - the JSON message received
@:return: Void
"""
if msg['type'] == 'vote':
#print str(self.id) + ": Got Vote Message----------"
message = Message.create_message_from_json(msg)
self.receive_vote(message)
if msg['type'] == 'heartbeat':
#print str(self.id) + "got ~~~HEARTBEAT~~~"
heart_beat = Message.create_message_from_json(msg)
if heart_beat.term >= self.current_term:
self.current_term = heart_beat.term
self.become_follower(heart_beat.leader)
if msg['type'] == 'voteRequest':
message = Message.create_message_from_json(msg)
if message.term > self.current_term:
self.become_follower(msg['leader'])
if msg['type'] in ['get', 'put']:
message = Message.create_message_from_json(msg)
redirect_message = message.create_redirect_message("FFFF")
self.send(redirect_message)
def create_msg_adiumLog(self):
# Logs of the format:
# (00:07:25)fetchgreebledonx:Do you know how you replace spaces in a string with nothing?
# Logs are passed to TXTMessageParser, which returns a list of dicts with
# timestamp, message_type, message, and sender
# message_type will be either message or sender, and for now we ignore status
date = self.filename.split("(")[1].split(")")[0].replace("|", "-")
try:
f = open(self.filename, "r")
data = f.read()
f.close()
p = TXTMessageParser()
p.parse(data)
for msg in [f for f in p.messages if f["message_type"] in ["message"]]:
m = Message(
msg["message"],
self.account,
self.contact,
msg["sender"],
self.service,
date + " " + msg["timestamp"],
)
m.insert(self.connection)
except IOError:
print "Error: %s" % self.filename
def exchange(self, vault=None):
'''
First thing first, the initiator issues a "exchange request".
In Cyclon protocol, the exchange request consists in generating a
subset of peers to exchange with the oldest peer. The "age" of a peer
is based on the how many exchanges have passed since I last updated
the state of that peer of my neighboring list.
'''
if self.itsExchanging():
return True
Cyclon.peerList.increaseAges()
try:
oldestPeer = Cyclon.peerList.getOldest()
except IndexError:
print ' * No peer available for exchanging'
return True
print '''Starting an exchange with ''' + oldestPeer.getName()
self.exchangeStarted()
print >> stderr, ''' * computing subset '''
shuffleSize = randint(1, self.getShuffleLength()) - 1
randomSubset = Cyclon.peerList.drop(oldestPeer).getSubset(shuffleSize)
sentSet = CyclonPeerList([ oldestPeer ]) + randomSubset
message = Message(kind='exchange',
source=Cyclon.myself.getName(),
destination=oldestPeer.getName())
randomSubset.insert(0, Cyclon.myself)
body = message.getBody()
print >> stderr, ''' * building message '''
for p in randomSubset:
node = body.appendChild(message.createElement('node'))
node.setAttribute('name', p.getName())
node.childNodes = p.getConnectionsXML()
print >> stderr, ''' * Sending request '''
message.send(
oldestPeer.transport, 3, 3
).addCallback(
self.exchangeReplyReceived, sentSet=sentSet
).addErrback(
self.removePeerErrback, peer=oldestPeer.getName(), success=False
).addErrback(
self.exchangeFinished, success=False
).addErrback(
self.exchange)
return True
sys.exit(0)
serverIp = sys.argv[1]
serverPort = 5425
filepath = sys.argv[2]
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # TCP 소켓을 생성한다.
try:
print("서버:{0}/{1}".format(serverIp, serverPort))
sock.connect((serverIp, serverPort)) # 접속 요청을 수락한다.
msgId = 0
reqMsg = Message()
filesize = os.path.getsize(filepath)
reqMsg.Body = BodyRequest(None)
reqMsg.Body.FILESIZE = filesize
reqMsg.Body.FILENAME = os.path.basename(filepath)
msgId += 1
reqMsg.Header = Header(None)
reqMsg.Header.MSGID = msgId
reqMsg.Header.MSGTYPE = message.REQ_FILE_SEND
reqMsg.Header.BODYLEN = reqMsg.Body.GetSize()
reqMsg.Header.FRAGMENTED = message.NOT_FRAGMENTED
reqMsg.Header.LASTMSG = message.LASTMSG
reqMsg.Header.SEQ = 0
MessageUtil.send(sock, reqMsg) # 클라이언트는 서버에 접속하자마자 파일 전송 요청 메세지를 보낸다.
data = self.sock.recv(16)
amount_received += len(data)
print('received "%s"' % data)
finally:
if (close_connection):
print("Closing socket")
self.sock.close()
print("Socket closed")
if __name__ == '__main__':
while True:
MESSAGE_SENDER = MessageSender("config.xml")
MESSAGE = Message("Test", \
"1", \
MESSAGE_SENDER.config.sensors[0].sensor_id, \
MESSAGE_SENDER.config.sensors[0].sensor_type, \
MESSAGE_SENDER.config.sensors[0].sensor_data_type, \
MESSAGE_SENDER.config.sensors[0].sensor_interval)
MESSAGE_SENDER.send_message(MESSAGE)
time.sleep(0.500)
# # Set up device readings
# tempReading = {'machineId':'001','sensorId':'001','sensorType':'Temp','Values':'30.1'}
# message = json.dumps(tempReading)
# message = message + '<EOF>'
# print ('sending "%s"' % message)
# # Create a TCP/)IP socket
# sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# # Connect the socket to the port where the server is listening
# server_address = ('localhost', 8010)
def nospace_wrapper(_):
"""Wrapper for nospace function."""
return Message(nospace(_['T']))
@command('nospace', __name__, help='Remove whitespaces from a string.',
usage='<text>')
def nospace_wrapper(_):
"""Wrapper for nospace function."""
return Message(nospace(_['T']))
@command('unspace', __name__, help='Remove extra whitespaces from a string.',
usage='<text>')
def unspace_wrapper(_):
"""Wrapper for unspace function."""
return Message(unspace(_['T']))
command('len', __name__, help='Return the length of a string.',
usage='<text>')(lambda _: Message(len(_['T'])))
def nospace(s):
"""Remove whitespaces from a string."""
return s.replace(' ', '')
def unspace(s):
"""Remove extra whitespaces from a string."""
# 'a b _ c d' to 'ab cd'
if max([len(x) for x in s.split()]) > 1:
return s
n = []
m = 0
for c in s:
def scrapeCommentsBot(url, channel_users, messageId):
""" Scrapes https://comments.bot/ website for comment feature extension."""
hdr = {
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Referer': 'https://cssspritegenerator.com',
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
'Accept-Encoding': 'none',
'Accept-Language': 'en-US,en;q=0.8',
'Connection': 'keep-alive'
}
soup = Bots.javaScriptLoadMoreHack(url, "bot")
if soup is None:
try:
req = Request(url, headers=hdr)
page = urlopen(req)
soup = BeautifulSoup(page, 'html.parser')
except:
logging.info("Could not open comments app.")
commentList = []
if soup is not None:
comments = soup.find_all('div', class_='comment-content')
count = 1
for comment in comments:
# Save comment Message
new_comment = Message()
new_comment.isComment = True
new_comment.parent = messageId
# Save Id
new_comment.id = str(messageId) + "." + str(count)
# Save Text
text_list = comment.find('div', class_='comment-text').contents
text_list = [
elem for elem in text_list if str(elem) != "<br/>"
] # Eliminate Html
new_comment.text = eliminateWhitespaces(' '.join(
[str(elem) for elem in text_list]))
# Save replyId
reply_text_list = comment.find('span',
class_='comment-reply-text')
if reply_text_list is not None:
reply_text = eliminateWhitespaces(' '.join(
[str(elem) for elem in reply_text_list]))
try:
new_comment.replyToMessageId = next(
(new_comment for new_comment in commentList
if new_comment.text == reply_text), None).id
except:
logging.info("Could not find quoted comment in Bot: " +
url)
count = count + 1
# Find not identified User
new_comment.sender_name = comment.find(
'div', class_='name-row').findChildren()[0].contents[0]
commentUser = telethon.types.User(0)
commentUser.first_name = new_comment.sender_name
channel_users.append(commentUser)
new_comment.timestamp = comment.find(
'div', class_='comment-date').findChildren()[0].contents[0]
# No identified Users
commentList.append(new_comment)
return commentList
def _update_cached_message_flags(self, highest_mod_seq, event_data_list):
# type: (int, t.List[AbstractEventData]) -> None
"""Update the flags on any cached messages.
"""
# we just check the highestmodseq and revert to full sync if they don't match
# this is kind of what thunderbird does https://wiki.mozilla.org/Thunderbird:IMAP_RFC_4551_Implementation
if highest_mod_seq is not None:
if self._highest_mod_seq == highest_mod_seq:
logger.debug("%s matching highest mod seq no flag update" % self)
return
logger.debug("%s started updating flags" % self)
# get all the flags for the old messages
fetch_data = self._imap_client.fetch('1:%d' % (self._last_seen_uid), [
'FLAGS']) # type: t.Dict[int, t.Dict[str, t.Any]]
# update flags in the cache
for message_schema in MessageSchema.objects.filter(folder_schema=self._schema):
assert isinstance(message_schema, MessageSchema)
# ignore cached messages that we just fetched
if message_schema.uid > self._last_seen_uid:
continue
# if we don't get any information about the message we have to remove it from the cache
if message_schema.uid not in fetch_data:
message_schema.delete()
logger.debug("%s deleted message with uid %d" %
(self, message_schema.uid))
continue
message_data = fetch_data[message_schema.uid]
# TODO make this more DRY
if 'SEQ' not in message_data:
logger.critical('Missing SEQ in message data')
logger.critical('Message data %s' % message_data)
continue
if 'FLAGS' not in message_data:
logger.critical('Missing FLAGS in message data')
logger.critical('Message data %s' % message_data)
continue
old_flags = set(message_schema.flags)
new_flags = set(message_data['FLAGS'])
if old_flags - new_flags:
# flag removed old flag exists which does not exist in new flags
event_data_list.append(RemovedFlagsData(Message(message_schema, self._imap_client), list(old_flags - new_flags)))
elif new_flags - old_flags:
# flag added, new flags exists which does not exist in old flags
event_data_list.append(NewFlagsData(Message(message_schema, self._imap_client), list(new_flags - old_flags)))
message_schema.flags = list(new_flags)
message_schema.msn = message_data['SEQ']
message_schema.save()
# TODO maybe trigger the user
logger.debug("%s updated flags" % self)
if highest_mod_seq is not None:
self._highest_mod_seq = highest_mod_seq
logger.debug("%s updated highest mod seq to %d" % (self, highest_mod_seq))
return True
if __name__ == '__main__':
server = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
server.bind(("", 0))
print("Server started at 0.0.0.0 port " + str(server.getsockname()[1]))
ctx = {}
plant = Plant(ctx)
workshop = Workshop(ctx)
designer = Designer(ctx)
plant.launch()
workshop.launch()
designer.launch()
# On dit à notre Logistics qu'on se connecte
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.sendto(pickle.dumps(Message(Message.CONNECTED, server.getsockname()[1])), ('localhost', 33333))
print(f"ctx:{ctx}")
while True:
data, addr = server.recvfrom(1024)
print(f"Data recv: {data}")
msg = pickle.loads(data)
plant.parent_pipe.send(msg)
import socket
import select
import sys
from message import Message
import pickle
host = '127.0.0.1'
port = 12345
mySocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
mySocket.connect((host, port))
_name = str(sys.argv[1])
m = Message(_name, "")
while True:
sockets_list = [sys.stdin, mySocket]
read_sockets, write_socket, error_socket = select.select(
sockets_list, [], [])
"""
There are two possible input situations. Either the
# user wants to give manual input to send to other people,
# or the server is sending a message to be printed on the
# screen. Select returns from sockets_list, the stream that
# is reader for input. So for example, if the server wants
# to send a message, then the if condition will hold true
# below.If the user wants to send a message, the else
# condition will evaluate as true
def run(message, matches, chat_id, step):
return [
Message(chat_id).set_text(
"🎗*Butely Bot*🎗\nbased on _telepot_\nfeedback:@Amrrezaebrahimi\nMade whit ❤️ in 🇮🇷",
parse_mode="markdown")
]
def _save_new_messages(self, last_seen_uid, event_data_list = None, urgent=False):
# type: (int, t.List[AbstractEventData]) -> None
"""Save any messages we haven't seen before
Args:
last_seen_uid (int): the max uid we have stored, should be 0 if there are no messages stored.
urgent (bool): if True, save only one email
"""
# add thread id to the descriptors if there is a thread id
descriptors = list(Message._descriptors) + ['X-GM-THRID'] if self._imap_account.is_gmail \
else list(Message._descriptors)
uid_criteria = ""
if urgent:
uid_criteria = '%d' % (last_seen_uid + 1)
else:
uid_criteria = '%d:*' % (last_seen_uid + 1)
fetch_data = self._imap_client.fetch(
uid_criteria, descriptors)
# seperate fetch in order to detect if the message is already read or not
header_data = self._imap_client.fetch(
uid_criteria, list(Message._header_descriptors))
# if there is only one item in the return field
# and we already have it in our database
# delete it to be safe and save it again
# TODO not sure why this happens maybe the folder._uid_next isn't getting updated properly
if len(fetch_data) == 1 and last_seen_uid in fetch_data:
already_saved = MessageSchema.objects.filter(folder_schema=self._schema, uid=last_seen_uid)
if already_saved:
logger.critical("%s found already saved message, deleting it" % self)
already_saved[0].delete()
logger.info("%s saving new messages" % (self))
for uid in fetch_data:
message_data = fetch_data[uid]
header = header_data[uid]
logger.debug("Message %d data: %s" % (uid, message_data))
if 'SEQ' not in message_data:
logger.critical('Missing SEQ in message data')
logger.critical('Message data %s' % message_data)
continue
if 'FLAGS' not in message_data:
logger.critical('Missing FLAGS in message data')
logger.critical('Message data %s' % message_data)
continue
if 'INTERNALDATE' not in message_data:
logger.critical('Missing INTERNALDATE in message data')
logger.critical('Message data %s' % message_data)
continue
if 'RFC822.SIZE' not in message_data:
logger.critical('Missing RFC822.SIZE in message data')
logger.critical('Message data %s' % message_data)
continue
# if 'ENVELOPE' not in message_data:
# logger.critical('Missing ENVELOPE in message data')
# logger.critical('Message data %s' % message_data)
# continue
if self._imap_account.is_gmail and 'X-GM-THRID' not in message_data:
logger.critical('Missing X-GM-THRID in message data')
logger.critical('Message data %s' % message_data)
continue
# check for supported thread algorithms here
if not self._imap_account.is_gmail:
capabilities = self._imap_client.capabilities()
capabilities = list(capabilities)
capabilities = filter(lambda cap: 'THREAD=' in cap, capabilities)
capabilities = [cap.replace('THREAD=', '') for cap in capabilities]
logger.critical("Add support for one of the following threading algorithms %s" % capabilities)
raise NotImplementedError("Unsupported threading algorithm")
# this is the date the message was received by the server
internal_date = message_data['INTERNALDATE'] # type: datetime
msn = message_data['SEQ']
flags = message_data['FLAGS']
if "\\Seen" not in flags:
self._imap_client.remove_flags(uid, ['\\Seen'])
# header = [h.replace('\r\n\t', ' ') for h in header]
header = header[list(Message._header_descriptors)[0]]
header = header.replace('\r\n\t', ' ')
header = header.replace('\r\n', ' ')
meta_data = {}
# figure out text encoding issue here
# logger.info(header[uid][list(Message._header_descriptors)[0]])
try:
header = self._parse_email_header(header)
except Exception as e:
logger.critical("header parsing problem %s %s, skip this message" % (header, e))
continue
try:
f_tmp = ""
header_field = ['Subject:', 'From:', 'To:', 'Cc:', 'CC:', 'Bcc:', 'Date:', 'In-Reply-To:', 'Message-Id:', 'Message-ID:', 'Message-id:']
for v in re.split('('+ "|".join(header_field) +')', header):
if not v:
continue
if v.strip() in header_field:
# Remove a colon and add to a dict
f_tmp = v[:-1].lower().strip()
else:
meta_data[f_tmp] = v.strip()
except Exception as e:
logger.critical("header parsing problem %s, skip this message" % e)
continue
# if we have a gm_thread_id set thread_schema to the proper thread
gm_thread_id = message_data.get('X-GM-THRID')
thread_schema = None
if gm_thread_id is not None:
result = self._imap_client.search(['X-GM-THRID', gm_thread_id])
logger.debug("thread messages %s, current message %d" % (result, uid))
thread_schema = self._find_or_create_thread(gm_thread_id)
logger.debug("message %d envelope %s" % (uid, meta_data))
try:
if internal_date.tzinfo is None or internal_date.tzinfo.utcoffset(internal_date) is None:
internal_date = timezone('US/Eastern').localize(internal_date)
# logger.critical("convert navie %s " % internal_date)
except Exception:
logger.critical("Internal date parsing error %s" % internal_date)
continue
try:
date = parser.parse(meta_data["date"])
# if date is naive then reinforce timezone
if date.tzinfo is None or date.tzinfo.utcoffset(date) is None:
date = timezone('US/Eastern').localize(date)
except Exception:
if "date" in meta_data:
logger.critical("Can't parse date %s, skip this message" % meta_data["date"])
continue
else:
date = internal_date
logger.info("Date not exist, put internal date instead")
# TODO seems like bulk email often not have a message-id
if "message-id" not in meta_data:
logger.critical("message-id not exist, skil this message %s" % meta_data)
continue
# create and save the message schema
message_schema = MessageSchema(imap_account=self._schema.imap_account,
folder_schema=self._schema,
uid=uid,
msn=msn,
flags=flags,
date=date,
subject="" if "subject" not in meta_data else meta_data['subject'],
message_id=meta_data["message-id"],
internal_date=internal_date,
_thread=thread_schema
)
if "from" in meta_data:
message_schema.from_m = self._find_or_create_contacts(meta_data['from'])[0]
# if envelope.from_ is not None:
# message_schema.from_m = self._find_or_create_contacts(envelope.from_)[0]
try:
message_schema.save()
except Exception as e:
logger.critical("%s failed to save message %d" % (self, uid))
logger.critical("%s stored last_seen_uid %d, passed last_seen_uid %d" % (self, self._last_seen_uid, last_seen_uid))
logger.critical("number of messages returned %d" % (len(fetch_data)))
# to prevent dup saved email
continue
if last_seen_uid != 0 and event_data_list is not None:
logger.critical(internal_date)
if tz.now() - internal_date < timedelta(seconds=5*60):
event_data_list.append(NewMessageData(Message(message_schema, self._imap_client)))
logger.debug("%s finished saving new messages..:" % self)
# create and save the message contacts
if "reply-to" in meta_data:
message_schema.reply_to.add(*self._find_or_create_contacts(meta_data["reply-to"]))
if "to" in meta_data:
message_schema.to.add(*self._find_or_create_contacts(meta_data["to"]))
if "cc" in meta_data:
message_schema.cc.add(*self._find_or_create_contacts(meta_data["cc"]))
if "bcc" in meta_data:
message_schema.bcc.add(*self._find_or_create_contacts(meta_data["bcc"]))
logger.debug("%s saved new message with uid %d" % (self, uid))
def sign_ssh_data(self, data):
m = Message()
m.add_string("ssh-ed25519")
m.add_string(self._signing_key.sign(data).signature)
return m
def _parse_signing_key_data(self, data, password):
from transport import Transport
# We may eventually want this to be usable for other key types, as
# OpenSSH moves to it, but for now this is just for Ed25519 keys.
# This format is described here:
# https://github.com/openssh/openssh-portable/blob/master/PROTOCOL.key
# The description isn't totally complete, and I had to refer to the
# source for a full implementation.
message = Message(data)
if message.get_bytes(len(OPENSSH_AUTH_MAGIC)) != OPENSSH_AUTH_MAGIC:
raise SSHException("Invalid key")
ciphername = message.get_text()
kdfname = message.get_text()
kdfoptions = message.get_binary()
num_keys = message.get_int()
if kdfname == "none":
# kdfname of "none" must have an empty kdfoptions, the ciphername
# must be "none"
if kdfoptions or ciphername != "none":
raise SSHException("Invalid key")
elif kdfname == "bcrypt":
if not password:
raise PasswordRequiredException(
"Private key file is encrypted")
kdf = Message(kdfoptions)
bcrypt_salt = kdf.get_binary()
bcrypt_rounds = kdf.get_int()
else:
raise SSHException("Invalid key")
if ciphername != "none" and ciphername not in Transport._cipher_info:
raise SSHException("Invalid key")
public_keys = []
for _ in range(num_keys):
pubkey = Message(message.get_binary())
if pubkey.get_text() != "ssh-ed25519":
raise SSHException("Invalid key")
public_keys.append(pubkey.get_binary())
private_ciphertext = message.get_binary()
if ciphername == "none":
private_data = private_ciphertext
else:
cipher = Transport._cipher_info[ciphername]
key = bcrypt.kdf(
password=b(password),
salt=bcrypt_salt,
desired_key_bytes=cipher["key-size"] + cipher["block-size"],
rounds=bcrypt_rounds,
# We can't control how many rounds are on disk, so no sense
# warning about it.
ignore_few_rounds=True,
)
decryptor = Cipher(cipher["class"](key[:cipher["key-size"]]),
cipher["mode"](key[cipher["key-size"]:]),
backend=default_backend()).decryptor()
private_data = (decryptor.update(private_ciphertext) +
decryptor.finalize())
message = Message(unpad(private_data))
if message.get_int() != message.get_int():
raise SSHException("Invalid key")
signing_keys = []
for i in range(num_keys):
if message.get_text() != "ssh-ed25519":
raise SSHException("Invalid key")
# A copy of the public key, again, ignore.
public = message.get_binary()
key_data = message.get_binary()
# The second half of the key data is yet another copy of the public
# key...
signing_key = nacl.signing.SigningKey(key_data[:32])
# Verify that all the public keys are the same...
assert (signing_key.verify_key.encode() == public == public_keys[i]
== key_data[32:])
signing_keys.append(signing_key)
# Comment, ignore.
message.get_binary()
if len(signing_keys) != 1:
raise SSHException("Invalid key")
return signing_keys[0]
def _valid_message(msg):
return Message.addrs(msg.id) is None and not msg.has_expired()
async def run(message, matches, chat_id, step):
if not is_group(message):
from_id = message['from']['id']
if step == 0:
user_steps[from_id] = {"name": "Soundcloud", "step": 1, "data": {}}
hide_keyboard = {'hide_keyboard': True, "selective": True}
return [
Message(chat_id).set_text(
"لطفا اسم موزیک یا نام خواننده را وارد کنید🤔",
reply_to_message_id=message['message_id'],
reply_markup=hide_keyboard)
]
if step == 1:
await sender(
Message(chat_id).set_text(
"اومم یکم صبر کن برم جستجو کنم ببینم چی گیرم میاد 😘😬",
parse_mode="markdown"))
user_steps[from_id] = {"name": "Soundcloud", "step": 2, "data": {}}
i = 0
show_keyboard = {'keyboard': [], "selective": True}
ttt = message['text'].replace(" ", "+")
for song in await search(ttt):
title, link, cover = song[0], song[1], song[2]
user_steps[from_id]['data'][title] = link
show_keyboard['keyboard'].append([title])
i += 1
if i == 20:
break
if len(show_keyboard['keyboard']) in [0, 1]:
hide_keyboard = {'hide_keyboard': True, 'selective': True}
del user_steps[from_id]
return [
Message(chat_id).set_text(
"چیزی نیافتم 😣",
reply_to_message_id=message['message_id'],
reply_markup=hide_keyboard,
parse_mode="markdown")
]
return [
Message(chat_id).set_text(
"یکی از اینارو انتخاب کن 🙃👇🏻",
reply_to_message_id=message['message_id'],
reply_markup=show_keyboard)
]
elif step == 2:
try:
await sender(
Message(chat_id).set_text(
"خب یکم صبر کن تا موزیکو دانلود کنم و برات بفرستم 🤓",
reply_to_message_id=message['message_id'],
reply_markup=key,
parse_mode="markdown"))
await downloader(
await
getfile(user_steps[from_id]['data'][message['text']]),
"tmp/{}.mp3".format(message['text']))
del user_steps[from_id]
bot.sendAudio(chat_id,
open("tmp/{}.mp3".format(message['text']), 'rb'),
title=message['text'],
performer="@SpntaBot")
os.remove("tmp/{}.mp3".format(message['text']))
except Exception as e:
del user_steps[from_id]
return [
Message(chat_id).set_text("داری اشتباه میزنی 😕",
parse_mode="markdown")
]
def build():
return Message({'@type': TrustPing.PING, '@id': str(uuid.uuid4())})
def broadcast(self, msg):
Message.register(msg.id, {msg.event_creator(), msg.sender})
self.plugin.broadcast(msg)
def _handle_UBM(self, command):
idx, network_id, account = self.__parse_network_and_account(command)
contact = self.__search_account(account, network_id)
if contact is not None:
message = Message(contact, command.payload)
self.emit("unmanaged-message-received", contact, message)
def setUp(self) -> None:
self.message_row = ['0.000224', 809, 0, 8, "07 a7 7f 8c 11 2f 00 10"]
self.message_object = Message(0.000224, 809, 0, 8, bytearray(b'\x07\xa7\x7f\x8c\x11/\x00\x10'))
def default(self, message):
response = Message(message)
response.status = 401
response.params['res'] = 'Invalid operation for state!'
self.context.socket.send(pickle.dumps(response))
def _handle_MSG(self, command):
message = Message(None, command.payload)
content_type = message.content_type
if content_type[0] == 'text/x-msmsgsprofile':
profile = {}
lines = command.payload.split("\r\n")
for line in lines:
line = line.strip()
if line:
name, value = line.split(":", 1)
profile[name] = value.strip()
self._client.profile._server_property_changed("profile", profile)
self.set_privacy(self._client.profile.privacy)
self._state = ProtocolState.SYNCHRONIZING
self._client.address_book.sync()
elif content_type[0] in \
('text/x-msmsgsinitialmdatanotification', \
'text/x-msmsgsoimnotification'):
if self._client.oim_box is not None:
self._client.oim_box._state = \
OIM.OfflineMessagesBoxState.NOT_SYNCHRONIZED
m = HTTPMessage()
m.parse(message.body)
mail_data = m.get_header('Mail-Data').strip()
if mail_data == 'too-large':
mail_data = None
self._client.oim_box.sync(mail_data)
if mail_data and \
content_type[0] == 'text/x-msmsgsinitialmdatanotification':
#Initial mail
start = mail_data.find('<IU>') + 4
end = mail_data.find('</IU>')
if start < end:
mailbox_unread = int(mail_data[start:end])
self._client.mailbox._initial_set(mailbox_unread)
elif content_type[0] == 'text/x-msmsgsinitialemailnotification':
#Initial mail (obsolete by MSNP11)
pass
elif content_type[0] == 'text/x-msmsgsemailnotification':
#New mail
m = HTTPMessage()
m.parse(message.body)
name = decode_rfc2047_string(m.get_header('From'))
address = m.get_header('From-Addr')
subject = decode_rfc2047_string(m.get_header('Subject'))
message_url = m.get_header('Message-URL')
post_url = m.get_header('Post-URL')
post_id = m.get_header('id')
dest = m.get_header('Dest-Folder')
if dest == 'ACTIVE':
self._client.mailbox._unread_mail_increased(1)
build = self._build_url_post_data
post_url, form_data = build(message_url, post_url, post_id)
self._client.mailbox._new_mail(name, address, subject,
post_url, form_data)
elif content_type[0] == 'text/x-msmsgsactivemailnotification':
#Movement of unread mail
m = HTTPMessage()
m.parse(message.body)
src = m.get_header('Src-Folder')
dest = m.get_header('Dest-Folder')
delta = int(m.get_header('Message-Delta'))
if src == 'ACTIVE':
self._client.mailbox._unread_mail_decreased(delta)
elif dest == 'ACTIVE':
self._client.mailbox._unread_mail_increased(delta)
def conf(self, message):
response = Message('CONFREPLY')
try:
transf_id = int(message.params['idtransf'])
status = message.params['status']
transf = self.context.server.transactions[transf_id]
if transf:
if status.upper() == 'APPROVED' or status.upper(
) == 'REPROVED':
if self.context.client.username == transf[
'destiny'].username:
if transf['status'] == 'pending':
response.status = 200
if status.upper() == 'APPROVED':
response.params[
'res'] = 'Transaction approved!'
elif status.upper() == 'REPROVED':
response.params[
'res'] = 'Transaction reproved!'
transf['status'] = status.lower()
transf['origin'].semaphore.release()
else:
response.status = 400
response.params[
'res'] = 'This transaction is already confirmed!'
else:
response.status = 400
response.params[
'res'] = 'This transaction is not for you!'
else:
response.status = 400
response.params['res'] = 'Invalid status!'
else:
response.status = 400
response.params['res'] = 'This transaction do not exists!'
except Exception as e:
response.status = 500
response.params['res'] = 'Internal Server Error!'
self.context.socket.send(pickle.dumps(response))
def sair(self, message):
response = Message('SAIRREPLY')
response.status = 200
response.params['res'] = 'Connection is going to close!'
self.context.socket.send(pickle.dumps(response))
def saq(self, message):
response = Message('SAQREPLY')
try:
quantity = float(message.params['qtd'])
if not quantity:
response.status = 400
response.params['res'] = 'Param qtd invalid!'
elif self.context.client.wallet.balance < quantity:
response.status = 400
response.params['res'] = 'Balance insufficient!'
else:
balance_before = self.context.client.wallet.balance
self.context.client.wallet.balance -= quantity
response.status = 200
response.params['balancebefore'] = balance_before
response.params[
'balanceafter'] = self.context.client.wallet.balance
except Exception as e:
response.status = 500
response.params['res'] = 'Internal Server Error!'
self.context.socket.send(pickle.dumps(response))
def transf(self, message):
response = Message('TRANSFREPLY')
try:
quantity = float(message.params['amount'])
client = self.context.server.find_user(message.params['destiny'])
if not quantity or not client:
response.status = 400
response.params['res'] = 'Param amount or destiny invalid!'
elif self.context.client.wallet.balance < quantity:
response.status = 400
response.params['res'] = 'Balance insufficient!'
else:
id = len(self.context.server.transactions)
self.context.server.transactions[id] = \
{
'origin': self.context.client,
'destiny': client,
'amount': quantity,
'status': 'pending'
}
self.context.client.semaphore.acquire()
self.context.client.semaphore.acquire()
response.status = 200
transf = self.context.server.transactions[id]
if transf['status'] == 'approved':
self.context.client.wallet.balance -= quantity
client.wallet.balance += quantity
response.params['res'] = 'Transaction approved!'
response.params[
'balanceafter'] = self.context.client.wallet.balance
elif transf['status'] == 'reproved':
response.params['res'] = 'Transaction reproved!'
response.params[
'balanceafter'] = self.context.client.wallet.balance
self.context.client.semaphore.release()
except Exception as e:
response.status = 500
response.params['res'] = 'Internal Server Error!'
self.context.socket.send(pickle.dumps(response))
def mail(self, prefetch=False, **kwargs):
search = ['ALL']
kwargs.get('read') and search.append('SEEN')
kwargs.get('unread') and search.append('UNSEEN')
kwargs.get('starred') and search.append('FLAGGED')
kwargs.get('unstarred') and search.append('UNFLAGGED')
kwargs.get('deleted') and search.append('DELETED')
kwargs.get('undeleted') and search.append('UNDELETED')
kwargs.get('draft') and search.append('DRAFT')
kwargs.get('undraft') and search.append('UNDRAFT')
kwargs.get('before') and search.extend(
['BEFORE',
kwargs.get('before').strftime(self.date_format)])
kwargs.get('after') and search.extend(
['SINCE', kwargs.get('after').strftime(self.date_format)])
kwargs.get('on') and search.extend(
['ON', kwargs.get('on').strftime(self.date_format)])
kwargs.get('header') and search.extend(
['HEADER',
kwargs.get('header')[0],
kwargs.get('header')[1]])
kwargs.get('sender') and search.extend(['FROM', kwargs.get('sender')])
kwargs.get('fr') and search.extend(['FROM', kwargs.get('fr')])
kwargs.get('to') and search.extend(['TO', kwargs.get('to')])
kwargs.get('cc') and search.extend(['CC', kwargs.get('cc')])
kwargs.get('subject') and search.extend(
['SUBJECT', kwargs.get('subject')])
kwargs.get('body') and search.extend(['BODY', kwargs.get('body')])
kwargs.get('label') and search.extend(
['X-GM-LABELS', kwargs.get('label')])
kwargs.get('attachment') and search.extend(['HAS', 'attachment'])
kwargs.get('query') and search.extend([kwargs.get('query')])
emails = []
# print search
response, data = self.gmail.imap.uid('SEARCH', *search)
if response == 'OK':
uids = filter(None, data[0].split(' ')) # filter out empty strings
for uid in uids:
if not self.messages.get(uid):
self.messages[uid] = Message(self, uid)
emails.append(self.messages[uid])
if prefetch and emails:
messages_dict = {}
for email in emails:
messages_dict[email.uid] = email
self.messages.update(
self.gmail.fetch_multiple_messages(messages_dict))
return emails
def __init__(self, src, req_id):
Message.__init__(self, src, None)
# The id that identifies the request itself
self.req_id = req_id
def test_extract_attachments():
message = Message("test_message")
task_common.extract_attachments(attachments_required, message)
def run(self):
c = Connection(self.config)
logging.config.dictConfig(self.config)
self.logger = logging.getLogger(__name__)
self.logger.info('Process {} Running'.format(self.id))
self.msgQ.put(Message('Process', self.id, 'Running'))
while True:
parkrun = self.inQ.get()
self.logger.debug(parkrun)
if parkrun is None:
self.logger.info('Process {} Exiting'.format(self.id))
self.msgQ.put(Message('Process', self.id, 'Exiting'))
break
self.logger.debug('Process {} got record {}'.format(
self.id, parkrun['EventURL']))
if parkrun['lastEvent'] is None: parkrun['lastEvent'] = 0
if self.mode == Mode.CHECKURLS:
if self.getURL(parkrun['URL']) is not None:
c.updateParkrunURL(parkrun['Name'], True, True)
self.msgQ.put(
Message('Process', self.id,
'Verified ' + parkrun['Name'] + ' valid'))
else:
c.updateParkrunURL(parkrun['Name'], True, False)
self.msgQ.put(
Message(
'Error', self.id, 'Could not verify ' +
parkrun['Name'] + ' as valid'))
if self.mode == Mode.NEWEVENTS:
self.logger.info(
'Process {} checking for new results for {}'.format(
self.id, parkrun['EventURL']))
self.msgQ.put(
Message('Process', self.id,
'Checking for new results for ' + parkrun['Name']))
parkrun['EventNumber'], parkrun[
'EventDate'], data = self.getLatestEvent(
parkrun['URL'] + parkrun['LatestResultsURL'])
if data is not None:
self.logger.debug(
'Event {} got {} events in history'.format(
parkrun['EventURL'], len(data)))
parkrun['Runners'] = len(data)
# Add the event if it's a new event
# Check the event has the correct number of runners
if not c.checkParkrunEvent(parkrun):
self.logger.info(
'Parkrun {} event {}: runners did not match - reimporting.'
.format(parkrun['Name'], parkrun['EventNumber']))
#if not, delete the old event record and re-import the data
self.msgQ.put(
Message(
'Process', self.id,
'Updating ' + parkrun['Name'] + ' event ' +
xstr(parkrun['EventNumber'])))
eventID = c.replaceParkrunEvent(parkrun)
self.logger.debug(
'getLastEvent found {} runners'.format(len(data)))
for row in data:
row['EventID'] = eventID
c.addParkrunEventPosition(row)
sleep(self.delay)
if self.mode == Mode.NORMAL:
data = self.getEventHistory(parkrun['URL'] +
parkrun['EventHistoryURL'])
if data is not None:
self.logger.debug(
'Event {} got {} events in history'.format(
parkrun['URL'], len(data)))
for row in data:
row['Name'] = parkrun['Name']
row['EventURL'] = parkrun['EventURL']
# Add the event if it's a new event
self.msgQ.put(
Message(
'Process', self.id, 'Checking ' + row['Name'] +
' event ' + xstr(row['EventNumber'])))
self.logger.debug(row)
self.logger.debug(
'Process {} Checking {} event {}'.format(
self.id, row['EventURL'],
xstr(row['EventNumber'])))
# Check the event has the correct number of runners
if not c.checkParkrunEvent(row):
#if not, delete the old event record and re-import the data
self.logger.info(
'Parkrun {} event {}: runners did not match - reimporting.'
.format(parkrun['EventURL'],
row['EventNumber']))
self.msgQ.put(
Message(
'Process', self.id,
'Updating ' + row['Name'] + ' event ' +
xstr(row['EventNumber'])))
eventID = c.replaceParkrunEvent(row)
eData = self.getEvent(
parkrun['URL'] + parkrun['EventNumberURL'],
row['EventNumber'])
if eData is not None:
self.logger.debug(
'getEvent found {} runners'.format(
len(eData)))
for eRow in eData:
eRow['EventID'] = eventID
c.addParkrunEventPosition(eRow)
sleep(self.delay)
else:
self.logger.debug('getEvent found no runners')
else:
self.logger.warning(
'Parkrun {} returns no history page.'.format(
parkrun['Name']))
c.execute(
"update p set p.LastUpdated = e.LastEvent from parkruns as p inner join (select ParkrunID, max(EventDate) as LastEvent from events group by ParkrunID) as e on p.ParkrunID = e.ParkrunID"
)
self.logger.debug('Sleeping for {} seconds'.format(self.delay))
sleep(self.delay)
c.close()
def request_values(self, num_iterations):
"""
Method invoked to start simulation. Prints out what clients have converged on what iteration.
Also prints out accuracy for each client on each iteration (what weights would be if not for the simulation) and federated accuaracy.
:param iters: number of iterations to run
"""
converged = {
} # maps client names to iteration of convergence. Contains all inactive clients
active_clients = set(self.directory.clients.keys())
for i in range(1, num_iterations + 1):
weights = {}
intercepts = {}
m = multiprocessing.Manager()
lock = m.Lock()
with ThreadPool(len(active_clients)) as calling_pool:
args = []
for client_name in active_clients:
client_instance = self.directory.clients[client_name]
body = {
'iteration':
i,
'lock':
lock,
'simulated_time':
config.LATENCY_DICT[self.name][client_name]
}
arg = Message(sender_name=self.name,
recipient_name=client_name,
body=body)
args.append((client_instance, arg))
messages = calling_pool.map(client_computation_caller, args)
server_logic_start = datetime.now()
vals = {
message.sender:
(message.body['weights'], message.body['intercepts'])
for message in messages
}
simulated_time = find_slowest_time(messages)
# add them to the weights_dictionary
for client_name, return_vals in vals.items():
client_weights, client_intercepts = return_vals
weights[client_name] = np.array(client_weights)
intercepts[client_name] = np.array(client_intercepts)
weights_np = list(
weights.values()) # the weights for this iteration!
intercepts_np = list(intercepts.values())
try:
averaged_weights = np.average(
weights_np, axis=0) # gets rid of security offsets
except:
raise ValueError(
'''DATA INSUFFICIENT: Some client does not have a sample from each class so dimension of weights is incorrect. Make
train length per iteration larger for each client to avoid this issue'''
)
averaged_intercepts = np.average(intercepts_np, axis=0)
self.averaged_weights[
i] = averaged_weights ## averaged weights for this iteration!!
self.averaged_intercepts[i] = averaged_intercepts
# add time server logic takes
server_logic_end = datetime.now()
server_logic_time = server_logic_end - server_logic_start
simulated_time += server_logic_time
with ThreadPool(len(active_clients)) as returning_pool:
args = []
for client_name in active_clients:
client_instance = self.directory.clients[client_name]
body = {
'iteration':
i,
'return_weights':
averaged_weights,
'return_intercepts':
averaged_intercepts,
'simulated_time':
simulated_time +
config.LATENCY_DICT[self.name][client_name]
}
message = Message(sender_name=self.name,
recipient_name=client_name,
body=body)
args.append((client_instance, message))
return_messages = returning_pool.map(client_weights_returner,
args)
simulated_time = find_slowest_time(return_messages)
server_logic_start = datetime.now()
clients_to_remove = set()
for message in return_messages:
if message.body[
'converged'] == True and message.sender not in converged: # converging
converged[message.sender] = i # iteration of convergence
clients_to_remove.add(message.sender)
server_logic_end = datetime.now()
server_logic_time = server_logic_end - server_logic_start
simulated_time += server_logic_time
if config.CLIENT_DROPOUT:
# tell the clients which other clients have dropped out
active_clients -= clients_to_remove
if len(
active_clients
) < 2: # no point in continuing if don't have at least 2 clients
self.print_convergences(converged)
return
with ThreadPool(len(active_clients)) as calling_pool:
args = []
for client_name in active_clients:
client_instance = self.directory.clients[client_name]
body = {
'clients_to_remove':
clients_to_remove,
'simulated_time':
simulated_time +
config.LATENCY_DICT[self.name][client_name],
'iteration':
i
}
message = Message(sender_name=self.name,
recipient_name=client_name,
body=body)
args.append((client_instance, message))
__ = calling_pool.map(client_agent_dropout_caller, args)
# at end of all iterations
self.print_convergences(converged)
def __parseCommentFromApp(comment, commentList, userList, count, messageId,
queryUser):
# Save comment Message
new_comment = Message()
new_comment.isComment = True
new_comment.parent = messageId
new_comment.id = str(messageId) + "." + str(count)
# Get all text elements of a comment
textList = None
try:
textList = comment.find_all('div', class_='bc-comment-text')
except:
logging.info("This thread has no comments")
# Save comment text and reply id
try:
# Find comment text and quoted comment id.
if len(textList) > 1:
try:
new_comment.text = textList[1].text
new_comment.replyToMessageId = next(
(new_comment for new_comment in commentList
if new_comment.text == textList[0].text), None).id
except AttributeError:
logging.info("Could not find quoted comment" +
", MessageId: " + str(new_comment.id))
# Find only comment text
else:
try:
new_comment.text = textList[0].text
except IndexError:
logging.info("Could not find comment text" +
", MessageId: " + str(messageId))
except:
traceback.print_exc()
logging.info("An error occurred reading comment text" +
", MessageId: " + str(messageId))
# Find not identified User
new_comment.sender_name = comment.find(
'span', class_='bc-comment-author-name').contents[0].contents[0]
# Find identified User and save User ich channel.users and save message.sendername
try:
identifierName = comment.find(
'span',
class_='bc-comment-author-name').contents[0]['href'].rsplit(
'/', 1)[-1]
except:
identifierName = ""
user = None
# Query user if identifier name was found
if not identifierName == "" and queryUser:
try:
user = Bots.getEntity(identifierName)
userList.append(user)
except ValueError:
# User for some reason not found.
pass
new_comment.username = identifierName
if user is not None:
new_comment.sender_name = concat(user.first_name, user.last_name)
new_comment.sender = user.id
else:
commentUser = telethon.types.User(0)
commentUser.first_name = new_comment.sender_name
new_comment.timestamp = comment.find('time')['datetime']
commentList.append(new_comment)
