def parse_wiki_event(date_str): for year, month, day in date_generator(date_str): event_date = '{}-{}-{}'.format(year, month, day) with PG_CONN.cursor() as cur: cmd = "SELECT event_raw FROM {table} where event_date='{event_date}' ORDER BY inserted_at DESC;" cmd = cmd.format(table=WIKIPEDIA_TABLE_RAW, event_date=event_date) cur.execute(cmd) event_raw = cur.fetchone()[0] soup = BeautifulSoup(event_raw, 'html.parser') try: event_table = soup.find('table', {'class': 'vevent'}) event_types = event_table.find('td', { 'class': 'description' }).find_all('dl') event_types_str = [o.text.strip() for o in event_types] log.debug(event_types_str) events_uls = event_table.find('td', { 'class': 'description' }).select('> ul') #log.debug(events_uls) #log.debug(len(events_uls)) # each ul correspond to an event_type # in one ul several events assert len(event_types_str) == len(events_uls) for event_type_str, uls in zip(event_types_str, events_uls): for event in uls.select('> li'): event_type = event_type_str event_title = event.find('a').text event_body = event.text event_sources = [ get_domain_from_url(o['href']) for o in event.find_all('a', { 'rel': 'nofollow', 'class': 'external text' }) ] cmd = ( 'INSERT INTO {table} (event_date, event_type, event_title, event_body, event_sources) ' 'VALUES (%s, %s, %s, %s, %s);') cmd = cmd.format(table=WIKIPEDIA_TABLE_PARSED_STG) cur.execute(cmd, (event_date, event_type, event_title, event_body, event_sources)) PG_CONN.commit() print 'EVENT TYPE ', event_type print 'EVENT TITLE ', event_title print 'EVENT BODY', event_body print 'EVENT SOURCES', event_sources except Exception, e: log.error('could not parse %s', event_date)
def scatter(self): log.debug(self.Id+" begin to draw ") plt.clf() cans = [] for line in self.lines: log.debug("line.xs="+str(line.xs)) log.debug("line.ys="+str(line.ys)) log.debug("plt atts="+str(line.plt)) #can = plt.plot(line.xs, line.ys, line.plt.pop("style", "-"), **line.plt) can = plt.scatter(x=line.xs, y=line.ys, s=self.kwargs.get("c", 1), **line.plt) cans.append(can) plt.grid(True) plt.xlabel(self.canvas.pop("xlabel", "X")) plt.ylabel(self.canvas.pop("ylabel", "X")) plt.xlim(xmax=self.canvas.pop("xmax", None)) self.extend(plt)#extend line plt.legend(**self.canvas) #loc='lower/upper left' if HOSTOS.startswith("Darwin"): pass plt.savefig(self.pngout) plt.savefig(self.pdfout) log.debug(self.Id+" fig save to "+self.pngout) plt.close() log.info(self.Id+" ends")
async def finish(self, s: Shlyapa): def player_array_to_dict(score_list): return dict([(self.players[i].name, v) for i, v in enumerate(score_list)]) s.calculate_results() total_score = player_array_to_dict(s.get_total_score_results()) explained_score = player_array_to_dict(s.get_explained_score_results()) guessed_score = player_array_to_dict(s.get_guessed_score_results()) score = {} for p in self.players: score[p.name] = { "total": total_score[p.name], "explained": explained_score[p.name], "guessed": guessed_score[p.name] } results = { "score": score, "explained": [player_array_to_dict(t) for t in s.get_explained_table_results()], "guessed": [player_array_to_dict(t) for t in s.get_guessed_table_results()] } await self.broadcast(message.Finish(results=results)) self.state = HatGame.ST_FINISH for p in self.players: """Reset players to ask new words""" p.reset() # noqa log.debug('Game finished')
def timbra_xml(self, xml, id_original=0): user_token = self._get_token(id_original) if not user_token: return '' method = 'timbra_xml' data = { 'xml': escape(xml.encode('ascii', 'xmlcharrefreplace').decode('utf-8')), 'rfc': self.rfc, 'user_token': user_token, 'transaccion_id': id_original, } data = self._SOAP[method].format(**data).encode('utf-8') headers = { 'SOAPAction': '"{}"'.format(self._ACTIONS[method]), 'Content-length': len(data), 'Content-type': 'text/xml; charset="UTF-8"' } s = Session() req = Request('POST', self.WS['timbrado'], data=data, headers=headers) prepped = req.prepare() try: response = s.send(prepped, timeout=TIMEOUT) res = xml2dict.parse(response.text) ok, res = self._check_fault(res) if ok: return '' return res['RespuestaTimbraXML']['ComprobanteXML']['DatosXML'] except exceptions.Timeout: self.error = 'Tiempo de espera agotado' except Exception as e: self.error = str(e) log.debug(e) return ''
def first_express_interest(self): if self.status == Controller.STATUS_OFF or self.status == Controller.STATUS_STEP_OFF: return if self.mydata.final_byte != None and self.mydata.next_byte >= self.mydata.final_byte: #we do not use is_all to check, since final_byte is more accurate and is_all -> final_byte log.debug("illegel request, do not express it, next_byte: %s" %(self.mydata.next_byte)) return chunkinfo = ChunkInfo() chunkinfo.beginT = datetime.datetime.now() chunkinfo.begin_byte = self.mydata.next_byte temp = self.chunkSizeEstimator.get_optimal_data_size() chunkinfo.end_byte = chunkinfo.begin_byte + temp - 1 chunkinfo.packetN = math.ceil((temp + CHUNK_HEADER_SIZE)/float(self.packet_max_data_size)) chunkinfo.status = 1 name = self.ndn_name name = self.ndn_name.append(ADAPTIVE_MOD_FLAG).append(str(temp)) #only leave the data size name = name.append(chunkinfo.begin_byte) chunkinfo.ndn_name = name self.chunkInfos.append(chunkinfo) self.mydata.unsatisfied_chunks_keyed_by_name[str(name)] = chunkinfo #packet_on_the_fly, 3 results, illegal, out-of-order, in-order and retransmission self.window.update_express(chunkinfo) self.express_interest(chunkinfo) self.mydata.next_byte = chunkinfo.end_byte + 1
def express_interest(self, chunkinfo): """this method may express illegal Interest, thus, re_express_interest and first_express_interest are in charge of checking; even that, there may also illegal Interest, due to unknown final_byte, leading to useless chunkinfo in chunkinfos and illegal Data(Nack) or Interest timeout (we do not use is_all to check, since final_byte is more accurate and is_all -> final_byte); thus, we need do_receiving_content to handle illegal Data """ assert chunkinfo != None, "chunkinfo == None" assert chunkinfo.endT == None, "chunkinfo.endT != None" selector = pyndn.Interest() selector.answerOriginKind = 0#producer generate every time selector.childSelctor = 1 selector.interestLifetime = self.rtoEstimator.get_rto() rst = self.handle.expressInterest(chunkinfo.ndn_name, self, selector) if rst != None and rst < 0: log.info("fail to express interest=%s with result %s" %(chunkinfo.ndn_name, rst)) self.window.update_nack(chunkinfo) chunkinfo.status = 0 else: chunkinfo.retxN += 1 log.debug("express interest=%s" %(chunkinfo.ndn_name))
async def get(self): ws = web.WebSocketResponse() await ws.prepare(self.request) user = self.request.user for _ws in self.request.app['websockets']: await _ws.send_str('%s joined' % user.login) self.request.app['websockets'].append(ws) async for msg in ws: if msg.type == WSMsgType.TEXT: if msg.data == 'close': await ws.close() else: try: message = Message(message=msg.data, user_id=user) await message.commit() except ValidationError as e: print(e) for _ws in self.request.app['websockets']: await _ws.send_str('{"user": "******", "msg": "%s"}' % (user.login, msg.data)) elif msg.type == WSMsgType.ERROR: log.debug('ws connection closed with exception %s' % ws.exception()) self.request.app['websockets'].remove(ws) for _ws in self.request.app['websockets']: await _ws.send_str('%s disconected' % user.login) log.debug('websocket connection closed') return ws
async def name(self, ws, data): name = data['name'] log.debug(f'user {name} logged in as {id(ws)}') player = Player(name=name, ws=ws) self.players[name] = player self.sockets[id(ws)] = player await self.game(ws)
def _get_token(self, transaccion_id=0): if not transaccion_id: transaccion_id = get_epoch() method = 'obtener_token' data = { 'rfc': self.rfc, 'transaccion_id': transaccion_id } data = self._SOAP[method].format(**data).encode('utf-8') headers = { 'SOAPAction': '"{}"'.format(self._ACTIONS[method]), 'Content-length': len(data), 'Content-type': 'text/xml; charset="UTF-8"' } s = Session() req = Request('POST', self.WS['seguridad'], data=data, headers=headers) prepped = req.prepare() try: response = s.send(prepped, timeout=TIMEOUT) res = xml2dict.parse(response.text) ok, res = self._check_fault(res) if ok: return '' token = res['RespuestaObtenerToken']['Token'] s = '{}|{}'.format(self.ID_INTEGRADOR, token) user_token = hashlib.sha1(s.encode()).hexdigest() return user_token except exceptions.Timeout: self.error = 'Tiempo de espera agotado' log.debug(self.error) print (self.error) except exceptions.RequestException as e: log.debug(e) print ('TOKEN', e) return ''
def upcall(self, kind, upcallInfo): if kind == pyndn.UPCALL_FINAL:#handler is about to be deregistered return pyndn.RESULT_OK if kind in [pyndn.UPCALL_INTEREST, pyndn.UPCALL_CONSUMED_INTEREST, pyndn.UPCALL_CONTENT_UNVERIFIED, pyndn.UPCALL_CONTENT_BAD]: log.error("unexpected kind: %s" %kind) return pyndn.RESULT_OK if kind == pyndn.UPCALL_CONTENT: self.is_all = True log.info("get data back: %s" %(upcallInfo.Interest.name)) self.stop() return pyndn.RESULT_OK elif kind == pyndn.UPCALL_INTEREST_TIMED_OUT: if self.turn == -1: return pyndn.RESULT_REEXPRESS else: if self.ist_sentN < self.turn: log.debug("timeout: %s" %(upcallInfo.Interest.name)) self.ist_sentN += 1 return pyndn.RESULT_REEXPRESS else: log.info("CANNOT get data back after %d trials: %s" %(self.turn, upcallInfo.Interest.name)) self.stop() return pyndn.RESULT_OK
async def ready(self, ws, msg: message.Ready): """Tell server that player is ready to guess/explain""" if self.state != HatGame.ST_PLAY: raise Exception( f"Invalid command 'ready' for game in state '{self.state}'") exp = self.turn.explaining gss = self.turn.guessing sent_by = self.sockets_map[id(ws)] if sent_by in (exp, gss): sent_by.ready() else: raise Exception('Wrong player sent ready command') log.debug( f'Pair state: explain({exp.name})={exp.state} guessing({gss.name})={gss.state}' ) if exp.state == Player.ST_READY and gss.state == Player.ST_READY: exp.play() gss.play() log.debug( f'Explanation started: explain {exp.name} guessing {gss.name}') self.timer = Timer(self.turn_timer, self.expired) await self.broadcast(message.Start()) await self.next_word()
async def post(self): ngmsg = message.Newgame.msg(await self.request.json()) game = HatGame(**ngmsg.args()) log.debug(f'New game request - {ngmsg.args()}') if game.id in self.request.app.games: return web.Response( status=500, content_type='application/json', text=json.dumps(message.Error(code=103, message='Duplicate game ID').data(), ensure_ascii=False) ) self.request.app.games[game.id] = game log.info(f"New game created id={game.id}, name='{game.game_name}''") headers = {} if NEED_CORS: headers = {'Access-Control-Allow-Origin': self.request.headers['Origin']} return web.Response( content_type='application/json', text=json.dumps(game.game_msg().data(), ensure_ascii=False), headers=headers )
def upcall(self, kind, upcallInfo): if kind != pyndn.UPCALL_INTEREST: log.warn("get kind: %s" %str(kind)) return pyndn.RESULT_OK co = self.prepare(upcallInfo) # try: # co = self.prepare(upcallInfo) # except: # thetype, value, traceback = sys.exc_info() # log.error("get exception: %s, %s, %s" %(thetype, value, traceback)) # co = None if co == None: log.warn("co == None") pass else: rst = self.handle.put(co) if rst < 0: log.warn("fail put content: %s, result: %s" %(co.ndn_name, rst)) else: log.debug("successfully put content, and returns %s"%(rst)) pass #("content: %s" %(co.ndn_name)) return pyndn.RESULT_INTEREST_CONSUMED
async def get_user_by_id(self, **kw): try: return self.get(id=self._id, ) except self.DoesNotExist: return False except Exception as e: log.debug(e) return False
def modpred(start, finish): try: result = df[df['DATE'].between(pd.to_datetime(start), pd.to_datetime(finish))] return result.to_json(orient="records") except Exception as e: log.debug(f'error: {e}') return json.dumps({"ERROR": str(repr(e)), "Data": None})
async def check_room(self, name, *args, **kwargs): try: return self.get(name=name, ) except self.DoesNotExist: return False except Exception as e: log.debug(e) return False
async def get_room_by_id(self, id, *args, **kwargs): try: return self.get(id=int(id), ) except self.DoesNotExist: return False except Exception as e: log.debug(e) return False
async def authorize(request, handler): session = await get_session(request) if (not session.get('uid')) and (not request.path.startswith('/login')): url = request.app.router['login'].url_for() log.debug('redirecting to {}'.format(str(url))) raise web.HTTPFound(url) response = await handler(request) return response
async def words(self, ws, data): words = data['words'] p = self.sockets[id(ws)] p.set_words(words) log.debug(f'user {p.name} sent words: {words}') # send prepare message to all players for p in self.players.values(): await self.prepare(p.socket)
def out_of_order_content(self, upcallInfo): """do nothing here, just buffer it and leave it to expected chunk come do not update the window when out-of-order, until in-order chunk is received """ name = str(upcallInfo.Interest.name) chunkinfo = self.mydata.unsatisfied_chunks_keyed_by_name.get(name) self.mydata.satisfied_chunkN += 1 self.window.update_receiving(chunkinfo) log.debug("received out-of-order Data: %s, out packet: %s" %(name, self.window.packet_on_the_fly))
def _check_fault(self, res): self.error = '' FAULT = 's:Fault' fault = res['s:Envelope']['s:Body'] if FAULT in fault: self.error = 'Código de Error: {}\n\n{}'.format( fault[FAULT]['faultcode'], fault[FAULT]['faultstring']['#text']) log.debug(self.error) return True, '' return False, fault
def update_loss(self, chunkinfo): self.packet_on_the_fly -= chunkinfo.packetN if self.is_fix == None: self._cwnd = int(self._cwnd/2.0) #sawtooth if self._cwnd < 1: self._cwnd = 1 self.threshold = self._cwnd #fast recovery log.debug("change cwnd to %s, threshold to %s, packet_on_the_fly to %s" %(self._cwnd, self.threshold, self.packet_on_the_fly)) self.sample()
def update_receiving(self, chunkinfo): self.packet_on_the_fly -= chunkinfo.packetN if self.is_fix == None: if self._cwnd < self.threshold: self._cwnd += chunkinfo.packetN #slow start else: self._cwnd += chunkinfo.packetN * 1.0/self._cwnd #additive increase if self._cwnd > MAX_WINDOWSN: self._cwnd = MAX_WINDOWSN self.sample() log.debug("change cwnd to %s, threshold to %s, packet_on_the_fly to %s" %(self._cwnd, self.threshold, self.packet_on_the_fly))
async def check_user(self, **kw): try: return self.get( login=self._login, password=self._password, ) except self.DoesNotExist: return False except Exception as e: log.debug(e) return False
async def get(self): session = await get_session(self.request) uid = session.get('uid') message = Message() messages = await message.get_messages('group_interns') log.debug(f'group_interns messages: {messages}') for message in messages['messages']: if message.get('uid') != None: if message['uid'] == uid: message['own'] = "yes" return messages
def re_express_interest(self, chunkinfo): if self.status == Controller.STATUS_OFF or self.status == Controller.STATUS_STEP_OFF: return if self.mydata.final_byte !=None and chunkinfo.begin_byte >= self.mydata.final_byte: #suppress the illegal reuqest log.debug("illegel request, do not re-express it: %s" %(chunkinfo.ndn_name)) elif self.is_all:#shouldn't happen, since already check before re-expressing log.error("already get all the requests, do not re-express it: %s" %(chunkinfo.ndn_name)) else: self.window.update_express(chunkinfo) self.chunkSizeEstimator.get_optimal_data_size() self.express_interest(chunkinfo)
async def words(self, ws, msg: message.Words): """Player sends it's words to server""" words = msg.words if len(words) < self.num_words: for wi in range(0, self.num_words): words.append(NOUNS.get_random_word()) p = self.sockets_map[id(ws)] p.words = words log.debug(f'user {p.name} sent words: {words}') await self.prepare()
def read(self, begin_byte, data_size): if self.fd.closed: self.fd = open(fpath) if begin_byte >= self.fsize: return None if self.fd.tell() != begin_byte: log.debug("move to %s" %(begin_byte)) self.fd.seek(begin_byte) data = self.fd.read(data_size) return data
async def get(self): log.debug('websocket new connection') ws = web.WebSocketResponse() self.request.app.websockets.append(ws) await ws.prepare(self.request) async for msg in ws: log.debug( f'websocket message received: {msg.type}: {msg.data.strip()}') if msg.type == WSMsgType.text: if msg.data == 'close': await ws.close() else: try: data = json.loads(msg.data) except Exception as e: await self.error(ws, 101, f'Broken message received {e}') continue if not 'cmd' in data: await self.error(ws, 102, f'Invalid message format {msg.data}') continue else: cmdtxt = data['cmd'] cmd = getattr(self.request.app.game, cmdtxt, None) if not callable(cmd): await self.error(ws, 103, f'Unknown command {cmdtxt}') continue try: log.debug(f'Received command {cmdtxt}') await cmd(ws, data) except Exception as e: log.exception('Error executing command {cmdtxt}: {e}') await self.error( ws, 104, f'Error executing command {cmdtxt}: {e}') elif msg.type == WSMsgType.error: log.debug('ws connection closed with exception %s' % ws.exception()) self.request.app.websockets.remove(ws) log.debug('websocket connection closed') return ws
async def get(self): session = await get_session(self.request) uid = 'user{0}'.format(random.randint(1, 1001)) uids = self.request.app['uids'] while uid in uids: uid = 'user{0}'.format(random.randint(1, 1001)) uids.append(uid) self.request.app['uids'] = uids session['uid'] = uid log.debug(uid) redirect(self.request, 'general')
async def expired(self): """Guessing time expired""" self.timer = None try: log.debug('Turn timer expired, stop turn') await self.broadcast(message.Stop(reason='timer')) self.turn.explaining.lastanswer() self.turn.guessing.finish() except Exception as e: log.error(f'Exception while process timer: {e}') log.exception()
def express_interest(self): name = self.ndn_name #name = self.ndn_name.append(800) name = self.ndn_name.append(ADAPTIVE_MOD_FLAG).append(800) #only leave the data size name = name.append(0) selector = pyndn.Interest() selector.answerOriginKind = 0#producer generate every time selector.childSelctor = 1 selector.interestLifetime = self.ist_lifetime self.handle.expressInterest(name, self, selector) log.debug("send Interest %s" %(name)) self.ist_sentN += 1
async def get(self): log.info("rest api request ... /pingmodel from " + str(self.request.remote)) if (self.request.headers.get("X-Auth-Token") != config.AUTHENTICATION_TOKEN_PINGMODEL): log.info("error authentication token ... " + "from " + str(self.request.remote) + " [" + str(self.request.headers.get("X-Auth-Token")) + "]") return web.Response(status=401, text=str('{"error":"authentication token"}')) log.info("authentication token ... " + "from " + str(self.request.remote) + " [Ok]") log.debug(str('{"status":"started"}')) text = '{"status":"started"}' return web.Response(text=text)
def fetch_wikipedia_raw_page(year, month, day): wiki_url = WIKIPEDIA_PAGE_TEMPLATE.format(YYYY=year, Month=MONTHS[month], D=day) log.debug('GET %s', wiki_url) r = requests.get(wiki_url) try: r.raise_for_status() except: log.error('NOT FOUND') return 'NOT FOUND: ' + wiki_url return r.text
async def get(self): session = await get_session(self.request) self.self_id = session.get('user') self.login = session.get('login') self.user = self.request.app['models']['user'] self.unread = self.request.app['models']['unread'] self.message = self.request.app['models']['message'] self.company = self.request.app['models']['company'] self.company_id = self.request.rel_url.query.get('company_id') ws = web.WebSocketResponse() await ws.prepare(self.request) my_companys = await self.company.get_company_by_user(self.self_id) for c in my_companys: self.request.app['websockets'][str(c['_id'])].append(ws) for _ws in self.request.app['websockets'][str(c['_id'])]: await _ws.send_json({ 'user_id': self.self_id, 'type': 'joined' }) self.request.app['online'][self.self_id] = ws async for msg in ws: if msg.type == WSMsgType.TEXT: data = json.loads(msg.data) if msg.data == 'close': await ws.close() else: if data['type'] == 'company_chat_mess': await self.recieve_company_chat_mess(data) elif data['type'] == 'private_chat_mess': await self.recieve_private_chat_mess(data) elif data['type'] == 'mess_notification': pass elif data['type'] == 'event_notification': pass elif msg.type == WSMsgType.ERROR: log.debug('ws connection closed with exception %s' % ws.exception()) for c in my_companys: self.request.app['websockets'][str(c['_id'])].remove(ws) await ws.close() return ws
async def get(self): ws = web.WebSocketResponse() await ws.prepare(self.request) session = await get_session(self.request) message = Message() session_user = session.get('user') username = session_user.get('username') for _ws in self.request.app['websockets']: await _ws.send_str('%s joined' % username) self.request.app['websockets'].append(ws) async for msg in ws: if msg.type == WSMsgType.text: for _ws in self.request.app['websockets']: time_chat = datetime.now() if len(history) > settings.MAX_MSG: del history[0] history.append({ 'time': time_chat, 'user': username, 'msg': msg.data }) await message.save_msg({ 'user_id': session_user.get('id'), 'created_at': time_chat, 'msg': msg.data }) await _ws.send_str('(%s) %s' % (username, msg.data)) elif msg.type == WSMsgType.error: log.debug('ws connection closed with exception %s' % ws.exception()) self.request.app['websockets'].remove(ws) for _ws in self.request.app['websockets']: await _ws.send_str('%s disconected' % username) log.debug('websocket connection closed') return ws
def make_data(invoice): data = {} try: tree = ET.parse(invoice).getroot() version = tree.attrib['version'] node = tree.find('{}Emisor'.format(PRE[version])) data['total'] = tree.attrib['total'] data['emisor_rfc'] = node.attrib['rfc'] node = tree.find('{}Receptor'.format(PRE[version])) data['receptor_rfc'] = node.attrib['rfc'] node = tree.find('{}Complemento/{}TimbreFiscalDigital'.format(PRE[version], PRE['TIMBRE'])) data['uuid'] = node.attrib['UUID'] return data except Exception as e: log.error('La factura no se pudo parsear, asegurate de que sea un CFDI válido') log.debug(e) return {}
async def guessed(self, ws, msg: message.Guessed): """Guessing User tell us of he guessed right""" sent_by = self.sockets_map[id(ws)] if sent_by != self.turn.explaining: raise ValueError( f'Only explaining player can send guessed command,' f' but sent by {sent_by} in {sent_by.state} state') if sent_by.state not in (Player.ST_PLAY, Player.ST_LAST_ANSWER): raise ValueError( f"Player {sent_by} can't sent guessed command while not in play" ) self.turn.guessed(result=msg.guessed) if msg.guessed: m = message.Explained(word=self.turn.word) else: m = message.Missed() await self.broadcast(m) if self.turn.explaining.state == Player.ST_LAST_ANSWER: """Answer after timer exhausted""" self.turn.explaining.finish() await self.next_move(explained_words=self.turn.result()) return if not self.has_words(): """End of turn""" if self.timer: self.timer.cancel() log.debug('No more words - next turn') await self.broadcast(message.Stop(reason='empty')) self.turn.explaining.finish() self.turn.guessing.finish() await self.next_move(explained_words=self.turn.result()) return await self.next_word()
async def get(self): message = Message() ws = web.WebSocketResponse() await ws.prepare(self.request) session = await get_session(self.request) uid = session.get('uid') #broadcast joining of new user join_msg = ('"%s has joined the chat"' % (uid)) #message for _ws in self.request.app['websockets_interns']: await _ws.send_str(join_msg) self.request.app['websockets_interns'].append(ws) #send client's id to this particular client for frontend await ws.send_str('{"myID": "%s"}' % (uid)) async for msg in ws: if msg.type == WSMsgType.TEXT: if msg.data == 'exit-chat': await ws.close() else: t = datetime.now(tzutc()) format_str = "%H:%M" t_string = t.strftime(format_str) json_message = '{{"from_user": "******",\ "uid": "{0}",\ "msg": "{1}",\ "time": "{2}"}}'.format(uid, msg.data, t_string) log.debug(json_message) await message.send_message('group_interns', json_message) for _ws in self.request.app['websockets_interns']: await _ws.send_str('{"user": "******", "msg": "%s", "time": "%s"}' % (uid, msg.data, t_string)) elif msg.type == WSMsgType.ERROR: log.debug('ws connection closed with exception {0}'.format(ws.exception())) #broadcast leaving of the user self.request.app['websockets_interns'].remove(ws) leave_msg = ('"%s has left the chat"' % (uid)) #message for _ws in self.request.app['websockets_interns']: await _ws.send_str(leave_msg) return ws
def insert_wikipedia_raw(date_str): with PG_CONN.cursor() as cur: c = 0 for year, month, day in date_generator(date_str): event_raw = fetch_wikipedia_raw_page(year, month, day) event_date = datetime(year, month, day) cmd = 'INSERT INTO {table} (event_date, event_raw) VALUES (%s, %s);' cmd = cmd.format(table=WIKIPEDIA_TABLE_RAW) log.debug(cmd) cur.execute(cmd, (event_date, event_raw)) c += 1 if c % 20: PG_CONN.commit() PG_CONN.commit() log.debug('Done.')
async def next_move(self): """Do next move, called on game start or after move finished""" explained_words = None if self.turn: self.turn.explaining.wait() self.turn.guessing.wait() explained_words = self.turn.result() # return to pool words which was miss-guessed by previous pair missed_words = self.turn.missed_words if len(missed_words): log.debug(f"Return #{len(missed_words)} words to hat") self.tour_words.extend(missed_words) s = self.shlyapa if explained_words is not None: # non-first move log.debug(f'Turn over, explained words={explained_words}') s.move_shlyapa(pair_explained_words=explained_words) if s.is_cur_tour_new(): await self.tour() if s.is_end(): await self.finish(s) return log.debug(f'New turn #{s.get_cur_turn()}') pair_idx = s.get_next_pair() exp = self.players[pair_idx.explaining] gss = self.players[pair_idx.guessing] self.turn = Turn(explaining=exp, guessing=gss) log.debug(f'In hat #{len(self.tour_words)}') log.debug(f'Pair selected: explain={exp} guessing={gss}') exp.begin() # noqa gss.begin() # noqa m = message.Turn(turn=s.get_cur_turn(), explain=exp.name, guess=gss.name) await self.broadcast(m)
def in_order_content(self, upcallInfo): """the best scenario, content is received in-order, however, we should check those buffered out-of-order chunks """ name = str(upcallInfo.Interest.name) chunkinfo = self.mydata.unsatisfied_chunks_keyed_by_name.pop(name) if not self.fout.closed: self.fout.write(chunkinfo.content) else: log.critical("fails to write content") self.mydata.accept_bytes += chunkinfo.data_size if not self.cache_data: chunkinfo.content = None chunkinfo.status = 2 #satisfied yet self.mydata.satisfied_chunkN += 1 self.mydata.expected_chunkI += 1 self.window.update_receiving(chunkinfo) log.debug("received in-order Data: %s, out packet: %s" %(name, self.window.packet_on_the_fly)) #check the out-of-order contents recevied before for name in self.mydata.unsatisfied_chunks_keyed_by_name.keys(): chunkinfo = self.mydata.unsatisfied_chunks_keyed_by_name[name] if chunkinfo.endT == None: break else: chunkinfo = self.mydata.unsatisfied_chunks_keyed_by_name.pop(name) if not self.fout.closed: self.fout.write(chunkinfo.content) else: log.critical("fails to write content") self.mydata.accept_bytes += chunkinfo.data_size if not self.cache_data: chunkinfo.content = None self.mydata.expected_chunkI += 1 chunkinfo.status = 2 #satisfied yet if self.mydata.final_byte == self.mydata.accept_bytes: self.is_all = True for chunkinfo in self.mydata.unsatisfied_chunks_keyed_by_name.itervalues(): log.warn(str(chunkinfo)) log.warn("------------------------ %s all the contents are received---------------------------" %(self.Id)) self.stop()
async def get(self): ws = web.WebSocketResponse() await ws.prepare(self.request) session = await get_session(self.request) user = User(self.request.db, {'id': session.get('user')}) login = await user.get_login() for _ws in self.request.app['websockets']: _ws.send_str('%s joined' % login) self.request.app['websockets'].append(ws) async for msg in ws: if msg.tp == MsgType.text: if msg.data == 'close': await ws.close() else: message = Message(self.request.db) result = await message.save(user=login, msg=msg.data) log.debug(result) for _ws in self.request.app['websockets']: _ws.send_str('(%s) %s' % (login, msg.data)) elif msg.tp == MsgType.error: log.debug('ws connection closed with exception %s' % ws.exception()) self.request.app['websockets'].remove(ws) for _ws in self.request.app['websockets']: _ws.send_str('%s disconected' % login) log.debug('websocket connection closed') return ws
async def get(self): ws = web.WebSocketResponse() await ws.prepare(self.request) session = await get_session(self.request) user = User(self.request.app.db, {'id': session.get('user')}) login = await user.get_login() for _ws in self.request.app['websockets']: _ws.send_str('%s joined' % login) self.request.app['websockets'].append(ws) async for msg in ws: if msg.type == WSMsgType.TEXT: if msg.data == 'close': await ws.close() else: message = Message(self.request.app.db) result = await message.save(user=login, msg=msg.data) log.debug(result) for _ws in self.request.app['websockets']: await _ws.send_str('{"user": "******", "msg": "%s"}' % (login, msg.data)) elif msg.type == WSMsgType.ERROR: log.debug('ws connection closed with exception %s' % ws.exception()) self.request.app['websockets'].remove(ws) for _ws in self.request.app['websockets']: _ws.send_str('%s disconected' % login) log.debug('websocket connection closed') return ws
async def get(self): ws = web.WebSocketResponse() await ws.prepare(self.request) session = await get_session(self.request) user = UserMySQL(id=session.get('user')) user = await user.get_user_by_id() for _ws in self.request.app['websockets']: _ws.send_str('%s joined' % user.login) self.request.app['websockets'].append(ws) async for msg in ws: if msg.tp == MsgType.text: if msg.data == 'close': await ws.close() else: message = MessageMySQL(user_id=session['user'], room_id=session['room_id']) message = await message.insert_message_to_db(msg=msg.data) log.debug('result:%s' % message) for _ws in self.request.app['websockets']: _ws.send_str('(%s) %s' % (user.login, msg.data)) elif msg.tp == MsgType.error: log.debug('ws connection closed with exception %s' % ws.exception()) self.request.app['websockets'].remove(ws) for _ws in self.request.app['websockets']: _ws.send_str('%s disconected' % user.login) log.debug('websocket connection closed') return ws
async def get(self): ws = web.WebSocketResponse() await ws.prepare(self.request) session = await get_session(self.request) async with self.request.app['db'].acquire() as conn: data = await self.request.post() login = await get_login(conn, session.get('user')) for _ws in self.request.app['websockets']: _ws.send_str(f'{login}s joined') self.request.app['websockets'].append(ws) async for msg in ws: if type(msg) == WSMsgType.TEXT: # if msg.data == 'close': async with self.request.app['db'].acquire() as conn: result = await save(conn, login, data['content']) log.debug(result) for _ws in self.request.app['websockets']: await _ws.send_str({ 'user_id': data['user_id'], 'content': data['content'], }) elif type(msg) == WSMsgType.ERROR: log.debug(f'ws connection closed with exception {ws.exception()}') self.request.app['websockets'].remove(ws) for _ws in self.request.app['websockets']: _ws.send_str(f'{login}s disconnected') log.debug('websocket connection closed') return ws
def bar(self): log.debug(self.Id+" begin to draw ") plt.clf() self.bars = [] Width = self.canvas.pop("width", 0.2) for i in range(len(self.lines)): line = self.lines[i] #for line in self.lines: print "line.xs=",line.xs print "line.ys=", line.ys xs = [x+i*Width for x in line.xs] bar = plt.bar(left=xs, height=line.ys, width=Width, bottom=0, align="center", **line.plt) self.bars.append(bar) #plt.legend( (p1[0], p2[0]), ('Men', 'Women') ) plt.legend((self.bars[i][0] for i in range(len(self.lines))), (self.lines[i].plt["label"] for i in range(len(self.lines)))) #for bar in self.bars: xs = [Width * len(self.lines)/2 + j for j in line.xs] plt.xticks(xs, line.xs) plt.grid(True) plt.xlabel(self.canvas.pop("xlabel", " ")) plt.ylabel(self.canvas.pop("ylabel", " ")) plt.legend(**self.canvas) plt.title(self.canvas.pop("title", " ")) #plt.xticklabels([]) self.extend(plt) plt.legend(**self.canvas) plt.savefig(self.pngout) plt.savefig(self.pdfout) log.debug(self.Id+" fig save to "+self.pngout) plt.close() log.info(self.Id+" finishes")
def do_meet_accident(self, kind, upcallInfo): name = str(upcallInfo.Interest.name) if not name in self.mydata.unsatisfied_chunks_keyed_by_name: #since it's not possible that two same Interest on the fly at the same time, it sholdn't happen log.error("timeout Interest not in the unsatisfied list, it should not happend: %s!!" %(name)) return chunkinfo = self.mydata.unsatisfied_chunks_keyed_by_name[name] self.chunkSizeEstimator.update_loss(chunkinfo) self.window.update_loss(chunkinfo) if kind == 4: log.debug("timeout, Interest=%s, out packet: %d" \ %(upcallInfo.Interest.name, self.window.packet_on_the_fly)) #log.warn("%s" %(upcallInfo)) else: log.warn("-----------------strange accident: kind=%s, Interest=%s------------------" %(kind, upcallInfo.Interest.name)) #window check here if self.window.packet_on_the_fly < self.window.get_cwnd(): #it's already make sure that the chunk is not satisfied yet, but it could be illegal self.re_express_interest(chunkinfo) else: chunkinfo.status = 0 #wait for re-expressing
def do_receive_content(self, kind, upcallInfo): """receive a contents, there are 4 different scenarios: duplicated content, in-order content, out-of-order content, illegal content """ name = str(upcallInfo.Interest.name) if not name in self.mydata.unsatisfied_chunks_keyed_by_name: log.debug(self.mydata.unsatisfied_chunks_keyed_by_name.keys()) #the chunkinfo is already satisfied by previous chunk (retransmission here) self.duplicate_content(upcallInfo) return chunkinfo = self.mydata.unsatisfied_chunks_keyed_by_name[name] chunkinfo.endT = datetime.datetime.now() chunkinfo.data_size = len(upcallInfo.ContentObject.content) chunkinfo.chunk_size = len(_pyndn.dump_charbuf(upcallInfo.ContentObject.ndn_data)) chunkinfo.content = upcallInfo.ContentObject.content temp = math.ceil((chunkinfo.chunk_size)/float(self.packet_max_data_size)) self.rtoEstimator.update(chunkinfo) fbi = upcallInfo.ContentObject.signedInfo.finalBlockID if fbi != None: if isinstance(fbi, str): fbi = pyndn.Name.seg2num(fbi) #log.info("***************final chunk id: %s" %(fbi)) if self.mydata.final_byte == None: #the first final block content self.mydata.final_byte = int(fbi) else: assert self.mydata.final_byte == int(fbi), "get different final block id, old %s and new %s" %(self.mydata.final_byte, int(fbi)) si = upcallInfo.ContentObject.signedInfo if si.type == pyndn.CONTENT_NACK: self.nack_content(upcallInfo) elif si.type == pyndn.CONTENT_DATA: if chunkinfo.packetN != temp: if self.mydata.final_byte != None and chunkinfo.end_byte > self.mydata.final_byte:#final chunk or illegal chunk log.debug("final chunk, thus size is shorter than expected") else: log.debug("expected packetN (%s) != real packetN (%s), final_byte (%s), upcallInfo: %s, chunksize:%s" %(chunkinfo.packetN, temp, self.mydata.final_byte, name, chunkinfo.chunk_size)) #chunkinfo.packetN = temp self.chunkSizeEstimator.update_receiving(chunkinfo) # if self.mydata.final_byte!=None and chunkinfo.end_byte < self.mydata.final_byte: # assert chunkinfo.data_size > 500, "chukinfo is strange, %s" %(chunkinfo) # if name == self.mydata.unsatisfied_chunks_keyed_by_name.keys()[0]: self.in_order_content(upcallInfo) else: self.out_of_order_content(upcallInfo) retxQ = [] for chunkinfo in self.mydata.unsatisfied_chunks_keyed_by_name.itervalues(): if chunkinfo.status == 0:#waiting for re-expressing retxQ.append(chunkinfo) if len(retxQ) == 2: break #here we do not check whether the request is legal or not for i in [0, 1]:#mulply add if self.window.packet_on_the_fly < self.window.get_cwnd(): #re-expressing is prior to request new if len(retxQ) != 0: chunkinfo = retxQ.pop(0) chunkinfo.status = 1 self.re_express_interest(chunkinfo) continue if self.mydata.final_byte== None: self.first_express_interest() elif self.mydata.final_byte!= None and self.mydata.next_byte < self.mydata.final_byte: self.first_express_interest() else: log.critical("unkown Data type: %s" %(upcallInfo.ContentObject))
async def game_detail_ws(request): ''' Game websocket handler ''' session = await get_session(request) user_id = int(session.get('user')) game_id = request.match_info['id'] ws = web.WebSocketResponse(autoclose=False) await ws.prepare(request) opened_ws = request.app['websockets'][game_id] games = Games(request.db) game_users = await games.get_users(game_id) data_users = [] for user in game_users: data_users.append({ 'id': user.id, 'login': user.login }) for _ws in opened_ws: _ws.send_str(json.dumps({'message': '{} joined'.format(user_id), 'status': STATUS['INFO']})) _ws.send_str(json.dumps({'message': 'update user list', 'status': STATUS['UPDATE_USERS'], 'users': data_users})) opened_ws.append(ws) async for msg in ws: if msg.tp == MsgType.text: if msg.data == 'close': await ws.close() else: data = json.loads(msg.data) try: # Check need attributes if 'i' not in data or 'j' not in data: raise Exception('Positions of move are required.') # Check if user in game if not any(user.id == user_id for user in game_users): raise Exception('You cannot play this game. Create your own to game.') game_info = await games.one(game_id) # Check if game was not ended if game_info.winner_id: raise Exception('Game is over.') game_moves = await games.get_moves(game_id) check_pairs_moves = ((game_info.config_size**2 - len(game_moves)) % 2 == 0) check_pairs_user = ((next(index for index, user in enumerate(game_users) if user.id == user_id) + 1) % 2 == 0) # Check if current user must move if check_pairs_user != check_pairs_moves: raise Exception('It is not your turn.') data_moves = [[0]*game_info.config_size for i in range(game_info.config_size)] for moves in game_moves: data_moves[moves.x][moves.y] = str(moves.users_id) # Check if the field is available if data_moves[data['i']][data['j']]: raise Exception('This field is not available.') data_moves[data['i']][data['j']] = str(user_id) # Save move await games.save_move(user_id, game_id, data['i'], data['j']) # Check game for winner winner_id = await check_for_winner(data_moves) # Draw if game_info.config_size**2 == (len(game_moves) + 1) and not winner_id: winner_id = DRAW # if we find winner or draw -> game is end. if winner_id: await games.finish_game(game_id, winner_id) # if user alone in game and make move next_user_id = -1 if len(game_users) == PLAYERS_IN_GAME: next_user_id = next(user.id for user in game_users if user.id != user_id) context = { 'status': STATUS['OK'], 'winner_id': winner_id, 'next_user_id': next_user_id, 'current_user_id': user_id, 'i': data['i'], 'j': data['j'], 'message': '{} made choice'.format(user_id) } except Exception as e: print(str(e)) context = { 'status': STATUS['ERROR'], 'message': str(e) } for _ws in opened_ws: _ws.send_str(json.dumps(context)) elif msg.tp == MsgType.error: log.debug('ws connection closed with exception {}'.format(ws.exception())) opened_ws.remove(ws) for _ws in opened_ws: _ws.send_str(json.dumps({'message': '{} disconected'.format(user_id), 'status': STATUS['INFO']})) log.debug('websocket connection closed') return ws
def get_optimal_data_size(self): """only data, not including the chunk header size Return: chunk_data_size: the size of data contained in a chunk, chunk header/signature ... are not included """ if (self.receivedN + self.lostN) == 0 or float(self.lostN)/(self.receivedN + self.lostN) == 0: if self.receivedN > 20: #quite reliable underlying layer protocol chunk_data_size = MAX_DATA_SIZE else: #just in prelimary stage chunk_data_size = DEFAULT_DATA_SIZE if (self.receivedN + self.lostN) != 0: self._loss_rate = float(self.lostN)/(self.receivedN + self.lostN) else: #ref to: loss_rate = float(self.lostN)/(self.receivedN + self.lostN) # 1- \omega self._loss_rate = loss_rate if loss_rate == 1: chunk_data_size = self._optimal_size/2 if chunk_data_size <= 1000: chunk_data_size = 1000 self._optimal_size = chunk_data_size log.debug("optimal chunk data size: %s, loss rate: %s" %(chunk_data_size, self._loss_rate)) self.sample() return chunk_data_size if self._optimal_size == 0 : self._optimal_size = DEFAULT_DATA_SIZE M = self.packet_max_data_size T = pow((1 - loss_rate),1 / math.ceil((CHUNK_HEADER_SIZE + self._optimal_size) /float( M ))) self.T = T D = CHUNK_HEADER_SIZE #\Delta #log.debug("lossrate=%s" %(loss_rate)) #print "loss_rate is %s" %loss_rate chunk_data_size = (-1*D*math.log(T) -pow((D*(math.log(T)))**2-4*M*math.log(T)*D, 0.5))/(2*math.log(T)) #data size of the chunk k = int (math.ceil((CHUNK_HEADER_SIZE + chunk_data_size) / M)) temp = k r1 = (T**temp) * (temp * M) / (CHUNK_HEADER_SIZE + temp*M) temp = k+1 r2 = (T**temp) * (temp * M) / (CHUNK_HEADER_SIZE + temp*M) if r1 > r2: chunk_data_size = k * M - CHUNK_HEADER_SIZE else: chunk_data_size = (k+1) * M - CHUNK_HEADER_SIZE if chunk_data_size > MAX_DATA_SIZE: chunk_data_size = MAX_DATA_SIZE self._optimal_size = chunk_data_size if self.is_fix != None: chunk_data_size = self.is_fix self._optimal_size = chunk_data_size log.debug("optimal chunk data size: %s, loss rate: %s" %(chunk_data_size, self._loss_rate)) self.sample() return chunk_data_size
def prepare(self, upcallInfo): ist = upcallInfo.Interest ist_name = ist.name flag_index = None #len(ist_name) - 2 #index of the end component for i in range(len(ist_name)-2): sub = ist_name[i] if sub == ADAPTIVE_MOD_FLAG: flag_index = i break if flag_index == None: log.error("not a flow consumer's interest, ignore: %s" %(ist_name)) return None expected_data_size = int(ist_name[flag_index+1]) begin_byte = int(ist_name[flag_index+2]) name = ist_name[:flag_index] #not include the flag name_str = str(name) if name_str in self.readers: reader = self.readers[name_str] log.debug("%s in reader" %(name_str)) else: if self.is_dir: subpath = ist_name[upcallInfo.matchedComps:flag_index] fpath = self.path for i in range(upcallInfo.matchedComps, flag_index): temp = ist_name[i] fpath = os.path.join(fpath, temp) #assume that matchedComps is the number of matched components, not index if not os.path.exists(fpath): log.critical("path %s from Interest %s does not exist" %(fpath, ist_name)) return None if os.path.isdir(fpath): log.critical("path %s from Interest %s is not a file" %(fpath, ist_name)) return None else:#not serve all the directory if upcallInfo.matchedComps != flag_index: log.critical("umatched ndn_name: %s, %s"%(ist_name, self.ndn_name)) return None else: fpath = self.path reader = Reader(fpath=fpath) self.readers[name_str] = reader log.debug("add a new reader: %s" %(name_str)) data = reader.read(begin_byte, expected_data_size) if data == None: log.critical("Interest %s: begin_byte %s > file_size %s" %(ist_name, begin_byte, reader.fsize)) nack = self._nack_template(ist_name, reader) return nack else: log.info("Interest: %s, expected_data_size: %s, begin_byte: %s, data_size: %s" \ %(ist.name, expected_data_size, begin_byte, len(data))) co = self._data_template(ist_name, data, reader.fsize, pyndn.CONTENT_DATA) return co