def parse_wiki_event(date_str):
for year, month, day in date_generator(date_str):
event_date = '{}-{}-{}'.format(year, month, day)
with PG_CONN.cursor() as cur:
cmd = "SELECT event_raw FROM {table} where event_date='{event_date}' ORDER BY inserted_at DESC;"
cmd = cmd.format(table=WIKIPEDIA_TABLE_RAW, event_date=event_date)
cur.execute(cmd)
event_raw = cur.fetchone()[0]
soup = BeautifulSoup(event_raw, 'html.parser')
try:
event_table = soup.find('table', {'class': 'vevent'})
event_types = event_table.find('td', {
'class': 'description'
}).find_all('dl')
event_types_str = [o.text.strip() for o in event_types]
log.debug(event_types_str)
events_uls = event_table.find('td', {
'class': 'description'
}).select('> ul')
#log.debug(events_uls)
#log.debug(len(events_uls))
# each ul correspond to an event_type
# in one ul several events
assert len(event_types_str) == len(events_uls)
for event_type_str, uls in zip(event_types_str, events_uls):
for event in uls.select('> li'):
event_type = event_type_str
event_title = event.find('a').text
event_body = event.text
event_sources = [
get_domain_from_url(o['href'])
for o in event.find_all('a', {
'rel': 'nofollow',
'class': 'external text'
})
]
cmd = (
'INSERT INTO {table} (event_date, event_type, event_title, event_body, event_sources) '
'VALUES (%s, %s, %s, %s, %s);')
cmd = cmd.format(table=WIKIPEDIA_TABLE_PARSED_STG)
cur.execute(cmd, (event_date, event_type, event_title,
event_body, event_sources))
PG_CONN.commit()
print 'EVENT TYPE ', event_type
print 'EVENT TITLE ', event_title
print 'EVENT BODY', event_body
print 'EVENT SOURCES', event_sources
except Exception, e:
log.error('could not parse %s', event_date)
def scatter(self):
log.debug(self.Id+" begin to draw ")
plt.clf()
cans = []
for line in self.lines:
log.debug("line.xs="+str(line.xs))
log.debug("line.ys="+str(line.ys))
log.debug("plt atts="+str(line.plt))
#can = plt.plot(line.xs, line.ys, line.plt.pop("style", "-"), **line.plt)
can = plt.scatter(x=line.xs, y=line.ys, s=self.kwargs.get("c", 1), **line.plt)
cans.append(can)
plt.grid(True)
plt.xlabel(self.canvas.pop("xlabel", "X"))
plt.ylabel(self.canvas.pop("ylabel", "X"))
plt.xlim(xmax=self.canvas.pop("xmax", None))
self.extend(plt)#extend line
plt.legend(**self.canvas)
#loc='lower/upper left'
if HOSTOS.startswith("Darwin"):
pass
plt.savefig(self.pngout)
plt.savefig(self.pdfout)
log.debug(self.Id+" fig save to "+self.pngout)
plt.close()
log.info(self.Id+" ends")
async def finish(self, s: Shlyapa):
def player_array_to_dict(score_list):
return dict([(self.players[i].name, v)
for i, v in enumerate(score_list)])
s.calculate_results()
total_score = player_array_to_dict(s.get_total_score_results())
explained_score = player_array_to_dict(s.get_explained_score_results())
guessed_score = player_array_to_dict(s.get_guessed_score_results())
score = {}
for p in self.players:
score[p.name] = {
"total": total_score[p.name],
"explained": explained_score[p.name],
"guessed": guessed_score[p.name]
}
results = {
"score":
score,
"explained":
[player_array_to_dict(t) for t in s.get_explained_table_results()],
"guessed":
[player_array_to_dict(t) for t in s.get_guessed_table_results()]
}
await self.broadcast(message.Finish(results=results))
self.state = HatGame.ST_FINISH
for p in self.players:
"""Reset players to ask new words"""
p.reset() # noqa
log.debug('Game finished')
def timbra_xml(self, xml, id_original=0):
user_token = self._get_token(id_original)
if not user_token:
return ''
method = 'timbra_xml'
data = {
'xml': escape(xml.encode('ascii', 'xmlcharrefreplace').decode('utf-8')),
'rfc': self.rfc,
'user_token': user_token,
'transaccion_id': id_original,
}
data = self._SOAP[method].format(**data).encode('utf-8')
headers = {
'SOAPAction': '"{}"'.format(self._ACTIONS[method]),
'Content-length': len(data),
'Content-type': 'text/xml; charset="UTF-8"'
}
s = Session()
req = Request('POST', self.WS['timbrado'], data=data, headers=headers)
prepped = req.prepare()
try:
response = s.send(prepped, timeout=TIMEOUT)
res = xml2dict.parse(response.text)
ok, res = self._check_fault(res)
if ok:
return ''
return res['RespuestaTimbraXML']['ComprobanteXML']['DatosXML']
except exceptions.Timeout:
self.error = 'Tiempo de espera agotado'
except Exception as e:
self.error = str(e)
log.debug(e)
return ''
def first_express_interest(self):
if self.status == Controller.STATUS_OFF or self.status == Controller.STATUS_STEP_OFF:
return
if self.mydata.final_byte != None and self.mydata.next_byte >= self.mydata.final_byte:
#we do not use is_all to check, since final_byte is more accurate and is_all -> final_byte
log.debug("illegel request, do not express it, next_byte: %s" %(self.mydata.next_byte))
return
chunkinfo = ChunkInfo()
chunkinfo.beginT = datetime.datetime.now()
chunkinfo.begin_byte = self.mydata.next_byte
temp = self.chunkSizeEstimator.get_optimal_data_size()
chunkinfo.end_byte = chunkinfo.begin_byte + temp - 1
chunkinfo.packetN = math.ceil((temp + CHUNK_HEADER_SIZE)/float(self.packet_max_data_size))
chunkinfo.status = 1
name = self.ndn_name
name = self.ndn_name.append(ADAPTIVE_MOD_FLAG).append(str(temp)) #only leave the data size
name = name.append(chunkinfo.begin_byte)
chunkinfo.ndn_name = name
self.chunkInfos.append(chunkinfo)
self.mydata.unsatisfied_chunks_keyed_by_name[str(name)] = chunkinfo
#packet_on_the_fly, 3 results, illegal, out-of-order, in-order and retransmission
self.window.update_express(chunkinfo)
self.express_interest(chunkinfo)
self.mydata.next_byte = chunkinfo.end_byte + 1
def express_interest(self, chunkinfo):
"""this method may express illegal Interest, thus, re_express_interest and first_express_interest are in charge of checking;
even that, there may also illegal Interest, due to unknown final_byte, leading to useless chunkinfo in chunkinfos and illegal Data(Nack) or Interest timeout
(we do not use is_all to check, since final_byte is more accurate and is_all -> final_byte);
thus, we need do_receiving_content to handle illegal Data
"""
assert chunkinfo != None, "chunkinfo == None"
assert chunkinfo.endT == None, "chunkinfo.endT != None"
selector = pyndn.Interest()
selector.answerOriginKind = 0#producer generate every time
selector.childSelctor = 1
selector.interestLifetime = self.rtoEstimator.get_rto()
rst = self.handle.expressInterest(chunkinfo.ndn_name, self, selector)
if rst != None and rst < 0:
log.info("fail to express interest=%s with result %s" %(chunkinfo.ndn_name, rst))
self.window.update_nack(chunkinfo)
chunkinfo.status = 0
else:
chunkinfo.retxN += 1
log.debug("express interest=%s" %(chunkinfo.ndn_name))
async def get(self):
ws = web.WebSocketResponse()
await ws.prepare(self.request)
user = self.request.user
for _ws in self.request.app['websockets']:
await _ws.send_str('%s joined' % user.login)
self.request.app['websockets'].append(ws)
async for msg in ws:
if msg.type == WSMsgType.TEXT:
if msg.data == 'close':
await ws.close()
else:
try:
message = Message(message=msg.data, user_id=user)
await message.commit()
except ValidationError as e:
print(e)
for _ws in self.request.app['websockets']:
await _ws.send_str('{"user": "******", "msg": "%s"}' %
(user.login, msg.data))
elif msg.type == WSMsgType.ERROR:
log.debug('ws connection closed with exception %s' %
ws.exception())
self.request.app['websockets'].remove(ws)
for _ws in self.request.app['websockets']:
await _ws.send_str('%s disconected' % user.login)
log.debug('websocket connection closed')
return ws
def scatter(self):
log.debug(self.Id+" begin to draw ")
plt.clf()
cans = []
for line in self.lines:
log.debug("line.xs="+str(line.xs))
log.debug("line.ys="+str(line.ys))
log.debug("plt atts="+str(line.plt))
#can = plt.plot(line.xs, line.ys, line.plt.pop("style", "-"), **line.plt)
can = plt.scatter(x=line.xs, y=line.ys, s=self.kwargs.get("c", 1), **line.plt)
cans.append(can)
plt.grid(True)
plt.xlabel(self.canvas.pop("xlabel", "X"))
plt.ylabel(self.canvas.pop("ylabel", "X"))
plt.xlim(xmax=self.canvas.pop("xmax", None))
self.extend(plt)#extend line
plt.legend(**self.canvas)
#loc='lower/upper left'
if HOSTOS.startswith("Darwin"):
pass
plt.savefig(self.pngout)
plt.savefig(self.pdfout)
log.debug(self.Id+" fig save to "+self.pngout)
plt.close()
log.info(self.Id+" ends")
async def name(self, ws, data):
name = data['name']
log.debug(f'user {name} logged in as {id(ws)}')
player = Player(name=name, ws=ws)
self.players[name] = player
self.sockets[id(ws)] = player
await self.game(ws)
def _get_token(self, transaccion_id=0):
if not transaccion_id:
transaccion_id = get_epoch()
method = 'obtener_token'
data = {
'rfc': self.rfc,
'transaccion_id': transaccion_id
}
data = self._SOAP[method].format(**data).encode('utf-8')
headers = {
'SOAPAction': '"{}"'.format(self._ACTIONS[method]),
'Content-length': len(data),
'Content-type': 'text/xml; charset="UTF-8"'
}
s = Session()
req = Request('POST', self.WS['seguridad'], data=data, headers=headers)
prepped = req.prepare()
try:
response = s.send(prepped, timeout=TIMEOUT)
res = xml2dict.parse(response.text)
ok, res = self._check_fault(res)
if ok:
return ''
token = res['RespuestaObtenerToken']['Token']
s = '{}|{}'.format(self.ID_INTEGRADOR, token)
user_token = hashlib.sha1(s.encode()).hexdigest()
return user_token
except exceptions.Timeout:
self.error = 'Tiempo de espera agotado'
log.debug(self.error)
print (self.error)
except exceptions.RequestException as e:
log.debug(e)
print ('TOKEN', e)
return ''
def upcall(self, kind, upcallInfo):
if kind == pyndn.UPCALL_FINAL:#handler is about to be deregistered
return pyndn.RESULT_OK
if kind in [pyndn.UPCALL_INTEREST, pyndn.UPCALL_CONSUMED_INTEREST, pyndn.UPCALL_CONTENT_UNVERIFIED, pyndn.UPCALL_CONTENT_BAD]:
log.error("unexpected kind: %s" %kind)
return pyndn.RESULT_OK
if kind == pyndn.UPCALL_CONTENT:
self.is_all = True
log.info("get data back: %s" %(upcallInfo.Interest.name))
self.stop()
return pyndn.RESULT_OK
elif kind == pyndn.UPCALL_INTEREST_TIMED_OUT:
if self.turn == -1:
return pyndn.RESULT_REEXPRESS
else:
if self.ist_sentN < self.turn:
log.debug("timeout: %s" %(upcallInfo.Interest.name))
self.ist_sentN += 1
return pyndn.RESULT_REEXPRESS
else:
log.info("CANNOT get data back after %d trials: %s" %(self.turn, upcallInfo.Interest.name))
self.stop()
return pyndn.RESULT_OK
async def ready(self, ws, msg: message.Ready):
"""Tell server that player is ready to guess/explain"""
if self.state != HatGame.ST_PLAY:
raise Exception(
f"Invalid command 'ready' for game in state '{self.state}'")
exp = self.turn.explaining
gss = self.turn.guessing
sent_by = self.sockets_map[id(ws)]
if sent_by in (exp, gss):
sent_by.ready()
else:
raise Exception('Wrong player sent ready command')
log.debug(
f'Pair state: explain({exp.name})={exp.state} guessing({gss.name})={gss.state}'
)
if exp.state == Player.ST_READY and gss.state == Player.ST_READY:
exp.play()
gss.play()
log.debug(
f'Explanation started: explain {exp.name} guessing {gss.name}')
self.timer = Timer(self.turn_timer, self.expired)
await self.broadcast(message.Start())
await self.next_word()
async def post(self):
ngmsg = message.Newgame.msg(await self.request.json())
game = HatGame(**ngmsg.args())
log.debug(f'New game request - {ngmsg.args()}')
if game.id in self.request.app.games:
return web.Response(
status=500,
content_type='application/json',
text=json.dumps(message.Error(code=103, message='Duplicate game ID').data(), ensure_ascii=False)
)
self.request.app.games[game.id] = game
log.info(f"New game created id={game.id}, name='{game.game_name}''")
headers = {}
if NEED_CORS:
headers = {'Access-Control-Allow-Origin': self.request.headers['Origin']}
return web.Response(
content_type='application/json',
text=json.dumps(game.game_msg().data(), ensure_ascii=False),
headers=headers
)
def upcall(self, kind, upcallInfo):
if kind != pyndn.UPCALL_INTEREST:
log.warn("get kind: %s" %str(kind))
return pyndn.RESULT_OK
co = self.prepare(upcallInfo)
# try:
# co = self.prepare(upcallInfo)
# except:
# thetype, value, traceback = sys.exc_info()
# log.error("get exception: %s, %s, %s" %(thetype, value, traceback))
# co = None
if co == None:
log.warn("co == None")
pass
else:
rst = self.handle.put(co)
if rst < 0:
log.warn("fail put content: %s, result: %s" %(co.ndn_name, rst))
else:
log.debug("successfully put content, and returns %s"%(rst))
pass
#("content: %s" %(co.ndn_name))
return pyndn.RESULT_INTEREST_CONSUMED
async def get_user_by_id(self, **kw):
try:
return self.get(id=self._id, )
except self.DoesNotExist:
return False
except Exception as e:
log.debug(e)
return False
def modpred(start, finish):
try:
result = df[df['DATE'].between(pd.to_datetime(start),
pd.to_datetime(finish))]
return result.to_json(orient="records")
except Exception as e:
log.debug(f'error: {e}')
return json.dumps({"ERROR": str(repr(e)), "Data": None})
async def check_room(self, name, *args, **kwargs):
try:
return self.get(name=name, )
except self.DoesNotExist:
return False
except Exception as e:
log.debug(e)
return False
async def get_room_by_id(self, id, *args, **kwargs):
try:
return self.get(id=int(id), )
except self.DoesNotExist:
return False
except Exception as e:
log.debug(e)
return False
async def authorize(request, handler):
session = await get_session(request)
if (not session.get('uid')) and (not request.path.startswith('/login')):
url = request.app.router['login'].url_for()
log.debug('redirecting to {}'.format(str(url)))
raise web.HTTPFound(url)
response = await handler(request)
return response
async def words(self, ws, data):
words = data['words']
p = self.sockets[id(ws)]
p.set_words(words)
log.debug(f'user {p.name} sent words: {words}')
# send prepare message to all players
for p in self.players.values():
await self.prepare(p.socket)
def out_of_order_content(self, upcallInfo):
"""do nothing here, just buffer it and leave it to expected chunk come
do not update the window when out-of-order, until in-order chunk is received
"""
name = str(upcallInfo.Interest.name)
chunkinfo = self.mydata.unsatisfied_chunks_keyed_by_name.get(name)
self.mydata.satisfied_chunkN += 1
self.window.update_receiving(chunkinfo)
log.debug("received out-of-order Data: %s, out packet: %s" %(name, self.window.packet_on_the_fly))
def _check_fault(self, res):
self.error = ''
FAULT = 's:Fault'
fault = res['s:Envelope']['s:Body']
if FAULT in fault:
self.error = 'Código de Error: {}\n\n{}'.format(
fault[FAULT]['faultcode'], fault[FAULT]['faultstring']['#text'])
log.debug(self.error)
return True, ''
return False, fault
def update_loss(self, chunkinfo):
self.packet_on_the_fly -= chunkinfo.packetN
if self.is_fix == None:
self._cwnd = int(self._cwnd/2.0) #sawtooth
if self._cwnd < 1:
self._cwnd = 1
self.threshold = self._cwnd #fast recovery
log.debug("change cwnd to %s, threshold to %s, packet_on_the_fly to %s" %(self._cwnd, self.threshold, self.packet_on_the_fly))
self.sample()
def update_receiving(self, chunkinfo):
self.packet_on_the_fly -= chunkinfo.packetN
if self.is_fix == None:
if self._cwnd < self.threshold:
self._cwnd += chunkinfo.packetN #slow start
else:
self._cwnd += chunkinfo.packetN * 1.0/self._cwnd #additive increase
if self._cwnd > MAX_WINDOWSN:
self._cwnd = MAX_WINDOWSN
self.sample()
log.debug("change cwnd to %s, threshold to %s, packet_on_the_fly to %s" %(self._cwnd, self.threshold, self.packet_on_the_fly))
async def check_user(self, **kw):
try:
return self.get(
login=self._login,
password=self._password,
)
except self.DoesNotExist:
return False
except Exception as e:
log.debug(e)
return False
async def get(self):
session = await get_session(self.request)
uid = session.get('uid')
message = Message()
messages = await message.get_messages('group_interns')
log.debug(f'group_interns messages: {messages}')
for message in messages['messages']:
if message.get('uid') != None:
if message['uid'] == uid:
message['own'] = "yes"
return messages
def re_express_interest(self, chunkinfo):
if self.status == Controller.STATUS_OFF or self.status == Controller.STATUS_STEP_OFF:
return
if self.mydata.final_byte !=None and chunkinfo.begin_byte >= self.mydata.final_byte: #suppress the illegal reuqest
log.debug("illegel request, do not re-express it: %s" %(chunkinfo.ndn_name))
elif self.is_all:#shouldn't happen, since already check before re-expressing
log.error("already get all the requests, do not re-express it: %s" %(chunkinfo.ndn_name))
else:
self.window.update_express(chunkinfo)
self.chunkSizeEstimator.get_optimal_data_size()
self.express_interest(chunkinfo)
async def words(self, ws, msg: message.Words):
"""Player sends it's words to server"""
words = msg.words
if len(words) < self.num_words:
for wi in range(0, self.num_words):
words.append(NOUNS.get_random_word())
p = self.sockets_map[id(ws)]
p.words = words
log.debug(f'user {p.name} sent words: {words}')
await self.prepare()
def read(self, begin_byte, data_size):
if self.fd.closed:
self.fd = open(fpath)
if begin_byte >= self.fsize:
return None
if self.fd.tell() != begin_byte:
log.debug("move to %s" %(begin_byte))
self.fd.seek(begin_byte)
data = self.fd.read(data_size)
return data
async def get(self):
log.debug('websocket new connection')
ws = web.WebSocketResponse()
self.request.app.websockets.append(ws)
await ws.prepare(self.request)
async for msg in ws:
log.debug(
f'websocket message received: {msg.type}: {msg.data.strip()}')
if msg.type == WSMsgType.text:
if msg.data == 'close':
await ws.close()
else:
try:
data = json.loads(msg.data)
except Exception as e:
await self.error(ws, 101,
f'Broken message received {e}')
continue
if not 'cmd' in data:
await self.error(ws, 102,
f'Invalid message format {msg.data}')
continue
else:
cmdtxt = data['cmd']
cmd = getattr(self.request.app.game, cmdtxt, None)
if not callable(cmd):
await self.error(ws, 103, f'Unknown command {cmdtxt}')
continue
try:
log.debug(f'Received command {cmdtxt}')
await cmd(ws, data)
except Exception as e:
log.exception('Error executing command {cmdtxt}: {e}')
await self.error(
ws, 104, f'Error executing command {cmdtxt}: {e}')
elif msg.type == WSMsgType.error:
log.debug('ws connection closed with exception %s' %
ws.exception())
self.request.app.websockets.remove(ws)
log.debug('websocket connection closed')
return ws
async def get(self):
session = await get_session(self.request)
uid = 'user{0}'.format(random.randint(1, 1001))
uids = self.request.app['uids']
while uid in uids:
uid = 'user{0}'.format(random.randint(1, 1001))
uids.append(uid)
self.request.app['uids'] = uids
session['uid'] = uid
log.debug(uid)
redirect(self.request, 'general')
async def expired(self):
"""Guessing time expired"""
self.timer = None
try:
log.debug('Turn timer expired, stop turn')
await self.broadcast(message.Stop(reason='timer'))
self.turn.explaining.lastanswer()
self.turn.guessing.finish()
except Exception as e:
log.error(f'Exception while process timer: {e}')
log.exception()
def express_interest(self):
name = self.ndn_name
#name = self.ndn_name.append(800)
name = self.ndn_name.append(ADAPTIVE_MOD_FLAG).append(800) #only leave the data size
name = name.append(0)
selector = pyndn.Interest()
selector.answerOriginKind = 0#producer generate every time
selector.childSelctor = 1
selector.interestLifetime = self.ist_lifetime
self.handle.expressInterest(name, self, selector)
log.debug("send Interest %s" %(name))
self.ist_sentN += 1
async def get(self):
log.info("rest api request ... /pingmodel from " +
str(self.request.remote))
if (self.request.headers.get("X-Auth-Token") !=
config.AUTHENTICATION_TOKEN_PINGMODEL):
log.info("error authentication token ... " + "from " +
str(self.request.remote) + " [" +
str(self.request.headers.get("X-Auth-Token")) + "]")
return web.Response(status=401,
text=str('{"error":"authentication token"}'))
log.info("authentication token ... " + "from " +
str(self.request.remote) + " [Ok]")
log.debug(str('{"status":"started"}'))
text = '{"status":"started"}'
return web.Response(text=text)
def fetch_wikipedia_raw_page(year, month, day):
wiki_url = WIKIPEDIA_PAGE_TEMPLATE.format(YYYY=year,
Month=MONTHS[month],
D=day)
log.debug('GET %s', wiki_url)
r = requests.get(wiki_url)
try:
r.raise_for_status()
except:
log.error('NOT FOUND')
return 'NOT FOUND: ' + wiki_url
return r.text
async def get(self):
session = await get_session(self.request)
self.self_id = session.get('user')
self.login = session.get('login')
self.user = self.request.app['models']['user']
self.unread = self.request.app['models']['unread']
self.message = self.request.app['models']['message']
self.company = self.request.app['models']['company']
self.company_id = self.request.rel_url.query.get('company_id')
ws = web.WebSocketResponse()
await ws.prepare(self.request)
my_companys = await self.company.get_company_by_user(self.self_id)
for c in my_companys:
self.request.app['websockets'][str(c['_id'])].append(ws)
for _ws in self.request.app['websockets'][str(c['_id'])]:
await _ws.send_json({
'user_id': self.self_id,
'type': 'joined'
})
self.request.app['online'][self.self_id] = ws
async for msg in ws:
if msg.type == WSMsgType.TEXT:
data = json.loads(msg.data)
if msg.data == 'close':
await ws.close()
else:
if data['type'] == 'company_chat_mess':
await self.recieve_company_chat_mess(data)
elif data['type'] == 'private_chat_mess':
await self.recieve_private_chat_mess(data)
elif data['type'] == 'mess_notification':
pass
elif data['type'] == 'event_notification':
pass
elif msg.type == WSMsgType.ERROR:
log.debug('ws connection closed with exception %s' %
ws.exception())
for c in my_companys:
self.request.app['websockets'][str(c['_id'])].remove(ws)
await ws.close()
return ws
async def get(self):
ws = web.WebSocketResponse()
await ws.prepare(self.request)
session = await get_session(self.request)
message = Message()
session_user = session.get('user')
username = session_user.get('username')
for _ws in self.request.app['websockets']:
await _ws.send_str('%s joined' % username)
self.request.app['websockets'].append(ws)
async for msg in ws:
if msg.type == WSMsgType.text:
for _ws in self.request.app['websockets']:
time_chat = datetime.now()
if len(history) > settings.MAX_MSG:
del history[0]
history.append({
'time': time_chat,
'user': username,
'msg': msg.data
})
await message.save_msg({
'user_id': session_user.get('id'),
'created_at': time_chat,
'msg': msg.data
})
await _ws.send_str('(%s) %s' % (username, msg.data))
elif msg.type == WSMsgType.error:
log.debug('ws connection closed with exception %s' %
ws.exception())
self.request.app['websockets'].remove(ws)
for _ws in self.request.app['websockets']:
await _ws.send_str('%s disconected' % username)
log.debug('websocket connection closed')
return ws
def make_data(invoice):
data = {}
try:
tree = ET.parse(invoice).getroot()
version = tree.attrib['version']
node = tree.find('{}Emisor'.format(PRE[version]))
data['total'] = tree.attrib['total']
data['emisor_rfc'] = node.attrib['rfc']
node = tree.find('{}Receptor'.format(PRE[version]))
data['receptor_rfc'] = node.attrib['rfc']
node = tree.find('{}Complemento/{}TimbreFiscalDigital'.format(PRE[version], PRE['TIMBRE']))
data['uuid'] = node.attrib['UUID']
return data
except Exception as e:
log.error('La factura no se pudo parsear, asegurate de que sea un CFDI válido')
log.debug(e)
return {}
async def guessed(self, ws, msg: message.Guessed):
"""Guessing User tell us of he guessed right"""
sent_by = self.sockets_map[id(ws)]
if sent_by != self.turn.explaining:
raise ValueError(
f'Only explaining player can send guessed command,'
f' but sent by {sent_by} in {sent_by.state} state')
if sent_by.state not in (Player.ST_PLAY, Player.ST_LAST_ANSWER):
raise ValueError(
f"Player {sent_by} can't sent guessed command while not in play"
)
self.turn.guessed(result=msg.guessed)
if msg.guessed:
m = message.Explained(word=self.turn.word)
else:
m = message.Missed()
await self.broadcast(m)
if self.turn.explaining.state == Player.ST_LAST_ANSWER:
"""Answer after timer exhausted"""
self.turn.explaining.finish()
await self.next_move(explained_words=self.turn.result())
return
if not self.has_words():
"""End of turn"""
if self.timer:
self.timer.cancel()
log.debug('No more words - next turn')
await self.broadcast(message.Stop(reason='empty'))
self.turn.explaining.finish()
self.turn.guessing.finish()
await self.next_move(explained_words=self.turn.result())
return
await self.next_word()
async def get(self):
message = Message()
ws = web.WebSocketResponse()
await ws.prepare(self.request)
session = await get_session(self.request)
uid = session.get('uid')
#broadcast joining of new user
join_msg = ('"%s has joined the chat"' % (uid)) #message
for _ws in self.request.app['websockets_interns']:
await _ws.send_str(join_msg)
self.request.app['websockets_interns'].append(ws)
#send client's id to this particular client for frontend
await ws.send_str('{"myID": "%s"}' % (uid))
async for msg in ws:
if msg.type == WSMsgType.TEXT:
if msg.data == 'exit-chat':
await ws.close()
else:
t = datetime.now(tzutc())
format_str = "%H:%M"
t_string = t.strftime(format_str)
json_message = '{{"from_user": "******",\
"uid": "{0}",\
"msg": "{1}",\
"time": "{2}"}}'.format(uid, msg.data, t_string)
log.debug(json_message)
await message.send_message('group_interns', json_message)
for _ws in self.request.app['websockets_interns']:
await _ws.send_str('{"user": "******", "msg": "%s", "time": "%s"}' % (uid, msg.data, t_string))
elif msg.type == WSMsgType.ERROR:
log.debug('ws connection closed with exception {0}'.format(ws.exception()))
#broadcast leaving of the user
self.request.app['websockets_interns'].remove(ws)
leave_msg = ('"%s has left the chat"' % (uid)) #message
for _ws in self.request.app['websockets_interns']:
await _ws.send_str(leave_msg)
return ws
def insert_wikipedia_raw(date_str):
with PG_CONN.cursor() as cur:
c = 0
for year, month, day in date_generator(date_str):
event_raw = fetch_wikipedia_raw_page(year, month, day)
event_date = datetime(year, month, day)
cmd = 'INSERT INTO {table} (event_date, event_raw) VALUES (%s, %s);'
cmd = cmd.format(table=WIKIPEDIA_TABLE_RAW)
log.debug(cmd)
cur.execute(cmd, (event_date, event_raw))
c += 1
if c % 20:
PG_CONN.commit()
PG_CONN.commit()
log.debug('Done.')
async def next_move(self):
"""Do next move, called on game start or after move finished"""
explained_words = None
if self.turn:
self.turn.explaining.wait()
self.turn.guessing.wait()
explained_words = self.turn.result()
# return to pool words which was miss-guessed by previous pair
missed_words = self.turn.missed_words
if len(missed_words):
log.debug(f"Return #{len(missed_words)} words to hat")
self.tour_words.extend(missed_words)
s = self.shlyapa
if explained_words is not None: # non-first move
log.debug(f'Turn over, explained words={explained_words}')
s.move_shlyapa(pair_explained_words=explained_words)
if s.is_cur_tour_new():
await self.tour()
if s.is_end():
await self.finish(s)
return
log.debug(f'New turn #{s.get_cur_turn()}')
pair_idx = s.get_next_pair()
exp = self.players[pair_idx.explaining]
gss = self.players[pair_idx.guessing]
self.turn = Turn(explaining=exp, guessing=gss)
log.debug(f'In hat #{len(self.tour_words)}')
log.debug(f'Pair selected: explain={exp} guessing={gss}')
exp.begin() # noqa
gss.begin() # noqa
m = message.Turn(turn=s.get_cur_turn(),
explain=exp.name,
guess=gss.name)
await self.broadcast(m)
def in_order_content(self, upcallInfo):
"""the best scenario, content is received in-order, however, we should check those buffered out-of-order chunks
"""
name = str(upcallInfo.Interest.name)
chunkinfo = self.mydata.unsatisfied_chunks_keyed_by_name.pop(name)
if not self.fout.closed:
self.fout.write(chunkinfo.content)
else:
log.critical("fails to write content")
self.mydata.accept_bytes += chunkinfo.data_size
if not self.cache_data:
chunkinfo.content = None
chunkinfo.status = 2 #satisfied yet
self.mydata.satisfied_chunkN += 1
self.mydata.expected_chunkI += 1
self.window.update_receiving(chunkinfo)
log.debug("received in-order Data: %s, out packet: %s" %(name, self.window.packet_on_the_fly))
#check the out-of-order contents recevied before
for name in self.mydata.unsatisfied_chunks_keyed_by_name.keys():
chunkinfo = self.mydata.unsatisfied_chunks_keyed_by_name[name]
if chunkinfo.endT == None:
break
else:
chunkinfo = self.mydata.unsatisfied_chunks_keyed_by_name.pop(name)
if not self.fout.closed:
self.fout.write(chunkinfo.content)
else:
log.critical("fails to write content")
self.mydata.accept_bytes += chunkinfo.data_size
if not self.cache_data:
chunkinfo.content = None
self.mydata.expected_chunkI += 1
chunkinfo.status = 2 #satisfied yet
if self.mydata.final_byte == self.mydata.accept_bytes:
self.is_all = True
for chunkinfo in self.mydata.unsatisfied_chunks_keyed_by_name.itervalues():
log.warn(str(chunkinfo))
log.warn("------------------------ %s all the contents are received---------------------------" %(self.Id))
self.stop()
async def get(self):
ws = web.WebSocketResponse()
await ws.prepare(self.request)
session = await get_session(self.request)
user = User(self.request.db, {'id': session.get('user')})
login = await user.get_login()
for _ws in self.request.app['websockets']:
_ws.send_str('%s joined' % login)
self.request.app['websockets'].append(ws)
async for msg in ws:
if msg.tp == MsgType.text:
if msg.data == 'close':
await ws.close()
else:
message = Message(self.request.db)
result = await message.save(user=login, msg=msg.data)
log.debug(result)
for _ws in self.request.app['websockets']:
_ws.send_str('(%s) %s' % (login, msg.data))
elif msg.tp == MsgType.error:
log.debug('ws connection closed with exception %s' % ws.exception())
self.request.app['websockets'].remove(ws)
for _ws in self.request.app['websockets']:
_ws.send_str('%s disconected' % login)
log.debug('websocket connection closed')
return ws
async def get(self):
ws = web.WebSocketResponse()
await ws.prepare(self.request)
session = await get_session(self.request)
user = User(self.request.app.db, {'id': session.get('user')})
login = await user.get_login()
for _ws in self.request.app['websockets']:
_ws.send_str('%s joined' % login)
self.request.app['websockets'].append(ws)
async for msg in ws:
if msg.type == WSMsgType.TEXT:
if msg.data == 'close':
await ws.close()
else:
message = Message(self.request.app.db)
result = await message.save(user=login, msg=msg.data)
log.debug(result)
for _ws in self.request.app['websockets']:
await _ws.send_str('{"user": "******", "msg": "%s"}' %
(login, msg.data))
elif msg.type == WSMsgType.ERROR:
log.debug('ws connection closed with exception %s' %
ws.exception())
self.request.app['websockets'].remove(ws)
for _ws in self.request.app['websockets']:
_ws.send_str('%s disconected' % login)
log.debug('websocket connection closed')
return ws
async def get(self):
ws = web.WebSocketResponse()
await ws.prepare(self.request)
session = await get_session(self.request)
user = UserMySQL(id=session.get('user'))
user = await user.get_user_by_id()
for _ws in self.request.app['websockets']:
_ws.send_str('%s joined' % user.login)
self.request.app['websockets'].append(ws)
async for msg in ws:
if msg.tp == MsgType.text:
if msg.data == 'close':
await ws.close()
else:
message = MessageMySQL(user_id=session['user'],
room_id=session['room_id'])
message = await message.insert_message_to_db(msg=msg.data)
log.debug('result:%s' % message)
for _ws in self.request.app['websockets']:
_ws.send_str('(%s) %s' % (user.login, msg.data))
elif msg.tp == MsgType.error:
log.debug('ws connection closed with exception %s' %
ws.exception())
self.request.app['websockets'].remove(ws)
for _ws in self.request.app['websockets']:
_ws.send_str('%s disconected' % user.login)
log.debug('websocket connection closed')
return ws
async def get(self):
ws = web.WebSocketResponse()
await ws.prepare(self.request)
session = await get_session(self.request)
async with self.request.app['db'].acquire() as conn:
data = await self.request.post()
login = await get_login(conn, session.get('user'))
for _ws in self.request.app['websockets']:
_ws.send_str(f'{login}s joined')
self.request.app['websockets'].append(ws)
async for msg in ws:
if type(msg) == WSMsgType.TEXT:
# if msg.data == 'close':
async with self.request.app['db'].acquire() as conn:
result = await save(conn, login, data['content'])
log.debug(result)
for _ws in self.request.app['websockets']:
await _ws.send_str({
'user_id': data['user_id'],
'content': data['content'],
})
elif type(msg) == WSMsgType.ERROR:
log.debug(f'ws connection closed with exception {ws.exception()}')
self.request.app['websockets'].remove(ws)
for _ws in self.request.app['websockets']:
_ws.send_str(f'{login}s disconnected')
log.debug('websocket connection closed')
return ws
def bar(self):
log.debug(self.Id+" begin to draw ")
plt.clf()
self.bars = []
Width = self.canvas.pop("width", 0.2)
for i in range(len(self.lines)):
line = self.lines[i]
#for line in self.lines:
print "line.xs=",line.xs
print "line.ys=", line.ys
xs = [x+i*Width for x in line.xs]
bar = plt.bar(left=xs, height=line.ys, width=Width, bottom=0, align="center", **line.plt)
self.bars.append(bar)
#plt.legend( (p1[0], p2[0]), ('Men', 'Women') )
plt.legend((self.bars[i][0] for i in range(len(self.lines))), (self.lines[i].plt["label"] for i in range(len(self.lines))))
#for bar in self.bars:
xs = [Width * len(self.lines)/2 + j for j in line.xs]
plt.xticks(xs, line.xs)
plt.grid(True)
plt.xlabel(self.canvas.pop("xlabel", " "))
plt.ylabel(self.canvas.pop("ylabel", " "))
plt.legend(**self.canvas)
plt.title(self.canvas.pop("title", " "))
#plt.xticklabels([])
self.extend(plt)
plt.legend(**self.canvas)
plt.savefig(self.pngout)
plt.savefig(self.pdfout)
log.debug(self.Id+" fig save to "+self.pngout)
plt.close()
log.info(self.Id+" finishes")
def do_meet_accident(self, kind, upcallInfo):
name = str(upcallInfo.Interest.name)
if not name in self.mydata.unsatisfied_chunks_keyed_by_name:
#since it's not possible that two same Interest on the fly at the same time, it sholdn't happen
log.error("timeout Interest not in the unsatisfied list, it should not happend: %s!!" %(name))
return
chunkinfo = self.mydata.unsatisfied_chunks_keyed_by_name[name]
self.chunkSizeEstimator.update_loss(chunkinfo)
self.window.update_loss(chunkinfo)
if kind == 4:
log.debug("timeout, Interest=%s, out packet: %d" \
%(upcallInfo.Interest.name, self.window.packet_on_the_fly))
#log.warn("%s" %(upcallInfo))
else:
log.warn("-----------------strange accident: kind=%s, Interest=%s------------------" %(kind, upcallInfo.Interest.name))
#window check here
if self.window.packet_on_the_fly < self.window.get_cwnd():
#it's already make sure that the chunk is not satisfied yet, but it could be illegal
self.re_express_interest(chunkinfo)
else:
chunkinfo.status = 0 #wait for re-expressing
def do_receive_content(self, kind, upcallInfo):
"""receive a contents, there are 4 different scenarios: duplicated content, in-order content, out-of-order content, illegal content
"""
name = str(upcallInfo.Interest.name)
if not name in self.mydata.unsatisfied_chunks_keyed_by_name:
log.debug(self.mydata.unsatisfied_chunks_keyed_by_name.keys())
#the chunkinfo is already satisfied by previous chunk (retransmission here)
self.duplicate_content(upcallInfo)
return
chunkinfo = self.mydata.unsatisfied_chunks_keyed_by_name[name]
chunkinfo.endT = datetime.datetime.now()
chunkinfo.data_size = len(upcallInfo.ContentObject.content)
chunkinfo.chunk_size = len(_pyndn.dump_charbuf(upcallInfo.ContentObject.ndn_data))
chunkinfo.content = upcallInfo.ContentObject.content
temp = math.ceil((chunkinfo.chunk_size)/float(self.packet_max_data_size))
self.rtoEstimator.update(chunkinfo)
fbi = upcallInfo.ContentObject.signedInfo.finalBlockID
if fbi != None:
if isinstance(fbi, str):
fbi = pyndn.Name.seg2num(fbi)
#log.info("***************final chunk id: %s" %(fbi))
if self.mydata.final_byte == None: #the first final block content
self.mydata.final_byte = int(fbi)
else:
assert self.mydata.final_byte == int(fbi), "get different final block id, old %s and new %s" %(self.mydata.final_byte, int(fbi))
si = upcallInfo.ContentObject.signedInfo
if si.type == pyndn.CONTENT_NACK:
self.nack_content(upcallInfo)
elif si.type == pyndn.CONTENT_DATA:
if chunkinfo.packetN != temp:
if self.mydata.final_byte != None and chunkinfo.end_byte > self.mydata.final_byte:#final chunk or illegal chunk
log.debug("final chunk, thus size is shorter than expected")
else:
log.debug("expected packetN (%s) != real packetN (%s), final_byte (%s), upcallInfo: %s, chunksize:%s" %(chunkinfo.packetN, temp, self.mydata.final_byte, name, chunkinfo.chunk_size))
#chunkinfo.packetN = temp
self.chunkSizeEstimator.update_receiving(chunkinfo)
# if self.mydata.final_byte!=None and chunkinfo.end_byte < self.mydata.final_byte:
# assert chunkinfo.data_size > 500, "chukinfo is strange, %s" %(chunkinfo)
#
if name == self.mydata.unsatisfied_chunks_keyed_by_name.keys()[0]:
self.in_order_content(upcallInfo)
else:
self.out_of_order_content(upcallInfo)
retxQ = []
for chunkinfo in self.mydata.unsatisfied_chunks_keyed_by_name.itervalues():
if chunkinfo.status == 0:#waiting for re-expressing
retxQ.append(chunkinfo)
if len(retxQ) == 2:
break
#here we do not check whether the request is legal or not
for i in [0, 1]:#mulply add
if self.window.packet_on_the_fly < self.window.get_cwnd():
#re-expressing is prior to request new
if len(retxQ) != 0:
chunkinfo = retxQ.pop(0)
chunkinfo.status = 1
self.re_express_interest(chunkinfo)
continue
if self.mydata.final_byte== None:
self.first_express_interest()
elif self.mydata.final_byte!= None and self.mydata.next_byte < self.mydata.final_byte:
self.first_express_interest()
else:
log.critical("unkown Data type: %s" %(upcallInfo.ContentObject))
async def game_detail_ws(request):
''' Game websocket handler '''
session = await get_session(request)
user_id = int(session.get('user'))
game_id = request.match_info['id']
ws = web.WebSocketResponse(autoclose=False)
await ws.prepare(request)
opened_ws = request.app['websockets'][game_id]
games = Games(request.db)
game_users = await games.get_users(game_id)
data_users = []
for user in game_users:
data_users.append({
'id': user.id,
'login': user.login
})
for _ws in opened_ws:
_ws.send_str(json.dumps({'message': '{} joined'.format(user_id),
'status': STATUS['INFO']}))
_ws.send_str(json.dumps({'message': 'update user list',
'status': STATUS['UPDATE_USERS'], 'users': data_users}))
opened_ws.append(ws)
async for msg in ws:
if msg.tp == MsgType.text:
if msg.data == 'close':
await ws.close()
else:
data = json.loads(msg.data)
try:
# Check need attributes
if 'i' not in data or 'j' not in data:
raise Exception('Positions of move are required.')
# Check if user in game
if not any(user.id == user_id for user in game_users):
raise Exception('You cannot play this game. Create your own to game.')
game_info = await games.one(game_id)
# Check if game was not ended
if game_info.winner_id:
raise Exception('Game is over.')
game_moves = await games.get_moves(game_id)
check_pairs_moves = ((game_info.config_size**2 - len(game_moves)) % 2 == 0)
check_pairs_user = ((next(index for index, user in enumerate(game_users) if user.id == user_id) + 1) % 2 == 0)
# Check if current user must move
if check_pairs_user != check_pairs_moves:
raise Exception('It is not your turn.')
data_moves = [[0]*game_info.config_size for i in range(game_info.config_size)]
for moves in game_moves:
data_moves[moves.x][moves.y] = str(moves.users_id)
# Check if the field is available
if data_moves[data['i']][data['j']]:
raise Exception('This field is not available.')
data_moves[data['i']][data['j']] = str(user_id)
# Save move
await games.save_move(user_id, game_id, data['i'], data['j'])
# Check game for winner
winner_id = await check_for_winner(data_moves)
# Draw
if game_info.config_size**2 == (len(game_moves) + 1) and not winner_id:
winner_id = DRAW
# if we find winner or draw -> game is end.
if winner_id:
await games.finish_game(game_id, winner_id)
# if user alone in game and make move
next_user_id = -1
if len(game_users) == PLAYERS_IN_GAME:
next_user_id = next(user.id for user in game_users if user.id != user_id)
context = {
'status': STATUS['OK'],
'winner_id': winner_id,
'next_user_id': next_user_id,
'current_user_id': user_id,
'i': data['i'],
'j': data['j'],
'message': '{} made choice'.format(user_id)
}
except Exception as e:
print(str(e))
context = {
'status': STATUS['ERROR'],
'message': str(e)
}
for _ws in opened_ws:
_ws.send_str(json.dumps(context))
elif msg.tp == MsgType.error:
log.debug('ws connection closed with exception {}'.format(ws.exception()))
opened_ws.remove(ws)
for _ws in opened_ws:
_ws.send_str(json.dumps({'message': '{} disconected'.format(user_id), 'status': STATUS['INFO']}))
log.debug('websocket connection closed')
return ws
def get_optimal_data_size(self):
"""only data, not including the chunk header size
Return:
chunk_data_size: the size of data contained in a chunk, chunk header/signature ... are not included
"""
if (self.receivedN + self.lostN) == 0 or float(self.lostN)/(self.receivedN + self.lostN) == 0:
if self.receivedN > 20: #quite reliable underlying layer protocol
chunk_data_size = MAX_DATA_SIZE
else: #just in prelimary stage
chunk_data_size = DEFAULT_DATA_SIZE
if (self.receivedN + self.lostN) != 0:
self._loss_rate = float(self.lostN)/(self.receivedN + self.lostN)
else:
#ref to:
loss_rate = float(self.lostN)/(self.receivedN + self.lostN) # 1- \omega
self._loss_rate = loss_rate
if loss_rate == 1:
chunk_data_size = self._optimal_size/2
if chunk_data_size <= 1000:
chunk_data_size = 1000
self._optimal_size = chunk_data_size
log.debug("optimal chunk data size: %s, loss rate: %s" %(chunk_data_size, self._loss_rate))
self.sample()
return chunk_data_size
if self._optimal_size == 0 :
self._optimal_size = DEFAULT_DATA_SIZE
M = self.packet_max_data_size
T = pow((1 - loss_rate),1 / math.ceil((CHUNK_HEADER_SIZE + self._optimal_size) /float( M )))
self.T = T
D = CHUNK_HEADER_SIZE #\Delta
#log.debug("lossrate=%s" %(loss_rate))
#print "loss_rate is %s" %loss_rate
chunk_data_size = (-1*D*math.log(T) -pow((D*(math.log(T)))**2-4*M*math.log(T)*D, 0.5))/(2*math.log(T)) #data size of the chunk
k = int (math.ceil((CHUNK_HEADER_SIZE + chunk_data_size) / M))
temp = k
r1 = (T**temp) * (temp * M) / (CHUNK_HEADER_SIZE + temp*M)
temp = k+1
r2 = (T**temp) * (temp * M) / (CHUNK_HEADER_SIZE + temp*M)
if r1 > r2:
chunk_data_size = k * M - CHUNK_HEADER_SIZE
else:
chunk_data_size = (k+1) * M - CHUNK_HEADER_SIZE
if chunk_data_size > MAX_DATA_SIZE:
chunk_data_size = MAX_DATA_SIZE
self._optimal_size = chunk_data_size
if self.is_fix != None:
chunk_data_size = self.is_fix
self._optimal_size = chunk_data_size
log.debug("optimal chunk data size: %s, loss rate: %s" %(chunk_data_size, self._loss_rate))
self.sample()
return chunk_data_size
def prepare(self, upcallInfo):
ist = upcallInfo.Interest
ist_name = ist.name
flag_index = None #len(ist_name) - 2 #index of the end component
for i in range(len(ist_name)-2):
sub = ist_name[i]
if sub == ADAPTIVE_MOD_FLAG:
flag_index = i
break
if flag_index == None:
log.error("not a flow consumer's interest, ignore: %s" %(ist_name))
return None
expected_data_size = int(ist_name[flag_index+1])
begin_byte = int(ist_name[flag_index+2])
name = ist_name[:flag_index] #not include the flag
name_str = str(name)
if name_str in self.readers:
reader = self.readers[name_str]
log.debug("%s in reader" %(name_str))
else:
if self.is_dir:
subpath = ist_name[upcallInfo.matchedComps:flag_index]
fpath = self.path
for i in range(upcallInfo.matchedComps, flag_index):
temp = ist_name[i]
fpath = os.path.join(fpath, temp)
#assume that matchedComps is the number of matched components, not index
if not os.path.exists(fpath):
log.critical("path %s from Interest %s does not exist" %(fpath, ist_name))
return None
if os.path.isdir(fpath):
log.critical("path %s from Interest %s is not a file" %(fpath, ist_name))
return None
else:#not serve all the directory
if upcallInfo.matchedComps != flag_index:
log.critical("umatched ndn_name: %s, %s"%(ist_name, self.ndn_name))
return None
else:
fpath = self.path
reader = Reader(fpath=fpath)
self.readers[name_str] = reader
log.debug("add a new reader: %s" %(name_str))
data = reader.read(begin_byte, expected_data_size)
if data == None:
log.critical("Interest %s: begin_byte %s > file_size %s" %(ist_name, begin_byte, reader.fsize))
nack = self._nack_template(ist_name, reader)
return nack
else:
log.info("Interest: %s, expected_data_size: %s, begin_byte: %s, data_size: %s" \
%(ist.name, expected_data_size, begin_byte, len(data)))
co = self._data_template(ist_name, data, reader.fsize, pyndn.CONTENT_DATA)
return co
