async def get_stations(line: str) -> dict:
# line = 'hokkaido/chitose-line'
logger.debug(f'expected: "hokkaido/chitose-line" got: {line}')
response = await request('GET', urljoin(DOMAIN, line))
if response:
tree = Selector(response.text)
return Dict.name_link(tree, XPATH_TO_STATIONS)
async def extract_station(station: str) -> list:
# station = "/hokkaido/chitose_00078-st/list"
logger.debug(f'expected: "/hokkaido/chitose_00078-st/list" got: {station}')
response = await request(
'GET', JSON_BY_STATION.format(Text.extract_station(station)))
if response:
return get_rooms_info(json.loads(response.text))
def received_hello(self, hello_id):
""" Registers the receival of a hello packet by one of the receivers.
If the link is anomalous, then decrease its frequency again.
Args:
hello_id (int): the id of the hello packet
"""
logger.debug(f"Received Hello Packet : {hello_id}")
if hello_id in list(self.pending_hello_pkts):
pkt_info = self.pending_hello_pkts[hello_id]
freq = pkt_info["freq"]
# if hello freq was increased, decrease it after anomalous behavior is no longer observed
if freq < self.hello_interval:
freqLinks = self.fault_freq[freq]
sId = pkt_info["space_switch_id"]
freqLinks.stageOneLinks[sId].discard(pkt_info["in_link"])
freqLinks.stageThreeLinks[sId].discard(pkt_info["out_link"])
decFreq = freq + 1
decFreqLinks = self.fault_freq[decFreq]
decFreqLinks.stageOneLinks[sId].add(pkt_info["in_link"])
decFreqLinks.stageThreeLinks[sId].add(pkt_info["in_link"])
del self.pending_hello_pkts[hello_id]
fault_links = [
(1, pkt_info["in_link"], pkt_info["space_switch_id"]),
(3, pkt_info["space_switch_id"], pkt_info["out_link"])
]
# Reset anomaly counter for the link
for link in fault_links:
if link in self.anomaly_count:
del self.anomaly_count[link]
else:
logger.info(f"Past threshold arrival of Hello Packet : {hello_id}")
pass
async def get_lines(city: str) -> list:
# city = 'https://www.good-monthly.com/okinawa/search/select_line.html'
logger.debug(f'expected: "https://www.good-monthly.com/okinawa/search/select_line.html" got: {city}')
response = await request('GET', city)
if response:
tree = Selector(response.text)
return Dict.get_name_link(tree)
async def get_stations(line: str) -> list:
# line = 'https://www.good-monthly.com/search/select_station.html?rosen_cd=523'
logger.debug(f'expected: "https://www.good-monthly.com/search/select_station.html?rosen_cd=523" got: {line}')
response = await request('GET', line)
if response:
tree = Selector(response.text)
return Dict.get_name_link(tree)
def read_data(self):
result = {}
for sheet_name in self.sheets:
logger.debug("reading %s" % sheet_name)
if sheet_name not in self.sheets_names:
# 不在自己需要的表明则跳出本次循环，不进行数据获取否则会出错
continue
# 选择表单
self.sh = self.wb[sheet_name]
# 按行读取数据转化为列表
rows_data = list(self.sh.rows)
# print(rows_data)
# 获取表单的表头信息
titles = []
for title in rows_data[0]:
titles.append(title.value)
# print(titles)
logger.info("读取到%s标题表头为：%s" % (sheet_name, titles))
# 定义一个空列表用来存储测试用例
cases = []
for case in rows_data[1:]:
# print(case)
data = []
for cell in case:
# 获取一条测试用例数据
data.append(cell.value)
# print(cell.value, type(cell.value))
case_data = dict(list(zip(titles, data)))
cases.append(case_data)
result[sheet_name] = cases
logger.debug("%s : %s" % (sheet_name, cases))
return result
def create_odds(self, season):
""" Create Odds """
for match in SeasonRepository(self.session).get_matches(season):
odds_list = MatchOddsScraper.extract(match)
if bool(odds_list):
fields = {MatchNotation.odds_list(): odds_list}
MatchRepository(self.session).update(match, fields)
logger.debug('Created odds: ' + str(match))
def create_matches(self, season):
matches = SeasonScraper.extract_matches(season)
for match in matches:
try:
MatchRepository(self.session).insert(match)
except DuplicateKeyError as error:
logger.debug(error)
pass
async def get_lines(city: str) -> dict:
# city = '/hokkaido/'
logger.debug(f'expected: "/hokkaido/" got: {city}')
response = await request('GET', LINK_TO_LINES.format(city))
if response:
with open('error.html', 'w') as f:
f.write(response.text)
tree = Selector(response.text)
return Dict.name_link(tree, XPATH_TO_LINES)
def initialize(self):
self.create_indexes()
for champ in ChampionshipUtil.list_championships():
try:
ChampionshipRepository(self.session).insert(champ)
logger.debug('Championship created: ' + str(champ))
except DuplicateKeyError as e:
logger.warning(e)
pass
ChampionshipManager.create_dir(champ)
logger.debug('Championship DIR created: ' + str(champ))
def request(cls, season, replace=False):
""" Request the season file to the site """
logger.debug('request(): ' + str(season))
if not SeasonManager.is_file_exists(season) or replace:
download = False
r = None
while not download:
try:
logger.debug('Requests: ' + cls.get_url(season))
r = requests.get(cls.get_url(season))
download = True
except Exception as e:
logger.exception(e)
SeasonManager.create_file(season, r.content)
async def broadcaster(self) -> None:
while True:
logger.debug(f"Start mailer for chats: {self.ACTIVE_CHATS}")
if not self.GAME_NEWS_SOURCE:
logger.warning(
"Empty game sources, setup a game sources by adding @Mailer.add_news_source() decorator"
)
informers = [informer() for informer in self.GAME_NEWS_SOURCE]
for sheduled_informer in asyncio.as_completed(informers):
updates = await sheduled_informer
for update in updates:
senders = [
self.send_safe_message(chat, update)
for chat in self.ACTIVE_CHATS
]
await asyncio.gather(*senders)
logger.debug(
f"Mailer successfully ended for chats: {self.ACTIVE_CHATS}")
await asyncio.sleep(self.update_frequency)
def initialize(self, initial_year, final_year):
""" Creates seasons """
self.create_indexes()
for champ in ChampionshipRepository(self.session).list():
for year in range(initial_year, final_year):
season = SeasonBusiness.create(champ,year)
try:
SeasonRepository(self.session).insert(season)
logger.debug('Season created: ' + str(season))
except DuplicateKeyError as e:
logger.warning(e)
pass
SeasonManager.create_dir(season)
logger.debug('Season DIR created: ' + str(season))
def construct_selection(self, args: Union[Dict[str, str],
Dict[str, List[str]]]):
"""Create a WHERE selection in normal disjoint form (AND of ORS) and prepare for the final time selection"""
logger.debug("Constructing a selection where query part with [%s]" %
args)
if not args:
return """ WHERE """
# Normalize to lists all the values in the dictionary
converted = {k: [v] for k, v in args.items() if type(v) != list}
query = {**args, **converted}
# constrct all the equivalences
query = [["{} = '{}'".format(k, x) for x in v]
for k, v in query.items()]
# Construct the ORs
query = ["(%s)" % (" OR ".join(v)) for v in query]
# Construct the AND
query = " AND ".join(query)
# Add the final And for the time selection
query += " AND "
return """ WHERE %s""" % query
async def send_safe_message(self,
chat_id: int,
text: str,
disable_notification: bool = False) -> bool:
"""
Safe messages sender
:param chat_id:
:param text:
:param disable_notification:
:return:
"""
try:
await super(Mailer, self).send_message(
chat_id, text, disable_notification=disable_notification)
except exceptions.BotBlocked:
logger.error(f"Target [ID:{chat_id}]: blocked by user")
self._delete_from_chats(chat_id)
except exceptions.ChatNotFound:
logger.error(f"Target [ID:{chat_id}]: invalid user/chat ID")
self._delete_from_chats(chat_id)
except exceptions.RetryAfter as e:
logger.error(
f"Target [ID:{chat_id}]: Flood limit is exceeded. Sleep {e.timeout} seconds."
)
await asyncio.sleep(e.timeout)
return await self.send_safe_message(chat_id, text)
except exceptions.UserDeactivated:
logger.error(f"Target [ID:{chat_id}]: user/chat is deactivated")
self._delete_from_chats(chat_id)
except exceptions.BotKicked:
logger.error(f"Bot was kicked from the group chat: {chat_id}")
self._delete_from_chats(chat_id)
except exceptions.TelegramAPIError:
logger.exception(f"Target [ID:{chat_id}]: failed")
else:
logger.debug(f"Target [ID:{chat_id}]: success")
return True
return False
async def extract_station(station: str) -> list:
# station = 'https://www.good-monthly.com/search/list_eki.html?rosen_eki_cd=483|7758'
logger.debug(f'expected: "https://www.good-monthly.com/search/list_eki.html?rosen_eki_cd=483|7758" got: {station}')
return await Rqst.get_extract_data(station)
def allotSlots(self, slotNumber):
"""When a time slot expires, send dispatch messages for all
scheduled packets.
Args:
slotNumber (int): The time slot for which dispatching
is being done
"""
for i in range(self.n):
# for each space switch in the given slot
sSwitch = self.network.spaceSwitches[i]
data = sSwitch.getSlotData(slotNumber)
finalQueue = sSwitch.queue
# generate a traffic matrix and get the best bipartite matching
for pkt in sSwitch.queue:
gSrc = pkt.src // self.n
gDest = pkt.dest // self.n
data.reqMat[gSrc][gDest] += 1
val, matching = mtchr.Matcher(data.reqMat).solve()
data.finalState = matching
# for each packet in the queue, check if it can be scheduled
for pkt in sSwitch.queue:
# check if packet src/dest matches space switch's config for this timeslot
if pkt.dest // self.n == matching[pkt.src // self.n]:
if pkt.src in data.transmissions.keys():
if pkt.wavelength in data.transmissions[
pkt.src].keys():
trnsmsnCount = data.transmissions[pkt.src][
pkt.wavelength]
else:
data.transmissions[pkt.src][pkt.wavelength] = 0
trnsmsnCount = data.transmissions[pkt.src][
pkt.wavelength]
else:
data.transmissions[pkt.src] = {}
data.transmissions[pkt.src]['count'] = 0
data.transmissions[pkt.src][pkt.wavelength] = 0
trnsmsnCount = data.transmissions[pkt.src][
pkt.wavelength]
if trnsmsnCount < MAX_TRANSMISSION_COUNT:
src = self.network.transmitters[pkt.src]
pkt.dispatchSlot = slotNumber
pkt.schedulingDelay = ((pkt.dispatchSlot + 1) *
self.slot) - pkt.arrivalTime
logger.debug(
f"[Packet {pkt.pktId}] : Wavelength Assigned = {pkt.wavelength}"
)
logger.debug(
f"[Packet {pkt.pktId}] : Space Switch Assigned = {i}"
)
logger.debug(
f"[Packet {pkt.pktId}] : Time Slot Assigned = {slotNumber}"
)
data.transmissions[pkt.src][pkt.wavelength] += 1
data.transmissions[pkt.src]['count'] += 1
src.onSchedule(pkt)
finalQueue.remove(pkt)
# queue with after removing scheduled packets
sSwitch.queue = finalQueue
async def _add_to_active_chat(self, msg: types.Message):
self.ACTIVE_CHATS.add(msg.chat.id)
if not Chats.get_or_none(Chats.recipient == msg.chat.id):
Chats.insert(recipient=msg.chat.id).execute()
logger.debug(f"New subsciber, chat id {msg.chat.id}")
await self._welcome_msg(msg)