def postinit(request):
if request.method != 'POST' or 'name' not in request.POST:
return HttpResponseBadRequest()
cookie = base64.b64encode(os.urandom(12)).decode()
player = Player(name=request.POST['name'], cookie=cookie)
player.save()
with transaction.atomic():
game, _ = Game.objects.select_for_update() \
.filter(status=Game.PENDING, player_count__lt=GAME_THRESHOLD) \
.order_by('create_time')[:1].get_or_create()
logger.debug('Game updating.')
game.player_count = F('player_count') + 1
game.save()
logger.debug('Game update success.{}'.format(game.player_count))
player.game = game
player.save()
logger.debug('Player update success.')
gid = game.id
msg, evt = noticenter_init.listen(gid)
if game.players.count() >= GAME_THRESHOLD:
game.status = Game.READY
game.save()
evt.wait()
msg = msg[0]
return HttpResponse(
json.dumps({
'pid': player.id,
'cookie': cookie,
'local_id': sorted(msg['players']).index(player.id) + 1,
'map': msg['map'],
}))
def read_csv_text(self, text, session):
objects = list()
lines = text.splitlines()
for line in lines:
if line == u''\
or line == u'player, date, time, item, itemID, itemString, response,' \
u' votes, class, instance, boss, gear1, gear2, responseID,' \
u' isAwardReason':
continue
entry = line.split(",")
if entry[0] == u'':
entry.pop(0)
player_name = entry[0].split("-")[0]
player_realm = entry[0].split("-")[1]
date = entry[1]
time = entry[2]
itemname = entry[3]
item = entry[5]
response = entry[6]
votes = int(entry[7])
wow_class = entry[8]
instance = entry[9]
boss = entry[10]
gear1 = entry[11]
gear2 = entry[12]
responseID = entry[13]
is_award_reason = entry[14]
instance_id = instance.split("-")[0]
instance_id = Codes.instance_codes[instance_id.lower()]
existing_player = session.query(Player)\
.filter(Player.name == player_name, Player.realm == player_realm).all()
player = None
if (len(existing_player) == 1):
player = existing_player[0]
elif (len(existing_player) > 1):
raise Exception(
"Multiple existing player_rel "
"with name {0} and realm {1} found in DB.".format(
player_name, player_realm))
else:
player = Player(name=player_name, realm=player_realm)
session.add(player)
iteminfo = self.unpack_item_string("Hitem:" + item.split("|")[2])
#RCLC output does not match the actual Wow item string format in that it removes the starting 'hitem'.
existing_loot = session.query(Loot)\
.filter(Loot.item_id == int(iteminfo["itemID"])).all()
new_time = datetime.strptime(date, "%d/%m/%y")
new_loot = LootAward(reason=response,
player_rel=player,
award_date=new_time)
session.add(new_loot)
if len(existing_loot) == 1:
new_loot.item_rel = existing_loot[0]
elif len(existing_loot) > 1:
raise Exception(
"Multiple loot instances of item {0} found.".format(
existing_loot[0].item_id))
else:
_loot = Loot(item_id=int(iteminfo["itemID"]),
instance=instance_id,
name=itemname)
session.add(_loot)
new_loot.item_rel = _loot
if gear1 != "nil":
gear1info = self.unpack_item_string("Hitem:" +
gear1.split("|")[2])
gear1name = gear1.split("|")[3]
gear1name = gear1name[2:-1]
existing_gear1 = session.query(Loot).filter(
Loot.item_id == int(gear1info["itemID"])).all()
if len(existing_gear1) == 1:
new_loot.replacement1_rel = existing_gear1[0]
elif len(existing_gear1) > 1:
raise Exception(
"Multiple loot instances of item {0} found.".format(
existing_gear1[0].item_id))
else:
_replacement1 = Loot(item_id=int(gear1info["itemID"]),
instance=-1,
name=gear1name)
session.add(_replacement1)
new_loot.replacement1_rel = _replacement1
else:
new_loot.replacement1_rel = None
if gear2 != "nil":
gear2info = self.unpack_item_string("Hitem" +
gear2.split("|")[2])
gear2name = gear2.split("|")[3]
gear2name = gear2name[2:-1]
existing_gear2 = session.query(Loot) \
.filter(Loot.item_id == int(gear2info["itemID"])).all()
if len(existing_gear2) == 1:
new_loot.replacement2_rel = existing_gear2[0]
elif len(existing_gear2) > 1:
raise Exception(
"Multiple loot instances of item {0} found.".format(
existing_gear2[0].item_id))
else:
_replacement2 = Loot(item_id=int(gear2info["itemID"]),
instance=-1,
name=gear2name)
session.add(_replacement2)
new_loot.replacement2_rel = _replacement2
else:
new_loot.replacement2_rel = None
objects.append(new_loot)
return objects
def read_tsv_text(self, text, session):
"""
:param text: Lines of TSV text representing loot council data. Lines should match
player date time item itemID itemString response votes class instance boss gear1 gear2 responseID isAwardReason
format.
:param session: Database session new loot awards should be added to.
:return: The LootAward objects created and added to the session.
"""
objects = list()
failed_lines = list()
lines = text.splitlines()
for line in lines:
try:
if line == u''\
or line == u'player date time item itemID itemString response' \
u' votes class instance boss gear1 gear2' \
u' responseID isAwardReason':
continue
data = line.split("\t")
player_name = data[0].split("-")[0]
player_realm = data[0].split("-")[1]
date = datetime.strptime(data[1], "%d/%m/%y")
time = data[2]
itemHL = data[3]
itemID = int(data[4])
itemString = data[5]
response = data[6]
votes = data[7]
wow_class = data[8]
instance = data[9]
boss = data[10]
gear1HL = data[11]
gear2HL = data[12]
responseID = data[13]
isAwardReason = data[14]
item_name = itemHL.split(",")[1]
item_name = item_name[1:-2]
gearIDregex = "wowhead.com/item=([0-9]+)"
gearBonusRegex = "&bonus=([0-9-:]+)"
gear1_name = gear1HL.split("\",")[1]
gear1_name = gear1_name[1:-2]
if gear1_name != u"nil":
gear1IDgroups = re.search(gearIDregex, gear1HL)
gear1ID = int(gear1IDgroups.group(1))
gear1BonusGroups = re.search(gearBonusRegex, gear1HL)
if gear1BonusGroups is not None:
gear1_bonus_IDs = gear1BonusGroups.group(1).split(":")
gear1_bonus_IDs = [int(i) for i in gear1_bonus_IDs]
else:
gear1_bonus_IDs = list()
gear2_name = gear2HL.split("\",")[1]
gear2_name = gear2_name[1:-2]
if gear2_name != u"nil":
gear2IDgroups = re.search(gearIDregex, gear2HL)
gear2ID = int(gear2IDgroups.group(1))
gear2BonusGroups = re.search(gearBonusRegex, gear2HL)
if gear2BonusGroups is not None:
gear2_bonus_IDs = gear2BonusGroups.group(1).split(":")
gear2_bonus_IDs = [int(i) for i in gear2_bonus_IDs]
else:
gear2_bonus_IDs = list()
instance_id = Codes.instance_codes[instance.split("-")
[0].lower()]
player = None
existing_player = session.query(Player)\
.filter(Player.name == player_name, Player.realm == player_realm).all()
if len(existing_player) == 1:
player = existing_player[0]
elif len(existing_player) > 1:
raise Exception(
"Multiple existing players with name {0} and realm {1} found."
.format(player_name, player_realm))
else:
bnet = BNet()
player_info = bnet.create_character_profile(
session, name=player_name, realm=player_realm)
player = Player(name=player_name,
realm=player_realm,
wow_class=player_info["class"])
session.add(player)
item_info = self.unpack_item_string("Hitem:" + itemString)
#RCLC output does not match the actual WoW item string format, it removes the starting "hitem",
#we add it here to preserve the purity of unpack_item_string
new_award = LootAward(reason=response,
player_rel=player,
award_date=date)
##session.add(new_award)
existing_items = session.query(Loot).filter(
Loot.item_id == itemID).all()
item_info["bonusIDs"] = [int(i) for i in item_info["bonusIDs"]]
matching = list()
for item in existing_items:
all_matching = True
if len(item.bonus_ids) != len(item_info["bonusIDs"]):
all_matching = False
for id in item.bonus_ids:
if id.bonus_id not in item_info["bonusIDs"]:
all_matching = False
break
if all_matching:
matching.append(item)
if len(matching) == 1:
new_award.item_rel = matching[0]
elif len(matching) > 1:
raise Exception(
"Multiple loot instances of item {0} with bonus IDs {1} found."
.format(matching[0].item_id, item_info["bonusIDs"]))
else:
new_item = Loot(item_id=itemID,
instance=instance_id,
name=item_name)
session.add(new_item)
new_award.item_rel = new_item
for id in item_info["bonusIDs"]:
new_item.bonus_ids.append(BonusID(bonus_id=id))
#We just get wowhead hyperlinks instead of item strings for the gear replacement, so itemID
#and bonus IDs extraction is different than how we do it with the awarded item.
if gear1_name != u"nil":
existing_gear1 = session.query(Loot).filter(
Loot.item_id == gear1ID).all()
matching = list()
for item in existing_gear1:
all_matching = True
if len(item.bonus_ids) != len(gear1_bonus_IDs):
all_matching = False
for id in item.bonus_ids:
if id.bonus_id not in gear1_bonus_IDs:
all_matching = False
break
if all_matching:
matching.append(item)
if len(matching) == 1:
new_award.replacement1_rel = matching[0]
elif len(matching) > 1:
raise Exception(
"Multiple loot instances of item {0} with bonus IDs {1} found."
.format(matching[0].item_id, gear1_bonus_IDs))
else:
new_gear1 = Loot(item_id=gear1ID,
instance=-1,
name=gear1_name)
session.add(new_gear1)
new_award.replacement1_rel = new_gear1
for id in gear1_bonus_IDs:
new_gear1.bonus_ids.append(BonusID(bonus_id=id))
if gear2_name != u"nil":
existing_gear2 = session.query(Loot).filter(
Loot.item_id == gear2ID).all()
matching = list()
for item in existing_gear2:
all_matching = True
if len(item.bonus_ids) != len(gear2_bonus_IDs):
all_matching = False
for id in item.bonus_ids:
if id.bonus_id not in gear2_bonus_IDs:
all_matching = False
break
if all_matching:
matching.append(item)
if len(matching) == 1:
new_award.replacement2_rel = matching[0]
elif len(matching) > 1:
raise Exception(
"Multiple loot instances of item {0} with bonus IDs {1} found."
.format(matching[0].item_id, gear2_bonus_IDs))
else:
new_gear2 = Loot(item_id=gear2ID,
instance=-1,
name=gear2_name)
session.add(new_gear2)
new_award.replacement2_rel = new_gear2
for id in gear2_bonus_IDs:
new_gear2.bonus_ids.append(BonusID(bonus_id=id))
#Check if the exact same award has already been made.
existing_awards = session.query(LootAward).filter(
LootAward.item_rel == new_award.item_rel,
LootAward.replacement1_rel == new_award.replacement1_rel,
LootAward.replacement2_rel == new_award.replacement2_rel,
LootAward.player_rel == new_award.player_rel).all()
if len(existing_awards) == 0:
objects.append(new_award)
session.add(new_award)
except Exception as e:
ex_type, ex, tb = sys.exc_info()
traceback.print_tb(tb)
failed_lines.append("Line: {0}\nError: {1}\n".format(
line,
str(e) + e.message))
return objects, failed_lines
def run(args):
"""
Runs the data scraper. Calls various apis and saves data out to the
database.
:param args: argparse Namespace containing command-line arguments
:return None
"""
init_logging(args.verbose)
logger = logging.getLogger(__name__)
logger.info('Starting...')
conf = read_config(args.config)
repo = get_database(args.database)
matches = replays.get_replays(7100)
for match in matches:
from db.models import Match, Player, Team
players = match['player_set']
if len(players) is not 6:
continue
players_stats = stats.get_player_stats(players, conf['apis']['stats']['key'])
team_0 = []
team_1 = []
players_to_save = []
missing_player_stats = False
for player in players:
new_player = Player(player)
if new_player.team == 0:
team_0.append(new_player)
else:
team_1.append(new_player)
if player['platform'] == '1':
new_player_stats = [s for s in players_stats if s['uniqueId'] == player['online_id']]
else:
new_player_stats = [s for s in players_stats if s['displayName'] == player['player_name']]
if len(new_player_stats) == 0:
missing_player_stats = True
break
else:
new_player.update_stats(new_player_stats[0]['stats'])
players_to_save.append(new_player)
if missing_player_stats:
continue
else:
for p in players_to_save:
repo.save_player(p)
db_team_0 = Team(team_0)
db_team_1 = Team(team_1)
repo.save_team(db_team_0)
repo.save_team(db_team_1)
time.sleep(conf['apis']['stats']['delay'])
def create_player(cls, attrs):
"""Create a new player in database
attrs are the attributes collected by the view"""
id = Tournament.set_player_id()
attrs["id"] = id
Player.create(attrs)
def add_player(idx, name, password=None, session=None):
""" Creates a new Player in DB.
"""
new_player = Player(id=idx, name=name, password=password)
session.add(new_player)
def make_scrape_round(tournament, round_no, date, url):
print "round ", round_no, "at", date
r = requests.get(url, verify=False)
soup = BeautifulSoup(r.content, "html.parser")
matches = soup.findAll("a", {"id": "match-link"})
for match in matches:
rounds = match.find("div", {
"class": "round_time"
}).find("div", {
"class": "round"
}).contents[0].strip().split("-")
gametype = rounds[0].strip().lower()
if not (gametype in ["ms", "mens singles", "men's singles"]
or 'ms' in gametype):
# only men's single
continue
details = match.find("div", {
"class": "score"
}).contents[0].strip().lower()
no_match = "no match" in details or "retirement" in details or "walkover" in details
if no_match:
continue
try:
player1_name = match.find("div", {
"class": "player1"
}).contents[0].strip().lower()
player2_name = match.find("div", {
"class": "player3"
}).contents[0].strip().lower()
except:
# no first/second player; walk over
continue
if player1_name == '' or player2_name == '':
continue
# rm digigts
player1_name = ''.join([i for i in player1_name if not i.isdigit()])
player2_name = ''.join([i for i in player2_name if not i.isdigit()])
# rm brackets
player1_name = ''.join([i for i in player1_name
if not i in '[]']).strip()
player2_name = ''.join([i for i in player2_name
if not i in '[]']).strip()
try:
player1 = get_player_name(player1_name)
except:
player1 = Player()
player1.name = player1_name
db.add(player1)
try:
player2 = get_player_name(player2_name)
except:
player2 = Player()
player2.name = player2_name
db.add(player2)
match_link = match.attrs['href']
match_link = match_link.replace("&stab=result", "&stab=match")
r = requests.get(match_link, verify=False)
soup = BeautifulSoup(r.content, "html.parser")
duration = soup.find("h3", {
"class": "match-duration"
}).contents[0].strip()
try:
duration = int(re.findall('\d+', duration)[0])
except:
duration = 0
try:
m = Match()
m.tournament = tournament
m.player1 = player1
m.player2 = player2
m.round_no = round_no
m.duration = duration
data = soup.find("div", {
"class": "live-profile-row"
}).findAll("tr")
ms1 = MatchStats()
ms1.max_consecutive_points = getint(
data[0].findChildren()[0].contents[0].strip())
ms1.game_points = getint(
data[1].findChildren()[0].contents[0].strip())
ms1.rallies_played = getint(
data[2].findChildren()[0].contents[0].strip())
ms1.rallies_won = getint(
data[3].findChildren()[0].contents[0].strip())
ms1.smash_winners = getint(
data[4].findChildren()[0].contents[0].strip())
ms1.net_winners = getint(
data[5].findChildren()[0].contents[0].strip())
ms1.clear_winners = getint(
data[6].findChildren()[0].contents[0].strip())
ms1.total_points_played = getint(
data[7].findChildren()[0].contents[0].strip())
ms1.total_points_won = getint(
data[8].findChildren()[0].contents[0].strip())
db.add(ms1)
ms2 = MatchStats()
ms2.max_consecutive_points = getint(
data[0].findChildren()[-1].contents[0].strip())
ms2.game_points = getint(
data[1].findChildren()[-1].contents[0].strip())
ms2.rallies_played = getint(
data[2].findChildren()[-1].contents[0].strip())
ms2.rallies_won = getint(
data[3].findChildren()[-1].contents[0].strip())
ms2.smash_winners = getint(
data[4].findChildren()[-1].contents[0].strip())
ms2.net_winners = getint(
data[5].findChildren()[-1].contents[0].strip())
ms2.clear_winners = getint(
data[6].findChildren()[-1].contents[0].strip())
ms2.total_points_played = getint(
data[7].findChildren()[-1].contents[0].strip())
ms2.total_points_won = getint(
data[8].findChildren()[-1].contents[0].strip())
db.add(ms2)
m.stat1 = ms1
m.stat2 = ms2
print "> added match ", m.player1.name, "vs", m.player2.name
db.add(m)
sets = match.find("div", {
"class": "score"
}).contents[0].strip().split(",")
for i, game_set in enumerate(sets):
scores = game_set.split("-")
s = Set()
s.match = m
s.index = i
s.score_1 = scores[0]
s.score_2 = scores[1]
db.add(s)
except:
# no match
continue