def upload(request): global timer c = all_page_infos(request) UploadFormSet = formset_factory(UploadFileForm, extra=5) if request.method == 'POST': formset = UploadFormSet(request.POST, request.FILES) replays = [] if formset.is_valid(): for form in formset: if form.cleaned_data: logger.info("upload by request.user=%r form.cleaned_data=%r", request.user, form.cleaned_data) ufile = form.cleaned_data['file'] short = form.cleaned_data['short'] long_text = form.cleaned_data['long_text'] tags = form.cleaned_data['tags'] timer = SrsTiming() timer.start("upload()") (path, written_bytes) = save_uploaded_file(ufile.read(), ufile.name) logger.info("User '%s' uploaded file '%s' with title '%s', parsing it now.", request.user, os.path.basename(path), short[:20]) if written_bytes != ufile.size: logger.warn("written_bytes=%r != ufile.size=%r", written_bytes, ufile.size) try: replay, msg = parse_uploaded_file(path, timer, tags, short, long_text, request.user) logger.debug("replay=%r msg=%r", replay, msg) replays.append((True, replay)) except parse_demo_file.BadFileType: form._errors = {'file': [u'Not a spring demofile: %s.' % ufile.name]} replays.append((False, 'Not a spring demofile: %s.' % ufile.name)) continue except AlreadyExistsError as exc: form._errors = {'file': [u'Uploaded replay already exists: "{}"'.format(exc.replay.title)]} replays.append((False, 'Uploaded replay already exists: <a href="{}">{}</a>.'.format( exc.replay.get_absolute_url(), exc.replay.title))) continue finally: timer.stop("upload()") logger.info("timings:\n%s", timer) if len(replays) == 0: logger.error("no replay created, this shouldn't happen") elif len(replays) == 1: if replays[0][0]: return HttpResponseRedirect(replays[0][1].get_absolute_url()) else: # fall through to get back on page with error msg pass else: c['replays'] = replays c["replay_details"] = True return render(request, 'multi_upload_success.html', c) else: # form = UploadFileForm() formset = UploadFormSet() c['formset'] = formset return render(request, 'upload.html', c)
def store_demofile_data(demofile, tags, path, filename, short, long_text, user): """ Store all data about this replay in the database, if replay exists, all argument except 'demofile' are ignored """ logger.info("tags='%s' path='%s' filename='%s' short='%s' long_text='%s' user='******'", tags, path, filename, short, long_text, user) logger.info("gameID='%s'", demofile.header["gameID"]) global timer if not timer: timer = SrsTiming() # pp = pprint.PrettyPrinter(depth=6) # logger.debug("demofile.__dict__: %s",pp.pformat(demofile.__dict__)) try: replay = Replay.objects.get(gameID=demofile.header["gameID"]) replay.download_count = 0 logger.debug("reparsing existing Replay(%d) %s (%s)", replay.id, replay, replay.gameID) except ObjectDoesNotExist: replay = Replay() logger.debug("new Replay: gameID: %s", demofile.header["gameID"]) replay.published = False if user is not None: replay.uploader = user if short is not None: replay.title = short # temp # copy match infos for key in ["versionString", "gameID", "wallclockTime"]: replay.__setattr__(key, demofile.header[key]) replay.unixTime = datetime.datetime.strptime(demofile.header["unixTime"], "%Y-%m-%d %H:%M:%S").replace( tzinfo=django.utils.timezone.get_current_timezone()) for key in ["autohostname", "gametype", "startpostype"]: if key in demofile.game_setup["host"]: replay.__setattr__(key, demofile.game_setup["host"][key]) # if match is <2 min long, don't rate it replay.notcomplete = demofile.header['gameTime'].startswith("0:00:") or demofile.header['gameTime'].startswith( "0:01:") logger.info("gameTime=%r -> replay.notcomplete=%r", demofile.header['gameTime'], replay.notcomplete) if path is not None: replay.filename = os.path.basename(path) replay.path = os.path.dirname(path) replay.save() UploadTmp.objects.create(replay=replay) logger.debug("replay(%d) gameID='%s' unixTime='%s' created", replay.pk, replay.gameID, replay.unixTime) # save AllyTeams timer.start(" allyteams") allyteams = {} at_depr = Allyteam.objects.filter(replay=replay, num=-1) if at_depr.exists(): logger.info("removing deprecated Allyteams: %s", at_depr.values_list('id')) at_depr.delete() for anum, val in demofile.game_setup['allyteam'].items(): defaults = {"numallies": val["numallies"], "winner": int(anum) in demofile.winningAllyTeams, "startrectbottom": val.get("startrectbottom"), "startrectleft": val.get("startrectleft"), "startrectright": val.get("startrectright"), "startrecttop": val.get("startrecttop")} allyteam, created = Allyteam.objects.get_or_create(replay=replay, num=anum, defaults=defaults) if not created: for k, v in defaults.items(): setattr(allyteam, k, v) allyteam.save() allyteams[anum] = allyteam timer.stop(" allyteams") logger.debug("replay(%d) allyteams: %s", replay.pk, [a.pk for a in allyteams.values()]) # save tags save_tags(replay, tags) # save map and mod options def truncate_option(k, v): if len(str(k)) > 512: k = "string to long, sorry" if len(str(v)) > 512: v = "string to long, sorry" return k, v timer.start(" map+modoptions") for k, v in demofile.game_setup['mapoptions'].items(): k, v = truncate_option(k, v) MapOption.objects.get_or_create(name=k, value=v, replay=replay) for k, v in demofile.game_setup['modoptions'].items(): k, v = truncate_option(k, v) ModOption.objects.get_or_create(name=k, value=v, replay=replay) timer.stop(" map+modoptions") logger.debug("replay(%d) added mapoptions, modoptions and tags: %s", replay.pk, replay.tags.all().values_list("name")) # save players and their accounts timer.start(" players and playeraccounts") # add late-connected spectators # TODO: add /joinas players for late_spec_name, late_spec_val in demofile.spectators.items(): if (hasattr(late_spec_val, "connected") and late_spec_val.connected and hasattr(late_spec_val, "playerNum") and late_spec_val.playerNum not in demofile.game_setup['player'].keys()): # found a spec that connected, but was not in the start script # -> late-connected # To add it, the problem is that we don't have the required # data for a proper Player object (we have only the name). We # try to find the correct Player[Account] by looking at existing # Players from previous Replays: players_w_late_spec_name = Player.objects.filter(name=late_spec_name, replay__id__lt=replay.id).order_by( "-id") if players_w_late_spec_name.exists(): player_w_late_spec_name = players_w_late_spec_name[0] accid = player_w_late_spec_name.account.accountid cc = player_w_late_spec_name.account.countrycode rank = player_w_late_spec_name.rank skill = player_w_late_spec_name.skill skilluncertainty = player_w_late_spec_name.skilluncertainty else: accid = None cc = "??" rank = 0 skill = "" skilluncertainty = -1 # add spectator to list of players from script demofile.game_setup['player'][late_spec_val.playerNum] = dict(accountid=accid, ally=-1, connected=True, countrycode=cc, name=late_spec_name, num=late_spec_val.playerNum, password='', rank=rank, skill=skill, skilluncertainty=skilluncertainty, spectator=1, startposx=-1) logger.debug("added late-spec %s", late_spec_name) players = dict() teams = list() for pnum in sorted(demofile.game_setup['player'].keys()): player = demofile.game_setup['player'][pnum] set_accountid(player) if player["countrycode"] is None: player["countrycode"] = "??" pa, _ = PlayerAccount.objects.get_or_create(accountid=player["accountid"], defaults={'countrycode': player["countrycode"], 'preffered_name': player["name"]}) if pa.preffered_name == "??" or pa.preffered_name == "": pa.preffered_name = player["name"] pa.save() if pa.countrycode == "??": pa.countrycode = player["countrycode"] pa.save() try: skill = player["skill"] except KeyError: skill = "" try: skilluncertainty = player["skilluncertainty"] except KeyError: skilluncertainty = -1 if player["rank"] is None: try: rank = Player.objects.filter(account=pa).order_by("-id")[0].rank except KeyError: rank = 0 else: rank = player["rank"] defaults = {"name": player["name"], "rank": rank, "skill": skill, "skilluncertainty": skilluncertainty, "spectator": bool(player["spectator"])} players[pnum], created = Player.objects.get_or_create(replay=replay, account=pa, defaults=defaults) if not created: for k, v in defaults.items(): setattr(players[pnum], k, v) if pa.accountid > 0 and not ( player['spectator'] == 1 and 'startposx' in player and player['startposx'] == -1): # if we found players w/o account, and now have a player with the # same name, but with an account - unify them # exclude freshly created players from late-join pac = Player.objects.filter(name=player["name"], account__accountid__lt=0) if pac.exists(): logger.info("replay(%d) found matching name-account info for previously accountless Player(s):", replay.pk) logger.info("replay(%d) PA.pk=%d Player(s).pk:%s", replay.pk, pa.pk, [(p.name, p.pk) for p in pac]) for pplayer in pac: try: pplayer.account.delete() except ObjectDoesNotExist: pass pplayer.account = pa pplayer.save() if "team" in player and player["team"] is not None: teams.append((players[pnum], player["team"])) # [(Player, "2"), ...] try: players[pnum].startposx = player["startposx"] players[pnum].startposy = player["startposy"] players[pnum].startposz = player["startposz"] except KeyError: logger.info("player has no start coords. (quit/kicked/not connected?) player: %s players[%d]: %s", player, pnum, players[pnum]) else: # this must be a spectator if not player["spectator"] == 1: logger.error("replay(%d) found player without team and not a spectator: %s", replay.pk, player) try: # late-join spec players[pnum].startposx = player["startposx"] except KeyError: pass players[pnum].save() timer.stop(" players and playeraccounts") logger.debug("replay(%d) saved PlayerAccounts and Players: %s", replay.pk, [Player.objects.filter(replay=replay)]) # save teams timer.start(" teams") team_depr = Team.objects.filter(replay=replay, num=-1) if team_depr.exists(): logger.info("removing deprecated Teams: %s", team_depr.values_list('id')) team_depr.delete() for tnum, val in demofile.game_setup['team'].items(): defaults = {"allyteam": allyteams[val["allyteam"]], "handicap": val["handicap"], "rgbcolor": floats2rgbhex(val.get("rgbcolor", 1.0)), "side": val["side"], "teamleader": players[val["teamleader"]]} if "rgbcolor" not in val: # probably zero-k, let's see and learn what stuff they use instead logger.error("No key 'rgbcolor' in game_setup of team %r, val=%r ", tnum, val) try: defaults["startposx"] = val["startposx"] defaults["startposy"] = val["startposy"] defaults["startposz"] = val["startposz"] except KeyError: pass team, created = Team.objects.get_or_create(replay=replay, num=tnum, defaults=defaults) if not created: for k, v in defaults.items(): setattr(team, k, v) team.save() # find Players for this Team teamplayers = [t[0] for t in teams if t[1] == tnum] if teamplayers: for player in teamplayers: player.team = team player.save() else: # team without player: this is a bot - create a Player for it logger.debug("replay(%d) team[%s] (Team(%d)) has no teamplayers, must be a bot", replay.pk, tnum, team.pk) bot_pa = PlayerAccount.objects.get(accountid=0) Player.objects.create(account=bot_pa, name="Bot (of " + team.teamleader.name + ")", rank=1, spectator=False, team=team, replay=replay) # add a "Bot" tag bottag, _ = Tag.objects.get_or_create(name="Bot") replay.tags.add(bottag) # detect single player and add tag try: if demofile.game_setup["host"]["hostip"] == "" and demofile.game_setup["host"]["ishost"] == 1: sptag, _ = Tag.objects.get_or_create(name="Single Player") replay.tags.add(sptag) except KeyError: pass timer.stop(" teams") logger.debug("replay(%d) saved Teams (%s)", replay.pk, Team.objects.filter(replay=replay).values_list('id')) # work around XTA and Zero-Ks usage of empty AllyTeams at_empty = Allyteam.objects.filter(replay=replay, team__isnull=True) if at_empty.exists(): logger.debug("replay(%s) deleting useless AllyTeams: %s", replay.pk, at_empty) at_empty.delete() timer.start(" map creation") # get / create map infos smap = springmaps.SpringMaps(demofile.game_setup["host"]["mapname"]) try: replay.map_info = Map.objects.get(name=demofile.game_setup["host"]["mapname"]) logger.debug("replay(%d) using existing map_info.pk=%d", replay.pk, replay.map_info.pk) smap.full_image_filename = MapImg.objects.get(startpostype=-1, map_info=replay.map_info).filename except ObjectDoesNotExist: # 1st time upload for this map: fetch info and full map, create thumb # for index page smap.fetch_info() if smap.map_info: startpos = str() for coord in smap.map_info[0]["metadata"]["StartPos"]: startpos += "%f,%f|" % (coord["x"], coord["z"]) startpos = startpos[:-1] height = smap.map_info[0]["metadata"]["Height"] width = smap.map_info[0]["metadata"]["Width"] full_img = smap.fetch_img() else: # no result - either api.springfiles.com is down or it didn't find the map err = "No map info from api.springfiles.com, using empty img for map '%s'." % demofile.game_setup["host"][ "mapname"] logger.error(err) startpos = "" height = 170 width = 340 shutil.copy(path_join(settings.MAPS_PATH, "__no_map_img.jpg"), path_join(settings.MAPS_PATH, smap.mapname, ".jpg")) full_img = path_join(settings.MAPS_PATH, smap.mapname, ".jpg") smap.make_home_thumb() if smap.map_info: metadata = smap.map_info[0] else: metadata = dict() replay.map_info = Map.objects.create(name=demofile.game_setup["host"]["mapname"], startpos=startpos, height=height, width=width, metadata=metadata) MapImg.objects.create(filename=full_img, startpostype=-1, map_info=replay.map_info) logger.debug("replay(%d) created new map_info and MapImg: map_info.pk=%d", replay.pk, replay.map_info.pk) replay.save() # # startpostype = 0: Fixed | # startpostype = 1: Random | # startpostype = 2: ChooseInGame | allyteam { .. startrectbottom, ... } # startpostype = 3: ChooseBeforeGame | team { startposx, startposz } ... # # relayhoststartpostype = 0 --> startpostype = 3 # relayhoststartpostype = 1 --> startpostype = 3 # relayhoststartpostype = 2 --> startpostype = 2 # relayhoststartpostype = 3 --> startpostype = 3 # # always make new img, as players will never choose exact same startpos try: mapfile = smap.create_map_with_boxes(replay) except: logger.error("FIXME: to broad exception handling.") logger.exception("error creating map img for Replay(%s) on map '%s'", replay.pk, replay.map_info.name) mapfile = "error creating map img" replay.map_img = MapImg.objects.create(filename=mapfile, startpostype=2, map_info=replay.map_info) replay.save() logger.debug("replay(%d) created new map_img(%d)", replay.pk, replay.map_img.pk) timer.stop(" map creation") timer.start(" tags + descriptions") # auto add tag 1v1 2v2 etc. autotag = set_autotag(replay) if short is not None and long_text is not None: # save descriptions save_desc(replay, short, long_text, autotag) timer.stop(" tags + descriptions") timer.start(" demofile.additional") # store if demofile doesn't contain the GAMEOVER msg, because of the spring94.1 bug: # http://springrts.com/mantis/view.php?id=3950 # http://springrts.com/mantis/view.php?id=3804 if not demofile.additional["gameover_frame"]: logger.debug("replay(%d) has no GAMEOVER msg", replay.pk) AdditionalReplayInfo.objects.get_or_create(replay=replay, key="gameover", value="") # BAward if "awards" in demofile.additional: ba_awards, _ = BAwards.objects.get_or_create(replay=replay) demo_awards = demofile.additional["awards"] for award_name in ["ecoKillAward", "fightKillAward", "effKillAward", "cowAward", "ecoAward", "dmgRecAward", "sleepAward"]: if type(demo_awards[award_name]) == tuple: aw1, aw2, aw3 = demo_awards[award_name] if aw1[0] > -1: setattr(ba_awards, award_name + "1st", players[aw1[0]]) setattr(ba_awards, award_name + "1stScore", aw1[1]) else: setattr(ba_awards, award_name + "1st", None) if aw2[0] > -1: setattr(ba_awards, award_name + "2nd", players[aw2[0]]) setattr(ba_awards, award_name + "2ndScore", aw2[1]) else: setattr(ba_awards, award_name + "2nd", None) if aw3[0] > -1: setattr(ba_awards, award_name + "3rd", players[aw3[0]]) setattr(ba_awards, award_name + "3rdScore", aw3[1]) else: setattr(ba_awards, award_name + "3rd", None) else: if demo_awards[award_name][0] > -1: setattr(ba_awards, award_name, players[demo_awards[award_name][0]]) setattr(ba_awards, award_name + "Score", demo_awards[award_name][1]) else: setattr(ba_awards, award_name, None) ba_awards.save() # XTAwards if "xtawards" in demofile.additional: for xta_award in demofile.additional["xtawards"]: team = Team.objects.get(replay=replay, num=xta_award["team"]) player = Player.objects.get(replay=replay, team=team) xt, cr = XTAwards.objects.get_or_create(replay=replay, isAlive=xta_award["isAlive"], player=player, unit=xta_award["name"], kills=xta_award["kills"], age=xta_award["age"]) logger.debug("XTA created: %s, award: %s", cr, xt) timer.stop(" demofile.additional") pp = pprint.PrettyPrinter(depth=6) for k, v in demofile.additional.items(): if k == "chat": v = "chat removed for shorter output" elif k == "awards": v = ba_awards logger.info("demofile.additional[%r]: %s", k, pp.pformat(v)) replay.published = True replay.save() UploadTmp.objects.filter(replay=replay).delete() return replay
def upload(request): global timer c = all_page_infos(request) UploadFormSet = formset_factory(UploadFileForm, extra=5) if request.method == 'POST': formset = UploadFormSet(request.POST, request.FILES) replays = [] if formset.is_valid(): for form in formset: if form.cleaned_data: logger.info( "upload by request.user=%r form.cleaned_data=%r", request.user, form.cleaned_data) ufile = form.cleaned_data['file'] short = form.cleaned_data['short'] long_text = form.cleaned_data['long_text'] tags = form.cleaned_data['tags'] timer = SrsTiming() timer.start("upload()") (path, written_bytes) = save_uploaded_file( ufile.read(), ufile.name) logger.info( "User '%s' uploaded file '%s' with title '%s', parsing it now.", request.user, os.path.basename(path), short[:20]) if written_bytes != ufile.size: logger.warn("written_bytes=%r != ufile.size=%r", written_bytes, ufile.size) try: replay, msg = parse_uploaded_file( path, timer, tags, short, long_text, request.user) logger.debug("replay=%r msg=%r", replay, msg) replays.append((True, replay)) except parse_demo_file.BadFileType: form._errors = { 'file': [u'Not a spring demofile: %s.' % ufile.name] } replays.append( (False, 'Not a spring demofile: %s.' % ufile.name)) continue except AlreadyExistsError as exc: form._errors = { 'file': [ u'Uploaded replay already exists: "{}"'.format( exc.replay.title) ] } replays.append(( False, 'Uploaded replay already exists: <a href="{}">{}</a>.' .format(exc.replay.get_absolute_url(), exc.replay.title))) continue finally: timer.stop("upload()") logger.info("timings:\n%s", timer) if len(replays) == 0: logger.error("no replay created, this shouldn't happen") elif len(replays) == 1: if replays[0][0]: return HttpResponseRedirect(replays[0][1].get_absolute_url()) else: # fall through to get back on page with error msg pass else: c['replays'] = replays c["replay_details"] = True return render(request, 'multi_upload_success.html', c) else: # form = UploadFileForm() formset = UploadFormSet() c['formset'] = formset return render(request, 'upload.html', c)