Пример #1
0
def load_transactions():

    print("loading {} transactions".format(soccer_db.transactions.count()))

    transactions = []

    for t in soccer_db.transactions.find():
        try:
            t.pop('_id')

            if t.get('team_from'):
                team_from = Team.objects.find(t['team_from'], create=True)
                team_from_id = team_from.id
            else:
                team_from_id= None

            if t.get('team_to'):
                team_to = Team.objects.find(t['team_to'], create=True)
                team_to_id = team_to.id
            else:
                team_to_id = None

            person = Bio.objects.find(t['person'])

            transactions.append({
                'ttype': t['ttype'],
                'person_id': person.id,
                'team_to_id': team_to_id,
                'team_from_id': team_from_id,
                'date': t['date'],
                })
        except:
            import pdb; pdb.set_trace()

    insert_sql("transactions_transaction", transactions)
Пример #2
0
def load_assists():
    print("\nloading assists\n")

    team_getter = make_team_getter()
    bio_getter = make_bio_getter()
    goal_getter = make_goal_getter()

    def create_assists(goal):

        #if goal['competition'] == 'Major League Soccer'  and goal['season'] == '1996':
        #    import pdb; pdb.set_trace()

        if not goal['assists']:
            return []

        if goal['assists'] == ['']:
            return []

        team_id = team_getter(goal['team'])
        bio_id = ogbio_id = None

        if goal['goal']:
            bio_id = bio_getter(goal['goal'])

        if not goal['date']:
            return {}

        d = datetime.date(goal['date'].year, goal['date'].month,
                          goal['date'].day)

        goal_id = goal_getter(team_id, bio_id, goal['minute'], d)
        if not goal_id:
            #import pdb; pdb.set_trace()
            print("Cannot create assists for %s" % goal)
            return []

        seen = set()

        for assister in goal['assists']:
            assist_ids = [bio_getter(e) for e in goal['assists']]
            for i, assist_id in enumerate(assist_ids, start=1):
                if assist_id and assist_id not in seen:
                    seen.add(assist_id)
                    assists.append({
                        'player_id': assist_id,
                        'goal_id': goal_id,
                        'order': i,
                    })

    assists = []

    i = 0
    for i, goal in enumerate(soccer_db.goals.find()):
        if i % 50000 == 0:
            print(i)
        create_assists(goal)

    print(i)
    print(len(assists))
    insert_sql('goals_assist', assists)
Пример #3
0
def generate_stats_generic(table, qs, make_key, update_dict):
    """
    Generate team, career, etc. stats.
    Maybe could improve this.
    """

    #print("Merging stats.")
    final_dict = {}

    # Don't try to add these items.
    excluded = ('player_id', 'team_id', 'competition_id', 'season_id', 'source_id')


    for stat in qs.values():

        # Set unaddable values to none.
        for k,v  in stat.items():
            if v in ('?', 'None', '-'):
                stat[k] = None



        # This determines what is filtered.
        # e.g., create all-time player stats with 
        # make_key = lambda s: s['player']
        key = make_key(stat) 
        # Create a new entry for this stat type
        if key not in final_dict:
            # This should set all necessary fields.
            final_dict[key] = stat
        else:
            # 
            d = final_dict[key]
            for key, value in stat.items():
                if key not in excluded: 
                    if not d[key]:
                        d[key] = value
                    else:
                        if value:
                            try:
                                d[key] += value
                            except:
                                import pdb; pdb.set_trace()
                                x = 5


    for key, stat in final_dict.items():
        stat.pop('id')
        for e in update_dict.keys():
            if e in stat:
                stat.pop(e)
        
        # update_dict seems unnecessary at this point. Those values aren't in the given stat.
        #stat.update(update_dict)

    insert_sql(table, list(final_dict.values()))
Пример #4
0
def vmadd():
    if 'username' not in session:
        return redirect('/login/')

    if request.method == 'POST':
        msg = sessionmsg()
        if msg['role'] != 0 and msg['role'] != 1:
            owner = msg['username']
            status = 0
            create_date = "1900-01-01"
            expire_date = "1900-01-01"
            #form_result数据样例{'project': u'1', 'term': u'1', 'used': u'22222222222', 'resource': u'2', 'system': u'os6'}
            form_result = request.form.to_dict()
            data = vmappend(form_result)
            field = [
                "hostname", "system", "term", "cpu", "mem", "disk", "project",
                "used", "ip", "mask", "gateway", "create_date", "expire_date",
                "owner", "status"
            ]
            data.update(create_date=create_date,
                        expire_date=expire_date,
                        owner=owner,
                        status=status)
            try:
                result = insert_sql(table_name, field, data)
                if result['code'] == 0:
                    result = {
                        'code': 0,
                        'msg': data['hostname'] + "虚机申请成功,等待审核"
                    }
            except:
                result = {'code': 1, 'msg': data['hostname'] + "虚机已存在"}
        else:
            result = {'code': 1, 'msg': "管理员无法申请账号"}
    return json.dumps(result)
Пример #5
0
def  vpnadd():
    if 'username' not in  session:
        return redirect('/login/')

    if request.method=='POST':
        msg = sessionmsg()
        if msg['role'] != 0 and msg['role'] != 1:
            #user_field  = ["id","username","name_cn","password","mobile","email","role","status"]
            #data={'username':msg['username']}
            #user_data = getone('user',data,user_field)
            #user_id = (user_data['msg']).get('id')
            owner = msg['username']
            status = 0
            #create_date = datetime.datetime.now()
            #expire_date = create_date + datetime.timedelta(days = 365)
            #create_date = create_date.strftime('%Y-%m-%d')
            #expire_date = expire_date.strftime('%Y-%m-%d')
            create_date = "1900-01-01"
            expire_date = "1900-01-01"
            field  = ["name","term","used","create_date","expire_date","owner","status"]
            data= {k:v[0] for k,v in dict(request.form).items()}
            data.update(create_date=create_date,expire_date=expire_date,owner=owner,status=status)
            try:
                result = insert_sql(table_name,field,data)
                if  result['code'] == 0:
                    result ={'code':0, 'msg':data['name'] + "账号申请成功,等待审核"}
            except:
                result ={'code':1, 'msg':data['name'] + "已存在"}
        else:
            result ={'code':1, 'msg':"管理员无法申请账号"}

    return  json.dumps(result)
Пример #6
0
def  vpnadd1():
    if 'username' not in  session:
        return redirect('/login/')

#获取当前用户列表,传入前端select添加vpn模态框
    if request.method=='GET':
        field  = "username"
        result = getall('user',field)
        print "result:"
        print result
        return json.dumps(result['msg'])
#添加vpn
    if request.method=='POST':
        status = 0
        create_date = "1900-01-01"
        expire_date = "1900-01-01"
        field  = ["name","term","used","create_date","expire_date","owner","status"]
        data= {k:v[0] for k,v in dict(request.form).items()}
        data.update(create_date=create_date,expire_date=expire_date,status=status)
#检查user表账号是否存在
        username=['username']
        data_check = {'username':data['owner']}
        result = check('user',username,data_check)
        if result['code'] == 1:
            result ={'code':1, 'msg':data['owner'] + "申请人账号不存在"}
            return  json.dumps(result)
#添加vpn数据
        try:
            result = insert_sql(table_name,field,data)
            if  result['code'] == 0:
                result ={'code':0, 'msg':data['name'] + "账号添加成功"}
        except:
            result ={'code':1, 'msg':data['name'] + "账号已存在"}
    return  json.dumps(result)
Пример #7
0
def generate_stats_generic(table, qs, make_key, update_dict):
    """
    Generate team, career, etc. stats.
    Maybe could improve this.
    """

    final_dict = {}
    excluded = ('player_id', 'team_id', 'competition_id', 'season_id', 'source_id') 

    for stat in qs.values():

        # Guard against unaddable values
        for k,v  in stat.items():
            if v in ('?', 'None', '-'):
                stat[k] = None

        # This determines what is filtered.
        # e.g., create all-time player stats with 
        # make_key = lambda s: s['player']
        key = make_key(stat) 
        # Create a new entry for this stat type
        if key not in final_dict:
            # This should set all necessary fields.
            final_dict[key] = stat
        else:
            d = final_dict[key]
            for key, value in stat.items():
                if key not in excluded: 
                    if not d[key]:
                        d[key] = value
                    else:
                        if value:
                            try:
                                d[key] += value
                            except:
                                import pdb; pdb.set_trace()
                                _ = 0


    for key, stat in final_dict.items():
        stat.pop('id')
        for e in update_dict.keys():
            if e in stat:
                stat.pop(e)


    insert_sql(table, list(final_dict.values()))
Пример #8
0
def generate_stats_generic(table, qs, make_key, update_dict):
    """
    Generate team, career, etc. stats.
    Maybe could improve this.
    """

    final_dict = {}
    excluded = ('player_id', 'team_id', 'competition_id', 'season_id',
                'source_id')

    for stat in qs.values():

        # Guard against unaddable values
        for k, v in stat.items():
            if v in ('?', 'None', '-'):
                stat[k] = None

        # This determines what is filtered.
        # e.g., create all-time player stats with
        # make_key = lambda s: s['player']
        key = make_key(stat)
        # Create a new entry for this stat type
        if key not in final_dict:
            # This should set all necessary fields.
            final_dict[key] = stat
        else:
            d = final_dict[key]
            for key, value in stat.items():
                if key not in excluded:
                    if not d[key]:
                        d[key] = value
                    else:
                        if value:
                            try:
                                d[key] += value
                            except:
                                import pdb
                                pdb.set_trace()
                                _ = 0

    for key, stat in final_dict.items():
        stat.pop('id')
        for e in update_dict.keys():
            if e in stat:
                stat.pop(e)

    insert_sql(table, list(final_dict.values()))
Пример #9
0
def namedadd():
    if 'username' not in session:
        return redirect('/login/')
    msg = sessionmsg()
    if request.method == 'POST':
        data = {k: v[0] for k, v in dict(request.form).items()}
        result = insert_sql('dns_records', fields, data)
        if result['code'] == 0:
            result = {'code': 0, 'msg': "Add Zone Successful"}
            return json.dumps(result)
Пример #10
0
def idcadd():
    if 'username' not in  session:
        return redirect('/login/')
    msg = sessionmsg()
    if request.method=='POST':
        idc = {k:v[0] for k,v in dict(request.form).items()}
        field = ['name','name_cn','address','adminer','phone']
        result = insert_sql('idc',field,idc)
        if  result['code'] == 0:
            result ={'code':0, 'msg':"IDC user success"}
            return  json.dumps(result)
Пример #11
0
def  add():
    if 'username' not in  session:
        return redirect('/login/')

    if request.method=='POST':
        field  = ["username","name_cn","password","mobile","email","role","status"]
        data= {k:v[0] for k,v in dict(request.form).items()}
        print data
        result = insert_sql('user',field,data)
        if  result['code'] == 0:
            result ={'code':0, 'msg':"add user success"}
    return  json.dumps(result)
Пример #12
0
def register():
    if request.method == 'POST':
        data = {k: v[0] for k, v in dict(request.form).items()}
        data['password'] = hashlib.md5(data['password'] + salt).hexdigest()
        field = ['id', 'username', 'password', 'role']
        result = insert_sql('user', fieald, data)
        print result
        if result['code'] == 0:
            return '用添加成功'
        else:
            return '用户添加失败'

    return render_template('user-add.html')
Пример #13
0
def addhost():
    if 'username' not in session:
        return redirect('/login/')

    if request.method == 'POST':
        data = dict((k, v[0]) for k, v in dict(request.form).items())

        field = ['username', 'password', 'ip']
        result = insert_sql('host', field, data)
        if result['code'] == 0:
            return '用添加成功'
        else:
            return '用户添加失败'
    return render_template('add-host.html')
Пример #14
0
def serveradd():
    if 'username' not in  session:
        return redirect('/login/')
    msg = sessionmsg()
    if request.method=='GET':
        idc   =  list('idc',idc_fields)
        cabinet   =  list('cabinet',cabinet_fields)
        return render_template('serveradd.html',msg=msg,idc=idc['msg'],cabinet=cabinet['msg'])
    if request.method=='POST':
       server  = {k:v[0] for k,v in dict(request.form).items()}
       result = insert_sql('server',server_field,server)
       if  result['code'] == 0:
           result ={'code':0, 'msg':"success"}
           return  json.dumps(result)
Пример #15
0
def load_transactions():

    print("loading {} transactions".format(soccer_db.transactions.count()))

    transactions = []

    for t in soccer_db.transactions.find():
        try:
            t.pop('_id')

            if t.get('team_from'):
                team_from = Team.objects.find(t['team_from'], create=True)
                team_from_id = team_from.id
            else:
                team_from_id = None

            if t.get('team_to'):
                team_to = Team.objects.find(t['team_to'], create=True)
                team_to_id = team_to.id
            else:
                team_to_id = None

            person = Bio.objects.find(t['person'])

            transactions.append({
                'ttype': t['ttype'],
                'person_id': person.id,
                'team_to_id': team_to_id,
                'team_from_id': team_from_id,
                'date': t['date'],
            })
        except:
            import pdb
            pdb.set_trace()

    insert_sql("transactions_transaction", transactions)
Пример #16
0
def cabinetadd():
    if 'username' not in session:
        return redirect('/login/')
    msg = sessionmsg()
    if request.method == 'GET':
        fields = ['id', 'name']
        result = list('idc', fields)
        return render_template('cabinetadd.html', msg=msg, idc=result['msg'])
    if request.method == 'POST':
        cabinet = {k: v[0] for k, v in dict(request.form).items()}
        field = ['name', 'idc_id', 'u_num', 'power']
        result = insert_sql('cabinet', field, cabinet)
        if result['code'] == 0:
            result = {'code': 0, 'msg': "success"}
            return json.dumps(result)
Пример #17
0
def jobadd():
        if 'username' not in  session:
             return redirect('/login/')
        msg = sessionmsg()
        if request.method=='GET':
            return render_template('jobadd.html',msg=msg)

        if request.method=='POST':
            job  = {k:v[0] for k,v in dict(request.form).items()}
            job['apply_name'] = session['username']
            field = ['apply_type','apply_name','apply_desc']
            result = insert_sql('job',field,job)
            if  result['code'] == 0:
                result ={'code':0, 'msg':"Job Add  success"}
                return  json.dumps(result)
Пример #18
0
def  add():
    if 'username' not in  session:
        return redirect('/login/')
    msg = sessionmsg()

    if request.method=='GET':
        msg = {'username':session['username'],'role':session['role']}
        return render_template('add.html',msg=msg)

    if request.method=='POST':
        field  = ["username","name_cn","password","mobile","email","role","status"]
        data= {k:v[0] for k,v in dict(request.form).items()}
        result = insert_sql('user',field,data)
        if  result['code'] == 0:
            result ={'code':0, 'msg':"add user success"}
    return  json.dumps(result)
Пример #19
0
def add():
    if 'username' not in session:
        return redirect('/login/')

    if request.method == 'POST':
        field = [
            "username", "name_cn", "password", "mobile", "email", "role",
            "status"
        ]
        data = {k: v[0] for k, v in dict(request.form).items()}
        data['password'] = hashlib.md5(data['password'] + salt).hexdigest()
        try:
            result = insert_sql('user', field, data)
            if result['code'] == 0:
                result = {'code': 0, 'msg': "add user success"}
            else:
                result = {'code': 1, 'msg': "add user failed"}
        except:
            result = {'code': 1, 'msg': "username already exsit"}
    return json.dumps(result)
Пример #20
0
def vmadd1():
    if 'username' not in session:
        return redirect('/login/')

#获取当前用户列表,传入前端select添加vpn模态框
    if request.method == 'GET':
        field = "username"
        result = getall('user', field)
        print "result:"
        print result
        return json.dumps(result['msg'])
#添加vm
    if request.method == 'POST':
        status = 0
        create_date = "1900-01-01"
        expire_date = "1900-01-01"
        form_result = request.form.to_dict()
        data = vmappend(form_result)
        field = [
            "hostname", "system", "term", "cpu", "mem", "disk", "project",
            "used", "ip", "mask", "gateway", "create_date", "expire_date",
            "owner", "status"
        ]
        data.update(create_date=create_date,
                    expire_date=expire_date,
                    status=status)
        #检查user表账号是否存在
        username = ['username']
        data_check = {'username': data['owner']}
        result = check('user', username, data_check)
        if result['code'] == 1:
            result = {'code': 1, 'msg': data['owner'] + "申请人账号不存在"}
            return json.dumps(result)
#添加vm数据
        try:
            result = insert_sql(table_name, field, data)
            if result['code'] == 0:
                result = {'code': 0, 'msg': data['hostname'] + "添加成功"}
        except:
            result = {'code': 1, 'msg': data['hostname'] + "添加失败"}
    return json.dumps(result)
Пример #21
0
def register():
    if request.method == 'POST':
        data = dict((k, v[0]) for k, v in dict(request.form).items())
        data['password'] = hashlib.md5(data['password'] + salt).hexdigest()
        # 1.判断用户输入用户名密码是否为空
        if not data["username"] or not data["password"]:
            error = 'UserName Or Password Not Null !'
            return render_template('register.html', error=error)

        # 2.判断该用户是否存在
        elif check_user('user_messages', data):
            error = u'用户 %s 已经存在' % (data["username"])
            return render_template('register.html', error=error)

        # 3.注册该用户信息至数据库
        else:
            field = ['username', 'password', 'role']
            if insert_sql('user_messages', data, field):
                return redirect("/login/")

    return render_template('register.html')
Пример #22
0
def load_goals():
    print("\nloading goals\n")

    team_getter = make_team_getter()
    bio_getter = make_bio_getter()
    game_getter = make_game_getter()
    gid_getter = make_gid_getter()

    l = []


    def create_goal(goal):

        team_id = team_getter(goal['team'])
        bio_id = ogbio_id = None

        if goal['goal']:
            bio_id = bio_getter(goal['goal'])

        if goal.get('own_goal_player'):
            ogbio_id = bio_getter(goal['own_goal_player'])


        # Tough to apply a goal without a date...
        if not goal['date']:
            return {}

        # Coerce to date to match dict.
        d = datetime.date(goal['date'].year, goal['date'].month, goal['date'].day)

        # Try gid first, fall back on team/date.
        game_id = None

        if 'gid' in goal:
            game_id = gid_getter(goal['gid'])

        if game_id is None:
            game_id = game_getter(team_id, d)

        if not game_id:
            print("Cannot create %s" % goal)
            return {}
        else:
            return {
                'date': goal['date'],
                'minute': goal['minute'],
                'team_id': team_id, 
                #'team_original_name': '',

                'player_id': bio_id, #player,
                'own_goal_player_id': ogbio_id,

                'game_id': game_id, 

                'own_goal': goal.get('own_goal', False),
                'penalty': goal.get('penalty', False),
                }

    i = 0 # if no goals.
    goals = []
    for i, goal in enumerate(soccer_db.goals.find()):
        if i % 50000 == 0:
            print(i)

        g = create_goal(goal)
        if g:
            goals.append(g)

    print(i)
    insert_sql('goals_goal', goals)

    events = []

    for goal in goals:
        event = {
            'game_id': goal['game_id'],
            'team_id': goal['team_id'],
            'minute': goal['minute'],
            'etype': 'goal',
            'subject_id': goal['player_id'],
            'object_id': None, 
            'description': '',
        }
        if goal['player_id']:
            events.append(event)


    insert_sql('events_event', events)
Пример #23
0
def load_assists():
    print("\nloading assists\n")

    team_getter = make_team_getter()
    bio_getter = make_bio_getter()
    goal_getter = make_goal_getter()

    def create_assists(goal):

        #if goal['competition'] == 'Major League Soccer'  and goal['season'] == '1996':
        #    import pdb; pdb.set_trace()
        


        if not goal['assists']:
            return []

        if goal['assists'] == ['']:
            return []



        team_id = team_getter(goal['team'])
        bio_id = ogbio_id = None

        if goal['goal']:
            bio_id = bio_getter(goal['goal'])


        if not goal['date']:
            return {}

        d = datetime.date(goal['date'].year, goal['date'].month, goal['date'].day)

        goal_id = goal_getter(team_id, bio_id, goal['minute'], d)
        if not goal_id:
            #import pdb; pdb.set_trace()
            print("Cannot create assists for %s" % goal)
            return []

        seen = set()

        for assister in goal['assists']:
            assist_ids = [bio_getter(e) for e in goal['assists']]
            for i, assist_id in enumerate(assist_ids, start=1):
                if assist_id and assist_id not in seen:
                    seen.add(assist_id)
                    assists.append({
                        'player_id': assist_id,
                        'goal_id': goal_id,
                        'order': i,
                        })

    assists = []

    i = 0
    for i, goal in enumerate(soccer_db.goals.find()):
        if i % 50000 == 0:
            print(i)
        create_assists(goal)

    print(i)
    print(len(assists))
    insert_sql('goals_assist', assists)
Пример #24
0
def load_fouls():

    print("loading fouls")
    print(soccer_db.fouls.count())



    team_getter = make_team_getter()
    bio_getter = make_bio_getter()
    game_getter = make_game_getter()
    gid_getter = make_gid_getter()

    l = []

    def create_foul(foul):
        
        if foul['team'] is None:
            return {}

        team_id = team_getter(foul['team'])

        bio_id = None

        bio_id = bio_getter(foul['name'])

        if foul['type'] == 'red':
            ftype = 'red'
        elif foul['type'] == 'yellow':
            ftype = 'yellow'
        else:
            import pdb; pdb.set_trace()

        if not foul['date']:
            return {}

        # Coerce to date to match dict.
        d = datetime.date(foul['date'].year, foul['date'].month, foul['date'].day)

        # Try gid first, fall back on team/date.
        game_id = None

        if 'gid' in foul:
            game_id = gid_getter(foul['gid'])

        if game_id is None:
            game_id = game_getter(team_id, d)

        if not game_id:
            print("Cannot create %s" % goal)
            return {}
        else:
            return {
                'game_id': game_id, 
                'team_id': team_id, 
                'minute': foul['minute'],
                
                'etype': ftype,
                #'team_original_name': '',

                'subject_id': bio_id, #player,

                'description': '',

                }

    i = 0 # if no goals.
    fouls = []
    for i, foul in enumerate(soccer_db.fouls.find()):
        if i % 50000 == 0:
            print(i)

        f = create_foul(foul)
        if f:
            fouls.append(f)

    insert_sql('events_event', fouls)
Пример #25
0
def load_standings():
    print("\n loading {} standings\n".format(soccer_db.standings.count()))

    team_getter = make_team_getter()
    competition_getter = make_competition_getter()
    season_getter = make_season_getter()

    # Create final standings from day-to-day standings
    final_standings = set()
    all_standings = set()
    max_game_standings = {}


    l = []
    for standing in soccer_db.standings.find().sort('team', 1):
        standing.pop('_id')
        
        competition_id = competition_getter(standing['competition'])
        season_id = season_getter(standing['season'], competition_id)
        team_id = team_getter(standing['team'])

        stage = standing.get('stage') or ''
        group = standing.get('group') or ''

        # make sure this is gone everywhere.
        # division = standing.get('division') or '' # division

        final = standing.get('final', False)


        d = {
                'competition_id': competition_id,
                'season_id': season_id,
                'group': group,
                'stage': stage,

                'team_id': team_id,
                'date': standing.get('date'),

                'games': standing['games'],
                'goals_for': standing.get('goals_for'),
                'goals_against': standing.get('goals_against'),
                'wins': standing['wins'],
                'shootout_wins': standing['shootout_wins'],
                'losses': standing['losses'],
                'shootout_losses': standing['shootout_losses'],
                'ties': standing['ties'],
                'points': standing.get('points'),
                'final': final,
                'deduction_reason': '',
                }
        l.append(d)

        """
        key = (standing['team'], standing['competition'], standing['season'])

        all_standings.add(key)
        if final:
            final_standings.add(key)

        if key not in max_game_standings or standing['games'] > max_game_standings[key]:
            max_game_standings[key] = d
            """

    insert_sql("standings_standing", l)

    # Handle this somewhere else.
    # Generate appropriate final standings.
    print("Generating final standings.")
    l2 = []
    for key in all_standings - final_standings:
        standing = max_game_standings[key].copy()
        #standing['final'] = True
        #standing['date'] = None
        l2.append(standing)

    insert_sql("standings_standing", l2)
Пример #26
0
 def insert_game_minutes():
     print("Generating {} game minutes".format(len(l)))
     insert_sql("games_gameminute", l)
Пример #27
0
def load_drafts():
    print("\nloading drafts\n")

    competition_getter = make_competition_getter()
    season_getter = make_season_getter()

    team_getter = make_team_getter()

    # Create the set of drafts.
    """
    for draft in soccer_db.drafts.find().sort('team', 1):
        draft.pop('_id')

        if draft['competition']:
            competition_id = competition_getter(draft['competition'])        
        else:
            competition_id = None

        season_id = season_getter(draft['season'], competition_id)

        Draft.objects.create(**{
                'name': draft['name'],
                'season_id': season_id,
                'competition_id': competition_id,
                'start': draft.get('start'),
                'end': draft.get('end'),
                })
    """

    print("\nloading {} picks\n".format(soccer_db.picks.count()))

    #import pdb; pdb.set_trace()

    # Create picks
    picks = []
    for pick in soccer_db.picks.find():

        # draft, text, player, position, team

        c = pick.get('competition')
        if c:
            competition_id = competition_getter(pick.get('competition'))
        else:
            competition_id = None

        season_id = season_getter(pick.get('season'), competition_id)

        #draft = Draft.objects.get(name=pick.get('draft'), competition_id=competition_id, season_id=season_id)
        draft = None

        if pick['team'] == 'Sean_Irish_LAGalaxy/Geneva':
            pass  #import pdb; pdb.set_trace()

        team_id = team_getter(pick['team'])

        if pick.get('former_team'):
            former_team_id = team_getter(pick['former_team'])
        else:
            former_team_id = None

        # Set the player reference.
        text = pick['text']

        # Draft picks were "drafted" in the MLS Allocation and Dispersal drafts.
        if "SuperDraft" in text:
            player_id = None
        elif text.lower() == 'pass':
            player_id = None
        else:
            player_id = Bio.objects.find(text).id
        """
        picks.append({
                #'draft_id': draft.id,
                'player_id': player_id,
                'team_id': team_id,
                'text': text,
                'position': pick.get('position') or '',
                'former_team_id': former_team_id,
                'number': pick['number'],
                })
        """

        picks.append({
            #'draft_id': draft.id,
            'ttype': 'draft pick',
            'date': pick['date'],
            'person_id': player_id,
            'team_to_id': team_id,
            'team_from_id': former_team_id,

            #'text': text,
            #'position': pick.get('position') or '',
            #'number': pick['number'],
        })

    #insert_sql("drafts_pick", list(picks))
    insert_sql("transactions_transaction", list(picks))
Пример #28
0
def load_lineups():
    # This should be removed in favor of substitutions
    # Ultimately subsumed by a combination game stats / substitutions
    # Too big to be worthwhile.


    # Need to do this with raw sql and standard dict management functions.
    print("\nloading lineups\n")
    from django.db import connection

    team_getter = make_team_getter()
    bio_getter = make_bio_getter()
    game_getter = make_game_getter()
    gid_getter = make_gid_getter()

    def create_appearance(a):

        if not a['name']:
            print(a)
            return None

        team_id = team_getter(a['team'])
        player_id = bio_getter(a['name'])

        game_id = None

        if 'gid' in a:
            game_id = gid_getter(a['gid'])

        if game_id is None:
            game_id = game_getter(team_id, a['date'])

            
        #game_id = game_getter(team_id, a['date'])

        if not game_id:

            import pdb; pdb.set_trace()

            print("Cannot create %s" % a)
            return {}

        if a['on'] is not None and a['off'] is not None:
            try:
                minutes = int(a['off']) - int(a['on'])
            except:
                print("Fail on %s" % str(a))
                minutes = None
        else:
            minutes = None

        return {
            'team_id': team_id,
            'game_id': game_id,
            'player_id': player_id,
            'on': a['on'],
            'off': a['off'],
            'minutes': minutes,
            'order': a.get('order', None),
            }

    # Create the appearance objects.
    l = []
    i = 0
    for i, a in enumerate(soccer_db.lineups.find()):
        u = create_appearance(a)
        if u:
            l.append(u)

        if i % 50000 == 0:
            print(i)

    print(i)
    print("Creating lineups")
    insert_sql('lineups_appearance', l)
Пример #29
0
def load_game_stats():
    print("\nloading game stats\n")

    team_getter = make_team_getter()
    bio_getter = make_bio_getter()
    source_getter = make_source_getter()
    game_getter = make_game_getter()
    gid_getter = make_gid_getter()
    game_result_getter = make_game_result_getter()

    birthdate_dict = dict(Bio.objects.exclude(birthdate=None).values_list("id", "birthdate"))
    
    print("\nprocessing")

    l = []    
    i = 0
    for i, stat in enumerate(soccer_db.gstats.find(timeout=False)): # no timeout because this query takes forever.
        if i % 50000 == 0:
            print(i)

        if stat['player'] == '':
            #import pdb; pdb.set_trace()
            continue

        
        try:
            bio_id = bio_getter(stat['player'])
        except:
            continue

        if bio_id is None:
            continue


        bd = birthdate_dict.get(bio_id)
        if stat['date'] and bd:
            # Coerce bd from datetime.date to datetime.time
            bdt = datetime.datetime.combine(bd, datetime.time())
            age = (stat['date'] - bdt).days / 365.25
        else:
            age = None

        team_id = team_getter(stat['team'])


        if 'gid' in stat:
            game_id = gid_getter(stat['gid'])
        else:
            game_id = game_getter(team_id, stat['date'])

        #game_id = game_getter(team_id, stat['date'])


        result = game_result_getter(team_id, stat['date'])

        if game_id is None or team_id is None:
            continue


        def c2i(key, coerce_none=True):
            # Coerce an integer

            if key in stat and stat[key] != None:
                if type(stat[key]) != int:
                    import pdb; pdb.set_trace()
                return stat[key]

            elif key in stat and stat[key] == None:
                return 0

            else:
                return None

        l.append({
            'player_id': bio_id,
            'team_id': team_id,
            'game_id': game_id,
            'games_started': c2i('games_started'),
            'games_played': c2i('games_played'),
            'minutes': c2i('minutes'),
            'goals': c2i('goals'),
            'assists': c2i('assists'),
            'shots': c2i('shots'),
            'shots_on_goal': c2i('shots_on_goal'),
            'fouls_committed': c2i('fouls_committed'),
            'fouls_suffered': c2i('fouls_suffered'),
            'yellow_cards': c2i('yellow_cards'),
            'red_cards': c2i('red_cards'),
            'on': c2i('on', False),
            'off': c2i('off', False),
            'age': age,
            'result': result,
            })

    print(i)


    insert_sql("stats_gamestat", l)
Пример #30
0
def load_drafts():
    print("\nloading drafts\n")

    competition_getter = make_competition_getter()
    season_getter = make_season_getter()

    team_getter = make_team_getter()

    # Create the set of drafts.

    """
    for draft in soccer_db.drafts.find().sort('team', 1):
        draft.pop('_id')

        if draft['competition']:
            competition_id = competition_getter(draft['competition'])        
        else:
            competition_id = None

        season_id = season_getter(draft['season'], competition_id)

        Draft.objects.create(**{
                'name': draft['name'],
                'season_id': season_id,
                'competition_id': competition_id,
                'start': draft.get('start'),
                'end': draft.get('end'),
                })
    """

    print("\nloading {} picks\n".format(soccer_db.picks.count()))

    #import pdb; pdb.set_trace()

    # Create picks
    picks = []
    for pick in soccer_db.picks.find():

        # draft, text, player, position, team

        c = pick.get('competition')
        if c:
            competition_id = competition_getter(pick.get('competition'))
        else:
            competition_id = None

        season_id = season_getter(pick.get('season'), competition_id)

        #draft = Draft.objects.get(name=pick.get('draft'), competition_id=competition_id, season_id=season_id)
        draft = None

        if pick['team'] == 'Sean_Irish_LAGalaxy/Geneva':
            pass #import pdb; pdb.set_trace()


        team_id = team_getter(pick['team'])

        if pick.get('former_team'):
            former_team_id = team_getter(pick['former_team'])
        else:
            former_team_id = None

        # Set the player reference.
        text = pick['text']

        # Draft picks were "drafted" in the MLS Allocation and Dispersal drafts.
        if "SuperDraft" in text:
            player_id = None
        elif text.lower() == 'pass':
            player_id = None
        else:
            player_id = Bio.objects.find(text).id

        """
        picks.append({
                #'draft_id': draft.id,
                'player_id': player_id,
                'team_id': team_id,
                'text': text,
                'position': pick.get('position') or '',
                'former_team_id': former_team_id,
                'number': pick['number'],
                })
        """

        picks.append({
            #'draft_id': draft.id,
            'ttype': 'draft pick',
            'date': pick['date'],
            'person_id': player_id,
            'team_to_id': team_id,
            'team_from_id': former_team_id,

            #'text': text,
            #'position': pick.get('position') or '',
            #'number': pick['number'],
                })


    #insert_sql("drafts_pick", list(picks))
    insert_sql("transactions_transaction", list(picks))
Пример #31
0
def load_games():
    print("\n loading {} games\n".format(soccer_db.games.count()))

    stadium_getter = make_stadium_getter()
    team_getter = make_team_getter()
    competition_getter = make_competition_getter()    
    source_getter = make_source_getter()
    bio_getter = make_bio_getter()

    city_getter = make_city_getter()
    country_getter = make_country_getter()

    season_getter = make_season_getter()

    games = []
    game_sources = []

    for game in soccer_db.games.find().sort('date', 1):

        if game.get('city') == 'Columbus Crew Stadium':
            import pdb; pdb.set_trace()

        # Apply stadium / state / country information.

        stadium_id = city_id = country_id = None
        if game.get('stadium'):

            
            stadium_id = stadium_getter(game['stadium'])
            s = Stadium.objects.get(id=stadium_id)
            if s.city:
                city_id = s.city.id
            else:
                city_id = None

        elif game.get('city'):
            city_id = city_getter(game['city']).id

        elif game.get('location'):

            country_id = country_getter(game['location'])
            if country_id is None:
                city_id = city_getter(game['location']).id

        competition_id = competition_getter(game['competition'])
        #game['competition'] = Competition.objects.get(id=game['competition'])

        #season_id = Season.objects.find(game['season'], competition_id).id # this!!
        season_id = season_getter(game['season'], competition_id)

        if game['season'] is None:
            import pdb; pdb.set_trace()

        team1_id = team_getter(game['team1'])
        team2_id = team_getter(game['team2'])

        home_team_id = None
        if game.get('home_team'):
            home_team_id = team_getter(game['home_team'])

        goals = (game['team1_score'] or 0) + (game['team2_score'] or 0)

        referee_id = linesman1_id = linesman2_id = linesman3_id = None
        if game['referee']:
            referee_id = bio_getter(game['referee'])

        if game.get('linesman1'):
            linesman1_id = bio_getter(game['linesman1'])

        if game.get('linesman2'):
            linesman2_id = bio_getter(game['linesman2'])

        if game.get('linesman3'):
            linesman3_id = bio_getter(game['linesman3'])


        if game.get('sources'):
            sources = sorted(set(game.get('sources')))
        elif game.get('source'):
            sources = [game['source']]
        else:
            sources = []

        
        for source in sources:
            if source.strip() == '':
                continue
            elif source.startswith('http'):
                source_url = source
            else:
                source_url = ''
            source_id = source_getter(source)
            t = (game['date'], team1_id, source_id, source_url)
            game_sources.append(t)

        result_unknown = game.get('result_unknown') or False
        not_played = game.get('not_played') or False
        forfeit = game.get('forfeit') or False
        minigame = game.get('minigame') or False
        indoor = game.get('indoor') or False

        minutes = game.get('minutes') or 90

        neutral = game.get('neutral') or False
        attendance = game.get('attendance')

        stage = game.get('stage') or ''
        group = game.get('group') or ''
        rnd = game.get('round') or ''


        # There are lots of problems with the NASL games, 
        # And probably ASL as well. Need to spend a couple
        # of hours repairing those schedules.

        if game['shootout_winner']:
            shootout_winner = team_getter(game['shootout_winner'])
        else:
            shootout_winner = None

        location = game.get('location', '')

        location = location or ''


        if 'gid' not in game:
            game['gid'] = get_id_by_time()


        games.append({
                'date': game['date'],
                'has_date': bool(game['date']),

                'team1_id': team1_id,
                'team1_original_name': game['team1_original_name'],
                'team2_id': team2_id,
                'team2_original_name': game['team2_original_name'],

                'team1_score': game['team1_score'],
                'official_team1_score': game.get('official_team1_score'),
                'team2_score': game['team2_score'],
                'official_team2_score': game.get('official_team2_score'),

                'shootout_winner_id': shootout_winner,

                'team1_result': game['team1_result'],
                'team2_result': game['team2_result'],

                'result_unknown': result_unknown,
                'not_played': not_played,
                'forfeit': forfeit,

                'goals': goals,
                'minigame': minigame,
                'indoor': indoor,

                'minutes': minutes,
                'competition_id': competition_id,
                'season_id': season_id,
                'stage': stage,
                'group': group,
                'round': rnd,


                'home_team_id': home_team_id,
                'neutral': neutral,

                'stadium_id': stadium_id,
                'city_id': city_id,
                'country_id': country_id,
                'location': location,
                'notes': game.get('notes', ''),
                'video': game.get('video', ''),
                'attendance': attendance,
                
                'referee_id': referee_id,
                'linesman1_id': linesman1_id,
                'linesman2_id': linesman2_id,
                'linesman3_id': linesman3_id,

                'merges': game['merges'],
                'gid': game['gid'],
                })


    print("Inserting {} games results.".format(len(games)))
    # Broke on massive attendance. 
    # Watch out for crazy integer values.
    insert_sql("games_game", games)

    print("Inserting games sources.")
    game_getter = make_game_getter()
    
    l = []
    for date, team_id, source_id, source_url in game_sources:

        # Don't call game_getter without date. Need to give games unique id's.
        if date:
            game_id = game_getter(team_id, date)
            if game_id:
                l.append({
                        'game_id': game_id,
                        'source_id': source_id,
                        'source_url': source_url,
                        })

    insert_sql("games_gamesource", l)
Пример #32
0
def load_bios():
    print("loading bios")

    cg = make_city_getter()


    # Find which names are used so we can only load these bios.
    # Huh? This is unnecessary.
    #fields = [('lineups', 'name'), ('goals', 'goal'), ('stats', 'name'), ('awards', 'recipient'), ('picks', 'text')]
    #names = set()

    # Add names to names field where they have been used.
    #for coll, key in fields:
    #    names.update([e[key] for e in soccer_db[coll].find()])

    # Load bios.
    for bio in soccer_db.bios.find().sort('name', 1):

        #if bio['name'] not in names:
            #print("Skipping %s" % bio['name'])
        #    continue

        bio.pop('_id')

        if not bio['name']:
            import pdb; pdb.set_trace()
            print("NO BIO: %s" % str(bio))
            continue

        # nationality should be many-to-many
        if 'nationality' in bio:
            bio.pop('nationality')

        bd = {}

        for key in 'name', 'height', 'birthdate', 'height', 'weight':
            if key in bio:
                bd[key] = bio[key] or None

        if bio.get('birthplace'):
            bd['birthplace'] = cg(bio['birthplace'])

        if bio.get('deathplace'):
            bd['deathplace'] = cg(bio['deathplace'])

        # Having unexpected problems here...
        bd['hall_of_fame'] = bio.get('hall_of_fame')
        if bd['hall_of_fame'] not in (True, False):
            bd['hall_of_fame'] = False

        Bio.objects.create(**bd)

    bio_getter = make_bio_getter()
    bio_ct_id = ContentType.objects.get(app_label='bios', model='bio').id

    images = []
    for bio in soccer_db.bios.find().sort('name', 1):
        if bio.get('img'):
            bid = bio_getter(bio['name'])
            fn = bio['img'].rsplit('/')[-1]
            
            images.append({
                    'filename': fn,
                    'content_type_id': bio_ct_id,
                    'object_id': bid,
                    })

    insert_sql("images_image", images)
Пример #33
0
def load_games():
    print("\n loading {} games\n".format(soccer_db.games.count()))

    stadium_getter = make_stadium_getter()
    team_getter = make_team_getter()
    competition_getter = make_competition_getter()
    source_getter = make_source_getter()
    bio_getter = make_bio_getter()

    city_getter = make_city_getter()
    country_getter = make_country_getter()

    season_getter = make_season_getter()

    games = []
    game_sources = []

    for game in soccer_db.games.find().sort('date', 1):

        if game.get('city') == 'Columbus Crew Stadium':
            import pdb
            pdb.set_trace()

        # Apply stadium / state / country information.

        stadium_id = city_id = country_id = None
        if game.get('stadium'):

            stadium_id = stadium_getter(game['stadium'])
            s = Stadium.objects.get(id=stadium_id)
            if s.city:
                city_id = s.city.id
            else:
                city_id = None

        elif game.get('city'):
            city_id = city_getter(game['city']).id

        elif game.get('location'):

            country_id = country_getter(game['location'])
            if country_id is None:
                city_id = city_getter(game['location']).id

        competition_id = competition_getter(game['competition'])
        #game['competition'] = Competition.objects.get(id=game['competition'])

        #season_id = Season.objects.find(game['season'], competition_id).id # this!!
        season_id = season_getter(game['season'], competition_id)

        if game['season'] is None:
            import pdb
            pdb.set_trace()

        team1_id = team_getter(game['team1'])
        team2_id = team_getter(game['team2'])

        home_team_id = None
        if game.get('home_team'):
            home_team_id = team_getter(game['home_team'])

        goals = (game['team1_score'] or 0) + (game['team2_score'] or 0)

        referee_id = linesman1_id = linesman2_id = linesman3_id = None
        if game['referee']:
            referee_id = bio_getter(game['referee'])

        if game.get('linesman1'):
            linesman1_id = bio_getter(game['linesman1'])

        if game.get('linesman2'):
            linesman2_id = bio_getter(game['linesman2'])

        if game.get('linesman3'):
            linesman3_id = bio_getter(game['linesman3'])

        if game.get('sources'):
            sources = sorted(set(game.get('sources')))
        elif game.get('source'):
            sources = [game['source']]
        else:
            sources = []

        for source in sources:
            if source.strip() == '':
                continue
            elif source.startswith('http'):
                source_url = source
            else:
                source_url = ''
            source_id = source_getter(source)
            t = (game['date'], team1_id, source_id, source_url)
            game_sources.append(t)

        result_unknown = game.get('result_unknown') or False
        not_played = game.get('not_played') or False
        forfeit = game.get('forfeit') or False
        minigame = game.get('minigame') or False
        indoor = game.get('indoor') or False

        minutes = game.get('minutes') or 90

        neutral = game.get('neutral') or False
        attendance = game.get('attendance')

        stage = game.get('stage') or ''
        group = game.get('group') or ''
        rnd = game.get('round') or ''

        # There are lots of problems with the NASL games,
        # And probably ASL as well. Need to spend a couple
        # of hours repairing those schedules.

        if game['shootout_winner']:
            shootout_winner = team_getter(game['shootout_winner'])
        else:
            shootout_winner = None

        location = game.get('location', '')

        location = location or ''

        if 'gid' not in game:
            game['gid'] = get_id_by_time()

        games.append({
            'date': game['date'],
            'has_date': bool(game['date']),
            'team1_id': team1_id,
            'team1_original_name': game['team1_original_name'],
            'team2_id': team2_id,
            'team2_original_name': game['team2_original_name'],
            'team1_score': game['team1_score'],
            'official_team1_score': game.get('official_team1_score'),
            'team2_score': game['team2_score'],
            'official_team2_score': game.get('official_team2_score'),
            'shootout_winner_id': shootout_winner,
            'team1_result': game['team1_result'],
            'team2_result': game['team2_result'],
            'result_unknown': result_unknown,
            'not_played': not_played,
            'forfeit': forfeit,
            'goals': goals,
            'minigame': minigame,
            'indoor': indoor,
            'minutes': minutes,
            'competition_id': competition_id,
            'season_id': season_id,
            'stage': stage,
            'group': group,
            'round': rnd,
            'home_team_id': home_team_id,
            'neutral': neutral,
            'stadium_id': stadium_id,
            'city_id': city_id,
            'country_id': country_id,
            'location': location,
            'notes': game.get('notes', ''),
            'video': game.get('video', ''),
            'attendance': attendance,
            'referee_id': referee_id,
            'linesman1_id': linesman1_id,
            'linesman2_id': linesman2_id,
            'linesman3_id': linesman3_id,
            'merges': game['merges'],
            'gid': game['gid'],
        })

    print("Inserting {} games results.".format(len(games)))
    # Broke on massive attendance.
    # Watch out for crazy integer values.
    insert_sql("games_game", games)

    print("Inserting games sources.")
    game_getter = make_game_getter()

    l = []
    for date, team_id, source_id, source_url in game_sources:

        # Don't call game_getter without date. Need to give games unique id's.
        if date:
            game_id = game_getter(team_id, date)
            if game_id:
                l.append({
                    'game_id': game_id,
                    'source_id': source_id,
                    'source_url': source_url,
                })

    insert_sql("games_gamesource", l)
Пример #34
0
def load_stats():
    print("\nloading stats\n")

    @timer
    def f():
        return

    team_getter, bio_getter, competition_getter, season_getter, source_getter = (
        make_team_getter(),
        make_bio_getter(),
        make_competition_getter(),
        make_season_getter(),
        make_source_getter(),
    )

    print("\nprocessing\n")

    l = []
    i = 0
    # for i, stat in enumerate(soccer_db.stats.find(timeout=False)): # no timeout because this query takes forever.
    for i, stat in enumerate(soccer_db.stats.find(no_cursor_timeout=True)):

        if i % 50000 == 0:
            print(i)

        if stat['name'] == '':
            #import pdb; pdb.set_trace()
            continue

        team_id = team_getter(stat['team'])
        bio_id = bio_getter(stat['name'])
        competition_id = competition_getter(stat['competition'])
        season_id = season_getter(stat['season'], competition_id)

        # cf game_sources stuff.
        """
        # change to sources!
        if stat.get('sources'): 
            sources = sorted(set(stat.get('sources')))
        elif stat.get('source'):
            sources = [stat['source']]
        else:
            sources = []

        for source in sources:
            if source.strip() == '':
                continue
            elif source.startswith('http'):
                source_url = source
            else:
                source_url = ''
            source_id = source_getter(source)
            #t = (game['date'], team1_id, source_id, source_url)
            #stat_sources.append(t)
        """

        if stat.get('source'):
            source_id = source_getter(stat['source'])

        else:
            source_id = None

        def c2i(key):
            # Coerce an integer

            if key in stat and stat[key] != None:
                if type(stat[key]) != int:
                    import pdb
                    pdb.set_trace()
                return stat[key]

            else:
                return None

        l.append({
            'player_id': bio_id,
            'team_id': team_id,
            'competition_id': competition_id,
            'season_id': season_id,
            'games_started': c2i('games_started'),
            'games_played': c2i('games_played'),
            'minutes': c2i('minutes'),
            'goals': c2i('goals'),
            'assists': c2i('assists'),
            'shots': c2i('shots'),
            'shots_on_goal': c2i('shots_on_goal'),
            'fouls_committed': c2i('fouls_committed'),
            'fouls_suffered': c2i('fouls_suffered'),
            'yellow_cards': c2i('yellow_cards'),
            'red_cards': c2i('red_cards'),
            'source_id': source_id,
        })

    print(i)

    insert_sql("stats_stat", l)
Пример #35
0
def load_fouls():

    print("loading fouls")
    print(soccer_db.fouls.count())

    team_getter = make_team_getter()
    bio_getter = make_bio_getter()
    game_getter = make_game_getter()
    gid_getter = make_gid_getter()

    l = []

    def create_foul(foul):

        if foul['team'] is None:
            return {}

        team_id = team_getter(foul['team'])

        bio_id = None

        bio_id = bio_getter(foul['name'])

        if foul['type'] == 'red':
            ftype = 'red'
        elif foul['type'] == 'yellow':
            ftype = 'yellow'
        else:
            import pdb
            pdb.set_trace()

        if not foul['date']:
            return {}

        # Coerce to date to match dict.
        d = datetime.date(foul['date'].year, foul['date'].month,
                          foul['date'].day)

        # Try gid first, fall back on team/date.
        game_id = None

        if 'gid' in foul:
            game_id = gid_getter(foul['gid'])

        if game_id is None:
            game_id = game_getter(team_id, d)

        if not game_id:
            print("Cannot create %s" % goal)
            return {}
        else:
            return {
                'game_id': game_id,
                'team_id': team_id,
                'minute': foul['minute'],
                'etype': ftype,
                #'team_original_name': '',
                'subject_id': bio_id,  #player,
                'description': '',
            }

    i = 0  # if no goals.
    fouls = []
    for i, foul in enumerate(soccer_db.fouls.find()):
        if i % 50000 == 0:
            print(i)

        f = create_foul(foul)
        if f:
            fouls.append(f)

    insert_sql('events_event', fouls)
Пример #36
0
def load_awards():

    print("\nloading awards\n")
    awards = set()
    award_dict = {}

    competition_getter = make_competition_getter()
    season_getter = make_season_getter()
    team_getter = make_team_getter()
    bio_getter = make_bio_getter()

    # Create the set of all awards.
    for item in soccer_db.awards.find():
        t = (item['competition'], item['award'], item.get('type', ''))
        awards.add(t)

    # Create all awards.
    for t in awards:
        competition, name, award_type = t

        # Using find because currently using NCAA awards but don't have ncaa standings.
        # Should be able to switch to competition_getter here.
        if competition:
            competition = Competition.objects.find(competition)

        a = Award.objects.create(competition=competition, name=name, type=award_type)
        award_dict[t] = a


    bio_ct_id = ContentType.objects.get(app_label='bios', model='bio').id
    team_ct_id = ContentType.objects.get(app_label='teams', model='team').id

    # Create awardItems.
    items = []
    for item in soccer_db.awards.find().sort('recipient', 1):
        item.pop('_id')

        # So we can have a season, a year, both, or neither for an award item
        award = award_dict[(item['competition'], item['award'], item.get('type', ''))]
        award_id = award.id

        # NCAA seasons don't exist.
        # Would be good to use get otherwise to ensure we have good data.
        if award.competition:
            competition_id = award.competition.id
            season_id = season_getter(item['season'], competition_id)
        else:
            season_id = None

        model_name = item.pop('model')
        if model_name == 'Bio':
            content_type_id = bio_ct_id
            object_id = bio_getter(item['recipient'])
        elif model_name == 'Team':
            content_type_id = team_ct_id
            object_id = team_getter(item['recipient'])
        else:
            import pdb; pdb.set_trace()
            raise

        items.append({
                'award_id': award_id,
                'season_id': season_id,
                'content_type_id': content_type_id,
                'object_id': object_id,
                })

    insert_sql("awards_awarditem", list(items))
Пример #37
0
def load_goals():
    print("\nloading goals\n")

    team_getter = make_team_getter()
    bio_getter = make_bio_getter()
    game_getter = make_game_getter()
    gid_getter = make_gid_getter()

    l = []

    def create_goal(goal):

        team_id = team_getter(goal['team'])
        bio_id = ogbio_id = None

        if goal['goal']:
            bio_id = bio_getter(goal['goal'])

        if goal.get('own_goal_player'):
            ogbio_id = bio_getter(goal['own_goal_player'])

        # Tough to apply a goal without a date...
        if not goal['date']:
            return {}

        # Coerce to date to match dict.
        d = datetime.date(goal['date'].year, goal['date'].month,
                          goal['date'].day)

        # Try gid first, fall back on team/date.
        game_id = None

        if 'gid' in goal:
            game_id = gid_getter(goal['gid'])

        if game_id is None:
            game_id = game_getter(team_id, d)

        if not game_id:
            print("Cannot create %s" % goal)
            return {}
        else:
            return {
                'date': goal['date'],
                'minute': goal['minute'],
                'team_id': team_id,
                #'team_original_name': '',
                'player_id': bio_id,  #player,
                'own_goal_player_id': ogbio_id,
                'game_id': game_id,
                'own_goal': goal.get('own_goal', False),
                'penalty': goal.get('penalty', False),
            }

    i = 0  # if no goals.
    goals = []
    for i, goal in enumerate(soccer_db.goals.find()):
        if i % 50000 == 0:
            print(i)

        g = create_goal(goal)
        if g:
            goals.append(g)

    print(i)
    insert_sql('goals_goal', goals)

    events = []

    for goal in goals:
        event = {
            'game_id': goal['game_id'],
            'team_id': goal['team_id'],
            'minute': goal['minute'],
            'etype': 'goal',
            'subject_id': goal['player_id'],
            'object_id': None,
            'description': '',
        }
        if goal['player_id']:
            events.append(event)

    insert_sql('events_event', events)
Пример #38
0
def load_game_stats():
    print("\nloading game stats\n")

    team_getter = make_team_getter()
    bio_getter = make_bio_getter()
    source_getter = make_source_getter()
    game_getter = make_game_getter()
    gid_getter = make_gid_getter()
    game_result_getter = make_game_result_getter()

    birthdate_dict = dict(
        Bio.objects.exclude(birthdate=None).values_list("id", "birthdate"))

    print("\nprocessing")

    l = []
    i = 0
    #for i, stat in enumerate(soccer_db.gstats.find(timeout=False)): # no timeout because this query takes forever.
    for i, stat in enumerate(soccer_db.gstats.find(no_cursor_timeout=True)):
        if i % 50000 == 0:
            print(i)

        if stat['player'] == '':
            #import pdb; pdb.set_trace()
            continue

        try:
            bio_id = bio_getter(stat['player'])
        except:
            continue

        if bio_id is None:
            continue

        bd = birthdate_dict.get(bio_id)
        if stat['date'] and bd:
            # Coerce bd from datetime.date to datetime.time
            bdt = datetime.datetime.combine(bd, datetime.time())
            age = (stat['date'] - bdt).days / 365.25
        else:
            age = None

        team_id = team_getter(stat['team'])

        if 'gid' in stat:
            game_id = gid_getter(stat['gid'])
        else:
            game_id = game_getter(team_id, stat['date'])

        #game_id = game_getter(team_id, stat['date'])

        result = game_result_getter(team_id, stat['date'])

        if game_id is None or team_id is None:
            continue

        def c2i(key, coerce_none=True):
            # Coerce an integer

            if key in stat and stat[key] != None:
                if type(stat[key]) != int:
                    import pdb
                    pdb.set_trace()
                return stat[key]

            elif key in stat and stat[key] == None:
                return 0

            else:
                return None

        l.append({
            'player_id': bio_id,
            'team_id': team_id,
            'game_id': game_id,
            'games_started': c2i('games_started'),
            'games_played': c2i('games_played'),
            'minutes': c2i('minutes'),
            'goals': c2i('goals'),
            'assists': c2i('assists'),
            'shots': c2i('shots'),
            'shots_on_goal': c2i('shots_on_goal'),
            'fouls_committed': c2i('fouls_committed'),
            'fouls_suffered': c2i('fouls_suffered'),
            'yellow_cards': c2i('yellow_cards'),
            'red_cards': c2i('red_cards'),
            'on': c2i('on', False),
            'off': c2i('off', False),
            'age': age,
            'result': result,
        })

    print(i)

    insert_sql("stats_gamestat", l)
Пример #39
0
def load_stats():
    print("\nloading stats\n")

    @timer
    def f():
        return 


    team_getter, bio_getter, competition_getter, season_getter, source_getter = (
        make_team_getter(), make_bio_getter(), make_competition_getter(), make_season_getter(), make_source_getter(),)
                
    print("\nprocessing\n")

    l = []    
    i = 0
    for i, stat in enumerate(soccer_db.stats.find(timeout=False)): # no timeout because this query takes forever.


        if i % 50000 == 0:
            print(i)

        if stat['name'] == '':
            #import pdb; pdb.set_trace()
            continue


        team_id = team_getter(stat['team'])
        bio_id = bio_getter(stat['name'])
        competition_id = competition_getter(stat['competition'])
        season_id = season_getter(stat['season'], competition_id)


        # cf game_sources stuff.
        """
        # change to sources!
        if stat.get('sources'): 
            sources = sorted(set(stat.get('sources')))
        elif stat.get('source'):
            sources = [stat['source']]
        else:
            sources = []

        for source in sources:
            if source.strip() == '':
                continue
            elif source.startswith('http'):
                source_url = source
            else:
                source_url = ''
            source_id = source_getter(source)
            #t = (game['date'], team1_id, source_id, source_url)
            #stat_sources.append(t)
        """

        if stat.get('source'):
            source_id = source_getter(stat['source'])

        else:
            source_id = None

        def c2i(key):
            # Coerce an integer

            if key in stat and stat[key] != None:
                if type(stat[key]) != int:
                    import pdb; pdb.set_trace()
                return stat[key]

            else:
                return None

        l.append({
            'player_id': bio_id,
            'team_id': team_id,
            'competition_id': competition_id,
            'season_id': season_id,
            'games_started': c2i('games_started'),
            'games_played': c2i('games_played'),
            'minutes': c2i('minutes'),
            'goals': c2i('goals'),
            'assists': c2i('assists'),
            'shots': c2i('shots'),
            'shots_on_goal': c2i('shots_on_goal'),
            'fouls_committed': c2i('fouls_committed'),
            'fouls_suffered': c2i('fouls_suffered'),
            'yellow_cards': c2i('yellow_cards'),
            'red_cards': c2i('red_cards'),
            'source_id': source_id,
            })

    print(i)

    insert_sql("stats_stat", l)