def new_team_squad_scrap(engine): """ Same as new_team_squad_scrap, only for current season. :param engine: :return: """ session = settings.create_session(engine) this_season_id = session.query(func.max(Season.id)).first()[0] session.close() try: print "Season id is", str(this_season_id) for division in [1, 2, 3]: session = settings.create_session(engine) print "Search for division", str(division) team_parser(session, this_season_id, division) squad_parser(session, this_season_id, division) conference_parser(session, this_season_id, division) session.close() except Exception, e: error_message = """ this_season_id: %s division: %s This combination may not exists. %s """ % (this_season_id, division, e) write_error_to_file(error_message) raise
def initial_game_detail_scrap(engine): """ Scrape detail message for each game :param engine: :return: """ session = settings.create_session(engine) games = session.query(Game).filter_by(has_detail=0).order_by(asc( Game.id)).all() session.close() try: for game_record in games: session = settings.create_session(engine) gamedetail_parser(session, game_record) session.close() except Exception, e: message = """ game_id: %s Something wrong in game_detail_parser %s """ % (game_record.id, e) write_error_to_file(message) raise
def new_schedule_game_player_scrap(engine): """ Same as new_schedule_game_player_scrap, only for current season. :param engine: :return: """ session = settings.create_session(engine) this_season_id = session.query(func.max(Season.id)).first()[0] squads = session.query(Squad).filter_by(season_id=this_season_id).all() session.close() try: for squad_record in squads: # If the squad_id != fake_id (non-ncaa squad id) if squad_record.id != 1: print "####", squad_record.id, str(datetime.datetime.now()) session = settings.create_session(engine) schedule_parser(session, squad_record) # game_parser() is inside schedule_parser() player_parser(session, squad_record) # squadmember_parser() is inside player_parser() session.close() except Exception, e: error_message = """ squad_id: %s This combination may not exists. %s """ % (squad_record.id, e) write_error_to_file(str(error_message)) raise
def new_game_detail_scrap(engine): """ Same as new_game_detail_scrap, only for current season. :param engine: :return: """ session = settings.create_session(engine) games = session.query(Game).filter_by(has_detail=0).order_by(desc(Game.id)).all() session.close() try: for game_record in games: print "%%%% game_id is "+str(game_record.id), str(datetime.datetime.now()) session = settings.create_session(engine) gamedetail_parser(session, game_record) session.close() except Exception, e: message = """ game_id: %s Something wrong in game_detail_parser %s """ % (game_record.id, e) write_error_to_file(message) raise
def new_season_stat_scrap(engine, gender): """ Same as new_season_stat_scrap, only for current season. :param engine: :param gender: :return: """ session = settings.create_session(engine) this_season_id = session.query(func.max(Season.id)).first()[0] squads = session.query(Squad).filter_by(season_id=this_season_id).all() session.close() try: for squad_record in squads: # If the squad_id != fake_id (non-ncaa squad id) if squad_record.id != 1: print "%%%% squad_id is " + str(squad_record.id), str( datetime.datetime.now()) session = settings.create_session(engine) season_stat_parser(session, squad_record, gender) session.close() except Exception, e: message = """ squad_id: %s team_id: %s season_id: %s This combination may not exists. %s """ % (squad_record.id, squad_record.season_id, squad_record.team_id, e) write_error_to_file(str(message)) raise
def new_season_stat_scrap(engine, gender): """ Same as new_season_stat_scrap, only for current season. :param engine: :param gender: :return: """ session = settings.create_session(engine) this_season_id = session.query(func.max(Season.id)).first()[0] squads = session.query(Squad).filter_by(season_id=this_season_id).all() session.close() try: for squad_record in squads: # If the squad_id != fake_id (non-ncaa squad id) if squad_record.id != 1: print "%%%% squad_id is "+str(squad_record.id), str(datetime.datetime.now()) session = settings.create_session(engine) season_stat_parser(session, squad_record, gender) session.close() except Exception, e: message = """ squad_id: %s team_id: %s season_id: %s This combination may not exists. %s """ % (squad_record.id, squad_record.season_id, squad_record.team_id, e) write_error_to_file(str(message)) raise
def new_game_detail_scrap(engine): """ Same as new_game_detail_scrap, only for current season. :param engine: :return: """ session = settings.create_session(engine) games = session.query(Game).filter_by(has_detail=0).order_by(desc( Game.id)).all() session.close() try: for game_record in games: print "%%%% game_id is " + str(game_record.id), str( datetime.datetime.now()) session = settings.create_session(engine) gamedetail_parser(session, game_record) session.close() except Exception, e: message = """ game_id: %s Something wrong in game_detail_parser %s """ % (game_record.id, e) write_error_to_file(message) raise
def initial_game_stat_scrap(engine): """ Scrape game_stat for SquadMember and Squad. :param engine: :return: """ session = settings.create_session(engine) games = session.query(Game).filter_by(has_stat=0).order_by(asc(Game.id)).all() session.close() try: for game_record in games: print "%%%% game_id is "+str(game_record.id), str(datetime.datetime.now()) session = settings.create_session(engine) game_stat_parser(session, game_record) session.close() except Exception, e: message = """ game_id: %s Something wrong in game_stat_parser %s """ % (game_record.id, e) write_error_to_file(message) raise
def initial_game_stat_scrap(engine): """ Scrape game_stat for SquadMember and Squad. :param engine: :return: """ session = settings.create_session(engine) games = session.query(Game).filter_by(has_stat=0).order_by(asc( Game.id)).all() session.close() try: for game_record in games: print "%%%% game_id is " + str(game_record.id), str( datetime.datetime.now()) session = settings.create_session(engine) game_stat_parser(session, game_record) session.close() except Exception, e: message = """ game_id: %s Something wrong in game_stat_parser %s """ % (game_record.id, e) write_error_to_file(message) raise
def initial_season_stat_scrap(engine, gender, season): """ Scrape season_stat for SquadMember and Squad. :param engine: :param gender: :param season: :return: """ session = settings.create_session(engine) if season: season_year = int(season) squads = session.query(Squad).filter_by(year=season_year).all() else: squads = session.query(Squad).all() session.close() try: for squad_record in squads: if squad_record.id != 1: print "%%%% squad_id is "+str(squad_record.id), str(datetime.datetime.now()) session = settings.create_session(engine) season_stat_parser(session, squad_record, gender) except Exception, e: message = """ squad_id: %s team_id: %s season_id: %s This combination may not exists. %s """ % (squad_record.id, squad_record.season_id, squad_record.team_id, e) write_error_to_file(str(message)) raise
def get_menu(): MenuIdJson = request.get_json() create_session(key="menuId", value=MenuIdJson["typeId"]) response = json.dumps({ "isFailure": True, "message": "Value Set" }, default=lambda o: o.__dict__) return response
def searched_menu(): SearchJson = request.get_json() create_session(key="checkin", value=SearchJson["checkin"]) create_session(key="checkout", value=SearchJson["checkout"]) create_session(key="rooms", value=SearchJson["rooms"]) create_session(key="input", value=SearchJson["input"]) return "/SearchMark"
def initial_schedule_game_player_scrap(engine, season): """ Scrape Schedule, Player, SquadMember based on Squad. Game_parser is inside schedule_parser. :param engine: :param season: :return: """ squad_id_finish_list = [] session = settings.create_session(engine) if season: season_year = int(season) squads = session.query(Squad).filter_by(year=season_year).all() else: squads = session.query(Squad).all() squadmembers_per_squad = session.query(SquadMember).group_by( SquadMember.squad_id).all() for squadmember_record in squadmembers_per_squad: squad_id_finish_list.append(squadmember_record.squad_id) session.close() try: for squad_record in squads: # If the squad_id != fake_id (non-ncaa squad id) if squad_record.id != 1: session = settings.create_session(engine) schedule_parser(session, squad_record) # game_parser() is inside schedule_parser() if squad_record.id not in squad_id_finish_list: player_parser(session, squad_record) # squadmember_parser() is inside player_parser() else: print 'Already scrapped, skip squad_id: ', squad_record.id session.close() except Exception, e: error_message = """ squad_id: %s This combination may not exists. %s """ % (squad_record.id, e) write_error_to_file(str(error_message)) raise
def initial_schedule_game_player_scrap(engine, season): """ Scrape Schedule, Player, SquadMember based on Squad. Game_parser is inside schedule_parser. :param engine: :param season: :return: """ squad_id_finish_list = [] session = settings.create_session(engine) if season: season_year = int(season) squads = session.query(Squad).filter_by(year=season_year).all() else: squads = session.query(Squad).all() squadmembers_per_squad = session.query(SquadMember).group_by(SquadMember.squad_id).all() for squadmember_record in squadmembers_per_squad: squad_id_finish_list.append(squadmember_record.squad_id) session.close() try: for squad_record in squads: # If the squad_id != fake_id (non-ncaa squad id) if squad_record.id != 1: session = settings.create_session(engine) schedule_parser(session, squad_record) # game_parser() is inside schedule_parser() if squad_record.id not in squad_id_finish_list: player_parser(session, squad_record) # squadmember_parser() is inside player_parser() else: print 'Already scrapped, skip squad_id: ', squad_record.id session.close() except Exception, e: error_message = """ squad_id: %s This combination may not exists. %s """ % (squad_record.id, e) write_error_to_file(str(error_message)) raise
def initial_team_squad_scrap(engine): """ Scrape Conference, Team and Squad based on season_id. :param engine: :return: """ session = settings.create_session(engine) seasons = session.query(Season).all() session.close() try: for season_record in seasons: season_id = season_record.id print "#####"+str(season_id), str(datetime.datetime.now()) for division in [1, 2, 3]: session = settings.create_session(engine) # Men&Women year 2010 only has division I if season_id == 10260 or season_id == 10261: if division != 1: print "$$$$" print division, season_id continue # Men&Women year 2011 only has division I and III if season_id == 10440 or season_id == 10420: if division == 2: print "$$$$" print division, season_id continue print "*****", str(datetime.datetime.now()) team_parser(session, season_id, division) squad_parser(session, season_id, division) conference_parser(session, season_id, division) session.close() except Exception, e: error_message = """ season_id: %s division: %s This combination may not exists. %s """ % (season_id, division, e) write_error_to_file(error_message) raise
def initial_team_squad_scrap(engine): """ Scrape Conference, Team and Squad based on season_id. :param engine: :return: """ session = settings.create_session(engine) seasons = session.query(Season).all() session.close() try: for season_record in seasons: season_id = season_record.id print "#####" + str(season_id), str(datetime.datetime.now()) for division in [1, 2, 3]: session = settings.create_session(engine) # Men&Women year 2010 only has division I if season_id == 10260 or season_id == 10261: if division != 1: print "$$$$" print division, season_id continue # Men&Women year 2011 only has division I and III if season_id == 10440 or season_id == 10420: if division == 2: print "$$$$" print division, season_id continue print "*****", str(datetime.datetime.now()) team_parser(session, season_id, division) squad_parser(session, season_id, division) conference_parser(session, season_id, division) session.close() except Exception, e: error_message = """ season_id: %s division: %s This combination may not exists. %s """ % (season_id, division, e) write_error_to_file(error_message) raise
def initial_season_stat_scrap(engine, gender, season): """ Scrape season_stat for SquadMember and Squad. :param engine: :param gender: :param season: :return: """ session = settings.create_session(engine) if season: season_year = int(season) squads = session.query(Squad).filter_by(year=season_year).all() else: squads = session.query(Squad).all() session.close() try: for squad_record in squads: if squad_record.id != 1: print "%%%% squad_id is " + str(squad_record.id), str( datetime.datetime.now()) session = settings.create_session(engine) season_stat_parser(session, squad_record, gender) except Exception, e: message = """ squad_id: %s team_id: %s season_id: %s This combination may not exists. %s """ % (squad_record.id, squad_record.season_id, squad_record.team_id, e) write_error_to_file(str(message)) raise
def initial_game_detail_scrap(engine): """ Scrape detail message for each game :param engine: :return: """ session = settings.create_session(engine) games = session.query(Game).filter_by(has_detail=0).order_by(asc(Game.id)).all() session.close() try: for game_record in games: session = settings.create_session(engine) gamedetail_parser(session, game_record) session.close() except Exception, e: message = """ game_id: %s Something wrong in game_detail_parser %s """ % (game_record.id, e) write_error_to_file(message) raise
def main(argv): gender = '' ptype = '' process = '' season = '' try: opts, args = getopt.getopt(argv, "g:t:p:s:", ["gender", "type=", "process=", "season="]) except getopt.GetoptError: print 'scheduler.py -g <gender> -t <type> -p <process> -s <season>' sys.exit(2) for opt, arg in opts: if opt in ("-g", "--gender"): gender = arg if gender != "men" and gender != "women": print "Gender wrong, please only use 'Men' or 'Women'." sys.exit() elif opt in ("-t", "--type"): ptype = arg elif opt in ("-p", "--process"): process = arg elif opt in ("-s", "--season"): season = arg engine = settings.create_engine(gender) # Create fake (non-ncaa) team and squad # fake team: id=1, name=non-ncaa team # fake squad: id=1, team_id=1, season_id=None, division=0, year=0 session = settings.create_session(engine) if session.query(Team).filter(Team.id == 1).first() is None: fake_team_record = Team("non-ncaa team", 1) session.add(fake_team_record) session.flush() if session.query(Squad).filter(Squad.team_id == 1).first() is None: fake_squad_record = Squad(0, None, 0, 1, None) session.add(fake_squad_record) session.flush() session.close() if ptype == "new": if process == "team_squad" or process == "1": new_team_squad_scrap(engine) elif process == "schedule_game_player" or process == "2": new_schedule_game_player_scrap(engine) elif process == "season_stat" or process == "3": new_season_stat_scrap(engine, gender) elif process == "game_stat" or process == "4": new_game_stat_scrap(engine) elif process == "game_detail" or process == "5": new_game_detail_scrap(engine) else: print "Process name not exists... Quit" sys.exit() elif ptype == "initial": if process == "team_squad" or process == "1": initial_team_squad_scrap(engine) elif process == "schedule_game_player" or process == "2": initial_schedule_game_player_scrap(engine, season) elif process == "season_stat" or process == "3": initial_season_stat_scrap(engine, gender, season) elif process == "game_stat" or process == "4": initial_game_stat_scrap(engine) elif process == "game_detail" or process == "5": initial_game_detail_scrap(engine) else: print "Process name not exists... Quit" sys.exit()
def confirmed_menu(): ConfirmJson = request.get_json() create_session(key="confirm", value=ConfirmJson["id"]) return "/confirmedDetails"