def update_db(data_path):
"""
Create an engine and session, query for existing game ids.
Add new files in data dir to.
Parameters
----------
data_path : Path
path to the pro match directory. Engine is at data_path / pro.db
"""
engine = make_engine("sqlite:///" + str(data_path / "pro.db"))
Session = sessionmaker(bind=engine)
session = Session()
try:
sql_games = set(list(zip(*session.query(Game.match_id).all()))[0])
except IndexError: # new db
sql_games = set()
cached = cached_games(data_path.resolve()) # JSON files on disk
new_games = (x for x in cached if int(x.stem) not in sql_games)
session = add_to_db(engine, new_games)
return engine, session
def main():
args = parser.parse_args()
store = os.path.expanduser(args.hdf_store)
data_dir = Path(os.path.expanduser(args.data_dir))
cached = cached_games(data_dir)
# first time. Generate the store
if not os.path.isfile(store):
pd.HDFStore(store)
with pd.get_store(store) as s:
try:
stored = s.select("drs")["match_id"].unique()
except KeyError:
stored = []
new_games = filter(lambda x: int(x.stem) not in stored, cached)
dfs = []
i = 0 # if no new games
for i, game in enumerate(new_games, 1):
dr = api.DetailsResponse.from_json(str(game))
dfs.append(format_df(dr))
else:
append_to_store(store, dfs)
print("Added {} games.".format(i))
def main():
args = parser.parse_args()
store = os.path.expanduser(args.hdf_store)
data_dir = Path(os.path.expanduser(args.data_dir))
cached = cached_games(data_dir)
# first time. Generate the store
if not os.path.isfile(store):
pd.HDFStore(store)
with pd.get_store(store) as s:
try:
stored = s.select('drs')['match_id'].unique()
except KeyError:
stored = []
new_games = filter(lambda x: int(x.stem) not in stored, cached)
dfs = []
i = 0 # if no new games
for i, game in enumerate(new_games, 1):
dr = api.DetailsResponse.from_json(str(game))
dfs.append(format_df(dr))
else:
append_to_store(store, dfs)
print("Added {} games.".format(i))
def update_db(data_path):
"""
Create an engine and session, query for existing game ids.
Add new files in data dir to.
Parameters
----------
data_path : Path
path to the pro match directory. Engine is at data_path / pro.db
"""
engine = make_engine("sqlite:///" + str(data_path / "pro.db"))
Session = sessionmaker(bind=engine)
session = Session()
try:
sql_games = set(list(zip(*session.query(Game.match_id).all()))[0])
except IndexError: # new db
sql_games = set()
cached = cached_games(data_path.resolve()) # JSON files on disk
new_games = (x for x in cached if int(x.stem) not in sql_games)
session = add_to_db(engine, new_games)
return engine, session
def get_new_details(match_ids, data_path):
with open(os.path.expanduser('~/') + 'Dropbox/bin/api-keys.txt') as f:
key = json.load(f)['steam']
h = api.API(key=key)
cached = [int(x.stem) for x in cached_games(data_path)] # fragile...
new_matches = (x for x in match_ids if int(x) not in cached)
details = {mid: h.get_match_details(mid) for mid in new_matches}
return details
def get_details(steam_id, key, data_dir):
"""
Take a steam_id and check for new games. Download details of any
new games to data_dir. Primarily called from the command line.
"""
cached = cached_games(data_dir)
h = api.API(key)
hr = h.get_match_history(account_id=steam_id)
new_ids = set(hr.match_ids) - set((int(x.stem.strip('details')) for x in cached))
if len(new_ids) == 0:
print("No new matches for {}".format(steam_id))
print("Fetching details on {} games".format(len(new_ids)))
for id_ in new_ids:
dr = h.get_match_details(id_)
with (data_dir / (str(dr.match_id) + '.json')).open('w') as f:
json.dump(dr.resp, f)
print("Added {}.".format(id_))
def get_details(steam_id, key, data_dir):
"""
Take a steam_id and check for new games. Download details of any
new games to data_dir. Primarily called from the command line.
"""
cached = cached_games(data_dir)
h = api.API(key)
hr = h.get_match_history(account_id=steam_id)
new_ids = set(hr.match_ids) - set(
(int(x.stem.strip('details')) for x in cached))
if len(new_ids) == 0:
print("No new matches for {}".format(steam_id))
print("Fetching details on {} games".format(len(new_ids)))
for id_ in new_ids:
dr = h.get_match_details(id_)
with (data_dir / (str(dr.match_id) + '.json')).open('w') as f:
json.dump(dr.resp, f)
print("Added {}.".format(id_))
def test_cached_games(self):
p = Path('.')
result = sorted(cached_games(p))
expected = [Path('1234.json'), Path('details12345678.json')]
self.assertEqual(result, expected)
def test_cached_games(self):
p = Path('.')
result = sorted(cached_games(p))
expected = [Path('1234.json'), Path('details12345678.json')]
self.assertEqual(result, expected)