def processUser(db, filename, geek, url): import library f = file(filename) changes = 0 sql = "select count(*) from users where geek = %s" data = db.execute(sql, [geek]) if int(data[0][0]) == 0: sql = "insert into users (geek) values (%s)" db.execute(sql, [geek]) for line in f.readlines(): if line.find("/images/user/") > 0: n = int(library.between(line, "/images/user/", "/")) sql = "update users set bggid = %s where geek = %s" db.execute(sql, [n, geek]) changes += 1 if line.find("/users?country=") > 0: s = library.between(line, "country=", '"') sql = "update users set country = %s where geek = %s" if s == "Australia": filename = "market_%s.html" % geek recordFile(db, filename, MARKET_URL % geek, "processMarket", geek, "User marketplace data") db.execute(sql, [s, geek]) changes += 1 if changes == 2: break f.close() return changes
def processMarket(db, filename, geek, url): import library data = [] f = file(filename) for line in f.readlines(): if line.find("<div class='storeheader'>") >= 0: gameid = library.between(line, '/game/', '"') elif line.find("More Info...") >= 0: itemid = library.between(line, 'itemid=', '"') if itemid and gameid: data.append((geek, gameid, itemid)) sql = "delete from market where geek = %s" db.execute(sql, [geek]) for (geek, gameid, itemid) in data: sql = "insert into market (geek, gameid, itemid) values (%s, %s, %s)" db.execute(sql, [geek, gameid, itemid]) return 1
def getMostVoters(filename): import library mv = [] with open(filename) as f: start = False for line in f.readlines(): line = line.strip() if not start and line.find("<th class='collection_bggrating'>") >= 0: start = True elif start and line.find('href="/boardgame/') >= 0: id = library.between(line, 'href="/boardgame/', '/') mv.append(int(id)) return mv
def getMostVoters(filename): import library mv = [] with open(filename) as f: start = False for line in f.readlines(): line = line.strip() if not start and line.find( "<th class='collection_bggrating'>") >= 0: start = True elif start and line.find('href="/boardgame/') >= 0: id = library.between(line, 'href="/boardgame/', '/') mv.append(int(id)) return mv
def processGame(db, filename, geek, url): import library, sitedata, os game = library.Thing() game.subdomain = "boardgame" if "/" not in filename: filename = os.path.join(sitedata.dbdir, filename) id = int(library.between(filename[filename.rfind('/'):], "/", ".")) game.id = id if _readFile(game, id, filename) is None: return 0 if game.rank > 0: sql = "update files set tillNextUpdate = '360:00:00' where filename = %s" db.execute(sql, [filename]) _saveToDatabase(db, game) return 1
def getTop50(filename): import library t50 = [] with open(filename) as f: start = False for line in f.readlines(): line = line.strip() if not start and line.find("<th class='collection_bggrating'>") >= 0: start = True elif start and line.find('href="/boardgame/') >= 0: id = library.between(line, 'href="/boardgame/', '/') id = int(id) if id not in t50: t50.append(id) if len(t50) == 50: break return t50
def getTop50(filename): import library t50 = [] with open(filename) as f: start = False for line in f.readlines(): line = line.strip() if not start and line.find( "<th class='collection_bggrating'>") >= 0: start = True elif start and line.find('href="/boardgame/') >= 0: id = library.between(line, 'href="/boardgame/', '/') id = int(id) if id not in t50: t50.append(id) if len(t50) == 50: break return t50
def processPlayed(db, filename, geek, url): import calendar, plays, datetime, library, logging existing = db.execute("select count(*) from monthsplayed where geek = %s", [geek]) noneBefore = existing[0][0] == 0l toAdd = [] for line in file(filename).readlines(): if line.find(">By date<") >= 0: s = library.between(line, '/end/', '"') fields = s.split("-") data = [geek, fields[1], fields[0]] toAdd.append(data) if len(toAdd) > 0: ensureGeek(db, geek) luData = {} lastUpdateTimes = db.execute( "select url, lastupdate from files where geek = %s and processMethod = 'processPlays'", [geek]) for (url, lu) in lastUpdateTimes: luData[url] = lu db.execute("delete from monthsplayed where geek = %s", [geek]) db.execute( "delete from files where geek = %s and processMethod = 'processPlays'", [geek]) for data in toAdd: m = int(data[1]) y = int(data[2]) db.execute( "insert into monthsplayed (geek, month, year) values ('%s', %s, %s)" % tuple(data)) playsFile = "played_%s_%02d_%d.xml" % (geek, m, y) url = plays.NEW_PLAYED_URL % (urllib.quote(geek), y, m, y, m) if m == 0 and y == 0: daysSince = 10000 url = "https://boardgamegeek.com/xmlapi2/plays?username=%s&mindate=0000-00-00&maxdate=0000-00-00&subtype=boardgame" % urllib.quote( geek) else: try: pd = datetime.date(y, m, calendar.monthrange(y, m)[1]) except calendar.IllegalMonthError: logging.error("IllegalMonthError %d %d %s" % (y, m, ` data `)) daysSince = (datetime.date.today() - pd).days if daysSince <= 3: tillNext = '24:00:00' elif daysSince <= 30: tillNext = '72:00:00' elif daysSince <= 60: tillNext = '168:00:00' else: tillNext = None description = "Plays for %d/%s" % (y, m) lu = luData.get(url) if tillNext is not None: if lu is not None: mtime = lu.strftime('%Y-%m-%d %H:%M:%S') sql2 = "insert into files (filename, url, processMethod, geek, lastupdate, tillNextUpdate, description) values ('%s', '%s', 'processPlays', '%s', '%s', '%s', '%s')" % ( playsFile, url, geek, mtime, tillNext, description) else: sql2 = "insert into files (filename, url, processMethod, geek, tillNextUpdate, description) values ('%s', '%s', 'processPlays', '%s', '%s', '%s')" % ( playsFile, url, geek, tillNext, description) else: # no automatic next update - manual only if lu is not None: mtime = lu.strftime('%Y-%m-%d %H:%M:%S') sql2 = "insert into files (filename, url, processMethod, geek, lastupdate, description) values ('% s', '%s', 'processPlays', '%s', '%s', '%s')" % ( playsFile, url, geek, mtime, description) else: sql2 = "insert into files (filename, url, processMethod, geek, description) values ('%s', '%s', 'processPlays', '%s', '%s')" % ( playsFile, url, geek, description) db.execute(sql2) sql3 = "update files set nextUpdate = addtime(lastUpdate, tillNextUpdate) where processMethod = 'processPlays' and geek = '%s'" % geek db.execute(sql3) sql4 = "delete from plays where geek = '%s' and date_format(playDate, '%%Y-%%m') not in (select distinct concat(right(concat('000',year), 4), '-', right(concat('0',month), 2)) from monthsplayed where geek = '%s')" % ( geek, geek) db.execute(sql4) return 1 elif noneBefore: return 1 else: print "nothing to add, check %s" % filename return 1
def processPlayed(db, filename, geek, url): import calendar, plays, datetime, library, logging existing = db.execute("select count(*) from monthsplayed where geek = %s", [geek]) noneBefore = existing[0][0] == 0l toAdd = [] for line in file(filename).readlines(): if line.find(">By date<") >= 0: s = library.between(line, '/end/', '"') fields = s.split("-") data = [geek, fields[1], fields[0]] toAdd.append(data) if len(toAdd) > 0: ensureGeek(db, geek) luData = {} lastUpdateTimes = db.execute("select url, lastupdate from files where geek = %s and processMethod = 'processPlays'", [geek]) for (url, lu) in lastUpdateTimes: luData[url] = lu db.execute("delete from monthsplayed where geek = %s", [geek]) db.execute("delete from files where geek = %s and processMethod = 'processPlays'", [geek]) for data in toAdd: m = int(data[1]) y = int(data[2]) db.execute("insert into monthsplayed (geek, month, year) values ('%s', %s, %s)" % tuple(data)) playsFile = "played_%s_%02d_%d.xml" % (geek, m, y) url = plays.NEW_PLAYED_URL % (urllib.quote(geek), y, m, y, m) if m == 0 and y == 0: daysSince = 10000 url = "https://boardgamegeek.com/xmlapi2/plays?username=%s&mindate=0000-00-00&maxdate=0000-00-00&subtype=boardgame" % urllib.quote(geek) else: try: pd = datetime.date(y, m, calendar.monthrange(y,m)[1]) except calendar.IllegalMonthError: logging.error("IllegalMonthError %d %d %s" % (y, m, `data`)) daysSince = (datetime.date.today() - pd).days if daysSince <= 3: tillNext = '24:00:00' elif daysSince <= 30: tillNext = '72:00:00' elif daysSince <= 60: tillNext = '168:00:00' else: tillNext = None description = "Plays for %d/%s" % (y,m) lu = luData.get(url) if tillNext is not None: if lu is not None: mtime = lu.strftime('%Y-%m-%d %H:%M:%S') sql2 = "insert into files (filename, url, processMethod, geek, lastupdate, tillNextUpdate, description) values ('%s', '%s', 'processPlays', '%s', '%s', '%s', '%s')" % (playsFile, url, geek, mtime, tillNext, description) else: sql2 = "insert into files (filename, url, processMethod, geek, tillNextUpdate, description) values ('%s', '%s', 'processPlays', '%s', '%s', '%s')" % (playsFile, url, geek, tillNext, description) else: # no automatic next update - manual only if lu is not None: mtime = lu.strftime('%Y-%m-%d %H:%M:%S') sql2 = "insert into files (filename, url, processMethod, geek, lastupdate, description) values ('% s', '%s', 'processPlays', '%s', '%s', '%s')" % (playsFile, url, geek, mtime, description) else: sql2 = "insert into files (filename, url, processMethod, geek, description) values ('%s', '%s', 'processPlays', '%s', '%s')" % (playsFile, url, geek, description) db.execute(sql2) sql3 = "update files set nextUpdate = addtime(lastUpdate, tillNextUpdate) where processMethod = 'processPlays' and geek = '%s'" % geek db.execute(sql3) sql4 = "delete from plays where geek = '%s' and date_format(playDate, '%%Y-%%m') not in (select distinct concat(right(concat('000',year), 4), '-', right(concat('0',month), 2)) from monthsplayed where geek = '%s')" % (geek, geek) db.execute(sql4) return 1 elif noneBefore: return 1 else: print "nothing to add, check %s" % filename return 1