def processUser(db, filename, geek, url):
import library
f = file(filename)
changes = 0
sql = "select count(*) from users where geek = %s"
data = db.execute(sql, [geek])
if int(data[0][0]) == 0:
sql = "insert into users (geek) values (%s)"
db.execute(sql, [geek])
for line in f.readlines():
if line.find("/images/user/") > 0:
n = int(library.between(line, "/images/user/", "/"))
sql = "update users set bggid = %s where geek = %s"
db.execute(sql, [n, geek])
changes += 1
if line.find("/users?country=") > 0:
s = library.between(line, "country=", '"')
sql = "update users set country = %s where geek = %s"
if s == "Australia":
filename = "market_%s.html" % geek
recordFile(db, filename, MARKET_URL % geek, "processMarket", geek, "User marketplace data")
db.execute(sql, [s, geek])
changes += 1
if changes == 2:
break
f.close()
return changes
def processUser(db, filename, geek, url):
import library
f = file(filename)
changes = 0
sql = "select count(*) from users where geek = %s"
data = db.execute(sql, [geek])
if int(data[0][0]) == 0:
sql = "insert into users (geek) values (%s)"
db.execute(sql, [geek])
for line in f.readlines():
if line.find("/images/user/") > 0:
n = int(library.between(line, "/images/user/", "/"))
sql = "update users set bggid = %s where geek = %s"
db.execute(sql, [n, geek])
changes += 1
if line.find("/users?country=") > 0:
s = library.between(line, "country=", '"')
sql = "update users set country = %s where geek = %s"
if s == "Australia":
filename = "market_%s.html" % geek
recordFile(db, filename, MARKET_URL % geek, "processMarket",
geek, "User marketplace data")
db.execute(sql, [s, geek])
changes += 1
if changes == 2:
break
f.close()
return changes
def processMarket(db, filename, geek, url):
import library
data = []
f = file(filename)
for line in f.readlines():
if line.find("<div class='storeheader'>") >= 0:
gameid = library.between(line, '/game/', '"')
elif line.find("More Info...") >= 0:
itemid = library.between(line, 'itemid=', '"')
if itemid and gameid:
data.append((geek, gameid, itemid))
sql = "delete from market where geek = %s"
db.execute(sql, [geek])
for (geek, gameid, itemid) in data:
sql = "insert into market (geek, gameid, itemid) values (%s, %s, %s)"
db.execute(sql, [geek, gameid, itemid])
return 1
def processMarket(db, filename, geek, url):
import library
data = []
f = file(filename)
for line in f.readlines():
if line.find("<div class='storeheader'>") >= 0:
gameid = library.between(line, '/game/', '"')
elif line.find("More Info...") >= 0:
itemid = library.between(line, 'itemid=', '"')
if itemid and gameid:
data.append((geek, gameid, itemid))
sql = "delete from market where geek = %s"
db.execute(sql, [geek])
for (geek, gameid, itemid) in data:
sql = "insert into market (geek, gameid, itemid) values (%s, %s, %s)"
db.execute(sql, [geek, gameid, itemid])
return 1
def getMostVoters(filename):
import library
mv = []
with open(filename) as f:
start = False
for line in f.readlines():
line = line.strip()
if not start and line.find("<th class='collection_bggrating'>") >= 0:
start = True
elif start and line.find('href="/boardgame/') >= 0:
id = library.between(line, 'href="/boardgame/', '/')
mv.append(int(id))
return mv
def getMostVoters(filename):
import library
mv = []
with open(filename) as f:
start = False
for line in f.readlines():
line = line.strip()
if not start and line.find(
"<th class='collection_bggrating'>") >= 0:
start = True
elif start and line.find('href="/boardgame/') >= 0:
id = library.between(line, 'href="/boardgame/', '/')
mv.append(int(id))
return mv
def processGame(db, filename, geek, url):
import library, sitedata, os
game = library.Thing()
game.subdomain = "boardgame"
if "/" not in filename:
filename = os.path.join(sitedata.dbdir, filename)
id = int(library.between(filename[filename.rfind('/'):], "/", "."))
game.id = id
if _readFile(game, id, filename) is None:
return 0
if game.rank > 0:
sql = "update files set tillNextUpdate = '360:00:00' where filename = %s"
db.execute(sql, [filename])
_saveToDatabase(db, game)
return 1
def processGame(db, filename, geek, url):
import library, sitedata, os
game = library.Thing()
game.subdomain = "boardgame"
if "/" not in filename:
filename = os.path.join(sitedata.dbdir, filename)
id = int(library.between(filename[filename.rfind('/'):], "/", "."))
game.id = id
if _readFile(game, id, filename) is None:
return 0
if game.rank > 0:
sql = "update files set tillNextUpdate = '360:00:00' where filename = %s"
db.execute(sql, [filename])
_saveToDatabase(db, game)
return 1
def getTop50(filename):
import library
t50 = []
with open(filename) as f:
start = False
for line in f.readlines():
line = line.strip()
if not start and line.find("<th class='collection_bggrating'>") >= 0:
start = True
elif start and line.find('href="/boardgame/') >= 0:
id = library.between(line, 'href="/boardgame/', '/')
id = int(id)
if id not in t50:
t50.append(id)
if len(t50) == 50:
break
return t50
def getTop50(filename):
import library
t50 = []
with open(filename) as f:
start = False
for line in f.readlines():
line = line.strip()
if not start and line.find(
"<th class='collection_bggrating'>") >= 0:
start = True
elif start and line.find('href="/boardgame/') >= 0:
id = library.between(line, 'href="/boardgame/', '/')
id = int(id)
if id not in t50:
t50.append(id)
if len(t50) == 50:
break
return t50
def processPlayed(db, filename, geek, url):
import calendar, plays, datetime, library, logging
existing = db.execute("select count(*) from monthsplayed where geek = %s",
[geek])
noneBefore = existing[0][0] == 0l
toAdd = []
for line in file(filename).readlines():
if line.find(">By date<") >= 0:
s = library.between(line, '/end/', '"')
fields = s.split("-")
data = [geek, fields[1], fields[0]]
toAdd.append(data)
if len(toAdd) > 0:
ensureGeek(db, geek)
luData = {}
lastUpdateTimes = db.execute(
"select url, lastupdate from files where geek = %s and processMethod = 'processPlays'",
[geek])
for (url, lu) in lastUpdateTimes:
luData[url] = lu
db.execute("delete from monthsplayed where geek = %s", [geek])
db.execute(
"delete from files where geek = %s and processMethod = 'processPlays'",
[geek])
for data in toAdd:
m = int(data[1])
y = int(data[2])
db.execute(
"insert into monthsplayed (geek, month, year) values ('%s', %s, %s)"
% tuple(data))
playsFile = "played_%s_%02d_%d.xml" % (geek, m, y)
url = plays.NEW_PLAYED_URL % (urllib.quote(geek), y, m, y, m)
if m == 0 and y == 0:
daysSince = 10000
url = "https://boardgamegeek.com/xmlapi2/plays?username=%s&mindate=0000-00-00&maxdate=0000-00-00&subtype=boardgame" % urllib.quote(
geek)
else:
try:
pd = datetime.date(y, m, calendar.monthrange(y, m)[1])
except calendar.IllegalMonthError:
logging.error("IllegalMonthError %d %d %s" %
(y, m, ` data `))
daysSince = (datetime.date.today() - pd).days
if daysSince <= 3:
tillNext = '24:00:00'
elif daysSince <= 30:
tillNext = '72:00:00'
elif daysSince <= 60:
tillNext = '168:00:00'
else:
tillNext = None
description = "Plays for %d/%s" % (y, m)
lu = luData.get(url)
if tillNext is not None:
if lu is not None:
mtime = lu.strftime('%Y-%m-%d %H:%M:%S')
sql2 = "insert into files (filename, url, processMethod, geek, lastupdate, tillNextUpdate, description) values ('%s', '%s', 'processPlays', '%s', '%s', '%s', '%s')" % (
playsFile, url, geek, mtime, tillNext, description)
else:
sql2 = "insert into files (filename, url, processMethod, geek, tillNextUpdate, description) values ('%s', '%s', 'processPlays', '%s', '%s', '%s')" % (
playsFile, url, geek, tillNext, description)
else:
# no automatic next update - manual only
if lu is not None:
mtime = lu.strftime('%Y-%m-%d %H:%M:%S')
sql2 = "insert into files (filename, url, processMethod, geek, lastupdate, description) values ('% s', '%s', 'processPlays', '%s', '%s', '%s')" % (
playsFile, url, geek, mtime, description)
else:
sql2 = "insert into files (filename, url, processMethod, geek, description) values ('%s', '%s', 'processPlays', '%s', '%s')" % (
playsFile, url, geek, description)
db.execute(sql2)
sql3 = "update files set nextUpdate = addtime(lastUpdate, tillNextUpdate) where processMethod = 'processPlays' and geek = '%s'" % geek
db.execute(sql3)
sql4 = "delete from plays where geek = '%s' and date_format(playDate, '%%Y-%%m') not in (select distinct concat(right(concat('000',year), 4), '-', right(concat('0',month), 2)) from monthsplayed where geek = '%s')" % (
geek, geek)
db.execute(sql4)
return 1
elif noneBefore:
return 1
else:
print "nothing to add, check %s" % filename
return 1
def processPlayed(db, filename, geek, url):
import calendar, plays, datetime, library, logging
existing = db.execute("select count(*) from monthsplayed where geek = %s", [geek])
noneBefore = existing[0][0] == 0l
toAdd = []
for line in file(filename).readlines():
if line.find(">By date<") >= 0:
s = library.between(line, '/end/', '"')
fields = s.split("-")
data = [geek, fields[1], fields[0]]
toAdd.append(data)
if len(toAdd) > 0:
ensureGeek(db, geek)
luData = {}
lastUpdateTimes = db.execute("select url, lastupdate from files where geek = %s and processMethod = 'processPlays'", [geek])
for (url, lu) in lastUpdateTimes:
luData[url] = lu
db.execute("delete from monthsplayed where geek = %s", [geek])
db.execute("delete from files where geek = %s and processMethod = 'processPlays'", [geek])
for data in toAdd:
m = int(data[1])
y = int(data[2])
db.execute("insert into monthsplayed (geek, month, year) values ('%s', %s, %s)" % tuple(data))
playsFile = "played_%s_%02d_%d.xml" % (geek, m, y)
url = plays.NEW_PLAYED_URL % (urllib.quote(geek), y, m, y, m)
if m == 0 and y == 0:
daysSince = 10000
url = "https://boardgamegeek.com/xmlapi2/plays?username=%s&mindate=0000-00-00&maxdate=0000-00-00&subtype=boardgame" % urllib.quote(geek)
else:
try:
pd = datetime.date(y, m, calendar.monthrange(y,m)[1])
except calendar.IllegalMonthError:
logging.error("IllegalMonthError %d %d %s" % (y, m, `data`))
daysSince = (datetime.date.today() - pd).days
if daysSince <= 3:
tillNext = '24:00:00'
elif daysSince <= 30:
tillNext = '72:00:00'
elif daysSince <= 60:
tillNext = '168:00:00'
else:
tillNext = None
description = "Plays for %d/%s" % (y,m)
lu = luData.get(url)
if tillNext is not None:
if lu is not None:
mtime = lu.strftime('%Y-%m-%d %H:%M:%S')
sql2 = "insert into files (filename, url, processMethod, geek, lastupdate, tillNextUpdate, description) values ('%s', '%s', 'processPlays', '%s', '%s', '%s', '%s')" % (playsFile, url, geek, mtime, tillNext, description)
else:
sql2 = "insert into files (filename, url, processMethod, geek, tillNextUpdate, description) values ('%s', '%s', 'processPlays', '%s', '%s', '%s')" % (playsFile, url, geek, tillNext, description)
else:
# no automatic next update - manual only
if lu is not None:
mtime = lu.strftime('%Y-%m-%d %H:%M:%S')
sql2 = "insert into files (filename, url, processMethod, geek, lastupdate, description) values ('% s', '%s', 'processPlays', '%s', '%s', '%s')" % (playsFile, url, geek, mtime, description)
else:
sql2 = "insert into files (filename, url, processMethod, geek, description) values ('%s', '%s', 'processPlays', '%s', '%s')" % (playsFile, url, geek, description)
db.execute(sql2)
sql3 = "update files set nextUpdate = addtime(lastUpdate, tillNextUpdate) where processMethod = 'processPlays' and geek = '%s'" % geek
db.execute(sql3)
sql4 = "delete from plays where geek = '%s' and date_format(playDate, '%%Y-%%m') not in (select distinct concat(right(concat('000',year), 4), '-', right(concat('0',month), 2)) from monthsplayed where geek = '%s')" % (geek, geek)
db.execute(sql4)
return 1
elif noneBefore:
return 1
else:
print "nothing to add, check %s" % filename
return 1