def month_iterator(start, end, unused_dt):
sy,sm,sd = SpinConfig.unix_to_cal(start) # starting year,month,day
ey,em,ed = SpinConfig.unix_to_cal(end) # ending year,month,day
while sy <= ey and ((sy < ey) or (sm <= em)):
lasty = sy
lastm = sm
# compute next month
sm += 1
if sm > 12:
sm = 1
sy += 1
yield (SpinConfig.cal_to_unix((lasty,lastm,1)), SpinConfig.cal_to_unix((sy,sm,1)))
def get_users_modified_since(mintime, maxtime):
sys.stderr.write(
'DEPRECATED use of SpinUserDB.get_users_modified_since(), this should be replaced with dbserver query\n'
)
import SpinLog, SpinJSON
assert mintime <= maxtime
modset = set([])
# quantize counter to days
day_begin = SpinConfig.cal_to_unix(SpinConfig.unix_to_cal(mintime))
while day_begin < maxtime:
log_file = SpinConfig.config[
'log_dir'] + '/' + SpinLog.time_to_date_string(
day_begin) + '-sessions.json'
day_begin += 24 * 60 * 60
#print 'CHECKING', log_file
try:
fd = open(log_file)
except:
# punt on any errors
return None
for line in fd.xreadlines():
# try to hold off JSON parsing as much as possible
if '0115_logged_in' not in line: continue
event = SpinJSON.loads(line)
etime = int(event['time'])
if etime >= mintime and etime < maxtime:
modset.add(int(event['user_id']))
if etime >= maxtime:
break
return modset
def is_satisfied(self, player, qdata):
if player.cooldown_active('birthday_' + self.tag):
return False
now_unix = player.get_absolute_time()
today = time.gmtime(now_unix)
if 'birthday_' + self.tag in player.history:
if player.history['birthday_' + self.tag] >= today.tm_year:
return False
if player.birthday:
birthday = time.gmtime(player.birthday)
birthday_unix = SpinConfig.cal_to_unix(
(today.tm_year, birthday.tm_mon, birthday.tm_mday))
start_unix = now_unix - 604800 # 7 days
return (start_unix <= birthday_unix) and (birthday_unix <=
now_unix)
return False
def birthday_to_birth_time(birthday):
m, d, y = map(int, birthday.split('/'))
return SpinConfig.cal_to_unix((y, m, d))
elif key == '--observation-day':
observation_day = int(val)
elif key == '--ltv-day':
ltv_day = int(val)
elif key == '--browser-bounce-rate':
browser_bounce_rate = float(val)
elif key == '--check-campaign':
check_campaign_name = val
elif key == '--check-demographic':
check_demographic = val
elif key == '--check-creation-range':
s1, s2 = val.split('-')
m1, d1, y1 = map(int, s1.split('/'))
m2, d2, y2 = map(int, s2.split('/'))
check_creation_range = [
SpinConfig.cal_to_unix((y1, m1, d1)),
SpinConfig.cal_to_unix((y2, m2, d2))
]
elif key == '--fudge-receipts':
fudge_receipts = float(val)
elif key == '--fudge-is-paying-user':
fudge_is_paying_user = float(val)
elif key == '--optimize-sku-price':
optimize_sku_price_param = val
elif key == '--min-spend':
min_spend = float(val)
elif key == '--verbose':
verbose = True
if build_model_name:
print >> sys.stderr, 'Building model "%s" from upcache "%s" using axes: %s' % (
def do_slave(task):
date = task['date']
game_id = task['game_id']
verbose = task['verbose']
dry_run = task['dry_run']
commit_interval = task['commit_interval']
start_time = SpinConfig.cal_to_unix((int(date[0:4]),int(date[4:6]),int(date[6:8])))
end_time = start_time + 86400
gamedata = SpinJSON.load(open(SpinConfig.gamedata_filename(override_game_id=game_id)))
STORE = {}
[get_store_items(STORE, sku) for sku in gamedata['store']['catalog']]
if verbose:
print >> sys.stderr, 'converting date', date, 'start_time', start_time, 'end_time', end_time, '...'
if not verbose: filterwarnings('ignore', category = MySQLdb.Warning)
cfg = SpinConfig.get_mysql_config(game_id+'_upcache')
con = MySQLdb.connect(*cfg['connect_args'], **cfg['connect_kwargs'])
store_table = cfg['table_prefix']+game_id+'_store'
s3 = SpinS3.S3(SpinConfig.aws_key_file())
bucket = 'spinpunch-logs'
batch = 0
total = 0
cur = con.cursor()
for entry in s3.list_bucket(bucket, prefix='%s/%s-%s-metrics.json' % (date[0:6], SpinConfig.game_id_long(override_game_id=game_id), date)):
filename = entry['name'].split('/')[-1]
if verbose: print >> sys.stderr, 'reading', filename
if entry['name'].endswith('.zip'):
tf = tempfile.NamedTemporaryFile(prefix='old_metrics_to_mysql-'+filename, suffix='.zip')
s3.get_file(bucket, entry['name'], tf.name)
unzipper = subprocess.Popen(['unzip', '-q', '-p', tf.name],
stdout = subprocess.PIPE)
elif entry['name'].endswith('.gz'):
fd = s3.get_open(bucket, entry['name'], allow_keepalive = False)
unzipper = subprocess.Popen(['gunzip', '-c', '-'],
stdin = fd.fileno(),
stdout = subprocess.PIPE)
for line in unzipper.stdout.xreadlines():
if '5120_buy_item' in line:
#and ('item:token' in line):
entry = SpinJSON.loads(line)
if entry['event_name'] != '5120_buy_item': continue
if 'price_currency' not in entry:
# old metric, need to fill in manually
if entry['items'][0]['spec'] in STORE:
entry['price_currency'] = 'item:token'
entry['price'] = STORE[entry['items'][0]['spec']]
if verbose: print >> sys.stderr, SpinJSON.dumps(entry)
if entry.get('price_currency','unknown') != 'item:token': continue
if '_id' in entry:
entry_id = entry['_id']
else:
id_generator.set_time(int(time.time()))
entry_id = id_generator.generate() # arbitrary
assert len(entry['items']) == 1
item = entry['items'][0]
keyvals = [('_id', entry_id),
('time', entry['time']),
('user_id', entry['user_id']),
('price', entry['price']),
('currency', entry['price_currency']),
('item', item['spec']),
('stack', item.get('stack',1))]
query = "INSERT INTO " + store_table + \
"("+', '.join(['`'+k+'`' for k,v in keyvals])+")"+ \
" VALUES ("+', '.join(['%s'] * len(keyvals)) +")"
if dry_run:
print >> sys.stderr, query, [v for k,v in keyvals]
else:
cur.execute(query, [v for k,v in keyvals])
batch += 1
total += 1
if commit_interval > 0 and batch >= commit_interval:
batch = 0
con.commit()
cur = con.cursor()
if verbose: print >> sys.stderr, total, 'inserted'
if not dry_run:
con.commit()
verbose = True
dry_run = False
parallel = 1
opts, args = getopt.gnu_getopt(sys.argv[1:], 'g:c:q', ['dry-run','parallel='])
for key, val in opts:
if key == '-g': game_id = val
elif key == '-c': commit_interval = int(val)
elif key == '-q': verbose = False
elif key == '--dry-run': dry_run = True
elif key == '--parallel': parallel = int(val)
if len(args) != 2:
print 'usage: %s 20130101 20130112' % sys.argv[0]
sys.exit(1)
start_time, end_time = map(lambda x: SpinConfig.cal_to_unix((int(x[0:4]),int(x[4:6]),int(x[6:8]))), args[0:2])
tasks = [{'date': '%04d%02d%02d' % SpinConfig.unix_to_cal(t),
'game_id': game_id,
'verbose': verbose, 'dry_run': dry_run, 'commit_interval':commit_interval,
'commit_interval': commit_interval} for t in range(start_time, end_time+86400, 86400)]
if parallel <= 1:
for task in tasks:
do_slave(task)
else:
SpinParallel.go(tasks, [sys.argv[0], '--slave'], on_error = 'break', nprocs=parallel, verbose = False)
def main(args):
global gamedata
global basedata
global hivedata
global lootdata
global time_now
opts, args = getopt.gnu_getopt(args, 'cg:', [])
game_id = 'ALL'
output = 'txt'
for key, val in opts:
if key == '-g':
game_id = val
if key == '-c':
output = 'csv'
time_now = int(time.time())
date_now = datetime.datetime.utcfromtimestamp(time_now)
gamedata = SpinJSON.load(open(SpinConfig.gamedata_filename()))
cur_week = SpinConfig.get_pvp_week(gamedata['matchmaking']['week_origin'], time_now)
today5pm = SpinConfig.cal_to_unix(SpinConfig.unix_to_cal(time_now)) + 17*60*60
if time_now < today5pm:
cur_day = ((date_now.weekday() + 3) % 7 + 1)
else:
cur_day = ((date_now.weekday() + 4) % 7 + 1)
if output == 'txt':
print 'Sauron v0.01.015 | %s, %s %s (Week %s, Day %s) %s GMT:<br>' % (date_now.strftime('%A')[:3].upper(), date_now.strftime("%B")[:3].upper(), date_now.strftime('%d, %Y'), cur_week, cur_day, date_now.strftime('%H:%M:%S'))
print '======================================================================='
print '\n TITLES :\n'
print ' TR - Thunder Run'
print ' BFM - Battlefront Mars'
print ''
print ' ALL TITLES :\n'
print ' TEST CALENDAR (Coming soon)'
print ' EVENT CALENDAR (Coming soon)'
print ''
print '======================================================================='
print ' LEGEND'
print '=======================================================================\n'
print ' EVENT DETAILS :\n'
print ' ONP: Ops Needs Points event'
print ' IMM: Immortal event'
print ' TUT: Tutorial event'
print ' ID: AI base/attack id'
print ' UNT: Number of AI units'
print ' DPS: Total harmful damage per second of AI units/turrets'
print ' HLT: Total max health of AI units/turrets'
print ' SPC: Total unit space taken up by AI units'
print ' TUR: Number of turrets'
print ' SPT: Total sprite count of AI buildings, units, and scenery for gauging frame rates.'
print ' CCL: Estimated difficulty in terms of CC level.'
print ''
if output == 'csv':
print 'Level, Type, Base ID, Estimate CCL, ONP, Unit Count, Unit Health, Unit DPS, Unit Space, Turret Count, Turret Health, Turret DPS, Sprite Count, Loot'
if game_id is 'ALL':
for id in GAME_IDS:
try:
gamedata = SpinJSON.load(open(SpinConfig.gamedata_filename(override_game_id = id)))
basedata = SpinJSON.load(open('./%s/built/%s_ai_bases_compiled.json' % (id, id)))
hivedata = SpinJSON.load(open('./%s/built/%s_hives_compiled.json' % (id, id)))
lootdata = SpinJSON.load(open('./%s/built/%s_loot_tables_compiled.json' % (id, id)))
except IOError:
print 'ERROR: Can\'t find compiled gamedata files for %s. Please run cd ../gameserver; ./make-gamedata.sh -u -g %s\n' % (id, id)
return
if output == 'txt':
print_event_details_txt(id)
elif output == 'csv':
print_event_details_csv(id)
else:
if game_id not in GAME_IDS:
print 'Invalid game id: %s' % game_id
return
try:
gamedata = SpinJSON.load(open(SpinConfig.gamedata_filename(override_game_id = game_id)))
basedata = SpinJSON.load(open('./%s/built/%s_ai_bases_compiled.json' % (game_id, game_id)))
hivedata = SpinJSON.load(open('./%s/built/%s_hives_compiled.json' % (game_id, game_id)))
lootdata = SpinJSON.load(open('./%s/built/%s_loot_tables_compiled.json' % (game_id, game_id)))
except IOError:
print 'ERROR: Can\'t find compiled gamedata files for %s. Please run cd ../gameserver; ./make-gamedata.sh -u -g %s\n' % (game_id, game_id)
return
if output == 'txt':
print_event_details_txt(game_id)
elif output == 'csv':
print_event_details_csv(game_id)
sql_util.do_insert(cur, sessions_table,
[('user_id',row['user_id']),
('start',row['in']),
('end',row['out'])] + \
sql_util.parse_brief_summary(row))
batch += 1
total += 1
for affected, dt in ((affected_days, 86400), (affected_hours, 3600)):
for t in xrange(dt*(row['in']//dt), dt*(row['out']//dt+1), dt):
affected.add(t)
# updated affected_months
# this assumes a session at most touches two months. Not likely you'd log in for more than a whole month!
for field in ('in','out'):
y,m,d = SpinConfig.unix_to_cal(row[field])
affected_months.add(SpinConfig.cal_to_unix((y,m,1))) # first of the month
if commit_interval > 0 and batch >= commit_interval:
batch = 0
con.commit()
if verbose: print total, 'inserted'
con.commit()
if verbose: print 'total', total, 'inserted', 'affecting', len(affected_months), 'month(s)', len(affected_days), 'day(s)', len(affected_hours), 'hour(s)'
# update summaries
# find range of sessions data available
cur.execute("SELECT MIN(start) AS min_time, MAX(start) AS max_time FROM "+sql_util.sym(sessions_table))
rows = cur.fetchall()
if rows and rows[0] and rows[0]['min_time'] and rows[0]['max_time']:
def do_slave(task):
date = task['date']
game_id = task['game_id']
verbose = task['verbose']
dry_run = task['dry_run']
start_time = SpinConfig.cal_to_unix(
(int(date[0:4]), int(date[4:6]), int(date[6:8])))
end_time = start_time + 86400
if verbose:
print >> sys.stderr, 'converting date', date, 'start_time', start_time, 'end_time', end_time, '...'
# gamedata = SpinJSON.load(open(SpinConfig.gamedata_filename(override_game_id = game_id)))
if not verbose: filterwarnings('ignore', category=MySQLdb.Warning)
quarries = SpinConfig.load(
SpinConfig.gamedata_component_filename('quarries_compiled.json'))
hives = SpinConfig.load(
SpinConfig.gamedata_component_filename('hives_compiled.json'))
# ensure that the spawn list is ordered by id_start - necessary for find_template() below
for spawn_list in quarries['spawn'], hives['spawn']:
spawn_list.sort(key=lambda x: x['id_start'])
cfg = SpinConfig.get_mysql_config(game_id + '_upcache')
con = MySQLdb.connect(*cfg['connect_args'], **cfg['connect_kwargs'])
battles_table = cfg['table_prefix'] + game_id + '_battles'
if 0:
# find any already-converted battles
cur = con.cursor()
cur.execute(
"SELECT COUNT(*) FROM %s WHERE time >= %%s and time < %%s" %
battles_table, (start_time, end_time))
row = cur.fetchone()
con.commit()
if row and row[0] > 0:
print >> sys.stderr, 'there are already', row[
0], 'entries in this time range, aborting!'
return
s3 = SpinS3.S3(SpinConfig.aws_key_file())
bucket = 'spinpunch-%sprod-battle-logs' % game_id
for entry in s3.list_bucket(bucket,
prefix='%s-battles-%s/%s' %
(game_id, date[0:6], date)):
filename = entry['name'].split('/')[-1]
event_time, attacker_id, defender_id, base_id = parse_battle_log_filename(
filename)
if (not base_id) or event_time < start_time or event_time >= end_time:
continue
if base_id[0] != 'v': continue # only look at hives
print >> sys.stderr, event_time, SpinLog.pretty_time(
time.gmtime(event_time)), filename
fd = s3.get_open(bucket, entry['name'], allow_keepalive=False)
unzipper = subprocess.Popen(['gunzip', '-c', '-'],
stdin=fd.fileno(),
stdout=subprocess.PIPE)
battle_start = None
battle_end = None
for line in unzipper.stdout.xreadlines():
if '3820_battle_start' in line:
battle_start = SpinJSON.loads(line)
elif '3830_battle_end' in line:
battle_end = SpinJSON.loads(line)
if (not battle_start) or (not battle_end): continue
base_template = find_template(hives['spawn'], int(base_id[1:]))
if not base_template:
sys.stderr.write('unknown hive %s\n' % base_id)
continue
# generate a fake summary
summary = {
'time':
event_time,
'attacker_id':
battle_start['attacker_user_id'],
'attacker_level':
battle_start['attacker_level'],
'attacker_outcome':
battle_end['battle_outcome'],
'defender_id':
battle_start['opponent_user_id'],
'defender_level':
battle_start['opponent_level'],
'defender_outcome':
'victory'
if battle_end['battle_outcome'] == 'defeat' else 'defeat',
'base_damage':
battle_end['base_damage'],
'base_id':
battle_start['base_id'],
'base_type':
'hive',
'base_template':
base_template,
'loot':
battle_end['loot']
}
cur = con.cursor()
cur.execute(
"SELECT battle_id FROM %s WHERE time = %%s and attacker_id = %%s and defender_id = %%s"
% battles_table, (event_time, battle_start['attacker_user_id'],
battle_start['opponent_user_id']))
row = cur.fetchone()
con.commit()
if row:
sys.stderr.write('appears to be a duplicate, skipping!\n')
continue
id_generator.set_time(int(time.time()))
battle_id = id_generator.generate() # arbitrary
keys = [
'battle_id',
]
values = [
battle_id,
]
for kname, ktype in battle_fields.iteritems():
path = kname.split(':')
probe = summary
val = None
for i in xrange(len(path)):
if path[i] not in probe:
break
elif i == len(path) - 1:
val = probe[path[i]]
break
else:
probe = probe[path[i]]
if val is not None:
keys.append(kname)
values.append(val)
query = "INSERT INTO " + battles_table + \
"("+', '.join(['`'+x+'`' for x in keys])+")"+ \
" VALUES ("+', '.join(['%s'] * len(values)) +")"
print >> sys.stderr, query
print >> sys.stderr, values
if not dry_run:
cur = con.cursor()
cur.execute(query, values)
con.commit()
opts, args = getopt.gnu_getopt(sys.argv[1:], 'g:c:q',
['dry-run', 'parallel='])
for key, val in opts:
if key == '-g': game_id = val
elif key == '-c': commit_interval = int(val)
elif key == '-q': verbose = False
elif key == '--dry-run': dry_run = True
elif key == '--parallel': parallel = int(val)
if len(args) != 2:
print 'usage: %s 20130101 20130112' % sys.argv[0]
sys.exit(1)
start_time, end_time = map(
lambda x: SpinConfig.cal_to_unix(
(int(x[0:4]), int(x[4:6]), int(x[6:8]))), args[0:2])
tasks = [{
'date': '%04d%02d%02d' % SpinConfig.unix_to_cal(t),
'game_id': game_id,
'verbose': verbose,
'dry_run': dry_run,
'commit_interval': commit_interval
} for t in range(start_time, end_time + 86400, 86400)]
if parallel <= 1:
for task in tasks:
do_slave(task)
else:
SpinParallel.go(tasks, [sys.argv[0], '--slave'],
on_error='break',
import SpinConfig
import SpinNoSQL
import sys, time, getopt
time_now = int(time.time())
if __name__ == '__main__':
opts, args = getopt.gnu_getopt(sys.argv[1:], 'g:yt',
['yesterday', 'date=', 'trailing'])
time_offset = 0
trailing = False
game_id = SpinConfig.config['game_id']
for key, val in opts:
if key == '--yesterday' or key == '-y': time_offset = -86400
elif key == '--date':
time_offset = SpinConfig.cal_to_unix(
(int(val[0:4]), int(val[4:6]), int(val[6:8]))) - time_now
elif key == '-g':
game_id = val
elif key == '--trailing' or key == '-t':
trailing = True
nosql_client = SpinNoSQL.NoSQLClient(
SpinConfig.get_mongodb_config(game_id))
day_start = 86400 * ((time_now + time_offset) // 86400)
if trailing:
dau = 0
time_range = [time_now - 86400, time_now]
else:
dau = nosql_client.dau_get(time_now + time_offset)
