def month_iterator(start, end, unused_dt): sy,sm,sd = SpinConfig.unix_to_cal(start) # starting year,month,day ey,em,ed = SpinConfig.unix_to_cal(end) # ending year,month,day while sy <= ey and ((sy < ey) or (sm <= em)): lasty = sy lastm = sm # compute next month sm += 1 if sm > 12: sm = 1 sy += 1 yield (SpinConfig.cal_to_unix((lasty,lastm,1)), SpinConfig.cal_to_unix((sy,sm,1)))
def get_users_modified_since(mintime, maxtime): sys.stderr.write( 'DEPRECATED use of SpinUserDB.get_users_modified_since(), this should be replaced with dbserver query\n' ) import SpinLog, SpinJSON assert mintime <= maxtime modset = set([]) # quantize counter to days day_begin = SpinConfig.cal_to_unix(SpinConfig.unix_to_cal(mintime)) while day_begin < maxtime: log_file = SpinConfig.config[ 'log_dir'] + '/' + SpinLog.time_to_date_string( day_begin) + '-sessions.json' day_begin += 24 * 60 * 60 #print 'CHECKING', log_file try: fd = open(log_file) except: # punt on any errors return None for line in fd.xreadlines(): # try to hold off JSON parsing as much as possible if '0115_logged_in' not in line: continue event = SpinJSON.loads(line) etime = int(event['time']) if etime >= mintime and etime < maxtime: modset.add(int(event['user_id'])) if etime >= maxtime: break return modset
def is_satisfied(self, player, qdata): if player.cooldown_active('birthday_' + self.tag): return False now_unix = player.get_absolute_time() today = time.gmtime(now_unix) if 'birthday_' + self.tag in player.history: if player.history['birthday_' + self.tag] >= today.tm_year: return False if player.birthday: birthday = time.gmtime(player.birthday) birthday_unix = SpinConfig.cal_to_unix( (today.tm_year, birthday.tm_mon, birthday.tm_mday)) start_unix = now_unix - 604800 # 7 days return (start_unix <= birthday_unix) and (birthday_unix <= now_unix) return False
def birthday_to_birth_time(birthday): m, d, y = map(int, birthday.split('/')) return SpinConfig.cal_to_unix((y, m, d))
elif key == '--observation-day': observation_day = int(val) elif key == '--ltv-day': ltv_day = int(val) elif key == '--browser-bounce-rate': browser_bounce_rate = float(val) elif key == '--check-campaign': check_campaign_name = val elif key == '--check-demographic': check_demographic = val elif key == '--check-creation-range': s1, s2 = val.split('-') m1, d1, y1 = map(int, s1.split('/')) m2, d2, y2 = map(int, s2.split('/')) check_creation_range = [ SpinConfig.cal_to_unix((y1, m1, d1)), SpinConfig.cal_to_unix((y2, m2, d2)) ] elif key == '--fudge-receipts': fudge_receipts = float(val) elif key == '--fudge-is-paying-user': fudge_is_paying_user = float(val) elif key == '--optimize-sku-price': optimize_sku_price_param = val elif key == '--min-spend': min_spend = float(val) elif key == '--verbose': verbose = True if build_model_name: print >> sys.stderr, 'Building model "%s" from upcache "%s" using axes: %s' % (
def do_slave(task): date = task['date'] game_id = task['game_id'] verbose = task['verbose'] dry_run = task['dry_run'] commit_interval = task['commit_interval'] start_time = SpinConfig.cal_to_unix((int(date[0:4]),int(date[4:6]),int(date[6:8]))) end_time = start_time + 86400 gamedata = SpinJSON.load(open(SpinConfig.gamedata_filename(override_game_id=game_id))) STORE = {} [get_store_items(STORE, sku) for sku in gamedata['store']['catalog']] if verbose: print >> sys.stderr, 'converting date', date, 'start_time', start_time, 'end_time', end_time, '...' if not verbose: filterwarnings('ignore', category = MySQLdb.Warning) cfg = SpinConfig.get_mysql_config(game_id+'_upcache') con = MySQLdb.connect(*cfg['connect_args'], **cfg['connect_kwargs']) store_table = cfg['table_prefix']+game_id+'_store' s3 = SpinS3.S3(SpinConfig.aws_key_file()) bucket = 'spinpunch-logs' batch = 0 total = 0 cur = con.cursor() for entry in s3.list_bucket(bucket, prefix='%s/%s-%s-metrics.json' % (date[0:6], SpinConfig.game_id_long(override_game_id=game_id), date)): filename = entry['name'].split('/')[-1] if verbose: print >> sys.stderr, 'reading', filename if entry['name'].endswith('.zip'): tf = tempfile.NamedTemporaryFile(prefix='old_metrics_to_mysql-'+filename, suffix='.zip') s3.get_file(bucket, entry['name'], tf.name) unzipper = subprocess.Popen(['unzip', '-q', '-p', tf.name], stdout = subprocess.PIPE) elif entry['name'].endswith('.gz'): fd = s3.get_open(bucket, entry['name'], allow_keepalive = False) unzipper = subprocess.Popen(['gunzip', '-c', '-'], stdin = fd.fileno(), stdout = subprocess.PIPE) for line in unzipper.stdout.xreadlines(): if '5120_buy_item' in line: #and ('item:token' in line): entry = SpinJSON.loads(line) if entry['event_name'] != '5120_buy_item': continue if 'price_currency' not in entry: # old metric, need to fill in manually if entry['items'][0]['spec'] in STORE: entry['price_currency'] = 'item:token' entry['price'] = STORE[entry['items'][0]['spec']] if verbose: print >> sys.stderr, SpinJSON.dumps(entry) if entry.get('price_currency','unknown') != 'item:token': continue if '_id' in entry: entry_id = entry['_id'] else: id_generator.set_time(int(time.time())) entry_id = id_generator.generate() # arbitrary assert len(entry['items']) == 1 item = entry['items'][0] keyvals = [('_id', entry_id), ('time', entry['time']), ('user_id', entry['user_id']), ('price', entry['price']), ('currency', entry['price_currency']), ('item', item['spec']), ('stack', item.get('stack',1))] query = "INSERT INTO " + store_table + \ "("+', '.join(['`'+k+'`' for k,v in keyvals])+")"+ \ " VALUES ("+', '.join(['%s'] * len(keyvals)) +")" if dry_run: print >> sys.stderr, query, [v for k,v in keyvals] else: cur.execute(query, [v for k,v in keyvals]) batch += 1 total += 1 if commit_interval > 0 and batch >= commit_interval: batch = 0 con.commit() cur = con.cursor() if verbose: print >> sys.stderr, total, 'inserted' if not dry_run: con.commit()
verbose = True dry_run = False parallel = 1 opts, args = getopt.gnu_getopt(sys.argv[1:], 'g:c:q', ['dry-run','parallel=']) for key, val in opts: if key == '-g': game_id = val elif key == '-c': commit_interval = int(val) elif key == '-q': verbose = False elif key == '--dry-run': dry_run = True elif key == '--parallel': parallel = int(val) if len(args) != 2: print 'usage: %s 20130101 20130112' % sys.argv[0] sys.exit(1) start_time, end_time = map(lambda x: SpinConfig.cal_to_unix((int(x[0:4]),int(x[4:6]),int(x[6:8]))), args[0:2]) tasks = [{'date': '%04d%02d%02d' % SpinConfig.unix_to_cal(t), 'game_id': game_id, 'verbose': verbose, 'dry_run': dry_run, 'commit_interval':commit_interval, 'commit_interval': commit_interval} for t in range(start_time, end_time+86400, 86400)] if parallel <= 1: for task in tasks: do_slave(task) else: SpinParallel.go(tasks, [sys.argv[0], '--slave'], on_error = 'break', nprocs=parallel, verbose = False)
def main(args): global gamedata global basedata global hivedata global lootdata global time_now opts, args = getopt.gnu_getopt(args, 'cg:', []) game_id = 'ALL' output = 'txt' for key, val in opts: if key == '-g': game_id = val if key == '-c': output = 'csv' time_now = int(time.time()) date_now = datetime.datetime.utcfromtimestamp(time_now) gamedata = SpinJSON.load(open(SpinConfig.gamedata_filename())) cur_week = SpinConfig.get_pvp_week(gamedata['matchmaking']['week_origin'], time_now) today5pm = SpinConfig.cal_to_unix(SpinConfig.unix_to_cal(time_now)) + 17*60*60 if time_now < today5pm: cur_day = ((date_now.weekday() + 3) % 7 + 1) else: cur_day = ((date_now.weekday() + 4) % 7 + 1) if output == 'txt': print 'Sauron v0.01.015 | %s, %s %s (Week %s, Day %s) %s GMT:<br>' % (date_now.strftime('%A')[:3].upper(), date_now.strftime("%B")[:3].upper(), date_now.strftime('%d, %Y'), cur_week, cur_day, date_now.strftime('%H:%M:%S')) print '=======================================================================' print '\n TITLES :\n' print ' TR - Thunder Run' print ' BFM - Battlefront Mars' print '' print ' ALL TITLES :\n' print ' TEST CALENDAR (Coming soon)' print ' EVENT CALENDAR (Coming soon)' print '' print '=======================================================================' print ' LEGEND' print '=======================================================================\n' print ' EVENT DETAILS :\n' print ' ONP: Ops Needs Points event' print ' IMM: Immortal event' print ' TUT: Tutorial event' print ' ID: AI base/attack id' print ' UNT: Number of AI units' print ' DPS: Total harmful damage per second of AI units/turrets' print ' HLT: Total max health of AI units/turrets' print ' SPC: Total unit space taken up by AI units' print ' TUR: Number of turrets' print ' SPT: Total sprite count of AI buildings, units, and scenery for gauging frame rates.' print ' CCL: Estimated difficulty in terms of CC level.' print '' if output == 'csv': print 'Level, Type, Base ID, Estimate CCL, ONP, Unit Count, Unit Health, Unit DPS, Unit Space, Turret Count, Turret Health, Turret DPS, Sprite Count, Loot' if game_id is 'ALL': for id in GAME_IDS: try: gamedata = SpinJSON.load(open(SpinConfig.gamedata_filename(override_game_id = id))) basedata = SpinJSON.load(open('./%s/built/%s_ai_bases_compiled.json' % (id, id))) hivedata = SpinJSON.load(open('./%s/built/%s_hives_compiled.json' % (id, id))) lootdata = SpinJSON.load(open('./%s/built/%s_loot_tables_compiled.json' % (id, id))) except IOError: print 'ERROR: Can\'t find compiled gamedata files for %s. Please run cd ../gameserver; ./make-gamedata.sh -u -g %s\n' % (id, id) return if output == 'txt': print_event_details_txt(id) elif output == 'csv': print_event_details_csv(id) else: if game_id not in GAME_IDS: print 'Invalid game id: %s' % game_id return try: gamedata = SpinJSON.load(open(SpinConfig.gamedata_filename(override_game_id = game_id))) basedata = SpinJSON.load(open('./%s/built/%s_ai_bases_compiled.json' % (game_id, game_id))) hivedata = SpinJSON.load(open('./%s/built/%s_hives_compiled.json' % (game_id, game_id))) lootdata = SpinJSON.load(open('./%s/built/%s_loot_tables_compiled.json' % (game_id, game_id))) except IOError: print 'ERROR: Can\'t find compiled gamedata files for %s. Please run cd ../gameserver; ./make-gamedata.sh -u -g %s\n' % (game_id, game_id) return if output == 'txt': print_event_details_txt(game_id) elif output == 'csv': print_event_details_csv(game_id)
sql_util.do_insert(cur, sessions_table, [('user_id',row['user_id']), ('start',row['in']), ('end',row['out'])] + \ sql_util.parse_brief_summary(row)) batch += 1 total += 1 for affected, dt in ((affected_days, 86400), (affected_hours, 3600)): for t in xrange(dt*(row['in']//dt), dt*(row['out']//dt+1), dt): affected.add(t) # updated affected_months # this assumes a session at most touches two months. Not likely you'd log in for more than a whole month! for field in ('in','out'): y,m,d = SpinConfig.unix_to_cal(row[field]) affected_months.add(SpinConfig.cal_to_unix((y,m,1))) # first of the month if commit_interval > 0 and batch >= commit_interval: batch = 0 con.commit() if verbose: print total, 'inserted' con.commit() if verbose: print 'total', total, 'inserted', 'affecting', len(affected_months), 'month(s)', len(affected_days), 'day(s)', len(affected_hours), 'hour(s)' # update summaries # find range of sessions data available cur.execute("SELECT MIN(start) AS min_time, MAX(start) AS max_time FROM "+sql_util.sym(sessions_table)) rows = cur.fetchall() if rows and rows[0] and rows[0]['min_time'] and rows[0]['max_time']:
def do_slave(task): date = task['date'] game_id = task['game_id'] verbose = task['verbose'] dry_run = task['dry_run'] start_time = SpinConfig.cal_to_unix( (int(date[0:4]), int(date[4:6]), int(date[6:8]))) end_time = start_time + 86400 if verbose: print >> sys.stderr, 'converting date', date, 'start_time', start_time, 'end_time', end_time, '...' # gamedata = SpinJSON.load(open(SpinConfig.gamedata_filename(override_game_id = game_id))) if not verbose: filterwarnings('ignore', category=MySQLdb.Warning) quarries = SpinConfig.load( SpinConfig.gamedata_component_filename('quarries_compiled.json')) hives = SpinConfig.load( SpinConfig.gamedata_component_filename('hives_compiled.json')) # ensure that the spawn list is ordered by id_start - necessary for find_template() below for spawn_list in quarries['spawn'], hives['spawn']: spawn_list.sort(key=lambda x: x['id_start']) cfg = SpinConfig.get_mysql_config(game_id + '_upcache') con = MySQLdb.connect(*cfg['connect_args'], **cfg['connect_kwargs']) battles_table = cfg['table_prefix'] + game_id + '_battles' if 0: # find any already-converted battles cur = con.cursor() cur.execute( "SELECT COUNT(*) FROM %s WHERE time >= %%s and time < %%s" % battles_table, (start_time, end_time)) row = cur.fetchone() con.commit() if row and row[0] > 0: print >> sys.stderr, 'there are already', row[ 0], 'entries in this time range, aborting!' return s3 = SpinS3.S3(SpinConfig.aws_key_file()) bucket = 'spinpunch-%sprod-battle-logs' % game_id for entry in s3.list_bucket(bucket, prefix='%s-battles-%s/%s' % (game_id, date[0:6], date)): filename = entry['name'].split('/')[-1] event_time, attacker_id, defender_id, base_id = parse_battle_log_filename( filename) if (not base_id) or event_time < start_time or event_time >= end_time: continue if base_id[0] != 'v': continue # only look at hives print >> sys.stderr, event_time, SpinLog.pretty_time( time.gmtime(event_time)), filename fd = s3.get_open(bucket, entry['name'], allow_keepalive=False) unzipper = subprocess.Popen(['gunzip', '-c', '-'], stdin=fd.fileno(), stdout=subprocess.PIPE) battle_start = None battle_end = None for line in unzipper.stdout.xreadlines(): if '3820_battle_start' in line: battle_start = SpinJSON.loads(line) elif '3830_battle_end' in line: battle_end = SpinJSON.loads(line) if (not battle_start) or (not battle_end): continue base_template = find_template(hives['spawn'], int(base_id[1:])) if not base_template: sys.stderr.write('unknown hive %s\n' % base_id) continue # generate a fake summary summary = { 'time': event_time, 'attacker_id': battle_start['attacker_user_id'], 'attacker_level': battle_start['attacker_level'], 'attacker_outcome': battle_end['battle_outcome'], 'defender_id': battle_start['opponent_user_id'], 'defender_level': battle_start['opponent_level'], 'defender_outcome': 'victory' if battle_end['battle_outcome'] == 'defeat' else 'defeat', 'base_damage': battle_end['base_damage'], 'base_id': battle_start['base_id'], 'base_type': 'hive', 'base_template': base_template, 'loot': battle_end['loot'] } cur = con.cursor() cur.execute( "SELECT battle_id FROM %s WHERE time = %%s and attacker_id = %%s and defender_id = %%s" % battles_table, (event_time, battle_start['attacker_user_id'], battle_start['opponent_user_id'])) row = cur.fetchone() con.commit() if row: sys.stderr.write('appears to be a duplicate, skipping!\n') continue id_generator.set_time(int(time.time())) battle_id = id_generator.generate() # arbitrary keys = [ 'battle_id', ] values = [ battle_id, ] for kname, ktype in battle_fields.iteritems(): path = kname.split(':') probe = summary val = None for i in xrange(len(path)): if path[i] not in probe: break elif i == len(path) - 1: val = probe[path[i]] break else: probe = probe[path[i]] if val is not None: keys.append(kname) values.append(val) query = "INSERT INTO " + battles_table + \ "("+', '.join(['`'+x+'`' for x in keys])+")"+ \ " VALUES ("+', '.join(['%s'] * len(values)) +")" print >> sys.stderr, query print >> sys.stderr, values if not dry_run: cur = con.cursor() cur.execute(query, values) con.commit()
opts, args = getopt.gnu_getopt(sys.argv[1:], 'g:c:q', ['dry-run', 'parallel=']) for key, val in opts: if key == '-g': game_id = val elif key == '-c': commit_interval = int(val) elif key == '-q': verbose = False elif key == '--dry-run': dry_run = True elif key == '--parallel': parallel = int(val) if len(args) != 2: print 'usage: %s 20130101 20130112' % sys.argv[0] sys.exit(1) start_time, end_time = map( lambda x: SpinConfig.cal_to_unix( (int(x[0:4]), int(x[4:6]), int(x[6:8]))), args[0:2]) tasks = [{ 'date': '%04d%02d%02d' % SpinConfig.unix_to_cal(t), 'game_id': game_id, 'verbose': verbose, 'dry_run': dry_run, 'commit_interval': commit_interval } for t in range(start_time, end_time + 86400, 86400)] if parallel <= 1: for task in tasks: do_slave(task) else: SpinParallel.go(tasks, [sys.argv[0], '--slave'], on_error='break',
import SpinConfig import SpinNoSQL import sys, time, getopt time_now = int(time.time()) if __name__ == '__main__': opts, args = getopt.gnu_getopt(sys.argv[1:], 'g:yt', ['yesterday', 'date=', 'trailing']) time_offset = 0 trailing = False game_id = SpinConfig.config['game_id'] for key, val in opts: if key == '--yesterday' or key == '-y': time_offset = -86400 elif key == '--date': time_offset = SpinConfig.cal_to_unix( (int(val[0:4]), int(val[4:6]), int(val[6:8]))) - time_now elif key == '-g': game_id = val elif key == '--trailing' or key == '-t': trailing = True nosql_client = SpinNoSQL.NoSQLClient( SpinConfig.get_mongodb_config(game_id)) day_start = 86400 * ((time_now + time_offset) // 86400) if trailing: dau = 0 time_range = [time_now - 86400, time_now] else: dau = nosql_client.dau_get(time_now + time_offset)