def stream_upcache(game_id, info=None): bucket, name = SpinConfig.upcache_s3_location(game_id) return SpinUpcacheIO.S3Reader(SpinS3.S3(SpinConfig.aws_key_file()), bucket, name, verbose=False, info=info)
def upcache_reader(upcache_path, info=None): if upcache_path.startswith('s3:'): reader = SpinUpcacheIO.S3Reader(SpinS3.S3(SpinConfig.aws_key_file()), SpinConfig.upcache_s3_location( SpinConfig.game())[0], upcache_path.split(':')[1], info=info) return reader else: return SpinUpcacheIO.LocalReader(upcache_path, info=info)
def __init__(self, game_id=None, key_file=None, userdb_bucket=None, playerdb_bucket=None, aistate_bucket=None, basedb_bucket=None, verbose=False): self.nbuckets = 100 if game_id is None: game_id = SpinConfig.config['game_id'] self.game_id = game_id self.userdb_bucket = userdb_bucket if userdb_bucket else SpinConfig.config.get( 'userdb_s3_bucket', None) self.playerdb_bucket = playerdb_bucket if playerdb_bucket else SpinConfig.config.get( 'playerdb_s3_bucket', None) self.aistate_bucket = aistate_bucket if aistate_bucket else SpinConfig.config.get( 'aistate_s3_bucket', None) self.basedb_bucket = basedb_bucket if basedb_bucket else SpinConfig.config.get( 'basedb_s3_bucket', None) self.s3con = SpinS3.S3( key_file if key_file else SpinConfig.aws_key_file(), verbose=verbose)
def iterate_from_s3(game_id, bucket, logname, start_time, end_time, verbose=True): assert start_time > 0 # to protect against same-time collisions, create a unique fake "PID" for MongoDB row _ids sha = hashlib.sha1() sha.update(game_id) dig = sha.digest() fake_pid = (ord(dig[1]) << 8) | ord(dig[0]) s3 = SpinS3.S3(SpinConfig.aws_key_file()) last_id_time = -1 id_serial = 0 for t in xrange(86400 * (start_time // 86400), 86400 * (end_time // 86400), 86400): # for each day y, m, d = SpinConfig.unix_to_cal(t) prefix = '%04d%02d/%s-%04d%02d%02d-%s' % ( y, m, SpinConfig.game_id_long(override_game_id=game_id), y, m, d, logname) for entry in s3.list_bucket(bucket, prefix=prefix): filename = entry['name'].split('/')[-1] if verbose: print 'reading', filename if entry['name'].endswith('.zip'): tf = tempfile.NamedTemporaryFile(prefix=logname + '-' + filename, suffix='.zip') s3.get_file(bucket, entry['name'], tf.name) unzipper = subprocess.Popen(['unzip', '-q', '-p', tf.name], stdout=subprocess.PIPE) elif entry['name'].endswith('.gz'): tf = tempfile.NamedTemporaryFile(prefix=logname + '-' + filename, suffix='.gz') s3.get_file(bucket, entry['name'], tf.name) unzipper = subprocess.Popen(['gunzip', '-c', tf.name], stdout=subprocess.PIPE) else: raise Exception('unhandled file extension: ' + entry['name']) for line in unzipper.stdout.xreadlines(): row = SpinJSON.loads(line) if row['time'] < start_time: continue # skip ahead elif row['time'] >= end_time: break if '_id' not in row: # synthesize a fake MongoDB row ID if row['time'] != last_id_time: last_id_time = row['time'] id_serial = 0 row['_id'] = SpinNoSQLId.creation_time_id(row['time'], pid=fake_pid, serial=id_serial) assert SpinNoSQLId.is_valid(row['_id']) id_serial += 1 # note: there's a small chance this could end up duplicating an event at the boundary of an S3 import and MongoDB import if verbose: print row yield row
# Copyright (c) 2015 SpinPunch. All rights reserved. # Use of this source code is governed by an MIT-style license that can be # found in the LICENSE file. # migrate on-disk aistate/ files to S3 bucket import sys, os, hashlib, glob, getopt import SpinS3 import SpinConfig import multiprocessing io_config = SpinConfig.config['gameservers'][ SpinConfig.config['gameservers'].keys()[0]].get('io', {}) assert io_config['aistate_use_s3'] con = SpinS3.S3(SpinConfig.aws_key_file()) bucket = SpinConfig.config['aistate_s3_bucket'] def make_s3_aistate_objname(filename): name = os.path.basename(filename) return hashlib.md5(name).hexdigest() + '-' + name def migrate_to_s3(filename): objname = make_s3_aistate_objname(filename) print '%-35s -> %-60s' % (filename, objname), # check if the file exists mtime = con.exists(bucket, objname) if mtime > 0:
def stream_userdb(): bucket, name = SpinConfig.upcache_s3_location(SpinConfig.game()) return SpinUpcacheIO.S3Reader(SpinS3.S3(SpinConfig.aws_key_file()), bucket, name).iter_all()
def do_slave(task): date = task['date'] game_id = task['game_id'] verbose = task['verbose'] dry_run = task['dry_run'] commit_interval = task['commit_interval'] start_time = SpinConfig.cal_to_unix((int(date[0:4]),int(date[4:6]),int(date[6:8]))) end_time = start_time + 86400 gamedata = SpinJSON.load(open(SpinConfig.gamedata_filename(override_game_id=game_id))) STORE = {} [get_store_items(STORE, sku) for sku in gamedata['store']['catalog']] if verbose: print >> sys.stderr, 'converting date', date, 'start_time', start_time, 'end_time', end_time, '...' if not verbose: filterwarnings('ignore', category = MySQLdb.Warning) cfg = SpinConfig.get_mysql_config(game_id+'_upcache') con = MySQLdb.connect(*cfg['connect_args'], **cfg['connect_kwargs']) store_table = cfg['table_prefix']+game_id+'_store' s3 = SpinS3.S3(SpinConfig.aws_key_file()) bucket = 'spinpunch-logs' batch = 0 total = 0 cur = con.cursor() for entry in s3.list_bucket(bucket, prefix='%s/%s-%s-metrics.json' % (date[0:6], SpinConfig.game_id_long(override_game_id=game_id), date)): filename = entry['name'].split('/')[-1] if verbose: print >> sys.stderr, 'reading', filename if entry['name'].endswith('.zip'): tf = tempfile.NamedTemporaryFile(prefix='old_metrics_to_mysql-'+filename, suffix='.zip') s3.get_file(bucket, entry['name'], tf.name) unzipper = subprocess.Popen(['unzip', '-q', '-p', tf.name], stdout = subprocess.PIPE) elif entry['name'].endswith('.gz'): fd = s3.get_open(bucket, entry['name'], allow_keepalive = False) unzipper = subprocess.Popen(['gunzip', '-c', '-'], stdin = fd.fileno(), stdout = subprocess.PIPE) for line in unzipper.stdout.xreadlines(): if '5120_buy_item' in line: #and ('item:token' in line): entry = SpinJSON.loads(line) if entry['event_name'] != '5120_buy_item': continue if 'price_currency' not in entry: # old metric, need to fill in manually if entry['items'][0]['spec'] in STORE: entry['price_currency'] = 'item:token' entry['price'] = STORE[entry['items'][0]['spec']] if verbose: print >> sys.stderr, SpinJSON.dumps(entry) if entry.get('price_currency','unknown') != 'item:token': continue if '_id' in entry: entry_id = entry['_id'] else: id_generator.set_time(int(time.time())) entry_id = id_generator.generate() # arbitrary assert len(entry['items']) == 1 item = entry['items'][0] keyvals = [('_id', entry_id), ('time', entry['time']), ('user_id', entry['user_id']), ('price', entry['price']), ('currency', entry['price_currency']), ('item', item['spec']), ('stack', item.get('stack',1))] query = "INSERT INTO " + store_table + \ "("+', '.join(['`'+k+'`' for k,v in keyvals])+")"+ \ " VALUES ("+', '.join(['%s'] * len(keyvals)) +")" if dry_run: print >> sys.stderr, query, [v for k,v in keyvals] else: cur.execute(query, [v for k,v in keyvals]) batch += 1 total += 1 if commit_interval > 0 and batch >= commit_interval: batch = 0 con.commit() cur = con.cursor() if verbose: print >> sys.stderr, total, 'inserted' if not dry_run: con.commit()
opts, args = getopt.gnu_getopt(sys.argv[1:], '', [ 'cache-only', 's3-userdb', 'from-s3-keyfile=', 'to-s3-keyfile=', 'from-s3-bucket=', 'to-s3-bucket=', 'to-mongodb=', 'nosql-deltas-only', 'cache-read=', 'cache-write=', 'cache-segments=', 'parallel=', 'cache-update-time=', 'use-dbserver=', 'include-developers', 'progress' ]) include_developers = False progress = False from_s3_bucket = None to_s3_bucket = None to_mongodb_config = None nosql_deltas_only = False s3_userdb = False to_s3_keyfile = from_s3_keyfile = SpinConfig.aws_key_file() use_dbserver = True cache_read = None cache_write = None cache_segments = 1 cache_update_time = -1 parallel = 1 time_now = int(time.time()) for key, val in opts: if key == '--include-developers': include_developers = True elif key == '--progress': progress = True elif key == '--s3-userdb': s3_userdb = True
gamedata = SpinJSON.load(open(SpinConfig.gamedata_filename())) time_now = int(time.time()) def get_leveled_quantity(qty, level): if type(qty) == list: return qty[level-1] return qty # return an iterator that streams the entire userdb one entry at a time def stream_userdb(): bucket, name = SpinConfig.upcache_s3_location(SpinConfig.game()) return SpinUpcacheIO.S3Reader(SpinS3.S3(SpinConfig.aws_key_file()), bucket, name).iter_all() driver = SpinUserDB.S3Driver(game_id = 'mf', key_file = SpinConfig.aws_key_file(), userdb_bucket = 'spinpunch-prod-userdb', playerdb_bucket = 'spinpunch-mfprod-playerdb') # 0.55 -> deviation -1.13 # 0.50 -> deviation -0.77 # 0.40 -> deviation -0.10 # 0.37 -> deviation +0.37 # 0.25 -> deviation +1.51 sd 2.25 # 0.25 (NOpvp) -> losers 11.62%, mean deviation = +1.29, RMS deviation = 2.052963 # 0.25 (YESpvp) -> losers 9.91%, mean deviation = +1.51, RMS deviation = 2.252234 # 0.20 (NOpvp) -> losers 3.61%, mean deviation = +2.06, RMS deviation = 2.608003 # 0.20 (YESpvp) -> losers 2.89%, mean deviation = +2.28, RMS deviation = 2.827710 # 0.20 (YESpvp,YESharvest) -> losers 2.37%, mean deviation = +2.46, RMS deviation = 3.008991 #XP_SCALE=0.200000, N=21850, losers 0.86%, mean deviation = +2.64, RMS deviation = 2.984100
def do_slave(task): date = task['date'] game_id = task['game_id'] verbose = task['verbose'] dry_run = task['dry_run'] start_time = SpinConfig.cal_to_unix( (int(date[0:4]), int(date[4:6]), int(date[6:8]))) end_time = start_time + 86400 if verbose: print >> sys.stderr, 'converting date', date, 'start_time', start_time, 'end_time', end_time, '...' # gamedata = SpinJSON.load(open(SpinConfig.gamedata_filename(override_game_id = game_id))) if not verbose: filterwarnings('ignore', category=MySQLdb.Warning) quarries = SpinConfig.load( SpinConfig.gamedata_component_filename('quarries_compiled.json')) hives = SpinConfig.load( SpinConfig.gamedata_component_filename('hives_compiled.json')) # ensure that the spawn list is ordered by id_start - necessary for find_template() below for spawn_list in quarries['spawn'], hives['spawn']: spawn_list.sort(key=lambda x: x['id_start']) cfg = SpinConfig.get_mysql_config(game_id + '_upcache') con = MySQLdb.connect(*cfg['connect_args'], **cfg['connect_kwargs']) battles_table = cfg['table_prefix'] + game_id + '_battles' if 0: # find any already-converted battles cur = con.cursor() cur.execute( "SELECT COUNT(*) FROM %s WHERE time >= %%s and time < %%s" % battles_table, (start_time, end_time)) row = cur.fetchone() con.commit() if row and row[0] > 0: print >> sys.stderr, 'there are already', row[ 0], 'entries in this time range, aborting!' return s3 = SpinS3.S3(SpinConfig.aws_key_file()) bucket = 'spinpunch-%sprod-battle-logs' % game_id for entry in s3.list_bucket(bucket, prefix='%s-battles-%s/%s' % (game_id, date[0:6], date)): filename = entry['name'].split('/')[-1] event_time, attacker_id, defender_id, base_id = parse_battle_log_filename( filename) if (not base_id) or event_time < start_time or event_time >= end_time: continue if base_id[0] != 'v': continue # only look at hives print >> sys.stderr, event_time, SpinLog.pretty_time( time.gmtime(event_time)), filename fd = s3.get_open(bucket, entry['name'], allow_keepalive=False) unzipper = subprocess.Popen(['gunzip', '-c', '-'], stdin=fd.fileno(), stdout=subprocess.PIPE) battle_start = None battle_end = None for line in unzipper.stdout.xreadlines(): if '3820_battle_start' in line: battle_start = SpinJSON.loads(line) elif '3830_battle_end' in line: battle_end = SpinJSON.loads(line) if (not battle_start) or (not battle_end): continue base_template = find_template(hives['spawn'], int(base_id[1:])) if not base_template: sys.stderr.write('unknown hive %s\n' % base_id) continue # generate a fake summary summary = { 'time': event_time, 'attacker_id': battle_start['attacker_user_id'], 'attacker_level': battle_start['attacker_level'], 'attacker_outcome': battle_end['battle_outcome'], 'defender_id': battle_start['opponent_user_id'], 'defender_level': battle_start['opponent_level'], 'defender_outcome': 'victory' if battle_end['battle_outcome'] == 'defeat' else 'defeat', 'base_damage': battle_end['base_damage'], 'base_id': battle_start['base_id'], 'base_type': 'hive', 'base_template': base_template, 'loot': battle_end['loot'] } cur = con.cursor() cur.execute( "SELECT battle_id FROM %s WHERE time = %%s and attacker_id = %%s and defender_id = %%s" % battles_table, (event_time, battle_start['attacker_user_id'], battle_start['opponent_user_id'])) row = cur.fetchone() con.commit() if row: sys.stderr.write('appears to be a duplicate, skipping!\n') continue id_generator.set_time(int(time.time())) battle_id = id_generator.generate() # arbitrary keys = [ 'battle_id', ] values = [ battle_id, ] for kname, ktype in battle_fields.iteritems(): path = kname.split(':') probe = summary val = None for i in xrange(len(path)): if path[i] not in probe: break elif i == len(path) - 1: val = probe[path[i]] break else: probe = probe[path[i]] if val is not None: keys.append(kname) values.append(val) query = "INSERT INTO " + battles_table + \ "("+', '.join(['`'+x+'`' for x in keys])+")"+ \ " VALUES ("+', '.join(['%s'] * len(values)) +")" print >> sys.stderr, query print >> sys.stderr, values if not dry_run: cur = con.cursor() cur.execute(query, values) con.commit()
import sys, os, getopt, time, tempfile import SpinS3 import SpinConfig import SpinNoSQL import bson.objectid import SpinLog import SpinParallel import subprocess time_now = int(time.time()) # autoconfigure based on config.json game_id = SpinConfig.config['game_id'] log_bucket = 'spinpunch-logs' s3_key_file_for_logs = SpinConfig.aws_key_file() TABLES = { 'chat': { 's3_name': 'chat', 'table_name': 'chat_buffer', 'compression': 'zip', 'retain_for': 30 * 86400 }, # credits log 'credits': { 's3_name': 'credits', 'table_name': 'log_credits', 'compression': 'zip', 'retain_for': -1
import sys, os, getopt, time, tempfile, shutil import SpinS3 import SpinUserDB import SpinConfig import SpinParallel import SpinSingletonProcess import subprocess date_str = time.strftime('%Y%m%d', time.gmtime()) # autoconfigure based on config.json game_id = SpinConfig.config['game_id'] backup_bucket = 'spinpunch-backups' backup_obj_prefix = '%s-player-data-%s/' % (SpinConfig.game_id_long(), date_str) s3_key_file_for_db = SpinConfig.aws_key_file() s3_key_file_for_backups = SpinConfig.aws_key_file() class NullFD(object): def write(self, stuff): pass def backup_s3_dir(title, bucket_name, prefix='', ignore_errors=False, verbose=False): # back up from S3 (either an entire bucket or one subdirectory) to one cpio.gz archive msg_fd = sys.stderr if verbose else NullFD()