def stream_upcache(game_id, info=None):
bucket, name = SpinConfig.upcache_s3_location(game_id)
return SpinUpcacheIO.S3Reader(SpinS3.S3(SpinConfig.aws_key_file()),
bucket,
name,
verbose=False,
info=info)
def upcache_reader(upcache_path, info=None):
if upcache_path.startswith('s3:'):
reader = SpinUpcacheIO.S3Reader(SpinS3.S3(SpinConfig.aws_key_file()),
SpinConfig.upcache_s3_location(
SpinConfig.game())[0],
upcache_path.split(':')[1],
info=info)
return reader
else:
return SpinUpcacheIO.LocalReader(upcache_path, info=info)
def __init__(self,
game_id=None,
key_file=None,
userdb_bucket=None,
playerdb_bucket=None,
aistate_bucket=None,
basedb_bucket=None,
verbose=False):
self.nbuckets = 100
if game_id is None: game_id = SpinConfig.config['game_id']
self.game_id = game_id
self.userdb_bucket = userdb_bucket if userdb_bucket else SpinConfig.config.get(
'userdb_s3_bucket', None)
self.playerdb_bucket = playerdb_bucket if playerdb_bucket else SpinConfig.config.get(
'playerdb_s3_bucket', None)
self.aistate_bucket = aistate_bucket if aistate_bucket else SpinConfig.config.get(
'aistate_s3_bucket', None)
self.basedb_bucket = basedb_bucket if basedb_bucket else SpinConfig.config.get(
'basedb_s3_bucket', None)
self.s3con = SpinS3.S3(
key_file if key_file else SpinConfig.aws_key_file(),
verbose=verbose)
def iterate_from_s3(game_id,
bucket,
logname,
start_time,
end_time,
verbose=True):
assert start_time > 0
# to protect against same-time collisions, create a unique fake "PID" for MongoDB row _ids
sha = hashlib.sha1()
sha.update(game_id)
dig = sha.digest()
fake_pid = (ord(dig[1]) << 8) | ord(dig[0])
s3 = SpinS3.S3(SpinConfig.aws_key_file())
last_id_time = -1
id_serial = 0
for t in xrange(86400 * (start_time // 86400), 86400 * (end_time // 86400),
86400): # for each day
y, m, d = SpinConfig.unix_to_cal(t)
prefix = '%04d%02d/%s-%04d%02d%02d-%s' % (
y, m, SpinConfig.game_id_long(override_game_id=game_id), y, m, d,
logname)
for entry in s3.list_bucket(bucket, prefix=prefix):
filename = entry['name'].split('/')[-1]
if verbose: print 'reading', filename
if entry['name'].endswith('.zip'):
tf = tempfile.NamedTemporaryFile(prefix=logname + '-' +
filename,
suffix='.zip')
s3.get_file(bucket, entry['name'], tf.name)
unzipper = subprocess.Popen(['unzip', '-q', '-p', tf.name],
stdout=subprocess.PIPE)
elif entry['name'].endswith('.gz'):
tf = tempfile.NamedTemporaryFile(prefix=logname + '-' +
filename,
suffix='.gz')
s3.get_file(bucket, entry['name'], tf.name)
unzipper = subprocess.Popen(['gunzip', '-c', tf.name],
stdout=subprocess.PIPE)
else:
raise Exception('unhandled file extension: ' + entry['name'])
for line in unzipper.stdout.xreadlines():
row = SpinJSON.loads(line)
if row['time'] < start_time: continue # skip ahead
elif row['time'] >= end_time: break
if '_id' not in row:
# synthesize a fake MongoDB row ID
if row['time'] != last_id_time:
last_id_time = row['time']
id_serial = 0
row['_id'] = SpinNoSQLId.creation_time_id(row['time'],
pid=fake_pid,
serial=id_serial)
assert SpinNoSQLId.is_valid(row['_id'])
id_serial += 1
# note: there's a small chance this could end up duplicating an event at the boundary of an S3 import and MongoDB import
if verbose: print row
yield row
# Copyright (c) 2015 SpinPunch. All rights reserved.
# Use of this source code is governed by an MIT-style license that can be
# found in the LICENSE file.
# migrate on-disk aistate/ files to S3 bucket
import sys, os, hashlib, glob, getopt
import SpinS3
import SpinConfig
import multiprocessing
io_config = SpinConfig.config['gameservers'][
SpinConfig.config['gameservers'].keys()[0]].get('io', {})
assert io_config['aistate_use_s3']
con = SpinS3.S3(SpinConfig.aws_key_file())
bucket = SpinConfig.config['aistate_s3_bucket']
def make_s3_aistate_objname(filename):
name = os.path.basename(filename)
return hashlib.md5(name).hexdigest() + '-' + name
def migrate_to_s3(filename):
objname = make_s3_aistate_objname(filename)
print '%-35s -> %-60s' % (filename, objname),
# check if the file exists
mtime = con.exists(bucket, objname)
if mtime > 0:
def stream_userdb():
bucket, name = SpinConfig.upcache_s3_location(SpinConfig.game())
return SpinUpcacheIO.S3Reader(SpinS3.S3(SpinConfig.aws_key_file()), bucket,
name).iter_all()
def do_slave(task):
date = task['date']
game_id = task['game_id']
verbose = task['verbose']
dry_run = task['dry_run']
commit_interval = task['commit_interval']
start_time = SpinConfig.cal_to_unix((int(date[0:4]),int(date[4:6]),int(date[6:8])))
end_time = start_time + 86400
gamedata = SpinJSON.load(open(SpinConfig.gamedata_filename(override_game_id=game_id)))
STORE = {}
[get_store_items(STORE, sku) for sku in gamedata['store']['catalog']]
if verbose:
print >> sys.stderr, 'converting date', date, 'start_time', start_time, 'end_time', end_time, '...'
if not verbose: filterwarnings('ignore', category = MySQLdb.Warning)
cfg = SpinConfig.get_mysql_config(game_id+'_upcache')
con = MySQLdb.connect(*cfg['connect_args'], **cfg['connect_kwargs'])
store_table = cfg['table_prefix']+game_id+'_store'
s3 = SpinS3.S3(SpinConfig.aws_key_file())
bucket = 'spinpunch-logs'
batch = 0
total = 0
cur = con.cursor()
for entry in s3.list_bucket(bucket, prefix='%s/%s-%s-metrics.json' % (date[0:6], SpinConfig.game_id_long(override_game_id=game_id), date)):
filename = entry['name'].split('/')[-1]
if verbose: print >> sys.stderr, 'reading', filename
if entry['name'].endswith('.zip'):
tf = tempfile.NamedTemporaryFile(prefix='old_metrics_to_mysql-'+filename, suffix='.zip')
s3.get_file(bucket, entry['name'], tf.name)
unzipper = subprocess.Popen(['unzip', '-q', '-p', tf.name],
stdout = subprocess.PIPE)
elif entry['name'].endswith('.gz'):
fd = s3.get_open(bucket, entry['name'], allow_keepalive = False)
unzipper = subprocess.Popen(['gunzip', '-c', '-'],
stdin = fd.fileno(),
stdout = subprocess.PIPE)
for line in unzipper.stdout.xreadlines():
if '5120_buy_item' in line:
#and ('item:token' in line):
entry = SpinJSON.loads(line)
if entry['event_name'] != '5120_buy_item': continue
if 'price_currency' not in entry:
# old metric, need to fill in manually
if entry['items'][0]['spec'] in STORE:
entry['price_currency'] = 'item:token'
entry['price'] = STORE[entry['items'][0]['spec']]
if verbose: print >> sys.stderr, SpinJSON.dumps(entry)
if entry.get('price_currency','unknown') != 'item:token': continue
if '_id' in entry:
entry_id = entry['_id']
else:
id_generator.set_time(int(time.time()))
entry_id = id_generator.generate() # arbitrary
assert len(entry['items']) == 1
item = entry['items'][0]
keyvals = [('_id', entry_id),
('time', entry['time']),
('user_id', entry['user_id']),
('price', entry['price']),
('currency', entry['price_currency']),
('item', item['spec']),
('stack', item.get('stack',1))]
query = "INSERT INTO " + store_table + \
"("+', '.join(['`'+k+'`' for k,v in keyvals])+")"+ \
" VALUES ("+', '.join(['%s'] * len(keyvals)) +")"
if dry_run:
print >> sys.stderr, query, [v for k,v in keyvals]
else:
cur.execute(query, [v for k,v in keyvals])
batch += 1
total += 1
if commit_interval > 0 and batch >= commit_interval:
batch = 0
con.commit()
cur = con.cursor()
if verbose: print >> sys.stderr, total, 'inserted'
if not dry_run:
con.commit()
opts, args = getopt.gnu_getopt(sys.argv[1:], '', [
'cache-only', 's3-userdb', 'from-s3-keyfile=', 'to-s3-keyfile=',
'from-s3-bucket=', 'to-s3-bucket=', 'to-mongodb=', 'nosql-deltas-only',
'cache-read=', 'cache-write=', 'cache-segments=', 'parallel=',
'cache-update-time=', 'use-dbserver=', 'include-developers', 'progress'
])
include_developers = False
progress = False
from_s3_bucket = None
to_s3_bucket = None
to_mongodb_config = None
nosql_deltas_only = False
s3_userdb = False
to_s3_keyfile = from_s3_keyfile = SpinConfig.aws_key_file()
use_dbserver = True
cache_read = None
cache_write = None
cache_segments = 1
cache_update_time = -1
parallel = 1
time_now = int(time.time())
for key, val in opts:
if key == '--include-developers':
include_developers = True
elif key == '--progress':
progress = True
elif key == '--s3-userdb':
s3_userdb = True
gamedata = SpinJSON.load(open(SpinConfig.gamedata_filename()))
time_now = int(time.time())
def get_leveled_quantity(qty, level):
if type(qty) == list:
return qty[level-1]
return qty
# return an iterator that streams the entire userdb one entry at a time
def stream_userdb():
bucket, name = SpinConfig.upcache_s3_location(SpinConfig.game())
return SpinUpcacheIO.S3Reader(SpinS3.S3(SpinConfig.aws_key_file()),
bucket, name).iter_all()
driver = SpinUserDB.S3Driver(game_id = 'mf', key_file = SpinConfig.aws_key_file(),
userdb_bucket = 'spinpunch-prod-userdb',
playerdb_bucket = 'spinpunch-mfprod-playerdb')
# 0.55 -> deviation -1.13
# 0.50 -> deviation -0.77
# 0.40 -> deviation -0.10
# 0.37 -> deviation +0.37
# 0.25 -> deviation +1.51 sd 2.25
# 0.25 (NOpvp) -> losers 11.62%, mean deviation = +1.29, RMS deviation = 2.052963
# 0.25 (YESpvp) -> losers 9.91%, mean deviation = +1.51, RMS deviation = 2.252234
# 0.20 (NOpvp) -> losers 3.61%, mean deviation = +2.06, RMS deviation = 2.608003
# 0.20 (YESpvp) -> losers 2.89%, mean deviation = +2.28, RMS deviation = 2.827710
# 0.20 (YESpvp,YESharvest) -> losers 2.37%, mean deviation = +2.46, RMS deviation = 3.008991
#XP_SCALE=0.200000, N=21850, losers 0.86%, mean deviation = +2.64, RMS deviation = 2.984100
def do_slave(task):
date = task['date']
game_id = task['game_id']
verbose = task['verbose']
dry_run = task['dry_run']
start_time = SpinConfig.cal_to_unix(
(int(date[0:4]), int(date[4:6]), int(date[6:8])))
end_time = start_time + 86400
if verbose:
print >> sys.stderr, 'converting date', date, 'start_time', start_time, 'end_time', end_time, '...'
# gamedata = SpinJSON.load(open(SpinConfig.gamedata_filename(override_game_id = game_id)))
if not verbose: filterwarnings('ignore', category=MySQLdb.Warning)
quarries = SpinConfig.load(
SpinConfig.gamedata_component_filename('quarries_compiled.json'))
hives = SpinConfig.load(
SpinConfig.gamedata_component_filename('hives_compiled.json'))
# ensure that the spawn list is ordered by id_start - necessary for find_template() below
for spawn_list in quarries['spawn'], hives['spawn']:
spawn_list.sort(key=lambda x: x['id_start'])
cfg = SpinConfig.get_mysql_config(game_id + '_upcache')
con = MySQLdb.connect(*cfg['connect_args'], **cfg['connect_kwargs'])
battles_table = cfg['table_prefix'] + game_id + '_battles'
if 0:
# find any already-converted battles
cur = con.cursor()
cur.execute(
"SELECT COUNT(*) FROM %s WHERE time >= %%s and time < %%s" %
battles_table, (start_time, end_time))
row = cur.fetchone()
con.commit()
if row and row[0] > 0:
print >> sys.stderr, 'there are already', row[
0], 'entries in this time range, aborting!'
return
s3 = SpinS3.S3(SpinConfig.aws_key_file())
bucket = 'spinpunch-%sprod-battle-logs' % game_id
for entry in s3.list_bucket(bucket,
prefix='%s-battles-%s/%s' %
(game_id, date[0:6], date)):
filename = entry['name'].split('/')[-1]
event_time, attacker_id, defender_id, base_id = parse_battle_log_filename(
filename)
if (not base_id) or event_time < start_time or event_time >= end_time:
continue
if base_id[0] != 'v': continue # only look at hives
print >> sys.stderr, event_time, SpinLog.pretty_time(
time.gmtime(event_time)), filename
fd = s3.get_open(bucket, entry['name'], allow_keepalive=False)
unzipper = subprocess.Popen(['gunzip', '-c', '-'],
stdin=fd.fileno(),
stdout=subprocess.PIPE)
battle_start = None
battle_end = None
for line in unzipper.stdout.xreadlines():
if '3820_battle_start' in line:
battle_start = SpinJSON.loads(line)
elif '3830_battle_end' in line:
battle_end = SpinJSON.loads(line)
if (not battle_start) or (not battle_end): continue
base_template = find_template(hives['spawn'], int(base_id[1:]))
if not base_template:
sys.stderr.write('unknown hive %s\n' % base_id)
continue
# generate a fake summary
summary = {
'time':
event_time,
'attacker_id':
battle_start['attacker_user_id'],
'attacker_level':
battle_start['attacker_level'],
'attacker_outcome':
battle_end['battle_outcome'],
'defender_id':
battle_start['opponent_user_id'],
'defender_level':
battle_start['opponent_level'],
'defender_outcome':
'victory'
if battle_end['battle_outcome'] == 'defeat' else 'defeat',
'base_damage':
battle_end['base_damage'],
'base_id':
battle_start['base_id'],
'base_type':
'hive',
'base_template':
base_template,
'loot':
battle_end['loot']
}
cur = con.cursor()
cur.execute(
"SELECT battle_id FROM %s WHERE time = %%s and attacker_id = %%s and defender_id = %%s"
% battles_table, (event_time, battle_start['attacker_user_id'],
battle_start['opponent_user_id']))
row = cur.fetchone()
con.commit()
if row:
sys.stderr.write('appears to be a duplicate, skipping!\n')
continue
id_generator.set_time(int(time.time()))
battle_id = id_generator.generate() # arbitrary
keys = [
'battle_id',
]
values = [
battle_id,
]
for kname, ktype in battle_fields.iteritems():
path = kname.split(':')
probe = summary
val = None
for i in xrange(len(path)):
if path[i] not in probe:
break
elif i == len(path) - 1:
val = probe[path[i]]
break
else:
probe = probe[path[i]]
if val is not None:
keys.append(kname)
values.append(val)
query = "INSERT INTO " + battles_table + \
"("+', '.join(['`'+x+'`' for x in keys])+")"+ \
" VALUES ("+', '.join(['%s'] * len(values)) +")"
print >> sys.stderr, query
print >> sys.stderr, values
if not dry_run:
cur = con.cursor()
cur.execute(query, values)
con.commit()
import sys, os, getopt, time, tempfile
import SpinS3
import SpinConfig
import SpinNoSQL
import bson.objectid
import SpinLog
import SpinParallel
import subprocess
time_now = int(time.time())
# autoconfigure based on config.json
game_id = SpinConfig.config['game_id']
log_bucket = 'spinpunch-logs'
s3_key_file_for_logs = SpinConfig.aws_key_file()
TABLES = {
'chat': {
's3_name': 'chat',
'table_name': 'chat_buffer',
'compression': 'zip',
'retain_for': 30 * 86400
},
# credits log
'credits': {
's3_name': 'credits',
'table_name': 'log_credits',
'compression': 'zip',
'retain_for': -1
import sys, os, getopt, time, tempfile, shutil
import SpinS3
import SpinUserDB
import SpinConfig
import SpinParallel
import SpinSingletonProcess
import subprocess
date_str = time.strftime('%Y%m%d', time.gmtime())
# autoconfigure based on config.json
game_id = SpinConfig.config['game_id']
backup_bucket = 'spinpunch-backups'
backup_obj_prefix = '%s-player-data-%s/' % (SpinConfig.game_id_long(),
date_str)
s3_key_file_for_db = SpinConfig.aws_key_file()
s3_key_file_for_backups = SpinConfig.aws_key_file()
class NullFD(object):
def write(self, stuff):
pass
def backup_s3_dir(title,
bucket_name,
prefix='',
ignore_errors=False,
verbose=False):
# back up from S3 (either an entire bucket or one subdirectory) to one cpio.gz archive
msg_fd = sys.stderr if verbose else NullFD()
