def stream_upcache(game_id, info=None):
bucket, name = SpinConfig.upcache_s3_location(game_id)
return SpinUpcacheIO.S3Reader(SpinS3.S3(SpinConfig.aws_key_file()),
bucket,
name,
verbose=False,
info=info)
def get_users_modified_since(mintime, maxtime):
sys.stderr.write(
'DEPRECATED use of SpinUserDB.get_users_modified_since(), this should be replaced with dbserver query\n'
)
import SpinLog, SpinJSON
assert mintime <= maxtime
modset = set([])
# quantize counter to days
day_begin = SpinConfig.cal_to_unix(SpinConfig.unix_to_cal(mintime))
while day_begin < maxtime:
log_file = SpinConfig.config[
'log_dir'] + '/' + SpinLog.time_to_date_string(
day_begin) + '-sessions.json'
day_begin += 24 * 60 * 60
#print 'CHECKING', log_file
try:
fd = open(log_file)
except:
# punt on any errors
return None
for line in fd.xreadlines():
# try to hold off JSON parsing as much as possible
if '0115_logged_in' not in line: continue
event = SpinJSON.loads(line)
etime = int(event['time'])
if etime >= mintime and etime < maxtime:
modset.add(int(event['user_id']))
if etime >= maxtime:
break
return modset
def upcache_reader(upcache_path, info=None):
if upcache_path.startswith('s3:'):
reader = SpinUpcacheIO.S3Reader(SpinS3.S3(SpinConfig.aws_key_file()),
SpinConfig.upcache_s3_location(
SpinConfig.game())[0],
upcache_path.split(':')[1],
info=info)
return reader
else:
return SpinUpcacheIO.LocalReader(upcache_path, info=info)
def month_iterator(start, end, unused_dt):
sy,sm,sd = SpinConfig.unix_to_cal(start) # starting year,month,day
ey,em,ed = SpinConfig.unix_to_cal(end) # ending year,month,day
while sy <= ey and ((sy < ey) or (sm <= em)):
lasty = sy
lastm = sm
# compute next month
sm += 1
if sm > 12:
sm = 1
sy += 1
yield (SpinConfig.cal_to_unix((lasty,lastm,1)), SpinConfig.cal_to_unix((sy,sm,1)))
def my_slave(input):
nosql_client = SpinNoSQL.NoSQLClient(
SpinConfig.get_mongodb_config(SpinConfig.config['game_id']))
table = TABLES[input['kind']]
do_upload(nosql_client, table, input['verbose'], input['dry_run'],
input['keep_local'])
do_clean(nosql_client, table, input['verbose'], input['dry_run'])
def do_main():
if not os.path.exists(log_dir):
os.mkdir(log_dir)
global exception_log
exception_log = SpinLog.DailyRawLog(log_dir+'/', '-exceptions.txt')
global raw_log
raw_log = exception_log # SpinLog.DailyRawLog(log_dir+'/', '-chatserver.txt', buffer = (not verbose))
global chat_log
chat_log = SpinLog.DailyJSONLog(log_dir+'/','-chat.json')
global nosql_client
nosql_client = SpinNoSQL.NoSQLClient(SpinConfig.get_mongodb_config(SpinConfig.config['game_id']),
identity = 'chatserver', max_retries = -1, # never give up
log_exception_func = lambda x: exception_log.event(server_time, x))
pf = Factory()
pf.protocol = ChatProtocolHandlers
myhost = SpinConfig.config['chatserver'].get('chat_listen_host','localhost')
myport = SpinConfig.config['chatserver']['chat_port']
reactor.listenTCP(myport, pf, interface=myhost)
# SIGHUP forces a full flush and spin_config reload
def handle_SIGHUP():
global server_time
server_time = int(time.time())
try:
reload_spin_config()
except:
exception_log.event(server_time, 'chatserver SIGHUP Exception: ' + traceback.format_exc())
signal.signal(signal.SIGHUP, lambda signum, frm: handle_SIGHUP())
print 'Chat server up and running on %s:%d' % (myhost, myport)
if daemonize:
Daemonize.daemonize()
# update PID file with new PID
open(pidfile, 'w').write('%d\n' % os.getpid())
# turn on Twisted logging
def log_exceptions(eventDict):
if eventDict['isError']:
if 'failure' in eventDict:
text = ((eventDict.get('why') or 'Unhandled Error')
+ '\n' + eventDict['failure'].getTraceback().strip())
else:
text = ' '.join([str(m) for m in eventDict['message']])
exception_log.event(server_time, ('chatserver (%d): ' % os.getpid()) + text)
def log_raw(eventDict):
text = log.textFromEventDict(eventDict)
if text is None or ('connection established' in text) or ('connection lost' in text):
return
raw_log.event(server_time, ('chatserver (%d): ' % os.getpid()) + text)
log.startLoggingWithObserver(log_raw)
log.addObserver(log_exceptions)
reactor.run()
def reconfig():
global interval
try:
SpinConfig.reload() # reload config.json file
new_interval = SpinConfig.config.get('latency_probe_interval', 10)
if new_interval != interval:
if interval:
bg_task.stop()
bg_task.start(new_interval, now=False)
interval = new_interval
except:
if exception_log:
exception_log.event(
int(time.time()),
'latency_probe SIGHUP Exception: ' + traceback.format_exc())
else:
raise
def iterate_from_mongodb(game_id, table_name, start_time, end_time):
nosql_client = SpinNoSQL.NoSQLClient(
SpinConfig.get_mongodb_config(game_id))
qs = {'time': {'$gt': start_time, '$lt': end_time}}
for row in nosql_client.log_buffer_table(table_name).find(qs):
row['_id'] = nosql_client.decode_object_id(row['_id'])
yield row
def do_reminders(game_id, dry_run = False):
filename = SpinConfig.gamedata_component_filename('dev_reminders.json', override_game_id = game_id)
if not os.path.exists(filename): return
data = SpinConfig.load(filename)
time_now = int(time.time())
sender_name = data['sender']
for reminder in data['reminders']:
subject = 'Automated reminder from %s' % sender_name
if reminder['body'] == '$XXX': # special case for six-X comments
body = get_six_X_comments(game_id, time_now)
else:
body = 'Due %s %s' % (pretty_print_time(abs(reminder['deadline']-time_now)), 'from now' if reminder['deadline']>time_now else 'ago') + '\n' + reminder['body']
if body:
SpinReminders.send_reminders(sender_name, reminder['notify'], subject, body, dry_run = dry_run)
def pretty_print_bom(bom):
total = bom_to_gamebucks(bom)
p = {}
for key, val in bom.iteritems():
if key == 'time':
p[str(key)] = SpinConfig.pretty_print_time(val) + pct(
time_to_gamebucks(val), total)
elif key in gamedata['resources']:
p[str(key)] = str(val) + pct(resource_to_gamebucks(val), total)
else:
p[str(key)] = val
return repr(p)
def g_update_user(seg):
sys.stderr.write('seg %d/%d start\n' % (seg, SEGMENTS - 1))
db_client = SpinNoSQL.NoSQLClient(
SpinConfig.get_mongodb_config(SpinConfig.config['game_id']))
id_range = db_client.get_user_id_range()
id_set = [
id for id in xrange(id_range[0], id_range[1] + 1)
if (id % SEGMENTS) == seg
]
for user_id in id_set:
update_user(user_id, db_client)
sys.stderr.write('seg %d/%d DONE\n' % (seg, SEGMENTS - 1))
def main():
global http_client, nosql_client, bg_task, exception_log
http_client = AsyncHTTP.AsyncHTTPRequester(
-1,
-1,
int(0.8 * interval),
-1,
lambda x: exception_log.event(int(time.time()), x)
if daemonize else sys.stderr.write(x + '\n'),
max_tries=1)
nosql_client = SpinNoSQL.NoSQLClient(SpinConfig.get_mongodb_config(
SpinConfig.config['game_id']),
max_retries=-1) # never give up
#log.startLogging(sys.stdout)
signal.signal(signal.SIGHUP, handle_SIGHUP)
bg_task = task.LoopingCall(bgtask_func)
if daemonize:
Daemonize.daemonize()
# update PID file with new PID
open(pidfile, 'w').write('%d\n' % os.getpid())
exception_log = SpinLog.DailyRawLog(
SpinConfig.config.get('log_dir', 'logs') + '/', '-exceptions.txt')
# turn on Twisted logging
def log_exceptions(eventDict):
if eventDict['isError']:
if 'failure' in eventDict:
text = ((eventDict.get('why') or 'Unhandled Error') +
'\n' + eventDict['failure'].getTraceback().strip())
else:
text = ' '.join([str(m) for m in eventDict['message']])
exception_log.event(int(time.time()), text)
def log_raw(eventDict):
return
log.startLoggingWithObserver(log_raw)
log.addObserver(log_exceptions)
bg_task.start(interval)
reactor.run()
def to_playerdb(player_filename, player, base_id):
gamedata['ai_bases'] = SpinJSON.load(
open(SpinConfig.gamedata_component_filename("ai_bases_compiled.json")))
base = gamedata['ai_bases']['bases'][str(base_id)]
my_base = []
townhall_level = -1
for building in base['buildings']:
if building['spec'] == gamedata['townhall']:
townhall_level = building.get('level', 1)
props = {
'spec': building['spec'],
'xy': building['xy'],
'level': building.get('level', 1)
}
if 'equipment' in building: props['equipment'] = building['equipment']
my_base.append(props)
for unit in base['units']:
props = {
'spec': unit['spec'],
'level': unit.get('level', 1),
'xy': unit['xy']
}
if 'orders' in unit: props['orders'] = unit['orders']
if 'patrol' in unit: props['patrol'] = unit['patrol']
my_base.append(props)
for scenery in base.get('scenery', []):
my_base.append({'spec': scenery['spec'], 'xy': scenery['xy']})
player['unit_repair_queue'] = []
player['my_base'] = my_base
player['tech'] = base['tech']
if townhall_level > 0:
player['history'][gamedata['townhall'] + '_level'] = townhall_level
if 'base_climate' in base: player['base_climate'] = base['base_climate']
if 'deployment_buffer' in base:
player['deployment_buffer'] = base['deployment_buffer']
atom = AtomicFileWrite.AtomicFileWrite(player_filename, 'w')
SpinJSON.dump(player, atom.fd, pretty=True)
atom.complete(fsync=False)
print 'wrote contents of AI base %d to %s!' % (base_id, player_filename)
def is_satisfied(self, player, qdata):
if player.cooldown_active('birthday_' + self.tag):
return False
now_unix = player.get_absolute_time()
today = time.gmtime(now_unix)
if 'birthday_' + self.tag in player.history:
if player.history['birthday_' + self.tag] >= today.tm_year:
return False
if player.birthday:
birthday = time.gmtime(player.birthday)
birthday_unix = SpinConfig.cal_to_unix(
(today.tm_year, birthday.tm_mon, birthday.tm_mday))
start_unix = now_unix - 604800 # 7 days
return (start_unix <= birthday_unix) and (birthday_unix <=
now_unix)
return False
def bgtask_func():
time_now = time.time()
nosql_client.set_time(int(time_now))
servers = []
rows = nosql_client.server_status_query({},
fields={
'_id': 1,
'type': 1,
'hostname': 1,
'game_http_port': 1,
'external_http_port': 1,
'server_time': 1
})
for row in rows:
key = row['server_name']
if row['type'] == 'proxyserver':
url = 'http://%s:%d/PING' % (row['hostname'],
row['external_http_port'])
elif row['type'] == SpinConfig.game():
url = 'http://%s:%d/CONTROLAPI?secret=%s&method=ping' % (
row['hostname'], row['game_http_port'],
SpinConfig.config['proxy_api_secret'])
else:
continue
servers.append((key, url))
for key, url in servers:
if key in requests:
#print 'still running'
continue
requests[key] = {
'start_time':
time_now,
'request':
http_client.queue_request(
time_now,
url,
functools.partial(on_response, key, True),
error_callback=functools.partial(on_response, key, False))
} # standin parameter
def __init__(self,
game_id=None,
key_file=None,
userdb_bucket=None,
playerdb_bucket=None,
aistate_bucket=None,
basedb_bucket=None,
verbose=False):
self.nbuckets = 100
if game_id is None: game_id = SpinConfig.config['game_id']
self.game_id = game_id
self.userdb_bucket = userdb_bucket if userdb_bucket else SpinConfig.config.get(
'userdb_s3_bucket', None)
self.playerdb_bucket = playerdb_bucket if playerdb_bucket else SpinConfig.config.get(
'playerdb_s3_bucket', None)
self.aistate_bucket = aistate_bucket if aistate_bucket else SpinConfig.config.get(
'aistate_s3_bucket', None)
self.basedb_bucket = basedb_bucket if basedb_bucket else SpinConfig.config.get(
'basedb_s3_bucket', None)
self.s3con = SpinS3.S3(
key_file if key_file else SpinConfig.aws_key_file(),
verbose=verbose)
def stream_userdb():
bucket, name = SpinConfig.upcache_s3_location(SpinConfig.game())
return SpinUpcacheIO.S3Reader(SpinS3.S3(SpinConfig.aws_key_file()), bucket,
name).iter_all()
try:
import simplejson as json
except:
import json
import SpinConfig
# load some standard Python libraries
import sys, time, calendar
import csv
# load SpinPunch Upcache S3 access library
import SpinS3, SpinUpcacheIO
# load gamedata so we can reference it if necessary
# e.g. gamedata['units']['motion_cannon']['armor']
gamedata = json.load(open(SpinConfig.gamedata_filename()))
time_now = int(time.time())
def stream_userdb():
bucket, name = SpinConfig.upcache_s3_location(SpinConfig.game())
return SpinUpcacheIO.S3Reader(SpinS3.S3(SpinConfig.aws_key_file()), bucket,
name).iter_all()
# main program
if __name__ == '__main__':
# days at which to evaluate spend_dX
SPEND_MARKS = (0, 1, 3, 5, 7, 14, 30, 60, 90, 120)
('val', 'FLOAT8 NOT NULL'),
('mtime', 'INT8 NOT NULL')
],
# note: index stat AFTER time so that we can do time range queries quickly
'indices': { 'by_id': {'unique':True, 'keys':[(id_field,'ASC'), ('time_scope','ASC'), ('time_loc','ASC'), ('stat','ASC'), ('space_scope','ASC'), ('space_loc','ASC'),
('extra_scope','ASC'), ('extra_loc','ASC')] },
'by_stat': {'keys': [('time_scope','ASC'), ('time_loc','ASC'), ('stat','ASC'), ('space_scope','ASC'), ('space_loc','ASC'),
('extra_scope','ASC'), ('extra_loc','ASC'), ('val','DESC')]},
'by_mtime': {'keys': [('time_scope','ASC'), ('time_loc','ASC'), ('mtime','ASC')]},
}
}
if __name__ == '__main__':
game_id = SpinConfig.game()
commit_interval = 1000
verbose = True
force = False
do_reset = False
do_mongo_drop = False
dry_run = 0
opts, args = getopt.gnu_getopt(sys.argv[1:], 'g:c:q', ['reset','mongo-drop','dry-run','force'])
for key, val in opts:
if key == '-g': game_id = val
elif key == '-c': commit_interval = int(val)
elif key == '-q': verbose = False
elif key == '--mongo-drop': do_mongo_drop = True
elif key == '--reset': do_reset = True
def birthday_to_birth_time(birthday):
m, d, y = map(int, birthday.split('/'))
return SpinConfig.cal_to_unix((y, m, d))
{
'name': 'country',
'values': ['us']
}, # ,'gb','au']},
{
'name':
'age_group',
'values':
SpinConfig.AGE_GROUPS.values() + ['MISSING'],
'mandatory':
lambda (player):
('birthday' in player and 'account_creation_time' in player),
'readfunc':
lambda (player): SpinConfig.AGE_GROUPS.get(
SpinConfig.years_old_to_age_group(
(player['account_creation_time'] - birthday_to_birth_time(
player['birthday'])) / (365 * 24 * 60 * 60)), 'MISSING')
}
]
def get_key(axes, player):
k = []
for axis in axes:
if 'mandatory' in axis:
if not axis['mandatory'](player): return None
else:
if axis['name'] not in player: return None
if 'readfunc' in axis:
try:
val = axis['readfunc'](player)
#!/usr/bin/env python
# Copyright (c) 2015 SpinPunch. All rights reserved.
# Use of this source code is governed by an MIT-style license that can be
# found in the LICENSE file.
import random
import SpinJSON # JSON reading/writing library
import SpinConfig
gamedata = SpinJSON.load(open(SpinConfig.gamedata_filename()))
ncells = 180
def makeRiflemenCluster(base, x, y):
base['units'].append({
"xy": [x, y],
"patrol":
1,
"force_level":
8,
"spec":
"rifleman",
"orders": [{
"dest": [x - 23, y + 1],
"state": 6,
"aggressive": False
}, {
"dest": [x + 12, y - 2],
"state": 6,
"patrol_origin": 1
}, {
def iterate_from_s3(game_id,
bucket,
logname,
start_time,
end_time,
verbose=True):
assert start_time > 0
# to protect against same-time collisions, create a unique fake "PID" for MongoDB row _ids
sha = hashlib.sha1()
sha.update(game_id)
dig = sha.digest()
fake_pid = (ord(dig[1]) << 8) | ord(dig[0])
s3 = SpinS3.S3(SpinConfig.aws_key_file())
last_id_time = -1
id_serial = 0
for t in xrange(86400 * (start_time // 86400), 86400 * (end_time // 86400),
86400): # for each day
y, m, d = SpinConfig.unix_to_cal(t)
prefix = '%04d%02d/%s-%04d%02d%02d-%s' % (
y, m, SpinConfig.game_id_long(override_game_id=game_id), y, m, d,
logname)
for entry in s3.list_bucket(bucket, prefix=prefix):
filename = entry['name'].split('/')[-1]
if verbose: print 'reading', filename
if entry['name'].endswith('.zip'):
tf = tempfile.NamedTemporaryFile(prefix=logname + '-' +
filename,
suffix='.zip')
s3.get_file(bucket, entry['name'], tf.name)
unzipper = subprocess.Popen(['unzip', '-q', '-p', tf.name],
stdout=subprocess.PIPE)
elif entry['name'].endswith('.gz'):
tf = tempfile.NamedTemporaryFile(prefix=logname + '-' +
filename,
suffix='.gz')
s3.get_file(bucket, entry['name'], tf.name)
unzipper = subprocess.Popen(['gunzip', '-c', tf.name],
stdout=subprocess.PIPE)
else:
raise Exception('unhandled file extension: ' + entry['name'])
for line in unzipper.stdout.xreadlines():
row = SpinJSON.loads(line)
if row['time'] < start_time: continue # skip ahead
elif row['time'] >= end_time: break
if '_id' not in row:
# synthesize a fake MongoDB row ID
if row['time'] != last_id_time:
last_id_time = row['time']
id_serial = 0
row['_id'] = SpinNoSQLId.creation_time_id(row['time'],
pid=fake_pid,
serial=id_serial)
assert SpinNoSQLId.is_valid(row['_id'])
id_serial += 1
# note: there's a small chance this could end up duplicating an event at the boundary of an S3 import and MongoDB import
if verbose: print row
yield row
# Copyright (c) 2015 SpinPunch. All rights reserved.
# Use of this source code is governed by an MIT-style license that can be
# found in the LICENSE file.
# migrate on-disk aistate/ files to S3 bucket
import sys, os, hashlib, glob, getopt
import SpinS3
import SpinConfig
import multiprocessing
io_config = SpinConfig.config['gameservers'][
SpinConfig.config['gameservers'].keys()[0]].get('io', {})
assert io_config['aistate_use_s3']
con = SpinS3.S3(SpinConfig.aws_key_file())
bucket = SpinConfig.config['aistate_s3_bucket']
def make_s3_aistate_objname(filename):
name = os.path.basename(filename)
return hashlib.md5(name).hexdigest() + '-' + name
def migrate_to_s3(filename):
objname = make_s3_aistate_objname(filename)
print '%-35s -> %-60s' % (filename, objname),
# check if the file exists
mtime = con.exists(bucket, objname)
if mtime > 0:
# Use of this source code is governed by an MIT-style license that can be
# found in the LICENSE file.
# instructions for running this script:
# go to the gamedata/ directory
# PYTHONPATH=../gameserver ./make_ai_base_example.py
# (optional: use > to write output to a file instead of printing to the console)
import SpinJSON # JSON reading/writing library
import SpinConfig
import AtomicFileWrite # little library for safely overwriting files atomically
import sys, copy, getopt, os, random # import some misc. Python libraries
import tr_base_generator_helper as bgh
# load in gamedata
gamedata = SpinJSON.load(open(SpinConfig.gamedata_filename()))
ncells = 180
class cannotFindCoordinateError(Exception):
# def __init__(self):
#
def __str__(self):
return "Error"
BASE_LAYOUT = {
"sectors": [{
"cluster": "toc_cluster",
"positioning": "near_the_middle"
}, {
batch = 0
con.commit()
cur = con.cursor()
if verbose: print >> sys.stderr, total, 'inserted'
if not dry_run:
con.commit()
if __name__ == '__main__':
if '--slave' in sys.argv:
SpinParallel.slave(do_slave)
sys.exit(0)
game_id = SpinConfig.game()
commit_interval = 1000
verbose = True
dry_run = False
parallel = 1
opts, args = getopt.gnu_getopt(sys.argv[1:], 'g:c:q', ['dry-run','parallel='])
for key, val in opts:
if key == '-g': game_id = val
elif key == '-c': commit_interval = int(val)
elif key == '-q': verbose = False
elif key == '--dry-run': dry_run = True
elif key == '--parallel': parallel = int(val)
if len(args) != 2:
def unit_cost_summary_schema(sql_util):
return {'fields': [('day', 'INT8 NOT NULL')] + \
sql_util.summary_out_dimensions() + \
[('specname', 'VARCHAR(128) NOT NULL'),
('location', 'VARCHAR(16) NOT NULL'),
# ---
('count', 'INT8'),
('gamebucks', 'FLOAT4 NOT NULL'),
],
'indices': {'master': {'unique':True, 'keys': [('day','ASC')] + [(dim,'ASC') for dim, kind in sql_util.summary_out_dimensions()] + [('specname','ASC'),('location','ASC')]}},
}
if __name__ == '__main__':
game_id = SpinConfig.game()
commit_interval = 1000
verbose = True
dry_run = False
do_prune = False
do_optimize = False
do_unit_cost = False
opts, args = getopt.gnu_getopt(
sys.argv[1:], 'g:c:q', ['prune', 'optimize', 'dry-run', 'unit-cost'])
for key, val in opts:
if key == '-g': game_id = val
elif key == '-c': commit_interval = int(val)
elif key == '-q': verbose = False
elif key == '--dry-run': dry_run = True
def do_slave(task):
date = task['date']
game_id = task['game_id']
verbose = task['verbose']
dry_run = task['dry_run']
commit_interval = task['commit_interval']
start_time = SpinConfig.cal_to_unix((int(date[0:4]),int(date[4:6]),int(date[6:8])))
end_time = start_time + 86400
gamedata = SpinJSON.load(open(SpinConfig.gamedata_filename(override_game_id=game_id)))
STORE = {}
[get_store_items(STORE, sku) for sku in gamedata['store']['catalog']]
if verbose:
print >> sys.stderr, 'converting date', date, 'start_time', start_time, 'end_time', end_time, '...'
if not verbose: filterwarnings('ignore', category = MySQLdb.Warning)
cfg = SpinConfig.get_mysql_config(game_id+'_upcache')
con = MySQLdb.connect(*cfg['connect_args'], **cfg['connect_kwargs'])
store_table = cfg['table_prefix']+game_id+'_store'
s3 = SpinS3.S3(SpinConfig.aws_key_file())
bucket = 'spinpunch-logs'
batch = 0
total = 0
cur = con.cursor()
for entry in s3.list_bucket(bucket, prefix='%s/%s-%s-metrics.json' % (date[0:6], SpinConfig.game_id_long(override_game_id=game_id), date)):
filename = entry['name'].split('/')[-1]
if verbose: print >> sys.stderr, 'reading', filename
if entry['name'].endswith('.zip'):
tf = tempfile.NamedTemporaryFile(prefix='old_metrics_to_mysql-'+filename, suffix='.zip')
s3.get_file(bucket, entry['name'], tf.name)
unzipper = subprocess.Popen(['unzip', '-q', '-p', tf.name],
stdout = subprocess.PIPE)
elif entry['name'].endswith('.gz'):
fd = s3.get_open(bucket, entry['name'], allow_keepalive = False)
unzipper = subprocess.Popen(['gunzip', '-c', '-'],
stdin = fd.fileno(),
stdout = subprocess.PIPE)
for line in unzipper.stdout.xreadlines():
if '5120_buy_item' in line:
#and ('item:token' in line):
entry = SpinJSON.loads(line)
if entry['event_name'] != '5120_buy_item': continue
if 'price_currency' not in entry:
# old metric, need to fill in manually
if entry['items'][0]['spec'] in STORE:
entry['price_currency'] = 'item:token'
entry['price'] = STORE[entry['items'][0]['spec']]
if verbose: print >> sys.stderr, SpinJSON.dumps(entry)
if entry.get('price_currency','unknown') != 'item:token': continue
if '_id' in entry:
entry_id = entry['_id']
else:
id_generator.set_time(int(time.time()))
entry_id = id_generator.generate() # arbitrary
assert len(entry['items']) == 1
item = entry['items'][0]
keyvals = [('_id', entry_id),
('time', entry['time']),
('user_id', entry['user_id']),
('price', entry['price']),
('currency', entry['price_currency']),
('item', item['spec']),
('stack', item.get('stack',1))]
query = "INSERT INTO " + store_table + \
"("+', '.join(['`'+k+'`' for k,v in keyvals])+")"+ \
" VALUES ("+', '.join(['%s'] * len(keyvals)) +")"
if dry_run:
print >> sys.stderr, query, [v for k,v in keyvals]
else:
cur.execute(query, [v for k,v in keyvals])
batch += 1
total += 1
if commit_interval > 0 and batch >= commit_interval:
batch = 0
con.commit()
cur = con.cursor()
if verbose: print >> sys.stderr, total, 'inserted'
if not dry_run:
con.commit()
'indices': {'master': {'unique':False, 'keys': [(interval,'ASC')]}},
}
def get_ad_skynet_from_query_string(qs):
if qs:
q = urlparse.parse_qs(qs)
if 'spin_atgt' in q:
return q['spin_atgt'][0]
elif 'spin_rtgt' in q:
return q['spin_rtgt'][0]
return None
if __name__ == '__main__':
game_id = SpinConfig.game()
commit_interval = 1000
verbose = True
do_prune = False
do_optimize = False
opts, args = getopt.gnu_getopt(sys.argv[1:], 'g:c:q',
['prune', 'optimize'])
for key, val in opts:
if key == '-g': game_id = val
elif key == '-c': commit_interval = int(val)
elif key == '-q': verbose = False
elif key == '--prune': do_prune = True
elif key == '--optimize': do_optimize = True
import sys, time, getopt
time_now = int(time.time())
if __name__ == '__main__':
yes_i_am_sure = False
opts, args = getopt.gnu_getopt(sys.argv[1:], '', ['yes-i-am-sure'])
for key, val in opts:
if key == '--yes-i-am-sure': yes_i_am_sure = True
if not yes_i_am_sure:
print '--yes-i-am-sure flag to confirm.'
sys.exit(1)
nosql_client = SpinNoSQL.NoSQLClient(
SpinConfig.get_mongodb_config(SpinConfig.config['game_id']))
tbl = nosql_client.db[nosql_client.dbconfig['table_prefix'] +
'message_table']
for msg in tbl.find({'recipient': {
'$type': 2
}}, {
'type': 1,
'recipient': 1
}):
if msg['type'] in ('mail', 'i_attacked_you', 'FBRTAPI_payment'):
print 'KEEPING', msg['_id'], msg['type']
tbl.update({'_id': msg['_id']},
{'$set': {
'recipient': int(msg['recipient'])
}})
