def my_slave(input):
nosql_client = SpinNoSQL.NoSQLClient(
SpinConfig.get_mongodb_config(SpinConfig.config['game_id']))
table = TABLES[input['kind']]
do_upload(nosql_client, table, input['verbose'], input['dry_run'],
input['keep_local'])
do_clean(nosql_client, table, input['verbose'], input['dry_run'])
def do_main():
if not os.path.exists(log_dir):
os.mkdir(log_dir)
global exception_log
exception_log = SpinLog.DailyRawLog(log_dir+'/', '-exceptions.txt')
global raw_log
raw_log = exception_log # SpinLog.DailyRawLog(log_dir+'/', '-chatserver.txt', buffer = (not verbose))
global chat_log
chat_log = SpinLog.DailyJSONLog(log_dir+'/','-chat.json')
global nosql_client
nosql_client = SpinNoSQL.NoSQLClient(SpinConfig.get_mongodb_config(SpinConfig.config['game_id']),
identity = 'chatserver', max_retries = -1, # never give up
log_exception_func = lambda x: exception_log.event(server_time, x))
pf = Factory()
pf.protocol = ChatProtocolHandlers
myhost = SpinConfig.config['chatserver'].get('chat_listen_host','localhost')
myport = SpinConfig.config['chatserver']['chat_port']
reactor.listenTCP(myport, pf, interface=myhost)
# SIGHUP forces a full flush and spin_config reload
def handle_SIGHUP():
global server_time
server_time = int(time.time())
try:
reload_spin_config()
except:
exception_log.event(server_time, 'chatserver SIGHUP Exception: ' + traceback.format_exc())
signal.signal(signal.SIGHUP, lambda signum, frm: handle_SIGHUP())
print 'Chat server up and running on %s:%d' % (myhost, myport)
if daemonize:
Daemonize.daemonize()
# update PID file with new PID
open(pidfile, 'w').write('%d\n' % os.getpid())
# turn on Twisted logging
def log_exceptions(eventDict):
if eventDict['isError']:
if 'failure' in eventDict:
text = ((eventDict.get('why') or 'Unhandled Error')
+ '\n' + eventDict['failure'].getTraceback().strip())
else:
text = ' '.join([str(m) for m in eventDict['message']])
exception_log.event(server_time, ('chatserver (%d): ' % os.getpid()) + text)
def log_raw(eventDict):
text = log.textFromEventDict(eventDict)
if text is None or ('connection established' in text) or ('connection lost' in text):
return
raw_log.event(server_time, ('chatserver (%d): ' % os.getpid()) + text)
log.startLoggingWithObserver(log_raw)
log.addObserver(log_exceptions)
reactor.run()
def iterate_from_mongodb(game_id, table_name, start_time, end_time):
nosql_client = SpinNoSQL.NoSQLClient(
SpinConfig.get_mongodb_config(game_id))
qs = {'time': {'$gt': start_time, '$lt': end_time}}
for row in nosql_client.log_buffer_table(table_name).find(qs):
row['_id'] = nosql_client.decode_object_id(row['_id'])
yield row
def g_update_user(seg):
sys.stderr.write('seg %d/%d start\n' % (seg, SEGMENTS - 1))
db_client = SpinNoSQL.NoSQLClient(
SpinConfig.get_mongodb_config(SpinConfig.config['game_id']))
id_range = db_client.get_user_id_range()
id_set = [
id for id in xrange(id_range[0], id_range[1] + 1)
if (id % SEGMENTS) == seg
]
for user_id in id_set:
update_user(user_id, db_client)
sys.stderr.write('seg %d/%d DONE\n' % (seg, SEGMENTS - 1))
def main():
global http_client, nosql_client, bg_task, exception_log
http_client = AsyncHTTP.AsyncHTTPRequester(
-1,
-1,
int(0.8 * interval),
-1,
lambda x: exception_log.event(int(time.time()), x)
if daemonize else sys.stderr.write(x + '\n'),
max_tries=1)
nosql_client = SpinNoSQL.NoSQLClient(SpinConfig.get_mongodb_config(
SpinConfig.config['game_id']),
max_retries=-1) # never give up
#log.startLogging(sys.stdout)
signal.signal(signal.SIGHUP, handle_SIGHUP)
bg_task = task.LoopingCall(bgtask_func)
if daemonize:
Daemonize.daemonize()
# update PID file with new PID
open(pidfile, 'w').write('%d\n' % os.getpid())
exception_log = SpinLog.DailyRawLog(
SpinConfig.config.get('log_dir', 'logs') + '/', '-exceptions.txt')
# turn on Twisted logging
def log_exceptions(eventDict):
if eventDict['isError']:
if 'failure' in eventDict:
text = ((eventDict.get('why') or 'Unhandled Error') +
'\n' + eventDict['failure'].getTraceback().strip())
else:
text = ' '.join([str(m) for m in eventDict['message']])
exception_log.event(int(time.time()), text)
def log_raw(eventDict):
return
log.startLoggingWithObserver(log_raw)
log.addObserver(log_exceptions)
bg_task.start(interval)
reactor.run()
sql_util = SpinSQLUtil.MySQLUtil()
if not verbose: sql_util.disable_warnings()
cfg = SpinConfig.get_mysql_config(game_id + '_upcache')
con = MySQLdb.connect(*cfg['connect_args'], **cfg['connect_kwargs'])
store_table = cfg['table_prefix'] + game_id + '_store'
store_daily_summary_table = cfg[
'table_prefix'] + game_id + '_store_daily_summary'
store_top_spenders_28d_table = cfg[
'table_prefix'] + game_id + '_store_top_spenders_28d'
unit_cost_table = cfg['table_prefix'] + game_id + '_unit_cost'
unit_cost_daily_summary_table = cfg[
'table_prefix'] + game_id + '_unit_cost_daily_summary'
nosql_client = SpinNoSQL.NoSQLClient(
SpinConfig.get_mongodb_config(game_id))
cur = con.cursor(MySQLdb.cursors.DictCursor)
sql_util.ensure_table(cur, store_table, store_schema(sql_util))
sql_util.ensure_table(cur, store_daily_summary_table,
store_summary_schema(sql_util))
sql_util.ensure_table(cur, store_top_spenders_28d_table,
store_top_spenders_schema(sql_util, 'day'))
if do_unit_cost:
sql_util.ensure_table(cur, unit_cost_table, unit_cost_schema(sql_util))
sql_util.ensure_table(cur, unit_cost_daily_summary_table,
unit_cost_summary_schema(sql_util))
con.commit()
# find most recent already-converted action
start_time = -1
if not verbose: sql_util.disable_warnings()
cfg = SpinConfig.get_pgsql_config(SpinConfig.config['game_id']+'_scores2')
if (not force) and \
(SpinConfig.in_maintenance_window(cfg, time_now = time_now) or SpinConfig.in_maintenance_window(cfg, time_now = time_now + 1800)): # allow for 30min to operate
if verbose: print 'in database maintenance window, aborting'
sys.exit(0)
with SpinSingletonProcess.SingletonProcess('scores2-to-sql-%s' % (game_id)):
con = psycopg2.connect(*cfg['connect_args'], **cfg['connect_kwargs'])
tbl = { 'player': cfg['table_prefix']+'player_scores2',
'alliance': cfg['table_prefix']+'alliance_scores2' }
nosql_client = SpinNoSQL.NoSQLClient(SpinConfig.get_mongodb_config(SpinConfig.config['game_id']))
mongo_scores = Scores2.MongoScores2(nosql_client)
cur = con.cursor()
for kind in tbl:
if verbose: print 'setting up tables, indices, and functions for', tbl[kind]
if do_reset and dry_run < 2:
cur.execute("DROP TABLE "+sql_util.sym(tbl[kind]))
if dry_run < 2: sql_util.ensure_table(cur, tbl[kind], scores2_schema(Scores2.ID_FIELD[kind]))
replacements = { '%kind': kind, '%id_field': Scores2.ID_FIELD[kind], '%tbl': tbl[kind] }
replace_expr = re.compile('|'.join([key.replace('$','\$') for key in replacements.iterkeys()]))
if dry_run < 2: cur.execute(replace_expr.sub(lambda match: replacements[match.group(0)], """
CREATE OR REPLACE FUNCTION upsert_%kind_score (p%id_field INT4, pstat VARCHAR,
ptime_scope VARCHAR, ptime_loc INT4,
import sys, time, getopt
time_now = int(time.time())
if __name__ == '__main__':
yes_i_am_sure = False
opts, args = getopt.gnu_getopt(sys.argv[1:], '', ['yes-i-am-sure'])
for key, val in opts:
if key == '--yes-i-am-sure': yes_i_am_sure = True
if not yes_i_am_sure:
print '--yes-i-am-sure flag to confirm.'
sys.exit(1)
nosql_client = SpinNoSQL.NoSQLClient(
SpinConfig.get_mongodb_config(SpinConfig.config['game_id']))
tbl = nosql_client.db[nosql_client.dbconfig['table_prefix'] +
'message_table']
for msg in tbl.find({'recipient': {
'$type': 2
}}, {
'type': 1,
'recipient': 1
}):
if msg['type'] in ('mail', 'i_attacked_you', 'FBRTAPI_payment'):
print 'KEEPING', msg['_id'], msg['type']
tbl.update({'_id': msg['_id']},
{'$set': {
'recipient': int(msg['recipient'])
}})
if key == '--include-developers':
include_developers = True
elif key == '--progress':
progress = True
elif key == '--s3-userdb':
s3_userdb = True
elif key == '--from-s3-keyfile':
from_s3_keyfile = val
elif key == '--to-s3-keyfile':
to_s3_keyfile = val
elif key == '--from-s3-bucket':
from_s3_bucket = val
elif key == '--to-s3-bucket':
to_s3_bucket = val
elif key == '--to-mongodb':
to_mongodb_config = SpinConfig.get_mongodb_config(val)
to_mongodb_config[
'tablename'] = to_mongodb_config['table_prefix'] + val
elif key == '--nosql-deltas-only':
nosql_deltas_only = True
elif key == '--use-dbserver':
use_dbserver = bool(int(val))
elif key == '--cache-read':
cache_read = val
elif key == '--cache-write':
cache_write = val
elif key == '--cache-segments':
cache_segments = int(val)
elif key == '--cache-update-time':
cache_update_time = int(val)
elif key == '--parallel':
MY_DOMAIN = SpinConfig.config[
'spin_token_domain'] # domain used for cookie
time_now = int(time.time())
url_parts = urlparse.urlparse(os.getenv('REQUEST_URI'))
path = url_parts.path.split('/')
if not path[0]: path = path[1:]
if not path[-1]: path = path[:-1]
assert len(path) == 1
assert path[0] == 'AUTH'
my_endpoint = 'https://' + SpinConfig.config['proxyserver'][
'external_listen_host'] + ':' + str(
SpinConfig.config['proxyserver']['external_ssl_port']) + '/AUTH'
if 'mongodb_servers' in SpinConfig.config:
dbconfig = SpinConfig.get_mongodb_config(
SpinConfig.config.get('spin_token_db', 'AUTH'))
dbcon = pymongo.MongoClient(*dbconfig['connect_args'],
**dbconfig['connect_kwargs'])
db = dbcon[dbconfig['dbname']]
tbl = db[dbconfig['table_prefix'] + 'csrf_state']
else:
print "need mongodb_servers config for AUTH"
sys.exit(1)
print 'Content-Type: text/html'
print 'Pragma: no-cache, no-store'
print 'Cache-Control: no-cache, no-store'
if ('code' not in args): # first hit
final_url = args['final_url'][-1]
final_url_domain = SpinGoogleAuth.url_to_domain(final_url)
cfg = SpinConfig.get_mysql_config(game_id+'_upcache')
con = MySQLdb.connect(*cfg['connect_args'], **cfg['connect_kwargs'])
# INPUTS
sessions_table = cfg['table_prefix']+game_id+'_sessions'
# OUTPUTS
sessions_daily_temp_table = cfg['table_prefix']+game_id+'_sessions_daily_temp'
sessions_daily_summary_table = cfg['table_prefix']+game_id+'_sessions_daily_summary'
sessions_hourly_temp_table = cfg['table_prefix']+game_id+'_sessions_hourly_temp'
sessions_hourly_summary_table = cfg['table_prefix']+game_id+'_sessions_hourly_summary'
sessions_monthly_temp_table = cfg['table_prefix']+game_id+'_sessions_monthly_temp'
sessions_monthly_summary_table = cfg['table_prefix']+game_id+'_sessions_monthly_summary'
nosql_client = SpinNoSQL.NoSQLClient(SpinConfig.get_mongodb_config(game_id))
cur = con.cursor(MySQLdb.cursors.DictCursor)
sql_util.ensure_table(cur, sessions_table, sessions_schema(sql_util))
sql_util.ensure_table(cur, sessions_daily_summary_table, sessions_summary_schema(sql_util, 'day', 'dau'))
sql_util.ensure_table(cur, sessions_hourly_summary_table, sessions_summary_schema(sql_util, 'hour', 'hau'))
sql_util.ensure_table(cur, sessions_monthly_summary_table, sessions_summary_schema(sql_util, 'month', 'mau'))
con.commit()
# set time range for MongoDB query
start_time = -1
end_time = time_now - SpinETL.MAX_SESSION_LENGTH # skip entries too close to "now" to ensure all sessions have been closed by max(end_time)
# find most recent already-converted event in SQL
cur.execute("SELECT start FROM "+sql_util.sym(sessions_table)+" ORDER BY start DESC LIMIT 1")
rows = cur.fetchall()
# main program
if __name__ == '__main__':
opts, args = getopt.gnu_getopt(sys.argv[1:], 'g:', ['dry-run', 'limit='])
dry_run = False
limit = -1
game_id = SpinConfig.game()
for key, val in opts:
if key == '--dry-run':
dry_run = True
elif key == '-g':
game_id = val
elif key == '--limit':
limit = int(val)
db_client = SpinNoSQL.NoSQLClient(SpinConfig.get_mongodb_config(SpinConfig.config['game_id']), identity = 'retention_incentive.py')
db_client.set_time(time_now)
fb_notifications_log = SpinLog.FBNotificationsLogFilter(SpinNoSQLLog.NoSQLJSONLog(db_client, 'log_fb_notifications'))
infile = open(args[0]) if args[0] != '-' else sys.stdin
gamedata = SpinJSON.load(open(SpinConfig.gamedata_filename(override_game_id = game_id)))
seen = 0
sent = 0
for line in infile.xreadlines():
if not line: break
user_id = int(line)
print user_id,
def connect_to_db():
nosql_client = SpinNoSQL.NoSQLClient(SpinConfig.get_mongodb_config(
SpinConfig.config['game_id']),
identity='retention_newbie.py')
nosql_client.set_time(time_now)
return nosql_client
import SpinConfig
import os, sys, getopt
if __name__ == '__main__':
whichdb = SpinConfig.config['game_id']
mode = 'connect'
opts, args = getopt.gnu_getopt(sys.argv[1:], '', ['sizes'])
for key, val in opts:
if key == '--sizes': mode = 'size'
if len(args) > 0:
whichdb = args[0]
conf = SpinConfig.get_mongodb_config(whichdb)
cmd_args = [
'mongo', '-u', conf['username'], '-p', conf['password'], '--host',
conf['host'], '--port',
'%d' % conf['port'], conf['dbname']
]
if mode == 'size':
cmd_args += [
'--eval',
'''db.getCollectionNames().forEach(function (x) { var stats = db.getCollection(x).stats(); if(!stats['ns']) { return; }; print(stats['ns']+': '+(stats['size']/(1024*1024)).toFixed(1) + ' MB, '+(stats['storageSize']/(1024*1024)).toFixed(1)+' MB on disk'); })'''
]
os.execvp(cmd_args[0], cmd_args)
import SpinNoSQL
import SpinLog
class NoSQLLogBase(SpinLog.Log):
def __init__(self, nosql_client, table_name, safe = False):
SpinLog.Log.__init__(self)
self.nosql_client = nosql_client
self.table_name = table_name
self.safe = safe
# note: we do not "own" the nosql_client connection, so do not close it
def close(self): SpinLog.Log.close(self)
class NoSQLJSONLog(NoSQLLogBase):
def event(self, t, props, reason = ''):
assert type(t) in (int, float)
self.nosql_client.log_record(self.table_name, t, props, safe = self.safe, reason = reason)
class NoSQLRawLog(NoSQLLogBase):
def event(self, t, text, reason = ''):
assert type(t) in (int, float)
self.nosql_client.log_record(self.table_name, t, {'text':unicode(text)}, safe = self.safe, reason = reason)
if __name__ == '__main__':
import SpinConfig
nosql_client = SpinNoSQL.NoSQLClient(SpinConfig.get_mongodb_config(SpinConfig.config['game_id']), identity = 'test')
mylog = NoSQLJSONLog(nosql_client, 'log_fb_conversion_pixels')
mylog.event(1389328096, {"user_id":1112,"event_name":"7510_adnetwork_event_sent","code":7510,"api":"fb_conversion_pixels","context":"1234","kpi":"ftd"})
myraw = NoSQLRawLog(nosql_client, 'log_fbrtapi')
myraw.event(1389328096, 'hello 1 2 3')
for key, val in opts:
if key == '-g': game_id = val
elif key == '-c': commit_interval = int(val)
elif key == '-q': verbose = False
elif key == '--dry-run': dry_run = True
elif key == '--fix-missing-data': fix_missing_data = True
sql_util = SpinSQLUtil.MySQLUtil()
if not verbose: sql_util.disable_warnings()
cfg = SpinConfig.get_mysql_config(
'skynet' if fix_missing_data else 'skynet_readonly')
con = MySQLdb.connect(*cfg['connect_args'], **cfg['connect_kwargs'])
adstats_table = cfg['table_prefix'] + 'adstats_hourly'
nosql_config = SpinConfig.get_mongodb_config('skynet')
nosql_client = pymongo.MongoClient(*nosql_config['connect_args'],
**nosql_config['connect_kwargs'])
nosql_db = nosql_client[nosql_config['dbname']]
cur = con.cursor()
if not dry_run:
sql_util.ensure_table(cur, adstats_table, adstats_schema)
con.commit()
# find most recent entry
start_time = -1
end_time = time_now - 60 # skip entries too close to "now" to ensure that all entries for a given second are present
cur = con.cursor(MySQLdb.cursors.DictCursor)
if dry_run:
row = None
