def my_slave(input): nosql_client = SpinNoSQL.NoSQLClient( SpinConfig.get_mongodb_config(SpinConfig.config['game_id'])) table = TABLES[input['kind']] do_upload(nosql_client, table, input['verbose'], input['dry_run'], input['keep_local']) do_clean(nosql_client, table, input['verbose'], input['dry_run'])
def do_main(): if not os.path.exists(log_dir): os.mkdir(log_dir) global exception_log exception_log = SpinLog.DailyRawLog(log_dir+'/', '-exceptions.txt') global raw_log raw_log = exception_log # SpinLog.DailyRawLog(log_dir+'/', '-chatserver.txt', buffer = (not verbose)) global chat_log chat_log = SpinLog.DailyJSONLog(log_dir+'/','-chat.json') global nosql_client nosql_client = SpinNoSQL.NoSQLClient(SpinConfig.get_mongodb_config(SpinConfig.config['game_id']), identity = 'chatserver', max_retries = -1, # never give up log_exception_func = lambda x: exception_log.event(server_time, x)) pf = Factory() pf.protocol = ChatProtocolHandlers myhost = SpinConfig.config['chatserver'].get('chat_listen_host','localhost') myport = SpinConfig.config['chatserver']['chat_port'] reactor.listenTCP(myport, pf, interface=myhost) # SIGHUP forces a full flush and spin_config reload def handle_SIGHUP(): global server_time server_time = int(time.time()) try: reload_spin_config() except: exception_log.event(server_time, 'chatserver SIGHUP Exception: ' + traceback.format_exc()) signal.signal(signal.SIGHUP, lambda signum, frm: handle_SIGHUP()) print 'Chat server up and running on %s:%d' % (myhost, myport) if daemonize: Daemonize.daemonize() # update PID file with new PID open(pidfile, 'w').write('%d\n' % os.getpid()) # turn on Twisted logging def log_exceptions(eventDict): if eventDict['isError']: if 'failure' in eventDict: text = ((eventDict.get('why') or 'Unhandled Error') + '\n' + eventDict['failure'].getTraceback().strip()) else: text = ' '.join([str(m) for m in eventDict['message']]) exception_log.event(server_time, ('chatserver (%d): ' % os.getpid()) + text) def log_raw(eventDict): text = log.textFromEventDict(eventDict) if text is None or ('connection established' in text) or ('connection lost' in text): return raw_log.event(server_time, ('chatserver (%d): ' % os.getpid()) + text) log.startLoggingWithObserver(log_raw) log.addObserver(log_exceptions) reactor.run()
def iterate_from_mongodb(game_id, table_name, start_time, end_time): nosql_client = SpinNoSQL.NoSQLClient( SpinConfig.get_mongodb_config(game_id)) qs = {'time': {'$gt': start_time, '$lt': end_time}} for row in nosql_client.log_buffer_table(table_name).find(qs): row['_id'] = nosql_client.decode_object_id(row['_id']) yield row
def g_update_user(seg): sys.stderr.write('seg %d/%d start\n' % (seg, SEGMENTS - 1)) db_client = SpinNoSQL.NoSQLClient( SpinConfig.get_mongodb_config(SpinConfig.config['game_id'])) id_range = db_client.get_user_id_range() id_set = [ id for id in xrange(id_range[0], id_range[1] + 1) if (id % SEGMENTS) == seg ] for user_id in id_set: update_user(user_id, db_client) sys.stderr.write('seg %d/%d DONE\n' % (seg, SEGMENTS - 1))
def main(): global http_client, nosql_client, bg_task, exception_log http_client = AsyncHTTP.AsyncHTTPRequester( -1, -1, int(0.8 * interval), -1, lambda x: exception_log.event(int(time.time()), x) if daemonize else sys.stderr.write(x + '\n'), max_tries=1) nosql_client = SpinNoSQL.NoSQLClient(SpinConfig.get_mongodb_config( SpinConfig.config['game_id']), max_retries=-1) # never give up #log.startLogging(sys.stdout) signal.signal(signal.SIGHUP, handle_SIGHUP) bg_task = task.LoopingCall(bgtask_func) if daemonize: Daemonize.daemonize() # update PID file with new PID open(pidfile, 'w').write('%d\n' % os.getpid()) exception_log = SpinLog.DailyRawLog( SpinConfig.config.get('log_dir', 'logs') + '/', '-exceptions.txt') # turn on Twisted logging def log_exceptions(eventDict): if eventDict['isError']: if 'failure' in eventDict: text = ((eventDict.get('why') or 'Unhandled Error') + '\n' + eventDict['failure'].getTraceback().strip()) else: text = ' '.join([str(m) for m in eventDict['message']]) exception_log.event(int(time.time()), text) def log_raw(eventDict): return log.startLoggingWithObserver(log_raw) log.addObserver(log_exceptions) bg_task.start(interval) reactor.run()
sql_util = SpinSQLUtil.MySQLUtil() if not verbose: sql_util.disable_warnings() cfg = SpinConfig.get_mysql_config(game_id + '_upcache') con = MySQLdb.connect(*cfg['connect_args'], **cfg['connect_kwargs']) store_table = cfg['table_prefix'] + game_id + '_store' store_daily_summary_table = cfg[ 'table_prefix'] + game_id + '_store_daily_summary' store_top_spenders_28d_table = cfg[ 'table_prefix'] + game_id + '_store_top_spenders_28d' unit_cost_table = cfg['table_prefix'] + game_id + '_unit_cost' unit_cost_daily_summary_table = cfg[ 'table_prefix'] + game_id + '_unit_cost_daily_summary' nosql_client = SpinNoSQL.NoSQLClient( SpinConfig.get_mongodb_config(game_id)) cur = con.cursor(MySQLdb.cursors.DictCursor) sql_util.ensure_table(cur, store_table, store_schema(sql_util)) sql_util.ensure_table(cur, store_daily_summary_table, store_summary_schema(sql_util)) sql_util.ensure_table(cur, store_top_spenders_28d_table, store_top_spenders_schema(sql_util, 'day')) if do_unit_cost: sql_util.ensure_table(cur, unit_cost_table, unit_cost_schema(sql_util)) sql_util.ensure_table(cur, unit_cost_daily_summary_table, unit_cost_summary_schema(sql_util)) con.commit() # find most recent already-converted action start_time = -1
if not verbose: sql_util.disable_warnings() cfg = SpinConfig.get_pgsql_config(SpinConfig.config['game_id']+'_scores2') if (not force) and \ (SpinConfig.in_maintenance_window(cfg, time_now = time_now) or SpinConfig.in_maintenance_window(cfg, time_now = time_now + 1800)): # allow for 30min to operate if verbose: print 'in database maintenance window, aborting' sys.exit(0) with SpinSingletonProcess.SingletonProcess('scores2-to-sql-%s' % (game_id)): con = psycopg2.connect(*cfg['connect_args'], **cfg['connect_kwargs']) tbl = { 'player': cfg['table_prefix']+'player_scores2', 'alliance': cfg['table_prefix']+'alliance_scores2' } nosql_client = SpinNoSQL.NoSQLClient(SpinConfig.get_mongodb_config(SpinConfig.config['game_id'])) mongo_scores = Scores2.MongoScores2(nosql_client) cur = con.cursor() for kind in tbl: if verbose: print 'setting up tables, indices, and functions for', tbl[kind] if do_reset and dry_run < 2: cur.execute("DROP TABLE "+sql_util.sym(tbl[kind])) if dry_run < 2: sql_util.ensure_table(cur, tbl[kind], scores2_schema(Scores2.ID_FIELD[kind])) replacements = { '%kind': kind, '%id_field': Scores2.ID_FIELD[kind], '%tbl': tbl[kind] } replace_expr = re.compile('|'.join([key.replace('$','\$') for key in replacements.iterkeys()])) if dry_run < 2: cur.execute(replace_expr.sub(lambda match: replacements[match.group(0)], """ CREATE OR REPLACE FUNCTION upsert_%kind_score (p%id_field INT4, pstat VARCHAR, ptime_scope VARCHAR, ptime_loc INT4,
import sys, time, getopt time_now = int(time.time()) if __name__ == '__main__': yes_i_am_sure = False opts, args = getopt.gnu_getopt(sys.argv[1:], '', ['yes-i-am-sure']) for key, val in opts: if key == '--yes-i-am-sure': yes_i_am_sure = True if not yes_i_am_sure: print '--yes-i-am-sure flag to confirm.' sys.exit(1) nosql_client = SpinNoSQL.NoSQLClient( SpinConfig.get_mongodb_config(SpinConfig.config['game_id'])) tbl = nosql_client.db[nosql_client.dbconfig['table_prefix'] + 'message_table'] for msg in tbl.find({'recipient': { '$type': 2 }}, { 'type': 1, 'recipient': 1 }): if msg['type'] in ('mail', 'i_attacked_you', 'FBRTAPI_payment'): print 'KEEPING', msg['_id'], msg['type'] tbl.update({'_id': msg['_id']}, {'$set': { 'recipient': int(msg['recipient']) }})
if key == '--include-developers': include_developers = True elif key == '--progress': progress = True elif key == '--s3-userdb': s3_userdb = True elif key == '--from-s3-keyfile': from_s3_keyfile = val elif key == '--to-s3-keyfile': to_s3_keyfile = val elif key == '--from-s3-bucket': from_s3_bucket = val elif key == '--to-s3-bucket': to_s3_bucket = val elif key == '--to-mongodb': to_mongodb_config = SpinConfig.get_mongodb_config(val) to_mongodb_config[ 'tablename'] = to_mongodb_config['table_prefix'] + val elif key == '--nosql-deltas-only': nosql_deltas_only = True elif key == '--use-dbserver': use_dbserver = bool(int(val)) elif key == '--cache-read': cache_read = val elif key == '--cache-write': cache_write = val elif key == '--cache-segments': cache_segments = int(val) elif key == '--cache-update-time': cache_update_time = int(val) elif key == '--parallel':
MY_DOMAIN = SpinConfig.config[ 'spin_token_domain'] # domain used for cookie time_now = int(time.time()) url_parts = urlparse.urlparse(os.getenv('REQUEST_URI')) path = url_parts.path.split('/') if not path[0]: path = path[1:] if not path[-1]: path = path[:-1] assert len(path) == 1 assert path[0] == 'AUTH' my_endpoint = 'https://' + SpinConfig.config['proxyserver'][ 'external_listen_host'] + ':' + str( SpinConfig.config['proxyserver']['external_ssl_port']) + '/AUTH' if 'mongodb_servers' in SpinConfig.config: dbconfig = SpinConfig.get_mongodb_config( SpinConfig.config.get('spin_token_db', 'AUTH')) dbcon = pymongo.MongoClient(*dbconfig['connect_args'], **dbconfig['connect_kwargs']) db = dbcon[dbconfig['dbname']] tbl = db[dbconfig['table_prefix'] + 'csrf_state'] else: print "need mongodb_servers config for AUTH" sys.exit(1) print 'Content-Type: text/html' print 'Pragma: no-cache, no-store' print 'Cache-Control: no-cache, no-store' if ('code' not in args): # first hit final_url = args['final_url'][-1] final_url_domain = SpinGoogleAuth.url_to_domain(final_url)
cfg = SpinConfig.get_mysql_config(game_id+'_upcache') con = MySQLdb.connect(*cfg['connect_args'], **cfg['connect_kwargs']) # INPUTS sessions_table = cfg['table_prefix']+game_id+'_sessions' # OUTPUTS sessions_daily_temp_table = cfg['table_prefix']+game_id+'_sessions_daily_temp' sessions_daily_summary_table = cfg['table_prefix']+game_id+'_sessions_daily_summary' sessions_hourly_temp_table = cfg['table_prefix']+game_id+'_sessions_hourly_temp' sessions_hourly_summary_table = cfg['table_prefix']+game_id+'_sessions_hourly_summary' sessions_monthly_temp_table = cfg['table_prefix']+game_id+'_sessions_monthly_temp' sessions_monthly_summary_table = cfg['table_prefix']+game_id+'_sessions_monthly_summary' nosql_client = SpinNoSQL.NoSQLClient(SpinConfig.get_mongodb_config(game_id)) cur = con.cursor(MySQLdb.cursors.DictCursor) sql_util.ensure_table(cur, sessions_table, sessions_schema(sql_util)) sql_util.ensure_table(cur, sessions_daily_summary_table, sessions_summary_schema(sql_util, 'day', 'dau')) sql_util.ensure_table(cur, sessions_hourly_summary_table, sessions_summary_schema(sql_util, 'hour', 'hau')) sql_util.ensure_table(cur, sessions_monthly_summary_table, sessions_summary_schema(sql_util, 'month', 'mau')) con.commit() # set time range for MongoDB query start_time = -1 end_time = time_now - SpinETL.MAX_SESSION_LENGTH # skip entries too close to "now" to ensure all sessions have been closed by max(end_time) # find most recent already-converted event in SQL cur.execute("SELECT start FROM "+sql_util.sym(sessions_table)+" ORDER BY start DESC LIMIT 1") rows = cur.fetchall()
# main program if __name__ == '__main__': opts, args = getopt.gnu_getopt(sys.argv[1:], 'g:', ['dry-run', 'limit=']) dry_run = False limit = -1 game_id = SpinConfig.game() for key, val in opts: if key == '--dry-run': dry_run = True elif key == '-g': game_id = val elif key == '--limit': limit = int(val) db_client = SpinNoSQL.NoSQLClient(SpinConfig.get_mongodb_config(SpinConfig.config['game_id']), identity = 'retention_incentive.py') db_client.set_time(time_now) fb_notifications_log = SpinLog.FBNotificationsLogFilter(SpinNoSQLLog.NoSQLJSONLog(db_client, 'log_fb_notifications')) infile = open(args[0]) if args[0] != '-' else sys.stdin gamedata = SpinJSON.load(open(SpinConfig.gamedata_filename(override_game_id = game_id))) seen = 0 sent = 0 for line in infile.xreadlines(): if not line: break user_id = int(line) print user_id,
def connect_to_db(): nosql_client = SpinNoSQL.NoSQLClient(SpinConfig.get_mongodb_config( SpinConfig.config['game_id']), identity='retention_newbie.py') nosql_client.set_time(time_now) return nosql_client
import SpinConfig import os, sys, getopt if __name__ == '__main__': whichdb = SpinConfig.config['game_id'] mode = 'connect' opts, args = getopt.gnu_getopt(sys.argv[1:], '', ['sizes']) for key, val in opts: if key == '--sizes': mode = 'size' if len(args) > 0: whichdb = args[0] conf = SpinConfig.get_mongodb_config(whichdb) cmd_args = [ 'mongo', '-u', conf['username'], '-p', conf['password'], '--host', conf['host'], '--port', '%d' % conf['port'], conf['dbname'] ] if mode == 'size': cmd_args += [ '--eval', '''db.getCollectionNames().forEach(function (x) { var stats = db.getCollection(x).stats(); if(!stats['ns']) { return; }; print(stats['ns']+': '+(stats['size']/(1024*1024)).toFixed(1) + ' MB, '+(stats['storageSize']/(1024*1024)).toFixed(1)+' MB on disk'); })''' ] os.execvp(cmd_args[0], cmd_args)
import SpinNoSQL import SpinLog class NoSQLLogBase(SpinLog.Log): def __init__(self, nosql_client, table_name, safe = False): SpinLog.Log.__init__(self) self.nosql_client = nosql_client self.table_name = table_name self.safe = safe # note: we do not "own" the nosql_client connection, so do not close it def close(self): SpinLog.Log.close(self) class NoSQLJSONLog(NoSQLLogBase): def event(self, t, props, reason = ''): assert type(t) in (int, float) self.nosql_client.log_record(self.table_name, t, props, safe = self.safe, reason = reason) class NoSQLRawLog(NoSQLLogBase): def event(self, t, text, reason = ''): assert type(t) in (int, float) self.nosql_client.log_record(self.table_name, t, {'text':unicode(text)}, safe = self.safe, reason = reason) if __name__ == '__main__': import SpinConfig nosql_client = SpinNoSQL.NoSQLClient(SpinConfig.get_mongodb_config(SpinConfig.config['game_id']), identity = 'test') mylog = NoSQLJSONLog(nosql_client, 'log_fb_conversion_pixels') mylog.event(1389328096, {"user_id":1112,"event_name":"7510_adnetwork_event_sent","code":7510,"api":"fb_conversion_pixels","context":"1234","kpi":"ftd"}) myraw = NoSQLRawLog(nosql_client, 'log_fbrtapi') myraw.event(1389328096, 'hello 1 2 3')
for key, val in opts: if key == '-g': game_id = val elif key == '-c': commit_interval = int(val) elif key == '-q': verbose = False elif key == '--dry-run': dry_run = True elif key == '--fix-missing-data': fix_missing_data = True sql_util = SpinSQLUtil.MySQLUtil() if not verbose: sql_util.disable_warnings() cfg = SpinConfig.get_mysql_config( 'skynet' if fix_missing_data else 'skynet_readonly') con = MySQLdb.connect(*cfg['connect_args'], **cfg['connect_kwargs']) adstats_table = cfg['table_prefix'] + 'adstats_hourly' nosql_config = SpinConfig.get_mongodb_config('skynet') nosql_client = pymongo.MongoClient(*nosql_config['connect_args'], **nosql_config['connect_kwargs']) nosql_db = nosql_client[nosql_config['dbname']] cur = con.cursor() if not dry_run: sql_util.ensure_table(cur, adstats_table, adstats_schema) con.commit() # find most recent entry start_time = -1 end_time = time_now - 60 # skip entries too close to "now" to ensure that all entries for a given second are present cur = con.cursor(MySQLdb.cursors.DictCursor) if dry_run: row = None