def create(cls, orig_cls, user_id, game): user_state = orig_cls() user_state.user_id = user_id user_state.state = chunking.blow_chunks(make_empty_user_state(user_id, game)) user_state.date_created = int(time.time()) user_state.date_modified = int(time.time()) return user_state
def _build_response(self, state, chunked, game, user_id): if not chunked: # get the state in a chunked format for sending along the wire # there will be only a master chunk with no chunk config specified state = chunking.blow_chunks(state) resp = self.Response() resp.game = game resp.user_id = user_id if state is None: resp.state = "" else: resp.state = state return resp
def save(self, user_id, state): """ Save the state for the specified user (but don't release the lock). user_id: string state: dict (not a json string) """ log.debug("uuss.UserState.save %r", user_id) user_id = str(user_id) # Debugging for FB20021 if user_id == 'null': raise Exception('"null" user_id in userstate.save') if not isinstance(state, dict): raise Exception('state not a dict for user_id %s' % user_id) check_user_id(user_id, state, game=self.game) req = proto.SaveRequest() req.game = self.game req.user_id = user_id req.state = chunking.blow_chunks(state, self.chunk_config) self._protocol_send_message(req) resp = self._recv_expected_message(proto.SaveResponse, self.game, user_id)
def do_processing(): load_name = 'user_state-modified-%s-%s' % (bucket, time.time()) mod_ids = open(filename, 'r').readlines() mod_ids = [id.replace('\n', '').strip() for id in mod_ids] log.info("%s ids found to process" % len(mod_ids)) mod_count = 0 batch_count = 1 while mod_ids: proc_ids = mod_ids[:max_batch_size] mod_ids = mod_ids[max_batch_size:] base_file_name = "%s.%s" % (load_name, batch_count) log.info("Writing %s", base_file_name) with open(os.path.join(tmp_dir, base_file_name+".usrtmp"), 'w') as user_file: with open(os.path.join(tmp_dir, base_file_name+".plrtmp"), 'w') as player_file: mod_count += len(proc_ids) for row in proc_ids: try: if filetype == "player": id = row.split(',')[0][2:-1] else: id = row.split("\t")[0][1:-1] state = userstate.backup(id) if isinstance(state, str): state = simplejson.loads(zlib.decompress(state)) raw_state = chunking.blow_chunks(state) user_line = gen_userstate_line(id, raw_state) player_line = gen_player_line(id, state) user_file.write(user_line+'\n') if player_line: player_file.write(player_line+'\n') except userstate_multi.UserstateException, e: log.exception(e) except Exception, e: # (jay) errors are bad here, but we don't want to keep the # rest of the userstates from being saved, so log it and go on log.exception(e)
def do_processing(onerun=False): for collection_name in mongo_db.collection_names(): proc_time = 0 if collection_name.startswith('user_state-modified-%s' % bucket): try: load_name = 'user_state-modified-%s-%s' % (bucket, time.time()) try: mongo_db[collection_name].rename(load_name) except pymongo.errors.OperationFailure: # effectively this makes sure the renamed collection exists if mongo_db[load_name].count() <= 0: raise log.error("Error encountered renaming collection %s to %s, but able to continue" % (collection_name, load_name)) modified = mongo_db[load_name] start_time = time.time() log.info("%s userstates to save for %s ..." % (modified.count(), load_name)) # prepare for userstate size monitoring userstates_processed = 0 max_userstate_size = -1 total_userstate_size = 0 mod_count = 0 mod_rows = list(modified.find()) batch_count = 1 while mod_rows: proc_rows = mod_rows[:max_batch_size] mod_rows = mod_rows[max_batch_size:] base_file_name = "%s.%s" % (load_name, batch_count) with open(os.path.join(tmp_dir, base_file_name+".usrtmp"), 'w') as user_file: with open(os.path.join(tmp_dir, base_file_name+".plrtmp"), 'w') as player_file: for row in proc_rows: mod_count += row['count'] try: if row['_id'] == 'null': log.error("null user_id encountered") else: if rebalancing: (state, chunked) = userstate.get(row['_id'], raw=True) else: (state, chunked) = userstate.backup(row['_id'], raw=True) if state is not None: if not chunked: # NOTE(jpatrin): Shouldn't happen, but just in case... if isinstance(state, str): state = simplejson.loads(zlib.decompress(state)) # NOTE(jpatrin): This will be just a single chunk for now, # but will serve until the state gets saved by the game raw_state = chunking.blow_chunks(state) else: raw_state = state state = chunking.reconstitute_chunks(raw_state, True) #state_zlib_json = zlib.compress(simplejson.dumps(state)) #user_line = gen_userstate_line(row['_id'], state_zlib_json) user_line = gen_userstate_line(row['_id'], raw_state) player_line = gen_player_line(row['_id'], state) user_file.write(user_line+'\n') if player_line: player_file.write(player_line+'\n') # keep userstate size for average and max size tracking userstates_processed += 1 userstate_size = len(raw_state) / 1024 total_userstate_size += userstate_size max_userstate_size = max(userstate_size, max_userstate_size) except userstate_multi.UserstateException, e: log.exception(e) except Exception, e: # (jay) errors are bad here, but we don't want to keep the # rest of the userstates from being saved, so log it and go on log.exception(e) # don't want the file reader to get to these before we're done, so keep # as temporary name until finished writing os.rename(os.path.join(tmp_dir, base_file_name+".usrtmp"), os.path.join(tmp_dir, base_file_name+".user")) os.rename(os.path.join(tmp_dir, base_file_name+".plrtmp"), os.path.join(tmp_dir, base_file_name+".player")) log.info("processed batch, %s remaining" % len(mod_rows)) batch_count += 1
log.debug("other_state: %s...", repr(other_state)[:200]) log.info('Release another userstate lock') req = proto.ReleaseLock() req.game = 'dane' req.user_id = u2 protocol.send_message(req) #RECEIVE LockReleased resp = protocol.recv_message() log.info('Save a userstate') state.setdefault('uuss_test', {'i': 0})['i'] += 1 req = proto.SaveRequest() req.game = 'dane' req.user_id = u1 req.state = chunking.blow_chunks(state) protocol.send_message(req) resp = protocol.recv_message() log.info('Release a userstate lock') req = proto.ReleaseLock() req.game = 'dane' req.user_id = u1 protocol.send_message(req) #RECEIVE LockReleased resp = protocol.recv_message() sock.shutdown(socket.SHUT_RDWR) except Exception, e: if isinstance(e, KeyboardInterrupt): raise