def update(self, events):
'''
Updates the screen when an event happens
@param - list of game events
'''
for event in events:
if not hasattr(event, 'key'):
continue
if event.type == KEYDOWN:
if event.key == K_RETURN:
if self.currLine == GameMenuScreenLine.NewGame:
gui = EnterSaveNameGUI()
save_name = gui.saveName
if len(save_name) <= 0:
continue
load(NEW_GAME_DIR)
State.save_name = save_name
State.push_screen(gameScreen.GameScreen(CURRENT_GAME_DIR))
elif self.currLine == GameMenuScreenLine.LoadGame:
State.push_screen(loadGameScreen.LoadGameScreen())
elif self.currLine == GameMenuScreenLine.Exit:
sys.exit(0)
else:
super(GameMenuScreen, self).interact(event)
def test_load_save_state(self):
filename = 'test_logs/state.dat'
my_id = tc.CLIENT_ID
#TODO: change state
state.save(my_id, [], filename)
#TODO:check file
state.load(filename)
def test_load_save_state(self):
filename = "test_logs/state.dat"
my_id = tc.CLIENT_ID
# TODO: change state
state.save(my_id, [], filename)
# TODO:check file
state.load(filename)
def play(): if state.canload(): state.load() scene.pop() scene.push(gamescene) scene.push(buildscene) else: state.reset() scene.pop() scene.push(gamescene) scene.push(cutscene)
def play(): if state.canload(): state.load() scene.pop() scene.push(gamescene) scene.push(buildscene) else: state.reset() scene.pop() scene.push(gamescene) scene.push(cutscene)
def predict():
saved = state.load('model')
#saved = None
if debug_mode:
saved = None
if saved == None:
train, y, test, _ = data.get()
ftrain, ftest, _ = fea_1.get()
ftrain2, ftest2, _ = fea_2.get()
train = pd.concat([train, ftrain, ftrain2], axis=1)
test = pd.concat([test, ftest, ftest2], axis=1)
print(train.shape, test.shape)
z = pd.DataFrame()
z['id'] = test.id
z['y'] = 0
v = pd.DataFrame()
v['id'] = train.id
v['y'] = y
cv, _ = run(train, y, test, v, z)
state.save('model', (v, z, cv, None))
else:
v, z, cv, _ = saved
return v, z, cv, _
def predict():
saved = state.load('model')
#saved = None
if debug_mode:
saved = None
if saved == None:
train, y, test, _ = data.get()
ftrain, ftest, _ = fea_1.get()
ftrain2, ftest2, _ = fea_2.get()
train = pd.concat([train, ftrain, ftrain2], axis=1)
test = pd.concat([test, ftest, ftest2], axis=1)
print(train.shape, test.shape)
z = pd.DataFrame()
z['id'] = test.id
z['y'] = 0
v = pd.DataFrame()
v['id'] = train.id
v['y'] = y
cv, _ = run(train, y, test, v, z)
state.save('model', (v, z, cv, None))
else:
v, z, cv, _ = saved
return v, z, cv, _
def __init__(self, dht_addr, state_filename,
routing_m_mod, lookup_m_mod,
experimental_m_mod,
private_dht_name):
#TODO: don't do this evil stuff!!!
message.private_dht_name = private_dht_name
if size_estimation:
self._size_estimation_file = open('size_estimation.dat', 'w')
self.state_filename = state_filename
saved_id, saved_bootstrap_nodes = state.load(self.state_filename)
if saved_id:
self._my_id = saved_id
else:
self._my_id = identifier.RandomId()
self._my_node = Node(dht_addr, self._my_id)
self._tracker = tracker.Tracker()
self._token_m = token_manager.TokenManager()
self._querier = Querier()
self._routing_m = routing_m_mod.RoutingManager(self._my_node,
saved_bootstrap_nodes)
self._lookup_m = lookup_m_mod.LookupManager(self._my_id)
self._experimental_m = experimental_m_mod.ExperimentalManager(self._my_node.id)
current_ts = time.time()
self._next_save_state_ts = current_ts + SAVE_STATE_DELAY
self._next_maintenance_ts = current_ts
self._next_timeout_ts = current_ts
self._next_main_loop_call_ts = current_ts
self._pending_lookups = []
def __init__( self ):
"""
Initialise a ScrambleSuitTransport object.
"""
log.error("\n\n################################################\n"
"Do NOT rely on ScrambleSuit for strong security!\n"
"################################################\n")
log.debug("Initialising %s." % const.TRANSPORT_NAME)
super(ScrambleSuitTransport, self).__init__()
# Load the server's persistent state from file.
if self.weAreServer:
self.srvState = state.load()
# Initialise the protocol's state machine.
log.debug("Switching to state ST_WAIT_FOR_AUTH.")
self.protoState = const.ST_WAIT_FOR_AUTH
# Buffer for outgoing data.
self.sendBuf = ""
# Buffer for inter-arrival time obfuscation.
self.choppingBuf = fifobuf.Buffer()
# AES instances to decrypt incoming and encrypt outgoing data.
self.sendCrypter = mycrypto.PayloadCrypter()
self.recvCrypter = mycrypto.PayloadCrypter()
# Packet morpher to modify the protocol's packet length distribution.
self.pktMorpher = packetmorpher.new(self.srvState.pktDist
if self.weAreServer else None)
# Inter-arrival time morpher to obfuscate inter arrival times.
self.iatMorpher = self.srvState.iatDist if self.weAreServer else \
probdist.new(lambda: random.random() %
const.MAX_PACKET_DELAY)
# Used to extract protocol messages from encrypted data.
self.protoMsg = message.MessageExtractor()
# Used by the server-side: `True' if the ticket is already
# decrypted but not yet authenticated.
self.decryptedTicket = False
# If we are in external mode we should already have a shared
# secret set up because of validate_external_mode_cli().
if self.weAreExternal:
assert(self.uniformDHSecret)
if self.weAreClient and not self.weAreExternal:
# As a client in managed mode, we get the shared secret
# from callback `handle_socks_args()' per-connection. Set
# the shared secret to None for now.
self.uniformDHSecret = None
self.uniformdh = uniformdh.new(self.uniformDHSecret, self.weAreServer)
def __enter__(self):
""" Resume or start the session. """
if self.resume and state.load():
state.basic_state.session.resume()
else:
state.basic_state.session = Session(self.show_greeting, self.prog)
self.thread = threading.Thread(target=state.basic_state.session.run,
args=(self.cmd, self.run, self.quit, self.wait))
self.thread.start()
def main():
pygame.init()
pygame.mixer.init()
sound.setvolume(0)
pygame.font.init()
graphics.setmode()
graphics.init()
if settings.playback:
state0, record = cPickle.load(open(data.filepath("record.pkl"), "rb"))
state.loadobj(state0)
scene.push(scenes.game if settings.skipmenu else scenes.menu)
elif settings.restart:
state.init()
scene.push(scenes.game if settings.skipmenu else scenes.menu)
else:
scene.push(scenes.menu)
if state.load():
scene.push(scenes.game)
clock = pygame.time.Clock()
if settings.playback:
for rstate, dt, events, kpress, mpos in record:
clock.tick(settings.fps)
s = scene.top()
random.setstate(rstate)
if settings.vidcap:
vidcap._mpos = mpos
s.think(dt, events, kpress, mpos)
s.draw(mpos)
pygame.display.flip()
return
if settings.getrecord:
record = []
state0 = cPickle.loads(cPickle.dumps(state.saveobj()))
while scene.top():
dt = min(0.001 * clock.tick(settings.fps), 0.5)
if settings.fixfps:
dt = 1.0 / settings.fps
s = scene.top()
while dt > 0:
tdt = min(dt, 0.1)
events = map(EventProxy, pygame.event.get())
kpress = pygame.key.get_pressed()
mpos = pygame.mouse.get_pos()
if settings.getrecord:
record.append((random.getstate(), tdt, events, kpress, mpos))
s.think(tdt, events, kpress, mpos)
dt -= tdt
s.draw(mpos)
pygame.display.flip()
graphics.quit()
if settings.getrecord:
cPickle.dump([state0, record], open(data.filepath("record.pkl"), "wb"))
def update(self, events):
'''
Updates the screen when an event happens
@param - list of game events
'''
for event in events:
if not hasattr(event, 'key'):
continue
if event.type == KEYDOWN:
if event.key == K_RETURN:
if self.currLine == LoadGameScreenLine.ReturnToMainMenu:
State.pop_screen()
continue
save_name = self.lines[self.currLine]
load(USER_SAVES_DIR + save_name)
State.save_name = save_name
State.pop_screen()
State.push_screen(gameScreen.GameScreen(CURRENT_GAME_DIR))
else:
super(LoadGameScreen, self).interact(event)
def setup(cls, transportConfig):
"""
Called once when obfsproxy starts.
"""
log.error(
"\n\n################################################\n"
"Do NOT rely on ScrambleSuit for strong security!\n"
"################################################\n"
)
util.setStateLocation(transportConfig.getStateLocation())
cls.weAreClient = transportConfig.weAreClient
cls.weAreServer = not cls.weAreClient
cls.weAreExternal = transportConfig.weAreExternal
# If we are server and in managed mode, we should get the
# shared secret from the server transport options.
if cls.weAreServer and not cls.weAreExternal:
cfg = transportConfig.getServerTransportOptions()
if cfg and "password" in cfg:
try:
cls.uniformDHSecret = base64.b32decode(util.sanitiseBase32(cfg["password"]))
except (TypeError, AttributeError) as error:
raise base.TransportSetupFailed("Password could not be base32 decoded (%s)" % error)
cls.uniformDHSecret = cls.uniformDHSecret.strip()
if cls.weAreServer:
if not hasattr(cls, "uniformDHSecret"):
log.debug("Using fallback password for descriptor file.")
srv = state.load()
cls.uniformDHSecret = srv.fallbackPassword
if len(cls.uniformDHSecret) != const.SHARED_SECRET_LENGTH:
raise base.TransportSetupFailed(
"Wrong password length (%d instead of %d)" % len(cls.uniformDHSecret), const.SHARED_SECRET_LENGTH
)
if not const.STATE_LOCATION:
raise base.TransportSetupFailed(
"No state location set. If you are using external mode, "
"please set it using the --data-dir switch."
)
state.writeServerPassword(cls.uniformDHSecret)
def setup(cls, transportConfig):
"""
Called once when obfsproxy starts.
"""
log.error("\n\n################################################\n"
"Do NOT rely on ScrambleSuit for strong security!\n"
"################################################\n")
util.setStateLocation(transportConfig.getStateLocation())
cls.weAreClient = transportConfig.weAreClient
cls.weAreServer = not cls.weAreClient
cls.weAreExternal = transportConfig.weAreExternal
# If we are server and in managed mode, we should get the
# shared secret from the server transport options.
if cls.weAreServer and not cls.weAreExternal:
cfg = transportConfig.getServerTransportOptions()
if cfg and "password" in cfg:
try:
cls.uniformDHSecret = base64.b32decode(
util.sanitiseBase32(cfg["password"]))
except (TypeError, AttributeError) as error:
raise base.TransportSetupFailed(
"Password could not be base32 decoded (%s)" % error)
cls.uniformDHSecret = cls.uniformDHSecret.strip()
if cls.weAreServer:
if not hasattr(cls, "uniformDHSecret"):
log.debug("Using fallback password for descriptor file.")
srv = state.load()
cls.uniformDHSecret = srv.fallbackPassword
if len(cls.uniformDHSecret) != const.SHARED_SECRET_LENGTH:
raise base.TransportSetupFailed(
"Wrong password length (%d instead of %d)" %
len(cls.uniformDHSecret), const.SHARED_SECRET_LENGTH)
if not const.STATE_LOCATION:
raise base.TransportSetupFailed(
"No state location set. If you are using external mode, " \
"please set it using the --data-dir switch.")
state.writeServerPassword(cls.uniformDHSecret)
def predict():
saved = state.load('model')
#saved = None
if saved == None:
train, y, test, _ = data.get()
z = pd.DataFrame()
z['id'] = test.id
z['y'] = 0
v = pd.DataFrame()
v['id'] = train.id
v['y'] = y
cv, _ = run(train, y, test, v, z)
state.save('model', (v, z, cv, None))
else:
v, z, cv, _ = saved
return v, z, cv, _
def predict():
saved = state.load('model')
#saved = None
if saved == None:
train, y, test, _ = data.get()
z = pd.DataFrame()
z['id'] = test.id
z['y'] = 0
v = pd.DataFrame()
v['id'] = train.id
v['y'] = y
cv, _ = run(train, y, test, v, z)
state.save('model', (v, z, cv, None))
else:
v, z, cv, _ = saved
return v, z, cv, _
def predict():
saved = state.load('model')
if saved == None:
import l1_1_ho_xgb_1
import l1_1_ho_xgb_2
import l1_3_ho_xgb_1
import l1_3_ho_xgb_2
import l1_3_ho_xgb_3
#import l1_1_keras_1
import l1_1_keras_2
import l1_3_keras_1
import l1_3_keras_2
vs, zs, cvs = [], [], []
for module in [l1_1_ho_xgb_1, l1_1_ho_xgb_2, l1_3_ho_xgb_1, l1_3_ho_xgb_2,
l1_1_keras_2, l1_3_keras_1, l1_3_ho_xgb_3, l1_3_keras_2 ]:
v, z, cv, _ = module.predict()
vs.append(v)
zs.append(z)
cvs.append(cv)
z = pd.DataFrame()
z['id'] = zs[-1].id
z['y'] = 0
v = pd.DataFrame()
v['id'] = vs[-1].id
v['y'] = vs[-1].y
for s in vs + zs:
s.drop(['id', 'y'], axis=1, inplace=True)
train = pd.concat(vs, axis=1)
test = pd.concat(zs, axis=1)
y = v.y
cv, _ = run(train, y, test, v, z)
#state.save('model', (v, z, cv, None))
else:
v, z, cv, _ = saved
return v, z, cv, _
def get_public_server_options( cls, transportOptions ):
"""
Return ScrambleSuit's BridgeDB parameters, i.e., the shared secret.
As a fallback mechanism, we return an automatically generated password
if the bridge operator did not use `ServerTransportOptions'.
"""
log.debug("Tor's transport options: %s" % str(transportOptions))
if not "password" in transportOptions:
log.warning("No password found in transport options (use Tor's " \
"`ServerTransportOptions' to set your own password)." \
" Using automatically generated password instead.")
srv = state.load()
transportOptions = {"password":
base64.b32encode(srv.fallbackPassword)}
cls.uniformDHSecret = srv.fallbackPassword
return transportOptions
def __init__(self, version_label,
my_node, state_filename,
routing_m_mod, lookup_m_mod,
experimental_m_mod,
private_dht_name,
bootstrap_mode):
if size_estimation:
self._size_estimation_file = open('size_estimation.dat', 'w')
self.state_filename = state_filename
saved_id, saved_bootstrap_nodes = state.load(self.state_filename)
my_addr = my_node.addr
self._my_id = my_node.id # id indicated by user
if not self._my_id:
self._my_id = saved_id # id loaded from file
if not self._my_id:
self._my_id = self._my_id = identifier.RandomId() # random id
self._my_node = Node(my_addr, self._my_id, version=version_label)
self.msg_f = message.MsgFactory(version_label, self._my_id,
private_dht_name)
self._querier = Querier()
self._routing_m = routing_m_mod.RoutingManager(
self._my_node, saved_bootstrap_nodes, self.msg_f)
self._responder = responder.Responder(self._my_id, self._routing_m,
self.msg_f, bootstrap_mode)
self._tracker = self._responder._tracker
self._lookup_m = lookup_m_mod.LookupManager(self._my_id, self.msg_f)
self._experimental_m = experimental_m_mod.ExperimentalManager(
self._my_node.id, self.msg_f)
current_ts = time.time()
self._next_save_state_ts = current_ts + SAVE_STATE_DELAY
self._next_maintenance_ts = current_ts
self._next_timeout_ts = current_ts
self._next_main_loop_call_ts = current_ts
self._pending_lookups = []
self._cached_lookups = {}
def __init__(self, version_label,
my_node, state_filename,
routing_m_mod, lookup_m_mod,
experimental_m_mod,
private_dht_name,
bootstrap_mode):
if size_estimation:
self._size_estimation_file = open('size_estimation.dat', 'w')
self.state_filename = state_filename
saved_id, saved_bootstrap_nodes = state.load(self.state_filename)
my_addr = my_node.addr
self._my_id = my_node.id # id indicated by user
if not self._my_id:
self._my_id = saved_id # id loaded from file
if not self._my_id:
self._my_id = self._my_id = identifier.RandomId() # random id
self._my_node = Node(my_addr, self._my_id, version=version_label)
self.msg_f = message.MsgFactory(version_label, self._my_id,
private_dht_name)
self._querier = Querier()
self._routing_m = routing_m_mod.RoutingManager(
self._my_node, saved_bootstrap_nodes, self.msg_f)
self._responder = responder.Responder(self._my_id, self._routing_m,
self.msg_f, bootstrap_mode)
self._tracker = self._responder._tracker
self._lookup_m = lookup_m_mod.LookupManager(self._my_id, self.msg_f)
self._experimental_m = experimental_m_mod.ExperimentalManager(
self._my_node.id, self.msg_f)
current_ts = time.time()
self._next_save_state_ts = current_ts + SAVE_STATE_DELAY
self._next_maintenance_ts = current_ts
self._next_timeout_ts = current_ts
self._next_main_loop_call_ts = current_ts
self._pending_lookups = []
self._cached_lookups = []
def predict():
saved = state.load('model')
#saved = None
if saved == None:
import l1_1_ho_xgb_1
import l1_1_keras_1
import l1_1_ho_xgb_2
import l1_1_keras_2
vs, zs, cvs = [], [], []
for module in [
l1_1_ho_xgb_1, l1_1_ho_xgb_2, l1_1_keras_1, l1_1_keras_2
]:
v, z, cv, _ = module.predict()
vs.append(v)
zs.append(z)
cvs.append(cv)
z = pd.DataFrame()
z['id'] = zs[-1].id
z['y'] = 0
v = pd.DataFrame()
v['id'] = vs[-1].id
v['y'] = vs[-1].y
for s in vs + zs:
s.drop(['id', 'y'], axis=1, inplace=True)
train = pd.concat(vs, axis=1)
test = pd.concat(zs, axis=1)
y = v.y
cv, _ = run(train, y, test, v, z)
#state.save('model', (v, z, cv, None))
else:
v, z, cv, _ = saved
return v, z, cv, _
def setup( cls, transportConfig ):
"""
Called once when obfsproxy starts.
"""
util.setStateLocation(transportConfig.getStateLocation())
cls.weAreClient = transportConfig.weAreClient
cls.weAreServer = not cls.weAreClient
cls.weAreExternal = transportConfig.weAreExternal
# If we are server and in managed mode, we should get the
# shared secret from the server transport options.
if cls.weAreServer and not cls.weAreExternal:
cfg = transportConfig.getServerTransportOptions()
if cfg and "password" in cfg:
try:
cls.uniformDHSecret = base64.b32decode(util.sanitiseBase32(
cfg["password"]))
except TypeError as error:
log.error(error.message)
raise base.PluggableTransportError("Given password '%s' " \
"is not valid Base32! Run " \
"'generate_password.py' to generate a good " \
"password." % cfg["password"])
cls.uniformDHSecret = cls.uniformDHSecret.strip()
if cls.weAreServer:
if not hasattr(cls, "uniformDHSecret"):
log.debug("Using fallback password for descriptor file.")
srv = state.load()
cls.uniformDHSecret = srv.fallbackPassword
state.writeServerDescriptor(cls.uniformDHSecret,
transportConfig.getBindAddr(),
cls.weAreExternal)
# Random wait if we are a batch job
import time
if not HYPERPARAMETERS["console"]:
wait = 100 * random.random()
print >> sys.stderr, "Waiting %f seconds..." % wait
time.sleep(wait)
# import vocabulary
## logging.info("Reading vocab")
## vocabulary.read()
#
import model
try:
print >> sys.stderr, ("Trying to read training state for %s %s..." % (newkeystr, rundir))
(translation_model, cnt, lastcnt, epoch) = state.load(rundir, newkeystr)
print >> sys.stderr, ("...success reading training state for %s %s" % (newkeystr, rundir))
print >> sys.stderr, logfile
print >> sys.stderr, "CONTINUING FROM TRAINING STATE"
except IOError:
print >> sys.stderr, ("...FAILURE reading training state for %s %s" % (newkeystr, rundir))
print >> sys.stderr, ("INITIALIZING")
translation_model = {}
print >> sys.stderr, "Loading initial embeddings from %s" % HYPERPARAMETERS["INITIAL_EMBEDDINGS"]
# TODO: If we want more than one model, we should SHARE the embeddings parameters
embeddings = cPickle.load(common.file.myopen(HYPERPARAMETERS["INITIAL_EMBEDDINGS"]))
print >> sys.stderr, "INITIALIZING TRAINING STATE"
all_l1 = {}
def run(train, y, test, v, z):
#cname = sys._getframe().f_code.co_name
cname = 'p'
train.drop('id', axis=1, inplace=True)
test.drop('id', axis=1, inplace=True)
from hyperopt import fmin, tpe, hp, STATUS_OK, Trials, space_eval
dtrain = xgb.DMatrix(train, y)
def step_xgb(params):
cv = xgb.cv(params=params,
dtrain=dtrain,
num_boost_round=10000,
early_stopping_rounds=50,
nfold=10,
seed=params['seed'])
score = cv.ix[len(cv)-1, 0]
print(cname, score, len(cv), params)
return dict(loss=score, status=STATUS_OK)
space_xgb = dict(
max_depth = hp.choice('max_depth', range(2, 8)),
subsample = hp.quniform('subsample', 0.6, 1, 0.05),
colsample_bytree = hp.quniform('colsample_bytree', 0.6, 1, 0.05),
learning_rate = hp.quniform('learning_rate', 0.005, 0.03, 0.005),
min_child_weight = hp.quniform('min_child_weight', 1, 6, 1),
gamma = hp.quniform('gamma', 0.5, 10, 0.05),
objective = 'binary:logistic',
eval_metric = 'logloss',
seed = 1,
silent = 1
)
trs = state.load('xgb_trials')
if trs == None:
tr = Trials()
else:
tr, _ = trs
if len(tr.trials) > 0:
print('reusing %d trials, best was:'%(len(tr.trials)), space_eval(space_xgb, tr.argmin))
best = tr.argmin
while len(tr.trials) < 15:
best = fmin(step_xgb, space_xgb, algo=tpe.suggest, max_evals=len(tr.trials) + 1, trials = tr)
state.save('xgb_trials', (tr, space_xgb))
xgb_params = space_eval(space_xgb, best)
print(xgb_params)
N_splits = 9
N_seeds = 1
skf = model_selection.StratifiedKFold(n_splits=N_splits, shuffle=True)
dtest = xgb.DMatrix(test)
for s in range(N_seeds):
scores = []
cname2 = cname + str(s)
v[cname2], z[cname2] = 0, 0
xgb_params['seed'] = s + 4242
for n, (itrain, ival) in enumerate(skf.split(train, y)):
dtrain = xgb.DMatrix(train.ix[itrain], y[itrain])
dvalid = xgb.DMatrix(train.ix[ival], y[ival])
watch = [(dtrain, 'train'), (dvalid, 'valid')]
clf = xgb.train(xgb_params, dtrain, 10000, watch, early_stopping_rounds=100, verbose_eval=False)
p = clf.predict(dvalid)
v.loc[ival, cname2] += p
score = metrics.log_loss(y[ival], p)
z[cname2] += clf.predict(dtest)
print(cname, 'seed %d step %d of %d: '%(xgb_params['seed'], n+1, skf.n_splits), score, state.now())
scores.append(score)
z[cname2] /= N_splits
cv = scores
z['y'] = z[cname2]
print('validation loss: ', cv, np.mean(cv), np.std(cv))
return cv, None
def test2_loadState( self ):
# load() should create the state file if it doesn't exist yet.
self.failIf(os.path.exists(self.stateFile))
self.failUnless(isinstance(state.load(), state.State))
def start_basic():
""" Load & run programs and commands and hand over to interactive mode. """
import program
import run
import error
import state
import devices
import disk
import cassette
import reset
import sound
import audio
do_reset = False
backend, console = None, None
exit_error = ''
try:
# resume from saved emulator state if requested and available
resume = config.get('resume') and state.load()
# choose the video and sound backends
backend, console = prepare_console()
# greet, load and run only if not resuming
if resume:
# override selected settings from command line
cassette.override()
disk.override()
# suppress double prompt
if not state.basic_state.execute_mode:
state.basic_state.prompt = False
run.start('', False, config.get('quit'))
else:
# load/run program
config.options['run'] = config.get(1) or config.get('run')
prog = config.get('run') or config.get('load')
if prog:
# on load, accept capitalised versions and default extension
with open_native_or_dos_filename(prog) as progfile:
program.load(progfile)
reset.clear()
print_greeting(console)
# start the interpreter (and get out if we ran with -q)
run.start(config.get('exec'), config.get('run'), config.get('quit'))
except error.RunError as e:
exit_error = error.get_message(e.err)
except error.Exit:
# pause before exit if requested
if config.get('wait'):
backend.video_queue.put(backend.Event(backend.VIDEO_SET_CAPTION, 'Press a key to close window'))
backend.video_queue.put(backend.Event(backend.VIDEO_SHOW_CURSOR, False))
state.console_state.keyb.pause = True
# this performs a blocking keystroke read if in pause state
backend.check_events()
except error.Reset:
do_reset = True
except KeyboardInterrupt:
if config.get('debug'):
raise
except Exception as e:
print traceback.print_exc()
exit_error = "Unhandled exception\n%s" % traceback.format_exc()
finally:
try:
audio.close()
except (NameError, AttributeError) as e:
logging.debug('Error on closing audio: %s', e)
try:
# fix the terminal on exit (important for ANSI terminals)
# and save display interface state into screen state
state.console_state.screen.close()
except (NameError, AttributeError) as e:
logging.debug('Error on closing screen: %s', e)
# delete state if resetting
if do_reset:
state.delete()
if plat.system == 'Android':
shutil.rmtree(plat.temp_dir)
else:
state.save()
try:
# close files if we opened any
devices.close_files()
except (NameError, AttributeError) as e:
logging.debug('Error on closing files: %s', e)
try:
devices.close_devices()
except (NameError, AttributeError) as e:
logging.debug('Error on closing devices: %s', e)
if exit_error:
logging.error(exit_error)
def run(train, y, test, v, z):
np.random.seed(1)
#cname = sys._getframe().f_code.co_name
train = train.values
test = test.values
from hyperopt import fmin, tpe, hp, STATUS_OK, Trials, space_eval
space_stack = hp.choice('stacking by', [
dict(type='BayesianRidge'),
dict(type='Lars'),
dict(type='LinearRegression'),
dict(type='Ridge'),
dict(type='SGDRegressor', random_state=1),
dict(
type='XGBRegressor',
max_depth=hp.choice('max_depth', range(2, 8)),
subsample=hp.quniform('subsample', 0.6, 1, 0.05),
colsample_bytree=hp.quniform('colsample_bytree', 0.6, 1, 0.05),
learning_rate=hp.quniform('learning_rate', 0.005, 0.03, 0.005),
min_child_weight=hp.quniform('min_child_weight', 1, 6, 1),
gamma=hp.quniform('gamma', 0, 10, 0.05),
reg_alpha=hp.quniform('alpha', 0, 1, 0.0001),
),
])
def get_lr(params):
t = params['type']
del params['type']
if t == 'BayesianRidge':
lr = linear_model.BayesianRidge(**params)
elif t == 'Lars':
lr = linear_model.Lars(**params)
elif t == 'LinearRegression':
lr = linear_model.LinearRegression(**params)
elif t == 'Ridge':
lr = linear_model.Ridge(**params)
elif t == 'SGDRegressor':
lr = linear_model.SGDRegressor(**params)
elif t == 'XGBRegressor':
lr = xgb.XGBRegressor(**params)
return lr
def step(params):
print(params, end=' ')
cv = model_selection.cross_val_score(get_lr(params),
train,
y,
cv=10,
scoring=metrics.make_scorer(
metrics.log_loss))
score = np.mean(cv)
print(score)
return dict(loss=score, status=STATUS_OK)
trs = state.load('trials')
if trs == None:
tr = Trials()
else:
tr, _ = trs
if len(tr.trials) > 0:
best = tr.argmin
print('reusing %d trials, best was:' % (len(tr.trials)),
space_eval(space_stack, best))
mt = max(50, len(tr.trials) + 1)
while len(tr.trials) < min(50, mt):
best = fmin(step,
space_stack,
algo=tpe.suggest,
max_evals=len(tr.trials) + 1,
trials=tr)
state.save('trials', (tr, space_stack))
params = space_eval(space_stack, best)
print('best params:', params)
lr = get_lr(params)
cv = model_selection.cross_val_score(lr,
train,
y,
cv=10,
scoring=metrics.make_scorer(
metrics.log_loss))
lr.fit(train, y)
z['p'] = np.clip(lr.predict(test), 1e-5, 1 - 1e-5)
z['y'] = z['p']
v['p'] = model_selection.cross_val_predict(lr, train, y, cv=10)
print('cv:', np.mean(cv), np.std(cv))
return cv, None
def start_basic():
""" Load & run programs and commands and hand over to interactive mode. """
import program
import run
import error
import state
import devices
import disk
import cassette
import reset
import sound
do_reset = False
backend, console = None, None
try:
# resume from saved emulator state if requested and available
resume = config.get('resume') and state.load()
# choose the video and sound backends
backend, console = prepare_console()
# greet, load and run only if not resuming
if resume:
# override selected settings from command line
cassette.override()
disk.override()
# suppress double prompt
if not state.basic_state.execute_mode:
state.basic_state.prompt = False
run.start('', False, config.get('quit'))
else:
# load/run program
config.options['run'] = config.get(0) or config.get('run')
prog = config.get('run') or config.get('load')
if prog:
# on load, accept capitalised versions and default extension
with open_native_or_dos_filename(prog) as progfile:
program.load(progfile)
reset.clear()
print_greeting(console)
# start the interpreter (and get out if we ran with -q)
run.start(config.get('exec'), config.get('run'),
config.get('quit'))
except error.RunError as e:
msg = error.get_message(e.err)
if console and config.get('wait'):
console.write_error_message(msg, None)
if backend and backend.video:
# close terminal to avoid garbled error message
backend.video.close()
backend.video = None
logging.error(msg)
except error.Exit:
pass
except error.Reset:
do_reset = True
except KeyboardInterrupt:
if config.get('debug'):
raise
except Exception as e:
logging.error("Unhandled exception\n%s", traceback.format_exc())
finally:
try:
# fix the terminal on exit (important for ANSI terminals)
# and save display interface state into screen state
state.console_state.screen.close()
except (NameError, AttributeError) as e:
logging.debug('Error on closing screen: %s', e)
# delete state if resetting
if do_reset:
state.delete()
if plat.system == 'Android':
shutil.rmtree(plat.temp_dir)
else:
state.save()
try:
# close files if we opened any
devices.close_files()
except (NameError, AttributeError) as e:
logging.debug('Error on closing files: %s', e)
try:
devices.close_devices()
except (NameError, AttributeError) as e:
logging.debug('Error on closing devices: %s', e)
try:
sound.audio.close()
except (NameError, AttributeError) as e:
logging.debug('Error on closing audio: %s', e)
def __init__( self, transportConfig ):
"""
Initialise a ScrambleSuitTransport object.
"""
log.error("\n\n################################################\n"
"Do NOT rely on ScrambleSuit for strong security!\n"
"################################################\n")
log.debug("Initialising %s." % const.TRANSPORT_NAME)
util.setStateLocation(transportConfig.getStateLocation())
# Load the server's persistent state from file.
if self.weAreServer:
self.srvState = state.load()
# Initialise the protocol's state machine.
log.debug("Switching to state ST_WAIT_FOR_AUTH.")
self.protoState = const.ST_WAIT_FOR_AUTH
# Buffers for incoming and outgoing data.
self.sendBuf = self.recvBuf = ""
# Buffer for inter-arrival time obfuscation.
self.choppingBuf = ""
# AES instances to decrypt incoming and encrypt outgoing data.
self.sendCrypter = mycrypto.PayloadCrypter()
self.recvCrypter = mycrypto.PayloadCrypter()
# Packet morpher to modify the protocol's packet length distribution.
self.pktMorpher = packetmorpher.new(self.srvState.pktDist
if self.weAreServer else None)
# Inter-arrival time morpher to obfuscate inter arrival times.
self.iatMorpher = self.srvState.iatDist if self.weAreServer else \
probdist.new(lambda: random.random() %
const.MAX_PACKET_DELAY)
if self.weAreServer:
# `True' if the ticket is already decrypted but not yet
# authenticated.
self.decryptedTicket = False
if not hasattr(self, 'uniformDHSecret'):
# As the server, we get the shared secret from the constructor.
cfg = transportConfig.getServerTransportOptions()
self.uniformDHSecret = base64.b32decode(cfg["password"])
self.uniformDHSecret = self.uniformDHSecret.strip()
else:
# As the client, we get the shared secret from obfsproxy calling
# `handle_socks_args()'.
if not hasattr(self, 'uniformDHSecret'):
self.uniformDHSecret = None
self.uniformdh = uniformdh.new(self.uniformDHSecret, self.weAreServer)
# Variables used to unpack protocol messages.
self.totalLen = self.payloadLen = self.flags = None
def run(state, train, y, test, v, z):
#cname = sys._getframe().f_code.co_name
cname = 'p'
train.drop('id', axis=1, inplace=True)
test.drop('id', axis=1, inplace=True)
from hyperopt import fmin, tpe, hp, STATUS_OK, Trials, space_eval
def step_et(params):
clf = ensemble.ExtraTreesRegressor(**params)
cv = model_selection.cross_val_score(clf,
train,
y,
scoring=metrics.make_scorer(
metrics.log_loss),
cv=10,
n_jobs=-2)
score = np.mean(cv)
print(cname, score, params)
return dict(loss=score, status=STATUS_OK)
space_et = dict(
n_estimators=hp.choice('n_estimators', range(50, 1500)),
#criterion = hp.choice('criterion', ["gini", "entropy"]),
min_samples_split=hp.choice('min_samples_split', range(2, 10)),
min_samples_leaf=hp.choice('min_samples_leaf', range(1, 10)),
max_features=hp.choice('max_features', range(1, 16)),
random_state=1)
trs = state.load('et_trials')
if trs == None or debug_mode:
tr = Trials()
else:
tr, _ = trs
if len(tr.trials) > 0:
print('reusing %d trials, best was:' % (len(tr.trials)),
space_eval(space_et, tr.argmin))
best = tr.argmin
while len(tr.trials) < 15:
best = fmin(step_et,
space_et,
algo=tpe.suggest,
max_evals=len(tr.trials) + 1,
trials=tr)
state.save('et_trials', (tr, space_et))
et_params = space_eval(space_et, best)
print(et_params)
N_splits = 9
N_seeds = 3
skf = model_selection.StratifiedKFold(n_splits=N_splits, shuffle=True)
cv = []
for s in range(N_seeds):
scores = []
cname2 = cname + str(s)
v[cname2], z[cname2] = 0, 0
et_params['random_state'] = s + 4242
for n, (itrain, ival) in enumerate(skf.split(train, y)):
clf = ensemble.ExtraTreesRegressor(**et_params)
clf.fit(train.ix[itrain], y[itrain])
p = clf.predict(train.ix[ival])
v.loc[ival, cname2] += p
score = metrics.log_loss(y[ival], p)
z[cname2] += clf.predict(test)
print(
cname, 'seed %d step %d of %d: ' %
(et_params['random_state'], n + 1, skf.n_splits), score,
state.now())
scores.append(score)
z[cname2] /= N_splits
cv.append(np.mean(scores))
print('seed %d loss: ' % (et_params['random_state']), scores,
np.mean(scores), np.std(scores))
z['y'] = z[cname2]
print('cv:', cv, np.mean(cv), np.std(cv))
return cv, None
def start_basic():
""" Load & run programs and commands and hand over to interactive mode. """
import program
import run
import error
import state
import devices
import disk
import cassette
import reset
import sound
do_reset = False
backend, console = None, None
try:
# resume from saved emulator state if requested and available
resume = config.get('resume') and state.load()
# choose the video and sound backends
backend, console = prepare_console()
# greet, load and run only if not resuming
if resume:
# override selected settings from command line
cassette.override()
disk.override()
# suppress double prompt
if not state.basic_state.execute_mode:
state.basic_state.prompt = False
run.start('', False, config.get('quit'))
else:
# load/run program
config.options['run'] = config.get(0) or config.get('run')
prog = config.get('run') or config.get('load')
if prog:
# on load, accept capitalised versions and default extension
with open_native_or_dos_filename(prog) as progfile:
program.load(progfile)
reset.clear()
print_greeting(console)
# start the interpreter (and get out if we ran with -q)
run.start(config.get('exec'), config.get('run'), config.get('quit'))
except error.RunError as e:
msg = error.get_message(e.err)
if console and config.get('wait'):
console.write_error_message(msg, None)
if backend and backend.video:
# close terminal to avoid garbled error message
backend.video.close()
backend.video = None
logging.error(msg)
except error.Exit:
pass
except error.Reset:
do_reset = True
except KeyboardInterrupt:
if config.get('debug'):
raise
except Exception as e:
logging.error("Unhandled exception\n%s", traceback.format_exc())
finally:
try:
# fix the terminal on exit (important for ANSI terminals)
# and save display interface state into screen state
state.console_state.screen.close()
except (NameError, AttributeError) as e:
logging.debug('Error on closing screen: %s', e)
# delete state if resetting
if do_reset:
state.delete()
if plat.system == 'Android':
shutil.rmtree(plat.temp_dir)
else:
state.save()
try:
# close files if we opened any
devices.close_files()
except (NameError, AttributeError) as e:
logging.debug('Error on closing files: %s', e)
try:
devices.close_devices()
except (NameError, AttributeError) as e:
logging.debug('Error on closing devices: %s', e)
try:
sound.audio.close()
except (NameError, AttributeError) as e:
logging.debug('Error on closing audio: %s', e)
# Give ScrambleSuit server operators a way to manually issue new session
# tickets for out-of-band distribution.
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("ip_addr", type=str, help="The IPv4 address of the "
"%s server." % const.TRANSPORT_NAME)
parser.add_argument("tcp_port", type=int, help="The TCP port of the %s "
"server." % const.TRANSPORT_NAME)
parser.add_argument("ticket_file", type=str, help="The file, the newly "
"issued ticket is written to.")
args = parser.parse_args()
print "[+] Loading server state file."
serverState = state.load()
print "[+] Generating new session ticket."
masterKey = mycrypto.strongRandom(const.MASTER_KEY_LENGTH)
ticket = SessionTicket(masterKey, serverState).issue()
print "[+] Writing new session ticket to `%s'." % args.ticket_file
tickets = dict()
server = IPv4Address('TCP', args.ip_addr, args.tcp_port)
tickets[str(server)] = [int(time.time()), masterKey, ticket]
util.writeToFile(yaml.dump(tickets), args.ticket_file)
print "[+] Success."
def run(state, train, y, test, v, z):
#cname = sys._getframe().f_code.co_name
cname = 'p'
train.drop('id', axis=1, inplace=True)
test.drop('id', axis=1, inplace=True)
from hyperopt import fmin, tpe, hp, STATUS_OK, Trials, space_eval
def step_rf(params):
clf = ensemble.RandomForestRegressor(**params)
cv = model_selection.cross_val_score(clf,
train, y,
scoring=metrics.make_scorer(metrics.log_loss),
cv = 10,
n_jobs = -2)
score = np.mean(cv)
print(cname, score, params)
return dict(loss=score, status=STATUS_OK)
space_rf = dict(
n_estimators = hp.choice('n_estimators', range(50, 1500)),
#criterion = hp.choice('criterion', ["gini", "entropy"]),
min_samples_split = hp.choice('min_samples_split', range(2, 10)),
min_samples_leaf = hp.choice('min_samples_leaf', range(1, 10)),
max_features = hp.choice('max_features', range(1, 16)),
random_state = 1
)
trs = state.load('rf_trials')
if trs == None or debug_mode:
tr = Trials()
else:
tr, _ = trs
if len(tr.trials) > 0:
print('reusing %d trials, best was:'%(len(tr.trials)), space_eval(space_rf, tr.argmin))
best = tr.argmin
while len(tr.trials) < 15:
best = fmin(step_rf, space_rf, algo=tpe.suggest, max_evals=len(tr.trials) + 1, trials = tr)
state.save('et_trials', (tr, space_rf))
rf_params = space_eval(space_rf, best)
print(rf_params)
N_splits = 9
N_seeds = 3
skf = model_selection.StratifiedKFold(n_splits=N_splits, shuffle=True)
cv = []
for s in range(N_seeds):
scores = []
cname2 = cname + str(s)
v[cname2], z[cname2] = 0, 0
rf_params['random_state'] = s + 4242
for n, (itrain, ival) in enumerate(skf.split(train, y)):
clf = ensemble.RandomForestRegressor(**rf_params)
clf.fit(train.ix[itrain], y[itrain])
p = clf.predict(train.ix[ival])
v.loc[ival, cname2] += p
score = metrics.log_loss(y[ival], p)
z[cname2] += clf.predict(test)
print(cname, 'seed %d step %d of %d: '%(rf_params['random_state'], n+1, skf.n_splits), score, state.now())
scores.append(score)
z[cname2] /= N_splits
cv.append(np.mean(scores))
print('seed %d loss: '%(rf_params['random_state']), scores, np.mean(scores), np.std(scores))
z['y'] = z[cname2]
print('cv:', cv, np.mean(cv), np.std(cv))
return cv, None
os.system("ln -s log %s " % (verboselogfile))
else:
print >> sys.stderr, "Logging to %s, not creating any link because of default settings" % logfile
import random, numpy
random.seed(miscglobals.RANDOMSEED)
numpy.random.seed(miscglobals.RANDOMSEED)
import vocabulary
# logging.info("Reading vocab")
# vocabulary.read()
import model
try:
print >> sys.stderr, ("Trying to read training state for %s %s..." % (newkeystr, rundir))
(m, cnt, epoch, get_train_minibatch) = state.load(rundir, newkeystr)
print >> sys.stderr, ("...success reading training state for %s %s" % (newkeystr, rundir))
print >> sys.stderr, logfile
logging.basicConfig(filename=logfile, level=logging.DEBUG)
# logging.basicConfig(filename=logfile, filemode="w", level=logging.DEBUG)
logging.info("CONTINUING FROM TRAINING STATE")
except IOError:
print >> sys.stderr, ("...FAILURE reading training state for %s %s" % (newkeystr, rundir))
print >> sys.stderr, ("INITIALIZING")
m = model.Model()
cnt = 0
epoch = 1
get_train_minibatch = examples.TrainingMinibatchStream()
logging.basicConfig(filename=logfile, filemode="w", level=logging.DEBUG)
logging.info("INITIALIZING TRAINING STATE")
def run(train, y, test, v, z):
np.random.seed(1)
#cname = sys._getframe().f_code.co_name
train = train.values
test = test.values
from hyperopt import fmin, tpe, hp, STATUS_OK, Trials, space_eval
space_stack = hp.choice('stacking by', [
dict( type = 'BayesianRidge' ),
dict( type = 'Lars' ),
dict( type = 'LinearRegression' ),
dict( type = 'Ridge' ),
dict( type = 'SGDRegressor', random_state = 1 ),
dict( type = 'XGBRegressor',
max_depth = hp.choice('max_depth', range(2, 8)),
subsample = hp.quniform('subsample', 0.6, 1, 0.05),
colsample_bytree = hp.quniform('colsample_bytree', 0.6, 1, 0.05),
learning_rate = hp.quniform('learning_rate', 0.005, 0.03, 0.005),
min_child_weight = hp.quniform('min_child_weight', 1, 6, 1),
gamma = hp.quniform('gamma', 0, 10, 0.05),
reg_alpha = hp.quniform('alpha', 0, 1, 0.0001),
),
])
def get_lr(params):
t = params['type']
del params['type']
if t == 'BayesianRidge':
lr = linear_model.BayesianRidge(**params)
elif t == 'Lars':
lr = linear_model.Lars(**params)
elif t == 'LinearRegression':
lr = linear_model.LinearRegression(**params)
elif t == 'Ridge':
lr = linear_model.Ridge(**params)
elif t == 'SGDRegressor':
lr = linear_model.SGDRegressor(**params)
elif t == 'XGBRegressor':
lr = xgb.XGBRegressor(**params)
return lr
def step(params):
print(params, end = ' ')
cv = model_selection.cross_val_score(get_lr(params),
train, y,
cv=10,
scoring=metrics.make_scorer(metrics.log_loss))
score = np.mean(cv)
print(score)
return dict(loss=score, status=STATUS_OK)
trs = state.load('trials')
if trs == None:
tr = Trials()
else:
tr, _ = trs
if len(tr.trials) > 0:
best = tr.argmin
print('reusing %d trials, best was:'%(len(tr.trials)), space_eval(space_stack, best))
mt = max(50, len(tr.trials) + 1)
while len(tr.trials) < min(50, mt):
best = fmin(step, space_stack, algo=tpe.suggest, max_evals=len(tr.trials) + 1, trials = tr)
state.save('trials', (tr, space_stack))
params = space_eval(space_stack, best)
print('best params:', params)
lr = get_lr(params)
cv = model_selection.cross_val_score(lr,
train, y,
cv=10,
scoring=metrics.make_scorer(metrics.log_loss))
lr.fit(train, y)
z['p'] = np.clip(lr.predict(test), 1e-5, 1-1e-5)
z['y'] = z['p']
v['p'] = model_selection.cross_val_predict(lr,
train, y,
cv=10)
print('cv:', np.mean(cv), np.std(cv))
return cv, None
os.system("ln -s log %s " % (verboselogfile))
else:
print >> sys.stderr, "Logging to %s, not creating any link because of default settings" % logfile
import random, numpy
random.seed(miscglobals.RANDOMSEED)
numpy.random.seed(miscglobals.RANDOMSEED)
import vocabulary
# logging.info("Reading vocab")
# vocabulary.read()
import model
try:
print >> sys.stderr, ("Trying to read training state for %s %s..." % (newkeystr, rundir))
(m, cnt, epoch, get_train_minibatch) = state.load(rundir, newkeystr)
print >> sys.stderr, ("...success reading training state for %s %s" % (newkeystr, rundir))
print >> sys.stderr, logfile
logging.basicConfig(filename=logfile, level=logging.DEBUG)
# logging.basicConfig(filename=logfile, filemode="w", level=logging.DEBUG)
logging.info("CONTINUING FROM TRAINING STATE")
except IOError:
print >> sys.stderr, ("...FAILURE reading training state for %s %s" % (newkeystr, rundir))
print >> sys.stderr, ("INITIALIZING")
m = model.Model()
cnt = 0
epoch = 1
get_train_minibatch = examples.TrainingMinibatchStream()
logging.basicConfig(filename=logfile, filemode="w", level=logging.DEBUG)
logging.info("INITIALIZING TRAINING STATE")
def run(train, y, test, v, z):
#cname = sys._getframe().f_code.co_name
cname = 'p'
train.drop('id', axis=1, inplace=True)
test.drop('id', axis=1, inplace=True)
from hyperopt import fmin, tpe, hp, STATUS_OK, Trials, space_eval
dtrain = xgb.DMatrix(train, y)
def step_xgb(params):
cv = xgb.cv(params=params,
dtrain=dtrain,
num_boost_round=10000,
early_stopping_rounds=50,
nfold=10,
seed=params['seed'])
score = cv.ix[len(cv) - 1, 0]
print(cname, score, len(cv), params)
return dict(loss=score, status=STATUS_OK)
space_xgb = dict(max_depth=hp.choice('max_depth', range(2, 9)),
subsample=hp.quniform('subsample', 0.6, 1, 0.05),
colsample_bytree=hp.quniform('colsample_bytree', 0.6, 1,
0.05),
learning_rate=hp.quniform('learning_rate', 0.005, 0.1,
0.005),
min_child_weight=hp.quniform('min_child_weight', 1, 6, 1),
gamma=hp.quniform('gamma', 0.5, 10, 0.05),
reg_alpha=hp.quniform('reg_alpha', 0, 1, 0.001),
objective='binary:logistic',
eval_metric='logloss',
seed=1,
silent=1)
trs = state.load('xgb_trials')
if trs == None or debug_mode:
tr = Trials()
else:
tr, _ = trs
if len(tr.trials) > 0:
print('reusing %d trials, best was:' % (len(tr.trials)),
space_eval(space_xgb, tr.argmin))
best = tr.argmin
while len(tr.trials) < 15:
best = fmin(step_xgb,
space_xgb,
algo=tpe.suggest,
max_evals=len(tr.trials) + 1,
trials=tr)
state.save('xgb_trials', (tr, space_xgb))
xgb_params = space_eval(space_xgb, best)
print(xgb_params)
N_splits = 9
N_seeds = 3
skf = model_selection.StratifiedKFold(n_splits=N_splits, shuffle=True)
dtest = xgb.DMatrix(test)
cv = []
for s in range(N_seeds):
scores = []
cname2 = cname + str(s)
v[cname2], z[cname2] = 0, 0
xgb_params['seed'] = s + 4242
for n, (itrain, ival) in enumerate(skf.split(train, y)):
dtrain = xgb.DMatrix(train.ix[itrain], y[itrain])
dvalid = xgb.DMatrix(train.ix[ival], y[ival])
watch = [(dtrain, 'train'), (dvalid, 'valid')]
clf = xgb.train(xgb_params,
dtrain,
10000,
watch,
early_stopping_rounds=100,
verbose_eval=False)
p = clf.predict(dvalid)
v.loc[ival, cname2] += p
score = metrics.log_loss(y[ival], p)
z[cname2] += clf.predict(dtest)
print(
cname, 'seed %d step %d of %d: ' %
(xgb_params['seed'], n + 1, skf.n_splits), score, state.now())
scores.append(score)
z[cname2] /= N_splits
cv.append(np.mean(scores))
print('seed %d loss: ' % (xgb_params['seed']), scores, np.mean(scores),
np.std(scores))
z['y'] = z[cname2]
print('cv:', cv, np.mean(cv), np.std(cv))
return cv, None
def start_basic():
""" Load & run programs and commands and hand over to interactive mode. """
import program
import run
import error
import state
import devices
import disk
import cassette
import reset
import sound
import audio
do_reset = False
backend, console = None, None
exit_error = ''
try:
# resume from saved emulator state if requested and available
resume = config.get('resume') and state.load()
# choose the video and sound backends
backend, console = prepare_console()
# greet, load and run only if not resuming
if resume:
# override selected settings from command line
cassette.override()
disk.override()
# suppress double prompt
if not state.basic_state.execute_mode:
state.basic_state.prompt = False
run.start('', False, config.get('quit'))
else:
# load/run program
config.options['run'] = config.get(0) or config.get('run')
prog = config.get('run') or config.get('load')
if prog:
# on load, accept capitalised versions and default extension
with open_native_or_dos_filename(prog) as progfile:
program.load(progfile)
reset.clear()
print_greeting(console)
# start the interpreter (and get out if we ran with -q)
run.start(config.get('exec'), config.get('run'),
config.get('quit'))
except error.RunError as e:
exit_error = e.message
except error.Exit:
# pause before exit if requested
if config.get('wait'):
backend.video_queue.put(
backend.Event(backend.VIDEO_SET_CAPTION,
'Press a key to close window'))
backend.video_queue.put(
backend.Event(backend.VIDEO_SHOW_CURSOR, False))
state.console_state.keyb.pause = True
# this performs a blocking keystroke read if in pause state
backend.check_events()
except error.Reset:
do_reset = True
except KeyboardInterrupt:
if config.get('debug'):
raise
except Exception as e:
exit_error = "Unhandled exception\n%s" % traceback.format_exc()
finally:
try:
audio.close()
except (NameError, AttributeError) as e:
logging.debug('Error on closing audio: %s', e)
try:
# fix the terminal on exit (important for ANSI terminals)
# and save display interface state into screen state
state.console_state.screen.close()
except (NameError, AttributeError) as e:
logging.debug('Error on closing screen: %s', e)
# delete state if resetting
if do_reset:
state.delete()
if plat.system == 'Android':
shutil.rmtree(plat.temp_dir)
else:
state.save()
try:
# close files if we opened any
devices.close_files()
except (NameError, AttributeError) as e:
logging.debug('Error on closing files: %s', e)
try:
devices.close_devices()
except (NameError, AttributeError) as e:
logging.debug('Error on closing devices: %s', e)
if exit_error:
logging.error(exit_error)
magtag.splash.append(battery_label)
print("Refreshing...")
time.sleep(magtag.display.time_to_refresh + 1)
magtag.display.refresh()
time.sleep(magtag.display.time_to_refresh + 1)
status.set(2, status.BLUE)
print("Lightly sleeping for one minute before entering deep sleep...")
if state.light_sleep():
render_and_light_sleep(magtag)
# Free space at startup.
diagnostics.print_free_mem()
state.load()
# Setup and run
magtag = MagTag()
default_magtag_splash_length = len(magtag.splash)
magtag.peripherals.speaker_disable = True
status.enable(magtag)
render_and_light_sleep(magtag)
# Free space at shutdown
diagnostics.print_free_mem()
print('Network on? {}'.format(magtag.network.enabled))
print("Sleeping for 1 day or until button A or B are pressed...")
magtag.peripherals.neopixel_disable = True
parser = argparse.ArgumentParser()
parser.add_argument("ip_addr",
type=str,
help="The IPv4 address of the "
"%s server." % const.TRANSPORT_NAME)
parser.add_argument("tcp_port",
type=int,
help="The TCP port of the %s "
"server." % const.TRANSPORT_NAME)
parser.add_argument("ticket_file",
type=str,
help="The file, the newly "
"issued ticket is written to.")
args = parser.parse_args()
print "[+] Loading server state file."
serverState = state.load()
print "[+] Generating new session ticket."
masterKey = mycrypto.strongRandom(const.MASTER_KEY_LENGTH)
ticket = SessionTicket(masterKey, serverState).issue()
print "[+] Writing new session ticket to `%s'." % args.ticket_file
tickets = dict()
server = IPv4Address('TCP', args.ip_addr, args.tcp_port)
tickets[str(server)] = [int(time.time()), masterKey, ticket]
util.writeToFile(yaml.dump(tickets), args.ticket_file)
print "[+] Success."
parser.add_option("-m",
"--modeldir",
dest="modeldir",
type="string",
help="directory from which to load model")
(options, args) = parser.parse_args()
assert len(args) == 0
assert options.modeldir is not None
import common.dump, yaml
parameters = common.dump.load_canonical_directory(options.modeldir)
import common.hyperparameters
common.hyperparameters.set(parameters, "attardi07_english_ptb")
HYPERPARAMETERS = common.hyperparameters.read("attardi07_english_ptb")
m = state.load(options.modeldir)
HLAYERS = HYPERPARAMETERS["hidden layers"]
if HLAYERS == 2:
w1, b1, wh, bh, w2, b2 = m
else:
w1, b1, w2, b2 = m
import examples, sys
import graph
import numpy as N
from vocabulary import labelmap
ODIM = labelmap.len
from common.mydict import sort as dictsort
for l in sys.stdin:
e = examples._example_from_string(l)
