def main():
  try:
    common.setup()
    data = common.send_coordinator('/status/cluster', {})
    common.pprint_status(data)
  except TypeError:
    print 'The coordinator is not running, or you sent the wrong secret.'
Exemplo n.º 2
0
def main():
    try:
        common.setup()
        data = common.send_coordinator('/status/cluster', {})
        common.pprint_status(data)
    except TypeError:
        print 'The coordinator is not running, or you sent the wrong secret.'
def main():
  common.setup()

  # CHANGE ME
  zone = 'us-central1-a'
  machtype = 'n1-standard-4-d'
  image = 'projects/google/images/ubuntu-12-04-v20120621'
  # Persistent disk, if any.
  disk = ''
  # If this is set, only this slave will have the disk mounted, and it'll be rw.
  # Otherwise, all slaves get the disk mounted ro
  rw_disk_instance = ''

  print 'Packaging up the stuff the coordinator will need...'
  # tar will insert directories, so flatten the view a bit
  subprocess.call(['cp', 'coordinator/coordinator.py', '.'])
  subprocess.call(['cp', 'coordinator/hadoop_cluster.py', '.'])
  subprocess.call(['tar', 'czf', 'coordinator.tgz', 'hadoop', 'gcelib',
                   'hadoop-tools.jar', 'cfg.py', 'util.py', 'coordinator.py',
                   'hadoop_cluster.py', 'start_setup.sh'])
  subprocess.call(['rm', 'coordinator.py', 'hadoop_cluster.py'])
  # Push to a fixed place for now
  subprocess.call(['gsutil', 'cp', 'coordinator.tgz',
                   cfg.gs_coordinators_tarball])
  subprocess.call(['rm', 'coordinator.tgz'])
  print

  print 'Launching coordinator...'
  util.api.insert_instance(
      name=cfg.coordinator, zone=zone,
      machineType=machtype, image=image,
      serviceAccounts=gce_shortcuts.service_accounts([cfg.compute_scope,
                                                      cfg.rw_storage_scope]),
      networkInterfaces=gce_shortcuts.network(),
      metadata=gce_shortcuts.metadata({
          'startup-script': open('start_setup.sh').read(),
          'bootstrap.sh': open('coordinator/bootstrap.sh').read(),
          'tarball': cfg.gs_coordinators_tarball,
          'gs_bucket': cfg.gs_bucket,
          'zone': zone,
          'machine_type': machtype,
          'image': image,
          'disk': disk,
          'rw_disk_instance': rw_disk_instance,
          'secret': cfg.secret
      }),
      blocking=True
  )
  print

  print 'Waiting for coordinator to come online...'
  while True:
    status, _ = util.get_status(cfg.coordinator)
    print status[1]
    if status == util.InstanceState.SNITCH_READY:
      break
    time.sleep(cfg.poll_delay_secs)
  print

  print 'Controller is ready to receive commands.'
def main():
  common.setup()
  if len(sys.argv) != 3 or sys.argv[1] != '-f':
    print 'USAGE: {0} -f hdfs_path'.format(common.script_name())
    print 'WARNING: This deletes hdfs_path from your cluster.'
    sys.exit(1)

  nix = sys.argv[2]
  common.send_coordinator('/job/clean', {'path': nix}, verify=True)
def main():
    common.setup()
    if len(sys.argv) != 3 or sys.argv[1] != '-f':
        print 'USAGE: {0} -f hdfs_path'.format(common.script_name())
        print 'WARNING: This deletes hdfs_path from your cluster.'
        sys.exit(1)

    nix = sys.argv[2]
    common.send_coordinator('/job/clean', {'path': nix}, verify=True)
def main():
    common.setup()
    usage = (
        'USAGE: {0} [1,2,3]\nPhase 1 generates data, phase 2 sorts it, and '
        'phase 3 validates it.'.format(common.script_name()))

    if len(sys.argv) != 2:
        print usage
        sys.exit(1)

    # Detect if the user has the jar
    jar = 'hadoop-examples-{0}.jar'.format(cfg.hadoop_version)
    if not os.path.exists(jar):
        print(
            'You need {0}, which contains the Terasort MapReduce job, in your '
            'current directory.').format(jar)
        print(
            'Please run the following commands to get it, then re-run this '
            'script.')
        print
        tarball = '{0}.tar.gz'.format(cfg.hadoop_fn)
        print 'wget http://{0}/{1}/{2}'.format(cfg.hadoop_url, cfg.hadoop_fn,
                                               tarball)
        print 'tar xzf {0}'.format(tarball)
        print 'cp {0}/{1} .'.format(cfg.hadoop_fn, jar)
        sys.exit(1)

    # TODO Figure out number of tasks programatically. The defaults are sometimes
    # 1.
    num_tasks = 100
    phase = sys.argv[1]

    job_args = []
    if phase == '1':
        gigabytes = 1000
        # Convert GB->bytes, then divide by 100
        hundred_bytes = gigabytes * (10**7)

        job_args = [
            'teragen', '-Dmapred.map.tasks={0}'.format(num_tasks),
            hundred_bytes, '/job_input/terasort'
        ]
    elif phase == '2':
        # The terasort driver automatically uses as many map tasks as possible.
        job_args = [
            'terasort', '-Dmapred.reduce.tasks={0}'.format(num_tasks),
            '/job_input/terasort', '/job_output/terasort'
        ]
    elif phase == '3':
        job_args = [
            'teravalidate', '/job_output/terasort', '/job_output/teravalidate'
        ]
    else:
        print usage
        sys.exit(1)

    common.start_job(jar, job_args)
def main():
    common.setup()

    if len(sys.argv) != 2:
        print 'USAGE: {0} num_slaves'.format(common.script_name())
        sys.exit(1)

    num_slaves = int(sys.argv[1])

    print 'Adding {0} slaves...'.format(num_slaves)
    common.send_coordinator('/hadoop/add_slaves', {'num_slaves': num_slaves})
def main():
  common.setup()

  if len(sys.argv) != 2:
    print 'USAGE: {0} num_slaves'.format(common.script_name())
    sys.exit(1)

  num_slaves = int(sys.argv[1])

  print 'Adding {0} slaves...'.format(num_slaves)
  common.send_coordinator('/hadoop/add_slaves', {'num_slaves': num_slaves})
def main():
  common.setup()
  jobtracker_port = setup_tunnel(util.name_to_ip(cfg.hadoop_jobtracker), 50030)
  namenode_port = setup_tunnel(util.name_to_ip(cfg.hadoop_namenode), 50070)
  print
  print ('ssh tunnels are running in the background to provide access to the '
         'Hadoop web interface. You can close the tunnels by killing the '
         'ssh process responsible. "ps aux | grep ssh" should list processes '
         'matching the commands indicated above.')
  print
  print '*** JobTracker: http://localhost:{0}'.format(jobtracker_port)
  print '*** NameNode: http://localhost:{0}'.format(namenode_port)
Exemplo n.º 10
0
def main():
    common.setup()
    jobtracker_port = setup_tunnel(util.name_to_ip(cfg.hadoop_jobtracker),
                                   50030)
    namenode_port = setup_tunnel(util.name_to_ip(cfg.hadoop_namenode), 50070)
    print
    print(
        'ssh tunnels are running in the background to provide access to the '
        'Hadoop web interface. You can close the tunnels by killing the '
        'ssh process responsible. "ps aux | grep ssh" should list processes '
        'matching the commands indicated above.')
    print
    print '*** JobTracker: http://localhost:{0}'.format(jobtracker_port)
    print '*** NameNode: http://localhost:{0}'.format(namenode_port)
def main():
    common.setup()
    usage = "USAGE: {0} [1,2,3]\nPhase 1 generates data, phase 2 sorts it, and " "phase 3 validates it.".format(
        common.script_name()
    )

    if len(sys.argv) != 2:
        print usage
        sys.exit(1)

    # Detect if the user has the jar
    jar = "hadoop-examples-{0}.jar".format(cfg.hadoop_version)
    if not os.path.exists(jar):
        print ("You need {0}, which contains the Terasort MapReduce job, in your " "current directory.").format(jar)
        print ("Please run the following commands to get it, then re-run this " "script.")
        print
        tarball = "{0}.tar.gz".format(cfg.hadoop_fn)
        print "wget http://{0}/{1}/{2}".format(cfg.hadoop_url, cfg.hadoop_fn, tarball)
        print "tar xzf {0}".format(tarball)
        print "cp {0}/{1} .".format(cfg.hadoop_fn, jar)
        sys.exit(1)

    # TODO Figure out number of tasks programatically. The defaults are sometimes
    # 1.
    num_tasks = 100
    phase = sys.argv[1]

    job_args = []
    if phase == "1":
        gigabytes = 1000
        # Convert GB->bytes, then divide by 100
        hundred_bytes = gigabytes * (10 ** 7)

        job_args = ["teragen", "-Dmapred.map.tasks={0}".format(num_tasks), hundred_bytes, "/job_input/terasort"]
    elif phase == "2":
        # The terasort driver automatically uses as many map tasks as possible.
        job_args = [
            "terasort",
            "-Dmapred.reduce.tasks={0}".format(num_tasks),
            "/job_input/terasort",
            "/job_output/terasort",
        ]
    elif phase == "3":
        job_args = ["teravalidate", "/job_output/terasort", "/job_output/teravalidate"]
    else:
        print usage
        sys.exit(1)

    common.start_job(jar, job_args)
def main():
  common.setup()

  if len(sys.argv) != 2:
    print 'USAGE: {0} num_slaves'.format(common.script_name())
    sys.exit(1)
  num_slaves = int(sys.argv[1])
  if num_slaves < cfg.needed_slaves:
    print ('Hadoop needs at least {0} slaves for filesystem '
           'replication').format(cfg.needed_slaves)
    sys.exit(1)

  print 'Setting up Hadoop...'
  common.send_coordinator('/hadoop/launch', {'num_slaves': num_slaves})
  common.wait_for_hadoop()
def main():
    common.setup()

    if len(sys.argv) != 2:
        print 'USAGE: {0} num_slaves'.format(common.script_name())
        sys.exit(1)
    num_slaves = int(sys.argv[1])
    if num_slaves < cfg.needed_slaves:
        print('Hadoop needs at least {0} slaves for filesystem '
              'replication').format(cfg.needed_slaves)
        sys.exit(1)

    print 'Setting up Hadoop...'
    common.send_coordinator('/hadoop/launch', {'num_slaves': num_slaves})
    common.wait_for_hadoop()
Exemplo n.º 14
0
def test_attack_receiving_player_with_keys_unable_to_move():
    envs, shared_state = setup(positions=[(1, 0), (1, 1)],
                               walls=[(1, 3)],
                               additional_keys=[(1, 2)])
    env = envs[0]
    env2 = envs[1]

    # Fully recharge
    for i in range(20):
        env.step(FourKeysActions.NOTHING)
        env2.step(FourKeysActions.NOTHING)

    env2.step(FourKeysActions.RIGHT)  # Pickup key
    env2.step(FourKeysActions.LEFT)

    env.step(FourKeysActions.ATTACK)

    assert env.attack_recharge == 1
    assert env2.move_recharge == 0
    assert_position(env2,
                    shared_state, (1, 1),
                    player=FourKeysBoardItems.PLAYER2)
    env2.step(FourKeysActions.RIGHT)
    assert_position(env2,
                    shared_state, (1, 1),
                    player=FourKeysBoardItems.PLAYER2)
Exemplo n.º 15
0
def submit(params_file):
    # Use the model to generate predictions
    #    with open(best_fn, "w") as f:
    #        json.dump({"best_params": best_params, "best_value": best_value,
    #                   "n_optuna_iterations": n_optuna_iterations, "tictoc_seconds": toc-tic}, f, indent=2, sort_keys=True)
    with open(params_file) as f:
        params = json.load(f)
    xgb_params, best_value = params["best_params"], params["best_value"]
    # here we use the fact that xgboost is deterministic, cf. https://www.kaggle.com/devinanzelmo/done-done-2-more-efficient
    model = xgboost.XGBRegressor(**xgb_params)
    SUBMISSION_DATE = datetime.now()
    _PREDICTION_MODES = common_prediction.get_prediction_methods()

    X, y, X_test = common.setup()
    X_train, X_valid, y_train, y_valid = train_test_split(X, y, random_state=0)

    for tag, pm in tqdm.tqdm(list(_PREDICTION_MODES.items())):
        pred_df = pm(model, X, y, X_train, y_train, X_valid, y_valid, X_test)
        pred_fn = f".tmp/predictions_{tag}_{SUBMISSION_DATE.strftime('%Y%m%d')}_optuna.csv"
        pred_df.to_csv(pred_fn, index=None)
        common.submit_prediction(
            pred_fn,
            competition_name="30-days-of-ml",
            notebook_fn=
            "/home/nailbiter/for/forjupiter/kaggle/20210817-30-days-ml/scripts-optuna.py",
            mse=best_value,
            suff=tag,
        )


#
    _URL = "https://www.kaggle.com/c/30-days-of-ml/submissions"
    #    _URL
    #    webbrowser.get("firefox").open(_URL)
    print(_URL)
Exemplo n.º 16
0
def test_attack_recharge_with_steps():
    envs, shared_state = setup(positions=[(1, 0)],
                               walls=[(1, 3)],
                               additional_keys=[(1, 2)])
    env = envs[0]
    env.step(FourKeysActions.NOTHING)
    assert env.attack_recharge == 1
def main():
  common.setup()
  usage = 'USAGE: {0} hdfs_src gs_dst'.format(common.script_name())
  if len(sys.argv) != 3:
    print usage
    sys.exit(1)

  src = sys.argv[1]
  if sys.argv[2].startswith('gs://') or not sys.argv[2].startswith('/'):
    print usage
    print ('gs_dst should be of the form /path/to/object. gs://{0} will be '
           'prefixed for you.').format(cfg.gs_bucket)
    sys.exit(1)
  dst = 'gs://{0}{1}'.format(cfg.gs_bucket, sys.argv[2])

  common.download(src, dst)
Exemplo n.º 18
0
def test_attack_miss_player_when_charged_not_adjacent():
    envs, shared_state = setup(positions=[(1, 0), (1, 1)],
                               walls=[(1, 3)],
                               additional_keys=[(1, 2)])
    env = envs[0]
    env2 = envs[1]

    # Fully recharge
    for i in range(20):
        env.step(FourKeysActions.NOTHING)
        env2.step(FourKeysActions.NOTHING)

    env2.step(FourKeysActions.RIGHT)  # Pickup key
    env2.step(FourKeysActions.LEFT)
    env2.step(FourKeysActions.RIGHT)

    assert env2.keys == 1
    assert count_keys(shared_state.board) == 4

    assert env.attack_recharge == ATTACK_FULL_CHARGE

    env.step(FourKeysActions.ATTACK)
    assert env.attack_recharge == ATTACK_FULL_CHARGE
    assert env2.keys == 1
    assert count_keys(shared_state.board) == 4
Exemplo n.º 19
0
def test_inside_board_wall_multi_step_block():
    envs, shared_state = setup(positions=[(1, 1)], walls=[(1, 3)])
    env = envs[0]
    assert_position(env, shared_state, (1, 1))
    env.step(FourKeysActions.RIGHT)
    env.step(FourKeysActions.RIGHT)
    assert_position(env, shared_state, (1, 2))
def main():
    common.setup()
    usage = 'USAGE: {0} hdfs_src gs_dst'.format(common.script_name())
    if len(sys.argv) != 3:
        print usage
        sys.exit(1)

    src = sys.argv[1]
    if sys.argv[2].startswith('gs://') or not sys.argv[2].startswith('/'):
        print usage
        print(
            'gs_dst should be of the form /path/to/object. gs://{0} will be '
            'prefixed for you.').format(cfg.gs_bucket)
        sys.exit(1)
    dst = 'gs://{0}{1}'.format(cfg.gs_bucket, sys.argv[2])

    common.download(src, dst)
Exemplo n.º 21
0
def test_key_not_block():
    envs, shared_state = setup(positions=[(1, 1)],
                               walls=[(1, 3)],
                               additional_keys=[(1, 2)])
    env = envs[0]
    assert_position(env, shared_state, (1, 1))
    env.step(FourKeysActions.RIGHT)
    assert_position(env, shared_state, (1, 2))
Exemplo n.º 22
0
def test_breadcrumbs_increment_in_place():
    envs, shared_state = setup()
    env = envs[0]
    env.step(FourKeysActions.DOWN)
    env.step(FourKeysActions.UP)
    env.step(FourKeysActions.DOWN)
    assert get_breadcrumb_value(env, (0, 0)) == 2
    assert get_breadcrumb_value(env, (1, 0)) == 1
Exemplo n.º 23
0
def test_key_pickup():
    envs, shared_state = setup(positions=[(1, 1)],
                               walls=[(1, 3)],
                               additional_keys=[(1, 2)])
    env = envs[0]
    assert env.keys == 0
    env.step(FourKeysActions.RIGHT)
    assert env.keys == 1
Exemplo n.º 24
0
def test_drop_key_remove_from_inventory():
    envs, shared_state = setup(positions=[(1, 1)],
                               walls=[(1, 3)],
                               additional_keys=[(1, 2)])
    env = envs[0]
    env.step(FourKeysActions.RIGHT)  # Pickup key
    assert env.keys == 1
    env.step(FourKeysActions.DROP_KEY)
    assert env.keys == 0
Exemplo n.º 25
0
def test_drop_key_add_to_board():
    envs, shared_state = setup(positions=[(1, 1)],
                               walls=[(1, 3)],
                               additional_keys=[(1, 2)])
    env = envs[0]
    env.step(FourKeysActions.RIGHT)  # Pickup key
    assert count_keys(shared_state.board) == 4
    env.step(FourKeysActions.DROP_KEY)
    assert count_keys(shared_state.board) == 5
Exemplo n.º 26
0
def test_key_remove_from_game():
    envs, shared_state = setup(positions=[(1, 1)],
                               walls=[(1, 3)],
                               additional_keys=[(1, 2)])
    env = envs[0]
    assert len(shared_state.keys) == 5
    assert count_keys(shared_state.board) == 5
    env.step(FourKeysActions.RIGHT)
    assert len(shared_state.keys) == 4
    assert count_keys(shared_state.board) == 4
Exemplo n.º 27
0
def test_breadcrumbs_cap_at_limit():
    envs, shared_state = setup()
    env = envs[0]

    for i in range(50):
        env.step(FourKeysActions.DOWN)
        env.step(FourKeysActions.UP)

    assert get_breadcrumb_value(env, (0, 0)) == 20
    assert get_breadcrumb_value(env, (1, 0)) == 20
Exemplo n.º 28
0
def test_attack_recharge_decrease_with_adjacent_players():
    envs, shared_state = setup(positions=[(1, 0), (1, 1)],
                               walls=[(1, 3)],
                               additional_keys=[(1, 2)])
    env = envs[0]

    # Recharge
    for i in range(20):
        env.step(FourKeysActions.NOTHING)

    assert env.attack_recharge == ATTACK_FULL_CHARGE
    env.step(FourKeysActions.ATTACK)
    assert env.attack_recharge == 1
Exemplo n.º 29
0
def test_attack_recharge_no_change_on_key_pickup():
    envs, shared_state = setup(positions=[(1, 1)],
                               walls=[(1, 3)],
                               additional_keys=[(1, 2)])
    env = envs[0]

    # Recharge
    for i in range(20):
        env.step(FourKeysActions.NOTHING)

    assert env.attack_recharge == ATTACK_FULL_CHARGE
    env.step(FourKeysActions.RIGHT)
    assert env.attack_recharge == ATTACK_FULL_CHARGE
Exemplo n.º 30
0
 def __init__(self, username, ip="127.0.0.1"):
     self.username = username
     self.ip = ip
     self.nextseqnum = 0
     self.expectedseqnum = 0
     self.base = 0
     self.n = 0
     self.sndpkt = {}
     self.receivers = {}
     self.sock = common.setup((self.ip, 0))
     self.port = self.sock.getsockname()[1]
     thread.start_new_thread(self.start_server, ())
     print self.username, "listening", (self.ip, self.port)
     self.send_details()
Exemplo n.º 31
0
def submit(optuna_results_file, submission_date):
    # FIXME: raise error if pipeline hash changes

    #taken from https://stackoverflow.com/a/13514318
    this_function_name = cast(types.FrameType,
                              inspect.currentframe()).f_code.co_name
    logger = logging.getLogger(__name__).getChild(this_function_name)

    m = re.match(
        r"^.tmp/optuna_results_([a-z0-9_]+)_\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}\.csv$",
        optuna_results_file)
    assert m is not None, m

    pipeline_name = m.group(1)
    logger.info(f"pipeline_name: {pipeline_name}")
    optuna_results_df = pd.read_csv(optuna_results_file)
    assert optuna_results_df.study_hash.nunique() == 1, list(
        optuna_results_df.study_hash.unique())
    get_pipeline, search_space, signature = retrieve_pipeline(pipeline_name)
    assert signature == optuna_results_df.study_hash.iloc[0]

    best_params = optuna_results_df.to_dict(
        orient="records")[optuna_results_df.value.argmax()]
    mse = best_params["value"]
    best_params = {
        k[len("params_"):]: v
        for k, v in best_params.items() if k.startswith("params_")
    }

    ppl = get_pipeline(best_params)

    _PREDICTION_MODES = common_prediction.get_prediction_methods()
    X, y, X_test = common.setup()
    #    X_train, X_valid, y_train, y_valid = train_test_split(X, y, random_state=0)
    #
    for tag, pm in tqdm.tqdm(list(_PREDICTION_MODES.items())):
        submission_df = pm(ppl, X, y, None, None, None, None, X_test)

        submission_hash = common.submit_prediction(
            submission_df=submission_df,
            notebook_fn=__file__,
            mse=mse,
            submission_date=submission_date,
            suff=str((ppl, best_params)),
        )

    _URL = f"https://www.kaggle.com/c/{common.COMPETITION_NAME}/submissions"
    webbrowser.get("firefox").open(_URL)
    print(_URL)
Exemplo n.º 32
0
def main():
    common.setup()
    run = raw_input('Really delete all your instances? [y/n] ')
    if run != 'y':
        print 'Never mind.'
        sys.exit()

    def nix(instance):
        logging.info('Shutting down %s', instance)
        util.api.delete_instance(instance=instance, blocking=True)

    sched = util.Scheduler(cfg.num_workers * 2)
    for name in util.get_instance_names():
        sched.schedule(nix, (name, ))

    # Block
    left = len(util.get_instance_names())
    while left:
        logging.info('%s instances still left', left)
        time.sleep(cfg.poll_delay_secs)
        left = len(util.get_instance_names())
    print

    print 'All gone!'
def main():
  common.setup()
  run = raw_input('Really delete all your instances? [y/n] ')
  if run != 'y':
    print 'Never mind.'
    sys.exit()

  def nix(instance):
    logging.info('Shutting down %s', instance)
    util.api.delete_instance(instance=instance, blocking=True)

  sched = util.Scheduler(cfg.num_workers * 2)
  for name in util.get_instance_names():
    sched.schedule(nix, (name,))

  # Block
  left = len(util.get_instance_names())
  while left:
    logging.info('%s instances still left', left)
    time.sleep(cfg.poll_delay_secs)
    left = len(util.get_instance_names())
  print

  print 'All gone!'
Exemplo n.º 34
0
def test_attack_player_movement_recharge_not_impacted_without_keys():
    envs, shared_state = setup(positions=[(1, 0), (1, 1)],
                               walls=[(1, 3)],
                               additional_keys=[(1, 2)])
    env = envs[0]
    env2 = envs[1]

    # Fully recharge
    for i in range(20):
        env.step(FourKeysActions.NOTHING)
        env2.step(FourKeysActions.NOTHING)

    assert env2.keys == 0
    assert env.attack_recharge == ATTACK_FULL_CHARGE
    env.step(FourKeysActions.ATTACK)

    assert env.attack_recharge == 1
    assert env2.move_recharge == MOVEMENT_FULL_CHARGE
    assert env2.attack_recharge == ATTACK_FULL_CHARGE
Exemplo n.º 35
0
def test_breadcrumbs_cap_at_limit():
    envs, shared_state = setup(side_length=15)
    env = envs[0]

    for i in range(15):
        for i in range(3):
            for i in range(15):
                env.step(FourKeysActions.DOWN)
            for i in range(15):
                env.step(FourKeysActions.UP)

        env.step(FourKeysActions.RIGHT)

    for i in range(10):
        env.step(FourKeysActions.NOTHING)

    assert count_all_breadcrumbs(env) == BREADCRUMB_LIFESPAN

    assert get_breadcrumb_value(env, (0, 0)) == 0
    assert get_breadcrumb_value(env, (0, 5)) == 7
Exemplo n.º 36
0
def test_attack_no_effect_below_recharge():
    envs, shared_state = setup(positions=[(1, 0), (1, 1)],
                               walls=[(1, 3)],
                               additional_keys=[(1, 2)])
    env = envs[0]
    env2 = envs[1]

    # Partial recharge
    for i in range(10):
        env.step(FourKeysActions.NOTHING)
        env2.step(FourKeysActions.NOTHING)

    env2.step(FourKeysActions.RIGHT)  # Pickup key
    env2.step(FourKeysActions.LEFT)

    assert env2.keys == 1

    assert env.attack_recharge < ATTACK_FULL_CHARGE
    env.step(FourKeysActions.ATTACK)
    assert env.attack_recharge > 1
    assert env2.keys == 1
Exemplo n.º 37
0
def submit_pipeline(params_file):
    # FIXME: merge into submit later
    # TODO: create entry for every submission
    # FIXME: raise error if pipeline hash changes

    with open(params_file) as f:
        params = json.load(f)
    best_params, best_value, (
        pipeline_name, saved_pipeline_signature
    ) = params["best_params"], params["best_value"], params["pipeline"]
    model, _, _ = _retrieve_pipeline(pipeline_name)
    #    print(model)
    # FIXME: make more robust
    model.set_params(**{f"xgb__{k}": v for k, v in best_params.items()})
    SUBMISSION_DATE = datetime.now()
    _PREDICTION_MODES = {
        k: v
        for k, v in common_prediction.get_prediction_methods().items()
        if k in ["full"]
    }

    X, y, X_test = common.setup()
    X_train, X_valid, y_train, y_valid = train_test_split(X, y, random_state=0)

    for tag, pm in tqdm.tqdm(list(_PREDICTION_MODES.items())):
        pred_df = pm(model, X, y, X_train, y_train, X_valid, y_valid, X_test)
        pred_fn = f".tmp/predictions_{tag}_{SUBMISSION_DATE.strftime('%Y%m%d')}_optuna.csv"
        pred_df.to_csv(pred_fn, index=None)
        common.submit_prediction(
            pred_fn,
            competition_name="30-days-of-ml",
            notebook_fn=
            "/home/nailbiter/for/forjupiter/kaggle/20210817-30-days-ml/scripts-optuna.py",
            mse=best_value,
            suff="__".join([tag, params_file]),
        )

    _URL = "https://www.kaggle.com/c/30-days-of-ml/submissions"
    #    webbrowser.get("firefox").open(_URL)
    print(_URL)
Exemplo n.º 38
0
def setup():
    common.setup()
Exemplo n.º 39
0
    else:
        OUTFILE = sys.stdout

    if options.bucket:
        bucket = conn.create_bucket(options.bucket)
    else:
        bucket = common.get_new_bucket()

    keys = []
    print >> OUTFILE, 'bucket: %s' % bucket.name
    print >> sys.stderr, 'setup complete, generating files'
    for profile in common.config.file_generation.groups:
        seed = random.random()
        files = get_random_files(profile[0], profile[1], profile[2], seed)
        keys += upload_objects(bucket, files, seed)

    print >> sys.stderr, 'finished sending files. generating urls'
    for key in keys:
        print >> OUTFILE, key.generate_url(30758400) #valid for 1 year

    print >> sys.stderr, 'done'


if __name__ == '__main__':
    common.setup()
    try:
        main()
    except Exception as e:
        traceback.print_exc()
        common.teardown()
Exemplo n.º 40
0
#!/usr/bin/python

from common import run_cmake
from common import setup
from common import benchmark
from common import src
from common import obj


import subprocess
import os
import platform
import sys
import shutil

setup(internet=True)

def build_stage(n):
    inst_dir = obj + '/llvm-inst%s' % n

    optimize=True
    plugin = False
    static = platform.system() != 'Darwin'
    if n == 1:
        CC = '/usr/bin/gcc'
        CXX = '/usr/bin/g++'
        AR = 'ar'
        asserts = False
        lto = False
    else:
        prev_inst_dir = obj + '/llvm-inst%s' % (n-1)
Exemplo n.º 41
0
def test_inside_board_move_left_ok():
    envs, shared_state = setup(positions=[(1, 1)])
    env = envs[0]
    assert_position(env, shared_state, (1, 1))
    env.step(FourKeysActions.LEFT)
    assert_position(env, shared_state, (1, 0))
Exemplo n.º 42
0
def main():
    # parse options
    (options, args) = parse_options()

    try:
        # setup
        common.setup()
        bucket = common.get_new_bucket()
        print "Created bucket: {name}".format(name=bucket.name)
        r = None
        if (options.rewrite):
            r = random.randint(0, 65535)
        q = gevent.queue.Queue()

        # main work
        print "Using file size: {size} +- {stddev}".format(size=options.file_size, stddev=options.stddev)
        print "Spawning {r} readers and {w} writers...".format(r=options.num_readers, w=options.num_writers)
        greenlets = []
        greenlets += [gevent.spawn(writer, options.duration, bucket,
            name=x,
            queue=q,
            file_size=options.file_size,
            file_stddev=options.stddev,
            quantity=options.quantity,
            file_name_seed=r
            ) for x in xrange(options.num_writers)]
        greenlets += [gevent.spawn(reader, options.duration, bucket,
                name=x,
                queue=q
                ) for x in xrange(options.num_readers)]
        gevent.spawn_later(options.duration, lambda: q.put(StopIteration))

        total_read = 0
        total_write = 0
        read_success = 0
        read_failure = 0
        write_success = 0
        write_failure = 0
        for item in q:
            print item
            if item.type == Result.TYPE_READER:
                if item.success:
                    read_success += 1
                    total_read += item.size
                else:
                    read_failure += 1
            elif item.type == Result.TYPE_WRITER:
                if item.success:
                    write_success += 1
                    total_write += item.size
                else:
                    write_failure += 1

        # overall stats
        print "--- Stats ---"
        print "Total Read:  {read} MB ({mbps} MB/s)".format(
            read=(total_read/1024.0),
            mbps=(total_read/1024.0/options.duration)
            )
        print "Total Write: {write} MB ({mbps} MB/s)".format(
            write=(total_write/1024.0),
            mbps=(total_write/1024.0/options.duration)
            )
        print "Read filures: {num} ({percent}%)".format(
            num=read_failure,
            percent=(100.0*read_failure/max(read_failure+read_success, 1))
            )
        print "Write failures: {num} ({percent}%)".format(
            num=write_failure,
            percent=(100.0*write_failure/max(write_failure+write_success, 1))
            )

        gevent.joinall(greenlets, timeout=1)
    except Exception as e:
        print e
    finally:
        # cleanup
        if options.cleanup:
            common.teardown()