def main():
try:
common.setup()
data = common.send_coordinator('/status/cluster', {})
common.pprint_status(data)
except TypeError:
print 'The coordinator is not running, or you sent the wrong secret.'
def main():
try:
common.setup()
data = common.send_coordinator('/status/cluster', {})
common.pprint_status(data)
except TypeError:
print 'The coordinator is not running, or you sent the wrong secret.'
def main():
common.setup()
# CHANGE ME
zone = 'us-central1-a'
machtype = 'n1-standard-4-d'
image = 'projects/google/images/ubuntu-12-04-v20120621'
# Persistent disk, if any.
disk = ''
# If this is set, only this slave will have the disk mounted, and it'll be rw.
# Otherwise, all slaves get the disk mounted ro
rw_disk_instance = ''
print 'Packaging up the stuff the coordinator will need...'
# tar will insert directories, so flatten the view a bit
subprocess.call(['cp', 'coordinator/coordinator.py', '.'])
subprocess.call(['cp', 'coordinator/hadoop_cluster.py', '.'])
subprocess.call(['tar', 'czf', 'coordinator.tgz', 'hadoop', 'gcelib',
'hadoop-tools.jar', 'cfg.py', 'util.py', 'coordinator.py',
'hadoop_cluster.py', 'start_setup.sh'])
subprocess.call(['rm', 'coordinator.py', 'hadoop_cluster.py'])
# Push to a fixed place for now
subprocess.call(['gsutil', 'cp', 'coordinator.tgz',
cfg.gs_coordinators_tarball])
subprocess.call(['rm', 'coordinator.tgz'])
print
print 'Launching coordinator...'
util.api.insert_instance(
name=cfg.coordinator, zone=zone,
machineType=machtype, image=image,
serviceAccounts=gce_shortcuts.service_accounts([cfg.compute_scope,
cfg.rw_storage_scope]),
networkInterfaces=gce_shortcuts.network(),
metadata=gce_shortcuts.metadata({
'startup-script': open('start_setup.sh').read(),
'bootstrap.sh': open('coordinator/bootstrap.sh').read(),
'tarball': cfg.gs_coordinators_tarball,
'gs_bucket': cfg.gs_bucket,
'zone': zone,
'machine_type': machtype,
'image': image,
'disk': disk,
'rw_disk_instance': rw_disk_instance,
'secret': cfg.secret
}),
blocking=True
)
print
print 'Waiting for coordinator to come online...'
while True:
status, _ = util.get_status(cfg.coordinator)
print status[1]
if status == util.InstanceState.SNITCH_READY:
break
time.sleep(cfg.poll_delay_secs)
print
print 'Controller is ready to receive commands.'
def main():
common.setup()
if len(sys.argv) != 3 or sys.argv[1] != '-f':
print 'USAGE: {0} -f hdfs_path'.format(common.script_name())
print 'WARNING: This deletes hdfs_path from your cluster.'
sys.exit(1)
nix = sys.argv[2]
common.send_coordinator('/job/clean', {'path': nix}, verify=True)
def main():
common.setup()
if len(sys.argv) != 3 or sys.argv[1] != '-f':
print 'USAGE: {0} -f hdfs_path'.format(common.script_name())
print 'WARNING: This deletes hdfs_path from your cluster.'
sys.exit(1)
nix = sys.argv[2]
common.send_coordinator('/job/clean', {'path': nix}, verify=True)
def main():
common.setup()
usage = (
'USAGE: {0} [1,2,3]\nPhase 1 generates data, phase 2 sorts it, and '
'phase 3 validates it.'.format(common.script_name()))
if len(sys.argv) != 2:
print usage
sys.exit(1)
# Detect if the user has the jar
jar = 'hadoop-examples-{0}.jar'.format(cfg.hadoop_version)
if not os.path.exists(jar):
print(
'You need {0}, which contains the Terasort MapReduce job, in your '
'current directory.').format(jar)
print(
'Please run the following commands to get it, then re-run this '
'script.')
print
tarball = '{0}.tar.gz'.format(cfg.hadoop_fn)
print 'wget http://{0}/{1}/{2}'.format(cfg.hadoop_url, cfg.hadoop_fn,
tarball)
print 'tar xzf {0}'.format(tarball)
print 'cp {0}/{1} .'.format(cfg.hadoop_fn, jar)
sys.exit(1)
# TODO Figure out number of tasks programatically. The defaults are sometimes
# 1.
num_tasks = 100
phase = sys.argv[1]
job_args = []
if phase == '1':
gigabytes = 1000
# Convert GB->bytes, then divide by 100
hundred_bytes = gigabytes * (10**7)
job_args = [
'teragen', '-Dmapred.map.tasks={0}'.format(num_tasks),
hundred_bytes, '/job_input/terasort'
]
elif phase == '2':
# The terasort driver automatically uses as many map tasks as possible.
job_args = [
'terasort', '-Dmapred.reduce.tasks={0}'.format(num_tasks),
'/job_input/terasort', '/job_output/terasort'
]
elif phase == '3':
job_args = [
'teravalidate', '/job_output/terasort', '/job_output/teravalidate'
]
else:
print usage
sys.exit(1)
common.start_job(jar, job_args)
def main():
common.setup()
if len(sys.argv) != 2:
print 'USAGE: {0} num_slaves'.format(common.script_name())
sys.exit(1)
num_slaves = int(sys.argv[1])
print 'Adding {0} slaves...'.format(num_slaves)
common.send_coordinator('/hadoop/add_slaves', {'num_slaves': num_slaves})
def main():
common.setup()
if len(sys.argv) != 2:
print 'USAGE: {0} num_slaves'.format(common.script_name())
sys.exit(1)
num_slaves = int(sys.argv[1])
print 'Adding {0} slaves...'.format(num_slaves)
common.send_coordinator('/hadoop/add_slaves', {'num_slaves': num_slaves})
def main():
common.setup()
jobtracker_port = setup_tunnel(util.name_to_ip(cfg.hadoop_jobtracker), 50030)
namenode_port = setup_tunnel(util.name_to_ip(cfg.hadoop_namenode), 50070)
print
print ('ssh tunnels are running in the background to provide access to the '
'Hadoop web interface. You can close the tunnels by killing the '
'ssh process responsible. "ps aux | grep ssh" should list processes '
'matching the commands indicated above.')
print
print '*** JobTracker: http://localhost:{0}'.format(jobtracker_port)
print '*** NameNode: http://localhost:{0}'.format(namenode_port)
def main():
common.setup()
jobtracker_port = setup_tunnel(util.name_to_ip(cfg.hadoop_jobtracker),
50030)
namenode_port = setup_tunnel(util.name_to_ip(cfg.hadoop_namenode), 50070)
print
print(
'ssh tunnels are running in the background to provide access to the '
'Hadoop web interface. You can close the tunnels by killing the '
'ssh process responsible. "ps aux | grep ssh" should list processes '
'matching the commands indicated above.')
print
print '*** JobTracker: http://localhost:{0}'.format(jobtracker_port)
print '*** NameNode: http://localhost:{0}'.format(namenode_port)
def main():
common.setup()
usage = "USAGE: {0} [1,2,3]\nPhase 1 generates data, phase 2 sorts it, and " "phase 3 validates it.".format(
common.script_name()
)
if len(sys.argv) != 2:
print usage
sys.exit(1)
# Detect if the user has the jar
jar = "hadoop-examples-{0}.jar".format(cfg.hadoop_version)
if not os.path.exists(jar):
print ("You need {0}, which contains the Terasort MapReduce job, in your " "current directory.").format(jar)
print ("Please run the following commands to get it, then re-run this " "script.")
print
tarball = "{0}.tar.gz".format(cfg.hadoop_fn)
print "wget http://{0}/{1}/{2}".format(cfg.hadoop_url, cfg.hadoop_fn, tarball)
print "tar xzf {0}".format(tarball)
print "cp {0}/{1} .".format(cfg.hadoop_fn, jar)
sys.exit(1)
# TODO Figure out number of tasks programatically. The defaults are sometimes
# 1.
num_tasks = 100
phase = sys.argv[1]
job_args = []
if phase == "1":
gigabytes = 1000
# Convert GB->bytes, then divide by 100
hundred_bytes = gigabytes * (10 ** 7)
job_args = ["teragen", "-Dmapred.map.tasks={0}".format(num_tasks), hundred_bytes, "/job_input/terasort"]
elif phase == "2":
# The terasort driver automatically uses as many map tasks as possible.
job_args = [
"terasort",
"-Dmapred.reduce.tasks={0}".format(num_tasks),
"/job_input/terasort",
"/job_output/terasort",
]
elif phase == "3":
job_args = ["teravalidate", "/job_output/terasort", "/job_output/teravalidate"]
else:
print usage
sys.exit(1)
common.start_job(jar, job_args)
def main():
common.setup()
if len(sys.argv) != 2:
print 'USAGE: {0} num_slaves'.format(common.script_name())
sys.exit(1)
num_slaves = int(sys.argv[1])
if num_slaves < cfg.needed_slaves:
print ('Hadoop needs at least {0} slaves for filesystem '
'replication').format(cfg.needed_slaves)
sys.exit(1)
print 'Setting up Hadoop...'
common.send_coordinator('/hadoop/launch', {'num_slaves': num_slaves})
common.wait_for_hadoop()
def main():
common.setup()
if len(sys.argv) != 2:
print 'USAGE: {0} num_slaves'.format(common.script_name())
sys.exit(1)
num_slaves = int(sys.argv[1])
if num_slaves < cfg.needed_slaves:
print('Hadoop needs at least {0} slaves for filesystem '
'replication').format(cfg.needed_slaves)
sys.exit(1)
print 'Setting up Hadoop...'
common.send_coordinator('/hadoop/launch', {'num_slaves': num_slaves})
common.wait_for_hadoop()
def test_attack_receiving_player_with_keys_unable_to_move():
envs, shared_state = setup(positions=[(1, 0), (1, 1)],
walls=[(1, 3)],
additional_keys=[(1, 2)])
env = envs[0]
env2 = envs[1]
# Fully recharge
for i in range(20):
env.step(FourKeysActions.NOTHING)
env2.step(FourKeysActions.NOTHING)
env2.step(FourKeysActions.RIGHT) # Pickup key
env2.step(FourKeysActions.LEFT)
env.step(FourKeysActions.ATTACK)
assert env.attack_recharge == 1
assert env2.move_recharge == 0
assert_position(env2,
shared_state, (1, 1),
player=FourKeysBoardItems.PLAYER2)
env2.step(FourKeysActions.RIGHT)
assert_position(env2,
shared_state, (1, 1),
player=FourKeysBoardItems.PLAYER2)
def submit(params_file):
# Use the model to generate predictions
# with open(best_fn, "w") as f:
# json.dump({"best_params": best_params, "best_value": best_value,
# "n_optuna_iterations": n_optuna_iterations, "tictoc_seconds": toc-tic}, f, indent=2, sort_keys=True)
with open(params_file) as f:
params = json.load(f)
xgb_params, best_value = params["best_params"], params["best_value"]
# here we use the fact that xgboost is deterministic, cf. https://www.kaggle.com/devinanzelmo/done-done-2-more-efficient
model = xgboost.XGBRegressor(**xgb_params)
SUBMISSION_DATE = datetime.now()
_PREDICTION_MODES = common_prediction.get_prediction_methods()
X, y, X_test = common.setup()
X_train, X_valid, y_train, y_valid = train_test_split(X, y, random_state=0)
for tag, pm in tqdm.tqdm(list(_PREDICTION_MODES.items())):
pred_df = pm(model, X, y, X_train, y_train, X_valid, y_valid, X_test)
pred_fn = f".tmp/predictions_{tag}_{SUBMISSION_DATE.strftime('%Y%m%d')}_optuna.csv"
pred_df.to_csv(pred_fn, index=None)
common.submit_prediction(
pred_fn,
competition_name="30-days-of-ml",
notebook_fn=
"/home/nailbiter/for/forjupiter/kaggle/20210817-30-days-ml/scripts-optuna.py",
mse=best_value,
suff=tag,
)
#
_URL = "https://www.kaggle.com/c/30-days-of-ml/submissions"
# _URL
# webbrowser.get("firefox").open(_URL)
print(_URL)
def test_attack_recharge_with_steps():
envs, shared_state = setup(positions=[(1, 0)],
walls=[(1, 3)],
additional_keys=[(1, 2)])
env = envs[0]
env.step(FourKeysActions.NOTHING)
assert env.attack_recharge == 1
def main():
common.setup()
usage = 'USAGE: {0} hdfs_src gs_dst'.format(common.script_name())
if len(sys.argv) != 3:
print usage
sys.exit(1)
src = sys.argv[1]
if sys.argv[2].startswith('gs://') or not sys.argv[2].startswith('/'):
print usage
print ('gs_dst should be of the form /path/to/object. gs://{0} will be '
'prefixed for you.').format(cfg.gs_bucket)
sys.exit(1)
dst = 'gs://{0}{1}'.format(cfg.gs_bucket, sys.argv[2])
common.download(src, dst)
def test_attack_miss_player_when_charged_not_adjacent():
envs, shared_state = setup(positions=[(1, 0), (1, 1)],
walls=[(1, 3)],
additional_keys=[(1, 2)])
env = envs[0]
env2 = envs[1]
# Fully recharge
for i in range(20):
env.step(FourKeysActions.NOTHING)
env2.step(FourKeysActions.NOTHING)
env2.step(FourKeysActions.RIGHT) # Pickup key
env2.step(FourKeysActions.LEFT)
env2.step(FourKeysActions.RIGHT)
assert env2.keys == 1
assert count_keys(shared_state.board) == 4
assert env.attack_recharge == ATTACK_FULL_CHARGE
env.step(FourKeysActions.ATTACK)
assert env.attack_recharge == ATTACK_FULL_CHARGE
assert env2.keys == 1
assert count_keys(shared_state.board) == 4
def test_inside_board_wall_multi_step_block():
envs, shared_state = setup(positions=[(1, 1)], walls=[(1, 3)])
env = envs[0]
assert_position(env, shared_state, (1, 1))
env.step(FourKeysActions.RIGHT)
env.step(FourKeysActions.RIGHT)
assert_position(env, shared_state, (1, 2))
def main():
common.setup()
usage = 'USAGE: {0} hdfs_src gs_dst'.format(common.script_name())
if len(sys.argv) != 3:
print usage
sys.exit(1)
src = sys.argv[1]
if sys.argv[2].startswith('gs://') or not sys.argv[2].startswith('/'):
print usage
print(
'gs_dst should be of the form /path/to/object. gs://{0} will be '
'prefixed for you.').format(cfg.gs_bucket)
sys.exit(1)
dst = 'gs://{0}{1}'.format(cfg.gs_bucket, sys.argv[2])
common.download(src, dst)
def test_key_not_block():
envs, shared_state = setup(positions=[(1, 1)],
walls=[(1, 3)],
additional_keys=[(1, 2)])
env = envs[0]
assert_position(env, shared_state, (1, 1))
env.step(FourKeysActions.RIGHT)
assert_position(env, shared_state, (1, 2))
def test_breadcrumbs_increment_in_place():
envs, shared_state = setup()
env = envs[0]
env.step(FourKeysActions.DOWN)
env.step(FourKeysActions.UP)
env.step(FourKeysActions.DOWN)
assert get_breadcrumb_value(env, (0, 0)) == 2
assert get_breadcrumb_value(env, (1, 0)) == 1
def test_key_pickup():
envs, shared_state = setup(positions=[(1, 1)],
walls=[(1, 3)],
additional_keys=[(1, 2)])
env = envs[0]
assert env.keys == 0
env.step(FourKeysActions.RIGHT)
assert env.keys == 1
def test_drop_key_remove_from_inventory():
envs, shared_state = setup(positions=[(1, 1)],
walls=[(1, 3)],
additional_keys=[(1, 2)])
env = envs[0]
env.step(FourKeysActions.RIGHT) # Pickup key
assert env.keys == 1
env.step(FourKeysActions.DROP_KEY)
assert env.keys == 0
def test_drop_key_add_to_board():
envs, shared_state = setup(positions=[(1, 1)],
walls=[(1, 3)],
additional_keys=[(1, 2)])
env = envs[0]
env.step(FourKeysActions.RIGHT) # Pickup key
assert count_keys(shared_state.board) == 4
env.step(FourKeysActions.DROP_KEY)
assert count_keys(shared_state.board) == 5
def test_key_remove_from_game():
envs, shared_state = setup(positions=[(1, 1)],
walls=[(1, 3)],
additional_keys=[(1, 2)])
env = envs[0]
assert len(shared_state.keys) == 5
assert count_keys(shared_state.board) == 5
env.step(FourKeysActions.RIGHT)
assert len(shared_state.keys) == 4
assert count_keys(shared_state.board) == 4
def test_breadcrumbs_cap_at_limit():
envs, shared_state = setup()
env = envs[0]
for i in range(50):
env.step(FourKeysActions.DOWN)
env.step(FourKeysActions.UP)
assert get_breadcrumb_value(env, (0, 0)) == 20
assert get_breadcrumb_value(env, (1, 0)) == 20
def test_attack_recharge_decrease_with_adjacent_players():
envs, shared_state = setup(positions=[(1, 0), (1, 1)],
walls=[(1, 3)],
additional_keys=[(1, 2)])
env = envs[0]
# Recharge
for i in range(20):
env.step(FourKeysActions.NOTHING)
assert env.attack_recharge == ATTACK_FULL_CHARGE
env.step(FourKeysActions.ATTACK)
assert env.attack_recharge == 1
def test_attack_recharge_no_change_on_key_pickup():
envs, shared_state = setup(positions=[(1, 1)],
walls=[(1, 3)],
additional_keys=[(1, 2)])
env = envs[0]
# Recharge
for i in range(20):
env.step(FourKeysActions.NOTHING)
assert env.attack_recharge == ATTACK_FULL_CHARGE
env.step(FourKeysActions.RIGHT)
assert env.attack_recharge == ATTACK_FULL_CHARGE
def __init__(self, username, ip="127.0.0.1"):
self.username = username
self.ip = ip
self.nextseqnum = 0
self.expectedseqnum = 0
self.base = 0
self.n = 0
self.sndpkt = {}
self.receivers = {}
self.sock = common.setup((self.ip, 0))
self.port = self.sock.getsockname()[1]
thread.start_new_thread(self.start_server, ())
print self.username, "listening", (self.ip, self.port)
self.send_details()
def submit(optuna_results_file, submission_date):
# FIXME: raise error if pipeline hash changes
#taken from https://stackoverflow.com/a/13514318
this_function_name = cast(types.FrameType,
inspect.currentframe()).f_code.co_name
logger = logging.getLogger(__name__).getChild(this_function_name)
m = re.match(
r"^.tmp/optuna_results_([a-z0-9_]+)_\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}\.csv$",
optuna_results_file)
assert m is not None, m
pipeline_name = m.group(1)
logger.info(f"pipeline_name: {pipeline_name}")
optuna_results_df = pd.read_csv(optuna_results_file)
assert optuna_results_df.study_hash.nunique() == 1, list(
optuna_results_df.study_hash.unique())
get_pipeline, search_space, signature = retrieve_pipeline(pipeline_name)
assert signature == optuna_results_df.study_hash.iloc[0]
best_params = optuna_results_df.to_dict(
orient="records")[optuna_results_df.value.argmax()]
mse = best_params["value"]
best_params = {
k[len("params_"):]: v
for k, v in best_params.items() if k.startswith("params_")
}
ppl = get_pipeline(best_params)
_PREDICTION_MODES = common_prediction.get_prediction_methods()
X, y, X_test = common.setup()
# X_train, X_valid, y_train, y_valid = train_test_split(X, y, random_state=0)
#
for tag, pm in tqdm.tqdm(list(_PREDICTION_MODES.items())):
submission_df = pm(ppl, X, y, None, None, None, None, X_test)
submission_hash = common.submit_prediction(
submission_df=submission_df,
notebook_fn=__file__,
mse=mse,
submission_date=submission_date,
suff=str((ppl, best_params)),
)
_URL = f"https://www.kaggle.com/c/{common.COMPETITION_NAME}/submissions"
webbrowser.get("firefox").open(_URL)
print(_URL)
def main():
common.setup()
run = raw_input('Really delete all your instances? [y/n] ')
if run != 'y':
print 'Never mind.'
sys.exit()
def nix(instance):
logging.info('Shutting down %s', instance)
util.api.delete_instance(instance=instance, blocking=True)
sched = util.Scheduler(cfg.num_workers * 2)
for name in util.get_instance_names():
sched.schedule(nix, (name, ))
# Block
left = len(util.get_instance_names())
while left:
logging.info('%s instances still left', left)
time.sleep(cfg.poll_delay_secs)
left = len(util.get_instance_names())
print
print 'All gone!'
def main():
common.setup()
run = raw_input('Really delete all your instances? [y/n] ')
if run != 'y':
print 'Never mind.'
sys.exit()
def nix(instance):
logging.info('Shutting down %s', instance)
util.api.delete_instance(instance=instance, blocking=True)
sched = util.Scheduler(cfg.num_workers * 2)
for name in util.get_instance_names():
sched.schedule(nix, (name,))
# Block
left = len(util.get_instance_names())
while left:
logging.info('%s instances still left', left)
time.sleep(cfg.poll_delay_secs)
left = len(util.get_instance_names())
print
print 'All gone!'
def test_attack_player_movement_recharge_not_impacted_without_keys():
envs, shared_state = setup(positions=[(1, 0), (1, 1)],
walls=[(1, 3)],
additional_keys=[(1, 2)])
env = envs[0]
env2 = envs[1]
# Fully recharge
for i in range(20):
env.step(FourKeysActions.NOTHING)
env2.step(FourKeysActions.NOTHING)
assert env2.keys == 0
assert env.attack_recharge == ATTACK_FULL_CHARGE
env.step(FourKeysActions.ATTACK)
assert env.attack_recharge == 1
assert env2.move_recharge == MOVEMENT_FULL_CHARGE
assert env2.attack_recharge == ATTACK_FULL_CHARGE
def test_breadcrumbs_cap_at_limit():
envs, shared_state = setup(side_length=15)
env = envs[0]
for i in range(15):
for i in range(3):
for i in range(15):
env.step(FourKeysActions.DOWN)
for i in range(15):
env.step(FourKeysActions.UP)
env.step(FourKeysActions.RIGHT)
for i in range(10):
env.step(FourKeysActions.NOTHING)
assert count_all_breadcrumbs(env) == BREADCRUMB_LIFESPAN
assert get_breadcrumb_value(env, (0, 0)) == 0
assert get_breadcrumb_value(env, (0, 5)) == 7
def test_attack_no_effect_below_recharge():
envs, shared_state = setup(positions=[(1, 0), (1, 1)],
walls=[(1, 3)],
additional_keys=[(1, 2)])
env = envs[0]
env2 = envs[1]
# Partial recharge
for i in range(10):
env.step(FourKeysActions.NOTHING)
env2.step(FourKeysActions.NOTHING)
env2.step(FourKeysActions.RIGHT) # Pickup key
env2.step(FourKeysActions.LEFT)
assert env2.keys == 1
assert env.attack_recharge < ATTACK_FULL_CHARGE
env.step(FourKeysActions.ATTACK)
assert env.attack_recharge > 1
assert env2.keys == 1
def submit_pipeline(params_file):
# FIXME: merge into submit later
# TODO: create entry for every submission
# FIXME: raise error if pipeline hash changes
with open(params_file) as f:
params = json.load(f)
best_params, best_value, (
pipeline_name, saved_pipeline_signature
) = params["best_params"], params["best_value"], params["pipeline"]
model, _, _ = _retrieve_pipeline(pipeline_name)
# print(model)
# FIXME: make more robust
model.set_params(**{f"xgb__{k}": v for k, v in best_params.items()})
SUBMISSION_DATE = datetime.now()
_PREDICTION_MODES = {
k: v
for k, v in common_prediction.get_prediction_methods().items()
if k in ["full"]
}
X, y, X_test = common.setup()
X_train, X_valid, y_train, y_valid = train_test_split(X, y, random_state=0)
for tag, pm in tqdm.tqdm(list(_PREDICTION_MODES.items())):
pred_df = pm(model, X, y, X_train, y_train, X_valid, y_valid, X_test)
pred_fn = f".tmp/predictions_{tag}_{SUBMISSION_DATE.strftime('%Y%m%d')}_optuna.csv"
pred_df.to_csv(pred_fn, index=None)
common.submit_prediction(
pred_fn,
competition_name="30-days-of-ml",
notebook_fn=
"/home/nailbiter/for/forjupiter/kaggle/20210817-30-days-ml/scripts-optuna.py",
mse=best_value,
suff="__".join([tag, params_file]),
)
_URL = "https://www.kaggle.com/c/30-days-of-ml/submissions"
# webbrowser.get("firefox").open(_URL)
print(_URL)
def setup():
common.setup()
else:
OUTFILE = sys.stdout
if options.bucket:
bucket = conn.create_bucket(options.bucket)
else:
bucket = common.get_new_bucket()
keys = []
print >> OUTFILE, 'bucket: %s' % bucket.name
print >> sys.stderr, 'setup complete, generating files'
for profile in common.config.file_generation.groups:
seed = random.random()
files = get_random_files(profile[0], profile[1], profile[2], seed)
keys += upload_objects(bucket, files, seed)
print >> sys.stderr, 'finished sending files. generating urls'
for key in keys:
print >> OUTFILE, key.generate_url(30758400) #valid for 1 year
print >> sys.stderr, 'done'
if __name__ == '__main__':
common.setup()
try:
main()
except Exception as e:
traceback.print_exc()
common.teardown()
#!/usr/bin/python
from common import run_cmake
from common import setup
from common import benchmark
from common import src
from common import obj
import subprocess
import os
import platform
import sys
import shutil
setup(internet=True)
def build_stage(n):
inst_dir = obj + '/llvm-inst%s' % n
optimize=True
plugin = False
static = platform.system() != 'Darwin'
if n == 1:
CC = '/usr/bin/gcc'
CXX = '/usr/bin/g++'
AR = 'ar'
asserts = False
lto = False
else:
prev_inst_dir = obj + '/llvm-inst%s' % (n-1)
def test_inside_board_move_left_ok():
envs, shared_state = setup(positions=[(1, 1)])
env = envs[0]
assert_position(env, shared_state, (1, 1))
env.step(FourKeysActions.LEFT)
assert_position(env, shared_state, (1, 0))
def main():
# parse options
(options, args) = parse_options()
try:
# setup
common.setup()
bucket = common.get_new_bucket()
print "Created bucket: {name}".format(name=bucket.name)
r = None
if (options.rewrite):
r = random.randint(0, 65535)
q = gevent.queue.Queue()
# main work
print "Using file size: {size} +- {stddev}".format(size=options.file_size, stddev=options.stddev)
print "Spawning {r} readers and {w} writers...".format(r=options.num_readers, w=options.num_writers)
greenlets = []
greenlets += [gevent.spawn(writer, options.duration, bucket,
name=x,
queue=q,
file_size=options.file_size,
file_stddev=options.stddev,
quantity=options.quantity,
file_name_seed=r
) for x in xrange(options.num_writers)]
greenlets += [gevent.spawn(reader, options.duration, bucket,
name=x,
queue=q
) for x in xrange(options.num_readers)]
gevent.spawn_later(options.duration, lambda: q.put(StopIteration))
total_read = 0
total_write = 0
read_success = 0
read_failure = 0
write_success = 0
write_failure = 0
for item in q:
print item
if item.type == Result.TYPE_READER:
if item.success:
read_success += 1
total_read += item.size
else:
read_failure += 1
elif item.type == Result.TYPE_WRITER:
if item.success:
write_success += 1
total_write += item.size
else:
write_failure += 1
# overall stats
print "--- Stats ---"
print "Total Read: {read} MB ({mbps} MB/s)".format(
read=(total_read/1024.0),
mbps=(total_read/1024.0/options.duration)
)
print "Total Write: {write} MB ({mbps} MB/s)".format(
write=(total_write/1024.0),
mbps=(total_write/1024.0/options.duration)
)
print "Read filures: {num} ({percent}%)".format(
num=read_failure,
percent=(100.0*read_failure/max(read_failure+read_success, 1))
)
print "Write failures: {num} ({percent}%)".format(
num=write_failure,
percent=(100.0*write_failure/max(write_failure+write_success, 1))
)
gevent.joinall(greenlets, timeout=1)
except Exception as e:
print e
finally:
# cleanup
if options.cleanup:
common.teardown()