def __init__(self, params, run_name='CASA_Timing', region='us-west-2',
key=None, secret=None, max_instances=10):
super(Controller, self).__init__()
self.params = params
self.njobs = params['njobs']
self.data_files = params['files']
self.max_instances = max_instances
self.key = key
self.secret = secret
self.credentials = {"aws_access_key_id": self.key,
"aws_secret_access_key": self.secret}
self.job_name = run_name + "_" + timestring()
self.request_queue_name = self.job_name + "_request"
self.request_queue = \
boto.sqs.connect_to_region(region,
aws_access_key_id=self.key,
aws_secret_access_key=self.secret).create_queue(self.request_queue_name)
self.result_queue_name = self.job_name + "_result"
self.result_queue = \
boto.sqs.connect_to_region(region,
aws_access_key_id=self.key,
aws_secret_access_key=self.secret).create_queue(self.result_queue_name)
def __init__(self, params, run_name='CASA_Timing', region='us-west-2',
key=None, secret=None, max_instances=10):
super(Controller, self).__init__()
self.params = params
self.njobs = params['njobs']
self.data_files = params['files']
self.max_instances = max_instances
self.key = key
self.secret = secret
self.credentials = {"aws_access_key_id": self.key,
"aws_secret_access_key": self.secret}
self.job_name = run_name + "_" + timestring()
self.request_queue_name = self.job_name + "_request"
self.request_queue = \
boto.sqs.connect_to_region(region,
aws_access_key_id=self.key,
aws_secret_access_key=self.secret).create_queue(self.request_queue_name)
self.result_queue_name = self.job_name + "_result"
self.result_queue = \
boto.sqs.connect_to_region(region,
aws_access_key_id=self.key,
aws_secret_access_key=self.secret).create_queue(self.result_queue_name)
def create_root_drive(name, path_to_key, image_description=None,
region='us-west-2',
orig_image_id="ami-5189a661",
install_kwargs={},
verbose=True,
auto_terminate=True):
'''
Creates the root drive for AstroCompute instances.
Parameters
----------
name : str
Name of the image. Note that the time will be appended to the name.
path_to_key : str
Path to the key to link to the VM. Note this key must be attached to
you AWS account.
image_description : str, optional
Description of the created image.
region : str, optional
Region to create image. orig_image_id must be correct for the region.
orig_image_id : str, optional
Image ID to base the new image on. This defaults to ubuntu 14.04.
Note the ami changes with the chosen region.
install_kwargs : dict, optional
Passed to install_packages.
verbose : bool, optional
Enables output of the process.
auto_terminate : bool, optional
When enabled, terminates the instance after the image has been
created.
'''
key_name = path_to_key.split("/")[-1].rstrip(".pem")
name += "_"+timestring()
instance = launch(key_name, region=region, image_id=orig_image_id)
if verbose:
print("Instance status: " + str(instance.state))
try:
install_packages(instance, path_to_key, verbose=verbose,
**install_kwargs)
except Exception, e:
warnings.warn("Something went wrong. Terminating instance.")
instance.terminate()
raise e
from controller import WORKER_SCRIPT
from launch_instance import launch
from upload_download_s3 import upload_to_s3, download_from_s3, remove_s3_bucket
from utils import timestring
from boto import sqs, ec2, s3
from boto import Config
import json
from time import sleep, time
import os
# Assuming that the testing system has the AWS config already set.
try:
proc_name = "aws_ska_test_worker_" + timestring()
region = "us-west-2"
# Read in credentials
config = Config()
config.load_credential_file(os.path.expanduser("~/.aws/credentials"))
info = config.items("default")[2:]
key = info[0][1]
secret = info[1][1]
# Create a test file and upload to S3
if not os.path.exists("tests/test.txt"):
test_string = "ALLGLORYTOTHEHYPNOTOAD"
with open("tests/test.txt", "w") as f:
f.write(test_string)
print("Uploading to S3")
from controller import WORKER_SCRIPT
from launch_instance import launch
from upload_download_s3 import upload_to_s3, download_from_s3, remove_s3_bucket
from utils import timestring
from boto import sqs, ec2, s3
from boto import Config
import json
from time import sleep, time
import os
# Assuming that the testing system has the AWS config already set.
try:
proc_name = "aws_ska_test_worker_" + timestring()
region = "us-west-2"
# Read in credentials
config = Config()
config.load_credential_file(os.path.join(os.path.expanduser("~"),".aws/credentials"))
info = config.items("default")[2:]
key = info[0][1]
secret = info[1][1]
# Create a test file and upload to S3
if not os.path.exists("tests/test.txt"):
test_string = "ALLGLORYTOTHEHYPNOTOAD"
with open("tests/test.txt", "w") as f:
f.write(test_string)
writer = csv.writer(file,
delimiter=",",
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
lst = [str(end_time), str(np.mean(trial_times))]
lst += [
str(m) for m, trial_id in sorted(max_metrics, reverse=True)[:3]
]
writer.writerow(lst)
# plots the most recently created results
if __name__ == "__main__":
args = parser.parse_args()
plot_scheduler(args.logdir,
args.metric,
save=args.save,
use_wall=args.wall)
plt.xlabel("wall" if args.wall else "iterations")
plt.ylabel(args.metric)
# plt.savefig("figures/figure" + timestring() + ".png")
if not os.path.exists("figures/"):
os.makedirs("figures/")
if args.show:
plt.savefig("figures/" + os.path.basename(args.logdir) +
"{}.png".format(timestring()))
plt.show()
