def fetch_file(uri, file=None, username=None, password=None):
"""
Fetch a file based on the URI provided.
If you do not pass in a file pointer a tempfile.NamedTemporaryFile,
or None if the file could not be retrieved is returned.
The URI can be either an HTTP url, or "s3://bucket_name/key_name"
"""
mssapi.log.info('Fetching %s' % uri)
if file is None:
file = tempfile.NamedTemporaryFile()
try:
if uri.startswith('s3://'):
bucket_name, key_name = uri[len('s3://'):].split('/', 1)
c = mssapi.connect_s3(aws_access_key_id=username,
aws_secret_access_key=password)
bucket = c.get_bucket(bucket_name)
key = bucket.get_key(key_name)
key.get_contents_to_file(file)
else:
if username and password:
passman = urllib.request.HTTPPasswordMgrWithDefaultRealm()
passman.add_password(None, uri, username, password)
authhandler = urllib.request.HTTPBasicAuthHandler(passman)
opener = urllib.request.build_opener(authhandler)
urllib.request.install_opener(opener)
s = urllib.request.urlopen(uri)
file.write(s.read())
file.seek(0)
except:
raise
mssapi.log.exception('Problem Retrieving file: %s' % uri)
file = None
return file
def fetch_file(uri, file=None, username=None, password=None):
"""
Fetch a file based on the URI provided.
If you do not pass in a file pointer a tempfile.NamedTemporaryFile,
or None if the file could not be retrieved is returned.
The URI can be either an HTTP url, or "s3://bucket_name/key_name"
"""
mssapi.log.info('Fetching %s' % uri)
if file is None:
file = tempfile.NamedTemporaryFile()
try:
if uri.startswith('s3://'):
bucket_name, key_name = uri[len('s3://'):].split('/', 1)
c = mssapi.connect_s3(aws_access_key_id=username,
aws_secret_access_key=password)
bucket = c.get_bucket(bucket_name)
key = bucket.get_key(key_name)
key.get_contents_to_file(file)
else:
if username and password:
passman = urllib.request.HTTPPasswordMgrWithDefaultRealm()
passman.add_password(None, uri, username, password)
authhandler = urllib.request.HTTPBasicAuthHandler(passman)
opener = urllib.request.build_opener(authhandler)
urllib.request.install_opener(opener)
s = urllib.request.urlopen(uri)
file.write(s.read())
file.seek(0)
except:
raise
mssapi.log.exception('Problem Retrieving file: %s' % uri)
file = None
return file
def __init__(self):
super(CopyBot, self).__init__()
self.wdir = mssapi.config.get('Pyami', 'working_dir')
self.log_file = '%s.log' % self.instance_id
self.log_path = os.path.join(self.wdir, self.log_file)
mssapi.set_file_logger(self.name, self.log_path)
self.src_name = mssapi.config.get(self.name, 'src_bucket')
self.dst_name = mssapi.config.get(self.name, 'dst_bucket')
self.replace = mssapi.config.getbool(self.name, 'replace_dst', True)
s3 = mssapi.connect_s3()
self.src = s3.lookup(self.src_name)
if not self.src:
mssapi.log.error('Source bucket does not exist: %s' % self.src_name)
dest_access_key = mssapi.config.get(self.name, 'dest_aws_access_key_id', None)
if dest_access_key:
dest_secret_key = mssapi.config.get(self.name, 'dest_aws_secret_access_key', None)
s3 = mssapi.connect(dest_access_key, dest_secret_key)
self.dst = s3.lookup(self.dst_name)
if not self.dst:
self.dst = s3.create_bucket(self.dst_name)
def main():
try:
opts, args = getopt.getopt(sys.argv[1:], 'a:b:c:g:hi:k:m:n:o:rs:w', [
'ami', 'bucket', 'class', 'group', 'help', 'inputqueue', 'keypair',
'module', 'numinstances', 'outputqueue', 'reload', 'script_name',
'wait'
])
except:
usage()
params = {
'module_name': None,
'script_name': None,
'class_name': None,
'script_bucket': None,
'group': 'default',
'keypair': None,
'ami': None,
'num_instances': 1,
'input_queue_name': None,
'output_queue_name': None
}
reload = None
wait = None
for o, a in opts:
if o in ('-a', '--ami'):
params['ami'] = a
if o in ('-b', '--bucket'):
params['script_bucket'] = a
if o in ('-c', '--class'):
params['class_name'] = a
if o in ('-g', '--group'):
params['group'] = a
if o in ('-h', '--help'):
usage()
if o in ('-i', '--inputqueue'):
params['input_queue_name'] = a
if o in ('-k', '--keypair'):
params['keypair'] = a
if o in ('-m', '--module'):
params['module_name'] = a
if o in ('-n', '--num_instances'):
params['num_instances'] = int(a)
if o in ('-o', '--outputqueue'):
params['output_queue_name'] = a
if o in ('-r', '--reload'):
reload = True
if o in ('-s', '--script'):
params['script_name'] = a
if o in ('-w', '--wait'):
wait = True
# check required fields
required = ['ami']
for pname in required:
if not params.get(pname, None):
print('%s is required' % pname)
usage()
if params['script_name']:
# first copy the desired module file to S3 bucket
if reload:
print('Reloading module %s to S3' % params['script_name'])
else:
print('Copying module %s to S3' % params['script_name'])
l = imp.find_module(params['script_name'])
c = mssapi.connect_s3()
bucket = c.get_bucket(params['script_bucket'])
key = bucket.new_key(params['script_name'] + '.py')
key.set_contents_from_file(l[0])
params['script_md5'] = key.md5
# we have everything we need, now build userdata string
l = []
for k, v in params.items():
if v:
l.append('%s=%s' % (k, v))
c = mssapi.connect_ec2()
l.append('aws_access_key_id=%s' % c.aws_access_key_id)
l.append('aws_secret_access_key=%s' % c.aws_secret_access_key)
for kv in args:
l.append(kv)
s = '|'.join(l)
if not reload:
rs = c.get_all_images([params['ami']])
img = rs[0]
r = img.run(user_data=s,
key_name=params['keypair'],
security_groups=[params['group']],
max_count=params.get('num_instances', 1))
print('AMI: %s - %s (Started)' % (params['ami'], img.location))
print('Reservation %s contains the following instances:' % r.id)
for i in r.instances:
print('\t%s' % i.id)
if wait:
running = False
while not running:
time.sleep(30)
[i.update() for i in r.instances]
status = [i.state for i in r.instances]
print(status)
if status.count('running') == len(r.instances):
running = True
for i in r.instances:
print('Instance: %s' % i.ami_launch_index)
print('Public DNS Name: %s' % i.public_dns_name)
print('Private DNS Name: %s' % i.private_dns_name)
def main():
try:
opts, args = getopt.getopt(
sys.argv[1:],
"a:b:c:g:hi:k:m:n:o:rs:w",
[
"ami",
"bucket",
"class",
"group",
"help",
"inputqueue",
"keypair",
"module",
"numinstances",
"outputqueue",
"reload",
"script_name",
"wait",
],
)
except:
usage()
params = {
"module_name": None,
"script_name": None,
"class_name": None,
"script_bucket": None,
"group": "default",
"keypair": None,
"ami": None,
"num_instances": 1,
"input_queue_name": None,
"output_queue_name": None,
}
reload = None
wait = None
for o, a in opts:
if o in ("-a", "--ami"):
params["ami"] = a
if o in ("-b", "--bucket"):
params["script_bucket"] = a
if o in ("-c", "--class"):
params["class_name"] = a
if o in ("-g", "--group"):
params["group"] = a
if o in ("-h", "--help"):
usage()
if o in ("-i", "--inputqueue"):
params["input_queue_name"] = a
if o in ("-k", "--keypair"):
params["keypair"] = a
if o in ("-m", "--module"):
params["module_name"] = a
if o in ("-n", "--num_instances"):
params["num_instances"] = int(a)
if o in ("-o", "--outputqueue"):
params["output_queue_name"] = a
if o in ("-r", "--reload"):
reload = True
if o in ("-s", "--script"):
params["script_name"] = a
if o in ("-w", "--wait"):
wait = True
# check required fields
required = ["ami"]
for pname in required:
if not params.get(pname, None):
print("%s is required" % pname)
usage()
if params["script_name"]:
# first copy the desired module file to S3 bucket
if reload:
print("Reloading module %s to S3" % params["script_name"])
else:
print("Copying module %s to S3" % params["script_name"])
l = imp.find_module(params["script_name"])
c = mssapi.connect_s3()
bucket = c.get_bucket(params["script_bucket"])
key = bucket.new_key(params["script_name"] + ".py")
key.set_contents_from_file(l[0])
params["script_md5"] = key.md5
# we have everything we need, now build userdata string
l = []
for k, v in params.items():
if v:
l.append("%s=%s" % (k, v))
c = mssapi.connect_ec2()
l.append("aws_access_key_id=%s" % c.aws_access_key_id)
l.append("aws_secret_access_key=%s" % c.aws_secret_access_key)
for kv in args:
l.append(kv)
s = "|".join(l)
if not reload:
rs = c.get_all_images([params["ami"]])
img = rs[0]
r = img.run(
user_data=s,
key_name=params["keypair"],
security_groups=[params["group"]],
max_count=params.get("num_instances", 1),
)
print("AMI: %s - %s (Started)" % (params["ami"], img.location))
print("Reservation %s contains the following instances:" % r.id)
for i in r.instances:
print("\t%s" % i.id)
if wait:
running = False
while not running:
time.sleep(30)
[i.update() for i in r.instances]
status = [i.state for i in r.instances]
print(status)
if status.count("running") == len(r.instances):
running = True
for i in r.instances:
print("Instance: %s" % i.ami_launch_index)
print("Public DNS Name: %s" % i.public_dns_name)
print("Private DNS Name: %s" % i.private_dns_name)
