예제 #1
0
파일: utils.py 프로젝트: yudai09/niftycloud
def fetch_file(uri, file=None, username=None, password=None):
    """
    Fetch a file based on the URI provided.
    If you do not pass in a file pointer a tempfile.NamedTemporaryFile,
    or None if the file could not be retrieved is returned.
    The URI can be either an HTTP url, or "s3://bucket_name/key_name"
    """
    niftycloud.log.info('Fetching %s' % uri)
    if file is None:
        file = tempfile.NamedTemporaryFile()
    try:
        if uri.startswith('s3://'):
            bucket_name, key_name = uri[len('s3://'):].split('/', 1)
            c = niftycloud.connect_s3(aws_access_key_id=username,
                                aws_secret_access_key=password)
            bucket = c.get_bucket(bucket_name)
            key = bucket.get_key(key_name)
            key.get_contents_to_file(file)
        else:
            if username and password:
                passman = urllib.request.HTTPPasswordMgrWithDefaultRealm()
                passman.add_password(None, uri, username, password)
                authhandler = urllib.request.HTTPBasicAuthHandler(passman)
                opener = urllib.request.build_opener(authhandler)
                urllib.request.install_opener(opener)
            s = urllib.request.urlopen(uri)
            file.write(s.read())
        file.seek(0)
    except:
        raise
        niftycloud.log.exception('Problem Retrieving file: %s' % uri)
        file = None
    return file
예제 #2
0
 def __init__(self):
     super(CopyBot, self).__init__()
     self.wdir = niftycloud.config.get('Pyami', 'working_dir')
     self.log_file = '%s.log' % self.instance_id
     self.log_path = os.path.join(self.wdir, self.log_file)
     niftycloud.set_file_logger(self.name, self.log_path)
     self.src_name = niftycloud.config.get(self.name, 'src_bucket')
     self.dst_name = niftycloud.config.get(self.name, 'dst_bucket')
     self.replace = niftycloud.config.getbool(self.name, 'replace_dst', True)
     s3 = niftycloud.connect_s3()
     self.src = s3.lookup(self.src_name)
     if not self.src:
         niftycloud.log.error('Source bucket does not exist: %s' % self.src_name)
     dest_access_key = niftycloud.config.get(self.name, 'dest_aws_access_key_id', None)
     if dest_access_key:
         dest_secret_key = niftycloud.config.get(self.name, 'dest_aws_secret_access_key', None)
         s3 = niftycloud.connect(dest_access_key, dest_secret_key)
     self.dst = s3.lookup(self.dst_name)
     if not self.dst:
         self.dst = s3.create_bucket(self.dst_name)
예제 #3
0
def main():
    try:
        opts, args = getopt.getopt(sys.argv[1:], 'a:b:c:g:hi:k:m:n:o:rs:w',
                                   ['ami', 'bucket', 'class', 'group', 'help',
                                    'inputqueue', 'keypair', 'module',
                                    'numinstances', 'outputqueue',
                                    'reload', 'script_name', 'wait'])
    except:
        usage()
    params = {'module_name': None,
              'script_name': None,
              'class_name': None,
              'script_bucket': None,
              'group': 'default',
              'keypair': None,
              'ami': None,
              'num_instances': 1,
              'input_queue_name': None,
              'output_queue_name': None}
    reload = None
    wait = None
    for o, a in opts:
        if o in ('-a', '--ami'):
            params['ami'] = a
        if o in ('-b', '--bucket'):
            params['script_bucket'] = a
        if o in ('-c', '--class'):
            params['class_name'] = a
        if o in ('-g', '--group'):
            params['group'] = a
        if o in ('-h', '--help'):
            usage()
        if o in ('-i', '--inputqueue'):
            params['input_queue_name'] = a
        if o in ('-k', '--keypair'):
            params['keypair'] = a
        if o in ('-m', '--module'):
            params['module_name'] = a
        if o in ('-n', '--num_instances'):
            params['num_instances'] = int(a)
        if o in ('-o', '--outputqueue'):
            params['output_queue_name'] = a
        if o in ('-r', '--reload'):
            reload = True
        if o in ('-s', '--script'):
            params['script_name'] = a
        if o in ('-w', '--wait'):
            wait = True

    # check required fields
    required = ['ami']
    for pname in required:
        if not params.get(pname, None):
            print('%s is required' % pname)
            usage()
    if params['script_name']:
        # first copy the desired module file to S3 bucket
        if reload:
            print('Reloading module %s to S3' % params['script_name'])
        else:
            print('Copying module %s to S3' % params['script_name'])
        l = imp.find_module(params['script_name'])
        c = niftycloud.connect_s3()
        bucket = c.get_bucket(params['script_bucket'])
        key = bucket.new_key(params['script_name'] + '.py')
        key.set_contents_from_file(l[0])
        params['script_md5'] = key.md5
    # we have everything we need, now build userdata string
    l = []
    for k, v in params.items():
        if v:
            l.append('%s=%s' % (k, v))
    c = niftycloud.connect_ec2()
    l.append('aws_access_key_id=%s' % c.aws_access_key_id)
    l.append('aws_secret_access_key=%s' % c.aws_secret_access_key)
    for kv in args:
        l.append(kv)
    s = '|'.join(l)
    if not reload:
        rs = c.get_all_images([params['ami']])
        img = rs[0]
        r = img.run(user_data=s, key_name=params['keypair'],
                    security_groups=[params['group']],
                    max_count=params.get('num_instances', 1))
        print('AMI: %s - %s (Started)' % (params['ami'], img.location))
        print('Reservation %s contains the following instances:' % r.id)
        for i in r.instances:
            print('\t%s' % i.id)
        if wait:
            running = False
            while not running:
                time.sleep(30)
                [i.update() for i in r.instances]
                status = [i.state for i in r.instances]
                print(status)
                if status.count('running') == len(r.instances):
                    running = True
            for i in r.instances:
                print('Instance: %s' % i.ami_launch_index)
                print('Public DNS Name: %s' % i.public_dns_name)
                print('Private DNS Name: %s' % i.private_dns_name)