コード例 #1
0
ファイル: models.py プロジェクト: nprapps/musicgame
    def deploy(self):
        """
        Deploy this quiz JSON to S3.
        """
        if not self.slug:
            return

        data = json.dumps(self.flatten())

        s3 = boto.connect_s3()

        gzip_buffer = StringIO()

        with gzip.GzipFile(fileobj=gzip_buffer, mode='w') as f:
            f.write(data)

        data = gzip_buffer.getvalue()

        s3 = boto.connect_s3()

        for bucket_name in app_config.S3_BUCKETS:
            bucket = s3.get_bucket(bucket_name)

            k = Key(bucket, '%s/live-data/games/%s.json' % (app_config.PROJECT_SLUG, self.slug))
            k.set_contents_from_string(data, headers={
                'Content-Type': 'application/json',
                'Content-Encoding': 'gzip',
                'Cache-Control': 'max-age=5'
            })
            k.set_acl('public-read')
コード例 #2
0
ファイル: loaders.py プロジェクト: gwu-libraries/pywb
    def load(self, url, offset, length):
        if not s3_avail:  #pragma: no cover
           raise IOError('To load from s3 paths, ' +
                          'you must install boto: pip install boto')

        aws_access_key_id = self.aws_access_key_id
        aws_secret_access_key = self.aws_secret_access_key

        parts = urlsplit(url)

        if parts.username and parts.password:
            aws_access_key_id = unquote_plus(parts.username)
            aws_secret_access_key = unquote_plus(parts.password)
            bucket_name = parts.netloc.split('@', 1)[-1]
        else:
            bucket_name = parts.netloc

        if not self.s3conn:
            try:
                self.s3conn = connect_s3(aws_access_key_id, aws_secret_access_key)
            except Exception:  #pragma: no cover
                self.s3conn = connect_s3(anon=True)

        bucket = self.s3conn.get_bucket(bucket_name)

        key = bucket.get_key(parts.path)

        if offset == 0 and length == -1:
            headers = {}
        else:
            headers = {'Range': BlockLoader._make_range_header(offset, length)}

        # Read range
        key.open_read(headers=headers)
        return key
コード例 #3
0
ファイル: s3.py プロジェクト: AkiraKane/pandas
def get_filepath_or_buffer(filepath_or_buffer, encoding=None,
                           compression=None):

    # Assuming AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and AWS_S3_HOST
    # are environment variables
    parsed_url = parse_url(filepath_or_buffer)
    s3_host = os.environ.get('AWS_S3_HOST', 's3.amazonaws.com')

    try:
        conn = boto.connect_s3(host=s3_host)
    except boto.exception.NoAuthHandlerFound:
        conn = boto.connect_s3(host=s3_host, anon=True)

    b = conn.get_bucket(parsed_url.netloc, validate=False)
    if compat.PY2 and (compression == 'gzip' or
                       (compression == 'infer' and
                        filepath_or_buffer.endswith(".gz"))):
        k = boto.s3.key.Key(b, parsed_url.path)
        filepath_or_buffer = BytesIO(k.get_contents_as_string(
            encoding=encoding))
    else:
        k = BotoFileLikeReader(b, parsed_url.path, encoding=encoding)
        k.open('r')  # Expose read errors immediately
        filepath_or_buffer = k
    return filepath_or_buffer, None, compression
コード例 #4
0
def main():
    parser = argparse.ArgumentParser('Check the multipart upload status')
    parser.add_argument('-c', '--cancel', action="store_true", help='cancel all the outstanding ')
    parser.add_argument('-f', '--force', action="store_true", help='force all the outstanding ')

    args = vars(parser.parse_args())

    if exists(join(expanduser('~'), '.aws/credentials')):
        # This relies on a ~/.aws/credentials file holding the '<aws access key>', '<aws secret key>'
        LOG.info("Using ~/.aws/credentials")
        s3_connection = boto.connect_s3(profile_name='chiles')
    else:
        # This relies on a ~/.boto or /etc/boto.cfg file holding the '<aws access key>', '<aws secret key>'
        LOG.info("Using ~/.boto or /etc/boto.cfg")
        s3_connection = boto.connect_s3()

    bucket = s3_connection.get_bucket(CHILES_BUCKET_NAME)

    one_day_ago = datetime.datetime.now() - datetime.timedelta(hours=24)
    for item in bucket.list_multipart_uploads():
        LOG.info('key_name: {0}, initiated: {1}'.format(item.key_name, item.initiated))
        date_initiated = datetime.datetime.strptime(item.initiated, '%Y-%m-%dT%H:%M:%S.%fZ')
        if (date_initiated < one_day_ago and args['cancel']) or args['force']:
            LOG.info('Cancelling {0}'.format(item.key_name))
            bucket.cancel_multipart_upload(item.key_name, item.id)
コード例 #5
0
ファイル: aws.py プロジェクト: yamins81/yamutils
def download_s3_bucket(bucket_name, target, credentials=None):
    if credentials:
        conn = boto.connect_s3(*credentials)
    else:
        conn = boto.connect_s3()
    b = conn.get_bucket(bucket_name)
    L = list(b.list())
    L.sort(lambda x, y: x.name > y.name)
    L = L[::-1]
    f_suffix = '_$folder$'
    for l in L:
        n = l.name
        if n.endswith(f_suffix):
            dirname = n[:-len(f_suffix)]
            pathname = os.path.join(target, dirname)
            if not os.path.exists(pathname):
                print(n)
                os.mkdir(pathname)
        else:
            pathname = os.path.join(target, n)
            dirn = os.path.split(pathname)[0]
            if dirn and not os.path.isdir(dirn):
                print('making directory %s' % dirn)
                os.makedirs(dirn)
            if not os.path.exists(pathname):
                print(n)
                l.get_contents_to_filename(pathname)
コード例 #6
0
ファイル: S3.py プロジェクト: DangerTweaker/Nimbus
    def __init__(self, username, access_key, secret_key,
                       rate_limit=None, host=None):

        self.username = username
        self.access_key = access_key
        self.secret_key = secret_key
        self.rate_limit = rate_limit
        self.rate_limiter = RateLimiter(self.rate_limit)
        self.callbacks = CallbackAggregator()
        self.multipart_status_callbacks = CallbackAggregator()
        self.host = host
        self.logger = logging.getLogger(__name__)

        if self.rate_limit:
            self.callbacks.add_callback(self.rate_limiter)

        if self.host =='s3.amazonaws.com':
            self.connection = boto.connect_s3(self.access_key, self.secret_key)
        else:
            self.connection = boto.connect_s3(aws_access_key_id=self.access_key,
                              aws_secret_access_key=self.secret_key,
                              is_secure=False,
                              host=self.host,
                              port=8773,
                              calling_format=boto_s3_connection_class.OrdinaryCallingFormat(),
                              path="/services/Walrus")

        if not self.connection:
            raise S3AuthError("check access_key and secret_key")

        self.bucket = self.connection.lookup(username)
        if not self.bucket:
            raise S3AuthError("check access_key and secret_key")
コード例 #7
0
ファイル: s3-mp-download.py プロジェクト: bagger/s3-multipart
def main():
    logging.basicConfig(level=logging.INFO)
    args = parser.parse_args()
    log.debug("Got args: %s" % args)


    # Check that src is a valid S3 url
    split_rs = urlparse.urlsplit(args.src)
    if split_rs.scheme != "s3":
        raise ValueError("'%s' is not an S3 url" % args.src)

    # Check that dest does not exist
    if os.path.exists(args.dest):
        if args.force:
            os.remove(args.dest)
        else:
            raise ValueError("Destination file '%s' exists, specify -f to"
                             " overwrite" % args.dest)

    # Split out the bucket and the key
    s3 = boto.connect_s3()
    bucket = s3.lookup(split_rs.netloc)
    key = bucket.get_key(split_rs.path)

    # Determine the total size and calculate byte ranges
    conn = boto.connect_s3()
    resp = conn.make_request("HEAD", bucket=bucket, key=key)
    size = int(resp.getheader("content-length"))
    logging.info("Got headers: %s" % resp.getheaders())

    # Skipping multipart if file is less than 1mb
    if size < 1024 * 1024:
        t1 = time.time()
        key.get_contents_to_filename(args.dest)
        t2 = time.time() - t1
        log.info("Finished single-part download of %0.2fM in %0.2fs (%0.2fMbps)" %
                (size, t2, size/t2))
    else:
        # Touch the file
        fd = os.open(args.dest, os.O_CREAT)
        os.close(fd)
    
        num_parts = args.num_processes

        def arg_iterator(num_parts):
            for min_byte, max_byte in gen_byte_ranges(size, num_parts):
                yield (bucket.name, key.name, args.dest, min_byte, max_byte)

        s = size / 1024 / 1024.
        try:
            t1 = time.time()
            pool = Pool(processes=args.num_processes)
            pool.map_async(do_part_download, arg_iterator(num_parts)).get(9999999)
            t2 = time.time() - t1
            log.info("Finished downloading %0.2fM in %0.2fs (%0.2fMbps)" %
                    (s, t2, s/t2))
        except KeyboardInterrupt:
            log.info("User terminated")
        except Exception, err:
            log.error(err)
コード例 #8
0
ファイル: worker.py プロジェクト: JohnCEarls/DataDirac
    def _get_data_files(self):
        """
        Retrieves metadata and parsed dataframe files
            (generated by utilities/hddata_process.py) from S3
        """
        s2f = self._s3_to_fname
        while not op.exists(op.join( self.working_dir, s2f(self.meta_file))):
            try:
                conn = boto.connect_s3()
                b = conn.get_bucket(self.ds_bucket)
                k = Key(b)
                k.key = self.meta_file
                k.get_contents_to_filename(
                        op.join( self.working_dir, s2f(self.meta_file)))
            except:
                time.sleep(random.random())

        while not op.exists(op.join( self.working_dir, s2f(self.data_file))):
            conn = boto.connect_s3()
            try:
                b = conn.get_bucket(self.ds_bucket)
                k = Key(b)
                k.key = self.data_file
                k.get_contents_to_filename( op.join( self.working_dir,
                                                            s2f(self.data_file)) )
            except S3ResponseError:
                self.logger.exception( 'Master has not generated files' )
                raise
            except OSError:
                time.sleep(random.random())
コード例 #9
0
ファイル: aws_fetcher.py プロジェクト: JohnGriffiths/nidata
    def fetch(self, files, force=False, check=False, verbose=1):
        assert (self.profile_name or
                (self.access_key and self.secret_access_key))

        files = Fetcher.reformat_files(files)  # allows flexibility
        import boto
        if self.profile_name is not None:
            s3 = boto.connect_s3(profile_name=self.profile_name)
        elif (self.access_key is not None and
              self.secret_access_key is not None):
            s3 = boto.connect_s3(self.access_key, self.secret_access_key)

        bucket_names = np.unique([opts.get('bucket') for f, rk, opts in files])
        files_ = []
        for bucket_name in bucket_names:  # loop over bucket names: efficient
            if bucket_name:  # bucket requested
                buck = s3.get_bucket(bucket_name)
            else:  # default to first bucket
                buck = s3.get_all_buckets()[0]

            for file_, remote_key, opts in files:
                if opts.get('bucket') != bucket_name:
                    continue  # get all files from the current bucket only.
                target_file = op.join(self.data_dir, file_)
                key = buck.get_key(remote_key)
                if not key:
                    warnings.warn('Failed to find key: %s' % remote_key)
                    files_.append(None)
                else:
                    do_download = force or not op.exists(target_file)
                    try:
                        do_download = (do_download or
                                       (check and nib.load(
                                        target_file).get_data() is None))
                    except IOError as ioe:
                        if verbose > 0:
                            print("Warning: %s corrupted, re-downloading "
                                  "(Error=%s)" % (target_file, ioe))
                        do_download = True

                    if do_download:
                        # Ensure destination directory exists
                        destination_dir = op.dirname(target_file)
                        if not op.isdir(destination_dir):
                            if verbose > 0:
                                print("Creating base directory %s" % (
                                    destination_dir))
                            os.makedirs(destination_dir)

                        if verbose > 0:
                            print("Downloading [%s]/%s to %s." % (
                                bucket_name or 'default bucket',
                                remote_key,
                                target_file))
                        with open(target_file, 'wb') as fp:
                            cb = partial(test_cb, t0=time.time())
                            key.get_contents_to_file(fp, cb=cb, num_cb=None)

                    files_.append(target_file)
        return files_
コード例 #10
0
ファイル: aws.py プロジェクト: Curezhang/odo
def get_s3_connection(aws_access_key_id=None, aws_secret_access_key=None,
                      anon=False, profile_name=None, **kwargs):
    import boto

    if profile_name:
        return boto.connect_s3(profile_name=profile_name)

    cfg = boto.Config()

    if aws_access_key_id is None:
        aws_access_key_id = cfg.get('Credentials', 'aws_access_key_id')

    if aws_access_key_id is None:
        aws_access_key_id = os.environ.get('AWS_ACCESS_KEY_ID')

    if aws_secret_access_key is None:
        aws_secret_access_key = cfg.get('Credentials', 'aws_secret_access_key')

    if aws_secret_access_key is None:
        aws_secret_access_key = os.environ.get('AWS_SECRET_ACCESS_KEY')

    # anon is False but we didn't provide any credentials so try anonymously
    anon = (not anon and
            aws_access_key_id is None and
            aws_secret_access_key is None)
    return boto.connect_s3(aws_access_key_id, aws_secret_access_key,
                           anon=anon)
コード例 #11
0
ファイル: s3.py プロジェクト: vadikgo/ansible-docker-test
def get_s3_connection(aws_connect_kwargs, location, rgw, s3_url):
    if s3_url and rgw:
        rgw = urlparse(s3_url)
        s3 = boto.connect_s3(
            is_secure=rgw.scheme == 'https',
            host=rgw.hostname,
            port=rgw.port,
            calling_format=OrdinaryCallingFormat(),
            **aws_connect_kwargs
        )
    elif is_fakes3(s3_url):
        fakes3 = urlparse(s3_url)
        s3 = S3Connection(
            is_secure=fakes3.scheme == 'fakes3s',
            host=fakes3.hostname,
            port=fakes3.port,
            calling_format=OrdinaryCallingFormat(),
            **aws_connect_kwargs
        )
    elif is_walrus(s3_url):
        walrus = urlparse(s3_url).hostname
        s3 = boto.connect_walrus(walrus, **aws_connect_kwargs)
    else:
        aws_connect_kwargs['is_secure'] = True
        try:
            s3 = connect_to_aws(boto.s3, location, **aws_connect_kwargs)
        except AnsibleAWSError:
            # use this as fallback because connect_to_region seems to fail in boto + non 'classic' aws accounts in some cases
            s3 = boto.connect_s3(**aws_connect_kwargs)
    return s3
コード例 #12
0
ファイル: fileutil.py プロジェクト: kurzgood/seldon-server
    def download_s3(self,fromPath,toPath):
        """download from S3 to local folder

        Args:
            fromPath (str): S3 URL
            toPath (str): local folder
        """
        if fromPath.startswith("s3n://"):
            noSchemePath = fromPath[6:]
        elif fromPath.startswith("s3://"):
            noSchemePath = fromPath[5:]
        parts = noSchemePath.split('/')
        bucket = parts[0]
        s3path = noSchemePath[len(bucket)+1:]
        if self.key:
            self.conn = boto.connect_s3(self.key,self.secret)
        else:
            self.conn = boto.connect_s3()
        print bucket, s3path, toPath
        b = self.conn.get_bucket(bucket)
        for k in b.list(prefix=s3path):
            basename = os.path.basename(k.name)
            fnew = toPath+"/"+basename
            print "copying ",k.name,"to",fnew
            k.get_contents_to_filename(fnew)
コード例 #13
0
ファイル: s3.py プロジェクト: amuelli/cloudpebble
    def configure(self):
        if settings.AWS_ENABLED:
            if settings.AWS_S3_FAKE_S3 is None:
                # The host must be manually specified in Python 2.7.9+ due to
                # https://github.com/boto/boto/issues/2836 this bug in boto with .s in
                # bucket names.
                host = settings.AWS_S3_HOST if settings.AWS_S3_HOST else NoHostProvided

                self.s3 = boto.connect_s3(
                    settings.AWS_ACCESS_KEY_ID,
                    settings.AWS_SECRET_ACCESS_KEY,
                    host=host,
                    calling_format=OrdinaryCallingFormat()
                )
            else:
                host, port = (settings.AWS_S3_FAKE_S3.split(':', 2) + [80])[:2]
                port = int(port)
                self.s3 = boto.connect_s3("key_id", "secret_key", is_secure=False, port=port,
                                          host=host, calling_format=OrdinaryCallingFormat())
                _ensure_bucket_exists(self.s3, settings.AWS_S3_SOURCE_BUCKET)
                _ensure_bucket_exists(self.s3, settings.AWS_S3_EXPORT_BUCKET)
                _ensure_bucket_exists(self.s3, settings.AWS_S3_BUILDS_BUCKET)

            self.buckets = {
                'source': self.s3.get_bucket(settings.AWS_S3_SOURCE_BUCKET),
                'export': self.s3.get_bucket(settings.AWS_S3_EXPORT_BUCKET),
                'builds': self.s3.get_bucket(settings.AWS_S3_BUILDS_BUCKET),
            }
            self.configured = True
        else:
            self.s3 = None
            self.buckets = None
コード例 #14
0
ファイル: s3_audit.py プロジェクト: pikselpalette/pmcf
    def record_stack(self, stack, destination, credentials):
        """
        S3 implementation.  Uploads stack definition to configured S3 bucket.

        :param stack: stack definition
        :type stack: str.
        :param destination: destination to copy stack to
        :type destination: str.
        :param credentials: credentials for copy command
        :type credentials: dict.
        :returns:  boolean
        :raises: :class:`pmcf.exceptions.AuditException`
        """

        LOG.info('recording stack definition to s3://%s/%s',
                 credentials['audit_output'], destination)
        try:
            s3_conn = None
            if credentials.get('use_iam_profile'):
                s3_conn = boto.connect_s3()
            else:
                s3_conn = boto.connect_s3(
                    aws_access_key_id=credentials['access'],
                    aws_secret_access_key=credentials['secret']
                )
            bucket = s3_conn.get_bucket(credentials['audit_output'])
            k = boto.s3.key.Key(bucket)
            k.key = destination
            k.set_contents_from_string(stack)
        except (boto.exception.S3ResponseError,
                boto.exception.BotoServerError), exc:
            raise AuditException(exc)
コード例 #15
0
def upload_s3():
    """
    Upload jar file to s3
    """
    source_path = JARFILE
    source_size = os.stat(source_path).st_size

    # create bucket
    import boto
    conn = boto.connect_s3()
    bucket = conn.create_bucket(S3_BUCKET)

    # upload
    c = boto.connect_s3()
    b = c.get_bucket(S3_BUCKET)
    # Create a multipart upload request
    mp = b.initiate_multipart_upload(os.path.basename(source_path))

    # Use a chunk size of 5 MiB
    chunk_size = 5242880
    chunk_count = int(math.ceil(source_size / float(chunk_size)))

    # Send the file parts, using FileChunkIO to create a file-like object
    # that points to a certain byte range within the original file. We
    # set bytes to never exceed the original file size.
    for i in range(chunk_count):
        offset = chunk_size * i
        bytes = min(chunk_size, source_size - offset)
        with FileChunkIO(source_path, 'r', offset=offset,
                         bytes=bytes) as fp:
            mp.upload_part_from_file(fp, part_num=i + 1)

    # Finish the upload
    mp.complete_upload()
    print("Jar uploaded to S3 bucket " + S3_BUCKET)
コード例 #16
0
def Storage(accessKey,storageKey):
    # Storge bucket
    storageConn = get_storage_connection(accessKey,storageKey)

    storageTable = PrettyTable (["Storage Buckets", "Total Size"])
    storageTable.align["Storage Buckets"] = "l"
    storageTable.padding_width = 1
    

    boto.connect_s3(calling_format=OrdinaryCallingFormat())
    reservationStorage = storageConn.get_all_buckets()
    totalSize = 0

    for bucket in reservationStorage:
        s3Bucket = bucket.name
        if (bucket.name != 'esbundles-c14334.Ubuntu-1204-PE-300-agent'):           
            for key in bucket.list():
                size = key.size
                totalSize = totalSize + size
        storageTable.add_row([s3Bucket,""])        
    #print 'TotalSize: ' + str(totalSize/1073741824) + ' GB'
    totalSize = totalSize/1073741824
    storageTable.add_row(["---------------------------------------------------", "------------"])
    storageTable.add_row(["Total Size in GB", totalSize])
    print storageTable
    file2write.writelines('\n')
    file2write.writelines(str(storageTable))
コード例 #17
0
ファイル: __init__.py プロジェクト: thecubed/caching-s3-proxy
    def fetch_s3_object(self, bucket, key):
        if not self.no_cache:
            m = hashlib.md5()
            m.update(bucket+key)
            cache_key = m.hexdigest()

            if cache_key in self.cache:
                self.logger.debug('cache hit for %s' % cache_key)
                obj = self.cache[cache_key]
                print obj
                return obj
            else:
                self.logger.debug('cache miss for %s' % cache_key)

            conn = boto.connect_s3()
            b = conn.get_bucket(bucket)
            k = b.get_key(key)
            if k:
                obj = (k.get_contents_as_string(), self.build_s3_meta(k))
                self.cache[cache_key] = obj
                return obj
            else:
                return None, None
        else:
            conn = boto.connect_s3()
            k = conn.get_bucket(bucket).get_key(key)
            if k:
                meta = self.build_s3_meta(k)
                return k.get_contents_as_string(), meta
            else:
                return None, None
コード例 #18
0
ファイル: s3.py プロジェクト: onufryk/pistachio
def create_connection(settings):
  """ Creates an S3 connection using credentials is skipauth is false """
  if settings.get('skipauth'):
    conn = boto.connect_s3()
  else:
    conn = boto.connect_s3(settings['key'], settings['secret'])
  return conn
コード例 #19
0
ファイル: amazon.py プロジェクト: goodcodeguy/amazon-utils
def uploadThumbs(src, bucket_name, aws_access_id, aws_access_secret):
  updated_keys = 0

  # connect to the bucket
  conn = boto.connect_s3(aws_access_id, aws_access_secret)

  bucket = conn.get_bucket(bucket_name)

  for dirname, dirnames, filenames in os.walk(src):
    for filename in filenames:
      name, ext = filename.split('.')
      if ext == 'jpg':

        if updated_keys >= 100:
          # Close and Reopen connection
          conn.close()

          conn = boto.connect_s3(aws_access_id, aws_access_secret)

        key_dir = dirname.replace(os.path.abspath(src), '')
        sys.stdout.write("saving: " + key_dir + "/" + filename)
        k = Key(bucket)
        k.key = key_dir + "/" + filename
        k.set_contents_from_filename(os.path.join(dirname, filename),cb=done_cb)
        k.set_acl('public-read')
        sys.stdout.write("\n")
        sys.stdout.flush()
        updated_keys = updated_keys + 1
コード例 #20
0
ファイル: fileutil.py プロジェクト: Faye2014/seldon-server
    def download_s3(self,fromPath,toPath):
        """download from S3 to local folder

        Args:
            fromPath (str): S3 URL
            toPath (str): local folder
        """
        if fromPath.startswith("s3n://"):
            noSchemePath = fromPath[6:]
        elif fromPath.startswith("s3://"):
            noSchemePath = fromPath[5:]
        parts = noSchemePath.split('/')
        bucket = parts[0]
        s3path = noSchemePath[len(bucket)+1:]
        if self.aws_key:
            self.conn = boto.connect_s3(self.aws_key,self.aws_secret)
        else:
            self.conn = boto.connect_s3()
        b = self.conn.get_bucket(bucket)
        for k in b.list(prefix=s3path):
            if not k.name.endswith("/"):
                basename = os.path.basename(k.name)
                fnew = toPath+"/"+basename
                logger.info("copying %s to %s",k.name,fnew)
                k.get_contents_to_filename(fnew)
コード例 #21
0
ファイル: click.py プロジェクト: laterpay/rubberjack-cli
def rubberjack(ctx, application, organisation, region, sigv4_host, bucket):
    """
    Main entry point into the rubberjack CLI.
    """

    ctx.obj = {}

    ctx.obj['application'] = application
    ctx.obj['organisation'] = organisation
    ctx.obj['region'] = region_from_name(region)
    ctx.obj['application_name'] = application_name = "{organisation}-{application}".format(organisation=organisation, application=application)

    ctx.obj['dev_environment_name'] = "{application_name}-dev".format(application_name=application_name)
    ctx.obj['live_environment_name'] = "{application_name}-live".format(application_name=application_name)

    if bucket is None:
        bucket = "{organisation}-rubberjack-ebdeploy".format(organisation=organisation)

    ctx.obj['bucket'] = bucket

    # boto doesn't use a default of None, it uses NoHostProvided, and I struggled to pass that myself
    if sigv4_host:
        s3 = boto.connect_s3(host=sigv4_host)
    else:
        s3 = boto.connect_s3()
    ctx.obj['s3'] = s3
    ctx.obj['beanstalk'] = boto.beanstalk.layer1.Layer1(region=ctx.obj['region'])
コード例 #22
0
ファイル: fileutil.py プロジェクト: Faye2014/seldon-server
    def copy_s3_file(self,fromPath,bucket,path):
        """copy from local file to S3 

        Args:
            fromPath (str): local file
            bucket (str): S3 bucket
            path (str): S3 prefix to add to files
        """
        if self.aws_key:
            self.conn = boto.connect_s3(self.aws_key,self.aws_secret)
        else:
            self.conn = boto.connect_s3()
        b = self.conn.get_bucket(bucket)
        source_size = os.stat(fromPath).st_size
        # Create a multipart upload request
        uploadPath = path
        logger.info("uploading to bucket %s path %s",bucket,uploadPath)
        mp = b.initiate_multipart_upload(uploadPath)
        chunk_size = 10485760
        chunk_count = int(math.ceil(source_size / float(chunk_size)))
        for i in range(chunk_count):
            offset = chunk_size * i
            bytes = min(chunk_size, source_size - offset)
            with FileChunkIO(fromPath, 'r', offset=offset,bytes=bytes) as fp:
                logger.info("uploading to s3 chunk %d/%d",(i+1),chunk_count)
                mp.upload_part_from_file(fp, part_num=i + 1)
        # Finish the upload
        logger.info("completing transfer to s3")
        mp.complete_upload()
コード例 #23
0
ファイル: s3.py プロジェクト: haiwen/seafobj
    def do_connect(self):
        if self.conf.path_style_request:
            calling_format=boto.s3.connection.OrdinaryCallingFormat()
        else:
            calling_format=boto.s3.connection.SubdomainCallingFormat()

        if self.conf.host is None:
            # If version 4 signature is used, boto requires 'host' parameter
            # Also there is a bug in AWS Frankfurt that causes boto doesn't work.
            # The current work around is to give specific service address, like
            # s3.eu-central-1.amazonaws.com instead of s3.amazonaws.com.
            if self.conf.use_v4_sig:
                self.conn = boto.connect_s3(self.conf.key_id, self.conf.key,
                                            host='s3.%s.amazonaws.com' % self.conf.aws_region,
                                            is_secure=self.conf.use_https,
                                            calling_format=calling_format)
            else:
                self.conn = boto.connect_s3(self.conf.key_id, self.conf.key, is_secure=self.conf.use_https,
                                            calling_format=calling_format)
        else:
            self.conn = boto.connect_s3(self.conf.key_id, self.conf.key,
                                        host='%s' % self.conf.host,
                                        port=self.conf.port,
                                        is_secure=self.conf.use_https,
                                        calling_format=calling_format)

        self.bucket = self.conn.get_bucket(self.conf.bucket_name)
コード例 #24
0
ファイル: genome.py プロジェクト: curoverse/bcbio-nextgen
def _get_ref_from_galaxy_loc(name, genome_build, loc_file, galaxy_dt, need_remap,
                             galaxy_config, data):
    """Retrieve reference genome file from Galaxy *.loc file.

    Reads from tool_data_table_conf.xml information for the index if it
    exists, otherwise uses heuristics to find line based on most common setups.
    """
    refs = [ref for dbkey, ref in _galaxy_loc_iter(loc_file, galaxy_dt, need_remap)
            if dbkey == genome_build]
    remap_fn = alignment.TOOLS[name].remap_index_fn
    need_remap = remap_fn is not None
    if len(refs) == 0:
        # if we have an S3 connection, try to download
        try:
            import boto
            boto.connect_s3()
        except:
            raise ValueError("Could not find reference genome file %s %s" % (genome_build, name))
        logger.info("Downloading %s %s from AWS" % (genome_build, name))
        cur_ref = _download_prepped_genome(genome_build, data, name, need_remap)
    # allow multiple references in a file and use the most recently added
    else:
        cur_ref = refs[-1]
    if need_remap:
        assert remap_fn is not None, "%s requires remapping function from base location file" % name
        cur_ref = os.path.normpath(utils.add_full_path(cur_ref, galaxy_config["tool_data_path"]))
        cur_ref = remap_fn(os.path.abspath(cur_ref))
    return cur_ref
コード例 #25
0
ファイル: worker.py プロジェクト: JohnCEarls/tcDirac
    def _getDataFiles(self,file_master=0):
        """
        Retrieves metadata and parsed dataframe files
            (generated by utilities/hddata_process.py) from S3
        """
        comm = self._comm
        working_dir = self._working_dir
        data_source_bucket = self._datasource_bucket
       
        if comm.rank == file_master:
            if not op.exists(op.join( working_dir,'metadata.txt')):
                conn = boto.connect_s3()
                b = conn.get_bucket(data_source_bucket)
                k = Key(b)
                k.key = 'metadata.txt'
                k.get_contents_to_filename(op.join( working_dir,'metadata.txt'))

        if comm.rank == file_master:
            if not op.exists(op.join( working_dir, 'trimmed_dataframe.pandas')):
                conn = boto.connect_s3()
                b = conn.get_bucket(self._working_bucket)
                k = Key(b)
                k.key ='trimmed_dataframe.pandas'
                k.get_contents_to_filename(op.join( working_dir,'trimmed_dataframe.pandas'))
        comm.barrier()
コード例 #26
0
def connect(aws_access_key_id=None, aws_secret_access_key=None):
    """ uses api_key and secret_key if available or
    falls back on ENV variables AWS_ACCESS_KEY_ID && AWS_SECRET_ACCESS_KEY """
    if aws_access_key_id or aws_secret_access_key is None:
        return boto.connect_s3()
    else:
        log.debug('Falling back to ENV variables $AWS_ACCESS_KEY && $AWS_SECRET_KEY')
        return boto.connect_s3(aws_access_key_id, aws_secret_access_key)
コード例 #27
0
ファイル: uri.py プロジェクト: 50onRed/smr
def get_s3_bucket(bucket_name, config):
    if bucket_name not in S3_BUCKETS:
        if config.aws_access_key and config.aws_secret_key:
            s3conn = boto.connect_s3(config.aws_access_key, config.aws_secret_key)
        else:
            s3conn = boto.connect_s3() # use local boto config or IAM profile
        S3_BUCKETS[bucket_name] = s3conn.get_bucket(bucket_name)
    return S3_BUCKETS[bucket_name]
コード例 #28
0
    def s3(self):
        if not self.__s3_conn:
            if self.config.AWS:
                self.__s3_conn = boto.connect_s3(self.config.AWS["key"], self.config.AWS["secret_key"])
            else:
                self.__s3_conn = boto.connect_s3()

        return self.__s3_conn
コード例 #29
0
ファイル: CFStack.py プロジェクト: ryno75/cumulus
    def __init__(self, mega_stack_name, name, params, template_name, region,
                 sns_topic_arn, tags=None, depends_on=None):
        self.logger = logging.getLogger(__name__)
        if mega_stack_name == name:
            self.cf_stack_name = name
        else:
            self.cf_stack_name = "%s-%s" % (mega_stack_name, name)
        self.mega_stack_name = mega_stack_name
        self.name = name
        self.yaml_params = params
        self.params = {}
        self.template_name = template_name
        self.template_body = ''
        self.template_url = False
        if depends_on is None:
            self.depends_on = None
        else:
            self.depends_on = []
            for dep in depends_on:
                if dep == mega_stack_name:
                    self.depends_on.append(dep)
                else:
                    self.depends_on.append("%s-%s" % (mega_stack_name, dep))
        self.region = region
        self.sns_topic_arn = sns_topic_arn

        # Safer than setting default value for tags = {}
        if tags is None:
            self.tags = {}
        else:
            self.tags = tags

        try:
            # catch S3 url template names
            m = re.match(r'(https?|s3)://([^/]+)/(.+$)', self.template_name)
            if m:
                protocol, bucket, key = m.groups()
                if protocol == 's3':
                    connect_s3().get_bucket(bucket).get_key(key).read()
                else:
                    if not requests.get(self.template_name).ok:
                        raise Exception
            else:
                open(self.template_name, 'r')
        except:
            self.logger.critical("Failed to open template file %s for stack %s"
                                 % (self.template_name, self.name))
            exit(1)

        # check params is a dict if set
        if self.yaml_params and type(self.yaml_params) is not dict:
            self.logger.critical(
                "Parameters for stack %s must be of type dict not %s",
                self.name, type(self.yaml_params))
            exit(1)

        self.cf_stacks = {}
        self.cf_stacks_resources = {}
コード例 #30
0
ファイル: views.py プロジェクト: milinp/WebScraper-V-1.0--
def home(request):
	global location
	userInput = UserInputForm

	if 'import' in request.POST:
		newdoc = Document(docfile = request.FILES['upload'])
		newdoc.save()
		location = newdoc.path()

		# creating S3 bucket connection
		conn = boto.connect_s3('AKIAIJZ56E33VC2GBG3Q', 'xfSWxuK9uGAsRwtwdJgIPBhiye0Z3ka5oRqRa8FD')
		bucket = conn.create_bucket('client1.bucket')
		k = Key(bucket)

		filename = str(request.FILES['upload'])
		filenameKey = re.sub('\.txt$', '', filename)

		print filenameKey
		
		k.key = filenameKey 
		k.set_contents_from_filename(location)
		return HttpResponseRedirect(reverse('upload.views.home'))
	else:
		form = DocumentForm() # An empty, unbound form

	if 'user_input' in request.POST:
		form = UserInputForm(request.POST)
		if form.is_valid():
			form.save()
			myfile = open(os.path.dirname(os.path.abspath(upload.__file__))+ "/media/Watchlists/userinput.txt", 'a')
			myfile.write(request.POST['keyword'] + "\n")
			myfile.close()

			location = os.path.dirname(os.path.abspath(upload.__file__))+ "/media/Watchlists/userinput.txt"

			conn = boto.connect_s3('AKIAIJZ56E33VC2GBG3Q', 'xfSWxuK9uGAsRwtwdJgIPBhiye0Z3ka5oRqRa8FD')
			bucket = conn.create_bucket('client1.bucket')
			k = Key(bucket)

			filenameKey = "userinput"

			print filenameKey
		
			k.key = filenameKey 
			k.set_contents_from_filename(location)
			return HttpResponseRedirect(reverse('upload.views.home'))
		else:
			form = UserInputForm()

	# Load documents for the list page
	documents = Document.objects.all()

	# Rendner list page with the documents and the form
	return render_to_response(
		'upload/parallax.html',
		{'documents': documents, 'form' : form, 'userInput' : userInput},
		context_instance = RequestContext(request)
	)