Beispiel #1
0
def add_thumbnail(file_name):
    ''' add a thumbnail for the specific file'''
    src_uri = boto.storage_uri(
        '{}/{}'.format(conf.photos_bucket_name, file_name), 'gs')
    dest_uri = boto.storage_uri(
        '{}/{}'.format(conf.thumbnails_bucket_name, file_name), 'gs')

    try:
        new_key = dest_uri.new_key()
    except boto.exception.NoAuthHandlerFound as e:
        logging.error(e)
        return None

    # Create a file-like object for holding the photo contents.
    photo = StringIO.StringIO()
    src_uri.get_key().get_file(photo)

    thumbnail = StringIO.StringIO()
    im = Image.open(photo)
    im.thumbnail((260, 260))
    im.save(thumbnail, 'JPEG')
    thumbnail.seek(0)
    # save the thumbnail
    try:
        new_key.set_contents_from_file(thumbnail)
        new_key.make_public()
    except boto.exception.GSResponseError as e:
        logging.error(e)
        # Do we have the credentials file set up?
        boto_cred_file = os.path.expanduser('~') + '/.boto'
        if not os.path.exists(boto_cred_file):
            logging.error(
                'Credentials file {} was not found.'.format(boto_cred_file))
Beispiel #2
0
    def x_test_basic(self):
        from ambry.client.bigquery import BigQuery
        bg = BigQuery()
        
        import StringIO
        import os
        import shutil
        import tempfile
        import time
        from gslib.third_party.oauth2_plugin import oauth2_plugin
        
        import boto
        
        # URI scheme for Google Cloud Storage.
        GOOGLE_STORAGE = 'gs'
        # URI scheme for accessing local files.
        LOCAL_FILE = 'file'
        project_id = 128975330021

        header_values = {
            "x-goog-api-version": "2",
            "x-goog-project-id": project_id
        }

        uri = boto.storage_uri('', GOOGLE_STORAGE)
        for bucket in uri.get_all_buckets():
            print bucket.name

        uri = boto.storage_uri(bucket.name, GOOGLE_STORAGE)
        for obj in uri.get_bucket():
            print '%s://%s/%s' % (uri.scheme, uri.bucket_name, obj.name)
Beispiel #3
0
def read_file(file_name, bucket_name):
    
    if(DEBUG): print "Reading File: " + file_name
 
    # Create a file-like object for holding the object contents.
    object_contents = StringIO.StringIO()

    src_uri = boto.storage_uri(bucket_name + '/' + file_name, GOOGLE_STORAGE)

    # get_file() doesn't return the file contents
    # it writes the file contents to "object_contents" instead
    src_uri.get_key(headers=HEADER_VALUES).get_file(object_contents)

    local_dst_uri = boto.storage_uri(file_name, LOCAL_FILE)

    bucket_dst_uri = boto.storage_uri(bucket_name + '/' + file_name, GOOGLE_STORAGE)

    for dst_uri in (local_dst_uri, bucket_dst_uri):
        object_contents.seek(0)
        dst_uri.new_key(headers=HEADER_VALUES).set_contents_from_file(object_contents)

    object_contents.close()

    if(DEBUG): print "Read data from GCS to file: " + file_name

    return True
 def _SetStatus(self, status):
   if self.base:
     uri_string = 'gs://%s/output/stitch.status' % self.base
     logger.info('Setting status %s to %s', status, uri_string)
     uri = boto.storage_uri(uri_string)
     uri.connect(config.gs_access_key,config.gs_secret_key)
     uri.new_key().set_contents_from_string(
         status, policy='public-read', headers={'Content-Type': 'text/plain'})
         
     # Construct a state object to upload
     state = {}
     state['status'] = status
     state['update_time'] = time.time()
     state['output_base'] = '%s/output' % self.base
     state['input'] = [{ 'full': os.path.basename(i), 'thumb': os.path.basename(t)} 
         for (i, t) in zip(self.input_files, self.input_thumbs)]
     if status == 'DONE':
       state['output'] = { 'full': 'stitch.jpg', 'thumb': 'stitch-thumb.jpg' }
     if status in ('DONE', 'FAILED'):
       state['log'] = 'stitch.log'
     uri_string = 'gs://%s/output/stitch.state' % self.base
     state_json = json.dumps(state, indent=2)
     logger.info('Uploading state to %s\n%s', uri_string, state_json)
     uri = boto.storage_uri(uri_string)
     uri.connect(config.gs_access_key,config.gs_secret_key)
     uri.new_key().set_contents_from_string(
         state_json, policy='public-read', headers={'Content-Type': 'text/plain'})
   else:
     logger.error('No upload path for status %s', status)
Beispiel #5
0
        def _Check1():
            listing1 = self.RunGsUtil(
                ['ls', '-la', suri(bucket1_uri)],
                return_stdout=True).split('\n')
            listing2 = self.RunGsUtil(
                ['ls', '-la', suri(bucket2_uri)],
                return_stdout=True).split('\n')
            # 2 lines of listing output, 1 summary line, 1 empty line from \n split.
            self.assertEquals(len(listing1), 4)
            self.assertEquals(len(listing2), 4)

            # First object in each bucket should match in size and version-less name.
            size1, _, uri_str1, _ = listing1[0].split()
            self.assertEquals(size1, str(len('data0')))
            self.assertEquals(storage_uri(uri_str1).object_name, 'k')
            size2, _, uri_str2, _ = listing2[0].split()
            self.assertEquals(size2, str(len('data0')))
            self.assertEquals(storage_uri(uri_str2).object_name, 'k')

            # Similarly for second object in each bucket.
            size1, _, uri_str1, _ = listing1[1].split()
            self.assertEquals(size1, str(len('longer_data1')))
            self.assertEquals(storage_uri(uri_str1).object_name, 'k')
            size2, _, uri_str2, _ = listing2[1].split()
            self.assertEquals(size2, str(len('longer_data1')))
            self.assertEquals(storage_uri(uri_str2).object_name, 'k')
Beispiel #6
0
def upload_to_gc(fileobj):
    filename = helpers.generate_filename(fileobj.name)
    ext = os.path.splitext(fileobj.name)[-1].lower()

    #
    ext = save_tmp_file(fileobj, filename, ext)
    dst_uri = boto.storage_uri(settings.GS_STORAGE_BUCKET + '/' +
                               filename + ext, settings.GS_URI_SCHEME)

    dst_uri.new_key().set_contents_from_filename("/tmp/" + filename + ext)
    gcloud_path = dst_uri.object_name
    os.remove("/tmp/" + filename + ext)
    if ext in ['.jpg']:
        # upload thumbnail
        dst_uri = boto.storage_uri(settings.GS_STORAGE_BUCKET + '/' +
                                   filename + ext + ".thumbnail.jpg",
                                   settings.GS_URI_SCHEME)
        dst_uri.new_key().set_contents_from_filename("/tmp/" + filename + ext +
                                                     '.thumbnail.jpg')

        os.remove("/tmp/" + filename + ext + ".thumbnail.jpg")
    elif ext in ['.dcm', '.dicom']:
        # upload thumbnail
        dst_uri = boto.storage_uri(settings.GS_STORAGE_BUCKET + '/' +
                                   filename + ext + ".thumbnail.jpg",
                                   settings.GS_URI_SCHEME)
        dst_uri.new_key().set_contents_from_filename("/tmp/" + filename + ext +
                                                     '.thumbnail.png')

        os.remove("/tmp/" + filename + ext + ".thumbnail.png")

    return (gcloud_path, ext)
Beispiel #7
0
def open(path, mode="rb", **kw):
    uri = urlparse(path)

    if uri.scheme in ['file', 's3', '']:
        return smart_open(path, mode=mode, **kw)
    elif uri.scheme in ['gs']:
        if mode in ('r', 'rb'):
            storage_uri = boto.storage_uri(uri.netloc, uri.scheme)
            bucket = storage_uri.get_bucket(uri.netloc)
            key = bucket.get_key(uri.path)
            if key is None:
                raise KeyError(uri.path)
            return GSOpenRead(key, **kw)
        elif mode in ('w', 'wb'):
            storage_uri = boto.storage_uri(uri.netloc + '/' + uri.path,
                                           uri.scheme)
            key = storage_uri.new_key()
            if key is None:
                raise KeyError(uri.path)
            return GSOpenWrite(key, **kw)
        else:
            raise NotImplementedError(
                "file mode %s not supported for %r scheme", mode, uri.scheme)
    else:
        raise NotImplementedError("scheme %r is not supported", uri.scheme)
Beispiel #8
0
 def test_invalid_scheme(self):
     uri_str = 'mars://bucket/object'
     try:
         boto.storage_uri(uri_str, validate=False,
             suppress_consec_slashes=False)
     except InvalidUriError as e:
         self.assertIn('Unrecognized scheme', e.message)
Beispiel #9
0
def download(srcName, dstName) :
    "Download the files."
    src_uri = boto.storage_uri(bucketName + "/" + srcName, "gs")
    dst_uri = boto.storage_uri(dstName, "file")
    
    "Append the object name to the directory name."
    dst_key_name = dst_uri.object_name# + os.sep + src_uri.object_name
    
    "Use the new destination key name to create a new destination URI."
    new_dst_uri = dst_uri.clone_replace_name(dst_key_name)
    print new_dst_uri
    
    "Create a new destination key object."
    dst_key = new_dst_uri.new_key()
    
    "Retrieve the source key and create a source key object."
    src_key = src_uri.get_key()
    
    "Create a temporary file to hold our copy operation."
    tmp = tempfile.TemporaryFile()
    src_key.get_file(tmp)
    tmp.seek(0)

    "Download the object."
    dst_key.set_contents_from_file(tmp)
    return
Beispiel #10
0
def download(srcName, dstName):
    "Download the files."
    src_uri = boto.storage_uri(bucketName + "/" + srcName, "gs")
    dst_uri = boto.storage_uri(dstName, "file")

    "Append the object name to the directory name."
    dst_key_name = dst_uri.object_name  # + os.sep + src_uri.object_name

    "Use the new destination key name to create a new destination URI."
    new_dst_uri = dst_uri.clone_replace_name(dst_key_name)
    print new_dst_uri

    "Create a new destination key object."
    dst_key = new_dst_uri.new_key()

    "Retrieve the source key and create a source key object."
    src_key = src_uri.get_key()

    "Create a temporary file to hold our copy operation."
    tmp = tempfile.TemporaryFile()
    src_key.get_file(tmp)
    tmp.seek(0)

    "Download the object."
    dst_key.set_contents_from_file(tmp)
    return
Beispiel #11
0
    def get_all_files(self):
        try:
            uri = boto.storage_uri('', GOOGLE_STORAGE)
            files = []
            bucketmap = dict()
            for b in uri.get_all_buckets():
                bucket_dic, map_path = self.get_files_from_bucket(b)
                if map_path != '':
                    # print map_path
                    cipher = AES.new(process_cipherkey('password'), AES.MODE_ECB, CYPHER_IV)
                    uri = boto.storage_uri(b.name + '/' + map_path, GOOGLE_STORAGE)
                    data = cipher.decrypt(uri.get_key().get_contents_as_string())
                    for line in data.split('\n'):
                        # print line
                        if line == 'bucket to dir':
                            continue
                        elif line == 'dir to bucket':
                            break
                        else:
                            dirs = line.split()
                            bucketmap[dirs[0]] = eval(dirs[1])

                for prefix in bucket_dic:
                    filename = b.name + '/' + prefix + '_' + bucket_dic[prefix][0] + '_' + str(bucket_dic[prefix][1])
                    files.append(filename)
                    
            return bucketmap, files 
        except IOError:
            print 'Cannot get google files'
Beispiel #12
0
def get_userobjects(user=None, buckets=[]):
    """
    retrieves objects matching the pattern with
    user. default generic case provided to return all
    files, but not currently used in the scope of project
    """
    logging.debug('get userobject bucket=%s user=%s %d' %
                  (str(buckets), str(user), len(buckets)))
    objects = []
    error_str = ''
    pattern = None
    if user != None:
        pattern = config.Status_log_pattern % (user)
    else:
        pattern = None
    try:
        ## get all the buckets under the storage with the given key id ##
        if len(buckets) == 0:
            logging.debug('querying all buckets')
            uri = boto.storage_uri('', config.Google_storage)
            buckets = [bucket.name for bucket in uri.get_all_buckets()]
        ## list of objects ##
        for bucket in buckets:
            uri = boto.storage_uri(bucket, config.Google_storage)
            for obj_uri in uri.get_bucket():
                if pattern != None:
                    m = re.match(pattern, obj_uri.name)
                    if m != None:
                        objects.append(Bunch(obj_uri=obj_uri, pid=m.group(2)))
                else:
                    #Note this case is currently not used
                    objects.append(Bunch(obj_uri=obj_uri, pid=None))
    except AttributeError, e:
        error_str = 'GSCloud::get_userlist Attribute Error %s' % (e)
        logging.error(error_str)
Beispiel #13
0
def download_object(bucket_name, object_name, debug_level):
    '''
    Performs a GET Object operation to download an object from the specified bucket.
    @param bucket_name: The name of the bucket that contains the object to download.
    @param object_name: The name of the object to download.
    @param debug_level: The level of debug messages to be printed.     Input parameters:
    '''
    try:

        # Destination directory.
        # Replace the directory with one of your choice.
        dest_dir = os.getenv("HOME") + "/tmp/downloads/"

        # Define the object URI
        uri = boto.storage_uri(bucket_name + "/" + object_name, GOOGLE_STORAGE, debug_level)

        # Create a file-like object to hold the object contents.
        object_contents = StringIO.StringIO()

        # Get the object contents.
        uri.get_key().get_file(object_contents)

        # Set the local destination path.
        local_dest_uri = boto.storage_uri(
            os.path.join(dest_dir, object_name), LOCAL_FILE)
        
        # Download the object to the local destination.
        object_contents.seek(0)
        local_dest_uri.new_key().set_contents_from_file(object_contents)
        object_contents.close()

    except boto.exception, e:
        logging.error("download_object, error occurred: %s", e)
Beispiel #14
0
    def download_object(bucket, filename, dest_dir):
        print 'FUNCTION: DOWNLOAD OBJECT'
        print ''

        # Call list_object first
        bucket_name = bucket.name #self.list_object()

        #filename = raw_input("Which file to download: ")
        #dest_dir = raw_input("Input the downloading directory: ")
        src_uri = boto.storage_uri(bucket_name + '/' + filename, GOOGLE_STORAGE)

        # Create a file-like object for holding the object contents.
        object_contents = StringIO.StringIO()

        # The unintuitively-named get_file() doesn't return the object
        # contents; instead, it actually writes the contents to
        # object_contents.
        try:
            src_uri.get_key().get_file(object_contents)

            dst_uri = boto.storage_uri(os.path.join(dest_dir, filename), LOCAL_FILE)
            object_contents.seek(0)
            dst_uri.new_key().set_contents_from_file(object_contents)

            object_contents.close()
        except:
            msg = 'Sorry, but fail to download file: "%s".' % filename
            return msg
        else:
            msg = 'Successfully downloaded "%s"!' % filename
            return msg
Beispiel #15
0
 def test_invalid_scheme(self):
     uri_str = 'mars://bucket/object'
     try:
         boto.storage_uri(uri_str, validate=False,
             suppress_consec_slashes=False)
     except InvalidUriError as e:
         self.assertIn('Unrecognized scheme', e.message)
Beispiel #16
0
    def x_test_basic(self):
        from ambry.client.bigquery import BigQuery
        bg = BigQuery()

        import StringIO
        import os
        import shutil
        import tempfile
        import time
        from gslib.third_party.oauth2_plugin import oauth2_plugin

        import boto

        # URI scheme for Google Cloud Storage.
        GOOGLE_STORAGE = 'gs'
        # URI scheme for accessing local files.
        LOCAL_FILE = 'file'
        project_id = 128975330021

        header_values = {
            "x-goog-api-version": "2",
            "x-goog-project-id": project_id
        }

        uri = boto.storage_uri('', GOOGLE_STORAGE)
        for bucket in uri.get_all_buckets():
            print bucket.name

        uri = boto.storage_uri(bucket.name, GOOGLE_STORAGE)
        for obj in uri.get_bucket():
            print '%s://%s/%s' % (uri.scheme, uri.bucket_name, obj.name)
Beispiel #17
0
def get_userobjects(user=None, buckets=[]):
    """
    retrieves objects matching the pattern with
    user. default generic case provided to return all
    files, but not currently used in the scope of project
    """
    logging.debug('get userobject bucket=%s user=%s %d' %(str(buckets), str(user), len(buckets))) 
    objects   = []
    error_str = ''
    pattern   = None
    if user != None:
        pattern=config.Status_log_pattern%(user)
    else:
        pattern=None
    try:
        ## get all the buckets under the storage with the given key id ##
        if len(buckets) == 0:
            logging.debug('querying all buckets')
            uri     = boto.storage_uri('', config.Google_storage)
            buckets = [bucket.name for bucket in uri.get_all_buckets()]
        ## list of objects ##
        for bucket in buckets:
            uri = boto.storage_uri(bucket, config.Google_storage)
            for obj_uri in uri.get_bucket():
                if pattern != None:
                    m = re.match(pattern, obj_uri.name)
                    if m != None:
                        objects.append(Bunch(obj_uri=obj_uri, pid=m.group(2)))
                else:
                    #Note this case is currently not used
                    objects.append(Bunch(obj_uri=obj_uri,pid=None))
    except AttributeError, e:
        error_str = 'GSCloud::get_userlist Attribute Error %s'% (e)
        logging.error(error_str)
Beispiel #18
0
def read_file(file_name, bucket_name):
    if (DEBUG):
        print "Reading File: " + file_name

    # Create a file-like object for holding the object contents.
    object_contents = StringIO.StringIO()

    src_uri = boto.storage_uri(bucket_name + '/' + file_name, GOOGLE_STORAGE)

    # get_file() doesn't return the file contents
    # it writes the file contents to "object_contents" instead
    src_uri.get_key(headers=HEADER_VALUES).get_file(object_contents)

    local_dst_uri = boto.storage_uri(file_name, LOCAL_FILE)

    bucket_dst_uri = boto.storage_uri(bucket_name + '/' + file_name,
                                      GOOGLE_STORAGE)

    for dst_uri in (local_dst_uri, bucket_dst_uri):
        object_contents.seek(0)
        dst_uri.new_key(
            headers=HEADER_VALUES).set_contents_from_file(object_contents)

    object_contents.close()

    if (DEBUG):
        print "Read data from GCS to file: " + file_name

    return True
Beispiel #19
0
 def test_roundtrip_versioned_gs_object_uri_parsed(self):
     uri_str = 'gs://bucket/obj#1359908801674000'
     uri = boto.storage_uri(uri_str, validate=False,
         suppress_consec_slashes=False)
     roundtrip_uri = boto.storage_uri(uri.uri, validate=False,
         suppress_consec_slashes=False)
     self.assertEqual(uri.uri, roundtrip_uri.uri)
     self.assertEqual(uri.is_version_specific, True)
 def _prepareTestFile(self, bucket, size=None):
     import boto
     fileName = 'testfile_%s' % uuid.uuid4()
     uri = 'gs://%s/%s' % (bucket.name, fileName)
     if size:
         with open('/dev/urandom', 'r') as readable:
             boto.storage_uri(uri).set_contents_from_string(readable.read(size))
     return uri
Beispiel #21
0
 def test_roundtrip_versioned_gs_object_uri_parsed(self):
     uri_str = 'gs://bucket/obj#1359908801674000'
     uri = boto.storage_uri(uri_str, validate=False,
         suppress_consec_slashes=False)
     roundtrip_uri = boto.storage_uri(uri.uri, validate=False,
         suppress_consec_slashes=False)
     self.assertEqual(uri.uri, roundtrip_uri.uri)
     self.assertEqual(uri.is_version_specific, True)
Beispiel #22
0
def upload_logical_node( logical_id, executable_path, parameter_file_path ):
  executable_file = open( executable_path, 'r' )
  executable_uri = boto.storage_uri( configs.bucket + '/' + logical_id + '/executable', GOOGLE_STORAGE)
  executable_uri.new_key().set_contents_from_file( executable_file )

  parameter_file = open( parameter_file_path, 'r' )
  parameter_file_uri = boto.storage_uri( configs.bucket + '/' + logical_id + '/parameters.default', GOOGLE_STORAGE)
  parameter_file_key = parameter_file_uri.get_key()
Beispiel #23
0
 def _prepareTestFile(self, bucket, size=None):
     import boto
     fileName = 'testfile_%s' % uuid.uuid4()
     uri = 'gs://%s/%s' % (bucket.name, fileName)
     if size:
         with open('/dev/urandom', 'r') as readable:
             boto.storage_uri(uri).set_contents_from_string(readable.read(size))
     return uri
Beispiel #24
0
def remove_metadata(logical_id):
    uri = boto.storage_uri(configs.bucket + "/" + logical_id + "/instances.txt", GOOGLE_STORAGE)
    uri.delete()

    uri = boto.storage_uri(configs.bucket + "/" + logical_id + "/executable", GOOGLE_STORAGE)
    uri.delete()

    uri = boto.storage_uri(configs.bucket + "/" + logical_id + "/params.default", GOOGLE_STORAGE)
    uri.delete()
  def CreateBucket(self, bucket_name=None, test_objects=0, storage_class=None,
                   provider=None, prefer_json_api=False):
    """Creates a test bucket.

    The bucket and all of its contents will be deleted after the test.

    Args:
      bucket_name: Create the bucket with this name. If not provided, a
                   temporary test bucket name is constructed.
      test_objects: The number of objects that should be placed in the bucket.
                    Defaults to 0.
      storage_class: storage class to use. If not provided we us standard.
      provider: Provider to use - either "gs" (the default) or "s3".
      prefer_json_api: If true, use the JSON creation functions where possible.

    Returns:
      StorageUri for the created bucket.
    """
    if not provider:
      provider = self.default_provider

    if prefer_json_api and provider == 'gs':
      json_bucket = self.CreateBucketJson(bucket_name=bucket_name,
                                          test_objects=test_objects,
                                          storage_class=storage_class)
      bucket_uri = boto.storage_uri(
          'gs://%s' % json_bucket.name.encode(UTF8).lower(),
          suppress_consec_slashes=False)
      self.bucket_uris.append(bucket_uri)
      return bucket_uri

    bucket_name = bucket_name or self.MakeTempName('bucket')

    bucket_uri = boto.storage_uri('%s://%s' % (provider, bucket_name.lower()),
                                  suppress_consec_slashes=False)

    if provider == 'gs':
      # Apply API version and project ID headers if necessary.
      headers = {'x-goog-api-version': self.api_version}
      headers[GOOG_PROJ_ID_HDR] = PopulateProjectId()
    else:
      headers = {}

    # Parallel tests can easily run into bucket creation quotas.
    # Retry with exponential backoff so that we create them as fast as we
    # reasonably can.
    @Retry(StorageResponseError, tries=7, timeout_secs=1)
    def _CreateBucketWithExponentialBackoff():
      bucket_uri.create_bucket(storage_class=storage_class, headers=headers)

    _CreateBucketWithExponentialBackoff()
    self.bucket_uris.append(bucket_uri)
    for i in range(test_objects):
      self.CreateObject(bucket_uri=bucket_uri,
                        object_name=self.MakeTempName('obj'),
                        contents='test %d' % i)
    return bucket_uri
Beispiel #26
0
 def _getUrlForTestFile(cls, size=None):
     import boto
     fileName = 'testfile_%s' % uuid.uuid4()
     bucket = cls._createExternalStore()
     uri = 'gs://%s/%s' % (bucket.name, fileName)
     if size:
         with open('/dev/urandom', 'r') as readable:
             boto.storage_uri(uri).set_contents_from_string(readable.read(size))
     return uri
  def CreateBucket(self, bucket_name=None, test_objects=0, storage_class=None,
                   provider=None, prefer_json_api=False):
    """Creates a test bucket.

    The bucket and all of its contents will be deleted after the test.

    Args:
      bucket_name: Create the bucket with this name. If not provided, a
                   temporary test bucket name is constructed.
      test_objects: The number of objects that should be placed in the bucket.
                    Defaults to 0.
      storage_class: storage class to use. If not provided we us standard.
      provider: Provider to use - either "gs" (the default) or "s3".
      prefer_json_api: If true, use the JSON creation functions where possible.

    Returns:
      StorageUri for the created bucket.
    """
    if not provider:
      provider = self.default_provider

    if prefer_json_api and provider == 'gs':
      json_bucket = self.CreateBucketJson(bucket_name=bucket_name,
                                          test_objects=test_objects,
                                          storage_class=storage_class)
      bucket_uri = boto.storage_uri(
          'gs://%s' % json_bucket.name.encode(UTF8).lower(),
          suppress_consec_slashes=False)
      self.bucket_uris.append(bucket_uri)
      return bucket_uri

    bucket_name = bucket_name or self.MakeTempName('bucket')

    bucket_uri = boto.storage_uri('%s://%s' % (provider, bucket_name.lower()),
                                  suppress_consec_slashes=False)

    if provider == 'gs':
      # Apply API version and project ID headers if necessary.
      headers = {'x-goog-api-version': self.api_version}
      headers[GOOG_PROJ_ID_HDR] = PopulateProjectId()
    else:
      headers = {}

    # Parallel tests can easily run into bucket creation quotas.
    # Retry with exponential backoff so that we create them as fast as we
    # reasonably can.
    @Retry(StorageResponseError, tries=7, timeout_secs=1)
    def _CreateBucketWithExponentialBackoff():
      bucket_uri.create_bucket(storage_class=storage_class, headers=headers)

    _CreateBucketWithExponentialBackoff()
    self.bucket_uris.append(bucket_uri)
    for i in range(test_objects):
      self.CreateObject(bucket_uri=bucket_uri,
                        object_name=self.MakeTempName('obj'),
                        contents='test %d' % i)
    return bucket_uri
Beispiel #28
0
 def _getUrlForTestFile(cls, size=None):
     import boto
     fileName = 'testfile_%s' % uuid.uuid4()
     bucket = cls._createExternalStore()
     uri = 'gs://%s/%s' % (bucket.name, fileName)
     if size:
         with open('/dev/urandom', 'r') as readable:
             boto.storage_uri(uri).set_contents_from_string(
                 readable.read(size))
     return uri
Beispiel #29
0
def upload_file(file_path, bucket, new_filename=None, thumb_bucket=None):
    '''
    Upload the file object to a bucket.
    Add the original file path to its object metadata.
    '''
    if not new_filename:
        fn = get_basename(file_path)
    else:
        fn = new_filename

    dest_uri = boto.storage_uri(bucket + '/' + fn, 'gs')

    try:
        new_key = dest_uri.new_key()
    except boto.exception.NoAuthHandlerFound as e:
        logging.error(e)
        return None

    try:
        file_obj = open(file_path)
    except IOError as e:
        logging.error(e)
        return None

    new_key.update_metadata({'path': file_path})
    try:
        new_key.set_contents_from_file(file_obj)
        new_key.make_public()
    except boto.exception.GSResponseError as e:
        logging.error(e)
        # Do we have the credentials file set up?
        boto_cred_file = os.path.expanduser('~') + '/.boto'
        if not os.path.exists(boto_cred_file):
            logging.error(
                'Credentials file {} was not found.'.format(boto_cred_file))

        return None

    if thumb_bucket:
        file_obj.seek(0)
        im = Image.open(file_obj)
        im.thumbnail((260, 260))
        thumbnail = StringIO.StringIO()
        im.save(thumbnail, 'JPEG')
        thumb_uri = boto.storage_uri('{}/{}'.format(thumb_bucket, fn), 'gs')
        new_key = thumb_uri.new_key()
        # save the thumbnail
        thumbnail.seek(0)
        new_key.set_contents_from_file(thumbnail)

    file_obj.close()
    return str(dest_uri)
Beispiel #30
0
def downloadGCImages():
    # http://127.0.0.1:53000/downloadGCImages
    print("inside downloadGCImages2")
    imagelist = []
    DOGS_BUCKET = "rajpics"
    CATS_BUCKET = "rajpics"
    # Listing objects
    uri = boto.storage_uri(DOGS_BUCKET, GOOGLE_STORAGE)
    for obj in uri.get_bucket():
        print('%s://%s/%s' % (uri.scheme, uri.bucket_name, obj.name))
        # print('  "%s"' % obj.get_contents_as_string())
        # , "image":obj.get_contents_as_string()
        imagelist.append({"bucket_name": uri.bucket_name, "name": obj.name})
        print("image added to json list")
        src_uri = boto.storage_uri(DOGS_BUCKET + '/' + obj.name,
                                   GOOGLE_STORAGE)

        # Create a file-like object for holding the object contents.
        object_contents = io.BytesIO()
        # object_contents = obj.get_contents_as_string()

        # The unintuitively-named get_file() doesn't return the object
        # contents; instead, it actually writes the contents to
        # object_contents.
        src_uri.get_key().get_file(object_contents)
        print("after get file")

        local_dst_uri = boto.storage_uri(os.path.join(tmplocaldir, obj.name),
                                         LOCAL_FILE)

        bucket_dst_uri = boto.storage_uri(CATS_BUCKET + '/' + obj.name,
                                          GOOGLE_STORAGE)

        print("before writing to local file")
        # for dst_uri in (local_dst_uri, bucket_dst_uri):
        #    print("before seek")
        #    object_contents.seek(0)
        #    print("after seek")
        #    dst_uri.new_key().set_contents_from_file(object_contents)
        #    print("after write")

        print("before seek")
        object_contents.seek(0)
        print("after seek")
        local_dst_uri.new_key().set_contents_from_file(object_contents)
        print("after write")
        object_contents.close()

        print("Images written to dir:" + tmplocaldir)

    return app.response_class(json.dumps(imagelist),
                              content_type='application/json')
Beispiel #31
0
 def TestGetPathBeforeFinalDir(self):
   """Tests _GetPathBeforeFinalDir() (unit test)"""
   self.assertEqual('gs://',
                    cp._GetPathBeforeFinalDir(storage_uri('gs://bucket/')))
   self.assertEqual('gs://bucket',
                    cp._GetPathBeforeFinalDir(storage_uri('gs://bucket/dir/')))
   self.assertEqual('gs://bucket',
                    cp._GetPathBeforeFinalDir(storage_uri('gs://bucket/dir')))
   self.assertEqual('gs://bucket/dir',
                    cp._GetPathBeforeFinalDir(
                        storage_uri('gs://bucket/dir/obj')))
   self.assertEqual('file://%s' % self.src_dir_root.rstrip('/'),
                    cp._GetPathBeforeFinalDir(storage_uri(
                        'file://%sdir0/' % self.src_dir_root)))
Beispiel #32
0
	def delete(self,widget):
		if self.remote_treeview.get_selection().count_selected_rows() == 0:
			print '請至少選擇一個文件'
			return 0
		row = self.remote_treeview.get_selection().get_selected_rows()[1][0]
		remote_name = self.remote_liststore[row][0]
		print remote_name
		if self.list_status == 'bucket':
			remote_uri = boto.storage_uri(remote_name,'gs')
			remote_uri.delete_bucket()
			self.refresh(self)
		elif self.list_status == 'object':
			remote_uri = boto.storage_uri(self.bucket+os.sep+remote_name,'gs')
			remote_uri.delete_key()
			self.refresh(self)
Beispiel #33
0
def create_bucket():
    import datetime
    now = time.time()
    bucket_name = 'lighthouse-%d' % now
    # Your project ID can be found at https://console.cloud.google.com/
    # If there is no domain for your project, then project_id = 'YOUR_PROJECT'
    project_id = project_name
    
    
    # Instantiate a BucketStorageUri object.
    if bucket_name:
        
        # Try to create the bucket.
        try:
           uri = boto.storage_uri(bucket_name, GOOGLE_STORAGE)
           # If the default project is defined,
           # you do not need the headers.
           # Just call: uri.create_bucket()
           header_values = {"x-goog-project-id": project_id}
           uri.create_bucket(headers=header_values)
           
        except:
             print('Failed to create bucket:')
             
    print('Successfully created bucket "%s"' % bucket_name)        
    return bucket_name
    def _test_storage_uri(uri_str,
                          default_scheme='file',
                          debug=0,
                          validate=True):
        """Convenience method for instantiating a testing instance of StorageUri.

    This makes it unnecessary to specify
    bucket_storage_uri_class=mock_storage_service.MockBucketStorageUri.
    Also naming the factory method this way makes it clearer in the test
    code that StorageUri needs to be set up for testing.

    Args, Returns, and Raises are same as for boto.storage_uri(), except there's
    no bucket_storage_uri_class arg.

    Args:
      uri_str: Uri string to create StorageUri for.
      default_scheme: Default scheme for the StorageUri
      debug: debug level to pass to the underlying connection (0..3)
      validate: If True, validate the resource that the StorageUri refers to.

    Returns:
      StorageUri based on the arguments.
    """
        return boto.storage_uri(uri_str, default_scheme, debug, validate,
                                util.GSMockBucketStorageUri)
Beispiel #35
0
    def cleanup(self, id, name, stats):
        gslocation = "gs://%s/%s" % (self.bucket, id)
        logging.info("Clearing down older '%s' backups from GS (%s)..." % (name, gslocation))

        uri = boto.storage_uri(self.bucket, 'gs')
        bucket = uri.get_bucket()

        # Gather list of potentials first
        candidates = []
        for key in bucket.list(prefix="%s/" % id):
            parsed_date = parser.parse(key.last_modified)
            candidates.append([parsed_date, key.name])
        candidates.sort()

        # Loop and purge unretainable copies
        removable_names = []
        if self.retention_copies > 0:
            names = [name for d, name in candidates]
            if len(names) > self.retention_copies:
                removable_names = names[0:(len(names) - self.retention_copies)]
        if self.retention_days > 0:
            for d, name in candidates:
                days = (datetime.datetime.now(tz.tzutc()) - d).days
                if days > self.retention_days:
                    removable_names.append(name)
        for name in removable_names:
            logging.info("Removing '%s'..." % name)
            bucket.get_key(name).delete()

        # Return number of copies left
        stats.retained_copies = len(candidates) - len(removable_names)
Beispiel #36
0
 def _hashTestFile(self, url):
     import boto
     from toil.jobStores.googleJobStore import GoogleJobStore
     projectID, uri = GoogleJobStore._getResources(urlparse.urlparse(url))
     uri = boto.storage_uri(uri)
     contents = uri.get_contents_as_string(headers=self.headers)
     return hashlib.md5(contents).hexdigest()
Beispiel #37
0
    def __getMatchingKeys(self, dataPath, filename=None, includeDirectories=False, recursive=False):
        parse = _BotoClient.parseQuery(dataPath)

        storageScheme = parse[0]
        bucketName = parse[1]
        keyName = parse[2]

        if storageScheme == 's3' or storageScheme == 's3n':
            conn = S3ConnectionWithAnon(*self.awsCredentialsOverride.credentials)
            bucket = conn.get_bucket(bucketName)
        elif storageScheme == 'gs':
            conn = boto.storage_uri(bucketName, 'gs')
            bucket = conn.get_bucket()
        else:
            raise NotImplementedError("No file reader implementation for URL scheme " + storageScheme)

        if filename:
            # check whether last section of dataPath refers to a directory
            if not keyName.endswith("/"):
                if self.checkPrefix(bucket, keyName + "/"):
                    # keyname is a directory, but we've omitted the trailing "/"
                    keyName += "/"
                else:
                    # assume keyname refers to an object other than a directory
                    # look for filename in same directory as keyname
                    slashIdx = keyName.rfind("/")
                    if slashIdx >= 0:
                        keyName = keyName[:(slashIdx+1)]
                    else:
                        # no directory separators, so our object is in the top level of the bucket
                        keyName = ""
            keyName += filename

        return (storageScheme, _BotoClient.retrieveKeys(bucket, keyName, prefix=parse[3], postfix=parse[4],
                                                        includeDirectories=includeDirectories, recursive=recursive))
Beispiel #38
0
def _ListObjects(storage_schema, bucket, prefix, host_to_connect=None):
    """List objects under a bucket given a prefix.

  Args:
    storage_schema: The address schema identifying a storage. e.g., "gs"
    bucket: Name of the bucket.
    prefix: A prefix to list from.
    host_to_connect: An optional endpoint string to connect to.

  Returns:
    A list of object names.
  """

    bucket_list_result = None
    if _useBotoApi(storage_schema):
        bucket_uri = boto.storage_uri(bucket, storage_schema)
        if host_to_connect is not None:
            bucket_uri.connect(host=host_to_connect)

        bucket_list_result = bucket_uri.list_bucket(prefix=prefix)
    else:
        bucket_list_result = _AZURE_BLOB_SERVICE.list_blobs(bucket,
                                                            prefix=prefix)

    list_result = []
    for k in bucket_list_result:
        list_result.append(k.name)

    return list_result
def ReadObjects(storage_schema, bucket, objects_to_read, latency_results=None,
                bandwidth_results=None, object_size=None, host_to_connect=None):
  """Read a bunch of objects.

  Args:
    storage_schema: The address schema identifying a storage. e.g., "gs"
    bucket: Name of the bucket.
    objects_to_read: A list of names of objects to read.
    latency_results: An optional list to receive latency results.
    bandwidth_results: An optional list to receive bandwidth results.
    object_size: Size of the object that will be read, used to calculate bw.
    host_to_connect: An optional endpoint string to connect to.
  """
  for object_name in objects_to_read:

    start_time = time.time()
    try:
      if _useBotoApi(storage_schema):
        object_path = '%s/%s' % (bucket, object_name)
        object_uri = boto.storage_uri(object_path, storage_schema)
        object_uri.connect(host=host_to_connect)
        object_uri.new_key().get_contents_as_string()
      else:
        _AZURE_BLOB_SERVICE.get_blob_to_bytes(bucket, object_name)

      latency = time.time() - start_time

      if latency_results is not None:
        latency_results.append(latency)

      if (bandwidth_results is not None and
          object_size is not None and latency > 0.0):
        bandwidth_results.append(object_size / latency)
    except:
      logging.exception('Failed to read object %s', object_name)
Beispiel #40
0
    def setUp(self):
        """
        Initializes for each test.
        """
        # Create the test bucket.
        hostname = socket.gethostname().split('.')[0]
        uri_base_str = 'gs://res-download-test-%s-%s-%s' % (
            hostname, os.getpid(), int(time.time()))
        self.src_bucket_uri = storage_uri('%s-dst' % uri_base_str)
        self.src_bucket_uri.create_bucket()

        # Create test source objects.
        self.empty_src_key_size = 0
        (self.empty_src_key_as_string, self.empty_src_key) = (
            self.build_input_object('empty', self.empty_src_key_size))
        self.small_src_key_size = 2 * 1024  # 2 KB.
        (self.small_src_key_as_string, self.small_src_key) = (
            self.build_input_object('small', self.small_src_key_size))
        self.larger_src_key_size = 500 * 1024  # 500 KB.
        (self.larger_src_key_as_string, self.larger_src_key) = (
            self.build_input_object('larger', self.larger_src_key_size))

        # Use a designated tmpdir prefix to make it easy to find the end of
        # the tmp path.
        self.tmpdir_prefix = 'tmp_resumable_download_test'

        # Create temp dir and name for download file.
        self.tmp_dir = tempfile.mkdtemp(prefix=self.tmpdir_prefix)
        self.dst_file_name = '%s%sdst_file' % (self.tmp_dir, os.sep)

        self.tracker_file_name = '%s%stracker' % (self.tmp_dir, os.sep)

        # Create file-like object for detination of each download test.
        self.dst_fp = open(self.dst_file_name, 'w')
        self.created_test_data = True
Beispiel #41
0
 def test_default_object_acls_storage_uri(self):
     """Test default object acls using storage_uri."""
     # create a new bucket
     bucket = self._MakeBucket()
     bucket_name = bucket.name
     uri = storage_uri('gs://' + bucket_name)
     # get default acl and make sure it's project-private
     acl = uri.get_def_acl()
     self.assertIsNotNone(re.search(PROJECT_PRIVATE_RE, acl.to_xml()))
     # set default acl to a canned acl and verify it gets set
     uri.set_def_acl('public-read')
     acl = uri.get_def_acl()
     # save public-read acl for later test
     public_read_acl = acl
     self.assertEqual(acl.to_xml(), ('<AccessControlList><Entries><Entry>'
       '<Scope type="AllUsers"></Scope><Permission>READ</Permission>'
       '</Entry></Entries></AccessControlList>'))
     # back to private acl
     uri.set_def_acl('private')
     acl = uri.get_def_acl()
     self.assertEqual(acl.to_xml(),
                      '<AccessControlList></AccessControlList>')
     # set default acl to an xml acl and verify it gets set
     uri.set_def_acl(public_read_acl)
     acl = uri.get_def_acl()
     self.assertEqual(acl.to_xml(), ('<AccessControlList><Entries><Entry>'
       '<Scope type="AllUsers"></Scope><Permission>READ</Permission>'
       '</Entry></Entries></AccessControlList>'))
     # back to private acl
     uri.set_def_acl('private')
     acl = uri.get_def_acl()
     self.assertEqual(acl.to_xml(),
                      '<AccessControlList></AccessControlList>')
Beispiel #42
0
 def test_upload_with_file_content_change_during_upload(self):
     """
     Tests resumable upload on a file that changes one byte of content
     (so, size stays the same) while upload in progress
     """
     test_file_size = 500 * 1024  # 500 KB.
     test_file = self.build_input_file(test_file_size)[1]
     harness = CallbackTestHarness(fail_after_n_bytes=test_file_size/2,
                                   fp_to_change=test_file,
                                   # Write to byte 1, as the CallbackTestHarness writes
                                   # 3 bytes. This will result in the data on the server
                                   # being different than the local file.
                                   fp_change_pos=1)
     res_upload_handler = ResumableUploadHandler(num_retries=1)
     dst_key = self._MakeKey(set_contents=False)
     bucket_uri = storage_uri('gs://' + dst_key.bucket.name)
     dst_key_uri = bucket_uri.clone_replace_name(dst_key.name)
     try:
         dst_key.set_contents_from_file(
             test_file, cb=harness.call,
             res_upload_handler=res_upload_handler)
         self.fail('Did not get expected ResumableUploadException')
     except ResumableUploadException, e:
         self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT)
         # Ensure the file size didn't change.
         test_file.seek(0, os.SEEK_END)
         self.assertEqual(test_file_size, test_file.tell())
         self.assertNotEqual(
             e.message.find('md5 signature doesn\'t match etag'), -1)
         # Ensure the bad data wasn't left around.
         try:
             dst_key_uri.get_key()
             self.fail('Did not get expected InvalidUriError')
         except InvalidUriError, e:
             pass
Beispiel #43
0
 def test_default_object_acls_storage_uri(self):
     """Test default object acls using storage_uri."""
     # create a new bucket
     bucket = self._MakeBucket()
     bucket_name = bucket.name
     uri = storage_uri('gs://' + bucket_name)
     # get default acl and make sure it's project-private
     acl = uri.get_def_acl()
     self.assertIsNotNone(
         re.search(PROJECT_PRIVATE_RE, acl.to_xml()),
         'PROJECT_PRIVATE_RE not found in ACL XML:\n' + acl.to_xml())
     # set default acl to a canned acl and verify it gets set
     uri.set_def_acl('public-read')
     acl = uri.get_def_acl()
     # save public-read acl for later test
     public_read_acl = acl
     self.assertEqual(acl.to_xml(), ('<AccessControlList><Entries><Entry>'
       '<Scope type="AllUsers"></Scope><Permission>READ</Permission>'
       '</Entry></Entries></AccessControlList>'))
     # back to private acl
     uri.set_def_acl('private')
     acl = uri.get_def_acl()
     self.assertEqual(acl.to_xml(),
                      '<AccessControlList></AccessControlList>')
     # set default acl to an xml acl and verify it gets set
     uri.set_def_acl(public_read_acl)
     acl = uri.get_def_acl()
     self.assertEqual(acl.to_xml(), ('<AccessControlList><Entries><Entry>'
       '<Scope type="AllUsers"></Scope><Permission>READ</Permission>'
       '</Entry></Entries></AccessControlList>'))
     # back to private acl
     uri.set_def_acl('private')
     acl = uri.get_def_acl()
     self.assertEqual(acl.to_xml(),
                      '<AccessControlList></AccessControlList>')
Beispiel #44
0
def store_executable( bucket, executable_name ):
  try:
    executable_file = open( executable_name , 'r' )
    uri = boto.storage_uri( bucket + '/executable', GOOGLE_STORAGE)
    uri.new_key().set_contents_from_file( executable_file )
  except IOError, e:
      print "can't open the executable file to read"
Beispiel #45
0
def store_param_file( bucket, param_file_name ):
  try:
    param_file = open( param_file_name , 'r' )
    uri = boto.storage_uri( bucket + '/params.default', GOOGLE_STORAGE)
    uri.new_key().set_contents_from_file( param_file )
  except IOError, e:
      print "can't open the parameter file to read"
Beispiel #46
0
  def test_cp_v_option(self):
    # Tests that cp -v option returns the created object's version-specific URI.
    bucket_uri = self.CreateVersionedBucket()
    k1_uri = self.CreateObject(bucket_uri=bucket_uri, contents='data1')
    k2_uri = self.CreateObject(bucket_uri=bucket_uri, contents='data2')
    g1 = k1_uri.generation

    # Case 1: Upload file to object using one-shot PUT.
    tmpdir = self.CreateTempDir()
    fpath1 = self.CreateTempFile(tmpdir=tmpdir, contents='data1')
    self._run_cp_minus_v_test('-v', fpath1, k2_uri.uri)

    # Case 2: Upload file to object using resumable upload.
    size_threshold = boto.config.get('GSUtil', 'resumable_threshold', TWO_MB)
    file_as_string = os.urandom(size_threshold)
    tmpdir = self.CreateTempDir()
    fpath1 = self.CreateTempFile(tmpdir=tmpdir, contents=file_as_string)
    self._run_cp_minus_v_test('-v', fpath1, k2_uri.uri)

    # Case 3: Upload stream to object.
    self._run_cp_minus_v_test('-v', '-', k2_uri.uri)

    # Case 4: Download object to file. For this case we just expect output of
    # gsutil cp -v to be the URI of the file.
    tmpdir = self.CreateTempDir()
    fpath1 = self.CreateTempFile(tmpdir=tmpdir)
    dst_uri = storage_uri(fpath1)
    stderr = self.RunGsUtil(['cp', '-v', suri(k1_uri), suri(dst_uri)],
                            return_stderr=True)
    self.assertIn('Created: %s' % dst_uri.uri, stderr.split('\n')[-2])

    # Case 5: Daisy-chain from object to object.
    self._run_cp_minus_v_test('-Dv', k1_uri.uri, k2_uri.uri)
def _ListObjects(storage_schema, bucket, prefix, host_to_connect=None):
  """List objects under a bucket given a prefix.

  Args:
    storage_schema: The address schema identifying a storage. e.g., "gs"
    bucket: Name of the bucket.
    prefix: A prefix to list from.
    host_to_connect: An optional endpoint string to connect to.

  Returns:
    A list of object names.
  """

  bucket_list_result = None
  if _useBotoApi(storage_schema):
    bucket_uri = boto.storage_uri(bucket, storage_schema)
    if host_to_connect is not None:
      bucket_uri.connect(host=host_to_connect)

    bucket_list_result = bucket_uri.list_bucket(prefix=prefix)
  else:
    bucket_list_result = _AZURE_BLOB_SERVICE.list_blobs(bucket, prefix=prefix)

  list_result = []
  for k in bucket_list_result:
    list_result.append(k.name)

  return list_result
def DeleteObjects(storage_schema, bucket, objects_to_delete,
                  host_to_connect=None,
                  objects_deleted=None):
  """Delete a bunch of objects.

  Args:
    storage_schema: The address schema identifying a storage. e.g., "gs"
    bucket: Name of the bucket.
    objects_to_delete: A list of names of objects to delete.
    host_to_connect: An optional endpoint string to connect to.
    objects_deleted: An optional list to record the objects that have been
        successfully deleted.
  """

  for object_name in objects_to_delete:
    try:
      if _useBotoApi(storage_schema):
        object_path = '%s/%s' % (bucket, object_name)
        object_uri = boto.storage_uri(object_path, storage_schema)
        if host_to_connect is not None:
          object_uri.connect(host=host_to_connect)

        object_uri.delete_key()
      else:
        _AZURE_BLOB_SERVICE.delete_blob(bucket, object_name)

      if objects_deleted is not None:
        objects_deleted.append(object_name)
    except:
      logging.exception('Caught exception while deleting object %s.',
                        object_name)
Beispiel #49
0
def DeleteObjects(storage_schema,
                  bucket,
                  objects_to_delete,
                  host_to_connect=None,
                  objects_deleted=None):
    """Delete a bunch of objects.

  Args:
    storage_schema: The address schema identifying a storage. e.g., "gs"
    bucket: Name of the bucket.
    objects_to_delete: A list of names of objects to delete.
    host_to_connect: An optional endpoint string to connect to.
    objects_deleted: An optional list to record the objects that have been
        successfully deleted.
  """

    for object_name in objects_to_delete:
        try:
            if _useBotoApi(storage_schema):
                object_path = '%s/%s' % (bucket, object_name)
                object_uri = boto.storage_uri(object_path, storage_schema)
                if host_to_connect is not None:
                    object_uri.connect(host=host_to_connect)

                object_uri.delete_key()
            else:
                _AZURE_BLOB_SERVICE.delete_blob(bucket, object_name)

            if objects_deleted is not None:
                objects_deleted.append(object_name)
        except:
            logging.exception('Caught exception while deleting object %s.',
                              object_name)
Beispiel #50
0
def get_path(path):

    prefix = ""
    if path.endswith(".gz") and ".tar.gz" not in path:
        prefix = "/vsigzip/"

    uri = parse_uri(path)

    if uri.scheme == "file":
        path = uri.uri_path if os.path.exists(uri.uri_path) else None

    elif uri.scheme == "s3":
        conn = connect_s3()
        bucket = conn.get_bucket(uri.bucket_id)
        key = bucket.lookup(uri.key_id)
        if prefix == "":
            prefix = "/"
        prefix += os.path.join(prefix, "vsicurl")
        path = key.generate_url(60 * 60) if key is not None else key

    elif uri.scheme == 'gs':
        storage_uri = boto.storage_uri(uri.bucket_id, uri.scheme)
        bucket = storage_uri.get_bucket(uri.bucket_id)
        key = bucket.lookup(uri.key_id)
        if prefix == "":
            prefix = "/"
        prefix += os.path.join(prefix, "vsicurl/")
        path = key.generate_url(60 * 60) if key is not None else key

    return prefix + path
  def CreateBucket(self, bucket_name=None, test_objects=0, storage_class=None):
    """Creates a test bucket.

    The bucket and all of its contents will be deleted after the test.

    Args:
      bucket_name: Create the bucket with this name. If not provided, a
                   temporary test bucket name is constructed.
      test_objects: The number of objects that should be placed in the bucket.
                    Defaults to 0.
      storage_class: storage class to use. If not provided we us standard.

    Returns:
      StorageUri for the created bucket.
    """
    bucket_name = bucket_name or self.MakeTempName('bucket')

    bucket_uri = boto.storage_uri('gs://%s' % bucket_name.lower(),
                                  suppress_consec_slashes=False)

    # Apply API version and project ID headers if necessary.
    headers = {'x-goog-api-version': self.api_version}
    self.proj_id_handler.FillInProjectHeaderIfNeeded(
        'test', bucket_uri, headers)

    bucket_uri.create_bucket(storage_class=storage_class, headers=headers)
    self.bucket_uris.append(bucket_uri)
    for i in range(test_objects):
      self.CreateObject(bucket_uri=bucket_uri,
                        object_name=self.MakeTempName('obj'),
                        contents='test %d' % i)
    return bucket_uri
 def test_create_with_retention_months(self):
     bucket_name = self.MakeTempName('bucket')
     bucket_uri = boto.storage_uri('gs://%s' % (bucket_name.lower()),
                                   suppress_consec_slashes=False)
     self.RunGsUtil(['mb', '--retention', '1m', suri(bucket_uri)])
     self.VerifyRetentionPolicy(
         bucket_uri, expected_retention_period_in_seconds=SECONDS_IN_MONTH)
Beispiel #53
0
def connection_with_gs(name):
    """
    Connect to GS
    """
    import boto
    conn = boto.storage_uri(name, 'gs')
    return conn
Beispiel #54
0
	def upload(self, obj):
		sys.stdout.write('Uploading %s/%s %s... ' % (obj.parent.name, obj.name, format_size(obj.size)))
		sys.stdout.flush()
		with open(obj.full_path, 'r') as localfile:
			dest_uri = boto.storage_uri(CONFIG['bucket_id'] + obj.full_path, 'gs')
			dest_uri.new_key().set_contents_from_file(localfile)
		sys.stdout.write('Done!\n')
def OutputBenchExpectations(bench_type, rev_min, rev_max, representation_alg):
  """Reads bench data from google storage, and outputs expectations.

  Ignores data with revisions outside [rev_min, rev_max] integer range. For
  bench data with multiple revisions, we use higher revisions to calculate
  expected bench values.
  bench_type is either 'micro' or 'skp', according to the flag '-b'.
  Uses the provided representation_alg for calculating bench representations.
  """
  if bench_type not in BENCH_TYPES:
    raise Exception('Not valid bench_type! (%s)' % BENCH_TYPES)
  expectation_dic = {}
  uri = boto.storage_uri(URI_BUCKET, GOOGLE_STORAGE_URI_SCHEME)
  for obj in uri.get_bucket():
    # Filters out non-bench files.
    if ((not obj.name.startswith('perfdata/%s' % BENCH_BUILDER_PREFIX) and
         not obj.name.startswith(
             'playback/perfdata/%s' % BENCH_BUILDER_PREFIX)) or
         obj.name.find('_data') < 0):
      continue
    if ((bench_type == 'micro' and obj.name.find('_data_skp_') > 0) or
        (bench_type == 'skp' and obj.name.find('_skp_') < 0)):
      # Skips wrong bench type.
      continue
    # Ignores uninterested platforms.
    platform = obj.name.split('/')[1]
    if not platform.startswith(BENCH_BUILDER_PREFIX):
      platform = obj.name.split('/')[2]
    if not platform.startswith(BENCH_BUILDER_PREFIX):
      continue  # Ignores non-platform object
    if platform not in PLATFORMS:
      continue
    # Filters by revision.
    to_filter = True
    for rev in range(rev_min, rev_max + 1):
      if '_r%s_' % rev in obj.name:
        to_filter = False
        break
    if to_filter:
      continue
    contents = cStringIO.StringIO()
    obj.get_file(contents)
    for point in bench_util.parse('', contents.getvalue().split('\n'),
                                  representation_alg):
      if point.config in CONFIGS_TO_FILTER:
        continue

      key = '%s_%s_%s,%s-%s' % (point.bench, point.config, point.time_type,
                                platform, representation_alg)
      # It is fine to have later revisions overwrite earlier benches, since we
      # only use the latest bench within revision range to set expectations.
      expectation_dic[key] = point.time
  keys = expectation_dic.keys()
  keys.sort()
  for key in keys:
    bench_val = expectation_dic[key]
    # Prints out expectation lines.
    print '%s,%.3f,%.3f,%.3f' % (key, bench_val,
                                 bench_val * BENCH_LB - BENCH_ALLOWED_NOISE,
                                 bench_val * BENCH_UB + BENCH_ALLOWED_NOISE)
    def testCompose(self):
        data1 = 'hello '
        data2 = 'world!'
        expected_crc = 1238062967

        b = self._MakeBucket()
        bucket_uri = storage_uri("gs://%s" % b.name)
        key_uri1 = bucket_uri.clone_replace_name("component1")
        key_uri1.set_contents_from_string(data1)
        key_uri2 = bucket_uri.clone_replace_name("component2")
        key_uri2.set_contents_from_string(data2)

        # Simple compose.
        key_uri_composite = bucket_uri.clone_replace_name("composite")
        components = [key_uri1, key_uri2]
        key_uri_composite.compose(components, content_type='text/plain')
        self.assertEquals(key_uri_composite.get_contents_as_string(),
                          data1 + data2)
        composite_key = key_uri_composite.get_key()
        cloud_crc32c = binascii.hexlify(composite_key.cloud_hashes['crc32c'])
        self.assertEquals(cloud_crc32c, hex(expected_crc)[2:])
        self.assertEquals(composite_key.content_type, 'text/plain')

        # Compose disallowed between buckets.
        key_uri1.bucket_name += '2'
        try:
            key_uri_composite.compose(components)
            self.fail('Composing between buckets didn\'t fail as expected.')
        except BotoClientError as err:
            self.assertEquals(err.reason,
                              'GCS does not support inter-bucket composing')
Beispiel #57
0
    def move_object(bucket_from, bucket_to, file):
        conn = connection.connect()
        bucket_from_name = bucket_from.name  #self.list_object()

        #src_uri = boto.storage_uri(bucket_from_name + '/' + file.name, 'gs')

        # Create a file-like object for holding the object contents.
        object_contents = StringIO.StringIO()

        try:
            from boto.s3.key import Key
            k = Key(bucket_from)
            k.key = file.name
            #k.key.get_file(object_contents)
            k.get_contents_to_file(object_contents)

            dst_uri = boto.storage_uri(bucket_to.name + '/' + file.name, 'gs')
            object_contents.seek(0)
            dst_uri.new_key().set_contents_from_file(object_contents)

            # bucket = conn.get_bucket(bucket_to.name)
            # key = bucket.new_key(file.name)
            #
            # key.set_contents_from_file(object_contents)

            object_contents.close()
        except:
            msg = 'Sorry, but fail to move file: "%s".' % file.name
            return msg
        else:
            msg = 'Successfully move "%s"!' % file.name
            return msg