# this method is used to get the size of the files
def getSize(fileName, prefix=""):
size = os.stat(pathForResource(fileName)).st_size
return ds3.FileObject(prefix + fileName, size)
# this method is used to get the os specific path for an object located in the resources folder
def pathForResource(resourceName):
encoding = sys.getfilesystemencoding()
currentPath = os.path.dirname(unicode(__file__, encoding))
return os.path.join(currentPath, "resources", resourceName)
# get the sizes for each file
fileList = map(getSize, fileList)
fileObjectList = ds3.FileObjectList(fileList)
# submit the put bulk request to DS3
bulkResult = client.put_bulk_job_spectra_s3(
ds3.PutBulkJobSpectraS3Request(bucketName, fileObjectList))
# the bulk request will split the files over several chunks if it needs to.
# we then need to ask what chunks we can send, and then send them making
# sure we don't resend the same chunks
# create a set of the chunk ids which will be used to track
# what chunks have not been sent
chunkIds = set(map(lambda x: x['ChunkId'], bulkResult.result['ObjectsList']))
# while we still have chunks to send
while len(chunkIds) > 0:
"resources/tale_of_two_cities.txt", "resources/ulysses.txt"
]
fileMap = {}
# this method is used to get the size of the files
def createDs3Obj(fileName):
size = os.stat(fileName).st_size
ds3ObjName = "prefix/" + fileName
fileMap[ds3ObjName] = fileName
return ds3.FileObject(ds3ObjName, size)
# get the sizes for each file
fileList = ds3.FileObjectList(list(map(createDs3Obj, fileList)))
# submit the put bulk request to DS3
bulkResult = client.put_bulk_job_spectra_s3(
ds3.PutBulkJobSpectraS3Request(bucketName, fileList))
# the bulk request will split the files over several chunks if it needs to
# we need to iterate over the chunks, ask the server for space to send
# the chunks, then send all the objects returned in the chunk
for chunk in bulkResult.result['ObjectsList']:
allocateChunk = client.allocate_job_chunk_spectra_s3(
ds3.AllocateJobChunkSpectraS3Request(chunk['ChunkId']))
for obj in allocateChunk.result['ObjectList']:
objectDataStream = open(fileMap[obj['Name']], "rb")
client.put_object(
ds3.PutObjectRequest(bucketName,
"folder/folder2/tale_of_two_cities.txt":
"resources/tale_of_two_cities.txt",
"folder/folder2/ulysses.txt": "resources/ulysses.txt"
}
# this method is used to get the size of the files
# we need two parameters because the S3 API wants the name that the object will take on the server, but the size obviously needs to come from the file on the current file system
def getSize(fileName, realFileName):
size = os.stat(realFileName).st_size
return ds3.FileObject(fileName, size)
# get the sizes for each file
fileList = ds3.FileObjectList(
map(lambda key: getSize(key, fileListMapping[key]),
fileListMapping.keys()))
# submit the put bulk request to DS3
bulkResult = client.put_bulk_job_spectra_s3(
ds3.PutBulkJobSpectraS3Request(bucketName, fileList))
# the bulk request will split the files over several chunks if it needs to.
# we then need to ask what chunks we can send, and then send them making
# sure we don't resend the same chunks
# create a set of the chunk ids which will be used to track
# what chunks have not been sent
chunkIds = set(map(lambda x: x['ChunkId'], bulkResult.result['ObjectsList']))
# while we still have chunks to send
"tale_of_two_cities.txt":"resources/tale_of_two_cities.txt",
"ulysses.txt":"resources/ulysses.txt",
"folder/beowulf.txt":"resources/beowulf.txt",
"folder/sherlock_holmes.txt":"resources/sherlock_holmes.txt",
"folder/folder2/tale_of_two_cities.txt":"resources/tale_of_two_cities.txt",
"folder/folder2/ulysses.txt":"resources/ulysses.txt"
}
# this method is used to get the size of the files
# we need two parameters because the S3 API wants the name that the object will take on the server, but the size obviously needs to come from the file on the current file system
def getSize(fileName, realFileName):
size = os.stat(realFileName).st_size
return ds3.FileObject(fileName, size)
# get the sizes for each file
fileList = ds3.FileObjectList([getSize(key, fileListMapping[key]) for key in list(fileListMapping.keys())])
# submit the put bulk request to DS3
bulkResult = client.put_bulk_job_spectra_s3(ds3.PutBulkJobSpectraS3Request(bucketName, fileList))
# the bulk request will split the files over several chunks if it needs to.
# we then need to ask what chunks we can send, and then send them making
# sure we don't resend the same chunks
# create a set of the chunk ids which will be used to track
# what chunks have not been sent
chunkIds = set([x['ChunkId'] for x in bulkResult.result['ObjectsList']])
# while we still have chunks to send
while len(chunkIds) > 0:
# get a list of the available chunks that we can send
# Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
# Licensed under the Apache License, Version 2.0 (the "License"). You may not use
# this file except in compliance with the License. A copy of the License is located at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# or in the "license" file accompanying this file.
# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.
import os
import tempfile
from ds3 import ds3
client = ds3.createClientFromEnv()
bucketName = "sdkexamples"
objectName = "123456795.txt"
objlist = ds3.FileObjectList([ds3.FileObject(objectName)])
tapes = client.get_physical_placement_for_objects_spectra_s3(
ds3.GetPhysicalPlacementForObjectsSpectraS3Request(bucketName, objlist))
for tape in tapes.result['TapeList']:
print(tape)
# specific language governing permissions and limitations under the License.
import os
import tempfile
from ds3 import ds3
client = ds3.createClientFromEnv()
bucketName = "books"
# this example assumes that a bucket named "books" and the following objects exist on the server (these are the same objects as are on the server if they are not deleted at the end of the bulk put example)
fileList = ["beowulf.txt", "sherlock_holmes.txt", "tale_of_two_cities.txt", "ulysses.txt"]
bucketContents = client.get_bucket(ds3.GetBucketRequest(bucketName))
objectList = ds3.FileObjectList([ds3.FileObject(obj['Key']) for obj in bucketContents.result['ContentsList']])
bulkGetResult = client.get_bulk_job_spectra_s3(ds3.GetBulkJobSpectraS3Request(bucketName, objectList))
# create a set of the chunk ids which will be used to track
# what chunks have not been retrieved
chunkIds = set([x['ChunkId'] for x in bulkGetResult.result['ObjectsList']])
# create a dictionary to map our retrieved objects to temporary files
# if you want to keep the retreived files on disk, this is not necessary
tempFiles={}
# while we still have chunks to retrieve
while len(chunkIds) > 0:
# get a list of the available chunks that we can get
availableChunks = client.get_job_chunks_ready_for_client_processing_spectra_s3(
ds3.GetJobChunksReadyForClientProcessingSpectraS3Request(bulkGetResult.result['JobId']))
from ds3 import ds3
client = ds3.createClientFromEnv()
bucketName = "books"
# this example assumes that a bucket named "books" and the following objects exist on the server (these are the same objects as are on the server if they are not deleted at the end of the bulk put example)
fileList = [
"beowulf.txt", "sherlock_holmes.txt", "tale_of_two_cities.txt",
"ulysses.txt"
]
bucketContents = client.get_bucket(ds3.GetBucketRequest(bucketName))
objectList = ds3.FileObjectList(
map(lambda obj: ds3.FileObject(obj['Key']),
bucketContents.result['ContentsList']))
bulkGetResult = client.get_bulk_job_spectra_s3(
ds3.GetBulkJobSpectraS3Request(bucketName, objectList))
# create a set of the chunk ids which will be used to track
# what chunks have not been retrieved
chunkIds = set(map(lambda x: x['ChunkId'],
bulkGetResult.result['ObjectsList']))
# create a dictionary to map our retrieved objects to temporary files
# if you want to keep the retreived files on disk, this is not necessary
tempFiles = {}
# while we still have chunks to retrieve
while len(chunkIds) > 0: