# get the sizes for each file
fileList = ds3.FileObjectList(list(map(createDs3Obj, fileList)))
# submit the put bulk request to DS3
bulkResult = client.put_bulk_job_spectra_s3(
ds3.PutBulkJobSpectraS3Request(bucketName, fileList))
# the bulk request will split the files over several chunks if it needs to
# we need to iterate over the chunks, ask the server for space to send
# the chunks, then send all the objects returned in the chunk
for chunk in bulkResult.result['ObjectsList']:
allocateChunk = client.allocate_job_chunk_spectra_s3(
ds3.AllocateJobChunkSpectraS3Request(chunk['ChunkId']))
for obj in allocateChunk.result['ObjectList']:
objectDataStream = open(fileMap[obj['Name']], "rb")
client.put_object(
ds3.PutObjectRequest(bucketName,
obj['Name'],
obj['Length'],
objectDataStream,
offset=int(obj['Offset']),
job=bulkResult.result['JobId']))
# we now verify that all our objects have been sent to DS3
bucketResponse = client.get_bucket(ds3.GetBucketRequest(bucketName))
for obj in bucketResponse.result['ContentsList']:
print(obj['Key'])
import os
import tempfile
from ds3 import ds3
client = ds3.createClientFromEnv()
bucketName = "books"
# this example assumes that a bucket named "books" and the following objects exist on the server (these are the same objects as are on the server if they are not deleted at the end of the bulk put example)
fileList = [
"beowulf.txt", "sherlock_holmes.txt", "tale_of_two_cities.txt",
"ulysses.txt"
]
bucketContents = client.get_bucket(ds3.GetBucketRequest(bucketName))
objectList = list([
ds3.Ds3GetObject(obj['Key'])
for obj in bucketContents.result['ContentsList']
])
bulkGetResult = client.get_bulk_job_spectra_s3(
ds3.GetBulkJobSpectraS3Request(bucketName, objectList))
# create a set of the chunk ids which will be used to track
# what chunks have not been retrieved
chunkIds = set([x['ChunkId'] for x in bulkGetResult.result['ObjectsList']])
# create a dictionary to map our retrieved objects to temporary files
# if you want to keep the retreived files on disk, this is not necessary
tempFiles = {}
from ds3 import ds3
# take bucketname (required) and prefix (optional) from command line
if len(sys.argv) < 2:
print('Usage: python getBuckey.py <bucketname> [prefix]')
exit(0)
bucket = sys.argv[1]
prefix = None
if len(sys.argv) > 2:
prefix = sys.argv[2]
# get_bucket returns max 1000 objects -- use pagination to get more
getMore = True
marker = None
client = ds3.createClientFromEnv()
while getMore:
# the call will return a list of 1000 bucket objects
resp = client.get_bucket(
ds3.GetBucketRequest(bucket, None, marker, 1000, prefix))
# set pagination variables for next call
getMore = resp.result["IsTruncated"] == 'true'
marker = resp.result["NextMarker"]
# extract and print what is wanted from get_bucket response; key is object name
for contents in resp.result["ContentsList"]:
print(contents['Key'])