# we have not sent it, and if not, send that object for chunk in chunks: if not chunk['ChunkId'] in chunkIds: continue chunkIds.remove(chunk['ChunkId']) for obj in chunk['ObjectList']: # it is possible that if we start resending a chunk, due to the program crashing, that # some objects will already be in cache. Check to make sure that they are not, and then # send the object to Spectra S3 if obj['InCache'] == 'false': realFileName = "resources/" + obj['Name'] client.put_object( ds3.PutObjectRequest(bucketName, obj['Name'], offset=int(obj['Offset']), job=bulkResult.result['JobId'], real_file_name=realFileName)) # we now verify that all our objects have been sent to DS3 bucketResponse = client.get_bucket(ds3.GetBucketRequest(bucketName)) for obj in bucketResponse.result['ContentsList']: print obj['Key'] # delete the bucket by first deleting all the objects, and then deleting the bucket for obj in bucketResponse.result['ContentsList']: client.delete_object(ds3.DeleteObjectRequest(bucketName, obj['Key'])) client.delete_bucket(ds3.DeleteBucketRequest(bucketName))
# get the sizes for each file fileList = ds3.FileObjectList(list(map(createDs3Obj, fileList))) # submit the put bulk request to DS3 bulkResult = client.put_bulk_job_spectra_s3( ds3.PutBulkJobSpectraS3Request(bucketName, fileList)) # the bulk request will split the files over several chunks if it needs to # we need to iterate over the chunks, ask the server for space to send # the chunks, then send all the objects returned in the chunk for chunk in bulkResult.result['ObjectsList']: allocateChunk = client.allocate_job_chunk_spectra_s3( ds3.AllocateJobChunkSpectraS3Request(chunk['ChunkId'])) for obj in allocateChunk.result['ObjectList']: objectDataStream = open(fileMap[obj['Name']], "rb") client.put_object( ds3.PutObjectRequest(bucketName, obj['Name'], obj['Length'], objectDataStream, offset=int(obj['Offset']), job=bulkResult.result['JobId'])) # we now verify that all our objects have been sent to DS3 bucketResponse = client.get_bucket(ds3.GetBucketRequest(bucketName)) for obj in bucketResponse.result['ContentsList']: print(obj['Key'])
ds3ObjName = "prefix/" + fileName fileMap[ds3ObjName] = fileName return ds3.Ds3PutObject(ds3ObjName, size) # get the sizes for each file objectList = list(map(createDs3PutObject, fileList)) # submit the put bulk request to DS3 bulkResult = client.put_bulk_job_spectra_s3(ds3.PutBulkJobSpectraS3Request(bucketName, objectList)) # the bulk request will split the files over several chunks if it needs to # we need to iterate over the chunks, ask the server for space to send # the chunks, then send all the objects returned in the chunk for chunk in bulkResult.result['ObjectsList']: allocateChunk = client.allocate_job_chunk_spectra_s3(ds3.AllocateJobChunkSpectraS3Request(chunk['ChunkId'])) for obj in allocateChunk.result['ObjectList']: objectDataStream = open(fileMap[obj['Name']], "rb") objectDataStream.seek(int(obj['Offset']), 0) client.put_object(ds3.PutObjectRequest(bucket_name=bucketName, object_name=obj['Name'], length=obj['Length'], stream=objectDataStream, offset=int(obj['Offset']), job=bulkResult.result['JobId'])) # we now verify that all our objects have been sent to DS3 bucketResponse = client.get_bucket(ds3.GetBucketRequest(bucketName)) for obj in bucketResponse.result['ContentsList']: print(obj['Key'])