print("Save in memory. Time [s]: ", t2 - t1)
# Remove pml
g = g[model.nbpml:-model.nbpml, model.nbpml:-model.nbpml] # remove padding
g = np.reshape(g, -1, order='F')
# Chunk up gradient and write to bucket. Add gradients to SQS queue
chunk_size = get_chunk_size(len(g), num_chunks)
idx_count = 0
for j in range(num_chunks):
# Save to bucket
file_ext = ''.join(random.sample(chars * 12, 12))
key = partial_gradient_path + 'chunk_' + str(j + 1) + '/' + gradient_name
gwrite = g[idx_count:idx_count + chunk_size[j]]
array_put(gwrite, bucket, key)
idx_count += chunk_size[j]
# Add to queue
queue_name = queue_names + str(j + 1)
msg = bucket + '&' + partial_gradient_path + '&' + full_gradient_path + '&' + \
gradient_name + '&' + file_ext + '&' + str(iteration) + '&1&' + str(batchsize) + \
'&' + str(j+1) + '&' + queue_name + '&' + variable_path + '&' + variable_name + \
'&' + str(step_length) + '&' + str(step_scaling)
# SQS url
url = sqs_client.list_queues(QueueNamePrefix=queue_name)['QueueUrls'][0]
idx = url.find('com') + 4
user_id = url[idx:idx + 12]
url_address = 'https://sqs.us-east-1.amazonaws.com/' + user_id + '/' + queue_name
print("Save in memory. Time [s]: ", t2 - t1)
# Remove pml
g = g[model.nbpml:-model.nbpml, model.nbpml:-model.nbpml] # remove padding
g = np.reshape(g, -1, order='F')
# Chunk up gradient and write to bucket. Add gradients to SQS queue
chunk_size = get_chunk_size(len(g), num_chunks)
idx_count = 0
for j in range(num_chunks):
# Save to bucket
file_ext = ''.join(random.sample(chars * 12, 12))
key = partial_gradient_path + 'chunk_' + str(j + 1) + '/' + gradient_name
gwrite = g[idx_count:idx_count + chunk_size[j]]
array_put(gwrite, bucket, key)
idx_count += chunk_size[j]
# Add to queue
queue_name = queue_names + str(j + 1)
msg = bucket + '&' + partial_gradient_path + '&' + full_gradient_path + '&' + \
gradient_name + '&' + file_ext + '&' + str(iteration) + '&1&' + str(batchsize) + \
'&' + str(j+1) + '&' + queue_name + '&' + variable_path + '&' + variable_name + \
'&' + str(step_length) + '&' + str(step_scaling)
# SQS url
url = 'https://sqs.us-east-1.amazonaws.com/851065145468/' + queue_name
# Send message
#sqs_client.send_message(QueueUrl=url, DelaySeconds=0, MessageBody=msg)
# Remove pml and extent back to full size
g = g[model.nbpml:-model.nbpml, model.nbpml:-model.nbpml] # remove padding
g = extent_gradient(shape_full, origin_full, shape, origin, spacing, g)
g = np.reshape(g, -1, order='F')
# Chunk up gradient and write to bucket. Add gradients to SQS queue
chunk_size = get_chunk_size(len(g), num_chunks)
idx_count = 0
for j in range(num_chunks):
# Save to bucket
file_ext = ''.join(random.sample(chars * 12, 12))
key = partial_gradient_path + 'chunk_' + str(
j + 1) + '/' + gradient_name + file_ext
gwrite = g[idx_count:idx_count + chunk_size[j]]
array_put(gwrite, bucket, key)
idx_count += chunk_size[j]
# Add to queue
queue_name = queue_names + str(j + 1)
msg = bucket + '&' + partial_gradient_path + '&' + full_gradient_path + '&' + \
gradient_name + '&' + file_ext + '&' + str(iteration) + '&1&' + str(batchsize) + \
'&' + str(j+1) + '&' + queue_name + '&' + variable_path + '&' + variable_name + \
'&' + str(step_length) + '&' + str(step_scaling)
# SQS url
url = 'https://sqs.us-east-1.amazonaws.com/851065145468/' + queue_name
# Send message
sqs_client.send_message(QueueUrl=url, DelaySeconds=0, MessageBody=msg)