def verify_program(program):
states = walk_program(program)
for p_idx, loop_vars in states:
r_call_abstract_with_scope = program[p_idx]
current_node = (p_idx, loop_vars)
scope = r_call_abstract_with_scope.scope.copy()
children = find_children(program, p_idx, loop_vars)
for p_idx_child, child_vars in children:
child_parents = find_parents(program, p_idx_child, child_vars)
assert current_node in child_parents
parents = find_parents(program, p_idx, loop_vars)
for p_idx_parent, parent_vars in parents:
parent_children = find_children(program, p_idx_parent, parent_vars)
assert current_node in parent_children
def test_cholesky_multi_repeats():
''' Insert repeated instructions into PC queue avoid double increments '''
print("RUNNING MULTI")
np.random.seed(1)
size = 256
shard_size = 30
repeats = 15
total_repeats = 150
np.random.seed(2)
print("Generating X")
X = np.random.randn(size, 128)
print("Generating A")
A = X.dot(X.T) + size * np.eye(X.shape[0])
shard_sizes = (shard_size, shard_size)
A_sharded = BigMatrix("cholesky_test_A_{0}".format(int(time.time())),
shape=A.shape,
shard_sizes=shard_sizes,
write_header=True)
A_sharded.free()
shard_matrix(A_sharded, A)
program, meta = cholesky(A_sharded)
states = compiler.walk_program(program.program.remote_calls)
L_sharded = meta["outputs"][0]
L_sharded.free()
pwex = pywren.default_executor()
executor = pywren.lambda_executor
config = npw.config.default()
print("PROGRAM HASH", program.hash)
cores = 1
program.start()
jobs = []
for c in range(cores):
p = mp.Process(target=job_runner.lambdapack_run,
args=(program, ),
kwargs={
'timeout': 3600,
'pipeline_width': 3
})
jobs.append(p)
p.start()
np.random.seed(0)
while (program.program_status() == lp.PS.RUNNING):
sqs = boto3.resource('sqs', region_name=program.control_plane.region)
time.sleep(0.5)
waiting = 0
running = 0
for i, queue_url in enumerate(program.queue_urls):
client = boto3.client('sqs')
print("Priority {0}".format(i))
attrs = client.get_queue_attributes(
QueueUrl=queue_url,
AttributeNames=[
'ApproximateNumberOfMessages',
'ApproximateNumberOfMessagesNotVisible'
])['Attributes']
print(attrs)
waiting += int(attrs["ApproximateNumberOfMessages"])
running += int(attrs["ApproximateNumberOfMessagesNotVisible"])
print("SQS QUEUE STATUS Waiting {0}, Running {1}".format(
waiting, running))
for i in range(repeats):
p = program.get_progress()
if (p is None):
continue
else:
p = int(p)
pc = int(np.random.choice(min(p, len(states)), 1))
node = states[pc]
queue = sqs.Queue(program.queue_urls[0])
total_repeats -= 1
if (total_repeats > 0):
print("Malicilously enqueueing node ", pc, node, total_repeats)
queue.send_message(MessageBody=json.dumps(node))
time.sleep(1)
#for p in jobs:
# p.join()
program.wait()
program.free()
L_npw = L_sharded.numpy()
L = np.linalg.cholesky(A)
z = np.argmax(np.abs(L - L_npw))
assert (np.allclose(L_npw, L))