def run(flow, cloudburst, requests, local, sckt=None):
latencies = []
if not local:
print = logging.info
bench_start = time.time()
for i in range(requests):
if i % 100 == 0:
logging.info(f'On request {i}...')
inp = Table([('user', StrType), ('recent', NumpyType)])
uid = np.random.randint(NUM_USERS)
recent = np.random.randint(0, NUM_PRODUCT_SETS, 5)
inp.insert([str(uid), recent])
start = time.time()
flow.run(inp).get()
end = time.time()
latencies.append(end - start)
bench_end = time.time()
print_latency_stats(latencies, "E2E", not local, bench_end - bench_start)
if sckt:
bts = cp.dumps(latencies)
sckt.send(bts)
def run(flow, cloudburst, requests, local, sckt=None):
schema = [('classify', StrType), ('translate', StrType)]
french = [
'Je m\'appelle Pierre.', 'Comment allez-vous aujourd\'hui?',
'La nuit est longue et froide, et je veux rentrer chez moi.',
'Tu es venue a minuit, mais je me suis déja couché.',
'On veut aller dehors mais il faut rester dedans.'
]
german = [
'Ich bin in Berliner.', 'Die katz ist saß auf dem Stuhl.',
'Sie schwimmt im Regen.',
'Ich gehe in den Supermarkt, aber mir ist kalt.',
'Ich habe nie gedacht, dass du Amerikanerin bist.'
]
english = [
'What is the weather like today?',
'Why does it rain so much in April?',
'I like running but my ankles hurt.',
'I should go home to eat dinner before it gets too late.',
'I would like to hang out with my friends, but I have to work.'
]
inputs = []
for _ in range(20):
table = Table(schema)
if random.random() < 0.5:
other = random.choice(french)
else:
other = random.choice(german)
vals = [other, random.choice(english)]
table.insert(vals)
inputs.append(table)
logging.info('Starting benchmark...')
latencies = []
bench_start = time.time()
for i in range(requests):
if i % 100 == 0:
logging.info(f'On request {i}...')
inp = random.choice(inputs)
start = time.time()
result = flow.run(inp).get()
end = time.time()
latencies.append(end - start)
bench_end = time.time()
print_latency_stats(latencies, "E2E", not local, bench_end - bench_start)
if sckt:
bts = cp.dumps(latencies)
sckt.send(bts)
def run(cloudburst: CloudburstConnection,
num_requests: int,
data_size: str,
do_optimize: bool):
def stage1(self, row: Row) -> bytes:
import numpy as np
return np.random.rand(row['size'])
def stage2(self, row: Row) -> int:
return 3
print(f'Creating flow with {data_size} ({DATA_SIZES[data_size]}) inputs.')
flow = Flow('colocate-benchmark', FlowType.PUSH, cloudburst)
f1 = flow.map(stage1)
p1 = f1.map(stage2, names=['val1'])
p2 = f1.map(stage2, names=['val2'])
p3 = f1.map(stage2, names=['val3'])
p4 = f1.map(stage2, names=['val4'])
p5 = f1.map(stage2, names=['val5'])
# p6 = f1.map(stage2, names=['val6'])
# p7 = f1.map(stage2, names=['val7'])
# p8 = f1.map(stage2, names=['val8'])
p1.join(p2).join(p3).join(p4).join(p5) # .join(p6).join(p7).join(p8)
if do_optimize:
flow = optimize(flow, rules=optimize_rules)
print('Flow has been optimized...')
flow.deploy()
print('Flow successfully deployed!')
latencies = []
inp = Table([('size', IntType)])
inp.insert([DATA_SIZES[data_size]])
print('Starting benchmark...')
for i in range(num_requests):
if i % 100 == 0 and i > 0:
print(f'On request {i}...')
start = time.time()
res = flow.run(inp).get()
end = time.time()
latencies.append(end - start)
print_latency_stats(latencies, 'E2E')
def run(cloudburst: CloudburstConnection, num_requests: int, gamma: int,
num_replicas: int):
def stage1(self, val: int) -> int:
return val + 1
def stage2(self, row: Row) -> float:
import time
from scipy.stats import gamma
delay = gamma.rvs(3.0, scale=row['scale']) * 10 / 1000 # convert to ms
time.sleep(delay)
return delay
def stage3(self, row: Row) -> float:
return row['val']
print(f'Creating flow with {num_replicas} replicas and' +
f' gamma={GAMMA_VALS[gamma]}')
flow = Flow('fusion-benchmark', FlowType.PUSH, cloudburst)
flow.map(stage1, col='val') \
.map(stage2, names=['val'], high_variance=True) \
.map(stage3, names=['val'])
optimize_rules['compete_replicas'] = num_replicas
flow = optimize(flow, rules=optimize_rules)
print('Flow has been optimized...')
flow.deploy()
print('Flow successfully deployed!')
latencies = []
inp = Table([('val', IntType), ('scale', FloatType)])
inp.insert([1, GAMMA_VALS[gamma]])
print('Starting benchmark...')
for i in range(num_requests):
if i % 100 == 0 and i > 0:
print(f'On request {i}...')
time.sleep(.300) # Sleep to let the queue drain.
start = time.time()
res = flow.run(inp).get()
end = time.time()
latencies.append(end - start)
print_latency_stats(latencies, 'E2E')
def run(flow, cloudburst, requests, local, sckt=None):
if not local:
if not os.path.exists('imagenet_sample.zip'):
raise RuntimeError(
'Expect to have the imagenet_sample directory locally.')
os.system('unzip imagenet_sample.zip')
else:
if not os.path.exists('imagenet_sample/imagenet'):
raise RuntimeError(
'Expect to have the imagenet_sample directory locally.')
prefix = 'imagenet_sample/imagenet'
files = os.listdir(prefix)
files = [os.path.join(prefix, fname) for fname in files]
inputs = []
logging.info('Loading input images...')
for fname in files:
table = Table([('img', NumpyType)])
img = np.array(Image.open(fname).convert('RGB').resize((224, 224)))
table.insert([img])
inputs.append(table)
logging.info('Starting benchmark...')
latencies = []
bench_start = time.time()
for i in range(requests):
if i % 100 == 0:
logging.info(f'On request {i}...')
inp = random.choice(inputs)
start = time.time()
result = flow.run(inp).get()
end = time.time()
latencies.append(end - start)
bench_end = time.time()
print_latency_stats(latencies, "E2E", not local, bench_end - bench_start)
if sckt:
bts = cp.dumps(latencies)
sckt.send(bts)
def run(cloudburst: CloudburstConnection, num_requests: int, num_fns: int,
data_size: str, do_optimize: bool):
def fusion_op(self, row: Row) -> bytes:
return row['data']
print(f'Creating flow with {num_fns} operators and {data_size}' +
f' ({DATA_SIZES[data_size]}) inputs.')
flow = Flow('fusion-benchmark', FlowType.PUSH, cloudburst)
marker = flow
for _ in range(num_fns):
marker = marker.map(fusion_op, names=['data'])
if do_optimize:
flow = optimize(flow, rules=optimize_rules)
print('Flow has been optimized...')
flow.deploy()
print('Flow successfully deployed!')
latencies = []
inp = Table([('data', BtsType)])
inp.insert([os.urandom(DATA_SIZES[data_size])])
print('Starting benchmark...')
for i in range(num_requests):
if i % 100 == 0 and i > 0:
print(f'On request {i}...')
start = time.time()
res = flow.run(inp).get()
end = time.time()
latencies.append(end - start)
print_latency_stats(latencies, 'E2E')
def run(name, kvs, num_requests, sckt):
name = 'locality-' + name
oids = cp.loads(kvs.get(name).reveal())
lambd = boto3.client('lambda', 'us-east-1')
latencies = []
epoch_latencies = []
epoch_kvs = []
epoch_comp = []
epoch_start = time.time()
epoch = 0
for _ in range(num_requests):
args = []
for _ in range(10):
args.append(sys_random.choice(oids))
start = time.time()
loc = str(uuid.uuid4())
body = {'args': args, 'loc': loc}
res = lambd.invoke(FunctionName=name, Payload=json.dumps(body))
res = json.loads(res['Payload'].read())
kvs, comp = res
end = time.time()
invoke = end - start
epoch_kvs.append(kvs)
epoch_comp.append(comp)
total = invoke + kvs
latencies.append(total)
epoch_latencies.append(total)
epoch_end = time.time()
if (epoch_end - epoch_start) > 10:
sckt.send(cp.dumps(epoch_latencies))
utils.print_latency_stats(epoch_latencies,
'EPOCH %d E2E' % (epoch), True)
utils.print_latency_stats(epoch_comp, 'EPOCH %d COMP' % (epoch),
True)
utils.print_latency_stats(epoch_kvs, 'EPOCH %d KVS' % (epoch),
True)
epoch += 1
epoch_latencies.clear()
epoch_kvs.clear()
epoch_comp.clear()
epoch_start = time.time()
return latencies, [], [], 0
def run_bench(bname, num_requests, cloudburst, kvs, sckt, create=False):
logging.info('Running benchmark %s, %d requests.' % (bname, num_requests))
if bname == 'composition':
total, scheduler, kvs, retries = composition.run(cloudburst, num_requests,
sckt)
elif bname == 'locality':
total, scheduler, kvs, retries = locality.run(cloudburst, num_requests,
create, sckt)
elif bname == 'redis' or bname == 's3':
total, scheduler, kvs, retries = lambda_locality.run(bname, kvs,
num_requests,
sckt)
elif bname == 'predserving':
total, scheduler, kvs, retries = predserving.run(cloudburst, num_requests,
sckt)
elif bname == 'mobilenet':
total, scheduler, kvs, retries = mobilenet.run(cloudburst, num_requests,
sckt)
elif bname == 'scaling':
total, scheduler, kvs, retries = scaling.run(cloudburst, num_requests,
sckt, create)
else:
logging.info('Unknown benchmark type: %s!' % (bname))
sckt.send(b'END')
return
# some benchmark modes return no results
if not total:
sckt.send(b'END')
logging.info('*** Benchmark %s finished. It returned no results. ***'
% (bname))
return
else:
sckt.send(b'END')
logging.info('*** Benchmark %s finished. ***' % (bname))
logging.info('Total computation time: %.4f' % (sum(total)))
if len(total) > 0:
utils.print_latency_stats(total, 'E2E', True)
if len(scheduler) > 0:
utils.print_latency_stats(scheduler, 'SCHEDULER', True)
if len(kvs) > 0:
utils.print_latency_stats(kvs, 'KVS', True)
logging.info('Number of KVS get retries: %d' % (retries))
table = Table([('img', StrType)])
img = base64.b64encode(open('panda.jpg', "rb").read()).decode('ascii')
table.insert([img])
cloudburst = CloudburstConnection(sys.argv[1], '3.226.122.35')
flow = Flow('ensemble-flow', FlowType.PUSH, cloudburst)
img = flow.map(transform, init=transform_init, names=['img'])
anet = img.map(alexnet_model, init=alexnet_init, names=['alexnet_index', 'alexnet_perc'])
rnet = img.map(resnet_model, init=resnet_init, names=['resnet_index', 'resnet_perc'])
anet.join(rnet).map(ensemble_predict, names=['class'])
flow.deploy()
from cloudburst.server.benchmarks.utils import print_latency_stats
import time
print('Starting benchmark...')
latencies = []
for _ in range(100):
start = time.time()
result = flow.run(table).get()
end = time.time()
time.sleep(1)
latencies.append(end - start)
print_latency_stats(latencies, "E2E")
def run(cloudburst_client, num_requests, sckt, create):
''' DEFINE AND REGISTER FUNCTIONS '''
dag_name = 'scaling'
if create:
def slp(cloudburst, x):
import time
time.sleep(.050)
return x
cloud_sleep = cloudburst_client.register(slp, 'sleep')
if cloud_sleep:
print('Successfully registered sleep function.')
else:
sys.exit(1)
''' TEST REGISTERED FUNCTIONS '''
sleep_test = cloud_sleep(2).get()
if sleep_test != 2:
print('Unexpected result from sleep(2): %s' % (str(sleep_test)))
sys.exit(1)
print('Successfully tested functions!')
''' CREATE DAG '''
functions = ['sleep']
success, error = cloudburst_client.register_dag(
dag_name, functions, [])
if not success:
print('Failed to register DAG: %s' % (CloudburstError.Name(error)))
sys.exit(1)
return [], [], [], 0
else:
''' RUN DAG '''
arg_map = {'sleep': [1]}
total_time = []
epoch_req_count = 0
epoch_latencies = []
epoch_start = time.time()
epoch = 0
for _ in range(num_requests):
start = time.time()
res = cloudburst_client.call_dag(dag_name, arg_map, True)
end = time.time()
if res is not None:
epoch_req_count += 1
total_time += [end - start]
epoch_latencies += [end - start]
epoch_end = time.time()
if epoch_end - epoch_start > 10:
if sckt:
sckt.send(cp.dumps((epoch_req_count, epoch_latencies)))
logging.info('EPOCH %d THROUGHPUT: %.2f' %
(epoch, (epoch_req_count / 10)))
utils.print_latency_stats(epoch_latencies,
'EPOCH %d E2E' % epoch, True)
epoch += 1
epoch_req_count = 0
epoch_latencies.clear()
epoch_start = time.time()
return total_time, [], [], 0
def run(cloudburst: CloudburstConnection, num_requests: int, batch_size: int,
gpu: bool):
with open('imagenet_classes.txt', 'r') as f:
classes = [line.strip() for line in f.readlines()]
cloudburst.put_object('imagenet-classes', classes)
def resnet_init_gpu(self, cloudburst):
import os
import torch
import torchvision
from torchvision import transforms
tpath = os.path.join(os.getenv('TORCH_HOME'), 'checkpoints')
self.resnet = torch.load(os.path.join(tpath, 'resnet101.model')).cuda()
self.resnet.eval()
self.transforms = transforms.Compose([
transforms.ToPILImage(),
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
self.classes = cloudburst.get('imagenet-classes')
def resnet_model_gpu(self, table: Table) -> str:
"""
AlexNet for image classification on ImageNet
"""
import torch
inputs = []
for row in table.get():
img = self.transforms(row['img'])
inputs.append(img)
inputs = torch.stack(inputs, dim=0).cuda()
output = self.resnet(inputs)
_, indices = torch.sort(output, descending=True)
indices = indices.cpu().detach().numpy()
result = []
for idx_set in indices:
index = idx_set[0]
result.append(self.classes[index])
return result
def resnet_init_cpu(self, cloudburst):
import os
import torch
import torchvision
from torchvision import transforms
tpath = os.path.join(os.getenv('TORCH_HOME'), 'checkpoints')
self.resnet = torch.load(os.path.join(tpath, 'resnet101.model'))
self.resnet.eval()
self.transforms = transforms.Compose([
transforms.ToPILImage(),
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
self.classes = cloudburst.get('imagenet-classes')
def resnet_model_cpu(self, table: Table) -> str:
"""
AlexNet for image classification on ImageNet
"""
import torch
inputs = []
for row in table.get():
img = self.transforms(row['img'])
inputs.append(img)
inputs = torch.stack(inputs, dim=0)
output = self.resnet(inputs)
_, indices = torch.sort(output, descending=True)
indices = indices.detach().numpy()
result = []
for idx_set in indices:
index = idx_set[0]
result.append(self.classes[index])
return result
print(f'Creating flow with size {batch_size} batches.')
flow = Flow('batching-benchmark', FlowType.PUSH, cloudburst)
if gpu:
flow.map(resnet_model_gpu,
init=resnet_init_gpu,
names=['class'],
gpu=True,
batching=True)
else:
flow.map(resnet_model_cpu,
init=resnet_init_cpu,
names=['class'],
batching=True)
flow.deploy()
print('Flow successfully deployed!')
latencies = []
inp = Table([('img', NumpyType)])
img = np.array(Image.open('panda.jpg').convert('RGB').resize((224, 224)))
inp.insert([img])
kvs = cloudburst.kvs_client
if gpu:
print('Starting GPU warmup...')
for _ in range(50):
flow.run(inp).get()
print('Finished warmup...')
print('Starting benchmark...')
for i in range(num_requests):
if i % 100 == 0 and i > 0:
print(f'On request {i}...')
futs = []
for _ in range(batch_size):
futs.append(flow.run(inp))
pending = set([fut.obj_id for fut in futs])
# Break these apart to batch the KVS get requests.
start = time.time()
while len(pending) > 0:
get_start = time.time()
response = kvs.get(list(pending))
for key in response:
if response[key] is not None:
pending.discard(key)
end = time.time()
latencies.append(end - start)
compute_time = np.mean(latencies) * num_requests
tput = (batch_size * num_requests) / (compute_time)
print('THROUGHPUT: %.2f' % (tput))
print_latency_stats(latencies, 'E2E')
if type(msg) == tuple:
epoch_thruput += msg[0]
new_tot = msg[1]
else:
new_tot = msg
epoch_total += new_tot
total += new_tot
epoch_recv += 1
if epoch_recv == sent_msgs:
epoch_end = time.time()
elapsed = epoch_end - epoch_start
thruput = epoch_thruput / elapsed
logging.info('\n\n*** EPOCH %d ***' % (epoch))
logging.info('\tTHROUGHPUT: %.2f' % (thruput))
utils.print_latency_stats(epoch_total, 'E2E', True)
epoch_recv = 0
epoch_thruput = 0
epoch_total.clear()
epoch_start = time.time()
epoch += 1
logging.info('*** END ***')
if len(total) > 0:
utils.print_latency_stats(total, 'E2E', True)
None)
elif bname == 'pred_serving':
total, scheduler, kvs, retries = predserving.run(cloudburst_client,
num_requests, None)
elif bname == 'avg':
total, scheduler, kvs, retries = dist_avg.run(cloudburst_client, num_requests,
None)
elif bname == 'center_avg':
total, scheduler, kvs, retries = centr_avg.run(cloudburst_client, num_requests,
None)
elif bname == 'summa':
total, scheduler, kvs, retries = summa.run(cloudburst_client, num_requests,
None)
elif bname == 'scaling':
total, scheduler, kvs, retries = scaling.run(cloudburst_client, num_requests,
None)
else:
print('Unknown benchmark type: %s!' % (bname))
print('Total computation time: %.4f' % (sum(total)))
if total:
utils.print_latency_stats(total, 'E2E')
if scheduler:
utils.print_latency_stats(scheduler, 'SCHEDULER')
if kvs:
utils.print_latency_stats(kvs, 'KVS')
if retries > 0:
print('Number of KVS get retries: %d' % (retries))
def run(cloudburst: CloudburstConnection,
num_requests: int,
data_size: str,
breakpoint: bool,
do_optimize: bool):
print('Creating data...')
size = DATA_SIZES[data_size]
for i in range(1, NUM_DATA_POINTS+1):
arr = np.random.rand(size)
cloudburst.put_object('data-' + str(i), arr)
def stage1(self, row: Row) -> (int, str):
idx = int(row['req_num'] / 10) + 1
key = 'data-%d' % (idx)
return idx, key
def stage2(self, row: Row) -> str:
import numpy as np
arr = row[row['key']]
return float(np.sum(arr))
print(f'Creating flow with {data_size} ({DATA_SIZES[data_size]}) inputs.')
flow = Flow('locality-benchmark', FlowType.PUSH, cloudburst)
flow.map(stage1, names=['index', 'key']) \
.lookup('key', dynamic=True) \
.map(stage2, names=['sum'])
optimize_rules['breakpoint'] = breakpoint
if do_optimize:
flow = optimize(flow, rules=optimize_rules)
print('Flow has been optimized...')
flow.deploy()
print('Flow successfully deployed!')
latencies = []
inp = Table([('req_num', IntType)])
if breakpoint:
print('Starting warmup...')
for i in range(NUM_DATA_POINTS):
inp = Table([('req_num', IntType)])
inp.insert([i * 10])
res = flow.run(inp).get()
print('Pausing to let cache metadata propagate...')
time.sleep(15)
print('Starting benchmark...')
for i in range(num_requests):
if i % 100 == 0 and i > 0:
print(f'On request {i}...')
inp = Table([('req_num', IntType)])
inp.insert([i])
start = time.time()
res = flow.run(inp).get()
end = time.time()
latencies.append(end - start)
with open('data.bts', 'wb') as f:
from cloudburst.shared.serializer import Serializer
ser = Serializer()
bts = ser.dump(latencies)
f.write(bts)
print_latency_stats(latencies, 'E2E')
def run(cloudburst_client, num_requests, create, sckt):
dag_name = 'locality'
kvs_key = 'LOCALITY_OIDS'
if create:
''' DEFINE AND REGISTER FUNCTIONS '''
def dot(cloudburst, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10):
import numpy as np
s1 = np.add(v1, v2)
s2 = np.add(v3, v4)
s3 = np.add(v5, v6)
s4 = np.add(v7, v8)
s5 = np.add(v9, v10)
s1 = np.add(s1, s2)
s2 = np.add(s3, s4)
s1 = np.add(s1, s2)
s1 = np.add(s1, s5)
return np.average(s1)
cloud_dot = cloudburst_client.register(dot, 'dot')
if cloud_dot:
logging.info('Successfully registered the dot function.')
else:
sys.exit(1)
''' TEST REGISTERED FUNCTIONS '''
refs = ()
for _ in range(10):
inp = np.zeros(OSIZE)
k = str(uuid.uuid4())
cloudburst_client.put_object(k, inp)
refs += (CloudburstReference(k, True),)
dot_test = cloud_dot(*refs).get()
if dot_test != 0.0:
print('Unexpected result from dot(v1, v2): %s' % (str(dot_test)))
sys.exit(1)
logging.info('Successfully tested function!')
''' CREATE DAG '''
functions = ['dot']
connections = []
success, error = cloudburst_client.register_dag(dag_name, functions,
connections)
if not success and error != DAG_ALREADY_EXISTS:
print('Failed to register DAG: %s' % (CloudburstError.Name(error)))
sys.exit(1)
# for the hot version
oid = str(uuid.uuid4())
arr = np.random.randn(OSIZE)
cloudburst_client.put_object(oid, arr)
cloudburst_client.put_object(kvs_key, [oid])
return [], [], [], 0
else:
''' RUN DAG '''
# num_data_objects = num_requests * 10 # for the cold version
# oids = []
# for i in range(num_data_objects):
# if i % 100 == 0:
# logging.info('On object %d.' % (i))
# array = np.random.rand(OSIZE)
# oid = str(uuid.uuid4())
# cloudburst_client.put_object(oid, array)
# oids.append(oid)
# logging.info('Finished creating data!')
# for the hot version
oids = cloudburst_client.get_object(kvs_key)
total_time = []
scheduler_time = []
kvs_time = []
retries = 0
log_start = time.time()
log_epoch = 0
epoch_total = []
for i in range(num_requests):
refs = []
# for ref in oids[(i * 10):(i * 10) + 10]: # for the cold version
# refs.append(CloudburstReference(ref, True))
for _ in range(10): # for the hot version
refs.append(CloudburstReference(oids[0], True))
start = time.time()
arg_map = {'dot': refs}
rid = cloudburst_client.call_dag(dag_name, arg_map, True)
end = time.time()
epoch_total.append(end - start)
total_time.append(end - start)
# start = time.time()
# rid.get()
# end = time.time()
# kvs_time.append(end - start)
log_end = time.time()
if (log_end - log_start) > 10:
if sckt:
sckt.send(cp.dumps(epoch_total))
utils.print_latency_stats(epoch_total, 'EPOCH %d E2E' %
(log_epoch), True)
epoch_total.clear()
log_epoch += 1
log_start = time.time()
return total_time, scheduler_time, kvs_time, retries
