def main(): "Create flows" sys.stdout.flush() # Initialize workload workload = Workload(args.workload) receivers = readReceivers() print "NUM RECEIVERS: %d" % len(receivers) flowStartCmd = "sudo python ./flowGenerator.py --src-ip %s --src-port %d --dest-ip %s --dest-port %d --num-packets %d --num-bands %d --max-packets %d --packet-size %d --out %s/send-%s-%d.txt > test.txt" #random.seed(1234568) print "STARTING AT TIME %f" % time() srcPort = 5000 start = time() while time() - start < args.time: lambd = args.load * args.bw * 1000000 / 8 / args.packet_size / workload.getAverageFlowSize() waitTime = random.expovariate(lambd) print "Sleeping for %f seconds..." % waitTime sys.stdout.flush() sleep(waitTime) # get random receiver i = random.randrange(len(receivers)) (dest_ip, dest_port) = receivers[i] numPackets = workload.getFlowSize() print "Sending %d packets from %s:%d to %s:%d" % (numPackets, args.src_ip, srcPort, dest_ip, dest_port) Popen(flowStartCmd % (args.src_ip, srcPort, dest_ip, dest_port, numPackets, args.num_bands, workload.getMaxFlowSize(), args.packet_size, args.output_dir, args.src_ip, srcPort), shell=True) srcPort += 1 print "ENDING AT TIME %f" % time()
def test_edit_workload(): # Workloads to compare against workload1 = Workload("test_workload(I_v3)", "", "https://tfhub.dev/google/imagenet/inception_v3/feature_vector/1", 4000, 0.01, 10, 10, 10, 100, -1, 100, False, 0, 0, 0, "") workload2 = Workload("Small_Workload", "", "https://tfhub.dev/google/imagenet/inception_v3/feature_vector/1", 200, 0.01, 10, 10, 10, 100, -1, 100, False, 0, 0, 0, "") # Call get_workloads WL_data = aixprt.get_workloads() assert len(WL_data) == 2 # Check that both workloads exist in the system assert compare_workloads(WL_data["test_workload(I_v3)"], workload1) assert compare_workloads(WL_data["Small_Workload"], workload2) # Now edit the first workload aixprt.edit_workload("test_workload(I_v3)", "workload_edited", "edited", "https://tfhub.dev/google/imagenet/inception_v3/feature_vector/1", "5000", "0.01", "10", "10", "20", "100", "-2", "100", "True", "0", "0", "0", "") workload1_edited = Workload("workload_edited", "edited", "https://tfhub.dev/google/imagenet/inception_v3/feature_vector/1", 5000, 0.01, 10, 10, 20, 100, -2, 100, True, 0, 0, 0, "") # Enure it was edited properly WL_data = aixprt.get_workloads() assert len(WL_data) == 2 assert not "test_workload(I_v3)" in WL_data assert "workload_edited" in WL_data assert compare_workloads(WL_data["workload_edited"], workload1_edited) # Now try to edit a workload to have all invalid parameters (with the exception of name, comment, and command) actual_invalid = aixprt.edit_workload("Small_Workload", "invalidWL", "", '', "cat", "0.0.0.0", "1.0", "False", "1.0", "asdga100", "--2", "1%00", "FTrue", "z", "e", "ro", "") expected_invalid = ["tfhub_model", "training_steps", "learning_rate", "testing_percentage", "validation_percentage", "eval_step_interval", "train_batch_size", "test_batch_size", "validation_batch_size", "flip_left_right", "random_crop", "random_scale", "random_brightness"] WL_data = aixprt.get_workloads() assert len(WL_data) == 2 assert len(expected_invalid) == 13 assert expected_invalid == actual_invalid # Ensure that the workload was not edited assert compare_workloads(WL_data["Small_Workload"], workload2)
def test_remove_workload(): expected_workload1 = Workload("test_workload(I_v3)", "", "https://tfhub.dev/google/imagenet/inception_v3/feature_vector/1", 4000, 0.01, 10, 10, 10, 100, -1, 100, False, 0, 0, 0, "") expected_workload2 = Workload("Small_Workload", "", "https://tfhub.dev/google/imagenet/inception_v3/feature_vector/1", 200, 0.01, 10, 10, 10, 100, -1, 100, False, 0, 0, 0, "") expected_workload3 = Workload("Smaller_Workload", "", "https://tfhub.dev/google/imagenet/inception_v3/feature_vector/1", 100, 0.01, 10, 10, 10, 100, -1, 100, False, 0, 0, 0, "") # Ensure file is setup properly WL_data = aixprt.get_workloads() assert len(WL_data) == 2 assert "test_workload(I_v3)" in WL_data assert "Small_Workload" in WL_data # Try to remove a non-existent workload assert aixprt.remove_workload("Not_in_file") == 0 assert len(WL_data) == 2 assert "test_workload(I_v3)" in WL_data assert "Small_Workload" in WL_data # Remove an existing workload actual_workload1 = aixprt.remove_workload("test_workload(I_v3)") WL_data = aixprt.get_workloads() assert len(WL_data) == 1 assert not "test_workload(I_v3)" in WL_data assert compare_workloads(expected_workload1, actual_workload1) # Remove an existing workload that is used in a suite suite1 = aixprt.get_suite("Small_Suite") assert len(suite1) == 2 actual_workload2 = aixprt.remove_workload("Small_Workload") WL_data = aixprt.get_workloads() assert len(WL_data) == 0 assert not "Small_Workload" in WL_data assert compare_workloads(expected_workload2, actual_workload2) suite1 = aixprt.get_suite("Small_Suite") assert len(suite1) == 1 assert not "Small_Workload" in suite1 # Remove a workload that occurs multiple times in multiple suites # Setup for this particular test: aixprt.add_workload("test_workload(I_v3)", "", 'https://tfhub.dev/google/imagenet/inception_v3/feature_vector/1', "4100", "0.01", "10", "10", "10", "100", "-1", "100", "False", "0", "0", "0", "") aixprt.add_workload("Small_Workload", "", 'https://tfhub.dev/google/imagenet/inception_v3/feature_vector/1', "200", "0.01", "10", "10", "10", "100", "-1", "100", "False", "0", "0", "0", "") aixprt.add_workload("Smaller_Workload", "", 'https://tfhub.dev/google/imagenet/inception_v3/feature_vector/1', "100", "0.01", "10", "10", "10", "100", "-1", "100", "False", "0", "0", "0", "") WL_data = aixprt.get_workloads() assert len(WL_data) == 3 suite2_A = aixprt.get_suite("Small_Suite") assert len(suite2_A) == 1 suite2_B = aixprt.get_suite("Smaller_Suite") assert len(suite2_B) == 2 # Now remove actual_workload3 = aixprt.remove_workload("Smaller_Workload") WL_data = aixprt.get_workloads() assert len(WL_data) == 2 assert not "Smaller_Workload" in WL_data assert compare_workloads(expected_workload3, actual_workload3) suite2_A = aixprt.get_suite("Small_Suite") assert len(suite2_A) == 0 assert not "Smaller_Workload" in suite2_A suite2_B = aixprt.get_suite("Smaller_Suite") assert len(suite2_B) == 0 assert not "Smaller_Workload" in suite2_B
def workload(): """ A fixture for a Workload. """ _workload = Workload() yield _workload _workload.terminate()
def generate_query(self): if 'workload' in self.scheme: rel_list = [(x.source, x.target, x.label) for x in self.relation_ins] wl = Workload(list(self.node_labels), rel_list, self.workload) queries = wl.generate_workload() filename = 'query.cypher' tl = Translate(filename) tl.translate(queries)
def test_get_workloads(): workload1 = Workload("test_workload(I_v3)", "", "https://tfhub.dev/google/imagenet/inception_v3/feature_vector/1", 4000, 0.01, 10, 10, 10, 100, -1, 100, False, 0, 0, 0, "") workload2 = Workload("Small_Workload", "", "https://tfhub.dev/google/imagenet/inception_v3/feature_vector/1", 200, 0.01, 10, 10, 10, 100, -1, 100, False, 0, 0, 0, "") # Call get_workloads WL_data = aixprt.get_workloads() # Check that there are only 2 workloads in the list and ensure both are correct assert len(WL_data) == 2 assert "test_workload(I_v3)" in WL_data assert "Small_Workload" in WL_data assert compare_workloads(WL_data["test_workload(I_v3)"], workload1) assert compare_workloads(WL_data["Small_Workload"], workload2)
def main(names, component_id=1): count = len(names) pp.figure(figsize=(14, 2 * count), facecolor='w', edgecolor='k') for i in range(count): data = Workload.aggregate(Workload.locate(names[i]), 'dynamic_power') step_count = data.shape[1] time = constant.TIME_STEP * np.arange(step_count) pp.subplot(count, 1, i + 1) pp.plot(time, data[component_id, :]) pp.xlim([time[0], time[-1]]) pp.xlabel('Time (s)') pp.ylabel('Power (W)') pp.show()
def process(name): data = Workload.aggregate(Workload.locate(name), 'dynamic_power') component_count, step_count = data.shape time = constant.TIME_STEP * np.arange(0, step_count) pp.figure(figsize=(14, 2 * component_count), facecolor='w', edgecolor='k') for i in range(component_count): pp.subplot(component_count, 1, i + 1) pp.plot(time, data[i, :]) pp.xlim([time[0], time[-1]]) pp.ylabel('Power (W)') if i == 0: pp.title(name) pp.xlabel('Time (s)')
def get_workloads(): """ Retrieves the current workloads in the workloads.json file and returns them as a dictionary of paired workload names and Workload objects :return: A dictionary with each workload name being used as keys to access the related workload objects """ data = {} json_path = Path("workloads.json") if (not json_path.is_file()): return data data = load_workloads() workloads_dict = {} for x in data['workloads']: wl = Workload(x['name'], x['comment'], x['tfhub model'], x['training steps'], x['learning rate'], x['testing percentage'], x['validation percentage'], x['eval step interval'], x['train batch size'], x['test batch size'], x['validation batch size'], x['flip left/right'], x['random crop'], x['random scale'], x['random brightness'], x['command']) workloads_dict[x['name']] = wl return workloads_dict
def generateWorkload(self): W = [] logging.basicConfig(filename='example.log', level=logging.DEBUG) creation_Time = 0 i = 0 TotalCPUTime = np.random.normal(self.mean_job_time, self.sd_job_time) TotalCPUTime = round(TotalCPUTime * self.time_unit) logging.info('Task' + str(i) + ":Creation Time:" + str(creation_Time) + ":Total CPU time:" + str(TotalCPUTime)) IO = IOGenerator(self.rate_io, TotalCPUTime, self.mean_io_time, self.sd_io_time, self.time_unit) iolist = IOList(IO.generateIOEvent()) T = Task("Task 0", 0, np.abs(creation_Time), np.abs(TotalCPUTime), iolist) W.append(T) for i in xrange(self.num_jobs): creation_Time = round( random.expovariate(self.rate_jobs) * self.time_unit) + W[-1].creationTime TotalCPUTime = np.random.normal(self.mean_job_time, self.sd_job_time) TotalCPUTime = round(TotalCPUTime * self.time_unit) logging.info('Task' + str(i) + ":Creation Time:" + str(creation_Time) + ":Total CPU time:" + str(TotalCPUTime)) IO = IOGenerator(self.rate_io, TotalCPUTime, self.mean_io_time, self.sd_io_time, self.time_unit) iolist = IOList(IO.generateIOEvent()) T = Task("Task " + str(i), 0, np.abs(creation_Time), np.abs(TotalCPUTime), iolist) W.append(T) self.work_load_list = W w = Workload(W) return w
def create_workload(self, name, image="busybox", network=None): """ Create a workload container inside this host container. """ workload = Workload(self, name, image=image, network=network) self.workloads.add(workload) return workload
def create_workload(self, name, image="busybox", network="bridge", ip=None, labels=[]): """ Create a workload container inside this host container. """ workload = Workload(self, name, image=image, network=network, ip=ip, labels=labels) self.workloads.add(workload) return workload
def remove_workload(workload_name): """ Removes a workload from the workloads.json :param workload_name: name of the workload being removed\n :return: the removed workload, if it was sucessfuly removed from the list; else, it returns 0\n """ # Opening workloads.json into data data = {} json_path = Path("workloads.json") if (not json_path.is_file()): return data data = load_workloads() removed_WL = {} index = 0 # Traverses through the list of workloads until it finds a matching name, # then it removes the workload from the list and saves it to removed_WL for x in data['workloads']: if x['name'] == workload_name: removed_WL = data['workloads'].pop(index) index += 1 # If a workload was removed, then update the json and return the removed workload if removed_WL: with open('workloads.json', 'w') as json_file: json.dump(data, json_file, indent=4) json_file.close() # Check if that workload was in any suites suite_appearances = is_workload_in_suites(workload_name) # If so, then remove them from the suites if not suite_appearances == 1: count = len(suite_appearances) - 1 while count >= 0: remove_workload_from_suite(suite_appearances[count][1], suite_appearances[count][0]) count -= 1 # Create an actual workload object with removed_WL removed_WL = Workload( removed_WL['name'], removed_WL['comment'], removed_WL['tfhub model'], removed_WL['training steps'], removed_WL['learning rate'], removed_WL['testing percentage'], removed_WL['validation percentage'], removed_WL['eval step interval'], removed_WL['train batch size'], removed_WL['test batch size'], removed_WL['validation batch size'], removed_WL['flip left/right'], removed_WL['random crop'], removed_WL['random scale'], removed_WL['random brightness'], removed_WL['command']) return removed_WL # Else, return 0 return 0
def main(names, draw=True): paths = [] for name in names: paths.extend(Workload.enumerate(name)) print('%-20s %10s %10s' % ('Benchmark', 'Components', 'Time, s')) count = len(paths) cores = np.zeros([count, 1]) cache = np.zeros([count, 1]) for i in range(count): name = '/'.join(paths[i].split('/')[-2:]).replace('.sqlite3', '') data = Workload.aggregate(paths[i], 'dynamic_power') [component_count, step_count] = data.shape sys.stdout.write( '%-20s %10d %10.2f' % (name, component_count, constant.TIME_STEP * step_count)) total = np.sum(data) index = int(max(1, np.floor(component_count / 4))) index = range(component_count - index) for j in range(component_count): local = np.sum(data[j, :]) sys.stdout.write(' %10.2e (%5.2f%%)' % (local, 100 * local / total)) if j in index: cores[i] += local else: cache[i] += local print() scale = cores + cache cores = 100 * cores / scale cache = 100 * cache / scale print('Average cache contribution: %.2f%%' % np.mean(cache)) if not draw: return index = list(range(count)) pp.figure(figsize=(14, 6), facecolor='w', edgecolor='k') pp.bar(index, cores, color='b') pp.bar(index, cache, color='y', bottom=cores) pp.ylim([0, 101]) pp.show()
def dummyGen(): ioList1 = IOList([IO(10, 5), IO(13, 7)]) ioList2 = IOList([]) # high priority IO instensive task t1 = Task("Task 1", 0, 3, 20, ioList1) # low priority CPU intensive task t2 = Task("Task 2", 10, 0, 15, ioList2) w = Workload([t1, t2]) return w
def _queryTrObjsByCatView(wlNames=None, areaIds=None, areaNames=None, branchNames=None, testerIds=None, bldTypes=None): def _argsWrapper(args): if not args: return [None] else: return args if not wlNames: wlIds = [None] else: from workload import Workload wlIds = Workload.getWorkloadIds(wlNames) if areaNames: areaIds = Area.getAreaIds(areaNames) elif areaIds: childrenIds = Area.getChildrenAreaIds(areaIds) if childrenIds: areaIds.extend(childrenIds) else: areaIds = [None] # The order for filterKey and filterLoop should be matched filterKey = ['workload', 'area__id__in', 'deliverables__build__branch__in', 'tester__id__in', 'deliverables__build__bldtype__in'] filterLoop = itertools.product(wlIds, areaIds, _argsWrapper(branchNames), _argsWrapper(testerIds), _argsWrapper(bldTypes)) filterLoop = list(filterLoop) trObjs = [] for item in filterLoop: filterMap = {} for i in range(len(item)): if item[i] is None: continue filterMap[filterKey[i]] = item[i] logger.info("Retrieving testrun data via RestAPI with filterInfo: %s", filterMap) tmpTrObjs = queryCatInfo('testrun', filterMap, limitDay=limitDay, limit=limitNumber, orderBy="-endtime") if tmpTrObjs: logger.info("Totally get %d testruns" % len(tmpTrObjs)) trObjs.extend(tmpTrObjs) return trObjs
def create_workload(self, base_name, image="busybox", network="bridge", ip=None, labels=[], namespace=None): """ Create a workload container inside this host container. """ name = base_name + "_" + \ ''.join([random.choice(string.ascii_letters) for ii in range(6)]).lower() workload = Workload(self, name, image=image, network=network, ip=ip, labels=labels, namespace=namespace) self.workloads.add(workload) return workload
def test_add_workload(): WL_data = aixprt.get_workloads() workload1 = Workload("test_workload1", "test comment", 'https://tfhub.dev/google/imagenet/inception_v3/feature_vector/1', 4100, 0.02, 10, 20, 10, 100, -2, 100, True, 0, 0, 0, "") # Test to ensure that the workload to be added does not exist in the file assert not "test_workload1" in WL_data # Then add and assert that it does exist in it aixprt.add_workload("test_workload1", "test comment", 'https://tfhub.dev/google/imagenet/inception_v3/feature_vector/1', "4100", "0.02", "10", "20", "10", "100", "-2", "100", "True", "0", "0", "0", "") WL_data = aixprt.get_workloads() assert "test_workload1" in WL_data assert compare_workloads(WL_data["test_workload1"], workload1) # Try to add the same workload again and ensure it does not assert len(WL_data) == 3 assert aixprt.add_workload("test_workload1", "test comment", 'https://tfhub.dev/google/imagenet/inception_v3/feature_vector/1', "4100", "0.02", "10", "20", "10", "100", "-2", "100", "True", "0", "0", "0", "") == 1 WL_data = aixprt.get_workloads() assert len(WL_data) == 3 # Try to add invalid workloads and ensure it does not # Workload with 1 thing wrong invalid_1 = aixprt.add_workload("wrong_workload", "test comment", 'https://tfhub.dev/google/imagenet/inception_v3/feature_vector/1', "4100", "0.02", "1.0", "20", "10", "100", "-2", "100", "True", "0", "0", "0", "") expected_invalid_1 = ["testing_percentage"] WL_data = aixprt.get_workloads() assert len(WL_data) == 3 assert len(invalid_1) == 1 assert invalid_1 == expected_invalid_1 # Workload with everything wrong (except for the comment and command parameters, which can both be empty strings) invalid_2 = aixprt.add_workload("", "", '', "cat", "0..02", "1.0", "False", "1.0", "asdga100", "--2", "1%00", "FTrue", "z", "e", "ro", "") expected_invalid_2 = ["name", "tfhub_model", "training_steps", "learning_rate", "testing_percentage", "validation_percentage", "eval_step_interval", "train_batch_size", "test_batch_size", "validation_batch_size", "flip_left_right", "random_crop", "random_scale", "random_brightness"] WL_data = aixprt.get_workloads() assert len(WL_data) == 3 assert len(invalid_2) == 14 #TODO fix after regex is corrected assert invalid_2 == expected_invalid_2 # Add a workload with a command WL_data = aixprt.get_workloads() assert len(WL_data) == 3 assert aixprt.add_workload("test_workload2", "", '', "", "", "", "", "", "", "", "", "", "", "", "", "help dir") == 0 WL_data = aixprt.get_workloads() assert len(WL_data) == 4
def parse_config(fpath): conf = yaml.load(open(fpath, 'r'), Loader=yaml.FullLoader)['workload'] if conf['interarrival'].lower().endswith('s'): conf['interarrival'] = int(conf['interarrival'].lower().rsplit('s', 1)[0]) if conf['duration'].lower().endswith('s'): conf['duration'] = int(conf['duration'].lower().rsplit('s', 1)[0]) elif conf['duration'].lower().endswith('m'): conf['duration'] = int(conf['duration'].lower().rsplit('m', 1)[0]) * 60 elif conf['duration'].lower().endswith('h'): conf['duration'] = int(conf['duration'].lower().rsplit('h', 1)[0]) * 3600 if conf['cache']['size'].lower().endswith('g'): conf['cache']['size'] = int(conf['cache']['size'].lower().rsplit( 'g', 1)[0]) * (1 << 30) return conf if __name__ == '__main__': conf_file = sys.argv[1] if len(sys.argv) > 1 else 'config.yaml' conf = parse_config(conf_file) env = simpy.Environment() workload = Workload(conf, type='simulation', env=env) env.run(until=conf['duration']) workload.dump_stats()
def add_workload(name, comment, tfhub_model, training_steps, learning_rate, testing_percentage, validation_percentage, eval_step_interval, train_batch_size, test_batch_size, validation_batch_size, flip_left_right, random_crop, random_scale, random_brightness, command=None): """ Adds a new workload entry to the workloads.json file :param name: Name of the new workload\n :param comment: Optional workload comment/description of the workload\n :param tfhub_model: URL pertaining to the machine learning model that is to be used from Tensorflow's github\n :param training_steps: The number of training steps that are run before ending\n :param learning_rate: Numerical rate pertaining to the rate of learning when training\n :param testing_percentage: Percentage of images to use as a test set\n :param validation_percentage: Percentage of images to use as a validation set\n :param eval_step_interval: Number of steps to evaluate the training results after\n :param train_batch_size: Number of images to train on at a time\n :param test_batch_size: Number of images to test on. "-1" causes the entire test set to be used\n :param validation_batch_size: Number of images to use in the evaluation batch. "-1" causes the entire validation set to be used\n :param flip_left_right: True or False, whether to randomly flip half of the training images horizontally\n :param random_crop: Percentage determining how much of a margin to randomly crop off the training images\n :param random_scale: Percentage determining how much to randomly scale the size of the training images by\n :param random_brightness: Percentage determining how much to randomly multiply the training image input pixels up or down by\n :param command: Optional console command to be run upon running this workload\n :return: A list of invalid parameters if an error has occured. A 0 is returned on success\n """ # Creating a new workload to validate parameters new_WL = Workload(name, comment, tfhub_model, training_steps, learning_rate, testing_percentage, validation_percentage, eval_step_interval, train_batch_size, test_batch_size, validation_batch_size, flip_left_right, random_crop, random_scale, random_brightness, command) invalid_params = "" invalid_params = new_WL.validate_parameters() # If parameters are valid, then try to open workloads.json if len(invalid_params) == 0: json_path = Path("workloads.json") if json_path.is_file(): data = load_workloads() for x in data['workloads']: # Check if a workload with the same name already exists; if so, don't add this new one if (x['name'] == name): print(f"Workload {name} already exists.") return 1 # If workloads.json exists and the workload is acceptable, then add it to the workloads dictionary else: data = {} data['workloads'] = [] data['workloads'].append({ 'name': new_WL.name, 'comment': new_WL.comment, 'tfhub model': new_WL.tfhub_model, 'training steps': new_WL.training_steps, 'learning rate': new_WL.learning_rate, 'testing percentage': new_WL.testing_percentage, 'validation percentage': new_WL.validation_percentage, 'eval step interval': new_WL.eval_step_interval, 'train batch size': new_WL.train_batch_size, 'test batch size': new_WL.test_batch_size, 'validation batch size': new_WL.validation_batch_size, 'flip left/right': new_WL.flip_left_right, 'random crop': new_WL.random_crop, 'random scale': new_WL.random_scale, 'random brightness': new_WL.random_brightness, 'command': new_WL.command }) # Put updated dictionary into workloads.json with open('workloads.json', 'w') as json_file: json.dump(data, json_file, indent=4) json_file.close() # Success return 0 else: return invalid_params
def edit_workload(original_name, new_name, comment, tfhub_model, training_steps, learning_rate, testing_percentage, validation_percentage, eval_step_interval, train_batch_size, test_batch_size, validation_batch_size, flip_left_right, random_crop, random_scale, random_brightness, command=None): """ Edits an existing workload in the workloads.json file :param original_name: Name of the workload to be edited\n :param new_name: Name of the new workload\n :param comment: Optional workload comment/description of the workload\n :param tfhub_model: URL pertaining to the machine learning model that is to be used from Tensorflow's github\n :param training_steps: The number of training steps that are run before ending\n :param learning_rate: Numerical rate pertaining to the rate of learning when training\n :param testing_percentage: Percentage of images to use as a test set\n :param validation_percentage: Percentage of images to use as a validation set\n :param eval_step_interval: Number of steps to evaluate the training results after\n :param train_batch_size: Number of images to train on at a time\n :param test_batch_size: Number of images to test on. "-1" causes the entire test set to be used\n :param validation_batch_size: Number of images to use in the evaluation batch. "-1" causes the entire validation set to be used\n :param flip_left_right: True or False, whether to randomly flip half of the training images horizontally\n :param random_crop: Percentage determining how much of a margin to randomly crop off the training images\n :param random_scale: Percentage determining how much to randomly scale the size of the training images by\n :param random_brightness: Percentage determining how much to randomly multiply the training image input pixels up or down by\n :param command: Optional console command to be run upon running this workload\n :return: A list of invalid parameters if an error has occured\n """ # Creating a updated workload to validate parameters new_WL = Workload(new_name, comment, tfhub_model, training_steps, learning_rate, testing_percentage, validation_percentage, eval_step_interval, train_batch_size, test_batch_size, validation_batch_size, flip_left_right, random_crop, random_scale, random_brightness, command) invalid_params = "" invalid_params = new_WL.validate_parameters() # If parameters are valid, then try to open workloads.json if len(invalid_params) == 0: data = {} json_path = Path("workloads.json") # Check if the workloads file exists in order to edit it if (not json_path.is_file()): print("There are currently no workloads to edit.") return # If it exists, then load the workloads from it data = load_workloads() # Edit the parameters of the workload corresponding to the original name for x in data['workloads']: if x['name'] == original_name: x['name'] = new_WL.name x['comment'] = new_WL.comment x['tfhub model'] = new_WL.tfhub_model x['training steps'] = new_WL.training_steps x['learning rate'] = new_WL.learning_rate x['testing percentage'] = new_WL.testing_percentage x['validation percentage'] = new_WL.validation_percentage x['eval step interval'] = new_WL.eval_step_interval x['train batch size'] = new_WL.train_batch_size x['test batch size'] = new_WL.test_batch_size x['validation batch size'] = new_WL.validation_batch_size x['flip left/right'] = new_WL.flip_left_right x['random crop'] = new_WL.random_crop x['random scale'] = new_WL.random_scale x['random brightness'] = new_WL.random_brightness x['command'] = new_WL.command # Put updated dictionary into workloads.json with open('workloads.json', 'w') as json_file: json.dump(data, json_file, indent=4) json_file.close() return 0 else: return invalid_params
import datetime from workload import Workload date = datetime.date(2018, 6, 1) workload = Workload() issues = workload.getWorkloadPerIssue(date) totalWorkload = 0 print "| Issue | Workload |" for issue in issues: totalWorkload += issues[issue] print "|[" + issue.key + "|" + workload.account + issue.key + "] " + issue.fields.summary + "| " + str( float(issues[issue]) / 3600) + "h|" print "| Total |*" + str(float(totalWorkload) / 3600) + "h*|"
from mongo import Mongo from pymongo import * from query import Query import json from bson import json_util from datetime import datetime from workload import Workload if __name__ == "__main__": # Connect to Mongo mongo = Mongo() db = mongo.getDb() pipeline = Workload().getWorkload() f = open("demofile.md", "w") lst = db.list_collection_names() lst.sort() f.write('# Execution stats for each query') f.write('\n') f.write('\n') for i in lst: f.write('- [' + i + '](#' + i + ')') f.write('\n') f.write('\n') for i in lst: f.write('## ')
def main(): # NOTE: you can set invoke count of each function below the chain definitions (the concurrencies variable) # aws lambda1 = LambdaFunction(url="https://lambda1.com", id=1) lambda2 = LambdaFunction(url="https://lambda2.com", id=2) lambda3 = LambdaFunction(url="https://lambda3.com", id=3) lambda4 = LambdaFunction(url="https://lambda4.com", id=4) ## Chain 0 ## node1_c0 = ChainNode(function=lambda1, nodeID=1, children=[], lastNodeIDs=[1], chainFunctionIDs=[1], args={}) chain0 = node1_c0 ## Chain 1 ## # L2 # / # L1 # \ # L3 node3_c1 = ChainNode(function=lambda3, nodeID=3, children=[], lastNodeIDs=[2,3], chainFunctionIDs=[1,2,3], args={}) node2_c1 = ChainNode(function=lambda2, nodeID=2, children=[], lastNodeIDs=[2,3], chainFunctionIDs=[1,2,3], args={}) node1_c1 = ChainNode(function=lambda1, nodeID=1, children=[node2_c1, node3_c1], lastNodeIDs=[2,3], chainFunctionIDs=[1,2,3], args={}) chain1 = node1_c1 ## Chain 2 ## # L3-L2-L1 node3_c2 = ChainNode(function=lambda1, nodeID=3, children=[], lastNodeIDs=[3], chainFunctionIDs=[1,2,3], args={}) node2_c2 = ChainNode(function=lambda2, nodeID=2, children=[node3_c2], lastNodeIDs=[3], chainFunctionIDs=[1,2,3], args={}) node1_c2 = ChainNode(function=lambda3, nodeID=1, children=[node2_c2], lastNodeIDs=[3], chainFunctionIDs=[1,2,3], args={}) chain2 = node1_c2 ## Chain 3 ## # L2 # / # L1 # \ # L3-L4-L4-L4 node6_c3 = ChainNode(function=lambda4, nodeID=6, children=[], lastNodeIDs=[6,2], chainFunctionIDs=[1,2,3,4,4,4], args={}) node5_c3 = ChainNode(function=lambda4, nodeID=5, children=[node6_c3], lastNodeIDs=[6,2], chainFunctionIDs=[1,2,3,4,4,4], args={}) node4_c3 = ChainNode(function=lambda4, nodeID=4, children=[node5_c3], lastNodeIDs=[6,2], chainFunctionIDs=[1,2,3,4,4,4], args={}) node3_c3 = ChainNode(function=lambda3, nodeID=3, children=[node4_c3], lastNodeIDs=[6,2], chainFunctionIDs=[1,2,3,4,4,4], args={}) node2_c3 = ChainNode(function=lambda2, nodeID=2, children=[], lastNodeIDs=[6,2], chainFunctionIDs=[1,2,3,4,4,4], args={}) node1_c3 = ChainNode(function=lambda1, nodeID=1, children=[node2_c1, node3_c1], lastNodeIDs=[6,2], chainFunctionIDs=[1,2,3,4,4,4], args={}) chain3 = node1_c3 ## Chain 4 ## # L2 # / # L1-L3 # \ # L4 node4_c4 = ChainNode(function=lambda4, nodeID=4, children=[], lastNodeIDs=[2,3,4], chainFunctionIDs=[1,2,3,4], args={}) node3_c4 = ChainNode(function=lambda3, nodeID=3, children=[], lastNodeIDs=[2,3,4], chainFunctionIDs=[1,2,3,4], args={}) node2_c4 = ChainNode(function=lambda2, nodeID=2, children=[], lastNodeIDs=[2,3,4], chainFunctionIDs=[1,2,3,4], args={}) node1_c4 = ChainNode(function=lambda1, nodeID=1, children=[node2_c4, node3_c4, node4_c4], lastNodeIDs=[2,3,4], chainFunctionIDs=[1,2,3,4], args={}) chain4 = node1_c4 ## Chain 5 ## # L2 # / # L1 # \ # L3-L3 node4_c5 = ChainNode(function=lambda3, nodeID=4, children=[], lastNodeIDs=[4,2], chainFunctionIDs=[1,2,3,3], args={}) node3_c5 = ChainNode(function=lambda3, nodeID=3, children=[node4_c5], lastNodeIDs=[4,2], chainFunctionIDs=[1,2,3,3], args={}) node2_c5 = ChainNode(function=lambda2, nodeID=2, children=[], lastNodeIDs=[4,2], chainFunctionIDs=[1,2,3,3], args={}) node1_c5 = ChainNode(function=lambda1, nodeID=1, children=[node2_c5, node3_c5], lastNodeIDs=[4,2], chainFunctionIDs=[1,2,3,3], args={}) chain5 = node1_c5 ###################################################################################################################### # NOTE: real applications ## ObjectDetection using openCV # L2 # / # L1 # \ # L3 objectDetection1 = LambdaFunction(url="image-preprocessing.com", id=1) objectDetection2 = LambdaFunction(url="opencv_maskrcnn.com", id=2) objectDetection3 = LambdaFunction(url="opencv_yolo.com", id=3) node3_c6 = ChainNode(function=objectDetection3, nodeID=3, children=[], lastNodeIDs=[2,3], chainFunctionIDs=[1,2,3], args={}) node2_c6 = ChainNode(function=objectDetection2, nodeID=2, children=[], lastNodeIDs=[2,3], chainFunctionIDs=[1,2,3], args={}) node1_c6 = ChainNode(function=objectDetection1, nodeID=1, children=[node2_c6, node3_c6], lastNodeIDs=[2,3], chainFunctionIDs=[1,2,3], args={}) chain6 = node1_c6 concurrencies = [ {"chain": chain0, "count": 0}, {"chain": chain1, "count": 1}, {"chain": chain2, "count": 1}, {"chain": chain3, "count": 0}, {"chain": chain4, "count": 0}, {"chain": chain5, "count": 1}, {"chain": chain6, "count": 0}, ] chain_fan2 = Workload("Fan2", 5, 20, chain1, 1, 1000) chain_linear = Workload("Linear", 5, 20, chain2, 2, 1000) chain_fan_linear = Workload("FanAndLinear", 5, 20, chain5, 5, 1000) # Set up Kafka # kafka_url = "localhost:9092" # kafka_pq_topic = "pending_queue" # producer_PQ = connect_kafka_producer(kafka_url) # # Producer Side Logging # kafka_psq_topic = "producer_side_logging_queue" # producer_PSQ = connect_kafka_producer(kafka_url) # Functions are invoked here chain_fan2.startWorkload(producer_PQ, producer_PSQ) chain_linear.startWorkload(producer_PQ, producer_PSQ) chain_fan_linear.startWorkload(producer_PQ, producer_PSQ) producer_PQ.close() producer_PSQ.close()
def main(file_path): workload = Workload(file_path) for t, workload in workload.next(): time.sleep(1) # sleep for 1 sec periodic_curl(workload)
from mongo import Mongo from workload import Workload from query import Query import pandas as pd from table import Table if __name__ == "__main__": mongo = Mongo() db = mongo.getDb() list_of_collections = db.list_collection_names() workload = Workload() pipeline = workload.getWorkload() data = [] for i in pipeline: dct = {} for j in list_of_collections: q = Query(db, j, i) dct[j] = q.getQueryExecTime() data.append(dct) df = pd.DataFrame(data) df_total = pd.DataFrame([dict(df.sum())]) list_of_queries= [] for i in range(len(pipeline)): list_of_queries.append('Query '+str(i+1)) df['Query'] = list_of_queries df.set_index('Query', inplace=True)
def test_compare_workloads(): workloadA_1 = Workload("Small_Workload", "", "https://tfhub.dev/google/imagenet/inception_v3/feature_vector/1", 200, 0.01, 10, 10, 10, 100, -1, 100, False, 0, 0, 0, "") workloadA_2 = Workload("Small_Workload", "", "https://tfhub.dev/google/imagenet/inception_v3/feature_vector/1", 200, 0.01, 10, 10, 10, 100, -1, 100, False, 0, 0, 0, "") workloadB = Workload("test_workload(I_v3)", "comment", "https://tfhub.dev/google/imagenet/inception_v3/feature_vector/1", 4000, 0.02, 30, 10, 110, 100, -3, 100, True, 0, 0, 0, "") assert compare_workloads(workloadA_1, workloadA_2) assert not compare_workloads(workloadA_1, workloadB)
def getAllStages(execStats): s = set() if type(execStats) == type({}): for i in execStats: if i == 'stage': s.add(execStats[i]) else: s.update(getAllStages(execStats[i])) elif type(execStats) == type([]): for i in execStats: s.update(getAllStages(i)) return s if __name__ == "__main__": p = Workload() mongo = Mongo() db = mongo.getDb() list_of_collections = db.list_collection_names() list_of_collections.sort() pipeline = p.getWorkload() f = open("list_operations.md", "w") for i in list_of_collections: f.write('## ') f.write(i) f.write('\n') f.write('\n') f.write('| Query | Operations |') f.write('\n') f.write('|---|---|') f.write('\n')