def main(argv): Conf.ParseArgs() Util.MkDirs(Conf.GetDir("output_dir")) dist_sq_thresholds = [0, 0.008, 0.01, 0.02, 0.05] #dist_sq_thresholds = [0.02] dist_sq_thresholds_str = [] for d in dist_sq_thresholds: dist_sq_thresholds_str.append(_NoTrailing0s(d)) reduced_files = [] reduced_file_sizes = [] with Cons.MT("Generating reduced size usa map ..."): for d in dist_sq_thresholds_str: fn_co_loc = "filter-out-almost-duplicate-points/.output/usa-map-smallsize-%s" % d reduced_files.append(fn_co_loc) if not os.path.exists(fn_co_loc): cmd = "cd filter-out-almost-duplicate-points && ./build-and-run.sh --dist_sq_threshold=%s" % d Util.RunSubp(cmd) reduced_file_sizes.append(os.path.getsize(fn_co_loc)) dn_out = "%s/.output" % os.path.dirname(__file__) fn_out = "%s/usa-map.pdf" % dn_out with Cons.MT("Plotting ..."): env = os.environ.copy() env["IN_FNS"] = " ".join(reduced_files) env["IN_FN_SIZES"] = " ".join(str(s) for s in reduced_file_sizes) env["DIST_SQ_THRESHOLDS"] = " ".join(dist_sq_thresholds_str) env["OUT_FN"] = fn_out Util.RunSubp("gnuplot %s/usa-map.gnuplot" % os.path.dirname(__file__), env=env) Cons.P("Created %s %d" % (fn_out, os.path.getsize(fn_out)))
def main(argv): Conf.ParseArgs() dn_out = Conf.Get("dn_result") Util.MkDirs(dn_out) SimTime.Init(Conf.Get("simulation_time_begin")) Plot.Plot()
def main(argv): Conf.ParseArgs() SimTime.Init(Conf.Get("simulation_time_begin")) Util.MkDirs(Conf.dn_result) Plot.Plot()
def main(argv): Conf.ParseArgs() # Get the simulation/simulated time begin/end by parsing a client log file # with the the same simulation time begin SimTime.Init() Plot.Plot()
def main(argv): Conf.ParseArgs() Util.MkDirs(Conf.dn_result) Plot()
def main(argv): Conf.ParseArgs() Util.MkDirs(Conf.GetDir("output_dir")) Plot.Plot()
def main(argv): Conf.ParseArgs() Util.MkDirs(Conf.GetDir("output_dir")) #dist_sq_thresholds = [ # 0 # , 0.008 # , 0.01 # , 0.02 # , 0.05] # Still quite big. 277 KB dist_sq_thresholds = [0.02] dist_sq_thresholds_str = [] for d in dist_sq_thresholds: dist_sq_thresholds_str.append(_NoTrailing0s(d)) fns_co_loc = [] co_loc_file_sizes = [] with Cons.MT("Generating reduced central office locations ..."): for d in dist_sq_thresholds_str: fn_co_loc = "filter-out-almost-duplicate-points/.output/centraloffices-wo-almost-dup-points-%s" % d fns_co_loc.append(fn_co_loc) if not os.path.exists(fn_co_loc): cmd = "cd filter-out-almost-duplicate-points && ./build-and-run.sh --dist_sq_threshold=%s" % d Util.RunSubp(cmd) co_loc_file_sizes.append(os.path.getsize(fn_co_loc)) dn_out = "%s/.output" % os.path.dirname(__file__) fn_out = "%s/central-office-locations.pdf" % dn_out with Cons.MT("Plotting ..."): env = os.environ.copy() env["IN_FNS"] = " ".join(fns_co_loc) env["IN_FN_SIZES"] = " ".join(str(s) for s in co_loc_file_sizes) env["DIST_SQ_THRESHOLDS"] = " ".join(dist_sq_thresholds_str) env["OUT_FN"] = fn_out Util.RunSubp("gnuplot %s/central-office-on-map.gnuplot" % os.path.dirname(__file__), env=env) Cons.P("Created %s %d" % (fn_out, os.path.getsize(fn_out))) sys.exit(0) if False: # Parallel processing params = [] for line in re.split(r"\s+", exps): t = line.split("/quizup/") if len(t) != 2: raise RuntimeError("Unexpected") job_id = t[0] exp_dt = t[1] params.append((job_id, exp_dt)) p = multiprocessing.Pool(8) p.map(Plot, params) else: for line in re.split(r"\s+", exps): t = line.split("/quizup/") if len(t) != 2: raise RuntimeError("Unexpected") job_id = t[0] exp_dt = t[1] Plot((job_id, exp_dt))
def main(argv): Conf.ParseArgs() SimTime.Init(Conf.Get("simulation_time_begin"))
def main(argv): Conf.ParseArgs() Util.MkDirs(Conf.GetDir("output_dir")) #job_id = "170829-181631" #exp_dt = "170829-221645.216" #job_id = "170829-192542" #exp_dt = "170830-014718.545" #exp_dt = "170830-042708.995" #exp_dt = "170830-054649.010" #exp_dt = "170830-153428.602" #exp_dt = "170830-203631.698" #job_id = "170830-163842" #exp_dt = "170830-212839.447" #job_id = "170830-202118" #exp_dt = "170831-005310.542" #exp_dt = "170831-011541.748" #job_id = "170831-113405" #exp_dt = "170831-153420.794" #job_id = "170831-110209" #exp_dt = "170831-154551.402" #job_id = "170831-111144" #exp_dt = "170831-155637.210" # Makes no sense #job_id = "170831-133109" #exp_dt = "170831-173125.214" #job_id = "170831-140421" #exp_dt = "170831-183201.386" # Resume from here #exp_dt = "170831-190015.298" #exp_dt = "170831-191848.134" #exp_dt = "170831-193818.890" #job_id = "170831-143603" #exp_dt = "170831-184202.667" #exp_dt = "170831-184334.331" #exp_dt = "170831-190944.219" #exp_dt = "170831-192627.550" # Done #job_id = "170831-155754" #exp_dt = "170831-195839.169" #exp_dt = "170831-200538.279" #exp_dt = "170831-200638.899" # Useless #job_id = "170831-124533" #exp_dt = "170831-230308.235" #job_id = "170831-192947" #exp_dt = "170831-234616.346" # Good one #exp_dt = "170831-235527.718" #exp_dt = "170831-235625.159" #job_id = "170902-170944" #exp_dt = "170902-214840.754" #job_id = "170904-094142" #exp_dt = "170904-142142.187" #job_id = "170904-185540" #exp_dt = "170904-225625.517" #job_id = "170904-220622" #exp_dt = "170905-020747.805" #job_id = "170904-221109" #exp_dt = "170905-025502.235" #job_id = "170904-221143" #exp_dt = "170905-024659.084" #job_id = "170904-224756" #exp_dt = "170905-024816.701" #job_id = "170905-010800" #exp_dt = "170905-050840.937" #job_id = "170905-011221" #exp_dt = "170905-051238.471" #job_id = "170905-011342" #exp_dt = "170905-053208.271" #job_id = "170905-011412" #exp_dt = "170905-055606.739" #job_id = "170905-104927" #exp_dt = "170905-144956.585" #job_id = "170905-105010" #exp_dt = "170905-145026.520" #job_id = "170905-105045" #exp_dt = "170905-145117.869" #job_id = "170905-105155" #exp_dt = "170905-145213.241" #job_id = "170905-124453" #exp_dt = "170905-165025.497" # Oscillation. Getting close. #job_id = "170905-125629" #exp_dt = "170905-165658.468" # System was too fast for 30 ms. Should go for lower. #job_id = "170905-125742" #exp_dt = "170905-165834.321" #job_id = "170905-125833" #exp_dt = "170905-165900.224" #job_id = "170905-174554" #exp_dt = "170905-214612.440" #job_id = "170905-174713" #exp_dt = "170905-214728.224" #job_id = "170905-174811" #exp_dt = "170905-214822.597" #job_id = "170905-174828" #exp_dt = "170905-214850.989" #job_id = "170905-204638" #exp_dt = "170906-004713.465" #job_id = "170905-205034" #exp_dt = "170906-005101.752" #job_id = "170905-205114" #exp_dt = "170906-005143.496" #job_id = "170905-205137" #exp_dt = "170906-005210.092" # Full range #job_id = "170906-010535" #exp_dt = "170906-050616.365" #job_id = "170906-010535" #exp_dt = "170906-050652.415" #job_id = "170906-010535" #exp_dt = "170906-050710.117" #job_id = "170906-010535" #exp_dt = "170906-050724.385" # Explore the load #job_id = "170906-161010" #exp_dt = "170906-205618.503" #job_id = "170906-161039" #exp_dt = "170906-205659.087" #job_id = "170906-161127" #exp_dt = "170906-205421.292" #job_id = "170906-161155" #exp_dt = "170906-205555.173" #job_id = "170906-161220" #exp_dt = "170906-205627.194" # See if you can observe the diurnal latency fluctuations without sla admin exps = """170907-012643/quizup/170907-060525.197 170907-012707/quizup/170907-060630.476 170907-012728/quizup/170907-060921.104 170907-012751/quizup/170907-060456.600 170907-012812/quizup/170907-060408.490 170907-012834/quizup/170907-060408.086 170907-012852/quizup/170907-060457.537 170907-012915/quizup/170907-061430.304 170907-012937/quizup/170907-060558.257 170907-012956/quizup/170907-060554.099 170907-013019/quizup/170907-061145.045 170907-013041/quizup/170907-060432.527""" exps = """170907-121318/quizup/170907-165713.370 170907-121403/quizup/170907-165712.779 170907-121440/quizup/170907-165438.591 170907-121522/quizup/170907-165443.768 170907-121609/quizup/170907-165218.496 170907-121653/quizup/170907-165922.539 170907-121740/quizup/170907-165900.958 170907-121822/quizup/170907-165822.323 170907-121904/quizup/170907-170209.779 170907-122924/quizup/170907-170908.771 170907-123114/quizup/170907-170943.678 170907-123205/quizup/170907-171222.564""" exps = """170907-123114/quizup/170907-170943.678""" exps = """170907-164049/quizup/170907-212518.984 170907-164110/quizup/170907-211905.715 170907-164130/quizup/170907-211822.147 170907-164153/quizup/170907-211909.870 170907-164222/quizup/170907-213001.374 170907-164246/quizup/170907-213336.369 170907-164304/quizup/170907-212518.303 170907-164326/quizup/170907-220020.817 170907-164348/quizup/170907-212049.945 170907-164410/quizup/170907-212045.491 170907-164429/quizup/170907-212755.838 170907-164452/quizup/170907-213236.709 170907-164514/quizup/170907-212552.729 170907-164536/quizup/170907-213455.280 170907-164557/quizup/170907-213013.051 170907-164619/quizup/170907-212735.136 170907-165226/quizup/170907-212208.496 170907-235300/quizup/170908-035329.045""" exps = """170907-235300/quizup/170908-035329.045""" exps = """170908-195136/quizup/170909-003325.826 170908-200335/quizup/170909-003326.821 170908-200450/quizup/170909-003330.257 170908-200648/quizup/170909-003333.434 170908-200711/quizup/170909-003334.323""" # Made a mistake. Oh man. #exps = """170909-133452/quizup/170909-181406.862 # 170909-234250/quizup/170910-034328.419 # 170909-234319/quizup/170910-034334.963 # 170909-234341/quizup/170910-034405.061 # 170909-234355/quizup/170910-034420.568""" exps = """170910-125635/quizup/170910-165648.445""" exps = """170910-130922/quizup/170910-175350.796 170910-130946/quizup/170910-175121.624 170910-131008/quizup/170910-175914.834 170910-215259/quizup/170911-023919.669 170910-215323/quizup/170911-023717.417 170910-215342/quizup/170911-024322.130 170910-215403/quizup/170911-024013.697""" exps = """170911-180901/quizup/170911-225243.953 170911-191601/quizup/170912-000330.251 170911-191706/quizup/170912-000344.161 170911-191727/quizup/170912-001211.393""" exps = """170912-090146/quizup/170912-134543.058 170912-094042/quizup/170912-142410.344""" exps = """170912-121125/quizup/170912-165921.714""" exps = """170912-090146/quizup/170912-134543.058""" exps = """170912-200314/quizup/170913-000846.906 170912-200725/quizup/170913-000923.763""" exps = """170912-235306/quizup/170913-035333.657 170912-235306/quizup/170913-055415.634 170912-235306/quizup/170913-075445.589 170912-235306/quizup/170913-095515.845 170912-235806/quizup/170913-035835.113 170912-235806/quizup/170913-055909.611 170912-235806/quizup/170913-075944.519 170912-235806/quizup/170913-100015.814 170912-235306/quizup/170913-115546.705 170912-235806/quizup/170913-120051.923""" # Figuring out st1 read IO range that has a good impact to latency. exps = """170914-000708/quizup/170914-040724.461 170914-000708/quizup/170914-040728.503 170914-000708/quizup/170914-060743.656 170914-000708/quizup/170914-060747.543 170914-000708/quizup/170914-080805.656 170914-000708/quizup/170914-080806.895 170914-000708/quizup/170914-100826.473 170914-000708/quizup/170914-100828.200 170914-000708/quizup/170914-120845.944 170914-000708/quizup/170914-120850.303""" #exps = """170914-000708/quizup/170914-080806.895""" exps = """170914-150736/quizup/170914-190749.869 170914-151351/quizup/170914-191411.718 170914-153055/quizup/170914-201130.757 170914-153118/quizup/170914-201130.361 170914-153137/quizup/170914-201250.023 170914-153158/quizup/170914-201306.037 170914-153221/quizup/170914-201648.453 170914-153245/quizup/170914-201252.468 170914-153305/quizup/170914-201050.453""" exps = """170914-231951/quizup/170915-032025.411 170914-232159/quizup/170915-032225.825 170914-232330/quizup/170915-040616.383 170914-232350/quizup/170915-040809.867 170914-232414/quizup/170915-040314.460 170914-232438/quizup/170915-035913.893 170914-232501/quizup/170915-040730.774 170914-232523/quizup/170915-040311.639 170914-232543/quizup/170915-035747.727""" exps = """170915-082816/quizup/170915-131440.110 170915-082837/quizup/170915-130901.347 170915-082858/quizup/170915-130841.880 170915-082920/quizup/170915-130954.426 170915-082943/quizup/170915-131048.454""" # RocksDB log was only for like 10 mins. No sla_admin there. exps = """170916-075803/quizup/170916-130854.144""" exps = """170916-092153/quizup/170916-132205.868 170916-092325/quizup/170916-132335.799 170916-092740/quizup/170916-140616.320 170916-092804/quizup/170916-141328.769 170916-092825/quizup/170916-140920.145 170916-092852/quizup/170916-141053.787 170916-092915/quizup/170916-141348.288 170916-092938/quizup/170916-140728.381""" exps = """170916-092153/quizup/170916-132205.868""" exps = """170916-220723/quizup/170917-020739.228 170916-220748/quizup/170917-020806.055 170916-220846/quizup/170917-025257.235 170916-220910/quizup/170917-025604.621 170916-220934/quizup/170917-025027.329 170916-220953/quizup/170917-025025.737 170916-221014/quizup/170917-025820.759 170916-221037/quizup/170917-024758.760 170917-084245/quizup/170917-132036.875 170917-084309/quizup/170917-132822.408""" exps = """170916-220723/quizup/170917-020739.228""" exps = """170917-130929/quizup/170917-174607.417 170917-122717/quizup/170917-162741.599 170917-123033/quizup/170917-163053.328 170917-130954/quizup/170917-174637.952 170917-131020/quizup/170917-174725.221 170917-131041/quizup/170917-174936.296 170917-131101/quizup/170917-175045.541 170917-131124/quizup/170917-175419.171 170917-131148/quizup/170917-175136.884""" # slow dev iops-based control. widh P and varying levels of D. exps = """170918-002609/quizup/170918-042726.771 170918-002740/quizup/170918-042801.188 170918-002849/quizup/170918-050530.376 170918-002912/quizup/170918-051309.315 170918-002934/quizup/170918-050504.193 170918-002956/quizup/170918-050530.365 170918-003019/quizup/170918-050834.786 170918-003039/quizup/170918-050825.371 170918-003101/quizup/170918-050226.142 170918-003125/quizup/170918-052241.852 170918-094956/quizup/170918-144341.674 170918-095022/quizup/170918-144018.607""" # Exploring xr_iops to see if we can get different latencies. exps = """170918-180857/quizup/170918-225647.066 170918-180920/quizup/170918-225506.963 170918-180944/quizup/170918-225646.945 170918-181005/quizup/170918-224938.751 170918-181026/quizup/170918-224942.079 170918-181047/quizup/170918-230536.228 170918-181111/quizup/170918-225031.376 170918-181138/quizup/170918-225625.807 170918-181200/quizup/170918-225711.266""" exps = """170918-222135/quizup/170919-025352.507 170918-222158/quizup/170919-030120.294 170918-222220/quizup/170919-030207.986 170918-222242/quizup/170919-030232.318 170918-222304/quizup/170919-030717.652 170918-222327/quizup/170919-030707.413""" exps = """170919-124331/quizup/170919-170007.870 170919-131403/quizup/170919-180125.967 170919-131424/quizup/170919-180158.349 170919-131442/quizup/170919-180139.970 170919-131505/quizup/170919-180424.920 170919-131524/quizup/170919-180125.980 170919-131545/quizup/170919-175120.370 170919-131604/quizup/170919-175703.244""" exps = """170919-180328/quizup/170919-223524.568 170919-180355/quizup/170919-224902.038 170919-180420/quizup/170919-224852.550 170919-180442/quizup/170919-224011.441 170919-180506/quizup/170919-225135.053 170919-180528/quizup/170919-225115.304 170919-180548/quizup/170919-224900.713 170919-180610/quizup/170919-224739.183 170919-180634/quizup/170919-225207.149 170919-180655/quizup/170919-225134.893 170919-180714/quizup/170919-230406.016""" # 100K, 150K, 200K. Try lower than 100K exps = """170927-094416/quizup/170927-134712.284""" if False: # Parallel processing params = [] for line in re.split(r"\s+", exps): t = line.split("/quizup/") if len(t) != 2: raise RuntimeError("Unexpected") job_id = t[0] exp_dt = t[1] params.append((job_id, exp_dt)) p = multiprocessing.Pool(8) p.map(Plot, params) else: for line in re.split(r"\s+", exps): t = line.split("/quizup/") if len(t) != 2: raise RuntimeError("Unexpected") job_id = t[0] exp_dt = t[1] Plot((job_id, exp_dt))
def main(argv): Conf.ParseArgs() SimTime.Init(Conf.Get("simulation_time_begin")) RocksDbLogReader.CalcStorageCost()