コード例 #1
0
def SstLevelInfo():
    # Set Conf.ExpStartTime(), if not already set.
    if Conf.ExpStartTime() is None:
        MutantLogReader.Get()

    fn = "%s/sst-info-by-time-by-levels-level-seps-%s" % (Conf.dn_result,
                                                          Conf.ExpStartTime())
    if os.path.isfile(fn):
        return fn

    sst_y_cord_level_sep_highs = SstYCord.LevelSepHigh()

    with Cons.MT(
            "Generating Sst info by time by levels: level separators data file ..."
    ):
        with open(fn, "w") as fo:
            fmt = "%1d %10d %10s"
            fo.write("%s\n" % Util.BuildHeader(
                fmt, "level level_mid_for_labels level_low_for_separators"))
            lh_prev = 0
            for l, lh in sorted(sst_y_cord_level_sep_highs.iteritems()):
                lm = (lh + lh_prev) / 2
                fo.write((fmt + "\n") % (l, lm, lh_prev))
                lh_prev = lh
        Cons.P("Created %s %d" % (fn, os.path.getsize(fn)))
    return fn
コード例 #2
0
def _ReadStoredLog():
	if Conf.ExpStartTime() is None:
		return None

	dn = "%s/work/mutant/misc/logs/cassandra" % os.path.expanduser("~")
	fn = "%s/system-%s" % (dn, Conf.ExpStartTime())
	if not os.path.isfile(fn):
		# If there is a 7z file, uncompress it
		fn_7z = "%s.7z" % fn
		if os.path.isfile(fn_7z):
			with Cons.MT("Found a 7z file. Uncompressing"):
				Util.RunSubp("7z e -o%s %s" % (dn, fn_7z))
		else:
			return None

	with Cons.MT("Reading the stored Cassandra Mutant log file %s" % fn, print_time=False):
		lines = []
		with open(fn) as fo:
			for line in fo.readlines():
				lines.append(line.strip())
				# Stop after reading n lines for testing
				if 0 < Conf.MaxCassLogLines():
					if Conf.MaxCassLogLines() < len(lines):
						break
		#Cons.P(len(lines))

		return lines
コード例 #3
0
def SstInfo():
    # Set Conf.ExpStartTime(), if not already set.
    if Conf.ExpStartTime() is None:
        MutantLogReader.Get()

    fn = "%s/sst-info-by-time-by-levels-%s" % (Conf.dn_result,
                                               Conf.ExpStartTime())
    if os.path.isfile(fn):
        return fn

    (sst_lives, memt_lives) = MemtSstLife.Get()

    with Cons.MT("Generating Sst info by time by levels data file ..."):
        #with open(fn_m, "w") as fo:
        #	fo.write("%s\n" % Memt.Header())
        #	for addr, l in sorted(_memt_lives.iteritems()):
        #		fo.write("%s\n" % l)
        #Cons.P("Created %s %d" % (fn_m, os.path.getsize(fn_m)))

        with open(fn, "w") as fo:
            fo.write("%s\n" % MemtSstLife.SstLife.Header())
            for sst_gen, l in sorted(sst_lives.iteritems()):
                fo.write("%s\n" % l)
        Cons.P("Created %s %d" % (fn, os.path.getsize(fn)))
    return fn
コード例 #4
0
def SstHeatAtLastTime():
    # Set Conf.ExpStartTime(), if not already set.
    if Conf.ExpStartTime() is None:
        MutantLogReader.Get()

    fn_hlt = "%s/sst-heat-last-time-%s" % (Conf.dn_result, Conf.ExpStartTime())
    if os.path.isfile(fn_hlt):
        return fn_hlt

    sst_lives = MemtSstLife.GetSstLives()

    with Cons.MT("Generating Sst heats at the last time ..."):
        # Gather heat info at n different times
        num_times = Conf.heatmap_by_time_num_times

        if Conf.ExpFinishTime() is None:
            MemtSstLife.SetExpEndTimeFromSstLives()

        min_sst_opened = None
        for sst_gen, sl in sorted(sst_lives.iteritems()):
            min_sst_opened = sl.Opened() if min_sst_opened is None else min(
                min_sst_opened, sl.Opened())

        # Start time is when the first Sstable is opened, not the experiment start
        # time, when no SSTable exists yet.
        #   Exp start time:          160927-143257.395
        #   First Sstable open time: 160927-143411.273
        st = datetime.datetime.strptime(min_sst_opened, "%y%m%d-%H%M%S.%f")
        et = datetime.datetime.strptime(Conf.ExpFinishTime(),
                                        "%y%m%d-%H%M%S.%f")
        dur = (et - st).total_seconds()

        sstgen_heat = []
        t = st + datetime.timedelta(seconds=(float(dur) *
                                             (num_times - 1) / num_times +
                                             time_offset_in_sec))
        for sst_gen, sl in sorted(sst_lives.iteritems()):
            h = sl.HeatAtTime(t)
            if h is None:
                continue
            sstgen_heat.append((sst_gen, h))

        sstgen_heat.sort(key=lambda sh: sh[1], reverse=True)

        # Note: Don't bother with the width proportional to the tablet size for now

        fmt = "%4d %1d %8.3f"
        with open(fn_hlt, "w") as fo:
            # y0 is smaller than y1 (y0 is placed higher in the plot than y1).
            fo.write("%s\n" % Util.BuildHeader(fmt, "sst_gen level heat"))

            for sh in sstgen_heat:
                sst_gen = sh[0]
                heat = sh[1]
                fo.write(
                    (fmt + "\n") % (sst_gen, sst_lives[sst_gen].level, heat))
        Cons.P("Created %s %d" % (fn_hlt, os.path.getsize(fn_hlt)))
    return fn_hlt
コード例 #5
0
def LoadSstLivesFromPlotDataFiles():
    global _sst_lives
    if _sst_lives is not None:
        return _sst_lives

    fn_sst_info_by_time = "%s/sst-info-by-time-by-levels-%s" % (
        Conf.dn_result, Conf.ExpStartTime())
    if not os.path.isfile(fn_sst_info_by_time):
        return None

    fn_sst_heat_by_time = "%s/sst-heat-by-time-by-levels-%s" % (
        Conf.dn_result, Conf.ExpStartTime())
    if not os.path.isfile(fn_sst_heat_by_time):
        return None

    # This is not needed for the by-levels-with-heat plot
    #	"%s/sst-info-by-time-by-levels-%s" % (Conf.dn_result, Conf.ExpStartTime())

    with Cons.MT("Loading Sst lives from (%s, %s) ..." %
                 (fn_sst_info_by_time, fn_sst_heat_by_time)):
        _sst_lives = {}
        with open(fn_sst_info_by_time) as fo:
            for line in fo.readlines():
                if len(line) == 0:
                    continue
                if line[0] == "#":
                    continue
                line = line.strip()
                #Cons.P(line)
                t = re.split(r" +", line)
                sst_gen = int(t[0])
                sl = SstLife(sst_gen)
                sl.SetInfoFromPlotData(t)
                _sst_lives[sst_gen] = sl

        with open(fn_sst_heat_by_time) as fo:
            sstgen_lines_tokens = {}

            for line in fo.readlines():
                if len(line) == 0:
                    continue
                if line[0] == "#":
                    continue
                line = line.strip()
                if len(line) == 0:
                    continue
                t = re.split(r" +", line)
                if len(t) == 0:
                    continue
                sst_gen = int(t[0])
                if sst_gen not in sstgen_lines_tokens:
                    sstgen_lines_tokens[sst_gen] = []
                sstgen_lines_tokens[sst_gen].append(t)

            for sstgen, lt in sstgen_lines_tokens.iteritems():
                _sst_lives[sstgen].SetHeatFromPlotData(lt)
        return _sst_lives
コード例 #6
0
def PlotSstAccDistAtSpecificTimes():
    # At the time m sec after the n-th SSTable is created (time t).  To get the
    # max_plot_hgieht, all plot data files need to be generated before plotting
    # the first one.
    plot_data_fns_at_n = {}
    with Cons.MT(
            "Generating plot data for SSTables by levels with heat at specific times ..."
    ):
        for (n, m) in Conf.times_sst_by_levels_with_heat:
            (fn_in_boxes,
             fn_in_level_seps) = SstByLevelsWithHeatAtSpecificTimes.Boxes(
                 n, m)
            plot_data_fns_at_n[n] = (fn_in_boxes, fn_in_level_seps)

    with Cons.MT(
            "Plotting SSTables by levels with heat at specific times ..."):
        dn = "%s/sst-by-level-by-ks-range-with-heat" % Conf.dn_result

        for n, (fn_in_boxes,
                fn_in_level_seps) in sorted(plot_data_fns_at_n.iteritems()):
            env = os.environ.copy()
            env["FN_IN_BOXES"] = fn_in_boxes
            env["FN_IN_LEVEL_INFO"] = fn_in_level_seps
            env["MAX_PLOT_HEIGHT"] = str(
                SstByLevelsWithHeatAtSpecificTimes.max_plot_height)
            fn_out = "%s/sst-by-level-by-ks-range-with-heat-%s-%s.pdf" % (
                dn, Conf.ExpStartTime(), n)
            env["FN_OUT"] = fn_out

            Util.RunSubp(
                "gnuplot %s/sst-by-level-by-ks-range-with-heat-at-specific-time.gnuplot"
                % os.path.dirname(__file__),
                env=env)
            Cons.P("Created %s %d" % (fn_out, os.path.getsize(fn_out)))
コード例 #7
0
def _ReadAndCacheCassLog():
	with Cons.MT("Reading Cassandra log ..."):
		lines = []
		found_reset = False
		lines1 = []

		# WARN  [main] 2016-09-20 02:20:39,250 MemSsTableAccessMon.java:115 - Mutant: ResetMon
		pattern = re.compile(r"WARN  \[(main|MigrationStage:\d+)\]" \
				" (?P<datetime>\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d,\d\d\d)" \
				" MemSsTableAccessMon.java:\d+ -" \
				" Mutant: (?P<event>ResetMon)")

		# Note: s0 only for now.
		dn = "%s/work/mutant/log/%s/s0/cassandra" % (os.path.expanduser("~"), Util0.JobId())
		fn = "%s/system.log" % dn
		Cons.P("fn=%s" % fn)
		with open(fn) as fo:
			for line in fo.readlines():
				line = line.strip()
				mo = pattern.match(line)
				if mo is not None:
					found_reset = True
					Conf.SetExpStartTime(Util0.ShortDateTime(mo.group("datetime")))
					del lines[:]
				lines.append(line)

		# Keep reading zipped files like system.log.1.zip, until ResetMon is found
		i = 1
		while found_reset == False:
			fn = "%s/system.log.%d.zip" % (dn, i)
			Cons.P("ResetMon not found. Reading more from file %s ..." % fn)
			with zipfile.ZipFile(fn, "r") as z:
				for fn1 in z.namelist():
					#Cons.P(fn1)
					for line in z.read(fn1).split("\n"):
						line = line.strip()
						#Cons.P(line)
						mo = pattern.match(line)
						if mo is not None:
							found_reset = True
							Conf.SetExpStartTime(Util0.ShortDateTime(mo.group("datetime")))
							del lines1[:]
						lines1.append(line)
			if len(lines1) != 0:
				lines1.extend(lines)
				lines = list(lines1)
				del lines1[:]
			i += 1

		fn = "%s/work/mutant/misc/logs/cassandra/system-%s" \
				% (os.path.expanduser("~"), Conf.ExpStartTime())
		with open(fn, "w") as fo:
			for line in lines:
				fo.write("%s\n" % line)
		Cons.P("Created a Cassandra log file %s %d" % (fn, os.path.getsize(fn)))

		return lines
コード例 #8
0
def PlotSstHeatAtLastTime():
  # X-axis: Sstables
  # Y-axis: Heat

  # Sst heatmap boxes by time
  env = os.environ.copy()
  env["FN_IN"] = SstHeatmapByTime.SstHeatAtLastTime()
  fn_out = "%s/sst-heat-at-last-time-%s.pdf" % (Conf.dn_result, Conf.ExpStartTime())
  env["FN_OUT"] = fn_out

  with Cons.MT("Plotting SSTable heatmap by time ..."):
    Util.RunSubp("gnuplot %s/sst-heat-at-last-time.gnuplot" % os.path.dirname(__file__), env=env)
    Cons.P("Created %s %d" % (fn_out, os.path.getsize(fn_out)))
コード例 #9
0
    def __init__(self):
        if Conf.ExpStartTime() is None:
            raise RuntimeError("Unexpected")
        dn = "%s/work/mutant/misc/logs/cassandra" % os.path.expanduser("~")
        self.fn_compaction_log = "%s/compaction-%s" % (dn, Conf.ExpStartTime())

        # If not exist and there is a 7z file, uncompress it
        if not os.path.isfile(self.fn_compaction_log):
            fn_7z = "%s.7z" % self.fn_compaction_log
            if os.path.isfile(fn_7z):
                with Cons.MT("Found a 7z file. Uncompressing"):
                    Util.RunSubp("7z e -o%s %s" % (dn, fn_7z))

        # If still not exist, Copy the current compaction log file
        if not os.path.isfile(self.fn_compaction_log):
            dn1 = "%s/work/mutant/log/%s/s0/cassandra" % (
                os.path.expanduser("~"), Util0.JobId())
            Util.RunSubp("cp %s/compaction.log %s" \
              % (dn1, self.fn_compaction_log))

        with Cons.MT("Reading compaction log %s" % self.fn_compaction_log,
                     print_time=False):
            self.Read()
コード例 #10
0
def PlotSstHeatmapByTime():
  # X-axis: time
  # Y-axis: total storage space (depending on the workload, it can grow as time
  # goes by)

  # Sst heatmap boxes by time
  env = os.environ.copy()
  (env["FN_IN_HEATMAP"], env["FN_IN_VERTICAL_LINES"], heat_max) = SstHeatmapByTime.Heatmap()
  env["HEAT_MAX"] = str(heat_max)
  fn_out = "%s/sst-heatmap-by-time-%s.pdf" % (Conf.dn_result, Conf.ExpStartTime())
  env["FN_OUT"] = fn_out

  with Cons.MT("Plotting SSTable heatmap by time ..."):
    Util.RunSubp("gnuplot %s/sst-heatmap-by-time.gnuplot" % os.path.dirname(__file__), env=env)
    Cons.P("Created %s %d" % (fn_out, os.path.getsize(fn_out)))
コード例 #11
0
def PlotTabletByTimeByLevelWithHeat():
  with Cons.MT("Plotting Memtable and SSTables by time by levels with heat ..."):
    env = os.environ.copy()

    # Memtables are not plotted for now
    #env["FN_IN_MEMT"] = GetMemtByTimeData()

    env["FN_IN_SST_INFO"] = SstInfoByTimeByLevel.SstInfo()
    env["FN_IN_SST_LEVEL_INFO"] = SstInfoByTimeByLevel.SstLevelInfo()
    env["FN_IN_SST_HEAT"] = SstInfoByTimeByLevel.SstHeatByTimeByLevel()

    fn_out = "%s/sst-by-time-by-levels-with-heat-%s.pdf" % (Conf.dn_result, Conf.ExpStartTime())
    env["FN_OUT"] = fn_out

    with Cons.MT("Plotting ..."):
      #Util.RunSubp("gnuplot %s/tablet-timeline.gnuplot" % os.path.dirname(__file__), env=env)
      Util.RunSubp("gnuplot %s/sst-by-time-by-levels-with-heat.gnuplot" % os.path.dirname(__file__), env=env)
      Cons.P("Created %s %d" % (fn_out, os.path.getsize(fn_out)))
コード例 #12
0
def SstHeatByTimeByLevel():
	# Set Conf.ExpStartTime(), if not already set.
	if Conf.ExpStartTime() is None:
		MutantLogReader.Get()

	fn = "%s/sst-heat-by-time-by-levels-%s" % (Conf.dn_result, Conf.ExpStartTime())
	if os.path.isfile(fn):
		return fn

	(sst_lives, memt_lives) = MemtSstLife.Get()

	with Cons.MT("Generating SSTable by time by levels with heat data ..."):
		with open(fn, "w") as fo:
			fmt = "%4d %1d %17s %17s %8.1f %13d %13d %8.3f %8d %6s"
			fo.write("%s\n" % Util.BuildHeader(fmt
				, "sst_gen level time0 time1 age_in_sec y0 y1 heat(num_reads_per_sec_per_mb) heat_color heat_color_hex"))

			num_heat_color_blocks_merged = 0
			num_heat_color_blocks = 0

			for sst_gen, sl in sorted(sst_lives.iteritems()):
				# Some SSTables don't have any access stats at all, thus no heat info
				if len(sl.heat_by_time) == 0:
					continue

				t0_begin = None
				t0_prev = None
				t1_prev = None
				heat_prev = None
				heat_color_prev = None

				for t0, v in sl.heat_by_time.items():
					t1 = v[0]
					heat = v[1]

					if t0_begin is None:
						t0_begin = t0

					v = heat / MemtSstLife.SstLife.max_heat
					heat_color = TempColor.Get(v)

					if heat_color == heat_color_prev:
						# No change in t0_prev, heat_color_prev
						t1_prev = t1
						heat_prev = heat
						num_heat_color_blocks_merged += 1
					else:
						if (t0_prev is not None) and (t1_prev is not None) \
								and (heat_prev is not None) \
								and (heat_color_prev is not None):
							fo.write((fmt + "\n") \
									% (sst_gen
										, sst_lives[sst_gen].level
										, t0_prev.strftime("%y%m%d-%H%M%S.%f")[:-3]
										, t1_prev.strftime("%y%m%d-%H%M%S.%f")[:-3]
										, (t1_prev - t0_begin).total_seconds()
										, sst_lives[sst_gen].y_cord_low, sst_lives[sst_gen].YcordHigh()
										, heat_prev
										, heat_color_prev
										, "%0.6X" % heat_color_prev
										))
							num_heat_color_blocks += 1
						t0_prev = t0
						t1_prev = t1
						heat_prev = heat
						heat_color_prev = heat_color

				# The last one
				if (t0_prev is not None) and (t1_prev is not None) \
						and (heat_prev is not None) \
						and (heat_color_prev is not None):
					fo.write((fmt + "\n") \
							% (sst_gen
								, sst_lives[sst_gen].level
								, t0_prev.strftime("%y%m%d-%H%M%S.%f")[:-3]
								, t1_prev.strftime("%y%m%d-%H%M%S.%f")[:-3]
								, (t1_prev - t0_begin).total_seconds()
								, sst_lives[sst_gen].y_cord_low, sst_lives[sst_gen].YcordHigh()
								, heat_prev
								, heat_color_prev
								, "%0.6X" % heat_color_prev
								))
					num_heat_color_blocks += 1

				fo.write("\n")
			Cons.P("num_heat_color_blocks_merged=%d" % num_heat_color_blocks_merged)
			Cons.P("num_heat_color_blocks       =%d" % num_heat_color_blocks)
		Cons.P("Created %s %d" % (fn, os.path.getsize(fn)))
	return fn
コード例 #13
0
def _ReadAndCacheCassLogUntilDtCrossesStNotTested():
	with Cons.MT("Reading Cassandra log ..."):
		# Note: s0 only for now.
		dn = "%s/work/mutant/log/%s/s0/cassandra" % (os.path.expanduser("~"), Util0.JobId())

		# Start from system.log and keep reading system.log.n.zip where n >= 1,
		# until you find two datetimes that cross the given exp start time
		st = Conf.ExpStartTime()
		ft = Conf.ExpFinishTime()
		dt_prev = None
		dt_crossed_st = False

		# WARN  [main] 2016-09-20 02:20:39,250 MemSsTableAccessMon.java:115 - Mutant: ...
		pattern = re.compile(r".+" \
				" (?P<datetime>\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d,\d\d\d)" \
				" .+")

		fn = "%s/system.log" % dn
		lines = []
		Cons.P("fn=%s" % fn)
		with open(fn) as fo:
			for line in fo.readlines():
				line = line.strip()
				mo = pattern.match(line)
				if mo is None:
					raise RuntimeError("Unexpected line=[%s]" % line)
				dt = Util0.ShortDateTime(mo.group("datetime"))
				if (st <= dt) and (dt <= ft):
					lines.append(line)
				if ft < dt:
					break
				if not dt_crossed_st:
					if (dt_prev is not None) and (dt_prev < st) and (st < dt):
						dt_crossed_st = True
				dt_prev = dt

		# Keep reading zipped files like system.log.1.zip, until ResetMon is found
		i = 1
		while dt_crossed_st == False:
			fn = "%s/system.log.%d.zip" % (dn, i)
			Cons.P("dt haven't crossed st yet. Reading more logs from file %s ..." % fn)
			lines1 = []
			with zipfile.ZipFile(fn, "r") as z:
				dt_prev = None
				for fn1 in z.namelist():
					#Cons.P(fn1)
					for line in z.read(fn1).split("\n"):
						line = line.strip()
						#Cons.P(line)
						mo = pattern.match(line)
						if mo is None:
							raise RuntimeError("Unexpected line=[%s]" % line)
						dt = Util0.ShortDateTime(mo.group("datetime"))
						if (st <= dt) and (dt <= ft):
							lines.append(line)
						if ft < dt:
							break
						if not dt_crossed_st:
							if (dt_prev is not None) and (dt_prev < st) and (st < dt):
								dt_crossed_st = True
						dt_prev = dt
						lines1.append(line)
			if len(lines1) != 0:
				lines1.extend(lines)
				lines = list(lines1)
				del lines1[:]
			i += 1

		fn = "%s/work/mutant/misc/logs/cassandra/system-%s" \
				% (os.path.expanduser("~"), Conf.ExpStartTime())
		with open(fn, "w") as fo:
			for line in lines:
				fo.write("%s\n" % line)
		Cons.P("Created a Cassandra log file %s %d" % (fn, os.path.getsize(fn)))

		return lines
コード例 #14
0
def Heatmap():
    # Set Conf.ExpStartTime(), if not already set.
    if Conf.ExpStartTime() is None:
        MutantLogReader.Get()

    fn_hm = "%s/sst-heatmap-by-time-%s" % (Conf.dn_result, Conf.ExpStartTime())
    fn_vl = "%s/sst-heatmap-by-time-vertical-lines-%s" % (Conf.dn_result,
                                                          Conf.ExpStartTime())
    if os.path.isfile(fn_hm) and os.path.isfile(fn_vl):
        return (fn_hm, fn_vl, _MaxHeatFromHeatmapByTimeData(fn_hm))

    sst_lives = MemtSstLife.GetSstLives()

    with Cons.MT("Generating Sst heatmap by time ..."):
        # Gather heat info at n different times
        num_times = Conf.heatmap_by_time_num_times

        if Conf.ExpFinishTime() is None:
            MemtSstLife.SetExpEndTimeFromSstLives()

        min_sst_opened = None
        for sst_gen, sl in sorted(sst_lives.iteritems()):
            min_sst_opened = sl.Opened() if min_sst_opened is None else min(
                min_sst_opened, sl.Opened())

        # Start time is when the first Sstable is opened, not the experiment start
        # time, when no SSTable exists yet.
        #   Exp start time:          160927-143257.395
        #   First Sstable open time: 160927-143411.273
        #st = datetime.datetime.strptime(Conf.ExpStartTime(), "%y%m%d-%H%M%S.%f")
        st = datetime.datetime.strptime(min_sst_opened, "%y%m%d-%H%M%S.%f")
        et = datetime.datetime.strptime(Conf.ExpFinishTime(),
                                        "%y%m%d-%H%M%S.%f")
        dur = (et - st).total_seconds()

        # { t0: {HeatBoxes} }
        time_heatboxes = {}
        vertical_lines = []
        for i in range(0, num_times):
            t0 = st + datetime.timedelta(seconds=(float(dur) * i / num_times +
                                                  time_offset_in_sec))
            t1 = st + datetime.timedelta(seconds=(float(dur) *
                                                  (i + 1) / num_times +
                                                  time_offset_in_sec))
            vertical_lines.append(t1)

            # Heat boxes are sorted by their heat and plotted with the heights
            # proportional to the size.
            boxes = []
            for sst_gen, sl in sorted(sst_lives.iteritems()):
                h = sl.HeatAtTime(t0)
                if h is None:
                    continue
                boxes.append(_Box(sl, t0, t1, h))
            boxes.sort(key=lambda b: b.heat, reverse=True)
            time_heatboxes[t0] = boxes

            Cons.ClearLine()
            Cons.Pnnl("%4d/%4d" % (i + 1, num_times))
        print ""
        del vertical_lines[-1]

        # Set y-coordinate of each box
        for t, boxes in sorted(time_heatboxes.iteritems()):
            total_size = 0
            for b in boxes:
                total_size += b.sl.Size()
            s = 0
            for b in boxes:
                b.y0 = float(s) / total_size
                s += b.sl.Size()
                b.y1 = float(s) / total_size

        # Make leftmost time to 0.
        t_first = None
        t_base = datetime.datetime(2000, 1, 1)
        for t, boxes in sorted(time_heatboxes.iteritems()):
            if t_first is None:
                t_first = t
            for b in boxes:
                b.t0 = t_base + (b.t0 - t_first)
                b.t1 = t_base + (b.t1 - t_first)
        for i in range(len(vertical_lines)):
            vertical_lines[i] = t_base + (vertical_lines[i] - t_first)

        fmt = "%4d %1d %17s %17s %6.4f %6.4f" \
          " %8.3f %8d %6s"
        with open(fn_hm, "w") as fo:
            fo.write("# heat_max=%f\n" % MemtSstLife.SstLife.max_heat)

            # y0 is smaller than y1 (y0 is placed higher in the plot than y1).
            fo.write("%s\n" % Util.BuildHeader(fmt, \
              "sst_gen level t0 t1 y0 y1" \
              " heat heat_color heat_color_hex"))

            for t, boxes in sorted(time_heatboxes.iteritems()):
                for b in boxes:
                    fo.write((fmt + "\n") % ( \
                      b.sl.sst_gen, b.sl.level
                      , b.t0.strftime("%y%m%d-%H%M%S.%f")[:-3], b.t1.strftime("%y%m%d-%H%M%S.%f")[:-3]
                      , b.y0, b.y1
                      , b.heat, b.heat_color, ("%0.6X" % b.heat_color)
                      ))
                fo.write("\n")
        Cons.P("Created %s %d" % (fn_hm, os.path.getsize(fn_hm)))

        with open(fn_vl, "w") as fo:
            for vl in vertical_lines:
                fo.write("%s\n" % vl.strftime("%y%m%d-%H%M%S.%f")[:-3])
        Cons.P("Created %s %d" % (fn_vl, os.path.getsize(fn_vl)))
    return (fn_hm, fn_vl, MemtSstLife.SstLife.max_heat)
コード例 #15
0
def Boxes(n, m):
    if Conf.ExpStartTime() is None:
        MutantLogReader.Get()

    dn = "%s/sst-by-level-by-ks-range-with-heat" % Conf.dn_result
    fn_boxes = "%s/%s-boxes-%d" % (dn, Conf.ExpStartTime(), n)
    fn_level_info = "%s/%s-level-info-%d" % (dn, Conf.ExpStartTime(), n)
    if os.path.isfile(fn_boxes) and os.path.isfile(fn_level_info):
        _UpdateMaxPlotHeight(fn_boxes)
        return (fn_boxes, fn_level_info)

    sst_lives = MemtSstLife.GetSstLives()

    with Cons.MT(
            "Generating SSTables by level by ks range with heat at specific time n=%d m=%s ..."
            % (n, m)):
        Util.MkDirs(dn)

        t = datetime.datetime.strptime(sst_lives[n].Opened(), "%y%m%d-%H%M%S.%f") \
          + datetime.timedelta(seconds=m)

        # Boxes representing SSTables
        boxes = []

        for sst_gen, sl in sorted(sst_lives.iteritems()):
            h = sl.HeatAtTime(t)
            if h is None:
                continue
            boxes.append(_Box(sl, h))

        # Sort boxes by level acendening, sst_gen decending. At the same level, the
        # younger SSTables (with bigger sst_gen number) tend to be hotter.
        boxes.sort(key=lambda b: b.sst_life.sst_gen, reverse=True)
        boxes.sort(key=lambda b: b.sst_life.level)

        # Box height: fixed or exponentially growing.
        fixed_box_height = True

        if fixed_box_height:
            box_height = 1
        else:
            box_height = 0.5
        level_labels_y = [box_height / 2.0]
        level_separators_y = []

        # Set y0 of each box
        cur_y0_max = 0
        prev_level = None
        for b in boxes:
            if fixed_box_height:
                box_height = 1
            else:
                box_height = 0.5 * math.pow(2, b.sst_life.level)

            if b.sst_life.level == 0:
                b.y0 = 0
                cur_y0_max = b.y0
            else:
                if b.sst_life.level == prev_level:
                    b.y0 = cur_y0_max
                else:
                    if fixed_box_height:
                        prev_box_height = 1
                    else:
                        prev_box_height = 0.5 * math.pow(2, prev_level)
                    b.y0 = cur_y0_max + prev_box_height + 2 * y_spacing
                    sep = cur_y0_max + prev_box_height + y_spacing
                    level_separators_y.append(sep)
                    level_labels_y.append(sep + y_spacing + box_height / 2.0)
                    cur_y0_max = b.y0
            b.y1 = b.y0 + box_height
            prev_level = b.sst_life.level
        global max_plot_height
        max_plot_height = cur_y0_max + box_height + y_spacing if max_plot_height is None \
          else max(max_plot_height, cur_y0_max + box_height + y_spacing)

        fn_boxes = "%s/%s-boxes-%d" % (dn, Conf.ExpStartTime(), n)
        with open(fn_boxes, "w") as fo:
            fmt = "%3d %1d %9d %6.2f %8d %6s %20d %20d %5.1f %5.1f"
            fo.write("%s\n" % Util.BuildHeader(fmt, "sst_gen level size heat(reads_per_sec_per_mb)" \
              " heat_color heat_color_hex" \
              " ks_first ks_last box_y0 box_y1"))
            for b in boxes:
                fo.write(
                    (fmt + "\n") % (b.sst_life.sst_gen, b.sst_life.level,
                                    b.sst_life.Size(), b.heat, b.color,
                                    "%0.6X" % b.color, b.sst_life.key_range[0],
                                    b.sst_life.key_range[1], b.y0, b.y1))
        Cons.P("Created %s %d" % (fn_boxes, os.path.getsize(fn_boxes)))

        with open(fn_level_info, "w") as fo:
            fmt = "%5s %5s"
            fo.write("%s\n" %
                     Util.BuildHeader(fmt, "level_label_y level_separator_y"))
            for i in range(max(len(level_labels_y), len(level_separators_y))):
                l = "-"
                if i < len(level_labels_y):
                    l = "%5.1f" % level_labels_y[i]
                s = "-"
                if i < len(level_separators_y):
                    s = "%5.1f" % level_separators_y[i]
                fo.write((fmt + "\n") % (l, s))
        Cons.P("Created %s %d" %
               (fn_level_info, os.path.getsize(fn_level_info)))

    return (fn_boxes, fn_level_info)