Example #1
0
    def test5_bReader(self):
        reader = binaryReader("{}/trace.vscsi".format(DAT_FOLDER),
                              init_params={"label":6, "real_time":7, "fmt": "<3I2H2Q"})

        cH = cHeatmap()
        bpr = cH.get_breakpoints(reader, 'r', time_interval=1000000)
        self.assertEqual(bpr[10], 53)
        bpv = cH.get_breakpoints(reader, 'v', time_interval=1000)
        self.assertEqual(bpv[10], 10000)

        cH.heatmap(reader, 'r', "hit_ratio_start_time_end_time",
                   num_of_pixels=100, num_of_threads=4, cache_size=2000)
        cH.heatmap(reader, 'r', "hit_ratio_start_time_end_time",
                   num_of_threads=4, cache_size=2000)
        cH.heatmap(reader, 'r', "rd_distribution",
                   num_of_pixels=1000, num_of_threads=4)
        cH.heatmap(reader, 'r', "future_rd_distribution",
                   num_of_pixels=1000, num_of_threads=4)
        cH.heatmap(reader, 'r', "hit_ratio_start_time_end_time",
                   num_of_pixels=100, algorithm="FIFO",
                   num_of_threads=4, cache_size=200)

        cH.diffHeatmap(reader, 'r', "hit_ratio_start_time_end_time",
                       num_of_pixels=24, cache_size=200,
                       algorithm1="LRU", algorithm2="Optimal",
                       cache_params2=None, num_of_threads=4)
Example #2
0
    def plotHRCs(self, op=None, clf=True, save=True, figname="HRC_multi_trace.pdf", **kwargs):
        if op is None:
            if len(self.hrs) == 0:
                self.get_hit_rate()
            for i in range(len(self.hrs)):
                label = self.readers[i].file_loc[self.readers[i].file_loc.rfind('/')+1:]
                plt.plot(np.arange(0, self.bin_size*(len(self.hrs[i])-2), self.bin_size),
                         self.hrs[i][:-2], label=label)

        elif op == "+" or op == "add" or op == "addition":
            assert 'mix_mode' in kwargs, "please provide mix_mode when doing trace mixing/addition"
            # the following only works on Linux
            mixed_trace = tempfile.NamedTemporaryFile()
            traceMixer.mix_asBinary(readers=self.readers, mix_mode="real_time", output=mixed_trace.name)
            mixed_reader = binaryReader(mixed_trace.name, init_params={"label": 1, "fmt": "<L"})
            if self.cache_name == "LRU":
                profiler = LRUProfiler(mixed_reader, cache_size=self.cache_size)
            else:
                profiler = cGeneralProfiler(mixed_reader, cache_name=self.cache_name,
                                            cache_size=self.cache_size, bin_size=self.bin_size,
                                            cache_params=self.cache_params, num_of_threads=self.num_of_threads)
            hr = profiler.get_hit_rate()
            plt.plot(np.arange(0, self.bin_size * (len(hr) - 2), self.bin_size),
                     hr[:-2], label="mixed")

        if save:
            plt.legend(loc="best")
            plt.xlabel("Cache size/items")
            plt.ylabel("Hit Ratio")
            plt.savefig(figname)
            if clf:
                plt.clf()
Example #3
0
 def test_reader_binary(self):
     reader = binaryReader("{}/trace.vscsi".format(DAT_FOLDER),
                           data_type='l',
                           init_params={
                               "label": 6,
                               "real_time": 7,
                               "fmt": "<3I2H2Q"
                           })
     lines = 0
     for _ in reader:
         lines += 1
     self.assertEqual(lines, 113872)
     reader.reset()
     first_request = reader.read_one_element()
     print(first_request)
     self.assertEqual(int(first_request), 42932745)
Example #4
0
    def get_hit_ratio_shards(self, sample_ratio=0.01, **kwargs):
        """
        experimental function
        :param sample_ratio:
        :param kwargs:
        :return:
        """
        from mimircache.cacheReader.tracePreprocesser import tracePreprocessor
        kargs = {}
        if 'cache_size' not in kwargs:
            kargs['cache_size'] = self.cache_size
        else:
            kargs['cache_size'] = kwargs['cache_size']

        pp = tracePreprocessor(self.reader)
        N1, N2, traceName, fmt = pp.prepare_for_shards(
            sample_ratio=sample_ratio, has_time=False)
        correction = N2 - N1
        print("correction: {}".format(correction))
        # correction = 0
        tempReader = binaryReader(traceName,
                                  init_params={
                                      "label": 1,
                                      "fmt": fmt
                                  })

        if self.block_unit_size != 0:
            print("not supported yet")
            return None
        else:
            hit_ratio = mimircache.c_LRUProfiler.get_hit_ratio_seq_shards(
                tempReader.cReader,
                sample_ratio=sample_ratio,
                correction=correction,
                **kargs)
        return hit_ratio
Example #5
0
    maxLine = range[1]
    if maxLine == -1:
        maxLine = len(reader)

    line_count = 0
    # reader.jumpN(range[0])
    for line in reader:
        line_count += 1
        if line_count - 1 < range[0]:  # -1 because we add 1 before checking
            continue
        if line_count - 1 >= maxLine:
            break
        writer.write(line)


if __name__ == "__main__":
    # vReaderToPReader(sys.argv[1], sys.argv[2])
    # reader = vscsiReader("../data/traces/w38_vscsi1.vscsitrace")
    # reader1 = vscsiReader("../data/trace.vscsi")
    # reader1 = vscsiReader("../data/traces/w106_vscsi1.vscsitrace")
    # reader2 = vscsiReader("../data/traces/w100_vscsi1.vscsitrace")
    # trace_mixer(reader1, reader2, mix_mode="round_robin", round_robin_n=2)
    # trace_mixer(reader1, reader2, mix_mode="real_time")
    # splitTrace(reader, 2, "w38Split")
    # splitTraceWithTime("/var/dept/scratch/jyan254/cache/mimircache/data/traces/w65_vscsi1.vscsitrace")
    with binaryReader("../data/trace.vscsi", "<3I2H2Q") as reader:
        with traceBinaryWriter("test.vscsi", "<3I2H2Q") as writer:
            extract_part_trace(reader, writer, (1, -1))
    with binaryReader("../utils/test.vscsi", "<3I2H2Q") as r:
        print(len(r))
Example #6
0
    def test_Optimal(self):
        reader = vscsiReader("{}/trace.vscsi".format(DAT_FOLDER))
        p = cGeneralProfiler(reader, "Optimal", cache_size=2000)
        hr = p.get_hit_ratio()
        self.assertAlmostEqual(hr[0], 0.0)
        self.assertAlmostEqual(hr[100], 0.28106996417045593)
        hc = p.get_hit_count()
        self.assertEqual(hc[10], 180)
        self.assertEqual(hc[0], 0)
        mr = p.get_miss_rate()
        self.assertAlmostEqual(mr[-1], 0.71893000602722168)
        hr = p.get_hit_ratio(begin=113852, end=113872, cache_size=5000)
        self.assertAlmostEqual(hr[1], 0.2)
        reader.close()

        reader = plainReader("{}/trace.txt".format(DAT_FOLDER))
        p = cGeneralProfiler(reader,
                             "Optimal",
                             cache_size=2000,
                             num_of_threads=4)
        hr = p.get_hit_ratio()
        self.assertAlmostEqual(hr[0], 0.0)
        self.assertAlmostEqual(hr[100], 0.28106996417045593)
        hc = p.get_hit_count()
        self.assertEqual(hc[10], 180)
        self.assertEqual(hc[0], 0)
        mr = p.get_miss_rate()
        self.assertAlmostEqual(mr[-1], 0.71893000602722168)
        p.plotHRC("test2.png", cache_unit_size=32 * 1024)
        reader.close()

        reader = csvReader("{}/trace.csv".format(DAT_FOLDER),
                           init_params={
                               "header": True,
                               'label': 5,
                               'delimiter': ','
                           })
        p = cGeneralProfiler(reader,
                             "Optimal",
                             cache_size=2000,
                             num_of_threads=4)
        hr = p.get_hit_ratio()
        self.assertAlmostEqual(hr[0], 0.0)
        self.assertAlmostEqual(hr[100], 0.28106996417045593)
        hc = p.get_hit_count()
        self.assertEqual(hc[10], 180)
        self.assertEqual(hc[0], 0)
        mr = p.get_miss_rate()
        self.assertAlmostEqual(mr[-1], 0.71893000602722168)
        reader.close()

        reader = binaryReader("{}/trace.vscsi".format(DAT_FOLDER),
                              init_params={
                                  "label": 6,
                                  "real_time": 7,
                                  "fmt": "<3I2H2Q"
                              })
        p = cGeneralProfiler(reader,
                             "Optimal",
                             cache_size=2000,
                             num_of_threads=4)
        hr = p.get_hit_ratio()
        self.assertAlmostEqual(hr[0], 0.0)
        self.assertAlmostEqual(hr[100], 0.28106996417045593)
        hc = p.get_hit_count()
        self.assertEqual(hc[10], 180)
        self.assertEqual(hc[0], 0)
        mr = p.get_miss_rate()
        self.assertAlmostEqual(mr[-1], 0.71893000602722168)
        reader.close()