Esempio n. 1
0
def main():
    #
    # Initialize the serializer. At the moment sliced loading is only supported by the Binary
    # archive
    #
    serializer_write = ser.Serializer(ser.OpenModeKind.Write, "./slice",
                                      "field", "Binary")

    #
    # Allocate 3D numpy arrays
    #
    field_in = np.random.rand(512, 512, 80)
    field_out = np.zeros((512, 512, 80))

    #
    # Write the numpy array to disk at savepoint `sp`
    #
    start = time.time()

    savepoint = ser.Savepoint('sp')
    serializer_write.write('field', savepoint, field_in)

    print("Serializer.write      : %8.2f s" % (time.time() - start))

    #
    # Initialize a serializer for reading.
    #
    serializer_read = ser.Serializer(ser.OpenModeKind.Read, "./slice", "field",
                                     "Binary")

    #
    # Assume we are only interested in a certain layer of the data (k = 50), we can use the slice
    # object (ser.Slice) to encode this information and instruct the serializer to only load
    # the desired data. Note that you still need to allocate memory for the whole field!
    #
    start = time.time()

    serializer_read.read_slice('field', savepoint, ser.Slice[:, :, 50],
                               field_out)

    print("Serializer.read_slice : %8.2f s" % (time.time() - start))
    assert (np.allclose(field_in[:, :, 50], field_out[:, :, 50]))

    #
    # You can of course load the full data and slice it afterwards with numpy which yields the same
    # result, though is most likely slower.
    #
    start = time.time()

    serializer_read.read('field', savepoint, field_out)

    print("Serializer.read       : %8.2f s" % (time.time() - start))
    assert (np.allclose(field_in[:, :, 50], field_out[:, :, 50]))

    #
    # Remove directory
    #
    import shutil
    shutil.rmtree("./slice")
Esempio n. 2
0
    def serialize(self, intent, stage_id, stage_name, fields):
        sp = ser.Savepoint(self.name + "__" + intent)
        sp.metainfo.insert("stage_id", stage_id)
        sp.metainfo.insert("stage_name", stage_name)
        sp.metainfo.insert("invocation_count", self.invocation_count)

        for name, field in fields.items():
            self.serializer.write(name, sp, field)
Esempio n. 3
0
def read_data(path, tile, ser_count, is_in):

    mode_str = "in" if is_in else "out"
    vars = IN_VARS if is_in else OUT_VARS

    if is_in:

        serializer = ser.Serializer(ser.OpenModeKind.Read, path,
                                    "Generator_rank" + str(tile))
        savepoint = ser.Savepoint(f"cloud_mp-{mode_str}-{ser_count:0>6d}")

    else:

        serializer = ser.Serializer(ser.OpenModeKind.Read, path,
                                    "Serialized_rank" + str(tile))
        savepoint = ser.Savepoint(f"cloud_mp-{mode_str}-x-{ser_count:0>6d}")

    return data_dict_from_var_list(vars, serializer, savepoint)
def read_data(tile, is_in, path, ser_count=0):
    """
    Read serialbox2 format data under `./data` folder with prefix of `Generator_rank{tile}`
    :param tile: specify the number of tile in data
    :type tile: int
    :param is_in: true means in, false means out
    :type is_in: boolean
    """
    # TODO: read_async and readbuffer
    serializer = ser.Serializer(ser.OpenModeKind.Read, path,
                                "Generator_rank" + str(tile))
    inoutstr = "in" if is_in else "out"
    sp = ser.Savepoint(f"samfshalcnv-{inoutstr}-{ser_count:0>6d}")
    vars = IN_VARS if is_in else OUT_VARS
    data = data_dict_from_var_list(vars, serializer, sp)

    return data
def read_input_x_index(tile, ser_count, indices, path):

    serializer = ser.Serializer(ser.OpenModeKind.Read, path,
                                "Generator_rank" + str(tile))
    sp = ser.Savepoint(f"samfshalcnv-in-{ser_count:0>6d}")
    vars = set(IN_VARS) - set(scalar_vars)
    data = data_dict_from_var_list(vars, serializer, sp)

    for key in data:

        ndim = len(data[key].shape)
        arr = data[key]

        if ndim == 1 and key != "fscav":
            data[key] = arr[indices]
        elif ndim == 2:
            data[key] = arr[indices, :]
        elif ndim == 3:
            data[key] = arr[indices, :, :]

    return data
Esempio n. 6
0
def main():
    N = 512
    M = 512
    K = 80
    savepoint = ser.Savepoint('sp')

    #
    # First, we write some data to disk ...
    #
    serializer_write = ser.Serializer(ser.OpenModeKind.Write, "./async",
                                      "Field", "Binary")

    field_1 = np.random.rand(N, M, K)
    field_2 = np.random.rand(N, M, K)
    field_3 = np.random.rand(N, M, K)
    field_4 = np.random.rand(N, M, K)
    field_5 = np.random.rand(N, M, K)
    field_6 = np.random.rand(N, M, K)

    serializer_write.write('field_1', savepoint, field_1)
    serializer_write.write('field_2', savepoint, field_2)
    serializer_write.write('field_3', savepoint, field_3)
    serializer_write.write('field_4', savepoint, field_4)
    serializer_write.write('field_5', savepoint, field_5)
    serializer_write.write('field_6', savepoint, field_6)

    #
    # ... and read it again.
    #
    serializer_read = ser.Serializer(ser.OpenModeKind.Read, "./async", "Field",
                                     "Binary")

    start = time.time()

    field_1_rd = serializer_read.read('field_1', savepoint)
    field_2_rd = serializer_read.read('field_2', savepoint)
    field_3_rd = serializer_read.read('field_3', savepoint)
    field_4_rd = serializer_read.read('field_4', savepoint)
    field_5_rd = serializer_read.read('field_5', savepoint)
    field_6_rd = serializer_read.read('field_6', savepoint)

    print("Serializer.read       : %8.2f s" % (time.time() - start))

    #
    # Read operations are usually embarrassingly parallel and we can leverage this parallelism by
    # launching the operations asynchronously. If the archive is not thread-safe or if the library
    # was not configured with `SERIALBOX_ASYNC_API` the method falls back to synchronous execution.
    # To synchronize the tasks in the end, we can add a blocking Serializer.wait_for_all().
    #
    start = time.time()

    field_1_rd_async = serializer_read.read_async('field_1', savepoint)
    field_2_rd_async = serializer_read.read_async('field_2', savepoint)
    field_3_rd_async = serializer_read.read_async('field_3', savepoint)
    field_4_rd_async = serializer_read.read_async('field_4', savepoint)
    field_5_rd_async = serializer_read.read_async('field_5', savepoint)
    field_6_rd_async = serializer_read.read_async('field_6', savepoint)
    serializer_read.wait_for_all()

    print("Serializer.read_async : %8.2f s" % (time.time() - start))

    #
    # Finally, we verify the read operations actually do the same.
    #
    assert (np.allclose(field_1_rd, field_1_rd_async))
    assert (np.allclose(field_2_rd, field_2_rd_async))
    assert (np.allclose(field_3_rd, field_3_rd_async))
    assert (np.allclose(field_4_rd, field_4_rd_async))
    assert (np.allclose(field_5_rd, field_5_rd_async))
    assert (np.allclose(field_6_rd, field_6_rd_async))

    #
    # Remove directory
    #
    import shutil
    shutil.rmtree("./async")
def read_serialization_partx(var_list, part, tile = 0, path = DATAPATH):
    serializer = ser.Serializer(ser.OpenModeKind.Read, path, "Serialized_rank"+str(tile))
    sp         = ser.Savepoint(f"samfshalcnv-part{part}-input")
    data       = data_dict_from_var_list(var_list, serializer, sp)
    
    return numpy_dict_to_gt4py_dict(data)
Esempio n. 8
0
def write():

    #
    # Create a Serializer for writing. Besides the open-policy, we have to specify the `directory`
    # in which the Serializer is created and the `prefix` of all files. In case the directory does
    # not exist, it will be created. In addition, if the directory is not empty, all fields with the
    # same `prefix` will be erased (this behaviour can be inhibited using the Append mode).
    #
    serializer = ser.Serializer(ser.OpenModeKind.Write, "./laplacian/",
                                "field")

    #
    # Allocate a 2D numpy array and fill it with some random numbers
    #
    phi = np.random.rand(10, 10)

    #
    # Register the field within the Serializer. Note that for the Python Interface this step is not
    # strictly necessary as it can be done implicitly in the write method (see below).
    #
    fieldmetainfo = ser.FieldMetainfo(ser.TypeID.Float64, phi.shape)
    serializer.register_field('phi', fieldmetainfo)

    #
    # Add some global meta-information to the serializer. Besides the usual `key = value` pair,
    # you can also add `key = {value1, ..., valueN}` pairs.
    #
    serializer.global_metainfo.insert('answer', 42)
    serializer.global_metainfo.insert('halos', [1, 1, 1, 1])

    #
    # Up to this point nothing has been written to disk. Using update_meta_data() will force a write
    # of all meta-information to the corresponding JSON files. Note that the meta-data is updated
    # after each call and thus a manual update of the meta-data is seldom required. If you are
    # curious you can inspect the files './laplacian/MetaData-field.json' and
    # './laplacian/ArchiveMetaData-field.json'
    #
    serializer.update_meta_data()

    #
    # We now apply the `laplacian_stencil` three times to phi. In each iteration we will create an
    # input and output savepoint where we save the current `phi` field (input) and `lap` field
    # (output)
    #
    for t in range(3):
        #
        # Create a Savepoint. Savepoints can have the same name as long as they have different
        # meta-information. In our case we will always store the current time step `t` as a
        # meta-information, thus making it unique.
        #
        savepoint_in = ser.Savepoint('laplacian-in')
        savepoint_in.metainfo.insert('time', t)

        #
        # Register the Savepoint.
        #
        serializer.register_savepoint(savepoint_in)

        #
        # Write phi to disk at our input savepoint. This will create the file `field_phi.dat` upon
        # first invocation and afterwards the data is appended.
        #
        serializer.write('phi', savepoint_in, phi)

        #
        # Apply the laplacian_stencil to phi
        #
        lap = laplacian_stencil(phi)

        #
        # Create the output savepoint. This time we directly initialize the meta-information of the
        # savepoint.
        #
        savepoint_out = ser.Savepoint('laplacian-out', {'time': t})

        #
        # Write lap to disk. Note that here we implicitly register the field `lap` upon first
        # invocation. Same goes for the output savepoint.
        #
        serializer.write('lap', savepoint_out, lap)

        #
        # Finally, we swap phi with lap
        #
        phi = lap