def dialaSetup():
    """
    This routine sets up the data structures needed for the following unit tests. This includes a copy of a LAMMPS walker with the alanine dipeptide and a 2D particle walker.
    """
    import lammpsWalker
    wlkr = lammpsWalker.lammpsWalker("input.diala", "log.diala.test")
    ref_config = np.loadtxt("diala.structure").flatten()
    ref_vel = np.loadtxt("vel.ref").flatten()

    return wlkr, ref_config, ref_vel
Пример #2
0
    def setImages(self, filename):
        """
        Here we initialize an array of walker objects for the execution of the
        string object.
        """
        # start an array of walkers to store the string configuration.
        self.images = []

        for i in range(self.nimages):
            wlkr = lammpsWalker.lammpsWalker(filename, index=i)
            self.images.append(wlkr)

        return 0
def main():
    """
    definition of main scripting file for debugging purposes.
    """
    # time the execution
    starttime = time()
    
    # in the serial version, lets just set rank=0 
    rank = 0
    
    #---------------SET PARAMETERS -----------
    if rank == 0: print "Setting up umbrella sampling parameters."
    
    params = {}
    
    # set the scratch directory for the calculation files
    params['scratchdir'] = "/Users/jtempkin/enhanced_sampling_toolkit/neus/debug_US"
    params['inputFilename'] = "/Users/jtempkin/enhanced_sampling_toolkit/neus/input.diala"
    params['logFilename'] = params['scratchdir'] + "/log"
    
    # here we will set the umbrella sampling parameters in the params dict
    params['ncells'] = 144
    params['cellWidth'] = 15
    params['nwalkers'] = 1
    params['walkerSteps'] = 50000
    params['stepLength'] = 10
    params['Ftype'] = 'transition'
    
    # lets set the dynamics parameters that are needed to specify the walker 
    params['temperature'] = 310.0
    params['timestep'] = 1.0

    #--------------- INITIALIZATION--------
    # only allow root rank to build files
    if rank == 0: 
        print "Building scratch directory."
        # construct the wkdir. This is where the intermediate dynamcs files will 
        # be written. 
        fileIO.makeWkdir(params['scratchdir'])

    # construct the umbrella data structure
    if rank == 0: print "Initializing the umbrella structure."
    
    # create the partition object 
    system = partition.partition(params['ncells'])


    # now we construct the umbrella windows
    if rank == 0: print "Building the umbrellas."
    
    # specify the list of boundaries, nboxes, 1/2 width of the windows
    umbParams = {}
    
    # right now, we will hardcode a 12x12 array using Erik's routine for gridding a space
    umbParams['cvrange'] = np.array([map(float,entry.split(",")) for entry in "-180,180,12,15;-180,180,12,15".split(";")])
    umbParams['wrapping'] = np.array(map(float, "1,1".split(',')))
    
    system.umbrellas = fileIO.createUmbrellas(umbParams)
    
    # build neighbor list
    #system.buildNeighborList()
    
    
    
    #----------------INITIALIZE THE ENTRY POINTS FROM FILE------------
    # now we provide a data structure for entrypoints for each umbrella:
    for i in range(len(system.umbrellas)):
        system.umbrellas[i].entryPoints = []
    
    
    # now load each entry point file from the data base and load as an array of
    # ctypes pointers
    for i in range(len(system.umbrellas)):
        # load the numpy array
        data = np.load("entryPoints/in_" + str(i) + "_w0.entryPoints.npy")
        # now add each as a ctypes points to the entry points library
        for j in range(data.shape[0]):
            system.umbrellas[i].entryPoints.append(data[j].ctypes.data_as(ctypes.POINTER(ctypes.c_double)))
    
    
    """
    #------------ GENERATE INITIAL ENTRY POINTS --------------
    # sample umbrellas and construct F
    if rank == 0: print "Seeding entry points from a conventional simulation."
    
    # this 
    for i in range(len(system.umbrellas)):
        print i
        
        print "starting walker"
        # lets instantiate a walker object to sample this window. 
        wlkr = lammpsWalker.lammpsWalker(params['inputFilename'])
        
        print "minimize"
        # minimize structure prior to dynamics
        wlkr.minimize()
        
        
        # set the dynamics to sample by langevin
        wlkr.command("fix 1 all nve")
        wlkr.command("fix 2 all langevin 310.0 310.0 30.0 20874")
    
        print "setting colvars"
        # set colvars for this walker (currently, alanine dipeptide dihedrals)
        wlkr.colvars.append(['dihedral', 5, 7, 9, 15])
        wlkr.colvars.append(['dihedral', 7, 9, 15, 17]) 
        
        # set an array of starting/stoping restraints for equilibration
        restraint = [[0.0, 100.0], [0.0, 100.0]]
        
        # now specify colvars to dynamics routines
        wlkr.setColvars()
        
        # equilibrate the walker to the target point in CV space
        wlkr.equilibrate(system.umbrellas[i].center, restraint, 100000)
        
        # enter the sampling routine. This sampling routine will simply generate the initial 
        # entry point distribution 
        try: 
            system.sample(wlkr, params['walkerSteps'], i, 0, params, rank)
        except errors.DynamicsError:
            print "Rank", rank, "sampling error occured in umbrella", i, "."
            continue

        # now we are done populating the samples array, close the walker
        wlkr.close()
    
    # now we write out the entrypoints for each umbrella:
    for i in range(len(system.umbrellas)):
        np.save(params['scratchdir'] + "/in_" + str(i) + "_w0.entryPoints", system.umbrellas[i].entryPoints)
    
    """
    
    
    #----------------MAIN LOOP----------------
    if rank == 0: print "Sampling via NEUS." 
    # this 
    for i in range(len(system.umbrellas)):
        print "Rank", rank, "sampling umbrella", i, "."
        # lets instantiate a walker object to sample this window. 

        wlkr = lammpsWalker.lammpsWalker(params['inputFilename'], params['logFilename'], index=i)
        
        wlkr.setTimestep(params['timestep'])
        
        # set the dynamics to sample by langevin
        wlkr.command("fix 1 all nve")
        # the langevin fix here sets the temperature to 310K and the friction
        # coefficient to 30 ps-1
        wlkr.command("fix 2 all langevin " + " ".join([str(params['temperature']), str(params['temperature'])]) + " 30.0 20874")    
        
        # set colvars for this walker (currently, alanine dipeptide dihedrals)
        wlkr.colvars.append(['dihedral', 5, 7, 9, 15])
        wlkr.colvars.append(['dihedral', 7, 9, 15, 17]) 

        # now specify colvars to dynamics routines
        wlkr.setColvars()

        # now we initialize the starting coordinates from the entry points library
        temp_indx = random.randint(0, len(system.umbrellas[i].entryPoints)-1)
        print system.umbrellas[i].entryPoints[temp_indx], temp_indx
        print wlkr.lmp
        wlkr.setConfig(system.umbrellas[i].entryPoints[temp_indx])
        
        wlkr.command("run 0 post no")
        
        print "drawing velocities"
        # draw the velocities uniformly for now
        wlkr.drawVel(distType = 'gaussian', temperature = params['temperature'])
        
        # enter the sampling routine. This sampling routine will simply generate the initial 
        # entry point distribution 
        try: 
            system.sampleNeus(wlkr, params['walkerSteps'], i, 0, params, rank)
        except errors.DynamicsError:
            print "Rank", rank, "sampling error occurred in umbrella", i, "."
            continue
            
        # now we are done populating the samples array, close the walker
        wlkr.close()
        del wlkr
        #gc.collect()

    
    
    
   
    #----------------WRITE OUT DATA-------------
    # now allow rank 0 to process data. 
    if rank == 0:
        print system.F
        fileIO.writeMat(system.F, params['scratchdir'] + "/F.out")
        
        """
        print "Entering eigenvalue routine."
        # solve eigenvalue problem for F
        system.getZ()
        fileIO.writeMat(system.z, params['scratchdir'] + "/z.out")
    
        print "Computing the sensitivities."
        bounds = system.getlogBound(system.F)
        fileIO.writeMat(bounds, params['scratchdir'] + "/bounds.out")
        """

    # now we will perform an analysis of the data and increase sampling of 
    # windows with high variance

    if rank == 0:
        print "Done!"
        print "Total wallclock time was: " + str(time() - starttime) + " seconds."

    
    return 0
def main():
    """
    definition of main scripting file for debugging purposes.
    """
    # time the execution
    starttime = time()
    
    # in the serial version, lets just set rank=0 
    rank = 0
    
    #---------------SET PARAMETERS -----------
    if rank == 0: print "Setting up umbrella sampling parameters."
    
    params = {}
    
    # set the scratch directory for the calculation files
    params['scratchdir'] = "/Users/jtempkin/enhanced_sampling_toolkit/umbrella_sampling/debug_US"
    params['inputFilename'] = "/Users/jtempkin/enhanced_sampling_toolkit/umbrella_sampling/input.diala"
    
    # here we will set the umbrella sampling parameters in the params dict
    params['ncells'] = 144
    params['cellWidth'] = 60.0
    params['nwalkers'] = 1
    params['walkerSteps'] = 10000
    params['stepLength'] = 10
    params['Ftype'] = 'transition'
    
    # lets set the dynamics parameters that are needed to specify the walker 
    params['temperature'] = 310.0

    #--------------- INITIALIZATION--------
    # only allow root rank to build files
    if rank == 0: 
        print "Building scratch directory."
        # construct the wkdir. This is where the intermediate dynamcs files will 
        # be written. 
        fileIO.makeWkdir(params['scratchdir'])

    # construct the umbrella data structure
    if rank == 0: print "Initializing the umbrella structure."
    
    # create the partition object 
    system = partition.partition(params['ncells'])


    # now we construct the umbrella windows
    if rank == 0: print "Building the umbrellas."
    
    # specify the list of boundaries, nboxes, 1/2 width of the windows
    umbParams = {}
    
    # right now, we will hardcore a 12x12 array using Erik's routine for gridding a space
    umbParams['cvrange'] = np.array([map(float,entry.split(",")) for entry in "-180,180,12,30;-180,180,12,30".split(";")])
    umbParams['wrapping'] = np.array(map(float, "1,1".split(',')))
    
    system.umbrellas = fileIO.createUmbrellas(umbParams)
    
    #------------ MAIN LOOP --------------
    # sample umbrellas and construct F
    if rank == 0: print "Entering main loop."
    
    for i in range(len(system.umbrellas)):
        print "Rank", rank, "sampling umbrella", i, "."
        
        print "init walker"
        # lets instantiate a walker object to sample this window. 
        wlkr = lammpsWalker.lammpsWalker(params['inputFilename'])
        
        print "minimize"
        # minimize structure prior to dynamics
        wlkr.minimize()
        
        # set the dynamics to sample by langevin
        wlkr.command("fix 1 all nve")
        wlkr.command("fix 2 all langevin 310.0 310.0 30.0 20874")
    
        
        # set colvars for this walker (currently, alanine dipeptide dihedrals)
        wlkr.colvars.append(['dihedral', 5, 7, 9, 15])
        wlkr.colvars.append(['dihedral', 7, 9, 15, 17]) 
        
        # set an array of starting/stoping restraints for equilibration
        restraint = [[0.0, 100.0], [0.0, 100.0]]
        
        
        print "setting colvars"
        # now specify colvars to dynamics routines
        wlkr.setColvars()
        
        print "equilibrating"
        # equilibrate the walker to the target point in CV space
        wlkr.equilibrate(system.umbrellas[i].center, restraint, 100000)
        

        # enter the sampling routine 
        try: 
            system.sample(wlkr, params['walkerSteps'], i, 0, params, rank)
        except errors.DynamicsError:
            print "Rank", rank, "sampling error occured in umbrella", i, "."
            continue

        
        # now we are done populating the samples array, close the walker
        wlkr.close()
    

    #----------------WRITE OUT DATA-------------
    # now allow rank 0 to process data. 
    if rank == 0:
        print system.F
        fileIO.writeMat(system.F, params['wkdir'] + "/F.out")
    
        print "Entering eigenvalue routine."
        # solve eigenvalue problem for F
        system.getZ()
        fileIO.writeMat(system.z, params['wkdir'] + "/z.out")
    
        print "Computing the sensitivities."
        bounds = system.getlogBound(system.F)
        fileIO.writeMat(bounds, params['wkdir'] + "/bounds.out")


    # now we will perform an analysis of the data and increase sampling of 
    # windows with high variance

    if rank == 0:
        print "Done!"
        print "Total wallclock time was: " + str(time() - starttime) + " seconds."

    
    return 0
def main():
    """
    definition of main scripting file for debugging purposes.
    """
    # time the execution
    starttime = time()
    
    #--------------MPI INIT---------------------
    # init communicator
    comm = MPI.COMM_WORLD
    # get proc rank 
    rank = comm.Get_rank()
    nprocs = comm.Get_size()
    
    #---------------SET PARAMETERS -----------
    if rank == 0: print "Setting up umbrella sampling parameters."
    
    params = {}
    
    # set the scratch directory for the calculation files
    params['scratchdir'] = "/Users/jeremytempkin/Documents/enhanced_sampling_toolkit/umbrella_sampling/debug_US"
    params['inputFilename'] = "/Users/jtempkin/enhanced_sampling_toolkit/data/input.diala"
    
    # here we will set the umbrella sampling parameters in the params dict
    params['ncells'] = 3
    params['cellWidth'] = 60.0
    params['nwalkers'] = 1
    params['walkerSteps'] = 1000
    params['stepLength'] = 10
    params['Ftype'] = 'transition'
    
    # lets set the dynamics parameters that are needed to specify the walker 
    
    params['inputFilename'] = 'syn.in'
    params['temperature'] = 310.0

    #--------------- INITIALIZATION--------
    # only allow root rank to build files
    if rank == 0: 
        print "Building scratch directory."
        # construct the wkdir. This is where the intermediate dynamcs files will 
        # be written. 
        fileIO.makeWkdir(params['scratchdir'])

    # construct the umbrella data structure
    if rank == 0: print "Initializing the umbrella structure."
    
    # create the partition object 
    system = basisFunctions.partition(params['ncells'])


    # now we construct the umbrella windows
    if rank == 0: print "Building the umbrellas."
    
    # specify the list of boundaries, nboxes, 1/2 width of the windows
    umbParams = {}
    
    # right now, we will hardcore a 12x12 array using Erik's routine for gridding a space
    umbParams['cvrange'] = np.array([map(float,entry.split(",")) for entry in "-180,180,12,30;-180,180,12,30".split(";")])
    umbParams['wrapping'] = np.array(map(float, "1,1".split(',')))
    
    system.umbrellas = fileIO.createUmbrellas(umbParams)
    
    #------------ MAIN LOOP --------------
    # sample umbrellas and construct F
    if rank == 0: print "Entering main loop."
    
    for i in range(rank, len(system.umbrellas), nprocs):
        print "Rank", rank, "sampling umbrella", i, "."
        
        # lets instantiate a walker object to sample this window. 
        wlkr = lammpsWalker.lammpsWalker(params['inputFilename'])
        
        # minimize structure prior to dynamics
        wlkr.minimize()
        
        # set colvars for this walker (currently, alanine dipeptide dihedrals)
        wlkr.colvars.append(['dihedral', 5, 7, 9, 15])
        wlkr.colvars.append(['dihedral', 7, 9, 15, 17]) 
        
        # set an array of starting/stoping restraints for equilibration
        restraint = [[0.0, 10.0], [0.0, 10.0]]
        
        # now specify colvars to dynamics routines
        wlkr.setColvars()
        
        # equilibrate the walker to the target point in CV space
        wlkr.equilibrate(system.umbrellas[i].center, restraint, 100000)
        
        """
        # enter the sampling routine 
        try: 
            system.sample(wlkr, params['walkerSteps'], i, 0, params, rank)
        except errors.DynamicsError:
            print "Rank", rank, "sampling error occured in umbrella", i, "."
            continue
        """
        
        # now we are done populating the samples array, close the walker
        wlkr.close()
    
    #-----------------MPI COMMUNICATION------------
    """
    Here we communicate the rows of F to each processor to set up and solve the 
    eigenvalue problem. 
    """
    

    # now reduce the F matrix at root, first making a send buffer, 
    system.Fbuff = copy.deepcopy(system.F)
    comm.Reduce([system.Fbuff, MPI.DOUBLE], [system.F, MPI.DOUBLE], op=MPI.SUM, root=0)
        
    # also share the F_error matrix
    system.F_error_buff = copy.deepcopy(system.F_error)
    comm.Reduce([system.F_error_buff, MPI.DOUBLE], [system.F_error, MPI.DOUBLE], op=MPI.SUM, root=0)
    
    # issue a global MPI barrier to ensure data has been recieved. 
    comm.Barrier()
    
    
    #----------------WRITE OUT DATA-------------
    # now allow rank 0 to process data. 
    if rank == 0:
        print system.F
        fileIO.writeMat(system.F, params['wkdir'] + "/F.out")
    
        print "Entering eigenvalue routine."
        # solve eigenvalue problem for F
        system.getZ()
        fileIO.writeMat(system.z, params['wkdir'] + "/z.out")
    
        print "Computing the sensitivities."
        bounds = system.getlogBound(system.F)
        fileIO.writeMat(bounds, params['wkdir'] + "/bounds.out")


    # now we will perform an analysis of the data and increase sampling of 
    # windows with high variance

    if rank == 0:
        print "Done!"
        print "Total wallclock time was: " + str(time() - starttime) + " seconds."

    
    return 0
# -*- coding: utf-8 -*-
"""
A profiling run that tests the dynamics of calling propagate through the lammpsWalker API vs running dynamcis directly.
"""

import lammpsWalker
import dynamics 
import os

nsteps = 1000000
stepLength = 5
temperature = 310.0
damping_coefficient = 30.0
scratchdir = os.path.abspath("data")
inputFilename = os.path.abspath("data/input.enk")
logFilename = scratchdir + "/log.lammps"

os.chdir(scratchdir)

# lets instantiate a walker object to sample this window.

wlkr = lammpsWalker.lammpsWalker(inputFilename, logFilename, index=0, debug=False)

# set langevin dynamics object and pass it to the walker.
dyn = dynamics.langevin(temperature, damping_coefficient, shake = True)            
wlkr.setDynamics(dyn)
wlkr.setTimestep(1.0)
wlkr.drawVel()

for i in range(nsteps / stepLength):
    wlkr.propagate(stepLength)
def main(debug=False):
    """
    definition of main scripting file for debugging purposes.
    """
    # time the execution
    startTime = time()
    
    if debug: print "WARNING: Debug passed as True. Will dump maximum set of internal values."

    #---------------SET SIMULATION PARAMETERS -----------

    params = {}
    
    params = setParameters(params)
    
    print "parameters initialized as: ", params
        
    print "Building scratch directory."
    # construct the wkdir. This is where the intermediate dynamics files will be written.
    fileIO.makeWkdir(params['scratchdir'])

    # make sure we've built the scratchdir since we'll assume it exists for the rest of the code
    assert os.path.exists(params['scratchdir']), "The scratch directory was not created."
    
    # we're going to move the wokring directory into the data directory
    os.chdir(os.path.dirname(params['inputFilename']))
 

    #--------------- INITIALIZATION OF PARTITION INSTANCE--------
    # construct the umbrella data structure
    # commented out below this block is the origional construction lines for the partition module. 
    # here we're simply loading this from a binary pickle written to file. This obviates the need to 
    # reconstruct the neighborlist every call
    if os.path.exists(params['systemFile']):
        with open(params['systemFile'], "r") as f_handle:
            system = pickle.load(f_handle)
    else:

        system = partition_mod.partition(params['ncells'])

        if debug: print system

        # now we construct the umbrella windows
        system.umbrellas = system.createUmbrellas(params['cvrange'], params['wrapping'], basisType=params['basisType'], neighborList=False)
        
        with open(params['systemFile'], "w") as f_handle:
            pickle.dump(system, f_handle, protocol=2)

    # double check we've created as many windows as we said in ncells
    assert len(system.umbrellas) == params['ncells'], "The number of cells specified does not equal the number of umbrellas created."

    # debug prints
    if debug:
        for i in range(len(system.umbrellas)):
            print "umbrella", i, system.umbrellas[i].center, system.umbrellas[i].width


    # ------------------ SETTING THE DYNAMICS PARAMETERS FOR SAMPLING THE WINDOWS ----------------
    
    # now set up partition with a list of associated ranks
    system.rank_window_index = range(len(system.umbrellas))
    
    # -------------- INITIALIZATION OF SYSTEM OBSERVABLES -------
    # here we add a list of observables to compute
    pmf_prototype = observables.pmf("pmf", np.zeros((180,180)))
    system.addObservable(pmf_prototype)
    """
    #  now we check to make sure we've correctly set up the local observables
    for window in system.umbrellas:
        assert hasattr(window, "local_observables"), str(rank) + ": The local observables in window " + str(system.umbrellas.index(window)) + " was not created."

    for window in system.umbrellas:
        assert len(window.local_observables) == len(system.observables), str(rank) + ": The number of window observables does not match the number of partition observables."
    """

    #----------------MAIN LOOP----------------
    print "Sampling via equilibrium US."
    for k in range(params['iterations']):

        # ---------- RESET DATA USED FOR CURRENT ITERATION -------------
        # reset the M matrix to zeros for this iteration of the algorithm
        # this matrix will get populated from the inner loop
        system.M = np.zeros((len(system.umbrellas), len(system.umbrellas)))
        system.nsamples_M = np.zeros((len(system.umbrellas), len(system.umbrellas)))
        
        # start a timer on root rank
        st = time()

        # this loop does one iteration of the update
        for i in range(10):
            print "sampling window", i
            # lets instantiate a walker object to sample this window.

            wlkr = lammpsWalker.lammpsWalker(params['inputFilename'], params['logFilename'], index=i, debug=debug)
            
            # set langevin dynamics object and pass it to the walker.
            dyn = dynamics.langevin(params['temperature'], params['damping_coefficient'], shake = True)            
            wlkr.setDynamics(dyn)
            
            wlkr.setTimestep(1.0)
            
            wlkr.drawVel()

            # set colvars for this walker
            wlkr.addColvars('c1', "dihedral", [9, 30, 37, 44])
            wlkr.addColvars('c2','dihedral', [30, 37, 44, 64])
            
            #wlkr.setOutput("traj.out", "dcd", params['scratchdir'] + "/" + str(i) + ".dcd", nSteps=1000)
            #let's add an output for writing out the configurations in hdf5 format so we have an estimate for the local distribution
            if os.path.exists(params['scratchdir'] + "/ep." + str(i) + ".h5py"):
                f_handle = h5py.File(params['scratchdir'] + "/ep." + str(i) + ".h5py", "r")
                config = random.choice(f_handle.keys())
                wlkr.setConfig(f_handle[config][:])
                wlkr.drawVel()
                f_handle.close()
            
            else:
                try:
                    if os.path.exists(params['startingPoints'] + "/start." + str(i) + ".npy"):
                        
                        config = np.load(params['startingPoints'] + "/start." + str(i) + ".npy")
                        
                        wlkr.setConfig(config)
                        
                        wlkr.drawVel()
                    else:
                        
                        dragWalker(wlkr, system.umbrellas[i], nSteps=5)
                
                        assert system.umbrellas[i](wlkr.getColvars(), system.umbrellas) > 0.0, "The walker in " + str(i) + " did not initialize in the support of the window."  
                    
                        np.save(params['startingPoints'] + "/start." + str(i), wlkr.getConfig())
                        
                except AssertionError:  
                    pass
            
            # enter the sampling routine. 
            system.sample_US(wlkr, params['walkerSteps'], i, k, params, debug=True)
            
            # now we are done populating the samples array, close the walker
            wlkr.close()
            del wlkr

            if debug: print system.umbrellas[i].local_observables[0].data

        # report time it too for root to move out of the barrier (ie all ranks have sampled)
        print "Elapsed time in sampling routine on root rank after barrier: ", time() - st

        # now let's time the communication section
        st = time()

        # ------------ EIGENVALUE PROBLEM --------------        

        """
        Step 2) solution of the eigenvalue problem at each rank
        """
        # at rank, compute G and z
        system.updateF('overlap')

        system.k += 1
        
        #system.computeZ()
        
        if debug: print system.z
        
        """
        Step 3) estimation of observables on each rank
        """
        system.computeObservables()
        

        """
        Step 4) all reduction of averaged observables to each processor
        """
        if debug: print system.observables[0].data

        """
        Step 5) writing local and global estimates of the observables on root rank only
        """

        f_handle = h5py.File(params['scratchdir'] + "/F.out", "a")            
        f_handle.create_dataset("F." + str(k), data=system.F)
        f_handle.close()
        
        f_handle = h5py.File(params['scratchdir'] + "/z.out", "a")            
        f_handle.create_dataset("z." + str(k), data=system.z)
        f_handle.close()
        
        for obs in system.observables:
            f_handle = h5py.File(params['scratchdir'] + "/" + obs.name, "a")
            f_handle.create_dataset(obs.name + "." + str(k), data=obs.data)
            f_handle.close()
        

        print "Elapsed time in communication routine on root rank after barrier: ", time() - st

    print "Done!"
    print "Total wall clock time was: " + str(time() - startTime) + " seconds."


    return 0