Exemple #1
0
backup()
db = Database()

# remove SKIRT-runs that have been archived from the do-list
records = filter(lambda record: db.select("runid="+str(record['runid']))[0]['runstatus']!='archived', records)
print "--> .. of which {} have not yet been archived".format(len(records))

# synchronize the results from each completed run
for record in records:
    runid = record['runid']
    print "--> Synchronizing results for run-id {}...".format(runid)

    # get the local and remote paths (creating the local directories if needed)
    skirtrun = SkirtRun(runid, create=True)
    local_runpath = skirtrun.runpath()
    local_inpath = skirtrun.inpath()
    local_outpath = skirtrun.outpath()
    local_vispath = skirtrun.vispath()
    remote_runpath = local_runpath.replace(config.results_path, config.cosma["results_path"])
    remote_inpath = local_inpath.replace(config.results_path, config.cosma["results_path"])
    remote_outpath = local_outpath.replace(config.results_path, config.cosma["results_path"])
    remote_vispath = local_vispath.replace(config.results_path, config.cosma["results_path"])

    # synchronize the files in each directory
    #  - skip subdirectories and symbolic links
    #  - for the vis directory, do not overwrite newer local versions
    error = subprocess.call(("rsync", "-htvz", cosma_prefix+remote_runpath+"/*", local_runpath+"/"))
    if error: raise ValueError("Error in rsync for run: " + str(error))
    error = subprocess.call(("rsync", "-htvz", cosma_prefix+remote_inpath+"/*", local_inpath+"/"))
    if error: raise ValueError("Error in rsync for in: " + str(error))
    error = subprocess.call(("rsync", "-htvz", cosma_prefix+remote_outpath+"/*", local_outpath+"/"))
Exemple #2
0
# verify that the offsite archive server is mounted
if len(runids) == 0:
    print "No SKIRT-runs to be archived."
    exit()
if not os.path.isdir(archive_path):
    print "Please mount the offsite archive server and try again."
    exit()

# open the database and process each SKIRT-run
db = Database()
for runid in runids:
    print "--> Creating archives for SKIRT-run {}...".format(runid)

    # get the paths for the in and out directories, and the corresponding zip archives
    skirtrun = SkirtRun(runid)
    indir = skirtrun.inpath()
    outdir = skirtrun.outpath()
    inzip = indir + ".zip"
    outzip = outdir + ".zip"

    # export the database record
    info = db.select("runid=" + str(runid))[0]
    infofile = open(os.path.join(indir, "database_record.txt"), "w")
    infofile.write("# SKIRT-run database record\n")
    for key in info.keys():
        infofile.write(("{} = {}\n").format(key, info[key]))
    infofile.close()

    # create the archives
    if os.path.isfile(inzip):
        os.remove(inzip)
Exemple #3
0
# remove SKIRT-runs that have been archived from the do-list
records = filter(
    lambda record: db.select("runid=" + str(record['runid']))[0]['runstatus']
    != 'archived', records)
print "--> .. of which {} have not yet been archived".format(len(records))

# synchronize the results from each completed run
for record in records:
    runid = record['runid']
    print "--> Synchronizing results for run-id {}...".format(runid)

    # get the local and remote paths (creating the local directories if needed)
    skirtrun = SkirtRun(runid, create=True)
    local_runpath = skirtrun.runpath()
    local_inpath = skirtrun.inpath()
    local_outpath = skirtrun.outpath()
    local_vispath = skirtrun.vispath()
    remote_runpath = local_runpath.replace(config.results_path,
                                           config.cosma["results_path"])
    remote_inpath = local_inpath.replace(config.results_path,
                                         config.cosma["results_path"])
    remote_outpath = local_outpath.replace(config.results_path,
                                           config.cosma["results_path"])
    remote_vispath = local_vispath.replace(config.results_path,
                                           config.cosma["results_path"])

    # synchronize the files in each directory
    #  - skip subdirectories and symbolic links
    #  - for the vis directory, do not overwrite newer local versions
    error = subprocess.call(
Exemple #4
0
# verify that the offsite archive server is mounted
if len(runids) == 0:
    print "No SKIRT-runs to be archived."
    exit()
if not os.path.isdir(archive_path):
    print "Please mount the offsite archive server and try again."
    exit()

# open the database and process each SKIRT-run
db = Database()
for runid in runids:
    print "--> Creating archives for SKIRT-run {}...".format(runid)

    # get the paths for the in and out directories, and the corresponding zip archives
    skirtrun = SkirtRun(runid)
    indir = skirtrun.inpath()
    outdir = skirtrun.outpath()
    inzip = indir + ".zip"
    outzip = outdir + ".zip"

    # export the database record
    info = db.select("runid=" + str(runid))[0]
    infofile = open(os.path.join(indir, "database_record.txt"), 'w')
    infofile.write('# SKIRT-run database record\n')
    for key in info.keys():
        infofile.write(("{} = {}\n").format(key, info[key]))
    infofile.close()

    # create the archives
    if os.path.isfile(inzip): os.remove(inzip)
    error = subprocess.call(("zip", "-rj", inzip, indir))