def completed_or_archived_runids_in_range(runidspec):
runids = sorted(runids_in_range(runidspec))
db = Database()
runids = filter(
lambda runid: db.select("runid=" + str(runid))[0]['runstatus'] in
('completed', 'archived'), runids)
db.close()
return runids
def completed_skirtruns(unless_filenames=None):
db = Database()
runids = sorted([ row['runid'] for row in db.select("runstatus='completed' or runstatus='archived'") ])
runs = [ SkirtRun(runid) for runid in runids ]
if unless_filenames!=None:
runs = filter(lambda run: not has_visualization_files(run,unless_filenames), runs)
db.close()
return runs
def completed_skirtruns(unless_filenames=None):
db = Database()
runids = sorted([
row['runid']
for row in db.select("runstatus='completed' or runstatus='archived'")
])
runs = [SkirtRun(runid) for runid in runids]
if unless_filenames != None:
runs = filter(
lambda run: not has_visualization_files(run, unless_filenames),
runs)
db.close()
return runs
from pts.eagle.skirtrun import SkirtRun # ----------------------------------------------------------------- cosma_prefix = "[email protected]:" # ----------------------------------------------------------------- # copy the cosma database to obiwan so we have a backup and we can open it locally print "--> Copying database from cosma to obiwan backup area..." cosma_database_path = os.path.join(config.cosma["database_path"], "SKIRT-runs.db") backup_database_path = os.path.join(config.backup_path, "cosma_SKIRT-runs_backup_"+config.timestamp()+".db") subprocess.call(("rsync", "-htz", cosma_prefix+cosma_database_path, backup_database_path)) # extract records for all completed SKIRT-runs from the local copy of the cosma database db = Database(backup_database_path) records = db.select("runstatus = 'completed'") db.close() print "--> Cosma has {} completed SKIRT-runs".format(len(records)) # backup and open the local database backup() db = Database() # remove SKIRT-runs that have been archived from the do-list records = filter(lambda record: db.select("runid="+str(record['runid']))[0]['runstatus']!='archived', records) print "--> .. of which {} have not yet been archived".format(len(records)) # synchronize the results from each completed run for record in records: runid = record['runid']
cosma_prefix = "[email protected]:" # ----------------------------------------------------------------- # copy the cosma database to obiwan so we have a backup and we can open it locally print "--> Copying database from cosma to obiwan backup area..." cosma_database_path = os.path.join(config.cosma["database_path"], "SKIRT-runs.db") backup_database_path = os.path.join( config.backup_path, "cosma_SKIRT-runs_backup_" + config.timestamp() + ".db") subprocess.call(("rsync", "-htz", cosma_prefix + cosma_database_path, backup_database_path)) # extract records for all completed SKIRT-runs from the local copy of the cosma database db = Database(backup_database_path) records = db.select("runstatus = 'completed'") db.close() print "--> Cosma has {} completed SKIRT-runs".format(len(records)) # backup and open the local database backup() db = Database() # remove SKIRT-runs that have been archived from the do-list records = filter( lambda record: db.select("runid=" + str(record['runid']))[0]['runstatus'] != 'archived', records) print "--> .. of which {} have not yet been archived".format(len(records)) # synchronize the results from each completed run
# dump it into file
outfile = open(outfilepath, 'w')
pickle.dump(collection, outfile)
outfile.close()
print "Created info collection " + outfilepath
# -----------------------------------------------------------------
# chain the command-line arguments into a query list
if len(sys.argv) <= 1: raise ValueError("This script expects one or more command-line arguments")
querylist = "('{}')".format("','".join(sys.argv[1:]))
namelist = "_".join(sys.argv[1:])
# get a list of SkirtRun objects for which to collect statistics, in order of run-id
db = Database()
query = "runstatus in ('completed','archived') and label in {0} and eaglesim in {0}".format(querylist)
runids = sorted([ row['runid'] for row in db.select(query) ])
skirtruns = [ SkirtRun(runid) for runid in runids ]
db.close()
# perform the collection
if len(skirtruns) > 0:
print "Collecting statistics from {} SKIRT-runs with label and eaglesim fields in {}...".format(len(skirtruns),querylist)
collect_info(skirtruns, os.path.join(config.collections_path,"{}_info_collection.dat".format(namelist)))
else:
print "There are no SKIRT-runs with label and eaglesim fields in {}.".format(querylist)
# -----------------------------------------------------------------
def completed_or_archived_runids_in_range(runidspec):
runids = sorted(runids_in_range(runidspec))
db = Database()
runids = filter(lambda runid: db.select("runid=" + str(runid))[0]["runstatus"] in ("completed", "archived"), runids)
db.close()
return runids
def completed_runids():
db = Database()
runids = sorted([row["runid"] for row in db.select("runstatus = 'completed'")])
db.close()
return runids
# construct a list of SkirtRun objects to be processed, depending on the command line
if len(sys.argv) > 1:
runids = completed_or_archived_runids_in_range(sys.argv[1])
else:
runids = completed_runids()
# verify that the offsite archive server is mounted
if len(runids) == 0:
print "No SKIRT-runs to be archived."
exit()
if not os.path.isdir(archive_path):
print "Please mount the offsite archive server and try again."
exit()
# open the database and process each SKIRT-run
db = Database()
for runid in runids:
print "--> Creating archives for SKIRT-run {}...".format(runid)
# get the paths for the in and out directories, and the corresponding zip archives
skirtrun = SkirtRun(runid)
indir = skirtrun.inpath()
outdir = skirtrun.outpath()
inzip = indir + ".zip"
outzip = outdir + ".zip"
# export the database record
info = db.select("runid=" + str(runid))[0]
infofile = open(os.path.join(indir, "database_record.txt"), "w")
infofile.write("# SKIRT-run database record\n")
for key in info.keys():
print "Building grey body fits for {} SKIRT-runs".format(len(skirtruns))
for skirtrun in skirtruns:
print "Building grey body fit for SKIRT-run {}...".format(
skirtrun.runid())
plotgreybodyfit(skirtrun)
# -----------------------------------------------------------------
# build info files
if vistype == 'infofile':
from pts.eagle.database import Database
from pts.eagle.makeinfofile import makeinfofile
print "Building info files for {} SKIRT-runs".format(len(skirtruns))
for skirtrun in skirtruns:
print "Building info file for SKIRT-run {}...".format(skirtrun.runid())
record = Database().select("runid = ?", (skirtrun.runid(), ))[0]
makeinfofile(skirtrun, record['snaptag'])
# -----------------------------------------------------------------
# build gas particle plots
if vistype == 'particles':
from pts.eagle.plotgasparticles import plotgasparticles
print "Building gas particle plots for {} SKIRT-runs".format(
len(skirtruns))
for skirtrun in skirtruns:
print "Building gas particle plot for SKIRT-run {}...".format(
skirtrun.runid())
plotgasparticles(skirtrun)
# -----------------------------------------------------------------
def completed_runids():
db = Database()
runids = sorted(
[row['runid'] for row in db.select("runstatus = 'completed'")])
db.close()
return runids
# construct a list of SkirtRun objects to be processed, depending on the command line
if len(sys.argv) > 1:
runids = completed_or_archived_runids_in_range(sys.argv[1])
else:
runids = completed_runids()
# verify that the offsite archive server is mounted
if len(runids) == 0:
print "No SKIRT-runs to be archived."
exit()
if not os.path.isdir(archive_path):
print "Please mount the offsite archive server and try again."
exit()
# open the database and process each SKIRT-run
db = Database()
for runid in runids:
print "--> Creating archives for SKIRT-run {}...".format(runid)
# get the paths for the in and out directories, and the corresponding zip archives
skirtrun = SkirtRun(runid)
indir = skirtrun.inpath()
outdir = skirtrun.outpath()
inzip = indir + ".zip"
outzip = outdir + ".zip"
# export the database record
info = db.select("runid=" + str(runid))[0]
infofile = open(os.path.join(indir, "database_record.txt"), 'w')
infofile.write('# SKIRT-run database record\n')
for key in info.keys():
