local_runpath = skirtrun.runpath() local_inpath = skirtrun.inpath() local_outpath = skirtrun.outpath() local_vispath = skirtrun.vispath() remote_runpath = local_runpath.replace(config.results_path, config.cosma["results_path"]) remote_inpath = local_inpath.replace(config.results_path, config.cosma["results_path"]) remote_outpath = local_outpath.replace(config.results_path, config.cosma["results_path"]) remote_vispath = local_vispath.replace(config.results_path, config.cosma["results_path"]) # synchronize the files in each directory # - skip subdirectories and symbolic links # - for the vis directory, do not overwrite newer local versions error = subprocess.call(("rsync", "-htvz", cosma_prefix+remote_runpath+"/*", local_runpath+"/")) if error: raise ValueError("Error in rsync for run: " + str(error)) error = subprocess.call(("rsync", "-htvz", cosma_prefix+remote_inpath+"/*", local_inpath+"/")) if error: raise ValueError("Error in rsync for in: " + str(error)) error = subprocess.call(("rsync", "-htvz", cosma_prefix+remote_outpath+"/*", local_outpath+"/")) if error: raise ValueError("Error in rsync for out: " + str(error)) error = subprocess.call(("rsync", "-htvz", "--update", cosma_prefix+remote_vispath+"/*", local_vispath+"/")) if error: raise ValueError("Error in rsync for vis: " + str(error)) # update the record in the local database if needed if db.updaterow(runid, record): db.commit() # close the database db.close() print "--> Done synchronizing {} completed SKIRT-runs".format(len(records)) # -----------------------------------------------------------------
os.remove(outzip) error = subprocess.call(("zip", "-rj", outzip, outdir)) if error: raise ValueError("Error while zipping the out directory: " + str(error)) # copy the archives offsite print "--> Copying archives for SKIRT-run {}...".format(runid) timestamp = config.timestamp() archdir = os.path.join(archive_path, "g-{0:04}".format(runid // 1000)) archin = os.path.join(archdir, "r-{0:04}-{1:03}_in_{2}.zip".format(runid // 1000, runid % 1000, timestamp)) archout = os.path.join(archdir, "r-{0:04}-{1:03}_out_{2}.zip".format(runid // 1000, runid % 1000, timestamp)) if not os.path.isdir(archdir): os.makedirs(archdir) error = subprocess.call(("cp", "-v", inzip, archin)) if error: raise ValueError("Error while copying the in archive: " + str(error)) error = subprocess.call(("cp", "-v", outzip, archout)) if error: raise ValueError("Error while copying the out archive: " + str(error)) # update the runstatus in the database db.updatestatus((runid,), "archived") db.commit() # close the database db.close() print "Done..." # -----------------------------------------------------------------
# synchronize the files in each directory # - skip subdirectories and symbolic links # - for the vis directory, do not overwrite newer local versions error = subprocess.call( ("rsync", "-htvz", cosma_prefix + remote_runpath + "/*", local_runpath + "/")) if error: raise ValueError("Error in rsync for run: " + str(error)) error = subprocess.call( ("rsync", "-htvz", cosma_prefix + remote_inpath + "/*", local_inpath + "/")) if error: raise ValueError("Error in rsync for in: " + str(error)) error = subprocess.call( ("rsync", "-htvz", cosma_prefix + remote_outpath + "/*", local_outpath + "/")) if error: raise ValueError("Error in rsync for out: " + str(error)) error = subprocess.call( ("rsync", "-htvz", "--update", cosma_prefix + remote_vispath + "/*", local_vispath + "/")) if error: raise ValueError("Error in rsync for vis: " + str(error)) # update the record in the local database if needed if db.updaterow(runid, record): db.commit() # close the database db.close() print "--> Done synchronizing {} completed SKIRT-runs".format(len(records)) # -----------------------------------------------------------------
str(error)) # copy the archives offsite print "--> Copying archives for SKIRT-run {}...".format(runid) timestamp = config.timestamp() archdir = os.path.join(archive_path, "g-{0:04}".format(runid // 1000)) archin = os.path.join( archdir, "r-{0:04}-{1:03}_in_{2}.zip".format(runid // 1000, runid % 1000, timestamp)) archout = os.path.join( archdir, "r-{0:04}-{1:03}_out_{2}.zip".format(runid // 1000, runid % 1000, timestamp)) if not os.path.isdir(archdir): os.makedirs(archdir) error = subprocess.call(("cp", "-v", inzip, archin)) if error: raise ValueError("Error while copying the in archive: " + str(error)) error = subprocess.call(("cp", "-v", outzip, archout)) if error: raise ValueError("Error while copying the out archive: " + str(error)) # update the runstatus in the database db.updatestatus((runid, ), 'archived') db.commit() # close the database db.close() print "Done..." # -----------------------------------------------------------------