def cache_check(simrun):
'''
Checks if a similar simrun exists and if this run is allows to be loaded from cache.
If so, it loads the cache and returns True, and False otherwise.
'''
simrun_matches = SimRun.objects.filter(enable_cachefrom=True, group_name=simrun.group_name, instr_displayname=simrun.instr_displayname, params_str=simrun.params_str, gravity=simrun.gravity, scanpoints=simrun.scanpoints, neutrons__gte = simrun.neutrons).order_by('-complete')
if len(simrun_matches) > 0:
simrun.data_folder = os.path.join(os.path.join(STATIC_URL.lstrip('/'), DATA_DIRNAME), simrun.__str__())
# Simple unix cp -r of data directory
process = subprocess.Popen("cp -r " + simrun_matches[0].data_folder + " " + simrun.data_folder,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
(stdout, stderr) = process.communicate()
# Run stream editor to replace "Completed" label with "Loaded cache data from"
process = subprocess.Popen("sed -i.bak s\"/Completed/Loaded\ cache\ data\ from/\" " + simrun.data_folder + "/browse*.html",
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
(stdout, stderr) = process.communicate()
simrun.complete = simrun_matches[0].complete
simrun.save()
return True
else:
return False
def init_processing(simrun):
''' creates data folder, copies instr files and updates simrun object '''
try:
simrun.data_folder = os.path.join(os.path.join(STATIC_URL.lstrip('/'), DATA_DIRNAME), simrun.__str__())
os.mkdir(simrun.data_folder)
simrun.save()
# copy instrument from sim folder to simrun data folder
instr_source = '%s/%s/%s.instr' % (SIM_DIR, simrun.group_name, simrun.instr_displayname)
instr = '%s/%s.instr' % (simrun.data_folder, simrun.instr_displayname)
p = subprocess.Popen(['cp','-p',instr_source, instr])
p.wait()
# symlink the .c and the .out files
src_c = '%s/%s/%s.c' % (SIM_DIR, simrun.group_name, simrun.instr_displayname)
src_out = '%s/%s/%s.out' % (SIM_DIR, simrun.group_name, simrun.instr_displayname)
ln_c = '%s/%s.c' % (simrun.data_folder, simrun.instr_displayname)
ln_out = '%s/%s.out' % (simrun.data_folder, simrun.instr_displayname)
p = subprocess.Popen(['cp','-p',src_c, ln_c])
p.wait()
p = subprocess.Popen(['cp','-p',src_out, ln_out])
p.wait()
# symlink the contents of sim/datafiles/
allfiles = [f for f in os.listdir('sim/datafiles/') if os.path.isfile(os.path.join('sim/datafiles/', f))]
if '.gitignore' in allfiles:
allfiles.remove('.gitignore')
for f in allfiles:
src = os.path.join('..', '..', '..', 'sim', 'datafiles', f)
ln = '%s/%s' % (simrun.data_folder, f)
os.symlink(src, ln)
except Exception as e:
raise Exception('init_processing: %s (%s)' % (type(e).__name__, e.__str__()))
def init_processing(simrun):
''' creates data folder, copies instr files and updates simrun object '''
try:
simrun.data_folder = os.path.join(os.path.join(STATIC_URL.lstrip('/'), DATA_DIRNAME), simrun.__str__())
os.mkdir(simrun.data_folder)
simrun.save()
# copy instrument from sim folder to simrun data folder
instr_source = '%s/%s/%s.instr' % (SIM_DIR, simrun.group_name, simrun.instr_displayname)
instr = '%s/%s.instr' % (simrun.data_folder, simrun.instr_displayname)
p = subprocess.Popen(['cp','-p',instr_source, instr])
p.wait()
# symlink the .c and the .out files
src_c = '%s/%s/%s.c' % (SIM_DIR, simrun.group_name, simrun.instr_displayname)
src_out = '%s/%s/%s.out' % (SIM_DIR, simrun.group_name, simrun.instr_displayname)
ln_c = '%s/%s.c' % (simrun.data_folder, simrun.instr_displayname)
ln_out = '%s/%s.out' % (simrun.data_folder, simrun.instr_displayname)
p = subprocess.Popen(['cp','-p',src_c, ln_c])
p.wait()
p = subprocess.Popen(['cp','-p',src_out, ln_out])
p.wait()
# symlink the contents of sim/datafiles/
allfiles = [f for f in os.listdir('sim/datafiles/') if os.path.isfile(os.path.join('sim/datafiles/', f))]
if '.gitignore' in allfiles:
allfiles.remove('.gitignore')
for f in allfiles:
src = os.path.join('..', '..', '..', 'sim', 'datafiles', f)
ln = '%s/%s' % (simrun.data_folder, f)
os.symlink(src, ln)
except Exception as e:
raise Exception('init_processing: %s (%s)' % (type(e).__name__, e.__str__()))
def handle(self, *args, **options):
''' implements main execution loop and debug run '''
# enable logging
logging.basicConfig(
level=logging.INFO,
format='%(threadName)-22s: %(message)s',
)
# ensure data output dir exists:
try:
data_basedir = os.path.join(STATIC_URL.lstrip('/'), DATA_DIRNAME)
if not os.path.exists(data_basedir):
os.mkdir(data_basedir)
except:
raise ExitException(
'Could not find or create base data folder, exiting (%s).' %
data_basedir)
# global error handling
try:
# debug run
if options['debug']:
work(threaded=False)
exit()
# main threaded execution loop:
sema = threading.BoundedSemaphore(MAX_THREADS)
_log("created semaphore with %d slots" % MAX_THREADS)
_log("looking for simruns...")
while True:
work(threaded=True, semaphore=sema)
time.sleep(1)
# ctr-c exits
except KeyboardInterrupt:
print("")
_log("shutdown requested, exiting...")
print("")
print("")
# handle exit-exception (programmatic shutdown)
except ExitException as e:
print("")
_log("exit exception raised, exiting (%s)" % e.__str__())
print("")
print("")
# global exception log to file
except Exception as e:
_log_error(e)
def handle(self, *args, **options):
''' implements main execution loop and debug run '''
# enable logging
logging.basicConfig(level=logging.INFO,
format='%(threadName)-22s: %(message)s',
)
# ensure data output dir exists:
try:
data_basedir = os.path.join(STATIC_URL.lstrip('/'), DATA_DIRNAME)
if not os.path.exists(data_basedir):
os.mkdir(data_basedir)
except:
raise ExitException('Could not find or create base data folder, exiting (%s).' % data_basedir)
# graceful exiting
try:
# debug run
if options['debug']:
work(threaded=False)
exit()
# main threaded execution loop:
logging.info("looking for simruns...")
while True:
work()
time.sleep(1)
# ctr-c exits
except KeyboardInterrupt:
print("")
logging.info("shutdown requested, exiting...")
print("")
print("")
# handle exit-exception (programmatic shutdown)
except ExitException as e:
print("")
logging.warning("exit exception raised, exiting (%s)" % e.__str__())
print("")
print("")