def view_tracking(request):
cxt_data = {'src_directories': SourceSimDir.objects.all(),
'untracked_simfiles': SimFile.get_untracked_sims(),
'simfiles': SimFile.get_tracked_sims()}
csrf_context = RequestContext(request, cxt_data, [config_processor])
return render_to_response('tracking.html', csrf_context)
def doeditsimfile(request, simfile_id):
# Open up the file in an editor:
sim_file = SimFile.get_tracked_sims(id=simfile_id)
cwd = os.getcwd()
os.chdir(os.path.split(sim_file.full_filename)[0])
data_dict = {'full_filename': sim_file.full_filename}
#cmds = MReOrgConfig.get_ns().get('drop_into_editor_cmds', ['xterm &'])
#cmds = MReOrgConfig.get_ns().get('drop_into_editor_cmds', ['xterm &'])
#cmds = MReOrgConfig.get_ns().get('drop_into_editor_cmds', ['xterm &'])
cmds = MReOrgConfig.config['Settings']['Curate']['drop_into_editor_cmds']
for cxt_data in cmds:
t = string.Template(cxt_data).substitute(**data_dict)
os.system(t)
os.chdir(cwd)
# Return to the previous page:
referer = request.META.get('HTTP_REFERER', None)
if referer is None:
return HttpResponseRedirect('/')
try:
redirect_to = urlsplit(referer, 'http', False)[2]
return HttpResponseRedirect(redirect_to)
except IndexError:
return HttpResponseRedirect('/')
def overview_delete_simfile(_request, simfile_id):
try:
sim_file = SimFile.get_tracked_sims(id=simfile_id)
sim_file.delete()
except:
pass
return simplejson.dumps({})
def overview_delete_simfile(_request, simfile_id):
try:
sim_file = SimFile.get_tracked_sims(id=simfile_id)
sim_file.delete()
except:
pass
return simplejson.dumps({})
def refreshsimlist(_request):
assert False, 'Is this used?'
ensure_config()
runconfig = _request.session['current_runconfig']
states = {}
for simfile in SimFile.get_tracked_sims():
states[simfile.id] = simfile.get_status()
return simplejson.dumps({'sim_file_states': states})
def overview_resubmit_simfile(_request, simfile_id):
try:
sim_file = SimFile.get_tracked_sims(id=simfile_id)
except:
return simplejson.dumps({})
# Existing simulation object
if not sim_file.simqueueentry_set.count():
SimQueueEntry.create(sim_file=sim_file, runconfig=_request.session["current_runconfig"])
return simplejson.dumps({})
def overview_resubmit_simfile(_request, simfile_id):
try:
sim_file = SimFile.get_tracked_sims(id=simfile_id)
except:
return simplejson.dumps({})
# Existing simulation object
if not sim_file.simqueueentry_set.count():
SimQueueEntry.create(sim_file=sim_file,
runconfig=_request.session['current_runconfig'])
return simplejson.dumps({})
def view_overview(request):
ensure_config(request)
sims =[sim for sim in SimFile.get_tracked_sims() if request.session['current_filegroup'].contains_simfile(sim) ]
return render_to_response(
'overview.html',
RequestContext(request,
{ 'simfiles': sims },
[config_processor]
)
)
def overview_resubmit_simfile_if_failure(_request, simfile_id):
try:
sim_file = SimFile.get_tracked_sims(id=simfile_id)
except:
return simplejson.dumps({})
if sim_file.get_status(runconfig=_request.session['current_runconfig']) == SimRunStatus.Success:
return simplejson.dumps({})
if not sim_file.simqueueentry_set.count():
SimQueueEntry.create(sim_file=sim_file, runconfig=_request.session['current_runconfig'])
return simplejson.dumps({})
def update_db_from_config():
import mreorg
from mreorg.curator.frontend.models import RunConfiguration, FileGroup, SimFile
from mreorg.curator.frontend.models import EnvironVar
# Update the FileGroups:
default_filegroups = mreorg.MReOrgConfig.get_ns().get(
'default_filegroups', {})
for fgname, fgglobs in default_filegroups.iteritems():
filenames = set()
for fgglob in fgglobs:
filenames.update(mreorg.glob2.glob(fgglob))
# Safely get the FileGroup:
fg = FileGroup.get_or_make(name=fgname)
assert not fg.is_special(), 'Trying to overwrite a builtin filegroup'
for filename in filenames:
simfile = SimFile.get_or_make(full_filename=filename)
if not fg.contains_simfile(simfile):
fg.simfiles.add(simfile)
fg.save()
print 'Updated FileGroup: %s' % (fgname, )
# Update the RunConfigurations:
default_runconfigs = mreorg.MReOrgConfig.get_ns().get(
'default_runconfigs', {})
for confname, confinfo in default_runconfigs.iteritems():
runconf = RunConfiguration.get_or_make(name=confname)
assert not runconf.is_special(
), 'Trying to add a builtin-configuration'
runconf.timeout = confinfo.get('timeout', None)
for (key, value) in confinfo.get('env_vars', {}).iteritems():
try:
envvar = runconf.environvar_set.get(key=key)
envvar.value = value
envvar.save()
except EnvironVar.DoesNotExist:
envvar = EnvironVar(key=key, value=value, config=runconf)
envvar.save()
runconf.save()
print 'Updated RunConfig: %s' % confname
# Add default locations:
mh_adddefault_locations()
# Rescan-filesystem:
rescan_filesystem()
print 'Finished Reconfiguring'
def overview_resubmit_simfile_if_failure(_request, simfile_id):
try:
sim_file = SimFile.get_tracked_sims(id=simfile_id)
except:
return simplejson.dumps({})
if sim_file.get_status() == SimRunStatus.Sucess:
return simplejson.dumps({})
if not sim_file.simqueueentry_set.count():
SimQueueEntry.create(sim_file=sim_file,
runconfig=_request.session['current_runconfig'])
return simplejson.dumps({})
def update_db_from_config():
import mreorg
from mreorg.curator.frontend.models import RunConfiguration, FileGroup, SimFile
from mreorg.curator.frontend.models import EnvironVar
# Update the FileGroups:
default_filegroups = mreorg.MReOrgConfig.get_ns().get('default_filegroups',{})
for fgname, fgglobs in default_filegroups.iteritems():
filenames = set()
for fgglob in fgglobs:
filenames.update(mreorg.glob2.glob(fgglob) )
# Safely get the FileGroup:
fg = FileGroup.get_or_make(name=fgname)
assert not fg.is_special(), 'Trying to overwrite a builtin filegroup'
for filename in filenames:
simfile = SimFile.get_or_make(full_filename=filename)
if not fg.contains_simfile(simfile):
fg.simfiles.add(simfile)
fg.save()
print 'Updated FileGroup: %s' % (fgname,)
# Update the RunConfigurations:
default_runconfigs = mreorg.MReOrgConfig.get_ns().get('default_runconfigs',{})
for confname, confinfo in default_runconfigs.iteritems():
runconf = RunConfiguration.get_or_make(name=confname)
assert not runconf.is_special(), 'Trying to add a builtin-configuration'
runconf.timeout = confinfo.get('timeout',None)
for (key, value) in confinfo.get('env_vars',{}).iteritems():
try:
envvar = runconf.environvar_set.get(key=key)
envvar.value = value
envvar.save()
except EnvironVar.DoesNotExist:
envvar = EnvironVar(key=key,value=value,config=runconf)
envvar.save()
runconf.save()
print 'Updated RunConfig: %s' % confname
# Add default locations:
mh_adddefault_locations()
# Rescan-filesystem:
rescan_filesystem()
print 'Finished Reconfiguring'
def overview_toggle_simfile_for_resubmit(_request, simfile_id):
ensure_config()
try:
sim_file = SimFile.get_tracked_sims(id=simfile_id)
except:
return simplejson.dumps({})
# Existing simulation object
if sim_file.simqueueentry_set.count():
sim_file.simqueueentry_set.all().delete()
else:
SimQueueEntry.create(sim_file=sim_file, runconfig=_request.session['current_runconfig'])
return simplejson.dumps({})
def view_simulation_failures(request):
ensure_config(request)
runconfig= request.session['current_runconfig']
simfiles = build_proxy_for_sim_files(SimFile.get_tracked_sims(), runconfig=runconfig)
cxt_data = {
'failed_simulations': [fo for fo in simfiles if fo.get_status() == SimRunStatus.UnhandledException],
'timeout_simulations': [fo for fo in simfiles if fo.get_status() == SimRunStatus.TimeOut],
'nonzero_exitcode_simulations': [fo for fo in simfiles if fo.get_status() == SimRunStatus.NonZeroExitCode],
'changed_simulations': [fo for fo in simfiles if fo.get_status() == SimRunStatus.FileChanged],
'notrun_simulations': [fo for fo in simfiles if fo.get_status() == SimRunStatus.NeverBeenRun],
}
csrf_context = RequestContext(request, cxt_data, [config_processor])
return render_to_response('view_simulation_failures.html',
csrf_context)
def update_queue(_request, action):
print 'Action', action
tracked_sim_files = SimFile.get_tracked_sims()
print 'Tracked sim files:', len(tracked_sim_files)
runconfig=_request.session['current_runconfig']
if action == 'add-all':
with transaction.commit_on_success():
for simfile in tracked_sim_files:
if simfile.simqueueentry_set.count() > 0:
continue
SimQueueEntry.create( sim_file=simfile, runconfig=runconfig)
return simplejson.dumps({})
if action == 'clear-all':
SimQueueEntry.objects.all() .filter(status = SimQueueEntryState.Waiting).delete()
return simplejson.dumps({})
# TODO! Move functionality into here!
if action == "add-all-failures":
with transaction.commit_on_success():
for simfile in tracked_sim_files:
if simfile.simqueueentry_set.count() > 0:
continue
if simfile.get_status(runconfig=runconfig) not in (SimRunStatus.TimeOut, SimRunStatus.UnhandledException, SimRunStatus.NonZeroExitCode, SimRunStatus.FileChanged,SimRunStatus.NeverBeenRun):
continue
SimQueueEntry.create( sim_file=simfile, runconfig=_request.session['current_runconfig'])
return simplejson.dumps({})
if action == "add-all-failures-not-timeout":
with transaction.commit_on_success():
for simfile in tracked_sim_files:
if simfile.simqueueentry_set.count() > 0:
continue
if simfile.get_status(runconfig=runconfig) not in (SimRunStatus.UnhandledException, SimRunStatus.NonZeroExitCode, SimRunStatus.FileChanged,SimRunStatus.NeverBeenRun):
continue
SimQueueEntry.create( sim_file=simfile, runconfig=_request.session['current_runconfig'])
return simplejson.dumps({})
else:
assert False, 'Unhandled aciton: %s'% action
print 'Action', action
return simplejson.dumps({})
def do_queue_add_sims(request):
if not request.method == 'POST':
return HttpResponseRedirect('/viewsimulationqueue')
print 'Queuing Sims:'
print request.POST
r = re.compile(r"""simid_(?P<id>\d+)""", re.VERBOSE)
sim_id_matches = [r.match(k) for k in request.POST]
sim_ids = [int(m.groupdict()['id']) for m in sim_id_matches if m]
for simfile_id in sim_ids:
sim_file = SimFile.get_tracked_sims(id = simfile_id)
qe = SimQueueEntry(simfile=sim_file)
qe.save()
# Avoid Duplication
return HttpResponseRedirect('/viewsimulationqueue')
def view_sim_output_summaries(request):
last_runs = {}
for simfile in SimFile.get_tracked_sims():
last_runs[simfile] = simfile.get_last_run(request.session['current_runconfig'])
print last_runs
runconfig = (request.session['current_runconfig'])
data =[]
for fg in FileGroup.objects.all():
sim_data = [ (sim, sim.get_last_run(runconfig)) for sim in fg.simfiles.all() ]
data.append( (fg, sim_data) )
#cxt_data = {'last_runs':last_runs}
cxt_data = {'data':data}
print cxt_data
csrf_context = RequestContext(request, cxt_data, [config_processor] )
return render_to_response('simulation_output_summaries.html',
csrf_context)
def overview_update_sim_gui(_request, simfile_id):
print 'Dajax call recieved', simfile_id
from views import ensure_config
ensure_config(_request)
print 'A'
sim_file = SimFile.get_tracked_sims(id=simfile_id)
runconfig = _request.session['current_runconfig']
print 'B'
last_run = sim_file.get_last_run(runconfig=runconfig)
print 'Bb'
exec_date = last_run.execution_data_string() if last_run is not None else ""
print 'C'
v = simplejson.dumps(
{'sim_id':simfile_id,
'state':sim_file.get_status(runconfig=runconfig),
'is_queued':sim_file.is_queued(runconfig=runconfig),
'latest_exec_id':last_run.id if last_run else "",
'latest_exec_date': exec_date,
} )
print 'Finsihed'
return v
def overview_update_sim_gui(_request, simfile_id):
print "Dajax call recieved", simfile_id
from views import ensure_config
ensure_config(_request)
print 'A'
sim_file = SimFile.get_tracked_sims(id=simfile_id)
runconfig = _request.session['current_runconfig']
print 'B'
last_run = sim_file.get_last_run(runconfig=runconfig)
print 'Bb'
exec_date = last_run.execution_data_string(
) if last_run is not None else ""
print 'C'
v = simplejson.dumps({
'sim_id': simfile_id,
'state': sim_file.get_status(runconfig=runconfig),
'is_queued': sim_file.is_queued(runconfig=runconfig),
'latest_exec_id': last_run.id if last_run else "",
'latest_exec_date': exec_date,
})
print 'Finsihed'
return v
def view_simulation_failures(request):
fileObjs = SimFile.get_tracked_sims()
cxt_data = {
'failed_simulations': [fo for fo in fileObjs if fo.get_status()
== SimRunStatus.UnhandledException],
'timeout_simulations': [fo for fo in fileObjs
if fo.get_status()
== SimRunStatus.TimeOut],
'nonzero_exitcode_simulations': [fo for fo in fileObjs
if fo.get_status() == SimRunStatus.NonZeroExitCode],
'changed_simulations': [fo for fo in fileObjs
if fo.get_status()
== SimRunStatus.FileChanged],
'notrun_simulations': [fo for fo in fileObjs if fo.get_status()
== SimRunStatus.NeverBeenRun],
}
csrf_context = RequestContext(request, cxt_data)
return render_to_response('view_simulation_failures.html',
csrf_context)
def overview_update_sim_gui_batch(_request, simfile_ids):
print 'Updating!'
ensure_config()
if isinstance(simfile_ids, int):
simfile_ids = [simfile_ids]
elif isinstance(simfile_ids, basestring):
simfile_ids = [int(tok) for tok in simfile_ids.split()]
else:
assert False
res = []
for simfile_id in simfile_ids:
try:
sim_file = SimFile.get_tracked_sims(id=simfile_id)
except SimFile.DoesNotExist:
continue
if not sim_file.does_file_exist():
sim_file.delete()
continue
runconfig = _request.session['current_runconfig']
last_run = sim_file.get_last_run(runconfig=runconfig)
exec_date = last_run.execution_data_string() if last_run is not None else ""
res.append( {'sim_id':simfile_id,
'state':sim_file.get_status(runconfig=runconfig),
'is_queued':sim_file.is_queued(runconfig=runconfig),
'latest_exec_id':last_run.id if last_run else "",
'latest_exec_date': str(exec_date),
'latest_exec_duration': last_run.execution_time if last_run else '' ,
} )
v = simplejson.dumps(res)
print 'Retunrign from AJax'
#print v
return v
def overview_update_sim_gui(_request, simfile_id):
print "Dajax call recieved", simfile_id
from views import ensure_config
ensure_config(_request)
print "A"
sim_file = SimFile.get_tracked_sims(id=simfile_id)
runconfig = _request.session["current_runconfig"]
print "B"
last_run = sim_file.get_last_run(runconfig=runconfig)
print "Bb"
exec_date = last_run.execution_data_string() if last_run is not None else ""
print "C"
v = simplejson.dumps(
{
"sim_id": simfile_id,
"state": sim_file.get_status(runconfig=runconfig),
"is_queued": sim_file.is_queued(runconfig=runconfig),
"latest_exec_id": last_run.id if last_run else "",
"latest_exec_date": exec_date,
}
)
print "Finsihed"
return v
def do_untrack_all(request):
for sim in SimFile.get_tracked_sims():
sim.tracking_status = TrackingStatus.NotTracked
sim.save()
return HttpResponseRedirect('/tracking')
def refreshsimlist(_request):
states = {}
for simfile in SimFile.get_tracked_sims():
states[simfile.id] = simfile.get_status()
return simplejson.dumps({"sim_file_states": states})
def write_to_database( cls, sim_run_info):
output_file_dir = mreorg.MReOrgConfig.get_image_store_dir()
print 'Saving details from script: ', sim_run_info.script_name
# We don't neeed to update this file every time:
if mreorg.MReOrgConfig.is_non_curated_file(sim_run_info.script_name):
return
try:
simfile = SimFile.get_tracked_sims( full_filename=sim_run_info.script_name)
except: # DoesNotExistError,e :
simfile = SimFile.create( full_filename=sim_run_info.script_name, tracked=True)
simfile.save()
# Create a simulation result object:
simres = SimFileRun(
simfile = simfile,
execution_date = datetime.datetime.now(),
execution_time = sim_run_info.time_taken,
return_code = sim_run_info.return_code,
std_out = sim_run_info.std_out,
std_err = sim_run_info.std_err,
exception_type = sim_run_info.exception_details[0],
exception_traceback = str(sim_run_info.exception_details[2]),
simulation_sha1hash = get_file_sha1hash(simfile.full_filename),
library_sha1hash = '00000',
runconfig = RunConfiguration.objects.get(id=int(os.environ['_MREORG_RUNCONFIGID']))
)
simres.save()
output_file_dir = mreorg.MReOrgConfig.get_image_store_dir()
# Create the images
for image_filename in sim_run_info.output_images:
if not image_filename.endswith('svg'):
continue
# Copy the output file:
try:
hashstr = mreorg.get_file_sha1hash( image_filename)
except:
hashstr=None
if hashstr == None:
continue
opfile1 = output_file_dir + '/' + hashstr + '.svg'
shutil.copyfile(image_filename, opfile1)
f_thumb = image_filename.replace(".svg","thumb.png")
os.system('convert %s -resize 400x300 %s'%(
pipes.quote(image_filename),pipes.quote(f_thumb)))
hashstr = hashlib.md5( open(f_thumb).read() ).hexdigest()
hashstr = mreorg.get_file_sha1hash( f_thumb)
opfile2 = output_file_dir + '/' + hashstr + ".png"
shutil.copyfile(f_thumb, opfile2)
im_obj = SimFileRunOutputImage(
original_name = image_filename,
hash_name = opfile1,
hash_thumbnailname = opfile2,
simulation = simres
)
im_obj.save()
def refreshsimlist(_request):
states = {}
for simfile in SimFile.get_tracked_sims():
states[simfile.id] = simfile.get_status()
return simplejson.dumps({'sim_file_states': states})
def write_to_database(cls, sim_run_info):
output_file_dir = mreorg.MReOrgConfig.get_image_store_dir()
print 'Saving details from script: ', sim_run_info.script_name
# We don't neeed to update this file every time:
if mreorg.MReOrgConfig.is_non_curated_file(sim_run_info.script_name):
return
simfile = SimFile.get_or_make(full_filename=sim_run_info.script_name, make_kwargs={'tracking_status':TrackingStatus.Tracked})
import time
# Create a simulation result object:
simres = SimFileRun(
simfile=simfile,
execution_date=datetime.datetime.now(),
execution_time=sim_run_info.time_taken,
return_code=sim_run_info.return_code,
std_out=sim_run_info.std_out,
std_err=sim_run_info.std_err,
exception_type=sim_run_info.exception_details[0],
exception_traceback=str(sim_run_info.exception_details[2]),
simulation_sha1hash=get_file_sha1hash(simfile.full_filename),
library_sha1hash='00000',
runconfig=RunConfiguration.objects.get(id=int(os.environ['_MREORG_RUNCONFIGID'])),
)
simres.save()
with transaction.commit_on_success():
output_file_dir = mreorg.MReOrgConfig.get_image_store_dir()
# Create the images
for image_filename in sim_run_info.output_images:
if not image_filename.endswith('svg'):
continue
# Copy the output file:
try:
hashstr = mreorg.get_file_sha1hash(image_filename)
except:
hashstr = None
if hashstr == None:
continue
opfile1 = output_file_dir + '/' + hashstr + '.svg'
shutil.copyfile(image_filename, opfile1)
f_thumb = image_filename.replace('.svg', 'thumb.png')
os.system('convert %s -resize 400x300 %s' % (pipes.quote(image_filename),pipes.quote(f_thumb)))
time.sleep(5) # Sometimes, its not ready!
#hashstr = hashlib.md5(open(f_thumb).read()).hexdigest()
hashstr = mreorg.get_file_sha1hash(f_thumb)
opfile2 = output_file_dir + '/' + hashstr + ".png"
shutil.copyfile(f_thumb, opfile2)
im_obj = SimFileRunOutputImage(
original_name=image_filename,
hash_name=opfile1,
hash_thumbnailname=opfile2,
simulation=simres)
im_obj.save()
def simfile_details(request, simfile_id):
sim_file = SimFile.get_tracked_sims(id=simfile_id)
cxt_data = {'simfile': sim_file}
csrf_context = RequestContext(request, cxt_data)
return render_to_response('simfile_details.html', csrf_context)
