Example #1
0
def cache_check(simrun):
    '''
    Checks if a similar simrun exists and if this run is allows to be loaded from cache.
    If so, it loads the cache and returns True, and False otherwise.
    '''
    simrun_matches = SimRun.objects.filter(enable_cachefrom=True, group_name=simrun.group_name, instr_displayname=simrun.instr_displayname, params_str=simrun.params_str, gravity=simrun.gravity, scanpoints=simrun.scanpoints, neutrons__gte = simrun.neutrons).order_by('-complete')
    if len(simrun_matches) > 0:
        simrun.data_folder = os.path.join(os.path.join(STATIC_URL.lstrip('/'), DATA_DIRNAME), simrun.__str__())
        # Simple unix cp -r of data directory
        process = subprocess.Popen("cp -r " + simrun_matches[0].data_folder + " " + simrun.data_folder,
                                                                  stdout=subprocess.PIPE,
                                                                  stderr=subprocess.PIPE,
                                                                  shell=True)
        (stdout, stderr) = process.communicate()
        # Run stream editor to replace "Completed" label with "Loaded cache data from"
        process = subprocess.Popen("sed -i.bak s\"/Completed/Loaded\ cache\ data\ from/\" " + simrun.data_folder + "/browse*.html",
                                                                                                     stdout=subprocess.PIPE,
                                                                                                     stderr=subprocess.PIPE,
                                                                                                     shell=True)
        (stdout, stderr) = process.communicate()
        simrun.complete = simrun_matches[0].complete
        simrun.save()
        return True
    else:
        return False
Example #2
0
def init_processing(simrun):
    ''' creates data folder, copies instr files and updates simrun object '''
    try: 
        simrun.data_folder = os.path.join(os.path.join(STATIC_URL.lstrip('/'), DATA_DIRNAME), simrun.__str__())
        os.mkdir(simrun.data_folder)
        simrun.save()
        
        # copy instrument from sim folder to simrun data folder 
        instr_source = '%s/%s/%s.instr' % (SIM_DIR, simrun.group_name, simrun.instr_displayname)
        instr = '%s/%s.instr' % (simrun.data_folder, simrun.instr_displayname)
        p = subprocess.Popen(['cp','-p',instr_source, instr])
        p.wait()

        # symlink the .c and the .out files
        src_c = '%s/%s/%s.c' % (SIM_DIR, simrun.group_name, simrun.instr_displayname)
        src_out = '%s/%s/%s.out' % (SIM_DIR, simrun.group_name, simrun.instr_displayname)
        ln_c = '%s/%s.c' % (simrun.data_folder, simrun.instr_displayname)
        ln_out = '%s/%s.out' % (simrun.data_folder, simrun.instr_displayname)
        p = subprocess.Popen(['cp','-p',src_c, ln_c])
        p.wait()
        p = subprocess.Popen(['cp','-p',src_out, ln_out])
        p.wait()

        # symlink the contents of sim/datafiles/
        
        allfiles = [f for f in os.listdir('sim/datafiles/') if os.path.isfile(os.path.join('sim/datafiles/', f))]
        if '.gitignore' in allfiles:
            allfiles.remove('.gitignore')
        for f in allfiles:
            src = os.path.join('..', '..', '..', 'sim', 'datafiles', f)
            ln = '%s/%s' % (simrun.data_folder, f)
            os.symlink(src, ln)
        
    except Exception as e:
        raise Exception('init_processing: %s (%s)' % (type(e).__name__, e.__str__()))
Example #3
0
def init_processing(simrun):
    ''' creates data folder, copies instr files and updates simrun object '''
    try: 
        simrun.data_folder = os.path.join(os.path.join(STATIC_URL.lstrip('/'), DATA_DIRNAME), simrun.__str__())
        os.mkdir(simrun.data_folder)
        simrun.save()
        
        # copy instrument from sim folder to simrun data folder 
        instr_source = '%s/%s/%s.instr' % (SIM_DIR, simrun.group_name, simrun.instr_displayname)
        instr = '%s/%s.instr' % (simrun.data_folder, simrun.instr_displayname)
        p = subprocess.Popen(['cp','-p',instr_source, instr])
        p.wait()

        # symlink the .c and the .out files
        src_c = '%s/%s/%s.c' % (SIM_DIR, simrun.group_name, simrun.instr_displayname)
        src_out = '%s/%s/%s.out' % (SIM_DIR, simrun.group_name, simrun.instr_displayname)
        ln_c = '%s/%s.c' % (simrun.data_folder, simrun.instr_displayname)
        ln_out = '%s/%s.out' % (simrun.data_folder, simrun.instr_displayname)
        p = subprocess.Popen(['cp','-p',src_c, ln_c])
        p.wait()
        p = subprocess.Popen(['cp','-p',src_out, ln_out])
        p.wait()

        # symlink the contents of sim/datafiles/
        
        allfiles = [f for f in os.listdir('sim/datafiles/') if os.path.isfile(os.path.join('sim/datafiles/', f))]
        if '.gitignore' in allfiles:
            allfiles.remove('.gitignore')
        for f in allfiles:
            src = os.path.join('..', '..', '..', 'sim', 'datafiles', f)
            ln = '%s/%s' % (simrun.data_folder, f)
            os.symlink(src, ln)
        
    except Exception as e:
        raise Exception('init_processing: %s (%s)' % (type(e).__name__, e.__str__()))
Example #4
0
    def handle(self, *args, **options):
        ''' implements main execution loop and debug run '''
        # enable logging
        logging.basicConfig(
            level=logging.INFO,
            format='%(threadName)-22s: %(message)s',
        )

        # ensure data output dir exists:
        try:
            data_basedir = os.path.join(STATIC_URL.lstrip('/'), DATA_DIRNAME)
            if not os.path.exists(data_basedir):
                os.mkdir(data_basedir)
        except:
            raise ExitException(
                'Could not find or create base data folder, exiting (%s).' %
                data_basedir)

        # global error handling
        try:
            # debug run
            if options['debug']:
                work(threaded=False)
                exit()

            # main threaded execution loop:
            sema = threading.BoundedSemaphore(MAX_THREADS)
            _log("created semaphore with %d slots" % MAX_THREADS)

            _log("looking for simruns...")
            while True:
                work(threaded=True, semaphore=sema)
                time.sleep(1)

        # ctr-c exits
        except KeyboardInterrupt:
            print("")
            _log("shutdown requested, exiting...")
            print("")
            print("")

        # handle exit-exception (programmatic shutdown)
        except ExitException as e:
            print("")
            _log("exit exception raised, exiting (%s)" % e.__str__())
            print("")
            print("")

        # global exception log to file
        except Exception as e:
            _log_error(e)
Example #5
0
 def handle(self, *args, **options):
     ''' implements main execution loop and debug run '''
     # enable logging 
     logging.basicConfig(level=logging.INFO,
                 format='%(threadName)-22s: %(message)s',
                 )
     
     # ensure data output dir exists: 
     try:
         data_basedir = os.path.join(STATIC_URL.lstrip('/'), DATA_DIRNAME)
         if not os.path.exists(data_basedir):
             os.mkdir(data_basedir)
     except:
         raise ExitException('Could not find or create base data folder, exiting (%s).' % data_basedir)            
     
     # graceful exiting
     try:
         # debug run
         if options['debug']:
             work(threaded=False)
             exit()
         
         # main threaded execution loop:
         logging.info("looking for simruns...")
         while True:
             work()
             time.sleep(1)
         
     # ctr-c exits
     except KeyboardInterrupt:
         print("")
         logging.info("shutdown requested, exiting...")
         print("")
         print("")
     
     # handle exit-exception (programmatic shutdown)
     except ExitException as e:
         print("")
         logging.warning("exit exception raised, exiting (%s)" % e.__str__())
         print("")
         print("")