Exemple #1
0
    def save(self, filename, verbose=None, use_dictionary=False):
        """ Save as an npz file, using an incremental method, which
        only uses as much /tmp space as required by each var at a time.
        if use_dictionary is a valid dictionary, save the values of
        ANY AND ONLY the LOCAL variables whose names are in the keys for
        this set.
        So if you have extracted a subset, and you specify 
        use_dictionary=locals(),
        only that subset is saved (both in array length, and variables chosen).
        Beware locals that are not your variables - e.g. mtrand.beta
        """
        if verbose == None: verbose = self.verbose
        st = seconds()
        if use_dictionary == False:
            save_dict = self.da  # the dict used to get data
        else:
            save_dict = use_dictionary
            print('Warning - saving only a subset')

        use_keys = []
        for k in self.da.keys():
            if k in save_dict.keys():
                use_keys.append(k)

        if verbose: print(' Saving only {k}'.format(k=use_keys))

        args = ','.join(["{k}=save_dict['{k}']".format(k=k) for k in use_keys])
        if verbose:
            print('lengths: {0} -999 indicates dodgy variable'.format(
                [mylen(save_dict[k]) for k in use_keys]))

        exec("np.savez_compressed(filename," + args + ")")
        self.name = filename

        if verbose: print(' in {dt:.1f} secs'.format(dt=seconds() - st))
Exemple #2
0
    def load(self):
        start_mem = report_mem(msg='load')
        st = seconds()
        if self.verbose>0: print('loading {nm}'.format(nm=self.name)), 
        if self.loaded:
            if self.verbose: ('print {nm} already loaded'.format(nm=self.name))

        else:
            dd = {}
            for k in self.da.keys():
                dd.update({k: self.da[k]})
                # dictionaries get stored as an object, need "to list"
                if hasattr(dd[k],'dtype'):
                    if dd[k].dtype == np.dtype('object'):
                        dd[k] = dd[k].tolist()
                        if self.verbose: 
                            print('object conversion for {k}'
                                  .format(k=k))
                    if (hasattr(dd[k],'dtype') and 
                        dd[k].dtype == np.dtype('object')):
                        dd[k] = dd[k].tolist()
                        print('*** Second object conversion for {k}!!!'
                              .format(k=k))

        if self.verbose: print(' in {dt:.1f} secs'.format(dt=seconds()-st))
        self.da = dd
        report_mem(start_mem)
Exemple #3
0
def dist_mp(cl_instance,
            instances,
            squared=None,
            averaged=None,
            threads=None,
            debug=0):
    """ multithreaded distance calculation using dist.pyx and nogil version
    set debug>=1 to to a comparison against single processor, GIL version

    >>> import numpy as np
    >>> x=dist_mp(np.random.random((14,)).astype(np.float32),np.random.random((1000,14)).astype(np.float32),debug=1,threads=3)

    """
    if threads is None:
        threads = 3

    n_workers = threads
    workers = []

    for w in range(n_workers):
        workers.append(ThreadWorker(dist))
    # allow 1 more, it may be short or empty, no prob.
    chunk = 1 + len(instances) // n_workers

    st = seconds()
    for (w, worker) in enumerate(workers):
        #print worker.start([big[w*chunk:(w+1)*chunk],2])
        worker.start([cl_instance, instances[w * chunk:(w + 1) * chunk, :]],
                     squared=squared,
                     averaged=averaged,
                     debug=debug)


#      NO! not this  kwargs=dict(squared=squared, debug=debug))

    while 'running' in [worker.status() for worker in workers]:
        pass  #print('waiting'),

    x = []
    for worker in workers:
        x = np.append(x, worker.get_results())
        tm = seconds() - st
    if debug > 0:
        from pyfusion.utils.dist import dist as dist_gil
        from numpy.testing import assert_array_almost_equal

        # extra \n to avoid thread output overwriting (mostly)
        if debug > 1:
            print(
                '\ncheck with safer, single thread version of dist_nogil - no output==OK'
            )
        x1 = dist_gil(cl_instance,
                      instances,
                      averaged=averaged,
                      squared=squared,
                      debug=debug)
        assert_array_almost_equal(x, x1)
    print('took {dt:0.3g} sec for {w} workers'.format(dt=tm, w=n_workers))

    return (x)
Exemple #4
0
    def load(self):
        start_mem = report_mem(msg='load')
        st = seconds()
        if self.verbose > 0: print('loading {nm}'.format(nm=self.name)),
        if self.loaded:
            if self.verbose: ('print {nm} already loaded'.format(nm=self.name))

        else:
            dd = {}
            for k in self.da.keys():
                dd.update({k: self.da[k]})
                # dictionaries get stored as an object, need "to list"
                if hasattr(dd[k], 'dtype'):
                    if dd[k].dtype == np.dtype('object'):
                        dd[k] = dd[k].tolist()
                        if self.verbose:
                            print('object conversion for {k}'.format(k=k))
                    if (hasattr(dd[k], 'dtype')
                            and dd[k].dtype == np.dtype('object')):
                        dd[k] = dd[k].tolist()
                        print('*** Second object conversion for {k}!!!'.format(
                            k=k))

        if self.verbose: print(' in {dt:.1f} secs'.format(dt=seconds() - st))
        self.da = dd
        report_mem(start_mem)
def read_text_pyfusion(files, target='^Shot .*', ph_dtype=None, plot=pl.isinteractive(), ms=100, hold=0, debug=0, quiet=1,  exception = Exception):
    """ Accepts a file or a list of files, returns a list of structured arrays
    See merge ds_list to merge and convert types (float -> pyfusion.prec_med
    """
    st = seconds(); last_update=seconds()
    file_list = files
    if len(np.shape(files)) == 0: file_list = [file_list]
    f='f8'
    if ph_dtype == None: ph_dtype = [('p12',f),('p23',f),('p34',f),('p45',f),('p56',f)]
    #ph_dtype = [('p12',f)]
    ds_list =[]
    comment_list =[]
    count = 0
    for (i,filename) in enumerate(file_list):
        if seconds() - last_update > 30:
            last_update = seconds()
            print('reading {n}/{t}: {f}'
                  .format(f=filename, n=i, t=len(file_list)))
        try:
            if pl.is_string_like(target): 
                skip = 1+find_data(filename, target,debug=debug)
            else: 
                skip = target
            if quiet == 0:
                print('{t:.1f} sec, loading data from line {s} of {f}'
                      .format(t = seconds()-st, s=skip, f=filename))
            #  this little bit to determine layout of data
            # very inefficient to read twice, but in a hurry!
            txt = np.loadtxt(fname=filename, skiprows=skip-1, dtype=str, 
                             delimiter='FOOBARWOOBAR')
            header_toks = txt[0].split()
            # is the first character of the 2nd last a digit?
            if header_toks[-2][0] in '0123456789': 
                if pyfusion.VERBOSE > 0: 
                    print('found new header including number of phases')
                n_phases = int(header_toks[-2])
                ph_dtype = [('p{n}{np1}'.format(n=n,np1=n+1), f) for n in range(n_phases)]
                
            if 'frlow' in header_toks:  # add the two extra fields
                fs_dtype= [ ('shot','i8'), ('t_mid','f8'), 
                            ('_binary_svs','i8'), 
                            ('freq','f8'), ('amp', 'f8'), ('a12','f8'),
                            ('p', 'f8'), ('H','f8'), 
                            ('frlow','f8'), ('frhigh', 'f8'),('phases',ph_dtype)]
            else:
                fs_dtype= [ ('shot','i8'), ('t_mid','f8'), 
                            ('_binary_svs','i8'), 
                            ('freq','f8'), ('amp', 'f8'), ('a12','f8'),
                            ('p', 'f8'), ('H','f8'), ('phases',ph_dtype)]

            ds_list.append(
                np.loadtxt(fname=filename, skiprows = skip, 
                           dtype= fs_dtype)
            )
            count += 1
            comment_list.append(filename)
        except ValueError, info:
            print('Conversion error while reading {f} with loadtxt - {info}'.format(f=filename, info=info))
Exemple #6
0
def check_time_order(dd, allow_equal=True):
    shot = dd['shot']
    last_check = seconds()
    for s in np.unique(dd['shot']):
        ws = np.where(s == shot)[0]
        if np.min(np.diff(dd['t_mid'][ws])) < 0:
            print('time order problem in ' + str(s))
        else:
            print('OK'),
            if seconds() - last_check > 20:
                print('\n')
                last_check = seconds()
Exemple #7
0
def get_fftw3_speed(arr, iters=10, direction=None, dtype=np.float32, **kwargs):
    """ measure the fftw3 speed for various data sizes by using
    plan with estimate, and running one instance.  If arr is int,
    then the elements are different array sizes, otherwise use the
    array.
    direction default - just fwd - use 'both' for both
    To "train": - the print allows you to ^c out.
    from pyfusion.utils.primefactors import get_fftw3_speed
    from pyfusion.utils.fftw3_bdb_utils import save_wisdom, load_wisdom
    from pyfusion.data.filters import next_nice_number
    from pyfusion.utils.primefactors import get_fftw3_speed
    for n in next_nice_number(None):
        print(n); 
        get_fftw3_speed(n, flags=['FFTW_MEASURE'],direction='both',dtype=np.float32)
    save_wisdom()

    Accepts all pyfftw.FFTW args e.g. planning_timelimit
    """
    import pyfftw
    from time import time as seconds
    if np.isscalar(arr): arr = np.array([arr])
    if np.issubdtype(arr.dtype, int):
        atimes = []
        for n in arr:
            atimes.append([
                n,
                get_fftw3_speed(np.ones(n, dtype=dtype),
                                direction=direction,
                                iters=iters,
                                **kwargs)
            ])
        return (np.array(atimes))
    else:  # do one example
        build_kwargs = dict(flags=['FFTW_ESTIMATE'])
        build_kwargs.update(kwargs)
        print(build_kwargs)
        simd_align = pyfftw.simd_alignment  # 16 at the moment.
        arr = pyfftw.n_byte_align(arr, simd_align)
        out = pyfftw.n_byte_align(
            np.ones(len(arr) / 2 + 1, dtype=np.complex64), simd_align)
        fwd = pyfftw.FFTW(arr, out, **build_kwargs)
        if direction == 'both':
            rev = pyfftw.FFTW(out,
                              arr,
                              direction='FFTW_BACKWARD',
                              **build_kwargs)
        st = seconds()
        for i in range(iters):
            fwd.execute()
            if direction == 'both':
                rev.execute()
        return ((seconds() - st) / iters)
Exemple #8
0
def timeinfo(message, outstream=sys.stdout):
    if show_times == 0: return
    global st, st_0
    try:
        dt = seconds()-st
    except:
        outstream.writelines('Time:' + message + '(first call)\n')
        st = seconds()
        st_0 = seconds()
        return
    outstream.writelines("Time: {m} in {dt:.2f}/{t:.2f}\n".format(m=message, dt=dt, t=seconds()-st_0))
    st = seconds()
    return
Exemple #9
0
def timeinfo(message):
    if show_times == 0: return
    global st, st_0
    try:
        dt = seconds()-st
    except:
        print('Time:' + message + '(first call)')
        st = seconds()
        st_0 = seconds()
        return
    print("Time: {m} in {dt:.2f}/{t:.2f}".format(m=message, dt=dt, t=seconds()-st_0))
    st = seconds()
    return
Exemple #10
0
def timeinfo(message):
    if show_times == 0: return
    global st, st_0
    try:
        dt = seconds()-st
    except:
        print('Time:' + message + '(first call)')
        st = seconds()
        st_0 = seconds()
        return
    print("Time: {m} in {dt:.2f}/{t:.2f}".format(m=message, dt=dt, t=seconds()-st_0))
    st = seconds()
    return
Exemple #11
0
    def report_mem(prev_values=None, msg=None):
        """ Show status of phy/virt mem, and if given previous values, 
        report differences
        requires the psutil module - avail in apt-get, but I used pip
        """
        if msg is None:
            msg = ""
        else:
            msg += ": "
        if type(prev_values) == type(""):  # catch error in prev_values
            msg += prev_values
            prev_values = None
            print("need msg= in call to report_mem")

        # available in linux is free + buffers + cached
        if hasattr(psutil, "swap_memory"):  # new version (0.7+)
            pm = psutil.virtual_memory().available  # was psutil.avail_phymem()
            vm = psutil.swap_memory().free  # was psutil.avail_virtmem()
        else:
            pm = psutil.phymem_usage().free  # was .avail_phymem()
            vm = psutil.virtmem_usage().free  # avail_virtmem()

        tim = seconds()
        print("{msg}{pm:.3g} GB phys mem, {vm:.3g} GB virt mem avail".format(msg=msg, pm=pm / 1e9, vm=vm / 1e9)),

        if prev_values is None:
            print()
        else:
            print(
                "- dt={dt:.2g}s, used {pm:.2g} GB phys, {vm:.2g} GB virt".format(
                    pm=(prev_values[0] - pm) / 1e9, vm=(prev_values[1] - vm) / 1e9, dt=tim - prev_values[2]
                )
            )
        return (pm, vm, tim)
Exemple #12
0
def timeinfo(message, outstream=sys.stdout):
    if show_times == 0: return
    global st, st_0
    try:
        dt = seconds() - st
    except:
        outstream.writelines('Time:' + message + '(first call)\n')
        st = seconds()
        st_0 = seconds()
        return
    msgstr = str("Time: {m} in {dt:.2f}/{t:.2f}\n".format(m=message,
                                                          dt=dt,
                                                          t=seconds() - st_0))
    outstream.writelines(msgstr)
    pyfusion.logging.info(msgstr)
    st = seconds()
    return
Exemple #13
0
def dist_mp(cl_instance, instances, squared = None, averaged = None, threads=None, debug=0):
    """ multithreaded distance calculation using dist.pyx and nogil version
    set debug>=1 to to a comparison against single processor, GIL version
    >>> import numpy as np
    >>> x=dist_mp(np.random.random((14,)).astype(np.float32),np.random.random((1000,14)).astype(np.float32),debug=1,threads=3)

    """
    if threads is None:
        threads = 3

    n_workers = threads
    workers = []

    for w in range(n_workers): workers.append(ThreadWorker(dist))
    # allow 1 more, it may be short or empty, no prob.
    chunk = 1+len(instances)//n_workers

    st = seconds()
    for (w, worker) in enumerate(workers):
        #print worker.start([big[w*chunk:(w+1)*chunk],2])
        worker.start([cl_instance, instances[w*chunk:(w+1)*chunk,:]],
                     squared=squared, averaged=averaged, debug=debug)
#      NO! not this  kwargs=dict(squared=squared, debug=debug))


    while 'running' in  [worker.status() for worker in workers]:
        pass #print('waiting'),

    x = [] 
    for worker in workers: 
        x = np.append(x, worker.get_results())
        tm = seconds()-st
    if debug>0:
        from pyfusion.utils.dist import dist as dist_gil
        from numpy.testing import assert_array_almost_equal

        # extra \n to avoid thread output overwriting (mostly)
        if debug>1: 
            print('\ncheck with safer, single thread version of dist_nogil - no output==OK')
        x1 = dist_gil(cl_instance, instances, 
                      averaged=averaged, squared=squared, debug=debug)
        assert_array_almost_equal(x, x1)
    print('took {dt:0.3g} sec for {w} workers'.format(dt=tm, w=n_workers))

    return(x)
Exemple #14
0
def get_fftw3_speed(arr, iters=10, direction=None, dtype=np.float32, **kwargs):
    """ measure the fftw3 speed for various data sizes by using
    plan with estimate, and running one instance.  If arr is int,
    then the elements are different array sizes, otherwise use the
    array.
    direction default - just fwd - use 'both' for both
    To "train": - the print allows you to ^c out.
    from pyfusion.utils.primefactors import get_fftw3_speed
    from pyfusion.utils.fftw3_bdb_utils import save_wisdom, load_wisdom
    from pyfusion.data.filters import next_nice_number
    from pyfusion.utils.primefactors import get_fftw3_speed
    for n in next_nice_number(None):
        print(n); 
        get_fftw3_speed(n, flags=['FFTW_MEASURE'],direction='both',dtype=np.float32)
    save_wisdom()

    Accepts all pyfftw.FFTW args e.g. planning_timelimit
    """
    import pyfftw
    from time import time as seconds

    if np.isscalar(arr):
        arr = np.array([arr])
    if np.issubdtype(arr.dtype, int):
        atimes = []
        for n in arr:
            atimes.append([n, get_fftw3_speed(np.ones(n, dtype=dtype), direction=direction, iters=iters, **kwargs)])
        return np.array(atimes)
    else:  # do one example
        build_kwargs = dict(flags=["FFTW_ESTIMATE"])
        build_kwargs.update(kwargs)
        simd_align = pyfftw.simd_alignment  # 16 at the moment.
        arr = pyfftw.n_byte_align(arr, simd_align)
        out = pyfftw.n_byte_align(np.ones(len(arr) / 2 + 1, dtype=np.complex64), simd_align)
        fwd = pyfftw.FFTW(arr, out, **build_kwargs)
        if direction == "both":
            rev = pyfftw.FFTW(out, arr, direction="FFTW_BACKWARD", **build_kwargs)
        st = seconds()
        for i in range(iters):
            fwd.execute()
            if direction == "both":
                rev.execute()
        return (seconds() - st) / iters
Exemple #15
0
    def wait(self):
        self.count += 1
        if self.maxcpu == 1:
            return ()

        avg_time = (seconds() - self.st - self.waited) / self.count
        self.last_wait = max(avg_time * (1 - self.maxcpu) / self.maxcpu, 0.01)
        print(avg_time, self.last_wait, self.count)
        self.waited += self.last_wait
        sleep(self.last_wait)
Exemple #16
0
    def wait(self):
        self.count += 1
        if self.maxcpu == 1:
            return()

        avg_time = (seconds() - self.st - self.waited)/self.count
        self.last_wait = max(avg_time*(1-self.maxcpu)/self.maxcpu,0.01)
        print(avg_time, self.last_wait, self.count)
        self.waited += self.last_wait
        sleep(self.last_wait)
Exemple #17
0
    def load(self, sel=None):
        start_mem = report_mem(msg="load")
        st = seconds()
        if sel is None:  # the arg overrides the object value
            sel = self.sel
        else:
            print("Overriding any decimation - selecting {s:,} instances".format(s=len(sel)))
            self.sel = sel

        if self.verbose > 0:
            print("loading {nm}".format(nm=self.name)),
        if self.loaded:
            if self.verbose:
                ("print {nm} already loaded".format(nm=self.name))

        else:
            dd = {}
            for k in self.da.keys():
                if sel is None:
                    dd.update({k: self.da[k]})
                else:  # selective (decimated to limit)
                    try:
                        dd.update({k: self.da[k][sel]})
                    except Exception as reason:
                        dd.update({k: self.da[k]})
                        print("{k} loaded in full: {reason}".format(k=k, reason=reason))
                # dictionaries get stored as an object, need "to list"
                debug_(self.debug, 2, key="limit")
                if hasattr(dd[k], "dtype"):
                    if dd[k].dtype == np.dtype("object"):
                        dd[k] = dd[k].tolist()
                        if self.verbose:
                            print("object conversion for {k}".format(k=k))
                    if hasattr(dd[k], "dtype") and dd[k].dtype == np.dtype("object"):
                        dd[k] = dd[k].tolist()
                        print("*** Second object conversion for {k}!!!".format(k=k))
        # key 'info' should be replaced by the more up-to-date self. copy
        self.da = dd
        self.update({"info": self.infodict}, check=False)
        if self.verbose:
            print(" in {dt:.1f} secs".format(dt=seconds() - st))
        report_mem(start_mem)
        return True
Exemple #18
0
    def __init__(self, maxcpu=0.5):
        self.maxcpu = maxcpu
        self.maxcpu = min(self.maxcpu,1)
        self.maxcpu = max(self.maxcpu,0.01)
        if self.maxcpu != maxcpu:
            print('changed maxcpu from {i} to {a}'.format(i=maxcpu, a=self.maxcpu))

        self.count = 0
        self.st = seconds()
        self.last_wait = 0
        self.waited = 0
Exemple #19
0
    def __init__(self, maxcpu=0.5):
        self.maxcpu = maxcpu
        self.maxcpu = min(self.maxcpu, 1)
        self.maxcpu = max(self.maxcpu, 0.01)
        if self.maxcpu != maxcpu:
            print('changed maxcpu from {i} to {a}'.format(i=maxcpu,
                                                          a=self.maxcpu))

        self.count = 0
        self.st = seconds()
        self.last_wait = 0
        self.waited = 0
Exemple #20
0
    def save(self, filename, verbose=None, use_dictionary=False):
        """ Save as an npz file, using an incremental method, which
        only uses as much /tmp space as required by each var at a time.
        if use_dictionary is a valid dictionary, save the values of
        ANY AND ONLY the LOCAL variables whose names are in the keys for
        this set.
        So if you have extracted a subset, and you specify 
        use_dictionary=locals(),
        only that subset is saved (both in array length, and variables chosen).
        Beware locals that are not your variables - e.g. mtrand.beta
        """
        if verbose == None: verbose = self.verbose
        st = seconds()
        if use_dictionary == False: 
            save_dict = self.da # the dict used to get data
        else: 
            save_dict = use_dictionary
            print('Warning - saving only a subset')


        use_keys = []
        for k in self.da.keys():
            if k in save_dict.keys():
                use_keys.append(k)

        if verbose: print(' Saving only {k}'.format(k=use_keys))


        args=','.join(["{k}=save_dict['{k}']".
                       format(k=k) for k in use_keys])
        if verbose:
             print('lengths: {0} -999 indicates dodgy variable'
                   .format([mylen(save_dict[k]) for k in use_keys]))

        exec("np.savez_compressed(filename,"+args+")")
        self.name = filename

        if verbose: print(' in {dt:.1f} secs'.format(dt=seconds()-st))
Exemple #21
0
    def report_mem(prev_values=None,msg=None):
        """ Show status of phy/virt mem, and if given previous values, 
        report differences
        requires the psutil module - avail in apt-get, but I used pip
        """
        if msg is None: msg=''
        else: msg += ': '
        pm = psutil.avail_phymem()
        vm = psutil.avail_virtmem()
        tim = seconds()
        print('{msg}{pm:.2g} GB phys mem, {vm:.2g} GB virt mem avail'
              .format(msg=msg, pm=pm/1e9, vm=vm/1e9)),

        if prev_values is None:
            print
        else:
            print('- dt={dt:.2g}s, used {pm:.2g} GB phys, {vm:.2g} GB virt'
                  .format(pm=(prev_values[0] - pm)/1e9,
                          vm=(prev_values[1] - vm)/1e9, dt = tim-prev_values[2]))
        return((pm,vm,tim))
Exemple #22
0
    def report_mem(prev_values=None, msg=None):
        """ Show status of phy/virt mem, and if given previous values, 
        report differences
        requires the psutil module - avail in apt-get, but I used pip
        """
        if msg is None: msg = ''
        else: msg += ': '
        pm = psutil.avail_phymem()
        vm = psutil.avail_virtmem()
        tim = seconds()
        print('{msg}{pm:.2g} GB phys mem, {vm:.2g} GB virt mem avail'.format(
            msg=msg, pm=pm / 1e9, vm=vm / 1e9)),

        if prev_values is None:
            print
        else:
            print('- dt={dt:.2g}s, used {pm:.2g} GB phys, {vm:.2g} GB virt'.
                  format(pm=(prev_values[0] - pm) / 1e9,
                         vm=(prev_values[1] - vm) / 1e9,
                         dt=tim - prev_values[2]))
        return ((pm, vm, tim))
Exemple #23
0
  def store_index(self, time, index) :
      """Store information about:
         * Time-satmp
         * Total intensity
         * Beam center
         * Estimated Particle Size
         * Estimated particle nr

      """

      self.tot_t[index]         = time.time()
      self.tot_s[index]         = time.seconds()
      self.tot_ns[index]        = time.nanoseconds()
      self.tot_fd[index]        = time.fiducial()

      self.tot_int[index]       = float(self.img.sum())
      self.tot_cx[index]        = self.cent[0]
      self.tot_cy[index]        = self.cent[1]
      self.tot_size[index]      = self.radius
      self.tot_score[index]     = self.score

      self.ave                 += self.img
Exemple #24
0
  def store_index(self, time, index, flag = 1) :
      """Store information about:
         * Time-stamp
         * Total intensity
         * Beam center
         * Estimated Particle Size
         * Estimated particle nr

      """

      if not hasattr(self, 'streak_m'):
         self.streak_m    = 0
      if not hasattr(self, 'streak_s'):
         self.streak_s    = 0


      self.tot_t[index]         = time.time()
      self.tot_s[index]         = time.seconds()
      self.tot_ns[index]        = time.nanoseconds()
      self.tot_fd[index]        = time.fiducial()

      self.tot_int[index]       = float(self.img.sum())

      if (self.peak is not None):
          self.tot_peak1_int[index] = self.peak1
          self.tot_peak2_int[index] = self.peak2

      self.tot_streak_m[index]  = self.streak_m
      self.tot_streak_s[index]  = self.streak_s

      self.tot_cx[index]        = self.cent[0]
      self.tot_cy[index]        = self.cent[1]
      self.tot_size[index]      = self.radius
      self.tot_score[index]     = self.score

      if flag :
         self.ave                  += self.img
Exemple #25
0
  def store_index(self, time, index, flag = 1) :
      """Store information about:
         * Time-stamp
         * Total intensity
         * Beam center
         * Estimated Particle Size
         * Estimated particle nr

      """

      if not hasattr(self, 'streak_m'):
         self.streak_m    = 0
      if not hasattr(self, 'streak_s'):
         self.streak_s    = 0


      self.tot_t[index]         = time.time()
      self.tot_s[index]         = time.seconds()
      self.tot_ns[index]        = time.nanoseconds()
      self.tot_fd[index]        = time.fiducial()

      self.tot_int[index]       = float(self.img.sum())

      if (self.peak is not None):
          self.tot_peak1_int[index] = self.peak1
          self.tot_peak2_int[index] = self.peak2

      self.tot_streak_m[index]  = self.streak_m
      self.tot_streak_s[index]  = self.streak_s

      self.tot_cx[index]        = self.cent[0]
      self.tot_cy[index]        = self.cent[1]
      self.tot_size[index]      = self.radius
      self.tot_score[index]     = self.score

      if flag :
         self.ave                  += self.img
Exemple #26
0
    def store_index2(self, time, index, flag=1):
        """Store information about:
         * Time-stamp
         * Total intensity
         * Beam center
         * Estimated Particle Size
         * Estimated particle nr

      """

        self.tot_t[index] = time.time()
        self.tot_s[index] = time.seconds()
        self.tot_ns[index] = time.nanoseconds()
        self.tot_fd[index] = time.fiducial()

        self.tot_int[index] = float(self.image.sum())

        self.tot_cx[index] = self.cent[0]
        self.tot_cy[index] = self.cent[1]
        self.tot_size[index] = self.radius
        self.tot_score[index] = self.score

        if flag:
            self.ave += self.image
Exemple #27
0
    def do_fetch(self):
        # Definitions:
        # data_utc: is meant to be the first and last timestamps of the saved
        #        or the retrieved data if not saved. (at least from 3 march 2016)
        # shot_f, shot_t: are beginning and end of shot (from programs) at least from 3 march 2016
        # utc: seems to be same as data_utc
        # seg_f_u: up til 20 march 2020, seg_f_u appears to be the *requested* segment, (now fixed - renamed to req_f_u)
        # utc0 - only 9.9 or so.  Should be in file or set in base.py

        # In this revision, the only changes are - allow for self.time_range,
        #   variable names f, t changed to f_u, t_u (the desired data utc)
        # and comment fixes,
        # in preparation for including time_range and other cleanups.

        # My W7X shots are either of the form from_utc, to_utc,
        #  or date (8dig) and shot (progId)
        # the format is in the acquisition properties, to avoid
        # repetition in each individual diagnostic

        t_start = seconds()
        if self.shot[1] > 1e9 or hasattr(
                self, 'time_range') and self.time_range is not None:
            # we have start and end in UTC instead of shot no
            # need the shot and utc to get t1 to set zero t
            if hasattr(self, 'time_range') and self.time_range is not None:
                if self.shot[1] > 1e9:
                    raise ValueError(
                        "Can't supply shot as a utc pair and specify a time_range"
                    )
                actual_shot = self.shot
                f_u = None  # set to None to make sure we don't use it
            else:
                f_u, t_u = self.shot  #  Initialize to the requested utc range
                actual_shot = get_shot_number([f_u, t_u])

            progs = get_programs(actual_shot)  # need shot to look up progs
            #  need prog to get trigger - not tested for OP1.1
            if len(progs) > 1:
                raise LookupError(
                    'fetch: too many programs found - covers > 1 shot?')
                #this_prog = [prog for prog in progs if (f_u >= progs[prog]['from'] and
                #                                    t_u <= progs[prog]['upto'])]
            if len(progs) == 1:
                this_prog = progs.values()[0]
                # on shot 20180724,10, this trigger[1] is an empty list
                trigger = this_prog['trigger']['1']
                # This fallback to trigger[0] mean that more rubbish shots are saved than
                # if we only look at the proper trigger (1) - here is an example
                # run pyfusion/examples/plot_signals shot_number=[20180724,10] diag_name="W7X_UTDU_LP15_I" dev_name='W7X'
                if len(trigger) == 0:  # example above
                    print('** No Trigger 1 on shot {s}'.format(s=actual_shot))
                    debug_(pyfusion.DEBUG,
                           0,
                           key="noTrig1",
                           msg="No Trigger 1 found")
                    # take any that exist, at 60
                    trigger = [
                        trr[0] + int(60 * 1e9)
                        for trr in this_prog['trigger'].values()
                        if len(trr) > 0
                    ]
                    if len(trigger) == 0:
                        raise LookupError(
                            'No Triggers at all on shot {s}'.format(
                                s=actual_shot))
                utc_0 = trigger[
                    0]  # utc_0 is the first trigger (usu 61 sec ahead of data)
            else:
                print(
                    'Unable to look up programs - assuming this is a test shot'
                )
                utc_0 = f_u  #   better than nothing - probably a 'private' test/calibration shot
            if f_u is None:  # shorthand for have time range
                f_u = utc_0 + int(1e9 * (self.time_range[0]))  # + 61))
                t_u = utc_0 + int(
                    1e9 *
                    (self.time_range[1]))  # + 61)) was 61 rel to prog['from']

        else:  # self.shot is an 8,3 digit shot and time range not specified
            actual_shot = self.shot
            f_u, t_u = get_shot_utc(
                actual_shot
            )  # f_u is the start of the overall shot - i.e about plasma time -61 sec.
            # at present, ECH usually starts 61 secs after t0
            # so it is usually sufficient to request a later start than t0
            pre_trig_secs = self.pre_trig_secs if hasattr(
                self, 'pre_trig_secs') else 0.3
            # should get this from programs really - code is already above. We need to just move it.
            pyfusion.utils.warn('fetch: still using hard-coded 61 secs')
            utc_0 = f_u + int(1e9 * (61))  # utc_0 is plasma initiation in utc
            f_u = utc_0 - int(
                1e9 * pre_trig_secs)  # f_u is the first time wanted

        # make sure we have the following defined regardless of how we got here
        shot_f_u, shot_t_u = get_shot_utc(actual_shot)
        req_f_u = f_u  # req_f_u is the start of the desired data segment - sim. for req_t_u
        req_t_u = t_u
        # A URL STYLE diagnostic - used for a one-off rather than an array
        # this could be moved to setup so that the error info is more complete
        if hasattr(self, 'url'):
            fmt = self.url  # add from= further down: +'_signal.json?from={req_f_u}&upto={req_t_u}'
            params = {}
            # check consistency -   # url should be literal - no params (only for fmt) - gain, units are OK as they are not params
            if hasattr(self, 'params'):
                pyfusion.utils.warn(
                    'URL diagnostic {n} should not have params <{p}>'.format(
                        n=self.config_name, p=self.params))
        else:  # a pattern-based one - used for arrays of probes
            if hasattr(self, 'fmt'):  # does the diagnostic have one?
                fmt = self.fmt
            elif hasattr(self.acq, 'fmt'):  # else use the acq.fmt
                fmt = self.acq.fmt
            else:  # so far we have no quick way to check the server is online
                raise LookupError('no fmt - perhaps pyfusion.cfg has been '
                                  'edited because the url is not available')

            params = eval('dict(' + self.params + ')')

        # Originally added to fix up erroneous ECH alias mapping if ECH - only
        #   6 sources work if I don't.  But it seems to help with many others
        # This implementation is kludgey but proves the principle, and
        #   means we don't have to refer to any raw.. signals
        #   would be nice if they made a formal way to do this.
        if 'upto' not in fmt:
            fmt += '_signal.json?from={req_f_u}&upto={req_t_u}'

        assert req_f_u == f_u, 'req_f_u error'
        assert req_t_u == t_u, 'req_t_u error'  #
        params.update(req_f_u=req_f_u, req_t_u=req_t_u, shot_f_u=shot_f_u)
        url = fmt.format(**params)  # substitute the channel params

        debug_(pyfusion.DEBUG, 2, key="url", msg="middle of work on urls")
        if np.any([
                nm in url for nm in
                'Rf,Tower5,MainCoils,ControlCoils,TrimCoils,Mirnov,Interfer,_NBI_'
                .split(',')
        ]):
            from pyfusion.acquisition.W7X.get_url_parms import get_signal_url
            # replace the main middle bit with the expanded one from the GUI
            tgt = url.split('KKS/')[1].split('/scaled')[0].split('_signal')[0]
            # construct a time filter for the shot
            self.tgt = tgt  # for the sake of error_info
            filt = '?filterstart={req_f_u}&filterstop={req_t_u}'.format(
                **params)
            # get the url with the filter
            url = url.replace(tgt, get_signal_url(tgt, filt)).split('KKS/')[-1]
            # take the filter back out - we will add the exact one later
            url = url.replace(filt, '/')

        # nSamples now needs a reduction mechanism http://archive-webapi.ipp-hgw.mpg.de/
        # minmax is increasingly slow for nSamples>10k, 100k hopeless
        # Should ignore the test comparing the first two elements of the tb
        # prevent reduction (NSAMPLES=...) to avoid the bug presently in codac
        if (('nSamples' not in url) and (pyfusion.NSAMPLES != 0)
                and not (hasattr(self, 'allow_reduction')
                         and int(self.allow_reduction) == 0)):
            url += '&reduction=minmax&nSamples={ns}'.format(
                ns=pyfusion.NSAMPLES)

        debug_(pyfusion.DEBUG, 2, key="url", msg="work on urls")
        # we need %% in pyfusion.cfg to keep py3 happy
        # however with the new get_signal_url, this will all disappear
        if sys.version < '3.0.0' and '%%' in url:
            url = url.replace('%%', '%')

        if 'StationDesc.82' in url:  # fix spike bug in scaled QRP data
            url = url.replace('/scaled/', '/unscaled/')

        if pyfusion.CACHE:
            # Needed for improperly configured cygwin systems: e.g.IPP Virual PC
            # Perhaps this should be executed at startup of pyfusion?
            cygbin = "c:\\cygwin\\bin"
            if os.path.exists(cygbin) and not cygbin in os.environ['path']:
                os.environ['path'] += ";" + cygbin
            print('using wget on {url}'.format(url=url))
            retcode = os.system('wget -x "{url}"'.format(url=url))
            #  retcode = os.system('c:\\cygwin\\bin\\bash.exe -c "/bin/wget {url}"'.format(url=url))
            debug_(retcode != 0,
                   level=1,
                   key='wget',
                   msg="wget error or DEBUG='wget'")
            # now read from the local copy - seems like urls need full paths
            # appears to be a feature! http://stackoverflow.com/questions/7857416/file-uri-scheme-and-relative-files
            # /home/bdb112/pyfusion/working/pyfusion/archive-webapi.ipp-hgw.mpg.de/ArchiveDB/codac/W7X/CoDaStationDesc.82/DataModuleDesc.181_DATASTREAM/7/Channel_7/scaled/_signal.json?from=1457626020000000000&upto=1457626080000000000&nSamples=10000
            # url = url.replace('http://','file:///home/bdb112/pyfusion/working/pyfusion/')
            url = url.replace('http://', 'file:/' + os.getcwd() + '/')
            if 'win' in os.sys.platform:
                # weven thoug it seems odd, want 'file:/c:\\cygwin\\home\\bobl\\pyfusion\\working\\pyfusion/archive-webapi.ipp-hgw.mpg.de/ArchiveDB/codac/W7X/CoDaStationDesc.82/DataModuleDesc.192_DATASTREAM/4/Channel_4/scaled/_signal.json@from=147516863807215960&upto=1457516863809815961'
                url = url.replace('?', '@')
            # nicer replace - readback still fails in Win, untested on unix systems
            print('now trying the cached copy we just grabbed: {url}'.format(
                url=url))
        if (req_f_u > shot_t_u) or (req_t_u < shot_f_u):
            pyfusion.utils.warn(
                'segment requested is outside the shot times for ' +
                str(actual_shot))
        if pyfusion.VERBOSE > 0:
            print(
                '======== fetching url over {dt:.1f} secs from {fr:.1f} to {tt:.1f} =========\n[{u}]'
                .format(u=url,
                        dt=(params['req_t_u'] - params['req_f_u']) / 1e9,
                        fr=(params['req_f_u'] - shot_f_u) / 1e9,
                        tt=(params['req_t_u'] - shot_f_u) / 1e9))

        # seems to take twice as long as timeout requested.
        # haven't thought about python3 for the json stuff yet
        # This is not clean - should loop for timeout in [pyfusion.TIMEOUT, 3*pyfusion.TIMEOUT]
        try:
            # dat = json.load(urlopen(url,timeout=pyfusion.TIMEOUT)) works
            # but follow example in
            #    http://webservices.ipp-hgw.mpg.de/docs/howtoREST.html#python
            #  Some extracts in examples/howtoREST.py
            # dat = json.loads(urlopen(url,timeout=pyfusion.TIMEOUT).read().decode('utf-8'))
            t_pre = seconds()
            # for long shots, adjust strategy and timeout to reduce memory consumption
            ONE = 4  #  memory conservation tricks only apply for DEBUG<1
            # Thin allows the cutoff value to be increased in come cases
            # uncomment the following two for testing the exception handler
            ## timeout = pyfusion.TIMEOUT
            ## raise  httplib.IncompleteRead('test')
            if (req_t_u - req_f_u) / 1e9 > pyfusion.VERY_LONG:
                size_MS = 2 * (
                    req_t_u - req_f_u
                ) / 1e9  # approximate - later on calc from dt i.e. MSamples
                if pyfusion.NSAMPLES != 0:  # allow for subsampled data
                    size_MS = pyfusion.NSAMPLES / 1e6
                timeout = 8 * size_MS + pyfusion.TIMEOUT  #  don't make timeout too small!
                print('On-the-fly conversion: Setting timeout to {tmo}'.format(
                    tmo=timeout))
                dat = json.load(urlopen(url, timeout=timeout))
                t_text = seconds()
            else:
                timeout = pyfusion.TIMEOUT
                txtform = urlopen(url, timeout=timeout).read()
                t_text = seconds()
                print('{tm} {tp:.2f} prep, {tt:.2f} fetch without decode, '.
                      format(tm=time.strftime('%H:%M:%S'),
                             tp=t_pre - t_start,
                             tt=t_text - t_pre)),
                sys.stdout.flush()
                dat = json.loads(txtform.decode('utf-8'))
                if pyfusion.DEBUG < ONE:
                    txtform = None  # release memory
            t_conv = seconds()
            #  for 10MS of mirnov 0.06 prep, 9.61 fetch 19.03 conv
            #print('{tp:.2f} prep, {tt:.2f} fetch {tc:.2f} conv'.
            print('{tc:.2f} conv'.format(tp=t_pre - t_start,
                                         tt=t_text - t_pre,
                                         tc=t_conv - t_text))
        except socket.timeout as reason:  #  the following url access is a slightly different form?
            # should check if this is better tested by the URL module
            print('{tm} {tp:.2f} prep, {tt:.2f} timeout. '.format(
                tp=t_pre - t_start,
                tt=seconds() - t_pre,
                tm=time.strftime('%H:%M:%S'))),
            print(
                '****** first timeout error, try again with longer timeout  *****'
            )
            timeout *= 3
            dat = json.load(urlopen(url, timeout=timeout))
        except MemoryError as reason:
            raise  # too dangerous to do anything else except to reraise
        except httplib.IncompleteRead as reason:
            msg = str(
                '** IncompleteRead after {tinc:.0f}/{timeout:.0f}s ** on {c}: {u} \n{r}'
                .format(tinc=seconds() - t_start,
                        c=self.config_name,
                        u=url,
                        r=reason,
                        timeout=timeout))
            pyfusion.logging.error(
                msg
            )  # don't want to disturb the original exception, so raise <nothing> i.e. reraise
            raise  # possibly a memory error really? - not the case for 4114 20180912.48
        except Exception as reason:
            if pyfusion.VERBOSE >= 0:
                print(
                    '**** Exception (Memory? out of disk space?) OR timeout of {timeout} **** on {c}: {u} \n{r}'
                    .format(c=self.config_name,
                            u=url,
                            r=reason,
                            timeout=timeout))

            raise  # re raises the last exception

        # this form will default to repair = 2 for all LP probes.
        #default_repair = -1 if 'Desc.82/' in url else 0
        # Override acq.repair with the probe value
        default_repair = int(self.repair) if hasattr(
            self, 'repair') else 2 if 'Desc.82/' in url else 0
        # this form follows the config file settings
        self.repair = int(self.repair) if hasattr(self,
                                                  'repair') else default_repair
        dimraw = np.array(dat['dimensions'])
        if ('nSamples' not in url):  # skip this check if we are decimating
            if np.abs(req_f_u - dimraw[0]) > 2000:
                print(
                    '** Start is delayed by >2 us {dtdel:,} relative to the request'
                    .format(dtdel=dimraw[0] - req_f_u))
            if (req_t_u - dimraw[-1]) > 2000:
                print(
                    '** End is earlier by >2 us {dtdel:,} relative to the request'
                    .format(dtdel=req_t_u - dimraw[-1]))

        output_data_utc = [dat['dimensions'][0], dat['dimensions'][-1]]
        if pyfusion.DEBUG < ONE:
            dat['dimensions'] = None  # release memory
        # adjust dim only (not dim_raw so that zero time remains at t1
        dim = dimraw - utc_0
        # decimation with NSAMPLES will make the timebase look wrong - so disable repair
        if pyfusion.NSAMPLES != 0 or self.repair == 0 or self.repair == -1:
            pass  # leave as is
        # need at least this clipping for Langmuir probes in Op1.1
        elif self.repair == 1:
            dim = np.clip(dim, 0, 1e99)
        elif self.repair == 2:
            dim, msg = regenerate_dim(dim)
            if msg is not None:
                print('shot {s}, {c}: {m}'.format(s=self.shot,
                                                  c=self.config_name,
                                                  m=msg))
        else:
            raise ValueError(
                'repair value of {r} not understood'.format(r=self.repair))

        if pyfusion.VERBOSE > 2: print('repair', self.repair)
        #ch = Channel(self.config_name,  Coords('dummy', (0,0,0)))
        # this probably should be in base.py
        coords = get_coords_for_channel(**self.__dict__)
        # used to be bare_chan? should we include - signs?
        ch = Channel(self.config_name, coords)
        scl = 1 / 3277.24 if dat['datatype'].lower() == 'short' else 1
        if self.repair == -1:
            output_data = TimeseriesData(timebase=Timebase(dimraw),
                                         signal=scl * Signal(dat['values']),
                                         channels=ch)
        else:
            output_data = TimeseriesData(timebase=Timebase(1e-9 * dim),
                                         signal=scl * Signal(dat['values']),
                                         channels=ch)
        output_data.meta.update({'shot': self.shot})
        output_data.utc = output_data_utc  #  this copy was saved earlier so we could delete the large array to save space
        output_data.units = dat['units'] if 'units' in dat else ''
        # this is a minor duplication - at least it gets saved via params
        params['data_utc'] = output_data.utc
        params['utc_0'] = utc_0  # hopefully t0 -- useful in npz files
        # Warning - this could slow things down! - but allows
        # corrupted time to be re-calculated as algorithms improve.
        # and storage as differences takes very little space.
        params['diff_dimraw'] = dimraw
        params['diff_dimraw'][1:] = np.diff(dimraw)
        if pyfusion.DEBUG < ONE:
            dimraw = None
        # NOTE!!! need float128 to process dimraw, and cumsum won't return ints
        # or automatically promote to 128bit (neither do simple ops like *, +)
        params['pyfusion_version'] = pyfusion.version.get_version()
        if pyfusion.VERBOSE > 0:
            print('shot {s}, config name {c}'.format(c=self.config_name,
                                                     s=self.shot))

        output_data.config_name = self.config_name
        debug_(pyfusion.DEBUG, 2, key='W7XDataFetcher')
        output_data.params = params

        ###  the total shot utc.  output_data.utc = [f_u, t_u]
        return output_data
Exemple #28
0
        print ("Example\n>>> conn.execute('select * from summ order by shot desc limit 1').fetchone()")
        sys.exit()

import MDSplus as MDS

shots = 0

# set these both to () to stop on errors
shot_exception = Exception  # () to see message - catches and displays all errors
node_exception = Exception  # ()  ditto for nodes.

errs = dict(shot=[])  # error list for the shot overall (i.e. if tree is not found)
for diag in diags.keys():
    errs.update({diag: []})  # error list for each diagnostic

start = seconds()
for s in srange:  # on t440p (83808,86450): # FY14-15 (86451,89155):#(36363,88891): #81399,81402):  #(81600,84084):
    datdic = dict(shot=s)
    shots += 1
    try:
        if (s % 10) == 0:
            print (s),
        else:
            print ("."),
        if (s % 100) == 0:
            print ("")
        tree = MDS.Tree("h1data", s)
        non_special = list(diags.keys())
        non_special.remove("mirnov_coh")
        for diag in non_special:
            try:
Exemple #29
0
            runfile = tempfile.mktemp()
            env = os.environ
            # env.update({'PYTHONSTARTUP':tmpfil}) only works in interactive
            with open(filename, 'rt') as pf:
                prog = pf.readlines()
            with open(runfile, 'wt') as tf:
                tf.write(prerun+'\n')
                tf.writelines(prog)
        else:
            env = None
            runfile = filename

        cmd = '{py} {file} {args}'.format(file=runfile, py=python_exe, args=args)
        print(cmd)

        st = seconds()
        sub_pipe = subprocess.Popen(cmd,  env=env, shell=True, stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE)
        """ Doesn't workthis way - seems to wait.
        for t in range(max_sec):
            print(t)
            sleep(1)
            if sub_pipe.poll():
                break
            if t == max_sec - 1:
                print('terminate')
                sub_pipe.kill()
        """
        (resp, errout) = sub_pipe.communicate()
        print(errout)
        have_error = (((errout != b'')
Exemple #30
0
shots = 0

# set these both to () to stop on errors
shot_exception = () # Exception # () to see message - catches and displays all errors
node_exception = () if debug > 0 else Exception # ()  ditto for nodes.

errs = dict(shot=[])  # error list for the shot overall (i.e. if tree is not found)
for diag in diags.keys():
    errs.update({diag:[]})  # error list for each diagnostic

print("""If off-line, set pyfusion.TIMEOUT=0 to prevent long delays
The search for valid shot numbers is now much quicker
""")
# will take a while (~ 0.7 sec/day?),  about 2 mins as of 10/2017  

start = seconds()
dev = pyfusion.getDevice(devname)  # 'H1Local')
if 'W7' in devname:
    if (len(np.shape(srange)) == 3) or (np.shape(srange) == (2,2)):
        ansexp = input('About to expand shot range: Continue?  (Y/n/q)')
        if len(ans)==0 or ans.lower()[0] != 'y':
            sys.exit()
    
    pyfusion.logging.info(str('Starting with {st}, shape is {shp}'
                              .format(st=srangestr, shp=np.shape(srange))))
    if len(np.shape(srange)) == 3: # a bunch of begin/ends
        srange = [sh for sr in srange for sh in expand_shot_range(*sr)]
    elif np.shape(srange) == (2,2):
        print('assume a simple range')
        srange = expand_shot_range(*srange)
    else: # already a list
Exemple #31
0
            print('changed maxcpu from {i} to {a}'.format(i=maxcpu,
                                                          a=self.maxcpu))

        self.count = 0
        self.st = seconds()
        self.last_wait = 0
        self.waited = 0

    def wait(self):
        self.count += 1
        if self.maxcpu == 1:
            return ()

        avg_time = (seconds() - self.st - self.waited) / self.count
        self.last_wait = max(avg_time * (1 - self.maxcpu) / self.maxcpu, 0.01)
        print(avg_time, self.last_wait, self.count)
        self.waited += self.last_wait
        sleep(self.last_wait)


if __name__ == '__main__':
    from time import time as seconds
    st = seconds()
    regulator = Regulator(1)
    for i in range(10000):
        regulator.wait()

    print('{nits} iterations with 100% CPU -> overhead of {dt:.2f}us per call'.
          format(nits=regulator.count,
                 dt=1e6 * (seconds() - regulator.st) / regulator.count))
Exemple #32
0
        print("Example\n>>> conn.execute('select * from summ order by shot desc limit 1').fetchone()")
        sys.exit()
shots = 0

# set these both to () to stop on errors
shot_exception = () # Exception # () to see message - catches and displays all errors
node_exception = () # Exception # ()  ditto for nodes.

errs = dict(shot=[])  # error list for the shot overall (i.e. if tree is not found)
for diag in diags.keys():
    errs.update({diag:[]})  # error list for each diagnostic

print('If off-line, set pyfusion.TIMEOUT=0 to prevent long delays')


start = seconds()
dev=pyfusion.getDevice(devname)  # 'H1Local')
from pyfusion.data.shot_range import shot_range as expand_shot_range
srange = ((20160309,1), (20160310,99))
srange = ((20160101,1), (20160310,99))
#srange = ((20160202,1), (20160202,99))
if 'W7' in devname:
    srange = expand_shot_range(*srange)
else:
    srange=range(92000,95948)

for sh in srange: # on t440p (83808,86450): # FY14-15 (86451,89155):#(36363,88891): #81399,81402):  #(81600,84084):
    if 'W7' in devname:
        datdic = dict(shot=sh[0]*1000+sh[1], date=sh[0], sshot=sh[1])
        if (sh[1] % 10) == 0:
            print(sh),
Exemple #33
0
exec(_var_defaults)
exec(process_cmd_line_args())

filters = tb.Filters(complevel=complevel, complib=complib)

dd = DA(DAfilename).da

if outfilename is None:
    (base, ext) = os.path.splitext(os.path.realpath(DAfilename))
    outfilename = base + os.path.extsep + 'h5'

outf = tb.openFile(outfilename, "a")

for var in dd.keys():

    st_copy = seconds()
    if var in [nd.name for nd in outf.listNodes('/')]:
        raise LookupError('{f} already has a node "{n}"'.format(
            f=outf.filename, n=var))
    val = dd[
        var]  # need to hold it in memory this way to avoid multiple access
    sizebytes = val.nbytes
    print('{dt:.1f}s to read {v} {GB:.2f} GB for {f}'.format(
        dt=seconds() - st_copy,
        GB=sizebytes / 1e9,
        f=os.path.split(outfilename)[-1],
        v=var))

    st_write = seconds()

    try:
Exemple #34
0
    def test_pcubed(self):
        import matplotlib.pyplot as plt
        from matplotlib import gridspec
        from pyonline.percentiles import Percentiles, PCubed
        from time import time as seconds

        n = 10000000
        random_state = np.random.RandomState(10)
        #X = random_state.exponential(0.00000000000001, size=(n,))
        X = random_state.normal(0, 1, size=(n, ))

        n_markers = 20

        def get_psquared():
            return Percentiles(n_markers=n_markers)

        def get_pcubed():
            return PCubed(n_markers=n_markers)

        # JIT compile the code
        get_psquared().partial_fit(X[:n_markers]).transform(X[:n_markers],
                                                            cubic_interp=True)
        get_pcubed().partial_fit(X[:n_markers]).transform(X[:n_markers],
                                                          cubic_interp=True)

        psquared = get_psquared()
        pcubed = get_pcubed()

        start_seconds = seconds()
        pcubed_percentiles = pcubed.partial_fit(X).transform(X,
                                                             cubic_interp=True)
        stop_seconds = seconds()

        start_seconds = seconds()
        percentiles = pcubed.partial_fit(X).transform(X, cubic_interp=True)
        stop_seconds = seconds()
        pcubed_seconds = stop_seconds - start_seconds

        print('P-Cubed in %.2f seconds!' % pcubed_seconds)

        true_percentiles = pd.Series(X).rank() / X.shape[0]
        marker_percentiles = to_percentile(percentile_model.markers)

        gs = gridspec.GridSpec(2, 2)
        top_left = plt.subplot(gs[0, :1])
        bottom_left = plt.subplot(gs[1, :1])
        right = plt.subplot(gs[:, 1:])

        plot_n = 10000
        plot_X = X[:plot_n]

        pd.DataFrame(
            dict(Actual=plot_X,
                 PSquare=percentile_model.inverse_transform(
                     true_percentiles.values[:plot_n]),
                 PCubed=percentile_model.inverse_transform(
                     true_percentiles.values[:plot_n], cubic_interp=True)),
            index=true_percentiles.values[:plot_n],
        ).sort_index().plot(ax=top_left, title="n = %d" % n, colormap='Paired')

        for marker in marker_percentiles:
            top_left.axvline(marker, alpha=0.5, color='grey')
            bottom_left.axvline(marker, alpha=0.5, color='grey')

        top_left.scatter(marker_percentiles, percentile_model.quantiles)

        if hasattr(percentile_model, 'curvature'):

            print(percentile_model.curvature)
            pd.DataFrame(dict(StressLeft=percentile_model.curvature[:-1],
                              StressRight=percentile_model.curvature[1:]),
                         index=marker_percentiles[1:-1]).plot(ax=bottom_left)

        pd.DataFrame(
            dict(Actual=true_percentiles[:10000],
                 Estimated=percentiles[:10000])).plot.scatter(ax=right,
                                                              x='Actual',
                                                              y='Estimated',
                                                              figsize=(9, 7))

        fig = plt.gcf()
        fig.set_size_inches(24, 12)
        plt.tight_layout()
        plt.show()

        self.assertTrue(True)
Exemple #35
0
def read_text_pyfusion(files, target=b'^Shot .*', ph_dtype=None, plot=pl.isinteractive(), ms=100, hold=0, debug=0, quiet=1,  maxcpu=1, exception = Exception):
    """ Accepts a file or a list of files, returns a list of structured arrays
    See merge ds_list to merge and convert types (float -> pyfusion.prec_med
    """
    regulator = pyfusion.utils.Regulator(maxcpu)
    st = seconds(); last_update=seconds()
    file_list = files
    if len(np.shape(files)) == 0: file_list = [file_list]
    f='f8'
    if ph_dtype is None: ph_dtype = [('p12',f),('p23',f),('p34',f),('p45',f),('p56',f)]
    #ph_dtype = [('p12',f)]
    ds_list =[]
    comment_list =[]
    count = 0
    for (i,filename) in enumerate(file_list):
        regulator.wait()
        if seconds() - last_update > 10:
            last_update = seconds()
            tot = len(file_list)
            print('read {n}/{t}: ETA {m:.1f}m {f}'
                  .format(f=filename, n=i, t=tot,
                          m=(seconds()-st)*(tot-i)/float(60*i)))

        try:
            if (isinstance(target,str) or isinstance(target,bytes)): 
                skip = 1+find_data(filename, target, debug=debug)
            elif isinstance(target, int): 
                skip = target
            else:
                raise Exception('target ({target}) is not recognised'.format(target=target))
            if quiet == 0:
                print('{t:.1f} sec, loading data from line {s} of {f}'
                      .format(t = seconds()-st, s=skip, f=filename))
            #  this little bit to determine layout of data
            # very inefficient to read twice, but in a hurry!
            if debug>2: print('skiprows = \n', skip-1)
            txt = np.loadtxt(fname=filename, skiprows=skip-1, dtype=bytes, 
                             delimiter='FOOBARWOOBAR',ndmin=1)
            header_toks = txt[0].split()
            # look for a version number first
            if header_toks[-1][-1] in b'0123456789.':
                version = float(header_toks.pop())
                if b'ersion' not in header_toks.pop():
                    raise ValueError('Error reading header in {f}'
                                     .format(f=filename))
            else: version=-1  # pre Aug 12 2013
            # noticed that the offset moved in 2015 - when did it  happen?
            phase_offs = -4 if sys.version>'3,' else -2
            # is the first character of the 2nd last a digit?
            if header_toks[phase_offs][0] in b'0123456789': 
                if pyfusion.VERBOSE > 0: 
                    print('header toks', header_toks)
                    print('found new header including number of phases')
                n_phases = int(header_toks[phase_offs])
                ph_dtype = [('p{n}{np1}'.format(n=n,np1=n+1), f) for n in range(n_phases)]
                
            if 'frlow' in header_toks:  # add the two extra fields
                fs_dtype= [ ('shot','i8'), ('t_mid','f8'), 
                            ('_binary_svs','u8'),    # f16 - really want u8 here,  but npyio 
                                                      #has problem converting 10000000000000000000000000
                                                      #OverflowError: Python int too large to convert to C long
                                                      # doesn't happen if text is read in directly with loadtxt
                            ('freq','f8'), ('amp', 'f8'), ('a12','f8'),
                            ('p', 'f8'), ('H','f8'), 
                            ('frlow','f8'), ('frhigh', 'f8'),('phases',ph_dtype)]
            else:
                fs_dtype= [ ('shot','i8'), ('t_mid','f8'), 
                            ('_binary_svs','u8'), 
                            ('freq','f8'), ('amp', 'f8'), ('a12','f8'),
                            ('p', 'f8'), ('H','f8'), ('phases',ph_dtype)]

            if version > 0.69:  # don't rely on precision
                fs_dtype.insert(-1,('cpkf', 'f8'))  # -1 is 1 before the end
                fs_dtype.insert(-1,('fpkf', 'f8'))  # they appear in this order
                
            if pyfusion.VERBOSE > 0: 
                print(version, fs_dtype, '\n')

            ds = np.loadtxt(fname=filename, skiprows = skip, 
                            dtype= fs_dtype, ndmin=1)  # ENSURE a 1D array

            if len(ds) > 0:
                ds_list.append(ds)
                count += 1
                # npz reads in python 2 can't cope with unicode - don't report errors unless really debugging
                comment_list.append(filename.encode(errors=['ignore','strict'][pyfusion.DBG() > 5]))
            else:
                print('no data in {f}'.format(f=filename))

        except ValueError as reason:
            print('Conversion error while processing {f} with loadtxt - {reason} {args}'
                  .format(f=filename, reason=reason, args=reason.args))
            traceback.print_exc()

        except exception as info:
            print('Other exception while reading {f} with loadtxt - {info} {a}'.format(f=filename, info=info, a=info.args))
            traceback.print_exc()
    print("{c} out of {t} files".format(c=count, t=len(file_list)))
    if plot>0 and len(ds_list)>0: 
        plot_fs_DA(ds_list[0], ms=ms)
    return(ds_list, comment_list)
Exemple #36
0
def read_text_pyfusion(files, target=b'^Shot .*', ph_dtype=None, plot=pl.isinteractive(), ms=100, hold=0, debug=0, quiet=1,  maxcpu=1, exception = Exception):
    """ Accepts a file or a list of files, returns a list of structured arrays
    See merge ds_list to merge and convert types (float -> pyfusion.prec_med
    """
    regulator = pyfusion.utils.Regulator(maxcpu)
    st = seconds(); last_update=seconds()
    file_list = files
    if len(np.shape(files)) == 0: file_list = [file_list]
    f='f8'
    if ph_dtype is None: ph_dtype = [('p12',f),('p23',f),('p34',f),('p45',f),('p56',f)]
    #ph_dtype = [('p12',f)]
    ds_list =[]
    comment_list =[]
    count = 0
    for (i,filename) in enumerate(file_list):
        regulator.wait()
        if seconds() - last_update > 10:
            last_update = seconds()
            tot = len(file_list)
            print('read {n}/{t}: ETA {m:.1f}m {f}'
                  .format(f=filename, n=i, t=tot,
                          m=(seconds()-st)*(tot-i)/float(60*i)))

        try:
            if (isinstance(target,str) or isinstance(target,bytes)): 
                skip = 1+find_data(filename, target, debug=debug)
            elif isinstance(target, int): 
                skip = target
            else:
                raise Exception('target ({target}) is not recognised'.format(target=target))
            if quiet == 0:
                print('{t:.1f} sec, loading data from line {s} of {f}'
                      .format(t = seconds()-st, s=skip, f=filename))
            #  this little bit to determine layout of data
            # very inefficient to read twice, but in a hurry!
            if debug>2: print('skiprows = \n', skip-1)
            txt = np.loadtxt(fname=filename, skiprows=skip-1, dtype=bytes, 
                             delimiter='FOOBARWOOBAR',ndmin=1)
            header_toks = txt[0].split()
            # look for a version number first
            if header_toks[-1][-1] in b'0123456789.':
                version = float(header_toks.pop())
                if b'ersion' not in header_toks.pop():
                    raise ValueError('Error reading header in {f}'
                                     .format(f=filename))
            else: version=-1  # pre Aug 12 2013
            # noticed that the offset moved in 2015 - when did it  happen?
            phase_offs = -4 if sys.version>'3,' else -2
            # is the first character of the 2nd last a digit?
            if header_toks[phase_offs][0] in b'0123456789': 
                if pyfusion.VERBOSE > 0: 
                    print('found new header including number of phases')
                n_phases = int(header_toks[phase_offs])
                ph_dtype = [('p{n}{np1}'.format(n=n,np1=n+1), f) for n in range(n_phases)]
                
            if 'frlow' in header_toks:  # add the two extra fields
                fs_dtype= [ ('shot','i8'), ('t_mid','f8'), 
                            ('_binary_svs','u8'),    # f16 - really want u8 here,  but npyio 
                                                      #has problem converting 10000000000000000000000000
                                                      #OverflowError: Python int too large to convert to C long
                                                      # doesn't happen if text is read in directly with loadtxt
                            ('freq','f8'), ('amp', 'f8'), ('a12','f8'),
                            ('p', 'f8'), ('H','f8'), 
                            ('frlow','f8'), ('frhigh', 'f8'),('phases',ph_dtype)]
            else:
                fs_dtype= [ ('shot','i8'), ('t_mid','f8'), 
                            ('_binary_svs','u8'), 
                            ('freq','f8'), ('amp', 'f8'), ('a12','f8'),
                            ('p', 'f8'), ('H','f8'), ('phases',ph_dtype)]

            if version > 0.69:  # don't rely on precision
                fs_dtype.insert(-1,('cpkf', 'f8'))  # -1 is 1 before the end
                fs_dtype.insert(-1,('fpkf', 'f8'))  # they appear in this order
                
            if pyfusion.VERBOSE > 0: 
                print(version, fs_dtype, '\n')

            ds = np.loadtxt(fname=filename, skiprows = skip, 
                            dtype= fs_dtype, ndmin=1)  # ENSURE a 1D array

            if len(ds) > 0:
                ds_list.append(ds)
                count += 1
                # npz reads in python 2 can't cope with unicode - don't report errors unless really debugging
                comment_list.append(filename.encode(errors=['ignore','strict'][pyfusion.DBG() > 5]))
            else:
                print('no data in {f}'.format(f=filename))

        except ValueError as info:
            print('Conversion error while processing {f} with loadtxt - {info} {args}'
                  .format(f=filename, info=info, args=info.args))
            traceback.print_exc()

        except exception as info:
            print('Other exception while reading {f} with loadtxt - {info} {a}'.format(f=filename, info=info, a=info.args))
            traceback.print_exc()
    print("{c} out of {t} files".format(c=count, t=len(file_list)))
    if plot>0 and len(ds_list)>0: 
        ds = ds_list[0]
        if hold == 0: pl.clf() # for the colorbar()
        pl.scatter(ds['t_mid'],ds['freq'],ms*ds['a12'],ds['amp'])
        pl.title('{s}, colour is amp, size is a12'.format(s=ds['shot'][0]))
        pl.colorbar()
    return(ds_list, comment_list)
Exemple #37
0
    def save(self, filename, verbose=None, sel=None, use_dictionary=False, tempdir=None, zipopt=-1):
        """ Save as an npz file, using an incremental method, which
        only uses as much /tmp space as required by each var at a time.
        Select which to save with sel: if sel is None, save all except
        for use_dictionary below.
        If use_dictionary is a valid dictionary, save the values of
        ANY AND ONLY the LOCAL variables whose names are in the keys for
        this set.
        So if you have extracted a subset, and you specify 
        use_dictionary=locals(), only that subset is saved (both in array 
        length, and variables chosen).
        Beware locals that are not your variables - e.g. mtrand.beta
        To avoid running out of space on tmp, or to speed up zip - 
        Now included as an argument
        (Note that the normal os.putenv() doesn't seem to write to
        THIS environment use the fudge below - careful - no guarantees)
        os.environ.__setitem__('TMPDIR',os.getenv('HOME'))
        actually - this seems OK
        os.environ['IGETFILE']='/data/datamining/myView/bin/linux/igetfile'

        reload tempfile
        tempfile.gettempdir()
        also ('ZIPOPT','"-1"')  (Now incorporated into args, not tested)
        ** superseded by zlib.Z_DEFAULT_COMPRESSION 0--9  (or -1 for default)
        """
        if verbose is None:
            verbose = self.verbose
        st = seconds()

        if tempdir is not None:
            os.environ.__setitem__("TMPDIR", tempdir)
            import tempfile

            reload(tempfile)  # in case it was already imported
            if tempfile.gettempdir() != tempdir:
                warn("failed to set tempdir = {t}: Value is {v}".format(t=tempdir, v=tempfile.gettempdir()))

        import zlib

        zlib.Z_DEFAULT_COMPRESSION = int(zipopt)
        # """ now obsolete zipfile calls zlin with Z_DEFAULT_COMPRESION arg
        # import os
        # print('overriding default zip compression to {z}'.format(z=zipopt))
        # os.environ.__setitem__('ZIPOPT', str(zipopt))
        # """

        if use_dictionary == False:
            save_dict = self.da  # the dict used to get data
        else:
            save_dict = use_dictionary
            print("Warning - saving only a subset")

        if sel is not None:
            use_keys = sel
        else:
            use_keys = []
            for k in self.da.keys():
                if k in save_dict.keys():
                    use_keys.append(k)

        if verbose:
            print(" Saving only {k}".format(k=use_keys))

        if "info" in use_keys:
            info = info_to_bytes(self["info"])  # avoid py3 unicode error

        args = ",".join(["{k}=save_dict['{k}']".format(k=k) for k in use_keys])
        if verbose:
            print("lengths: {0} -999 indicates dodgy variable".format([mylen(save_dict[k]) for k in use_keys]))

        if "mask" in self.da:
            self.infodict.update({"valid_keys": self.masked.valid_keys})
            self.da["info"] = np.array(self.infodict)
            self.update({"info": self.infodict}, check=False)

        if self.debug:
            print("saving " + filename, args)

        exec("np.savez_compressed(filename," + args + ")")
        self.name = filename

        if verbose:
            print(" in {dt:.1f} secs".format(dt=seconds() - st))
        sys.exit()

import MDSplus as MDS

shots = 0

# set these both to () to stop on errors
shot_exception = ()  # () to see message - catches and displays all errors
node_exception = Exception  # ()  ditto for nodes.

errs = dict(
    shot=[])  # error list for the shot overall (i.e. if tree is not found)
for diag in diags.keys():
    errs.update({diag: []})  # error list for each diagnostic

start = seconds()
for s in srange:  # on t440p (83808,86450): # FY14-15 (86451,89155):#(36363,88891): #81399,81402):  #(81600,84084):
    datdic = dict(shot=s)
    shots += 1
    try:
        if (s % 10) == 0:
            print(s),
        else:
            print('.'),
        if (s % 100) == 0: print('')
        tree = MDS.Tree('magpie2', s)
        non_special = [
            diag for diag in list(diags) if diags[diag][2] != Special
        ]  # if 'dim' in diag]
        #non_special.remove('mirnov_coh')
        for diag in non_special:
Exemple #39
0
def read_text_pyfusion(files,
                       target='^Shot .*',
                       ph_dtype=None,
                       plot=pl.isinteractive(),
                       ms=100,
                       hold=0,
                       debug=0,
                       quiet=1,
                       exception=Exception):
    """ Accepts a file or a list of files, returns a list of structured arrays
    See merge ds_list to merge and convert types (float -> pyfusion.prec_med
    """
    st = seconds()
    last_update = seconds()
    file_list = files
    if len(np.shape(files)) == 0: file_list = [file_list]
    f = 'f8'
    if ph_dtype == None:
        ph_dtype = [('p12', f), ('p23', f), ('p34', f), ('p45', f), ('p56', f)]
    #ph_dtype = [('p12',f)]
    ds_list = []
    comment_list = []
    count = 0
    for (i, filename) in enumerate(file_list):
        if seconds() - last_update > 30:
            last_update = seconds()
            print('reading {n}/{t}: {f}'.format(f=filename,
                                                n=i,
                                                t=len(file_list)))
        try:
            if pl.is_string_like(target):
                skip = 1 + find_data(filename, target, debug=debug)
            else:
                skip = target
            if quiet == 0:
                print('{t:.1f} sec, loading data from line {s} of {f}'.format(
                    t=seconds() - st, s=skip, f=filename))
            #  this little bit to determine layout of data
            # very inefficient to read twice, but in a hurry!
            txt = np.loadtxt(fname=filename,
                             skiprows=skip - 1,
                             dtype=str,
                             delimiter='FOOBARWOOBAR')
            header_toks = txt[0].split()
            # is the first character of the 2nd last a digit?
            if header_toks[-2][0] in '0123456789':
                if pyfusion.VERBOSE > 0:
                    print('found new header including number of phases')
                n_phases = int(header_toks[-2])
                ph_dtype = [('p{n}{np1}'.format(n=n, np1=n + 1), f)
                            for n in range(n_phases)]

            if 'frlow' in header_toks:  # add the two extra fields
                fs_dtype = [('shot', 'i8'), ('t_mid', 'f8'),
                            ('_binary_svs', 'i8'), ('freq', 'f8'),
                            ('amp', 'f8'), ('a12', 'f8'), ('p', 'f8'),
                            ('H', 'f8'), ('frlow', 'f8'), ('frhigh', 'f8'),
                            ('phases', ph_dtype)]
            else:
                fs_dtype = [('shot', 'i8'), ('t_mid', 'f8'),
                            ('_binary_svs', 'i8'), ('freq', 'f8'),
                            ('amp', 'f8'), ('a12', 'f8'), ('p', 'f8'),
                            ('H', 'f8'), ('phases', ph_dtype)]

            ds_list.append(
                np.loadtxt(fname=filename, skiprows=skip, dtype=fs_dtype))
            count += 1
            comment_list.append(filename)
        except ValueError, info:
            print('Conversion error while reading {f} with loadtxt - {info}'.
                  format(f=filename, info=info))
Exemple #40
0
exec(_var_defaults)
exec(process_cmd_line_args())

filters=tb.Filters(complevel=complevel, complib=complib)

dd = DA(DAfilename).da

if outfilename is None:
    (base, ext) = os.path.splitext(os.path.realpath(DAfilename))
    outfilename = base + os.path.extsep + 'h5'

outf = tb.openFile(outfilename, "a")

for var in dd.keys():

    st_copy = seconds()
    if var in [nd.name for nd in outf.listNodes('/')]:
        raise LookupError('{f} already has a node "{n}"'
                          .format(f=outf.filename, n=var))
    val = dd[var]  # need to hold it in memory this way to avoid multiple access
    sizebytes = val.nbytes
    print('{dt:.1f}s to read {v} {GB:.2f} GB for {f}'
          .format(dt=seconds()-st_copy, 
                  GB = sizebytes/1e9, f=os.path.split(outfilename)[-1], v=var))

    st_write = seconds()

    try:
        var_atom = tb.atom.Atom.from_dtype(numpy.dtype(val.dtype))
    except Exception as reason:
        print('failed to copy {v}, reason: {r} {a}'
Exemple #41
0
    def __init__(self, name="none", datadir="..", system="none"):
        if name == "none":
            name = str(int(seconds()))
        self.name = name

        global startdir
        self.startdir = startdir
        self.counter = 1
        os.chdir(datadir)
        self.datadir = datadir
        self.saveddir = self.startdir
        self.talkfailed = False
        self.speakversion = "none"
        if system == "none":
            self.system = platform.system()
        else:
            self.system = system
        if self.system == "Darwin":
            try:
                from AppKit import NSSpeechSynthesizer

                #from AppKit import NSSpeechSynthesizer
                def wsay(afile):
                    #https://stackoverflow.com/questions/12758591/python-text-to-speech-in-macintosh
                    #NSSpeechSynthesizer.availableVoices()
                    speechSynthesizer = NSSpeechSynthesizer.alloc().init()
                    speechSynthesizer.setVoice_(
                        'com.apple.speech.synthesis.voice.karen')
                    x = open(afile, "r")
                    txt = x.read()
                    txt = txt.split("====")
                    txt = txt[0]
                    x.close()
                    speechSynthesizer.startSpeakingString_(txt)

                self.speakversion = "Mac NSSpeechSynthesizer"
            except:

                def wsay(afile):
                    x = open(afile, "r")
                    txt = x.read()
                    txt = txt.split("====")
                    txt = txt[0]
                    x.close()
                    txt = txt.replace('"', "'")
                    cmd = "echo \"" + txt + "\" | say -v Karen --quality=128"
                    x = os.popen(cmd, "r")

                self.speakversion = "Mac say"
#local
        if self.system == "local":
            #			print("local")
            def wsay(afile):
                if self.talkfailed: return ()
                from os.path import isfile
                afile = afile + ".wave"
                if isfile(afile):
                    pass
                else:
                    return
                ofile = """<!DOCTYPE html>
<html>
<head>
<title></title>
</head>
<body>
<audio controls autoplay>
<source src="../OUT" type="audio/wave">
Your browser does not support the audio element.
</audio>
</body>
</html>
"""
                ofile = ofile.replace("OUT", afile)
                ofile = ofile.replace("..", self.datadir)
                #print(ofile)
                try:
                    display(HTML(ofile))
                except:
                    print("talk failed")
                    self.talkfailed = True

            self.speakversion = "local "

        if self.system == "Windows" or self.system == "Linux":
            try:
                ##### conda create --name slides Jupiter Matplotlib scipy pandas
                ##### conda activate slides
                ##### pip install gtts
                from gtts import gTTS

                def wsay(afile):
                    global speakversion
                    if self.talkfailed: return ()
                    x = open(afile, "r")
                    txt = x.read()
                    txt = txt.split("====")
                    txt = txt[0]
                    x.close()
                    if len(txt) > 0:
                        tts = gTTS(text=txt, lang='en')
                        tts.save(self.name + ".mp3")
                    ofile = """<!DOCTYPE html>
<html>
<head>
	<title></title>
</head>
<body>
<audio controls autoplay>
  <source src="../OUT.mp3" type="audio/mp3">
  Your browser does not support the audio element.
</audio>
</body>
</html>
"""
                    ofile = ofile.replace("OUT", self.name)
                    ofile = ofile.replace("..", self.datadir)
                    try:
                        if len(txt) > 0:
                            #print(ofile)
                            display(HTML(ofile))
                        #os.remove(self.name+".mp3")
                    except:
                        print("talk failed")
                        self.talkfailed = True

                self.speakversion = "gtts (Internet Connection) "
            except:

                def wsay(afile):
                    global speakversion
                    x = open(afile, "r")
                    txt = x.read()
                    txt = txt.split("====")
                    txt = txt[0]
                    x.close()
                    #tts = gTTS(text=txt, lang='en')
                    print("no speech module found")

        username = getpass.getuser()
        self.wsay = wsay

        if (self.datadir == ".."):
            dir = os.getcwd()
        else:
            dir = datadir
        self.datadir = dir
        #os.chdir(dir)

        # text to be spoken
        txt = []

        # images to show, either jpg or png files
        lfiles = os.listdir()
        img = []
        for i in lfiles:
            if i.find("jpeg") > 0:
                img.append(i)
        if len(img) == 0:
            img = []
            for i in lfiles:
                if i.find("png") > 0:
                    img.append(i)
        img.sort()

        # optional text to show after the slide
        # that can be copy/pasted
        out = []

        for s in range(1, len(img) + 1):
            t = str("%4.4d" % (s))
            #t=dir+"/"+t
            txt.append(t)
            out.append("")
            #outfile=str("i%3.3d" % (s))
            try:
                #outfile=open(outfile,"r")
                outfile = open(t, "r")
                buff = outfile.read()
                buff = buff.split("====")
                if (len(buff) > 1):
                    buff = buff[1]
                else:
                    buff = ""
                out[-1] = buff
                #out[-1]=outfile.read()
                out[-1] = out[-1].replace("USER", username)
            except:
                pass
        self.out = out
        self.txt = txt
        self.img = img
Exemple #42
0
_var_defaults = """
DFfilename='/data/datamining/DA/PF2_130813_50_5X_1.5_5b_rms_1_diags_comp5.h5'
keep_open = 1  # for read, should keep open!
debug=1
"""
exec(_var_defaults)
exec(process_cmd_line_args())

mem = report_mem('init')
df = tb.open_file(DFfilename, 'r')
dd = {}

for nd in df.list_nodes('/'):
    var = nd.name
    st_copy = seconds()

    v = df.get_node('/' + var)
    dd.update({var: v})

    dt_copy = seconds() - st_copy
    if debug > 0:
        print('{var} in {dt_copy:.1f}s '.format(var=var, dt_copy=dt_copy))

report_mem(mem)

# This is model for a way to extract selected shots from a huge data set.
st_access = seconds()
n = 100  # take a little bit so it doean't take too long,  n=10000 gets all
for k in dd.keys():
    for i in range(2000):
Exemple #43
0
            runfile = tempfile.mktemp()
            env = os.environ
            # env.update({'PYTHONSTARTUP':tmpfil}) only works in interactive
            with open(filename, 'rt') as pf:
                prog = pf.readlines()
            with open(runfile, 'wt') as tf:
                tf.write(prerun+'\n')
                tf.writelines(prog)
        else:
            env = None
            runfile = filename

        cmd = '{py} {file} {args}'.format(file=runfile, py=python_exe, args=args)
        print(cmd)

        st = seconds()
        sub_pipe = subprocess.Popen(cmd,  env=env, shell=True, stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE)
        """ Doesn't workthis way - seems to wait.
        for t in range(max_sec):
            print(t)
            sleep(1)
            if sub_pipe.poll():
                break
            if t == max_sec - 1:
                print('terminate')
                sub_pipe.kill()
        """
        (resp, errout) = sub_pipe.communicate()
        print(errout)
        if ((errout != b'') and (not 'warn' in errout.lower())) or (sub_pipe.returncode != 0):
Exemple #44
0
_var_defaults = """
DFfilename='/data/datamining/DA/PF2_130813_50_5X_1.5_5b_rms_1_diags_comp5.h5'
keep_open = 1  # for read, should keep open!
debug=1
"""
exec(_var_defaults)
exec(process_cmd_line_args())

mem = report_mem('init')
df = tb.open_file(DFfilename,'r')
dd = {}

for nd in df.list_nodes('/'):
    var = nd.name
    st_copy = seconds()

    v = df.get_node('/'+var)
    dd.update({var:v})
    
    dt_copy = seconds() - st_copy
    if debug>0:
        print('{var} in {dt_copy:.1f}s '.format(var=var, dt_copy=dt_copy))

report_mem(mem)

# This is model for a way to extract selected shots from a huge data set.
st_access = seconds()
n=100  # take a little bit so it doean't take too long,  n=10000 gets all
for k in dd.keys():
    for i in range(2000):
Exemple #45
0
        self.maxcpu = max(self.maxcpu,0.01)
        if self.maxcpu != maxcpu:
            print('changed maxcpu from {i} to {a}'.format(i=maxcpu, a=self.maxcpu))

        self.count = 0
        self.st = seconds()
        self.last_wait = 0
        self.waited = 0

        
    def wait(self):
        self.count += 1
        if self.maxcpu == 1:
            return()

        avg_time = (seconds() - self.st - self.waited)/self.count
        self.last_wait = max(avg_time*(1-self.maxcpu)/self.maxcpu,0.01)
        print(avg_time, self.last_wait, self.count)
        self.waited += self.last_wait
        sleep(self.last_wait)

if __name__ == '__main__':
    from time import time as seconds
    st = seconds()
    regulator = Regulator(1)
    for i in range(10000):
        regulator.wait()
    
    print('{nits} iterations with 100% CPU -> overhead of {dt:.2f}us per call'
          .format(nits=regulator.count, dt=1e6*(seconds()-regulator.st)/regulator.count))