def main3(): logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO) path = '/home/gujt/work/data/Aquant2-ctree/0/' exe = Execute() exe.build_model() #exe.load_model() #exit() count = 0 for dir in os.walk(path): for file in dir[2]: print 'read.',file f = open(path+file) line1 = f.readline() start = now() while line1: if count > 10: exit() count += 1 line2 = f.readline() lines = [line1, line2] try: exe.train_sentence(lines) except: '[BUG]',line2 finally: line1 = f.readline() print '[',count,']', (now()-start),'s' start = now() f.close() print 'train ok.' exe.save_weights('../data/weights.bin')
def distinct_patterns_over_windows(self, window_sizes=None, trials=None, save_couplings=False, remove_zeros=False): """ Returns tuple: counts, entropies [, couplings] counts, entropies: arrays of size 2 x T x WSizes (0: empirical from model sample, 1: dynamics from learned model on sample) Parameters ---------- window_sizes : Type, optional Description (default None) trials : Type, optional Description (default None) save_couplings : bool, optional Description (default False) remove_zeros : bool, optional Description (default False) Returns ------- Value : Type Description """ if window_sizes is None: window_sizes = [1] trials = trials or range(self._original_spikes.T) counts = np.zeros((2, len(trials), len(window_sizes))) #entropies = np.zeros((2, len(trials), len(window_sizes))) couplings = {} tot_learn_time = 0 for ws, window_size in enumerate(window_sizes): couplings[window_size] = [] for c, trial in enumerate(trials): hdlog.info("Trial %d | ws %d" % (trial, window_size)) self._window_size = window_size t = now() self.fit(trials=[trial], remove_zeros=remove_zeros) diff = now() - t hdlog.info("[%1.3f min]" % (diff / 60.)) tot_learn_time += diff if save_couplings: couplings[ws].append(self._learner.network.J.copy()) self.chomp() #entropies[0, c, ws] = self._raw_patterns.entropy() counts[0, c, ws] = len(self._raw_patterns) #entropies[1, c, ws] = self._hopfield_patterns.entropy() counts[1, c, ws] = len(self._hopfield_patterns) hdlog.info("Total learn time: %1.3f mins" % (tot_learn_time / 60.)) self._learn_time = tot_learn_time if save_couplings: return counts, couplings return counts
def __locked_handle_service_change_state( self, state, service, start_time, ): """Handles a state change for a service and returns whether the state change was successful, """ check_time = now() try: service.assert_() except PgctlUserMessage as error: state_change_result = self.__locked_handle_state_change_exception( state, service, error, start_time, check_time, ) return state_change_result else: # TODO: debug() takes a lambda debug('loop: check_time %.3f', now() - check_time) if self._should_display_state(state): pgctl_print(state.strings.changed, service.name) service.service.message(state) return StateChangeResult.SUCCESS
def your_call_is_important_to_us(): if pyclosure['t0']: how_long = " (%.0f secs now)" % (now() - pyclosure['t0']) else: pyclosure['t0'] = now() how_long = "" the_logger.info("Waiting for %s%s..." % (what, how_long))
def deep_search(task, timeout=10*60): state = State.from_task(task) solution = Solution(task) solution.metrics = state.estimator.metrics deadline = now() + timeout # Python has no tail-recursion optimization. # Even more, python has a limit on recursion depth # So, we need to write big loops iteratively while state: # when we try to .negate() init state we will obtain None and will exit from the loop solution.steps += 1 if not state.is_feasible: state = state.parent continue if state.is_all_covered(): solution.store_result(state) state = state.negate() # try to deselect the current set or rollback to the parent state continue if state.get_optimistic_cost() >= solution.best_cost: if now() > deadline: # we get to this place often enough to stop in time, # and we get to it not on the each iteration, so we will not check the time too frequently return solution state = state.negate() # try to deselect the current set or rollback to the parent state continue state = state.next_child() solution.proven_as_optimal = True # we have not terminated on timeout, so we have explored all the tree return solution
def write_func(ident): with lock.write_lock(): enter_time = now() time.sleep(WORK_TIMES[ident % len(WORK_TIMES)]) exit_time = now() start_stops.append((lock.WRITER, enter_time, exit_time)) time.sleep(NAPPY_TIME)
def count_or_show_by_generator(gen, count_enable, row_count, col_count): """ gen: a generator returned by find_solutions_* count_enable: bool, only count solutions/configurations, don't show them """ if count_enable: print('Calculating, please wait... (Control+C to cancel)') tm0 = now() try: solution_count = sum(1 for _ in gen) except KeyboardInterrupt: print('\nGoodbye') return delta = now() - tm0 print('Number of Unique Configurations: %s' % solution_count) print('Running Time: %.4f seconds' % delta) else: print('Found Configurations:\n') for board in gen: print(format_board(board, row_count, col_count)) try: input('Press Enter to see the next, Control+C to exit') except KeyboardInterrupt: print('\nGoodbye') break
def main(): global verbose, f, db, co, ldap, auth, start parser = argparse.ArgumentParser() parser.add_argument('-v', "--verbose", action="count", default=0) parser.add_argument('-m', "--mail-file") parser.add_argument('-s', "--spread", default=ldapconf('MAIL', 'spread', None)) parser.add_argument('-i', "--ignore-size", dest="max_change", action="store_const", const=100) parser.add_argument('-a', "--no-auth-data", dest="auth", action="store_false", default=True) args = parser.parse_args() verbose = args.verbose auth = args.auth db = Factory.get('Database')() co = Factory.get('Constants')(db) start = now() curr = now() if verbose: logger.debug("Loading the EmailLDAP module...") ldap = Factory.get('EmailLDAP')(db) if verbose: logger.debug(" done in %d sec." % (now() - curr)) spread = args.spread if spread is not None: spread = map_spreads(spread, int) f = ldif_outfile('MAIL', args.mail_file, max_change=args.max_change) get_data(spread) end_ldif_outfile('MAIL', f)
def __init__(sz, start_position, r_winch, c, looptime, armflag): #Lengths set the initial configuration of the system. # Lengths: array 1x4 [L0, L1, L2, L3] #create spiral zipper object #physical notes #sz ckbot direction: CCW subtracts tether and CW adds/releases tether. sz.startposition = start_position sz.start_detected = False sz.entered = False sz.tether_subtract_CCW = True sz.looptime = looptime #control speed to be enforced sz.timeold = 0 sz.timeold2 = 0 sz.goal_prev = start_position sz.goal_start = now() sz.goal_stop =now() sz.target_achieved = [0, 0] sz.target_reached = False sz.P = 10*np.eye(2) #System covariance. Trust in initial Process Model Conditions sz.Q = 0.01*np.eye(2) #System noise Covariance. What we think the initial process noise is sz.R = 1*np.eye(2) #Sensor Noise Covariance. What we think of the sensor noise is. got_port = [1, 1, 1] ser = [0, 0, 0] try: ser[0] = serial.Serial('/dev/ttyACM0', 57600) except Exception, e: print "No Arduino on ACM0" print str(e) got_port[0] = 0
def compare_find_solutions_time(): """ run and compare the running time of 3 implementations of find_solutions """ row_count, col_count, count_by_symbol = input_problem() time_list = [] func_list = ( find_solutions_s, find_solutions_r, find_solutions_q, find_solutions_s, find_solutions_r, find_solutions_q, ) for func in func_list: # pylint! tm0 = now() for _ in func(row_count, col_count, count_by_symbol): pass delta = now() - tm0 time_list.append(delta) print('%.4f seconds (%s)' % (delta, func))
def pb47(): n = 0 n1 = [] n2 = [] n3 = [] print(now()) primes = simpleseive(200000) print(now()) while n < 200000: m = n n4 = [] for p in primes: #print(m,p) if p > m: break if m % p == 0: while m % p == 0: m = m / p #print(m,p) n4.append(p) if len(n4) == 5: break if len(n4) == len(n3) == len(n2) == len(n1)== 4: print(n-3,n-2,n-1,n,n*(n-1)*(n-2)*(n-3)) return #print(n1,n2,n3,n4) n1,n2,n3 = n2 + [],n3 + [],n4 + [] n += 1 print(now())
def wait_net_service(host, port, timeout=None): import socket from time import sleep, time as now log('Waiting for web server: ' + host + ':' + str(port)) s = socket.socket() if timeout: end = now() + timeout while True: try: if timeout: if now() > end: log('ERROR! Network sockets connect waiting timeout!') return False s.connect((host, port)) except socket.timeout: sleep(0.1) pass except socket.error: sleep(0.1) pass else: s.close() return True
def change_status(target, status): target_name = target.getName() old_status = None if target_name in busy_players: value = busy_players[target_name] time_left = value[1] - now() if time_left > 0: msg(target, "&cYou must wait %.2fs untill you can change your status" % time_left) return old_status = value[0] if old_status is status: if status is True: msg(target, "&cYou are already SUPER busy") elif status is False: msg(target, "&cYou are already busy") else: msg(target, "&cYou weren't busy yet") return busy_players[target_name] = (status, now() + busy_status_change_timeout) if status is True: broadcast(None, target.getDisplayName() + " &7is now SUPER busy") elif status is False: broadcast(None, target.getDisplayName() + " &7is now busy") else: broadcast(None, target.getDisplayName() + " &7is not busy anymore")
def run(self): while 1: sleep(self.expected - now()) if now() > self.expected: message = "WARNING: Message not processed for {0} seconds, giving up\n".format(self.timeout) sys.stderr.write(message) os.kill(os.getpid(),9)
def main(): path = input("path:") all_size = {} total_file = 0 total_delete = 0 start = now() for file in os.listdir(path): total_file += 1 real_path = os.path.join(path, file) if os.path.isfile(real_path) == True: size = os.stat(real_path).st_size name_and_md5 = [real_path, ''] if size in all_size.keys(): new_md5 = getmd5(real_path) if all_size[size][1] == '': all_size[size][1] = getmd5(all_size[size][0]) if new_md5 in all_size[size]: print ('删除'), file total_delete += 1 else: all_size[size].append(new_md5) else: all_size[size] = name_and_md5 end = now() time_last = end - start print ('文件总数:'), total_file print ('删除个数:'), total_delete print ('耗时:'), time_last, '秒'
def wait_net_service(server, port, timeout=None): """ Wait for network service to appear @param timeout: in seconds, if None or 0 wait forever @return: True of False, if timeout is None may return only True or throw unhandled network exception """ s = socket.socket() if timeout: from time import time as now # time module is needed to calc timeout shared between two exceptions end = now() + timeout while True: try: if timeout: next_timeout = end - now() if next_timeout < 0: return False else: s.settimeout(next_timeout) s.connect((server, port)) except socket.timeout, err: # this exception occurs only if timeout is set if timeout: return False except socket.error, err: # catch timeout exception from underlying network library # this one is different from socket.timeout pass
def _wait_net_service(server, port, timeout=None): """ Wait for network service to appear @param timeout: in seconds, if None or 0 wait forever @return: True of False, if timeout is None may return only True or throw unhandled network exception """ s = socket.socket() if timeout: end = now() + timeout while True: try: if timeout: next_timeout = end - now() if next_timeout < 0: return False else: s.settimeout(next_timeout) s.connect((server, port)) except socket.timeout: # this exception occurs only if timeout is set if timeout: return False except socket.error: pass else: s.close() return True
def main(): path = u'D:\\test2' all_md5 = {} all_size = {} total_file = 0 total_delete = 0 start = now() print("start") for file in os.listdir(path): total_file += 1 real_path = os.path.join(path, file) if os.path.isfile(real_path): size = os.stat(real_path).st_size name_and_md5 = [real_path, ''] if size in all_size.keys(): print('finded') new_md5 = getmd5(real_path) if all_size[size][1] == '': all_size[size][1] = getmd5(all_size[size][0]) if new_md5 in all_size[size]: print("DELETE:" + file) os.remove(path+'\\'+file) total_delete += 1 else: all_size[size].append(new_md5) else: all_size[size] = name_and_md5 end = now() time_last = end - start print('TOTAL NUMBER:', total_file) print('DELETED NUMBER:', total_delete) print('TIME COST:', time_last, 'SEC')
def __init__(self, test_data_fn): start = now() if os.path.isfile(test_data_fn): print("Reading test data...") self.prepop_rows, self.idens, self.props, self.rows = \ pickle.load(open(test_data_fn, 'rb')) else: print("Generating test data...") random.seed(4) # 4 chosen by fair dice roll. Guaranteed to be random forms = [gen_random_form() for x in range(NUM_FORMS)] # FIXME: don't use random.choice!!! Super duper slow self.prepop_rows = flatten(_rows_from_tufo(gen_random_tufo(random.choice(forms))) for x in range(NUM_PREEXISTING_TUFOS)) tufos = [gen_random_tufo(random.choice(forms)) for x in range(NUM_TUFOS)] self.idens = [t[0] for t in tufos] self.props = [get_random_keyval(t[1]) for t in tufos] random.shuffle(self.idens) random.shuffle(self.props) self.rows = flatten(_rows_from_tufo(x) for x in tufos) pickle.dump((self.prepop_rows, self.idens, self.props, self.rows), open(test_data_fn, 'wb')) print("Test data generation took: %.2f" % (now() - start)) print('addRows: # Tufos:%8d, # Rows: %8d' % (NUM_TUFOS, len(self.rows))) print('len count: small:%d, medium:%d, large:%d, huge:%d' % (small_count, medium_count, large_count, huge_count))
def main(): path = raw_input("Path: ") all_size = {} total_file = 0 total_delete = 0 start = now() for file in os.listdir(path): total_file += 1; real_path = os.path.join(path, file) if os.path.isfile(real_path) == True: filesize = os.stat(real_path).st_size name_md5 = [real_path, ''] if filesize in all_size.keys(): new_md5 = getmd5(real_path) if all_size[filesize][1] == '': all_size[filesize][1] = getmd5(all_size[filesize][0]) if new_md5 in all_size[filesize]: total_delete += 1 os.remove(real_path) print 'Delete ', file else: all_size[filesize].append(new_md5) else: all_size[filesize] = name_md5 end = now() time_last = end - start print 'File total: ', total_file print 'Del total: ', total_delete print 'Time consuming: ', time_last, 's'
def any_unknown(self, message, *args, **kwargs): found = False key = (message, repr(args), repr(kwargs)) try: value = self._cache.peek(key) except KeyError: pass else: found = True if not self._cache.hasExpired(key): raise StopIteration(value) if self._backoffStarted: if self._backoffStarted + self._backoffTimeout < now(): self._backoffStarted = None elif found: raise StopIteration(value) else: raise BackoffException() try: value = yield self.any.unknown(message, *args, **kwargs) self._cache[key] = value except (SystemExit, KeyboardInterrupt, AssertionError): raise except Exception, e: if self._backoffTimeout and isinstance(e, TimeoutException): self._backoffStarted = now() if not (self._returnCachedValueInCaseOfException and found): raise BackoffException() if not (self._returnCachedValueInCaseOfException and found): raise
def make_the_timeseries(): print('Read and filter data') # Load data from the star time, flux = kic.getdata(ID, kernelsize, quarter, sigma, noisecut) #time = time[:((len(time)+1)//2)] #flux = flux[:((len(flux)+1)//2)] assert len(time) == len(flux) # Calculate and print Nyquist-frequency dt = np.diff(time) nyquist = 1 / (2 * np.median(dt)) print('Nyquist frequency: %s µHz' % str(nyquist)) # Plot the time series """ plt.figure() plt.plot(time, flux, 'k.') plt.xlabel(r'Relative time [Ms]') plt.ylabel(r'Photometry') plt.xlim([np.amin(time), np.amax(time)]) plt.savefig('%s_time.pdf' % (starname), bbox_inches='tight') """ # Save data in textfile print('Write %d entries to %s' % (len(time), ts)) timerStart = now() savenpz(ts, np.transpose([time, flux])) elapsedTime = now() - timerStart print('Took %.2f s' % elapsedTime)
def generate_test_cases(cases=5): logfile = os.path.join(os.path.expanduser("~"), "sn", "py", "spikes", "qn", "bigcases.h5") h5 = h5py.File(logfile, "w") t = 0.0 for i in range(cases): n = 2000 + 0 * np.random.randint(10, 1000) a = np.random.randn(n) x = np.random.randn(n) * 3 A = power(n, 2) args = (A, a) lam = np.random.rand() * 0.01 m = 8 maxit = 100 tic = now() res = lbfgs.bfgsl1(f, x, lam=lam, args=args, debug=False, log=None) t += now() - tic subgroup = h5.create_group("case-%03d" % i) subgroup.create_dataset("A", data=A) subgroup.create_dataset("a", data=a) subgroup.create_dataset("x", data=x) subgroup.create_dataset("r", data=res) subgroup.create_dataset("lam", data=np.array([lam])) print "Computation time: ", t h5.close()
def _wait_until_ready(self, timeout=5, raise_if_not_ready=True): # we're only ready when it's possible to connect to the CrossBar # over TCP - so let's just try it. end = now() + timeout ready = False while not ready: timeout = end - now() if timeout < 0: if raise_if_not_ready: raise ConnectionError( 'Failed to connect to CrossBar over {}: {}:{}'.format( self.ipv, self.host, self.port) ) else: return ready try: self.try_connection() except ConnectionError: pass else: ready = True return ready
def __locked_change_state(self, state): """the critical section of __change_state""" pgctl_print(state.strings.changing, commafy(self.service_names)) services = [state(service) for service in self.services] failed = [] start_time = now() while services: for service in services: try: service.change() except Unsupervised: pass # handled in state assertion, below for service in tuple(services): check_time = now() try: service.assert_() except PgctlUserMessage as error: if timeout(service.name, error, state.strings.change, start_time, service.get_timeout(), check_time): services.remove(service) failed.append(service.name) else: # TODO: debug() takes a lambda debug('loop: check_time %.3f', now() - check_time) pgctl_print(state.strings.changed, service.name) service.service.message(state) services.remove(service) time.sleep(self.poll) return failed
def find_best_lin(train,test): best_r=0 T=[] for pen in [0.1,0.5,1,10,50]: start=now() clf=svm.LinearSVC(C=pen,class_weight="auto") clf.fit(train[:,2:18],train[:,1]) finish=now() T.append(finish-start) scores=clf.predict(test[:,2:18]) print pen scaled_score=scores # for i in range(len(scores)): # scaled_score[i]=(scores[i]-min(scores))/(max(scores)-min(scores)) fpr, tpr, thresholds = roc_curve(test[:,1], scaled_score) roc_auc = auc(fpr, tpr) print roc_auc r_score=clf.score(test[:,2:18],test[:,1]) if best_r<roc_auc: best_clf=clf best_r=roc_auc best_pen=pen best_scores=scaled_score return best_pen,best_r,best_clf,best_scores,T
def get_CR_delta(sz): #compare previous value and current value for modules # returns the delta in radians assums CCW is positive and CCW is defined facing the servo front #get current readings #Assume the current value of the sensor is stored and the previous value is correct. timeelapsed = now() - sz.timeold diff = [0.0,0.0,0.0,0.0] #diff = [0.0, 0.0] for i in range(4): diff[i] = sz.theta_curr[i]-sz.theta_prev[i] if abs(diff[i])>sz.limit_t: #calculate valid delta and switch signs to be correct direction if sz.theta_curr[i] >= sz.theta_prev[i]:# diff is +ve, so the solution should be -ve diff[i] = ((sz.max_t-sz.theta_curr[i])+(sz.theta_prev[i]-sz.min_t)+1)*(-1) else: # diff is negative, therfore the solution should be positive diff[i] = ((sz.max_t-sz.theta_prev[i])+(sz.theta_curr[i]-sz.min_t)+1) else: #tether length update based on IMU only diff[i] = diff[i] #valid calculation #print "motor position change " #print np.array(diff)#*sz.radian_const sz.timeold = now() if timeelapsed < .2: return np.array(diff)*sz.radian_const # convert ticks to radians else: return [0.0,0.0,0.0,0.0]
def oneclass_best_rbf(train_oneclass,test): best_r=0 best_gamma=0 T=[] S=[] #for para in [2e-6,2e-3,0.02,0.05,0.1,0.2,1]: #for para in [2e-12,2e-8,2e-6,2e-4,0.02]: for para in [2e-6]: clf_oneclass=svm.OneClassSVM(kernel='rbf',gamma=para) start=now() clf_oneclass.fit(train_oneclass[:,2:18]) finish=now() T.append(finish-start) print para scores=clf_oneclass.decision_function(test[:,2:18]) fpr, tpr, thresholds = roc_curve(test[:,1], scores) roc_auc = auc(fpr, tpr) print roc_auc #score=0.0 #for i in range(np.shape(test)[0]): #if ((predict[i]==-1.0)&(test[i,1]==0)) or ((predict[i]==1.0)&(test[i,1]==1)): #score=score+1.0 #score=score/np.shape(test)[0] #S.append(score) if best_r<roc_auc: best_r=roc_auc best_gamma=para best_clf=clf_oneclass return best_gamma,best_r,T,best_clf,S
def writeSynFile(self, altIndexList): """ Build .syn file """ if not altIndexList: return log.info("Sorting %s synonyms..." % len(altIndexList)) t0 = now() altIndexList.sort( key=lambda x: sortKeyBytes(x[0]) ) # 28 seconds with old sort key (converted from custom cmp) # 0.63 seconds with my new sort key # 0.20 seconds without key function (default sort) log.info("Sorting %s synonyms took %.2f seconds" % ( len(altIndexList), now() - t0, )) log.info("Writing %s synonyms..." % len(altIndexList)) t0 = now() with open(self._filename+".syn", "wb") as synFile: synFile.write(b"".join([ b_alt + b"\x00" + intToBinStr(wordIndex, 4) for b_alt, wordIndex in altIndexList ])) log.info("Writing %s synonyms took %.2f seconds" % ( len(altIndexList), now() - t0, ))
def wrapped(*args, **kwargs): start = now() result = func(*args, **kwargs) end = now() ms_delta = (end - start) * 1000 print "Execution time: {0}ms".format(ms_delta) return result
def __init__(self, numbering_mode=None, pin_rs=None, pin_rw=None, pin_e=None, pins_data=None, pin_backlight=None, backlight_mode='active_low', backlight_enabled=True, cols=20, rows=4, dotsize=8, charmap='A02', auto_linebreaks=True, compat_mode=False): """ Character LCD controller. The default pin numbers are based on the BOARD numbering scheme (1-26). You can save 1 pin by not using RW. Set ``pin_rw`` to ``None`` if you want this. :param pin_rs: Pin for register select (RS). Default: ``15``. :type pin_rs: int :param pin_rw: Pin for selecting read or write mode (R/W). Set this to ``None`` for read only mode. Default: ``18``. :type pin_rw: int :param pin_e: Pin to start data read or write (E). Default: ``16``. :type pin_e: int :param pins_data: List of data bus pins in 8 bit mode (DB0-DB7) or in 4 bit mode (DB4-DB7) in ascending order. Default: ``[21, 22, 23, 24]``. :type pins_data: list of int :param pin_backlight: Pin for controlling backlight on/off. Set this to ``None`` for no backlight control. Default: ``None``. :type pin_backlight: int :param backlight_mode: Set this to either ``active_high`` or ``active_low`` to configure the operating control for the backlight. Has no effect if pin_backlight is ``None`` :type backlight_mode: str :param backlight_enabled: Whether the backlight is enabled initially. Default: ``True``. Has no effect if pin_backlight is ``None`` :type backlight_enabled: bool :param numbering_mode: Which scheme to use for numbering of the GPIO pins, either ``GPIO.BOARD`` or ``GPIO.BCM``. Default: ``GPIO.BOARD`` (1-26). :type numbering_mode: int :param rows: Number of display rows (usually 1, 2 or 4). Default: ``4``. :type rows: int :param cols: Number of columns per row (usually 16 or 20). Default ``20``. :type cols: int :param dotsize: Some 1 line displays allow a font height of 10px. Allowed: ``8`` or ``10``. Default: ``8``. :type dotsize: int :param charmap: The character map used. Depends on your LCD. This must be either ``A00`` or ``A02``. Default: ``A02``. :type charmap: str :param auto_linebreaks: Whether or not to automatically insert line breaks. Default: ``True``. :type auto_linebreaks: bool :param compat_mode: Whether to run additional checks to support older LCDs that may not run at the reference clock (or keep up with it). :type compat_mode: bool """ # Configure compatibility mode self.compat_mode = compat_mode if compat_mode: self.last_send_event = now() # Set attributes if numbering_mode == GPIO.BCM or numbering_mode == GPIO.BOARD: self.numbering_mode = numbering_mode else: raise ValueError( 'Invalid GPIO numbering mode: numbering_mode=%s, ' 'must be either GPIO.BOARD or GPIO.BCM.\n' 'See https://gist.github.com/dbrgn/77d984a822bfc9fddc844f67016d0f7e ' 'for more details.' % numbering_mode) if pin_rs is None: raise ValueError('pin_rs is not defined.') if pin_e is None: raise ValueError('pin_e is not defined.') if len(pins_data) == 4: # 4 bit mode self.data_bus_mode = c.LCD_4BITMODE block1 = [None] * 4 elif len(pins_data) == 8: # 8 bit mode self.data_bus_mode = c.LCD_8BITMODE block1 = pins_data[:4] else: raise ValueError('There should be exactly 4 or 8 data pins.') block2 = pins_data[-4:] self.pins = PinConfig(rs=pin_rs, rw=pin_rw, e=pin_e, d0=block1[0], d1=block1[1], d2=block1[2], d3=block1[3], d4=block2[0], d5=block2[1], d6=block2[2], d7=block2[3], backlight=pin_backlight, mode=numbering_mode) self.backlight_mode = backlight_mode # Call superclass super(CharLCD, self).__init__(cols, rows, dotsize, charmap=charmap, auto_linebreaks=auto_linebreaks) # Set backlight status if pin_backlight is not None: self.backlight_enabled = backlight_enabled
False incremental 4 0.790 False incremental 5 3.087 switched to benchmarking search function False incremental 3 0.035 False incremental 4 0.698 False incremental 5 2.558 ''' if __name__ == '__main__': difficultPosition = ''' r . b q . . . r p p p p n k p p . . n b . p . . . . . . p . . . . . P . N . . . P . . P B N . . . P . . P P P P R . . Q K B . R''' test_board = [line for line in difficultPosition.replace(' ', '').split()] test_board.reverse() startTime = now() # main([test_board], 50, 0) _possible_moves = moves(test_board, True) _possible_moves.sort(key=lambda x: x[1], reverse=True) bestMove = search(_possible_moves, True, 2) print('{:.3f}'.format(now() - startTime)) print(total_moves) print('\n'.join(' '.join(piece for piece in row) for row in bestMove.__reversed__()) + '\n')
def writeGeneral(self) -> None: """ Build StarDict dictionary in general case. Every item definition may consist of an arbitrary number of articles. sametypesequence option is not used. """ dictMark = 0 altIndexList = [] # list of tuples (b"alternate", entryIndex) dictFile = open(self._filename + ".dict", "wb") idxFile = open(self._filename + ".idx", "wb") indexFileSize = 0 t0 = now() wordCount = 0 defiFormatCounter = Counter() if not isdir(self._resDir): os.mkdir(self._resDir) entryIndex = -1 while True: entry = yield if entry is None: break if entry.isData(): entry.save(self._resDir) continue entryIndex += 1 entry.detectDefiFormat() # call no more than once defiFormat = entry.defiFormat defiFormatCounter[defiFormat] += 1 if defiFormat not in ("m", "h", "x"): log.error(f"invalid defiFormat={defiFormat}, using 'm'") defiFormat = "m" words = entry.l_word # list of strs word = words[0] # str defi = self.fixDefi(entry.defi, defiFormat) # defi is str for alt in words[1:]: altIndexList.append((alt.encode("utf-8"), entryIndex)) b_dictBlock = (defiFormat + defi).encode("utf-8") + b"\x00" dictFile.write(b_dictBlock) blockLen = len(b_dictBlock) b_idxBlock = word.encode("utf-8") + b"\x00" + \ uint32ToBytes(dictMark) + \ uint32ToBytes(blockLen) idxFile.write(b_idxBlock) dictMark += blockLen indexFileSize += len(b_idxBlock) wordCount += 1 dictFile.close() idxFile.close() if not os.listdir(self._resDir): os.rmdir(self._resDir) log.info(f"Writing dict file took {now()-t0:.2f} seconds") log.debug("defiFormatsCount = " + pformat(defiFormatCounter.most_common())) self.writeSynFile(altIndexList) self.writeIfoFile( wordCount, indexFileSize, len(altIndexList), )
def _animation(f1): global s, app # Open socket try: s = socket(AF_INET, SOCK_DGRAM) s.bind(("", APRIL_DATA_PORT)) s.setblocking(0) # Server socket srv = socket(AF_INET, SOCK_STREAM) srv.bind(("0.0.0.0", 8080)) srv.listen(2) except: app.stop() raise client = None logfile = None # Axes for arena display ax = array( [min(ref[:, 0]), max(ref[:, 0]), min(ref[:, 1]), max(ref[:, 1])]) * 1.2 # Allowed tag IDS allow = set(corners + waypoints + ROBOT_TAGID) # Array size; must store all allowed tags N = max(allow) + 1 # Array holding all point locations pts = zeros((N, 4, 3), dtype=float) # Indicator array for point that are assumed static # and whose locations will be lowpass filtered statics = zeros(N, dtype=bool) statics[corners + waypoints] = True # Legend for point update indicator strings lbl = array(list(".+ld:*LD")) # (CONST) Reference locations for tag corners ang0 = array([-1 + 1j, 1 + 1j, 1 - 1j, -1 - 1j]) * -1j # (CONST) Point on a circle circ = exp(1j * linspace(0, 2 * pi, 16)) ### Initial values for variables # Configure sensor line-types sensorF = Sensor(':om', lw=2) sensorB = Sensor(':oc', lw=2) # Last message received from TagStreamer msg = None # Last waypoint visited M = 0 # Dynamic zoom scale for robot view zoom = None # Last homography prj = None # Start time T0 = now() # Time last waypoint message was sent lastWay = T0 - WAY_RATE - 1 # Number of messages received nmsg = 0 # ### MAIN LOOP ### # while len(waypoints) > M + 1: # continue until goal is reached # ### Read data from April # try: while True: # read data as fast as possible msg = s.recv(1 << 16) nmsg += 1 except SocketError, se: # until we've run out; last message remains in m pass # make sure we got something if not msg: continue # Parse tag information from UDP packet dat = json_loads(msg) msg = '' # Collect allowed tags h = empty_like(pts) h[:] = nan for d in dat: nm = d['i'] if not nm in allow: continue #if ~isnan(h[nm,0,0]): # print '(dup)', p = asarray(d['p']) / 100 h[nm, :, :2] = p h[nm, :, 2] = 1 # # at this point, all observed tag locations are in the dictionary h # ### Update pts array # # Tags seen in this frame fidx = ~isnan(h[:, 0, 0]) # Tags previously unseen uidx = isnan(pts[:, 0, 0]) # Tags to update directly: non static, or static and first time seen didx = (uidx & fidx) | ~statics if any(didx): pts[didx, ...] = h[didx, ...] # Tags to update with lowpass: static, seen and previously seen lidx = fidx & statics & ~uidx if any(lidx): pts[lidx, ...] *= (1 - alpha) pts[lidx, ...] += alpha * h[lidx, ...] # Print indicator telling operator what we did progress("%7.2f %5d " % (now() - T0, nmsg) + lbl[didx + 2 * lidx + 4 * fidx].tostring(), sameLine=True) # # Collect the corner tags and estimate homography nprj = None try: roi = array([mean(pts[nm], 0) for nm in corners]) # Homography mapping roi to ref nprj = fitHomography(roi, ref) except KeyError, ck: progress("-- missing corner %s" % str(c))
def train(): tok_path = get_tokenizer() model, vocab = get_mxnet_kogpt2_model(ctx=ctx) # tok = SentencepieceTokenizer(tok_path, num_best=0, alpha=0) data = pd.read_csv('Chatbot_data/ChatbotData.csv') max_len = opt.max_seq_len train_set = ChatDataset(data, tok_path, vocab, max_len=max_len) batch_size = opt.batch_size train_dataloader = mx.gluon.data.DataLoader(train_set, batch_size=batch_size, num_workers=5, shuffle=True) kogptqa = KoGPT2Chat(model) kogptqa.hybridize() # softmax cross entropy loss for classification loss_function = gluon.loss.SoftmaxCrossEntropyLoss() loss_function.hybridize() num_epochs = opt.num_epoch lr = 5e-5 trainer = gluon.Trainer(kogptqa.collect_params(), 'bertadam', { 'learning_rate': lr, 'epsilon': 1e-8, 'wd': 0.01 }) # LayerNorm과 Bias에는 Weight Decay를 적용하지 않는다. for _, v in kogptqa.collect_params('.*beta|.*gamma|.*bias').items(): v.wd_mult = 0.0 params = [ p for p in kogptqa.collect_params().values() if p.grad_req != 'null' ] # learning rate warmup accumulate = opt.accumulate step_size = batch_size * accumulate if accumulate else batch_size num_train_examples = len(train_set) num_train_steps = int(num_train_examples / step_size * num_epochs) warmup_ratio = 0.1 num_warmup_steps = int(num_train_steps * warmup_ratio) step_num = 0 all_model_params = kogptqa.collect_params() log_interval = 50 neg = -1e18 # Set grad_req if gradient accumulation is required if accumulate and accumulate > 1: for p in params: p.grad_req = 'add' start_train = time.now() max_gpu_load = 0 gpu_memory = 0 for epoch_id in range(num_epochs): step_loss = 0 for batch_id, (token_ids, mask, label) in enumerate(train_dataloader): if GPUtil.getGPUs()[0].load > max_gpu_load: max_gpu_load = GPUtil.getGPUs()[0].load if step_num < num_warmup_steps: new_lr = lr * step_num / num_warmup_steps else: non_warmup_steps = step_num - num_warmup_steps offset = non_warmup_steps / (num_train_steps - num_warmup_steps) new_lr = lr - offset * lr trainer.set_learning_rate(new_lr) with mx.autograd.record(): # load data to GPU or GPU token_ids = token_ids.as_in_context(ctx) mask = mask.as_in_context(ctx) label = label.as_in_context(ctx) # forward computation out = kogptqa(token_ids) masked_out = nd.where( mask.expand_dims(axis=2).repeat(repeats=out.shape[2], axis=2), out, neg * nd.ones_like(out)) # loss for responses exincluding MASK and PAD ls = loss_function(masked_out, label).sum() / mask.sum() # backward computation ls.backward() if not accumulate or (batch_id + 1) % accumulate == 0: trainer.allreduce_grads() nlp.utils.clip_grad_global_norm(params, 1) trainer.update(accumulate if accumulate else 1) step_num += 1 if accumulate and accumulate > 1: # set grad to zero for gradient accumulation all_model_params.zero_grad() step_loss += ls.asscalar() if GPUtil.getGPUs()[0].load > max_gpu_load: max_gpu_load = GPUtil.getGPUs()[0].load gpu_memory = GPUtil.getGPUs()[0].memoryUsed if step_num % log_interval == 0 and step_num > 0: print( '[Epoch {} Batch {}/{}] loss={:.4f}, lr={:.10f}, train ppl={:.3f}' .format(epoch_id + 1, batch_id + 1, len(train_dataloader), step_loss / log_interval, trainer.learning_rate, math.exp(step_loss / log_interval))) step_loss = 0 print(1234) log_train(gpu_memory, max_gpu_load, start_train, num_epochs, batch_size) logging.info('saving model file to {}'.format(opt.model_params)) kogptqa.save_parameters(opt.model_params)
def render_time(self, ctx, data): ctx.remember(now(), itodo.ITimer) return ctx.tag
def writeGeneral(self) -> None: """ Build StarDict dictionary in general case. Every item definition may consist of an arbitrary number of articles. sametypesequence option is not used. """ dictMark = 0 altIndexList = [] # list of tuples (b"alternate", wordIndex) dictFile = open(self._filename + ".dict", "wb") idxFile = open(self._filename + ".idx", "wb") indexFileSize = 0 t0 = now() wordCount = 0 defiFormatCounter = Counter() if not isdir(self._resDir): os.mkdir(self._resDir) entryI = -1 for entry in self._glos: if entry.isData(): entry.save(self._resDir) continue entryI += 1 words = entry.getWords() # list of strs word = words[0] # str defis = entry.getDefis() # list of strs entry.detectDefiFormat() # call no more than once defiFormat = entry.getDefiFormat() defiFormatCounter[defiFormat] += 1 if defiFormat not in ("m", "h"): defiFormat = "m" b_dictBlock = b"" for alt in words[1:]: altIndexList.append((alt.encode("utf-8"), entryI)) b_dictBlock += (defiFormat + defis[0]).encode("utf-8") + b"\x00" for altDefi in defis[1:]: b_dictBlock += (defiFormat + altDefi).encode("utf-8") + b"\x00" dictFile.write(b_dictBlock) blockLen = len(b_dictBlock) b_idxBlock = word.encode("utf-8") + b"\x00" + \ intToBinStr(dictMark, 4) + \ intToBinStr(blockLen, 4) idxFile.write(b_idxBlock) dictMark += blockLen indexFileSize += len(b_idxBlock) wordCount += 1 dictFile.close() idxFile.close() if not os.listdir(self._resDir): os.rmdir(self._resDir) log.info(f"Writing dict file took {now()-t0:.2f} seconds") log.debug("defiFormatsCount = " + pformat(defiFormatCounter.most_common())) self.writeSynFile(altIndexList) self.writeIfoFile(wordCount, indexFileSize, len(altIndexList))
def saveLiveConfLoop(): tm = now() if tm - lastLiveConfChangeTime > saveLiveConfDelay: saveLiveConf() return False ## Finish loop return True ## Continue loop
def run(): t0 = now() # parse command line options known_args, beam_args = runtime_args() # BigQuery utility bq_utils = BigQueryUtils() options = PipelineOptions(beam_args) options.view_as(SetupOptions).save_main_session = True with beam.Pipeline(options=options) as p: rows = (p | beam.io.ReadFromText(known_args.input, skip_header_lines=1) | beam.ParDo(BeamReadCSV(header_cols=FLIGHTS_CSV_COLUMNS)) | beam.ParDo(BeamTransformRecords(), date_fmt='%Y-%m-%d', time_fmt='%H%M')) # load the routes table into a lookup dict sql = f"""select airline, src, dest from {known_args.routes_table}""" routes = bq_utils.execute_as_dict(sql, keycols=['airline', 'src', 'dest']) # lookup routes rows, routes_rejects, missing_routes = ( rows | beam.ParDo(BeamLookupRoute(), routes=routes).with_outputs( 'rejects', 'missing_routes', main='main')) # write parquet output files output = (rows | beam.io.WriteToParquet( os.path.join(known_args.output, 'flights'), schema=datamodel_flights_parquet_schema(), file_name_suffix='.parquet')) # write missing routes to another output as CSV output_routes = ( missing_routes | "gbr" >> beam.GroupByKey() # calculate distinct missing routes | "missing_routes_csv" >> beam.Map( lambda e: ','.join(list(e[0])) ) # csv output the key (e[0] of key value tuple) which is (airline,src,dest) | "missing_routes_out" >> beam.io.WriteToText( os.path.join(known_args.output, 'rejects/missing-routes'), file_name_suffix='.csv', header='airline,src,dest')) # alternative: write (simple) newline delimited json output files # a very flexible output file format for bigquery and other big data tools # much slower to write and larger in size than binary formats such as Parquet, ORC, or Avro # but provides flexibility over schema for smaller data files # larger file sizes should use Avro, Parquet, ORC. Avro provides fastest write speeds where # parquet and orc provide faster read performance for analytical queries # output = (rows # | beam.Map(lambda e: {k: v if k != 'flight_date' else v.strftime('%Y-%m-%d') for k, v in e.items()}) # convert flight_date back to string type for json conversion # | beam.Map(lambda e: json.dumps(e)) # json dump row # | beam.io.WriteToText(os.path.join(known_args.output, 'flights'), # file_name_suffix='.json') # ) logger.info("beam pipiline completed.") # create bigquery external table and insert into bq flights table bq_utils.create_external_table(known_args.flights_ext_table, source_uris=os.path.join( known_args.output, "flights*.parquet"), source_format='PARQUET', delete_if_exists=True) # create and replace existing bigquery flights table bq_utils.create_table(known_args.flights_table, schema=datamodel_flights_bigquery_schema(), delete_if_exists=True) # insert into table as select (itas) statement sql = f""" INSERT INTO `{known_args.flights_table}` SELECT a.day_of_week, a.flight_date, a.airline, a.tailnumber, a.flight_number, a.src, a.src_city, a.src_state, a.dest, a.dest_city, a.dest_state, PARSE_TIME('%H:%M:%S', a.departure_time) as departure_time, PARSE_TIME('%H:%M:%S', a.actual_departure_time) as actual_departure_time, a.departure_delay, a.taxi_out, PARSE_TIME('%H:%M:%S', a.wheels_off) as wheels_off, PARSE_TIME('%H:%M:%S', a.wheels_on) as wheels_on, a.taxi_in, PARSE_TIME('%H:%M:%S', a.arrival_time) as arrival_time, PARSE_TIME('%H:%M:%S', a.actual_arrival_time) as actual_arrival_time, a.arrival_delay, a.cancelled, a.cancellation_code, a.flight_time, a.actual_flight_time, a.air_time, a.flights, a.distance, a.airline_delay, a.weather_delay, a.nas_delay, a.security_delay, a.late_aircraft_delay, -- CONCAT(a.flight_date, '_', a.airline, '_', a.flight_number) AS flightDate_airline_flightNumber FROM `{known_args.flights_ext_table}` a """ # insert records form parquet external table into final bq managed flights table r = bq_utils.execute(sql) logger.info(f"total time: {(now() - t0):,.6f} secs")
def was_unused_last(self, seconds): return now() > self.__last_access + seconds
def refresh(self): """ Set the time of last access to now. """ self.__last_access = now() return self
def power(time, flux, minfreq, maxfreq, step, chunksize): """ This function returns the power spectrum of the desired star. Arguments: - 'time': Time in megaseconds from the timeserie analysis. - 'flux': Photometry data from the timeserie analysis. - 'minfreq': The lower bound for the frequency interval - 'maxfreq': The upper bound for the frequency interval - 'step': The spacing between frequencies. - 'chunksize': Define a chunk for the least mean square Fourier in order to save time. """ # Import modules import numpy as np from time import time as now # Generate cyclic frequencies freq = np.arange(minfreq, maxfreq, step) # Generate list to store the calculated power power = np.zeros((len(freq), 1)) # Convert frequencies to angular frequencies nu = 2 * np.pi * freq # Iterate over the frequencies timerStart = now() # After this many frequencies, print progress info print_every = 75e6 // len(time) # Ensure chunksize divides print_every print_every = (print_every // chunksize) * chunksize for i in range(0, len(nu), chunksize): # Define chunk j = min(i + chunksize, len(nu)) rows = j - i if i % print_every == 0: # Info-print elapsedTime = now() - timerStart if i == 0: totalTime = 462 / 6000 * len(nu) else: totalTime = (elapsedTime / i) * len(nu) print("Progress: %.2f%% (%d..%d of %d) " "Elapsed: %.2f s Estimated total: %.2f s" % (np.divide(100.0 * i, len(nu)), i, j, len(nu), elapsedTime, totalTime)) """ The outer product is calculated. This way, the product between time and ang. freq. will be calculated elementwise; one column per frequency. This is done in order to save computing time. """ nutime = np.outer(time, nu[i:j]) """ An array with the measured flux is made so it has the same size as "nutime", since we want to multiply the two. """ fluxrep = np.repeat(flux[:, np.newaxis], repeats=rows, axis=1) # The Fourier subroutine sin_nutime = np.sin(nutime) cos_nutime = np.cos(nutime) s = np.sum(sin_nutime * fluxrep, axis=0) c = np.sum(cos_nutime * fluxrep, axis=0) ss = np.sum(sin_nutime**2, axis=0) cc = np.sum(cos_nutime**2, axis=0) sc = np.sum(sin_nutime * cos_nutime, axis=0) alpha = ((s * cc) - (c * sc)) / ((ss * cc) - (sc**2)) beta = ((c * ss) - (s * sc)) / ((ss * cc) - (sc**2)) power[i:j] = np.reshape(alpha**2 + beta**2, (rows, 1)) power = power.reshape(-1, 1) freq = freq.reshape(-1, 1) elapsedTime = now() - timerStart print('Computed power spectrum with chunk size %d in %.2f s' % (chunksize, elapsedTime)) return (freq, power)
'connected': timing_connected_layer, 'dropout': timing_dropout_layer, 'input': timing_input_layer, 'logistic': timing_logistic_layer, 'maxpool': timing_maxpool_layer, 'route': timing_route_layer, 'shortcut': timing_shortcut_layer, 'shuffler': timing_shuffler_layer, 'softmax': timing_softmax_layer, 'upsample': timing_upsample_layer, 'yolo': timing_yolo_layer } timing = [] for input_shape in input_shapes: tic = now() times = timing_layers[args.layer](input_shape) toc = now() print('Elapsed time: {:.3f} sec'.format(toc - tic)) timing += [pd.DataFrame(times)] timing = pd.concat(timing) if args.out: timing.to_csv(args.out, sep=',', header=True, index=False) else: print(timing)
def updateLine(self): self.label.set_label(self.lines[self.index]) self.startXpos = -self.label.width if self.label.rtl else screenWidth self.startTime = now()
def writeCompact(self, defiFormat): """ Build StarDict dictionary with sametypesequence option specified. Every item definition consists of a single article. All articles have the same format, specified in defiFormat parameter. Parameters: defiFormat - format of article definition: h - html, m - plain text """ dictMark = 0 altIndexList = [] # list of tuples (b"alternate", wordIndex) dictFile = open(self._filename + ".dict", "wb") idxFile = open(self._filename + ".idx", "wb") indexFileSize = 0 t0 = now() wordCount = 0 if not isdir(self._resDir): os.mkdir(self._resDir) entryI = -1 for entry in self._glos: if entry.isData(): entry.save(self._resDir) continue entryI += 1 words = entry.getWords() # list of strs word = words[0] # str defis = entry.getDefis() # list of strs b_dictBlock = b"" for alt in words[1:]: altIndexList.append((alt.encode("utf-8"), entryI)) b_dictBlock += (defis[0]).encode("utf-8") for altDefi in defis[1:]: b_dictBlock += b"\x00" + (altDefi).encode("utf-8") dictFile.write(b_dictBlock) blockLen = len(b_dictBlock) b_idxBlock = word.encode("utf-8") + b"\x00" + \ intToBinStr(dictMark, 4) + \ intToBinStr(blockLen, 4) idxFile.write(b_idxBlock) dictMark += blockLen indexFileSize += len(b_idxBlock) wordCount += 1 dictFile.close() idxFile.close() if not os.listdir(self._resDir): os.rmdir(self._resDir) log.info(f"Writing dict file took {now()-t0:.2f} seconds") log.debug("defiFormat = " + pformat(defiFormat)) self.writeSynFile(altIndexList) self.writeIfoFile( wordCount, indexFileSize, len(altIndexList), defiFormat, )
def updateFocusTime(*args): global focusTime focusTime = now()
def did_complete(self, order): print "Completed order " + ("matches" if order == self.pending_order else "does not match") + " the pending order" self.executed_orders.append([order, now()]) self.pending_order = None
def log(*args): if not printed: print() printed.append(1) print("\t ->", now(), *args)
def render_end(self, ctx, data): return ctx.tag["%.3f" % (now() - itodo.ITimer(ctx))]
# Initializes all necessary parameters to run model session = tf.InteractiveSession() # Session to run model train_writer = tf.summary.FileWriter('convNetSummary', session.graph) # File to write reporting saver = tf.train.Saver( ) # Handler for simple initialization or restoration of save files tf.train.start_queue_runners( ) # Coordinator to enqueue records with FixedLengthRecordReader # Initializes value of variable appropriately cifar10_model.load_variable_value(saver, session, reset_variables) to_print_1 = '%d, train CE: %g, train ACC: %g.' to_print_2 = '%d, train CE: %g, train ACC: %g. Inputs per second: %0.3f. Seconds per batch: %0.3f.' start_time = now() for index in range(100000): # Runs one training/summary step _, cross_entropy_value, accuracy_value, summary = session.run( [train_step, train_cross_entropy, train_accuracy, merged]) # Periodically tests evolution in accuracy for test set and training set if index % progress_report_interval == 0: # Saves current state of weights saver.save(session, variable_save_file) if index == 0: print(to_print_1 % (index, cross_entropy_value, accuracy_value))
# If robot not seen --> we're done if isnan(zc[ROBOT_TAGID]): yield continue # ### robot tag related updates # # robot tag corners rbt = z[ROBOT_TAGID, ...] # robot heading angle phasor ang = mean((rbt - zc[ROBOT_TAGID]) / ang0) ang /= abs(ang) # If logging data --> put into log if logfile is not None: lo = [ now(), zc[ROBOT_TAGID].real, zc[ROBOT_TAGID].imag, ang, int(zc[waypoints[M]].real), int(zc[waypoints[M]].imag) ] logfile.write(", ".join(["%.3f" % x for x in lo]) + "\n") # indicate robot a1.plot(zc[ROBOT_TAGID].real, zc[ROBOT_TAGID].imag, '*r', ms=15) # ### robot relative view # a2 = f1.add_subplot(122) # # Show the waypoints c = zc[waypoints] vc = ~isnan(c) cr = (c - zc[ROBOT_TAGID]) / ang
def render_starttimer(self, ctx, data): ctx.remember(now(), ITimer) return ctx.tag
def writeCompact(self, defiFormat): """ Build StarDict dictionary with sametypesequence option specified. Every item definition consists of a single article. All articles have the same format, specified in defiFormat parameter. Parameters: defiFormat - format of article definition: h - html, m - plain text """ dictMark = 0 altIndexList = [] # list of tuples (b"alternate", entryIndex) dictFile = open(self._filename + ".dict", "wb") idxFile = open(self._filename + ".idx", "wb") indexFileSize = 0 t0 = now() wordCount = 0 if not isdir(self._resDir): os.mkdir(self._resDir) entryIndex = -1 while True: entry = yield if entry is None: break if entry.isData(): entry.save(self._resDir) continue entryIndex += 1 words = entry.l_word # list of strs word = words[0] # str defi = self.fixDefi(entry.defi, defiFormat) # defi is str for alt in words[1:]: altIndexList.append((alt.encode("utf-8"), entryIndex)) b_dictBlock = defi.encode("utf-8") dictFile.write(b_dictBlock) blockLen = len(b_dictBlock) b_idxBlock = word.encode("utf-8") + b"\x00" + \ uint32ToBytes(dictMark) + \ uint32ToBytes(blockLen) idxFile.write(b_idxBlock) dictMark += blockLen indexFileSize += len(b_idxBlock) wordCount += 1 dictFile.close() idxFile.close() if not os.listdir(self._resDir): os.rmdir(self._resDir) log.info(f"Writing dict file took {now()-t0:.2f} seconds") self.writeSynFile(altIndexList) self.writeIfoFile( wordCount, indexFileSize, len(altIndexList), defiFormat=defiFormat, )
def render_stoptimer(self, ctx, data): start = ITimer(ctx) return ctx.tag['%s' % (now()-start)]
def _wait(self, parsed_args): from time import time as now, sleep min_score = parsed_args.score delay = parsed_args.delay deadline = now() + delay descr = [] ko = -1 exc_msg = ("Timeout ({0}s) while waiting for the services to get a " "score >= {1}, {2}") def maybe_unlock(allsrv): if not parsed_args.unlock: return if not allsrv: return self.app.client_manager.conscience.unlock_score(allsrv) def check_deadline(): if now() > deadline: if ko < 0: msg = exc_msg.format(delay, min_score, "proxy and/or conscience not ready") else: msg = exc_msg.format(delay, min_score, "still %d are not." % ko) for srv in descr: if srv['score'] < min_score: self.log.warn("%s %s %s", srv['type'], srv.get('id', None), srv['score']) raise Exception(msg) interval = _sleep_interval(0.0, 1.0, 2.0, 4.0) types = parsed_args.types if not parsed_args.types: while True: check_deadline() sleep(next(interval)) try: types = self.app.client_manager.conscience.service_types() break except OioNetworkException as exc: self.log.debug("Proxy error: %s", exc) except ServiceBusy as exc: self.log.debug("Conscience busy: %s", exc) interval = _sleep_interval(0.0, 1.0, 2.0, 4.0) while True: check_deadline() maybe_unlock(descr) sleep(next(interval)) descr = [] ko = -1 try: for typ in types: tmp = self.app.client_manager.conscience.all_services(typ) for srv in tmp: srv['type'] = typ descr += tmp except OioNetworkException as exc: self.log.debug("Proxy error: %s", exc) continue except ServiceBusy as exc: self.log.debug("Conscience busy: %s", exc) continue ko = len([s['score'] for s in descr if s['score'] < min_score]) if ko > 0: self.log.debug("Still %d services down", ko) continue # If a minimum has been specified, let's check we have enough # services if parsed_args.count: ok = len([s for s in descr if s['score'] >= min_score]) if ok < parsed_args.count: self.log.debug("Only %d services up", ok) continue # No service down, and enough services, we are done. for srv in descr: yield srv['type'], srv['addr'], srv['score'] return
# # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see <http://www.gnu.org/licenses/lgpl.txt>. # Also avalable in /usr/share/common-licenses/LGPL on Debian systems # or /usr/share/licenses/common/LGPL/license.txt on ArchLinux import time from time import localtime, strftime from time import time as now from gi.repository.GObject import timeout_add from gi.repository import GdkPixbuf from scal3.ui_gtk import * from scal3.ui_gtk.font_utils import * time_rem = lambda: int(1000 * (1.01 - now() % 1)) class ClockLabel(gtk.Label): def __init__(self, bold=False, seconds=True, selectable=False): gtk.Label.__init__(self) self.set_use_markup(True) self.set_selectable(selectable) self.bold = bold self.seconds = seconds self.running = False #self.connect('button-press-event', self.button_press) self.start() #??? def start(self): self.running = True
def _wait(self): """Rate limit the number of send events.""" end = self.last_send_event + COMPAT_MODE_WAIT_TIME while now() < end: pass
def runMCMCmodel(args): """ Simulate the survey data and run the MCMC luminosity calibration model. Parameters ---------- args - Command line arguments """ mcmcParams = args['mcmcString'] surveyParams = args['surveyString'] priorParams = args['priorsString'] maxIter = int(mcmcParams[0]) burnIter = int(mcmcParams[1]) thinFactor = int(mcmcParams[2]) walkerFactor = int(mcmcParams[3]) minParallax = float(surveyParams[1]) maxParallax = float(surveyParams[2]) meanAbsoluteMagnitude = float(surveyParams[3]) varianceAbsoluteMagnitude = float(surveyParams[4]) if surveyParams[5] == 'Inf': magLim = np.Inf else: magLim = float(surveyParams[5]) simulatedSurvey = U.UniformDistributionSingleLuminosity( int(surveyParams[0]), float(surveyParams[1]), float(surveyParams[2]), float(surveyParams[3]), float(surveyParams[4]), surveyLimit=magLim) #simulatedSurvey.setRandomNumberSeed(53949896) simulatedSurvey.generateObservations() numberOfStarsInSurvey = simulatedSurvey.numberOfStarsInSurvey # Calculate initial guesses for the true parallaxes and absolute magnitudes of the stars. clippedObservedParallaxes = simulatedSurvey.observedParallaxes.clip( minParallax, maxParallax) initialAbsMagGuesses = simulatedSurvey.observedMagnitudes + 5.0 * np.log10( clippedObservedParallaxes) - 10.0 meanAbsoluteMagnitudeGuess = initialAbsMagGuesses.mean() # Initial guesses for hyper parameters (mean absolute magnitude and sigma^2) # # Mean absolute magnitude uniform on (meanAbsMagLow, meanAbsMagHigh) meanAbsMagLow = float(priorParams[0]) meanAbsMagHigh = float(priorParams[1]) # Variance has 1/x distribution with lower and upper limit as prior varianceLow = float(priorParams[2]) varianceHigh = float(priorParams[3]) varianceInit = (varianceHigh - varianceLow) / (np.log(varianceHigh) - np.log(varianceLow)) initialParameters = np.concatenate( (np.array([meanAbsoluteMagnitudeGuess, varianceInit]), clippedObservedParallaxes, initialAbsMagGuesses)) # Parameters for emcee ln-posterior function posteriorDict = { 'minParallax': minParallax, 'maxParallax': maxParallax, 'muLow': meanAbsMagLow, 'muHigh': meanAbsMagHigh, 'varLow': varianceLow, 'varHigh': varianceHigh } observations = np.concatenate((simulatedSurvey.observedParallaxes, simulatedSurvey.observedMagnitudes)) observationalErrors = inverseVariance( np.concatenate( (simulatedSurvey.parallaxErrors, simulatedSurvey.magnitudeErrors))) # MCMC sampler parameters ndim = 2 * numberOfStarsInSurvey + 2 nwalkers = walkerFactor * ndim # Generate initial positions for each walker initialPositions = [np.empty((ndim)) for i in xrange(nwalkers)] initialPositions[0] = initialParameters for i in xrange(nwalkers - 1): ranMeanAbsMag = np.random.rand() * (meanAbsMagHigh - meanAbsMagLow) + meanAbsMagLow ranVariance = random_oneOverX(varianceLow, varianceHigh, 1) ranParallaxes = np.zeros_like(clippedObservedParallaxes) for j in xrange(numberOfStarsInSurvey): #if (i<nwalkers/2): ranParallaxes[j] = clippedObservedParallaxes[ j] + simulatedSurvey.parallaxErrors[j] * np.random.randn() #else: # ranParallaxes[j]=random_oneOverXFourth(minParallax,maxParallax,1) ranAbsMag = np.sqrt(ranVariance) * np.random.randn( numberOfStarsInSurvey) + ranMeanAbsMag initialPositions[i + 1] = np.concatenate( (np.array([ranMeanAbsMag, ranVariance]), ranParallaxes.clip(minParallax, maxParallax), ranAbsMag)) print '** Building sampler **' sampler = emcee.EnsembleSampler( nwalkers, ndim, UniformSpaceDensityGaussianLFBookemcee, threads=4, args=[posteriorDict, observations, observationalErrors]) # burn-in print '** Burn in **' start = now() pos, prob, state = sampler.run_mcmc(initialPositions, burnIter) print '** Finished burning in **' print ' Time (s): ', now() - start print 'Median acceptance fraction: ', np.median( sampler.acceptance_fraction) print( 'Acceptance fraction IQR: {0}'.format( np.percentile(sampler.acceptance_fraction, 25)) + ' -- {0}'.format(np.percentile(sampler.acceptance_fraction, 75))) correlationTimes = sampler.acor print 'Autocorrelation times: ' print ' Mean absolute magnitude: ', correlationTimes[0] print ' Variance absolute magnitude: ', correlationTimes[1] print ' Median for parallaxes: ', np.median( correlationTimes[2:numberOfStarsInSurvey + 2]) print ' Median for magnitudes: ', np.median( correlationTimes[numberOfStarsInSurvey + 2:]) print # final chain sampler.reset() start = now() print '** Starting sampling **' sampler.run_mcmc(pos, maxIter, rstate0=state, thin=thinFactor) print '** Finished sampling **' print ' Time (s): ', now() - start print 'Median acceptance fraction: ', np.median( sampler.acceptance_fraction) print( 'Acceptance fraction IQR: {0}'.format( np.percentile(sampler.acceptance_fraction, 25)) + ' -- {0}'.format(np.percentile(sampler.acceptance_fraction, 75))) correlationTimes = sampler.acor print 'Autocorrelation times: ' print ' Mean absolute magnitude: ', correlationTimes[0] print ' Variance absolute magnitude: ', correlationTimes[1] print ' Median for parallaxes: ', np.median( correlationTimes[2:numberOfStarsInSurvey + 2]) print ' Median for magnitudes: ', np.median( correlationTimes[numberOfStarsInSurvey + 2:]) # Extract the samples of the posterior distribution chain = sampler.flatchain # Point estimates of mean Absolute Magnitude and its standard deviation. meanAbsoluteMagnitudeSamples = chain[:, 0].flatten() varAbsoluteMagnitudeSamples = chain[:, 1].flatten() estimatedAbsMag = meanAbsoluteMagnitudeSamples.mean() errorEstimatedAbsMag = meanAbsoluteMagnitudeSamples.std() estimatedVarMag = varAbsoluteMagnitudeSamples.mean() errorEstimatedVarMag = varAbsoluteMagnitudeSamples.std() print "emcee estimates" print "mu_M={:4.2f}".format(estimatedAbsMag) + " +/- {:4.2f}".format( errorEstimatedAbsMag) print "sigma^2_M={:4.2f}".format(estimatedVarMag) + " +/- {:4.2f}".format( errorEstimatedVarMag) # Plot results # MAP estimates muDensity = gaussian_kde(meanAbsoluteMagnitudeSamples) mapValueMu = scipy.optimize.fmin(lambda x: -1.0 * muDensity(x), np.median(meanAbsoluteMagnitudeSamples), maxiter=1000, ftol=0.0001) varDensity = gaussian_kde(varAbsoluteMagnitudeSamples) mapValueVar = scipy.optimize.fmin(lambda x: -1.0 * varDensity(x), np.median(varAbsoluteMagnitudeSamples), maxiter=1000, ftol=0.0001) fig = plt.figure(figsize=(12, 8.5)) fig.add_subplot(2, 2, 1) x = np.linspace(meanAbsoluteMagnitudeSamples.min(), meanAbsoluteMagnitudeSamples.max(), 500) plt.plot(x, muDensity(x), 'k-') plt.axvline(meanAbsoluteMagnitude, linewidth=2, color="red") plt.xlabel("$\\mu_M$") plt.ylabel("$P(\\mu_M)$") fig.add_subplot(2, 2, 2) x = np.linspace(varAbsoluteMagnitudeSamples.min(), varAbsoluteMagnitudeSamples.max(), 500) plt.plot(x, varDensity(x), 'k-') plt.axvline(varianceAbsoluteMagnitude, linewidth=2, color="red") plt.xlabel("$\\sigma^2_M$") plt.ylabel("$P(\\sigma^2_M)$") fig.add_subplot(2, 2, 3) plt.hexbin(meanAbsoluteMagnitudeSamples, varAbsoluteMagnitudeSamples, C=None, bins='log', cmap=cm.gray_r) plt.xlabel("$\\mu_M$") plt.ylabel("$\\sigma^2_M$") plt.figtext(0.55, 0.4, "$\\widetilde{\\mu_M}=" + "{:4.2f}".format(estimatedAbsMag) + "$ $\\pm$ ${:4.2f}$".format(errorEstimatedAbsMag), ha='left') plt.figtext( 0.75, 0.4, "$\\mathrm{MAP}(\\widetilde{\\mu_M})=" + "{:4.2f}".format(mapValueMu[0]) + "$") plt.figtext(0.55, 0.35, "$\\widetilde{\\sigma^2_M}=" + "{:4.2f}".format(estimatedVarMag) + "$ $\\pm$ ${:4.2f}$".format(errorEstimatedVarMag), ha='left') plt.figtext( 0.75, 0.35, "$\\mathrm{MAP}(\\widetilde{\\sigma^2_M})=" + "{:4.2f}".format(mapValueVar[0]) + "$") titelA = ("$N_\\mathrm{stars}" + "={0}".format(numberOfStarsInSurvey) + "$, True values: $\\mu_M={0}".format(meanAbsoluteMagnitude) + "$, $\\sigma^2_M={0}".format(varianceAbsoluteMagnitude) + "$") titelB = ("Iterations = {0}".format(maxIter) + ", Burn = {0}".format(burnIter) + ", Thin = {0}".format(thinFactor)) plt.suptitle(titelA + "\\quad\\quad " + titelB) titelA = ("$N_\\mathrm{stars}" + "={0}".format(numberOfStarsInSurvey) + "$, True values: $\\mu_M={0}".format(meanAbsoluteMagnitude) + "$, $\\sigma^2_M={0}".format(varianceAbsoluteMagnitude) + "$") titelB = ("Iterations = {0}".format(maxIter) + ", Burn = {0}".format(burnIter) + ", Thin = {0}".format(thinFactor)) plt.suptitle(titelA + "\\quad\\quad " + titelB) titelC = [] titelC.append("MCMC sampling with emcee") titelC.append("$N_\\mathrm{walkers}" + "={0}".format(nwalkers) + "$, $N_\\mathrm{dim}" + "={0}".format(ndim) + "$") plt.figtext(0.55, 0.15, titelC[0], horizontalalignment='left') plt.figtext(0.60, 0.10, titelC[1], horizontalalignment='left') priorInfo = [] priorInfo.append( "Prior on $\\mu_M$: flat $\\quad{0}".format(meanAbsMagLow) + "<\\mu_M<{0}".format(meanAbsMagHigh) + "$") priorInfo.append( "Prior on $\\sigma^2_M$: $1/\\sigma^2_M\\quad{0}".format(varianceLow) + "<\\sigma^2_M<{0}".format(varianceHigh) + "$") plt.figtext(0.55, 0.25, priorInfo[0], horizontalalignment='left') plt.figtext(0.55, 0.20, priorInfo[1], horizontalalignment='left') if (args['pdfOutput']): plt.savefig('luminosityCalibrationResultsEmcee.pdf') elif (args['pngOutput']): plt.savefig('luminosityCalibrationResultsEmcee.png') elif (args['epsOutput']): plt.savefig('luminosityCalibrationResultsEmcee.eps') else: plt.show()
def new_experiment(self, **experiment_info): # # load from file # self.ensure_loaded() # add basic data to the experiment # there are 3 levels: # - self.collection_keeper.local_data (root) # - self.experiment_keeper # - self.experiment_info_keeper self.experiment_keeper = self.collection_keeper.sub_record_keeper( experiment_number=self. prev_experiment_local_data["experiment_number"] + 1 if not self.prev_experiment_local_data["had_error"] else self.prev_experiment_local_data["experiment_number"], error_number=self.prev_experiment_local_data["error_number"] + 1, had_error=True, experiment_start_time=now(), ) # create experiment record keeper if len(experiment_info) == 0: self.experiment_info_keeper = self.experiment_keeper else: self.experiment_info_keeper = self.experiment_keeper.sub_record_keeper( **experiment_info) def save_experiment(_, error, traceback): # mutate the root one based on having an error or not no_error = error is None experiment_info = self.experiment_keeper.local_data experiment_info["experiment_end_time"] = now() experiment_info["experiment_duration"] = experiment_info[ "experiment_end_time"] - experiment_info[ "experiment_start_time"] if no_error: experiment_info["had_error"] = False experiment_info["error_number"] = 0 # refresh the all_record_keepers dict # especially after mutating the self.experiment_keeper.local_data # (this ends up acting like a set, but keys are based on mutable values) self.record_keepers = { super_hash(each_value): each_value for each_value in self.record_keepers.values() } # # save to file # # ensure folder exists import os os.makedirs(os.path.dirname(self.file_path), exist_ok=True) self.prev_experiment_local_data = self.experiment_keeper.local_data data = (self.collection_keeper.local_data, self.experiment_keeper.local_data, self.record_keepers, self._records) # update self encase multiple experiments are run without re-reading the file print("Saving " + str(len(self._records)) + " records") large_pickle_save(data, self.file_path) print("Records saved to: " + self.file_path) # re-throw the error if not no_error: print( f'There was an error when running an experiment. Experiment collection: "{self.collection_name}"' ) print(f'However, the partial experiment data was saved') experiment_number = self.experiment_keeper.local_data[ "experiment_number"] error_number = self.experiment_keeper.local_data[ "error_number"] print( f'This happend on:\n dict(experiment_number={experiment_number}, error_number={error_number})' ) raise error return Experiment(experiment_info_keeper=self.experiment_info_keeper, save_experiment=save_experiment)
def animateStart(self): self.updateLine() self.startTime = now() self.animateUpdate()