def create_analytic(*, model: Any, instance: Any, request: Any) -> None: analytic_signal.send(sender=model, instance=instance, request=request) logger.success(f"analytic data was created for {instance}")
optimizer = torch.optim.Adam( [ { 'params': dt_model.gp_layer.parameters() }, { 'params': likelihood.parameters() }, ], lr=0.01, ) dt_model.train() likelihood.train() # We would have to set a new loss object each time. mll = gpytorch.mlls.VariationalELBO(likelihood, dt_model.gp_layer, num_data=num_tasks) optimizer.zero_grad() output = dt_model(test_shape_x) loss = -mll(output, test_shape_y) logger.error(output.rsample().size()) logger.success(loss) loss.backward() optimizer.step() new_shape_x = torch.randn(2, 4, 5, 5) output = dt_model(new_shape_x)
def create_animation( frame_folder: Path, animation_folder: Path, observable: str, animation_type: str, time_slice: Optional[int] = None, frame_rate: Optional[int] = 10, ) -> None: """ Method for creating animations from generated volumetric figures. Args: frame_folder: folder path to figures that will be be stitched together. animation_folder: folder path to place animations in. observable: observable we are creating an animation.. animation_type: format of animation. Available: 'gif', 'avi' or 'mp4' time_slice: optional, eucl time slice. frame_rate: frames per second of animation. Raises: NameError: if animation_type is not recognized. """ # Removes spaces observable = observable.replace(" ", "_") input_paths = frame_folder / "frame_t%02d.png" if time_slice: animation_path = animation_folder / ( f"{observable.lower()}_{time_slice}.{animation_type}" ) else: animation_path = animation_folder / ( f"{observable.lower()}.{animation_type}" ) if animation_type == "gif": cmd = [ "convert", "-delay", "1", "-loop", "0", str(frame_folder / "*.png"), str(animation_path), ] elif animation_type == "mp4": cmd = [ "ffmpeg", "-r", str(frame_rate), "-start_number", "0", "-i", str(input_paths), "-c:v", "libx264", "-crf", "0", "-preset", "veryslow", "-c:a", "libmp3lame", "-b:a", "320k", "-y", str(animation_path), ] elif animation_type == "avi": cmd = [ "ffmpeg", "-r", str(frame_rate), "-i", str(input_paths), "-y", "-qscale:v", "0", str(animation_path), ] else: raise NameError( f"{animation_type} is not a recognized animation type." ) logger.info(f"Running command: {' '.join(cmd)}") proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) _ = proc.stdout.read() logger.success(f"Animation {animation_path} created.")
logger.info("Starting the segmenter control process (step 4/4)") segmenter_thread = planktoscope.segmenter.SegmenterProcess( shutdown_event, "/home/pi/data" ) segmenter_thread.start() # Starts the module process # Uncomment here as needed # logger.info("Starting the module process") # module_thread = planktoscope.module.ModuleProcess(shutdown_event) # module_thread.start() logger.info("Starting the display module") display = planktoscope.display.Display() logger.success("Looks like everything is set up and running, have fun!") planktoscope.light.ready() while run: # TODO look into ways of restarting the dead threads logger.trace("Running around in circles while waiting for someone to die!") if not stepper_thread.is_alive(): logger.error("The stepper process died unexpectedly! Oh no!") break if imager_thread and not imager_thread.is_alive(): logger.error("The imager process died unexpectedly! Oh no!") break if not segmenter_thread.is_alive(): logger.error("The segmenter process died unexpectedly! Oh no!") break time.sleep(1)
def main(): """ Main gameloop driver. """ logger.info("Starting game") pygame.init() screen = pygame.display.set_mode((WIDTH, HEIGHT)) clock = pygame.time.Clock() screen.fill(pygame.Color("white")) gs = GameState() load_images() running = True # Flag for when a move is made, triggers get_valid_moves-call move_made = False valid_moves = gs.get_valid_moves() # Keeps track og last clicked square: (x, y) selected_square = () # Keeps track of player clicks: [(x, y), (x, y)] player_clicks = [] while running: for event in pygame.event.get(): if event.type == pygame.QUIT: logger.info("Exiting") running = False elif event.type == pygame.MOUSEBUTTONDOWN: pos = pygame.mouse.get_pos() col, row = pos[0] // SQUARE_SIZE, pos[1] // SQUARE_SIZE # Same square clicked twice if selected_square == (row, col): # Undo last clicks selected_square = () player_clicks = [] else: selected_square = (row, col) player_clicks.append(selected_square) if len(player_clicks) == 2: move = Move(gs.board, player_clicks[0], player_clicks[1]) if move in valid_moves: move = valid_moves[valid_moves.index(move)] gs.make_move(move) move_made = True selected_square = () player_clicks = [] logger.success(f"Move: {move.PGN}") else: logger.error(f"{move.move_notation()} - Not a valid move") player_clicks = [selected_square] elif event.type == pygame.KEYDOWN: # Undo move on left arrow-key if event.key == pygame.K_LEFT: gs.undo_move() # As valid_moves has been refreshed with the last move, undoing it needs to generate a new move-list again move_made = True if move_made: valid_moves = gs.get_valid_moves() move_made = False if gs.stalemate or gs.checkmate: running = False if gs.stalemate: logger.info("Stalemate") if gs.checkmate: logger.info("Mate!") draw_game_state(screen, gs) clock.tick(MAX_FPS) pygame.display.flip()
def run(self, show_progress_bar=None): """Main routine of the simulation. Note: Make sure to call :meth:`setup` prior to this function. Arguments: show_progress_bar (:obj:`bool`, optional): Whether to show fancy progress bar via tqdm. By default, only show if stdout is a terminal and Veros is running on a single process. """ vs = self.state logger.info('\nStarting integration for {0[0]:.1f} {0[1]}'.format(time.format_time(vs.runlen))) start_time, start_iteration = vs.time, vs.itt profiler = None pbar = progress.get_progress_bar(vs, use_tqdm=show_progress_bar) with handlers.signals_to_exception(): try: with pbar: while vs.time - start_time < vs.runlen: with vs.timers['diagnostics']: diagnostics.write_restart(vs) if vs.itt - start_iteration == 3 and rs.profile_mode and rst.proc_rank == 0: # when using bohrium, most kernels should be pre-compiled by now profiler = diagnostics.start_profiler() with vs.timers['main']: self.set_forcing(vs) if vs.enable_idemix: idemix.set_idemix_parameter(vs) with vs.timers['eke']: eke.set_eke_diffusivities(vs) with vs.timers['tke']: tke.set_tke_diffusivities(vs) with vs.timers['momentum']: momentum.momentum(vs) with vs.timers['temperature']: thermodynamics.thermodynamics(vs) if vs.enable_eke or vs.enable_tke or vs.enable_idemix: advection.calculate_velocity_on_wgrid(vs) with vs.timers['eke']: if vs.enable_eke: eke.integrate_eke(vs) with vs.timers['idemix']: if vs.enable_idemix: idemix.integrate_idemix(vs) with vs.timers['tke']: if vs.enable_tke: tke.integrate_tke(vs) utilities.enforce_boundaries(vs, vs.u[:, :, :, vs.taup1]) utilities.enforce_boundaries(vs, vs.v[:, :, :, vs.taup1]) if vs.enable_tke: utilities.enforce_boundaries(vs, vs.tke[:, :, :, vs.taup1]) if vs.enable_eke: utilities.enforce_boundaries(vs, vs.eke[:, :, :, vs.taup1]) if vs.enable_idemix: utilities.enforce_boundaries(vs, vs.E_iw[:, :, :, vs.taup1]) momentum.vertical_velocity(vs) with vs.timers['plugins']: for plugin in self._plugin_interfaces: with vs.timers[plugin.name]: plugin.run_entrypoint(vs) vs.itt += 1 vs.time += vs.dt_tracer pbar.advance_time(vs.dt_tracer) self.after_timestep(vs) with vs.timers['diagnostics']: if not diagnostics.sanity_check(vs): raise RuntimeError('solution diverged at iteration {}'.format(vs.itt)) if vs.enable_neutral_diffusion and vs.enable_skew_diffusion: isoneutral.isoneutral_diag_streamfunction(vs) diagnostics.diagnose(vs) diagnostics.output(vs) # NOTE: benchmarks parse this, do not change / remove logger.debug(' Time step took {:.2f}s', vs.timers['main'].get_last_time()) # permutate time indices vs.taum1, vs.tau, vs.taup1 = vs.tau, vs.taup1, vs.taum1 except: logger.critical('Stopping integration at iteration {}', vs.itt) raise else: logger.success('Integration done\n') finally: diagnostics.write_restart(vs, force=True) timing_summary = [ '', 'Timing summary:', ' setup time = {:.2f}s'.format(vs.timers['setup'].get_time()), ' main loop time = {:.2f}s'.format(vs.timers['main'].get_time()), ' momentum = {:.2f}s'.format(vs.timers['momentum'].get_time()), ' pressure = {:.2f}s'.format(vs.timers['pressure'].get_time()), ' friction = {:.2f}s'.format(vs.timers['friction'].get_time()), ' thermodynamics = {:.2f}s'.format(vs.timers['temperature'].get_time()), ' lateral mixing = {:.2f}s'.format(vs.timers['isoneutral'].get_time()), ' vertical mixing = {:.2f}s'.format(vs.timers['vmix'].get_time()), ' equation of state = {:.2f}s'.format(vs.timers['eq_of_state'].get_time()), ' EKE = {:.2f}s'.format(vs.timers['eke'].get_time()), ' IDEMIX = {:.2f}s'.format(vs.timers['idemix'].get_time()), ' TKE = {:.2f}s'.format(vs.timers['tke'].get_time()), ' diagnostics and I/O = {:.2f}s'.format(vs.timers['diagnostics'].get_time()), ' plugins = {:.2f}s'.format(vs.timers['plugins'].get_time()), ] timing_summary.extend([ ' {:<22} = {:.2f}s'.format(plugin.name, vs.timers[plugin.name].get_time()) for plugin in vs._plugin_interfaces ]) logger.debug('\n'.join(timing_summary)) if profiler is not None: diagnostics.stop_profiler(profiler)
def print_and_upload_result(self, name, metrics, line): with open(self.out, "rb") as out: lines = out.read().splitlines() result = lines[-line:] LOG.success(f"Result for {self.name}, uploaded under {name}:") self._print_upload_perf(name, metrics, result)
def Conectar(self): T = str(M.b64encode(self.email.encode())) self.arqInit = str(R.logs_path) + '/' + T + '_init.conf' if os.path.exists(self.arqInit): os.remove(self.arqInit) if self.contareal: self.tipoconta = 'REAL' else: self.tipoconta = 'PRACTICE' if self.ent_tipo == 'P': self.ent_valor1 = round(self.valorinicial * self.ent_valor1 / 100, 2) self.ent_gale1 = round(self.valorinicial * self.ent_gale1 / 100, 2) self.ent_gale2 = round(self.valorinicial * self.ent_gale2 / 100, 2) if self.usarsoros: self.ent_valor1 = round(self.valorinicial * self.percent / 100, 2) self.ent_gale1 = 0 self.ent_gale2 = 0 if self.qtdgales > 0: self.ent_gale1 = 2 if self.qtdgales > 1: self.ent_gale2 = 2 else: if self.qtdgales < 1: self.ent_gale1 = 0 if self.qtdgales < 2: self.ent_gale2 = 0 if self.loadLista() == 0: A.info( Idioma.traducao('Lista vazia ou com dia/horário expirados.')) A.info(Idioma.traducao('Aguarde, conectando a IQ...')) print(Idioma.traducao('Aguarde, conectando a IQ...')) self.View.janela.Refresh() N.createapiconnection(self.email, self.senha, self.tipoconta) conect = API.instance().connection if conect: if self.VerificarLicenca(): C.instance().actual_balance = 0 if self.tipostop == 'P': X = self.stopgain / 100 Z = self.stoploss / 100 C.instance().win_limit = self.valorinicial * X C.instance().stop_limit = self.valorinicial * Z * -1 else: C.instance().win_limit = self.stopgain C.instance().stop_limit = self.stoploss * -1 A.info('Versão: ' + self.versaoapp) A.success( Idioma.traducao('Tipo de conta:') + ' {}', self.tipoconta) A.info(Idioma.traducao('Parâmetros iniciais:')) A.success( Idioma.traducao('Valor inicial: $') + '{}', round(self.valorinicial, 2)) A.success( Idioma.traducao('Quantidade de gales:') + ' {}', self.qtdgales) A.success( Idioma.traducao('Payout mínimo:') + ' {}', self.payoutmin) if self.prestop: A.success(Idioma.traducao('Pré-Stop Loss: Ligado')) if self.esperarIQ: A.success(Idioma.traducao('Resultado Resp. IQ')) else: A.success(Idioma.traducao('Resultado por Taxas')) A.success('Delay: {}', self.delay) if self.priorid == 0: A.success( Idioma.traducao('Prioridade') + ': ' + Idioma.traducao('Maior Payout')) elif self.priorid == 1: A.success( Idioma.traducao('Prioridade') + ': ' + Idioma.traducao('Digital')) elif self.priorid == 2: A.success( Idioma.traducao('Prioridade') + ': ' + Idioma.traducao('Binárias')) if self.naonoticia: A.success(Idioma.traducao('Não operar em notícia')) if self.tendusar: if self.tendemasma: A.success( Idioma.traducao('Não Operar Contra') + ': ' + Idioma.traducao('Usar EMA5 + SMA20')) else: A.success( Idioma.traducao('Não Operar Contra') + ': ' + Idioma.traducao('Quant. Velas') + ': ' + str(self.tendvelas)) if not self.usarsoros: A.info(Idioma.traducao('Entradas fixas:')) A.success( Idioma.traducao('Entrada: $') + '{}', round(self.ent_valor1, 2)) if self.ent_gale1 > 0: A.success( Idioma.traducao('Gale 1: $') + '{}', round(self.ent_gale1, 2)) if self.ent_gale2 > 0: A.success( Idioma.traducao('Gale 2: $') + '{}', round(self.ent_gale2, 2)) else: A.info('Soros:') if self.modelo == 'A': A.success(Idioma.traducao('Modelo: Agressivo')) elif self.modelo == 'M': A.success(Idioma.traducao('Modelo: Moderado')) else: A.success(Idioma.traducao('Modelo: Conservador')) A.success( Idioma.traducao('1ª entrada: %') + '{} | ' + Idioma.traducao('Valor: $') + '{}', self.percent, round(self.ent_valor1, 2)) A.warning( 'WIN %{} - ' + Idioma.traducao('Parar de operar quando atingir: $') + '{}', self.stopgain, round(C.instance().win_limit, 2)) A.warning( 'LOSS %{} - ' + Idioma.traducao('Parar de operar quando atingir: $') + '{}', self.stoploss, round(C.instance().stop_limit, 2)) print('Versão: ' + self.versaoapp) print(Idioma.traducao('Tipo de conta:'), self.tipoconta) print(Idioma.traducao('Parâmetros iniciais:')) print(Idioma.traducao('Quantidade de gales:'), self.qtdgales) print(Idioma.traducao('Payout mínimo:'), self.payoutmin) if self.prestop: print(Idioma.traducao('Pré-Stop Loss: Ligado')) if self.esperarIQ: print(Idioma.traducao('Resultado Resp. IQ')) else: print(Idioma.traducao('Resultado por Taxas')) print('Delay: ', self.delay) if self.priorid == 0: print( Idioma.traducao('Prioridade') + ': ' + Idioma.traducao('Maior Payout')) elif self.priorid == 1: print( Idioma.traducao('Prioridade') + ': ' + Idioma.traducao('Digital')) elif self.priorid == 2: print( Idioma.traducao('Prioridade') + ': ' + Idioma.traducao('Binárias')) if self.naonoticia: print(Idioma.traducao('Não operar em notícia')) if self.tendusar: if self.tendemasma: print( Idioma.traducao('Não Operar Contra') + ': ' + Idioma.traducao('Usar EMA5 + SMA20')) else: print( Idioma.traducao('Não Operar Contra') + ': ' + Idioma.traducao('Quant. Velas') + ': ' + str(self.tendvelas)) if not self.usarsoros: print(Idioma.traducao('Entradas fixas:')) print(Idioma.traducao('Entrada: $'), round(self.ent_valor1, 2)) if self.ent_gale1 > 0: print(Idioma.traducao('Gale 1: $'), round(self.ent_gale1, 2)) if self.ent_gale2 > 0: print(Idioma.traducao('Gale 2: $'), round(self.ent_gale2, 2)) else: print('Soros:') if self.modelo == 'A': print(Idioma.traducao('Modelo: Agressivo')) elif self.modelo == 'M': print(Idioma.traducao('Modelo: Moderado')) else: print(Idioma.traducao('Modelo: Conservador')) print((Idioma.traducao('1ª entrada: %') + '{0} | ' + Idioma.traducao('Valor: $') + '{1}').format( self.percent, round(self.ent_valor1, 2))) self.View.janela['valorinic'].update( value=(round(self.valorinicial, 2))) self.View.janela['saldoatual'].update( value=(C.instance().balance)) self.View.janela['stopgainp'].update(value=(self.stopgain)) self.View.janela['stopgainv'].update( value=(round(C.instance().win_limit, 2))) self.View.janela['stoplossp'].update(value=(self.stoploss)) self.View.janela['stoplossv'].update( value=(round(C.instance().stop_limit, 2))) self.View.janela.Refresh() if not self.usarsoros: self.valorinicial = 0 C.instance().sorosgale.config_ini(self.valorinicial, self.percent, self.modelo) return True return False
def sync_state(self, pixelArray): """ Code to run when in "sync" state Synchronising with prospective optical gating for phase-locked triggering. """ logger.debug("Processing frame in prospective optical gating mode.") # Gets the phase (in frames) and arrays of SADs between the current frame and the referencesequence currentPhaseInFrames, sad, self.pog_settings = pog.phase_matching( pixelArray, self.ref_frames, settings=self.pog_settings) logger.trace(sad) # Convert phase to 2pi base current_phase = ( 2 * np.pi * (currentPhaseInFrames - self.pog_settings["numExtraRefFrames"]) / self.pog_settings["reference_period"]) # rad # Calculate cumulative phase (phase) from delta phase (current_phase - last_phase) if len(self.frame_history) == 0: # i.e. first frame logger.debug( "First frame, using current phase as cumulative phase.") delta_phase = 0 phase = current_phase self.last_phase = current_phase else: delta_phase = current_phase - self.last_phase while delta_phase < -np.pi: delta_phase += 2 * np.pi phase = self.frame_history[-1].metadata[ "unwrapped_phase"] + delta_phase self.last_phase = current_phase # Evicts the oldest entry in frame_history if it exceeds the history length that we are meant to be retaining # Note: deletion of first list element is potentially a performance issue, # although we are hopefully capping the length low enough that it doesn't become a real bottleneck if len(self.frame_history) >= self.settings["frame_buffer_length"]: del self.frame_history[0] # Append PixelArray object to frame_history list with its metadata pixelArray.metadata["unwrapped_phase"] = phase pixelArray.metadata["sad_min"] = np.argmin(sad) self.frame_history.append(pixelArray) logger.debug( "Current time: {0} s; cumulative phase: {1} (delta:{2:+f}) rad; sad: {3}", self.frame_history[-1].metadata["timestamp"], self.frame_history[-1].metadata["unwrapped_phase"], delta_phase, self.frame_history[-1].metadata["sad_min"], ) # If we have at least one period of phase history, have a go at predicting a future trigger time # (Note that this prediction can be disabled by enabling "phase_stamp_only" in pog_settings this_predicted_trigger_time_s = None sendTriggerNow = 0 if (len(self.frame_history) > self.pog_settings["reference_period"] and self.pog_settings["phase_stamp_only"] != True): logger.debug("Predicting trigger...") # TODO: JT writes: this seems as good a place as any to highlight the general issue that the code is not doing a great job of precise timing. # It determines a delay time before sending the trigger, but then executes a bunch more code. # Oh and, more importantly, that delay time is then treated relative to “current_time_s”, which is set *after* doing the phase-matching. # That is going to reduce accuracy and precision, and also makes me even more uncomfortable in terms of future-proofing. # I think it would be much better to pass around absolute times, not deltas. # Gets the trigger response logger.trace("Predicting next trigger.") time_to_wait_seconds = pog.predict_trigger_wait( pa.get_metadata_from_list( self.frame_history, ["timestamp", "unwrapped_phase", "sad_min"]), self.pog_settings, fitBackToBarrier=True, ) logger.trace("Time to wait: {0} s.".format(time_to_wait_seconds)) # frame_history is an nx3 array of [timestamp, phase, argmin(SAD)] # phase (i.e. frame_history[:,1]) should be cumulative 2Pi phase # targetSyncPhase should be in [0,2pi] this_predicted_trigger_time_s = ( self.frame_history[-1].metadata["timestamp"] + time_to_wait_seconds) # Captures the image if time_to_wait_seconds > 0: logger.info("Possible trigger after: {0}s", time_to_wait_seconds) ( time_to_wait_seconds, sendTriggerNow, self.pog_settings, ) = pog.decide_trigger( self.frame_history[-1].metadata["timestamp"], time_to_wait_seconds, self.pog_settings, ) if sendTriggerNow != 0: logger.success( "Sending trigger (reason: {0}) at time ({1} plus {2}) s", sendTriggerNow, self.frame_history[-1].metadata["timestamp"], time_to_wait_seconds, ) # Trigger only self.trigger_fluorescence_image_capture( this_predicted_trigger_time_s) # Update trigger iterator (for adaptive algorithm) self.trigger_num += 1 # Update PixelArray with predicted trigger time and trigger type self.frame_history[-1].metadata[ "predicted_trigger_time_s"] = this_predicted_trigger_time_s self.frame_history[-1].metadata["trigger_type_sent"] = sendTriggerNow logger.debug( "Current time: {0} s; predicted trigger time: {1} s; trigger type: {2}", self.frame_history[-1].metadata["timestamp"], self.frame_history[-1].metadata["predicted_trigger_time_s"], self.frame_history[-1].metadata["trigger_type_sent"], ) # store this phase now to calculate the delta phase for the next frame self.last_phase = float(current_phase)
def __init__(self, running_file='running'): self.running_file = running_file def start_scan(self): with open(self.running_file, 'w') as f: f.write(str(1)) def check_for_flag(self, flag): pass def is_scan_done(self): with open(self.running_file, 'r') as f: running = bool(int(f.read().strip('\n'))) return not running def wait_for_scan_done(self): while not md.is_scan_done(): time.sleep(1) if __name__ == "__main__": md = MockDriver() logger.info('Starting scan') md.start_scan() md.wait_for_scan_done() logger.success('Scan done') # print(md.check_for_flag("4f6a142e-56ff-f229-4d1d-aa055c000de7"))
def run(args): hosts = ["localhost", "localhost", "localhost"] with infra.ccf.network(hosts, args.build_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb) as network: first_node, (backups) = network.start_and_join(args) term_info = {} long_msg = "X" * (2**14) # first timer determines after how many seconds each node will be suspended timeouts = [] t = random.uniform(1, 10) LOG.info( f"Initial timer for node {first_node.node_id} is {t} seconds...") timeouts.append((t, first_node)) for backup in backups: t = random.uniform(1, 10) LOG.info( f"Initial timer for node {backup.node_id} is {t} seconds...") timeouts.append((t, backup)) for t, node in timeouts: tm = Timer( t, timeout, args=[node, True, args.election_timeout / 1000], ) tm.start() with first_node.node_client() as mc: check_commit = infra.checker.Checker(mc) check = infra.checker.Checker() clients = [] with contextlib.ExitStack() as es: LOG.info("Write messages to nodes using round robin") clients.append( es.enter_context(first_node.user_client(format="json"))) for backup in backups: clients.append( es.enter_context(backup.user_client(format="json"))) node_id = 0 for id in range(1, TOTAL_REQUESTS): node_id += 1 c = clients[node_id % len(clients)] try: resp = c.rpc("LOG_record", {"id": id, "msg": long_msg}) except Exception: LOG.info("Trying to access a suspended node") try: cur_primary, cur_term = network.find_primary() term_info[cur_term] = cur_primary.node_id except Exception: LOG.info("Trying to access a suspended node") id += 1 # wait for the last request to commit final_msg = "Hello world!" check_commit( c.rpc("LOG_record", { "id": 1000, "msg": final_msg }), result=True, ) check( c.rpc("LOG_get", {"id": 1000}), result={"msg": final_msg}, ) # check that a new node can catch up after all the requests new_node = network.create_and_trust_node( lib_name=args.package, host="localhost", args=args, ) assert new_node # give new_node a second to catch up time.sleep(1) with new_node.user_client(format="json") as c: check( c.rpc("LOG_get", {"id": 1000}), result={"msg": final_msg}, ) # assert that view changes actually did occur assert len(term_info) > 1 LOG.success( "----------- terms and primaries recorded -----------") for term, primary in term_info.items(): LOG.success(f"term {term} - primary {primary}")
def index(): get_templates() form = GenerateForm() form.platform.choices = get_templates() if request.method == 'GET': return render_template('index.html', form=form) if form.validate_on_submit(): print("Valid form submission") # Do BundleGen work outputdir = os.path.abspath( os.path.join(TMP_DIR, Utils.get_random_string(5))) selected_platform = STBPlatform(form.platform.data) if not selected_platform.found_config(): print(f"Could not find config for platform {form.platform.data}") raise AppError("Could not find platform") img_url = "" creds = "" if form.image_url.data: # If downloading from URL, just use that as-is img_url = form.image_url.data # Add creds if given if form.registry_uname.data and form.registry_password.data: creds = f"{form.registry_uname.data}:{form.registry_password.data}" elif form.uploaded_img.data: # Got an uploaded image (hopefully a tar.gz!) f = form.uploaded_img.data filename = secure_filename(f.filename) upload_filepath = os.path.join(UPLOAD_FOLDER, filename) f.save(upload_filepath) # Extract tar # Extract the .tar to a temp directory img_temp_path = tempfile.mkdtemp() with tarfile.open(upload_filepath) as tar: tar.extractall(img_temp_path) # Delete tar os.remove(upload_filepath) img_url = f"oci:{img_temp_path}:latest" creds = None if not img_url: print("IMG URL is empty") raise AppError("Image URL cannot be empty") # Download Image img_downloader = ImageDownloader() img_path = img_downloader.download_image( img_url, creds, selected_platform.get_config()) if not img_path: logger.error("Failed to donwload image") raise AppError("Image download failed") # Unpack the image with umoci tag = ImageDownloader().get_image_tag(img_url) img_unpacker = ImageUnpackager() img_unpacker.unpack_image(img_path, tag, outputdir) # Delete the downloaded image now we've unpacked it logger.info(f"Deleting {img_path}") shutil.rmtree(img_path) # Load app metadata app_metadata_image_path = os.path.join(outputdir, "rootfs", "appmetadata.json") image_metadata_exists = os.path.exists(app_metadata_image_path) app_metadata_dict = {} custom_app_metadata = form.app_metadata.data if not image_metadata_exists and not custom_app_metadata: # No metadata at all logger.error( f"Cannot find app metadata file in OCI image and none provided to BundleGen" ) raise AppError("No Metadata provided") elif (not image_metadata_exists and custom_app_metadata) or (image_metadata_exists and custom_app_metadata): # Use custom metadata app_metadata_dict = json.loads(custom_app_metadata) else: # Load metadata from image with open(app_metadata_image_path) as metadata: app_metadata_dict = json.load(metadata) # remove app metadata from image rootfs if image_metadata_exists: os.remove(app_metadata_image_path) # Begin processing. Work in the output dir where the img was unpacked to processor = BundleProcessor(selected_platform.get_config(), outputdir, app_metadata_dict, False, form.lib_match.data) if not processor.check_compatibility(): # Not compatible - delete any work done so far shutil.rmtree(outputdir) raise AppError("App incompatible") success = processor.begin_processing() if not success: logger.warning("Failed to produce bundle") raise AppError("Something went wrong") tarball_name = app_metadata_dict["id"] + Utils.get_random_string(6) Utils.create_tgz(outputdir, tarball_name) logger.success( f"Successfully generated bundle at {tarball_name}.tar.gz") # Move to persistant storage print(f"Moving '{tarball_name}.tar.gz' to {BUNDLE_STORE_DIR}") shutil.move(f'{tarball_name}.tar.gz', BUNDLE_STORE_DIR) return jsonify(success=True)
def __exit__(self, *exc): self.end = maya.now()._epoch delta = self.end - self.start logger.success(f"It took {delta}s") logger.success(f"It took {(delta*1000)}ms") return False
def run(self, actions, battery, lat_long_actual, box): logger.success('Start communication') init_hour = self.time_hour() init_minute = self.time_minute(init_hour) while True: if self.connection is None: self.connection = self.board.open_connection(port=self.port) sleep(0.9) try: message_board = self.connection.readline().decode('utf-8', 'replace') if str(message_board[:8]) == str(self.PATTERN): if self.board.digit_verify(str(message_board)): self.board.send_message(connection=self.connection, message=self.board.SEND_OK) message_board = message_board.split(',') # basculamento if message_board[2] == '1': logger.debug('{}'.format(message_board)) box[0] = int(message_board[3]) code = self.code_cart(box=box[0], battery=battery[:]) if code is None: self.board.send_message(connection=self.connection, message=self.NOT_FOUND) logger.info('Not identified :(') else: logger.success('Caixote {}, basculou na carreta: {} '.format(box[0], code)) attempts = 1 while attempts <= 5: self.board.send_message(connection=self.connection, message='$PNEUD,G,1,{}'.format(code)) logger.debug('Send: {}'.format(attempts)) sleep(0.5) attempts += 1 if message_board[2] == '2': # camera off and analise reboot if message_board[3] == '0': if self.preview_state == 1: # actions[0] = 0 self.preview_state = int(message_board[3]) actual_hour = self.time_hour() if actual_hour[0] < init_hour[0]: actual_hour[0] = int(actual_hour[0]) + 24 actual_minute = self.time_minute(actual_hour) reboot = actual_minute - init_minute if reboot >= self.reboot: logger.info('Reboot system') sleep(0.8) os.system('sudo reboot') # camera start elif message_board[3] == '1': if self.preview_state == 0: # actions[0] = 1 self.preview_state = int(message_board[3]) try: lat_long_actual[0], lat_long_actual[1] = self.convert_coord( array=[float(message_board[4]), float(message_board[5])]) except ValueError: logger.error('Invalid GPS') except UnicodeError: logger.error('Unicode error') except IOError: self.close_connection() logger.error('Serial error') except AttributeError: self.close_connection() logger.error('Invalid argument')
def _x_call_gce(self, version_only=False): """ Call GCE on kmerfreq table to estimate genome characters """ if version_only: # print the version proc = sps.Popen( [self.binaries["gce"], "-V"], stderr=sps.STDOUT, stdout=sps.PIPE, ) out = proc.communicate()[0].decode().split("\n") out = [i for i in out if "Version" in i][0] vers = out.split()[-1] return vers # prerun commands resfile = os.path.join(self.workdir, self.srr, self.srr + ".kmer.freq.stat") cmd1 = ['cat', resfile] cmd2 = ['grep', '#Kmer indivdual number'] # (sic) null = "cat {workdir}/{srr}/{srr}.kmer.freq.stat | grep '#Kmer indiv'" logger.info("Executing {}:".format(null)) logger.debug("Executing: {}".format(" ".join(cmd1))) proc1 = sps.Popen(cmd1, stderr=sps.STDOUT, stdout=sps.PIPE) proc2 = sps.Popen(cmd2, stdin=proc1.stdout, stdout=sps.PIPE) out = proc2.communicate() if proc2.returncode: logger.error(out[0].decode()) # store ikmer result ikmer = out[0].decode().strip().split()[-1] logger.success("Kmer individual number: {}".format(ikmer)) # write kmer 2 column sub result file res2col = resfile + ".2colum" logger.info("Parsing 2-columns file to: {}".format(res2col)) arr = pd.read_csv(resfile, skiprows=7, sep="\t", header=None) arr = arr.iloc[:, :2] arr.to_csv(res2col, index=False, sep="\t", header=None) # Run in homozygous mode logger.info("Running 'gce' in homozygous mode to estimate coverage") null = "{gce} -g " + ikmer + " -f {res.2col}" logger.info("Executing: {}".format(null)) cmd = [ self.binaries["gce"], "-g", ikmer, "-f", res2col, ] logger.debug(" ".join(cmd)) proc = sps.Popen(cmd, stderr=sps.STDOUT, stdout=sps.PIPE) self.gce1out = proc.communicate() if proc.returncode: logger.error(self.gce1out[0].decode()) # write to a tmp file parse = self.gce1out[0].decode().split("Final estimation table:") parse = parse[-1].strip().split("\n") headers, data = parse[:2] headers = headers.strip().split("\t") data = data.strip().split("\t") self.h0dict = {} for i, j in zip(headers, data): self.h0dict[i] = j logger.success("GCE H0 {}: {}".format(j)) # Run in heterozygous mode logger.info("Running 'gce' in heterozygous mode.") null = "{gce} -g " + ikmer + " -f {res.2col} -H 1 -c {coverage}" logger.info("Executing: {}".format(null)) cmd = [ self.binaries["gce"], "-g", ikmer, "-f", res2col, "-H", "1", "-c", str(int(float(round(self.h0dict["coverage_depth"])))) ] logger.debug(" ".join(cmd)) proc = sps.Popen(cmd, stderr=sps.STDOUT, stdout=sps.PIPE) self.gce2out = proc.communicate() if proc.returncode: logger.error(self.gce2out[0].decode()) # write to a tmp file parse = self.gce2out[0].decode().split("Final estimation table:") parse = coverage[-1].strip().split("\n") logger.success("GCE genome size: {}".format(coverage)) logger.success("GCE heterozygosity: {}".format(coverage)) logger.success("GCE coverage depth: {}".format(coverage))
def determine_state(self, pixelArray, modeString="determine period"): """ Code to run when in "determine" state Determine period mode (default behaviour requires user input). In this mode we obtain a minimum number of frames, determine a period and then return. It is assumed that the user (or cli/flask app) then runs the user_select_ref_frame function (and updates the state) before running analyse again with the new state. """ logger.debug("Processing frame in {0} mode.".format(modeString)) # Adds new frame to buffer self.ref_buffer.append(pixelArray) # Impose an upper limit on the buffer length, to protect against performance degradation # in cases where we are not succeeding in identifying a period # Note: deletion of first list element is potentially a performance issue, # although we are hopefully capping the length low enough that it doesn't become a real bottleneck ref_buffer_duration = (self.ref_buffer[-1].metadata["timestamp"] - self.ref_buffer[0].metadata["timestamp"]) if (("min_heart_rate_hz" in self.settings) and (ref_buffer_duration > 1.0 / self.settings["min_heart_rate_hz"])): logger.debug("Trimming buffer to duration {0}".format( 1.0 / self.settings["min_heart_rate_hz"])) del self.ref_buffer[0] # Calculate period from determine_reference_period.py logger.info("Attempting to determine new reference period.") self.ref_frames, self.pog_settings = ref.establish( self.ref_buffer, self.period_guesses, self.pog_settings) if self.ref_frames is not None: # We were provided with ref_frames as a list, and this is helpful because it means we could still access # the PixelArray metadata at this point if we wished. # However, long-term we want to store a 3D array because that is what oga expects to work with. # We therefore make that conversion here self.ref_frames = np.array(self.ref_frames) # Automatically select a target frame and barrier # This can be overriden by the user/controller later self.pog_settings = pog.pick_target_and_barrier_frames( self.ref_frames, self.pog_settings) # Determine barrier frames self.pog_settings = pog.determine_barrier_frames(self.pog_settings) # Save the period ref.save_period(self.ref_frames, self.settings["period_dir"]) logger.success("Period determined.") self.justRefreshedRefFrames = True # Flag that a slow action took place if self.automatic_target_frame: logger.info( "Period determined and target frame automatically selected; switching to prospective optical gating mode." ) # Automatically switch to the "sync" state, using the default reference frame. # The user is expected to change the reference frame later, via a GUI, if they wish to self.start_sync_with_ref_frame( self.pog_settings["referenceFrame"]) self.state = "sync" else: # If we aren't using the automatically determined period # We raise the stop flag, which returns the current state to the user/app # The user/app can then select a target frame # The user/app will also need to call the adaptive system # see user_select_ref_frame() self.stop = True
def write_results_to_file(result_dict: Dict): logger.info(f"| ----- Results will be written into file.") with open("census2010.py", "w") as res_file: res_file.write(f"'Results' = {pprint.pformat(result_dict)}") logger.success(f"| ----- Results are in the census2010.py file.")
def trigger_fluorescence_image_capture(self, delay): """As this is the base server, this function just outputs a log that a trigger would have been sent.""" logger.success("A fluorescence image would be triggered now.")
def run(get_command, args): if args.fixed_seed: seed(getpass.getuser()) hosts = args.nodes if not hosts: hosts = ["local://localhost"] * minimum_number_of_local_nodes(args) args.initial_user_count = 3 LOG.info("Starting nodes on {}".format(hosts)) with infra.network.network(hosts, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb) as network: network.start_and_join(args) primary, backups = network.find_nodes() command_args = get_command_args(args, get_command) if args.use_jwt: jwt_key_priv_pem, _ = infra.crypto.generate_rsa_keypair(2048) jwt_cert_pem = infra.crypto.generate_cert(jwt_key_priv_pem) jwt_kid = "my_key_id" jwt_issuer = "https://example.issuer" # Add JWT issuer with tempfile.NamedTemporaryFile(prefix="ccf", mode="w+") as metadata_fp: jwt_cert_der = infra.crypto.cert_pem_to_der(jwt_cert_pem) der_b64 = base64.b64encode(jwt_cert_der).decode("ascii") data = { "issuer": jwt_issuer, "jwks": { "keys": [{ "kty": "RSA", "kid": jwt_kid, "x5c": [der_b64] }] }, } json.dump(data, metadata_fp) metadata_fp.flush() network.consortium.set_jwt_issuer(primary, metadata_fp.name) jwt = infra.crypto.create_jwt({}, jwt_key_priv_pem, jwt_kid) command_args += ["--bearer-token", jwt] nodes_to_send_to = filter_nodes(primary, backups, args.send_tx_to) clients = [] client_hosts = [] if args.one_client_per_backup: if not backups: raise Exception( "--one-client-per-backup was set but no backup was found") client_hosts = ["localhost"] * len(backups) else: if args.client_nodes: client_hosts.extend(args.client_nodes) if args.num_localhost_clients: client_hosts.extend(["localhost"] * int(args.num_localhost_clients)) if not client_hosts: client_hosts = ["localhost"] for client_id, client_host in enumerate(client_hosts): node = nodes_to_send_to[client_id % len(nodes_to_send_to)] remote_client = configure_remote_client(args, client_id, client_host, node, command_args) clients.append(remote_client) if args.network_only: for remote_client in clients: LOG.info( f"Client can be run with: {remote_client.remote.get_cmd()}" ) while True: time.sleep(60) else: for remote_client in clients: remote_client.start() hard_stop_timeout = 90 try: with cimetrics.upload.metrics(complete=False) as metrics: tx_rates = infra.rates.TxRates(primary) start_time = time.time() while True: stop_waiting = True for i, remote_client in enumerate(clients): done = remote_client.check_done() # all the clients need to be done LOG.info( f"Client {i} has {'completed' if done else 'not completed'} running ({time.time() - start_time:.2f}s / {hard_stop_timeout}s)" ) stop_waiting = stop_waiting and done if stop_waiting: break if time.time() > start_time + hard_stop_timeout: raise TimeoutError( f"Client still running after {hard_stop_timeout}s" ) time.sleep(5) tx_rates.get_metrics() for remote_client in clients: perf_result = remote_client.get_result() LOG.success( f"{args.label}/{remote_client.name}: {perf_result}" ) # TODO: Only results for first client are uploaded # https://github.com/microsoft/CCF/issues/1046 if remote_client == clients[0]: LOG.success( f"Uploading results for {remote_client.name}") metrics.put(args.label, perf_result) else: LOG.warning( f"Skipping upload for {remote_client.name}") primary, _ = network.find_primary() with primary.client() as nc: r = nc.get("/node/memory") assert r.status_code == http.HTTPStatus.OK.value results = r.body.json() tx_rates.insert_metrics(**results) # Construct name for heap metric, removing ^ suffix if present heap_peak_metric = args.label if heap_peak_metric.endswith("^"): heap_peak_metric = heap_peak_metric[:-1] heap_peak_metric += "_mem" peak_value = results["peak_allocated_heap_size"] metrics.put(heap_peak_metric, peak_value) LOG.info(f"Rates:\n{tx_rates}") tx_rates.save_results(args.metrics_file) for remote_client in clients: remote_client.stop() except Exception: LOG.error("Stopping clients due to exception") for remote_client in clients: remote_client.stop() raise
def start_and_join(self, args): """ Starts a CCF network. :param args: command line arguments to configure the CCF nodes. """ self.common_dir = get_common_folder_name(args.workspace, args.label) assert (args.gov_script is not None ), "--gov-script argument must be provided to start a network" self._setup_common_folder(args.gov_script) mc = max(1, args.initial_member_count) initial_members_info = [] for i in range(mc): initial_members_info += [( i, (i < args.initial_recovery_member_count), { "is_operator": True } if (i < args.initial_operator_count) else None, )] self.consortium = infra.consortium.Consortium( self.common_dir, self.key_generator, self.share_script, initial_members_info, args.participants_curve, authenticate_session=not args.disable_member_session_auth, ) initial_users = list(range(max(0, args.initial_user_count))) self.create_users(initial_users, args.participants_curve) primary = self._start_all_nodes(args) self.wait_for_all_nodes_to_catch_up(primary) LOG.success("All nodes joined network") self.consortium.activate(primary) if args.js_app_script: LOG.error( "--js-app-script is deprecated - update to --js-app-bundle instead" ) infra.proc.ccall("cp", args.js_app_script, args.binary_dir).check_returncode() self.consortium.set_js_app(remote_node=primary, app_script_path=args.js_app_script) if args.js_app_bundle: self.consortium.deploy_js_app(remote_node=primary, app_bundle_path=args.js_app_bundle) for path in args.jwt_issuer: self.consortium.set_jwt_issuer(remote_node=primary, json_path=path) self.consortium.add_users(primary, initial_users) LOG.info(f"Initial set of users added: {len(initial_users)}") self.consortium.open_network(remote_node=primary) self.status = ServiceStatus.OPEN LOG.success("***** Network is now open *****")
logger.debug(choice) net = self.supernet.sample(choice) for j in range(len(self.obj_functions)): f_res[j][i] = self.obj_functions[j](net) show = "" for name, res in zip(self.obj_names, [res[i] for res in f_res]): show += "%s:%.6f, " % (name, res) logger.info("PopID: {} | Choice {} | Result {}", i, choice, show) out["F"] = anp.column_stack(f_res) if __name__ == "__main__": supernet = model.__dict__[args.model]().to(device) supernet.load_state_dict( torch.load(args.load_path) ["state_dict"]) problem = MyProblem(supernet) res = search(problem, pop_size=20, n_offsprings=10, n_generations=10, seed=123) X = np.array(list(map(lambda x: x.X, res.pop))) F = np.array(list(map(lambda x: x.F, res.pop))) logger.info("get popuation") logger.success(X) logger.info("get result") logger.success(F) os.makedirs("saved-pops", exist_ok=True) torch.save(res.pop, "saved-pops/fair_pop.pkl")
return d ########################################################## # MAIN ########################################################## padding: int = snakemake.params.padding apply_filters: bool = snakemake.params.get("apply_filters", False) only_alt: bool = snakemake.params.get("only_alt", False) adjust_pos: bool = snakemake.params.get("adjust_pos", False) logger.info("Extracting gene names from panel...") with open(snakemake.input.panel) as istream: genes = extract_genes_from_panel(istream) logger.success(f"Extracted {len(genes)} genes from the panel") logger.info("Extracting intervals for genes from GFF...") with open(snakemake.input.annotation) as istream: ivtree = extract_intervals_for_genes_from_gff(genes, istream, padding) logger.success(f"Intervals extracted for {len(ivtree)} genes") logger.info( "Extracting those VCF records that fall within the gene intervals and altering " "their CHROM and POS accordingly...") vcf_reader = VCF(snakemake.input.vcf) logger.debug("Adding genes to header...") for iv in ivtree: vcf_reader.add_to_header( f"##contig=<ID={iv.data[0]},length={iv.end-iv.begin}>")
def convert_json_to_dataframe(self): license_list = [] # Iterate through the accounts for account in self.json_to_parse: accountName = account['accountName'] accountDomain = account['accountDomain'] accountStatus = account['accountStatus'] accountType = account['accountType'] account_dict = {'accountName': account['accountName'], 'accountDomain': account['accountDomain'], 'accountStatus': account['accountStatus'], 'accountType': account['accountType']} #Iterate through the "roles" for role_dict in account['roles']: # logger.info(' working on role_dict: {}'.format(role_dict)) role = role_dict['role'] account_dict['role'] = role # we are only interested, at least at this time, in the following roles. # "Appended VA USER" is a faux user created by the SmartAccountSDK due to some wonkiness in how # the BU Production Test domain is acting. if role in ['Virtual Account Administrator', 'Virtual Account User', 'APPENDED VA USER']: account_dict['virtualAccount'] = role_dict['virtualAccount'] account_dict['virtualAccount_status'] = "" account_dict['statusMessage'] = "" licenses_array = [] # There will be "assigned licenses" or licenses. Choose which one we have run into. if 'assignedLicenses' not in role_dict.keys(): # Check to see if there are issues. if 'licenses' not in role_dict.keys(): logger.error('No licenses or assignedLicenses found!!') continue licenses_array = role_dict['licenses'] logger.success('Licenses found!!') else: logger.success('assignedLicenses found!!') licenses_array = role_dict['assignedLicenses']['licenses'] account_dict['virtualAccount_status'] = role_dict['assignedLicenses']['status'] account_dict['statusMessage'] = role_dict['assignedLicenses']['statusMessage'] license_info_dict = {} account_dict_copy = {} # work through the array of licenses for license_dict in licenses_array: # logger.info(' working on license_dict: {}'.format(license_dict)) license_info_dict['license'] = license_dict['license'] license_info_dict['assignedLicenses_quantity'] = license_dict['quantity'] license_info_dict['inUse'] = license_dict['inUse'] license_info_dict['available'] = license_dict['available'] license_info_dict['ahaApps'] = license_dict['ahaApps'] license_info_dict['billingType'] = license_dict['billingType'] license_info_dict['pendingQuantity'] = license_dict['pendingQuantity'] license_info_dict['reserved'] = license_dict['reserved'] license_info_dict['isPortable'] = license_dict['isPortable'] license_info_dict['assignedLicenses_status'] = license_dict['status'] license_info_dict_copy = license_info_dict.copy() license_detail_dict = {} # Each license could have multiple instances. i.e. quantities purchase at different times # iterate through them for license_detail in license_dict['licenseDetails']: # logger.info(' working on license_detail: {}'.format(license_detail)) if type(license_detail['startDate']) is not type(None): if 'Z' in license_detail['startDate']: license_detail['startDate'] = \ pd.to_datetime(license_detail['startDate'], format='%m/%d/%y %H:%M', infer_datetime_format=True, utc=True).to_pydatetime() else: license_detail['startDate'] = \ pd.to_datetime(license_detail['startDate'], infer_datetime_format=True, utc=True).to_pydatetime() if type(license_detail['endDate']) is not type(None): if 'Z' in license_detail['endDate']: license_detail['endDate'] = \ pd.to_datetime(license_detail['endDate'], format='%m/%d/%y %H:%M', infer_datetime_format=True, utc=True).to_pydatetime() else: license_detail['endDate'] = \ pd.to_datetime(license_detail['endDate'], infer_datetime_format=True, utc=True).to_pydatetime() # Once we have everything, put into the list of licenses. # We create a copy of the account dict, because we want rows of data in the dataframe. # the information gathered earlier in the account_dict is common for all rows associated # with the account. i.e. accountName, virtualAccount, etc. account_dict_copy = account_dict.copy() account_dict_copy.update(license_info_dict_copy) account_dict_copy.update(license_detail) license_list.append(account_dict_copy) # Last but not least, create the Pandas Dataframe. df = pd.DataFrame(license_list) logger.info('CSSMJSONParser, convert_json_to_dataframe end') return df
def run(get_command, args): if args.fixed_seed: seed(getpass.getuser()) hosts = args.nodes if not hosts: hosts = ["localhost"] * minimum_number_of_local_nodes(args) LOG.info("Starting nodes on {}".format(hosts)) with infra.network.network(hosts, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb) as network: network.start_and_join(args) primary, backups = network.find_nodes() command_args = get_command_args(args, get_command) nodes_to_send_to = filter_nodes(primary, backups, args.send_tx_to) clients = [] client_hosts = [] if args.one_client_per_backup: if not backups: raise Exception( "--one-client-per-backup was set but no backup was found") client_hosts = ["localhost"] * len(backups) else: if args.client_nodes: client_hosts.extend(args.client_nodes) if args.num_localhost_clients: client_hosts.extend(["localhost"] * int(args.num_localhost_clients)) if not client_hosts: client_hosts = ["localhost"] for client_id, client_host in enumerate(client_hosts): node = nodes_to_send_to[client_id % len(nodes_to_send_to)] remote_client = configure_remote_client(args, client_id, client_host, node, command_args) clients.append(remote_client) if args.network_only: for remote_client in clients: LOG.info( f"Client can be run with: {remote_client.remote.get_cmd()}" ) while True: time.sleep(60) else: for remote_client in clients: remote_client.start() hard_stop_timeout = 90 try: with cimetrics.upload.metrics(complete=False) as metrics: tx_rates = infra.rates.TxRates(primary) start_time = time.time() while True: stop_waiting = True for i, remote_client in enumerate(clients): done = remote_client.check_done() # all the clients need to be done LOG.info( f"Client {i} has {'completed' if done else 'not completed'} running ({time.time() - start_time:.2f}s / {hard_stop_timeout}s)" ) stop_waiting = stop_waiting and done if stop_waiting: break if time.time() > start_time + hard_stop_timeout: raise TimeoutError( f"Client still running after {hard_stop_timeout}s" ) time.sleep(5) tx_rates.get_metrics() for remote_client in clients: perf_result = remote_client.get_result() LOG.success( f"{args.label}/{remote_client.name}: {perf_result}" ) # TODO: Only results for first client are uploaded # https://github.com/microsoft/CCF/issues/1046 if remote_client == clients[0]: LOG.success( f"Uploading results for {remote_client.name}") metrics.put(args.label, perf_result) else: LOG.warning( f"Skipping upload for {remote_client.name}") LOG.info(f"Rates:\n{tx_rates}") tx_rates.save_results(args.metrics_file) for remote_client in clients: remote_client.stop() except Exception: LOG.error("Stopping clients due to exception") for remote_client in clients: remote_client.stop() raise
async def wordle(app: Ariadne, message: MessageChain, group: Group, member: Member, single_game: ArgResult, dic: RegexResult, length: ArgResult, get_help: ArgResult, give_up: ArgResult, statistic: ArgResult) -> NoReturn: if get_help.matched: await app.sendGroupMessage( group, MessageChain( "Wordle文字游戏\n" "答案为指定长度单词,发送对应长度单词即可\n" "灰色块代表此单词中没有此字母\n" "黄色块代表此单词中有此字母,但该字母所处位置不对\n" "绿色块代表此单词中有此字母且位置正确\n" "猜出单词或用光次数则游戏结束\n" "发起游戏:/wordle -l=5 -d=SAT,其中-l/-length为单词长度,-d/-dic为指定词典,默认为5和CET4\n" "中途放弃:/wordle -g 或 /wordle -giveup\n" "查看数据统计:/wordle -s 或 /wordle -statistic\n" "查看提示:/wordle -hint\n" f"注:目前包含词典:{'、'.join(word_dics)}")) return None if statistic.matched: data = await get_member_statistic(group, member) await app.sendGroupMessage( group, MessageChain( f"用户 {member.name}\n" f"共参与{data[4]}场游戏,其中胜利{data[0]}场,失败{data[1]}场\n" f"一共猜对{data[2]}次,猜错{data[3]}次,共使用过{data[5]}次提示,再接再厉哦~"), quote=message.getFirst(Source)) return None if give_up.matched: return None await mutex.acquire() if group.id in group_running and group_running[group.id]: await app.sendGroupMessage(group, MessageChain("本群已有正在运行中的游戏实例,请等待本局游戏结束!")) mutex.release() return None else: if dic.matched: dic = dic.result.asDisplay().split('=')[1].strip() if dic not in word_dics: await app.sendGroupMessage( group, MessageChain(f"没有找到名为{dic}的字典!已有字典:{'、'.join(word_dics)}")) mutex.release() return None else: group_word_dic[group.id] = dic elif group.id not in group_word_dic: group_word_dic[group.id] = DEFAULT_DIC group_running[group.id] = True mutex.release() single = single_game.matched length = int(length.result.asDisplay().split('=') [1].strip()) if length.matched else 5 if length not in word_list[group_word_dic[group.id]].keys(): await app.sendGroupMessage( group, MessageChain( f"单词长度错误,词库中没有长度为{length}的单词=!" f"目前词库({group_word_dic[group.id]})中" f"只有长度为{'、'.join([str(i) for i in sorted(word_list[group_word_dic[group.id]].keys())])}的单词!" )) await mutex.acquire() group_running[group.id] = False mutex.release() return None wordle_instance = Wordle(length, dic=group_word_dic[group.id]) logger.success(f"成功创建 Wordle 实例,单词为:{wordle_instance.word}") await app.sendGroupMessage( group, MessageChain([ Image(data_bytes=wordle_instance.get_board_bytes()), Plain( f"\n你有{wordle_instance.row}次机会猜出单词,单词长度为{wordle_instance.length},请发送单词" ) ]), quote=message.getFirst(Source)) game_end = False try: while not game_end: game_end = await inc.wait(WordleWaiter(wordle_instance, group, member if single else None), timeout=300) except asyncio.exceptions.TimeoutError: await app.sendGroupMessage(group, MessageChain("游戏超时,进程结束"), quote=message.getFirst(Source)) await mutex.acquire() group_running[group.id] = False mutex.release()
def process_single_event(min_periods, max_periods, taper_tmin_tmax, asdf_filename, waveform_length, sampling_rate, output_directory, logfile, correct_cea, cea_correction_file): # with pyasdf.ASDFDataSet(asdf_filename) as ds: tmin, tmax = map(float, taper_tmin_tmax.split(",")) with pyasdf.ASDFDataSet(asdf_filename, mode="r") as ds: # load cea correction file if(correct_cea): correction_data = pd.read_csv(cea_correction_file, sep="|", comment="#", names=[ "network", "station", "eventno", "mean", "std", "median", "mad", "starttime", "endtime"]) correction_data["starttime"] = correction_data["starttime"].apply( modify_time) correction_data["endtime"] = correction_data["endtime"].apply( modify_time) # add logger information logger.add(logfile, format="{time} {level} {message}", level="INFO") # some parameters event = ds.events[0] origin = event.preferred_origin() or event.origins[0] event_time = origin.time event_latitude = origin.latitude event_longitude = origin.longitude for min_period, max_period in zip(min_periods, max_periods): # log if(isroot): logger.success( f"[{rank}/{size}] start to process {asdf_filename} from {min_period}s to {max_period}s") if(correct_cea): logger.success( f"[{rank}/{size}] will correct cea dataset according to the cea station orientation information") f2 = 1.0 / tmax f3 = 1.0 / tmin f1 = 0.5 * f2 f4 = 2.0 * f3 pre_filt = (f1, f2, f3, f4) # log if(isroot): logger.success( f"[{rank}/{size}] {asdf_filename} is filtered with {f1} {f2} {f3} {f4}") def process_function(st, inv): # log logger.info( f"[{rank}/{size}] processing {inv.get_contents()['stations'][0]}") # there are possibility that some stations has multiple loc codes or use HH stations. (should avoid in the future) st = filter_st(st, inv) # overlap the previous trace status_code = check_st_numberlap(st, inv) if(status_code == -1): return elif(status_code == 0): pass elif(status_code == 1): # merge may have roblem (samplign rate is not equal) try: st.merge(method=1, fill_value=0, interpolation_samples=0) except: logger.error( f"[{rank}/{size}] {inv.get_contents()['stations'][0]} error in merging") return else: raise Exception("unknown status code") status_code = check_time(st, event_time, waveform_length, inv) if(status_code == 0): pass elif(status_code == -1): logger.error( f"[{rank}/{size}] {inv.get_contents()['stations'][0]} error in cutting data") return else: raise Exception("unknown status code") # trim will automatically use starttime if starttime>eventtime st.trim(event_time, event_time+waveform_length) st.detrend("demean") st.detrend("linear") st.taper(max_percentage=0.05, type="hann") # st.remove_response(output="DISP", pre_filt=pre_filt, zero_mean=False, # taper=False, inventory=inv, water_level=None) # here we should use PZ files to remove the response. st = remove_response(st, pre_filt=pre_filt, inv=inv) # the same of removing response with sac st.detrend("demean") st.detrend("linear") st.interpolate(sampling_rate=sampling_rate) # ! have problem here (all value is zero) station_latitude = inv[0][0].latitude station_longitude = inv[0][0].longitude # baz is calculated using station and event's location # for cea stations, we can directly add an angle to it _, baz, _ = gps2dist_azimuth(station_latitude, station_longitude, event_latitude, event_longitude) network = inv.get_contents()['networks'][0] if(correct_cea and (network in CEA_NETWORKS)): baz = func_correct_cea( baz, inv, event_time, correction_data) if(baz == None): logger.error( f"[{rank}/{size}] {inv.get_contents()['stations'][0]} error in correcting orientation") return # we have to limit baz to be in [0,360) baz = np.mod(baz, 360) components = [tr.stats.channel[-1] for tr in st] if "N" in components and "E" in components: # there may be some problem in rotating (time span is not equal for three channels) try: st.rotate(method="NE->RT", back_azimuth=baz) except: logger.error( f"[{rank}/{size}] {inv.get_contents()['stations'][0]} error in rotating") return else: logger.error( f"[{rank}/{size}] {inv.get_contents()['stations'][0]} doesn't have both N and E") return # bandpass filter st.filter("bandpass", freqmin=1.0/max_period, freqmax=1.0/min_period, corners=2, zerophase=True) # Convert to single precision to save space. for tr in st: tr.data = np.require(tr.data, dtype="float32") return st tag_name = "preprocessed_%is_to_%is" % ( int(min_period), int(max_period)) tag_map = { "raw": tag_name } output_name_head = asdf_filename.split("/")[-1].split(".")[0] ds.process(process_function, join( output_directory, output_name_head+"."+tag_name + ".h5"), tag_map=tag_map) if(rank == 0): logger.success( f"[{rank}/{size}] success in processing {asdf_filename} from {min_period}s to {max_period}s")
def cmd(ceph_args, output, com): cfg = config.get() mg_paths = cfg["paths"] if output in ["json", "json-pretty"]: commands_dir = "must_gather_commands_json_output" json_add = "_--format_json-pretty" else: commands_dir = "must_gather_commands" json_add = "" file_name = "{}_{}{}".format(com, "_".join(ceph_args), json_add) if file_name.startswith("ceph_config_show_"): file_name = file_name.replace("ceph_config_show_", "config_") ceph_file = os.path.join("ceph", commands_dir, file_name) lg.debug("ceph_file: {}".format(ceph_file)) ceph_cmd_paths = {} i = 1 for p in mg_paths: ceph_cmd_path = os.path.join(p, ceph_file) if os.path.isfile(ceph_cmd_path): lg.info("Command output file found: {}".format(ceph_cmd_path)) ceph_cmd_paths[i] = ceph_cmd_path i += 1 if ceph_cmd_paths: for i, cp in ceph_cmd_paths.items(): with open(cp, "r") as lf: print(lf.read()) if len(mg_paths) > 1: lg.opt(colors=True).success("^^^<e>[{}]</>^^^\n".format(i)) else: suggestions = {} i = 1 for p in mg_paths: try: files = os.listdir( os.path.join(p, "ceph", "must_gather_commands")) file_match = "{}_{}".format(com, "_".join(ceph_args)) sugg = [] sugg.extend([ "omg " + f.replace("_", " ") for f in files if f.startswith(file_match) ]) sugg.extend([ "omg ceph config show {}".format(f.replace("config_", "")) for f in files if f.startswith("config_") ]) if sugg: suggestions[i] = sugg except Exception as e: lg.debug(e) pass i += 1 if suggestions: lg.success("\nNote: Output of following commands are available:\n") for i, sugg in suggestions.items(): lg.success("\n".join(sugg)) lg.opt(colors=True).success("^^^<e>[{}]</>^^^\n".format(i)) else: lg.error("Command output not found in any of the" " {} must-gather paths".format(len(mg_paths)))
def _x_fasterqd(self, version_only=False): """ Call fasterq-dump on SRR id to convert .sra file to .fastq files. """ if version_only: # print the version proc = sps.Popen( [self.binaries["fasterq-dump"], "-V"], stderr=sps.STDOUT, stdout=sps.PIPE, ) out = proc.communicate()[0].decode().split()[-1] return out # command and logger message cmd = [ self.binaries["fasterq-dump"], self.srr, "-O", os.path.join(self.workdir, self.srr), "-t", os.path.join(self.workdir, self.srr), ] null = "{fasterq-dump} {srr} -O {workdir}/{srr} -t {workdir}/{srr}" logger.info("Executing: {}".format(null)) logger.debug("Executing: {}".format(" ".join(cmd))) # call the tool proc = sps.Popen(cmd, stderr=sps.STDOUT, stdout=sps.PIPE) out = proc.communicate() if proc.returncode: logger.error("Failed: {}".format(out[0].decode())) logger.error( "If you encountered a disk full error but believe you have " "sufficient space available in your working dir then the " "is being caused by the insane behavior of the sra-tools " "package which hides large tmp file in obscure places. " "You can turn off this behavior by running 'vdb-config -i'." "Turn off the 'enable local file caching' option.") raise TypeError(out[0].decode()) # write a tmp SRR.lib file libfile = os.path.join(self.workdir, self.srr, "{}_files.lib".format(self.srr)) fastqs = glob.glob(os.path.join(self.workdir, self.srr, "*.fastq")) with open(libfile, 'w') as out: out.write("\n".join(fastqs)) # show file size result f1 = self.srr + "_1.fastq" fastq1 = os.path.join(self.workdir, self.srr, f1) size1 = os.path.getsize(fastq1) size1 = round(size1 / 1e9, 2) f2 = self.srr + "_2.fastq" fastq2 = os.path.join(self.workdir, self.srr, f2) size2 = os.path.getsize(fastq2) size2 = round(size2 / 1e9, 2) logger.success("Fastq dumped {} ({} Gb)".format(f1, size1)) logger.success("Fastq dumped {} ({} Gb)".format(f2, size2))
def setup(self): self.remote.setup() LOG.success(f"Remote client {self.name} setup")
def train( self, train_dataloader: DataIterator, validation_dataloader: DataIterator, ) -> Dict[str, float]: mi_not_improved = 0 for epoch in range(self._epochs): # Train self._pytorch_model.train() logger.info("Training") train_metrics = self._fit(train_dataloader) # Log metrics only on master with run_on_rank_zero decorator training_util.log_metrics( mode_str="Training", info={ "epoch": epoch, "aggressive": self._aggressive }, metrics=train_metrics, ) # Validation logger.info("Validation") validation_metrics = self.evaluate(validation_dataloader, info={ "epoch": epoch, "aggressive": self._aggressive }) # Check mutual info to finish aggressive training if needed if self._aggressive and self._model.is_kl_used: mi_not_improved += 1 # 5 is an expected number of aggressive epochs based on experiments from the paper if mi_not_improved == 5: self._aggressive = False logger.info("Stop aggressive burning.") if self._metric_patience: self._metric_patience(validation_metrics) # Save model state only on master if self._is_master: self._save_checkpoint( validation_metrics, is_best_so_far=self._metric_patience.improved if self._metric_patience else True, save_dict={ "model": self._model.state_dict(), "encoder_optimizer": self._encoder_optimizer.state_dict(), "decoder_optimizer": self._decoder_optimizer.state_dict(), "encoder_scheduler": self._encoder_scheduler.state_dict(), "decoder_scheduler": self._decoder_scheduler.state_dict(), **validation_metrics }, ) # Wait for master process to save new checkpoint if self._distributed: dist.barrier() if self._metric_patience.should_stop if self._metric_patience else False: logger.success("Patience reached. Stop training.") logger.info("Best metrics: {}".format( json.dumps(self._metric_patience.best_metrics, ensure_ascii=False, indent=2))) break return self._metric_patience.best_metrics if self._metric_patience else validation_metrics