def _establish_connection(self, run_id, dict_dag): try: static_info = self._prepare_static_info() r = requests.post(self.broadcast_address, json={ "run_id": run_id, "dag_json": dict_dag, "pipeline_files": static_info }) if r.status_code != 201: logger.error( colored_print( "ERROR: There was a problem sending data to the server" "with reason: {}".format(r.reason))) sys.exit(1) except requests.exceptions.ConnectionError: logger.error( colored_print( "ERROR: Could not establish connection with server. The server" " may be down or there is a problem with your internet " "connection.", "red_bold")) sys.exit(1)
def _close_connection(self, report_id): """Sends a delete request for the report JSON hash Parameters ---------- report_id : str Hash of the report JSON as retrieved from :func:`~_get_report_hash` """ logger.debug( "Closing connection and sending DELETE request to {}".format( self.broadcast_address)) try: r = requests.delete(self.broadcast_address, json={"run_id": report_id}) if r.status_code != 202: logger.error( colored_print( "ERROR: There was a problem sending data to the server" "with reason: {}".format(r.reason))) except requests.exceptions.ConnectionError: logger.error( colored_print( "ERROR: Could not establish connection with server. The server" " may be down or there is a problem with your internet " "connection.", "red_bold")) sys.exit(1)
def broadcast_status(self): logger.info(colored_print("Preparing broadcast data...", "green_bold")) run_hash = self._get_run_hash() dict_dag = self._dag_file_to_dict() _broadcast_sent = False self._establish_connection(run_hash, dict_dag) stay_alive = True try: while stay_alive: if not _broadcast_sent: self._print_msg(run_hash) _broadcast_sent = True self.update_inspection() if self.send: self._send_status_info(run_hash) self.send = False sleep(self.refresh_rate) except FileNotFoundError: logger.error( colored_print( "ERROR: nextflow log and/or trace files are no longer " "reachable!", "red_bold")) except Exception as e: logger.error("ERROR: ", sys.exc_info()[0]) finally: logger.info("Closing connection") self._close_connection(run_hash)
def check_arguments(args): passed = True # Check if no args are passed if len(sys.argv) == 1: logger.info( colored_print("Please provide one of the supported " "arguments!", "red_bold")) passed = False # Check if output argument is valid # Check if output file was provided, if it is not a directory, and if # the directory exists if not args.output_nf \ or os.path.isdir(args.output_nf) \ or (os.path.dirname(args.output_nf) and not os.path.isdir(os.path.dirname(args.output_nf))): logger.info( colored_print( "Please provide a valid output file and " "location!", "red_bold")) passed = False return passed
def guess_process(query_str, process_map): """ Function to guess processes based on strings that are not available in process_map. If the string has typos and is somewhat similar (50%) to any process available in flowcraft it will print info to the terminal, suggesting the most similar processes available in flowcraft. Parameters ---------- query_str: str The string of the process with potential typos process_map: The dictionary that contains all the available processes """ save_list = [] # loops between the processes available in process_map for process in process_map: similarity = SequenceMatcher(None, process, query_str) # checks if similarity between the process and the query string is # higher than 50% if similarity.ratio() > 0.5: save_list.append(process) # checks if any process is stored in save_list if save_list: logger.info(colored_print( "Maybe you meant:\n\t{}".format("\n\t".join(save_list)), "white")) logger.info(colored_print("Hint: check the available processes by using " "the '-l' or '-L' flag.", "white"))
def validate_build_arguments(args): if not args.tasks and not args.recipe and not args.check_only \ and not args.detailed_list and not args.short_list: logger.error( colored_print( "At least one of these options is required: -t, -r, -c, " "-l, -L", "red_bold")) sys.exit(1) if (args.tasks or args.recipe) and not args.output_nf: logger.error( colored_print( "Please provide the path and name of the pipeline file using the" " -o option.", "red_bold")) sys.exit(1) if args.output_nf: parsed_output_nf = (args.output_nf if args.output_nf.endswith(".nf") else "{}.nf".format(args.output_nf)) opath = parsed_output_nf if os.path.dirname(opath): parent_dir = os.path.dirname(opath) if not os.path.exists(parent_dir): logger.error( colored_print( "The provided directory '{}' does not exist.".format( parent_dir), "red_bold")) sys.exit(1) return parsed_output_nf
def validate_build_arguments(args): # Skip all checks when listing the processes if args.detailed_list or args.short_list: return # Skill all checks when exporting parameters AND providing at least one # component if args.export_params: # Check if components provided if not args.tasks: logger.error( colored_print( "At least one component needs to be provided via the -t option" " when exporting parameters in JSON format")) sys.exit(1) return # When none of the main run options is specified if not args.tasks and not args.recipe and not args.check_only \ and not args.detailed_list and not args.short_list: logger.error( colored_print( "At least one of these options is required: -t, -r, -c, " "-l, -L", "red_bold")) sys.exit(1) # When the build mode is active via tasks or recipe, but no output file # option has been provided if (args.tasks or args.recipe) and not args.check_recipe \ and not args.output_nf: logger.error( colored_print( "Please provide the path and name of the pipeline file using the" " -o option.", "red_bold")) sys.exit(1) if args.output_nf: if not os.path.basename(args.output_nf): logger.error( colored_print( "Output pipeline path '{}' missing a name (only the directory " "path was provided)".format(args.output_nf), "red_bold")) sys.exit(1) parsed_output_nf = (args.output_nf if args.output_nf.endswith(".nf") else "{}.nf".format(args.output_nf.strip())) opath = parsed_output_nf if os.path.dirname(opath): parent_dir = os.path.dirname(opath) if not os.path.exists(parent_dir): logger.error( colored_print( "The provided directory '{}' does not exist.".format( parent_dir), "red_bold")) sys.exit(1) return parsed_output_nf
def broadcast_report(self): logger.info( colored_print("Preparing to broacast reports...", "green_bold")) report_hash = self._get_report_id() # When in watch mode, if self.watch: logger.info( colored_print("\tFetching pipeline run status", "green_bold")) self._update_pipeline_status() logger.info( colored_print("\tSending initial request to test service", "green_bold")) self._init_live_reports(report_hash) logger.info( colored_print("\tInitial parsing of trace file", "green_bold")) self.update_trace_watch() self._print_msg(report_hash) logger.debug("Establishing connection...") stay_alive = True _broadcast_sent = False try: while stay_alive: # When not in watch mode, send the report JSON once if not _broadcast_sent and not self.watch: self._send_report(report_hash) self._print_msg(report_hash) _broadcast_sent = True # When in watch mode, continuously monitor the trace file for # updates if self.watch: self.update_trace_watch() self.update_log_watch() # When new report JSON files are available, send then # via a PUT request if self.send: self._send_live_report(report_hash) self.send = False sleep(self.refresh_rate) except FileNotFoundError as e: print(e) logger.error( colored_print("ERROR: Report JSON file is not reachable!", "red_bold")) except Exception as e: logger.exception("ERROR: " + e) finally: logger.info("Closing connection") self._close_connection(report_hash)
def list_recipes(full=False): """Method that iterates over all available recipes and prints their information to the standard output Parameters ---------- full : bool If true, it will provide the pipeline string along with the recipe name """ logger.info( colored_print("\n===== L I S T O F R E C I P E S =====\n", "green_bold")) # This will iterate over all modules included in the recipes subpackage # It will return the import class and the module name, algon with the # correct prefix prefix = "{}.".format(recipes.__name__) for importer, modname, _ in pkgutil.iter_modules(recipes.__path__, prefix): # Import the current module _module = importer.find_module(modname).load_module(modname) # Fetch all available classes in module _recipe_classes = [ cls for cls in _module.__dict__.values() if isinstance(cls, type) ] # Iterate over each Recipe class, and check for a match with the # provided recipe name. for cls in _recipe_classes: recipe_cls = cls() if hasattr(recipe_cls, "name"): logger.info( colored_print("=> {}".format(recipe_cls.name), "blue_bold")) if full: logger.info( colored_print("\t {}".format(recipe_cls.__doc__), "purple_bold")) logger.info( colored_print( "Pipeline string: {}\n".format( recipe_cls.pipeline_str), "yellow_bold")) sys.exit(0)
def _init_live_reports(self, report_id): """Sends a POST request to initialize the live reports Parameters ---------- report_id : str Hash of the report JSON as retrieved from :func:`~_get_report_hash` """ logger.debug("Sending initial POST request to {} to start report live" " update".format(self.broadcast_address)) try: with open(".metadata.json") as fh: metadata = [json.load(fh)] except: metadata = [] start_json = {"data": {"results": metadata}} try: requests.post(self.broadcast_address, json={ "run_id": report_id, "report_json": start_json, "status": self.status_info }) except requests.exceptions.ConnectionError: logger.error( colored_print( "ERROR: Could not establish connection with server. The server" " may be down or there is a problem with your internet " "connection.", "red_bold")) sys.exit(1)
def _print_msg(self, run_id): report_address = "{}reports/broadcast/{}".format(self.app_address, run_id) logger.info(colored_print( "The pipeline reports are available in the following link:", "green_bold")) logger.info("{}".format(report_address))
def _close_connection(self, run_id): try: r = requests.delete(self.broadcast_address, json={"run_id": run_id}) if r.status_code != 202: logger.error( colored_print( "ERROR: There was a problem sending data to the server" "with reason: {}".format(r.reason))) except requests.exceptions.ConnectionError: logger.error( colored_print( "ERROR: Could not establish connection with server. The server" " may be down or there is a problem with your internet " "connection.", "red_bold")) sys.exit(1)
def _print_msg(self, run_id): inspect_address = "{}inspect/{}".format(self.app_address, run_id) logger.info( colored_print( "Starting broadcast. You can see the pipeline progress on the " "link below:", "green_bold")) logger.info("{}".format(inspect_address))
def inspect(args): try: nf_inspect = NextflowInspector(args.trace_file, args.refresh_rate, args.pretty, args.url) if args.mode == "overview": nf_inspect.display_overview() if args.mode == "broadcast": nf_inspect.broadcast_status() except eh.InspectionError as ie: logger.error(colored_print(ie.value, "red_bold")) sys.exit(1) except eh.LogError as le: logger.error(colored_print(le.value, "red_bold")) sys.exit(1)
def report(args): try: fc_report = FlowcraftReport(report_file=args.report_file, trace_file=args.trace_file, log_file=args.log_file, watch=args.watch, ip_addr=args.url) fc_report.broadcast_report() except eh.ReportError as re: logger.error(colored_print(re.value, "red_bold")) sys.exit(1) except eh.LogError as le: logger.error(colored_print(le.value, "red_bold")) sys.exit(1)
def check_arguments(args): # Check if no args are passed if len(sys.argv) == 1: logger.info( colored_print("Please provide one of the supported " "arguments!", "red_bold")) return False return True
def _set_channels(self): """Sets the main channels for the pipeline This method will parse de the :attr:`~Process.processes` attribute and perform the following tasks for each process: - Sets the input/output channels and main input forks and adds them to the process's :attr:`assemblerflow.process.Process._context` attribute (See :func:`~NextflowGenerator.set_channels`). - Automatically updates the main input channel of the first process of each lane so that they fork from the user provide parameters (See :func:`~NextflowGenerator._update_raw_input`). - Check for the presence of secondary inputs and adds them to the :attr:`~NextflowGenerator.secondary_inputs` attribute. - Check for the presence of secondary channels and adds them to the :attr:`~NextflowGenerator.secondary_channels` attribute. Notes ----- **On the secondary channel setup**: With this approach, there can only be one secondary link start for each type of secondary link. For instance, If there are two processes that start a secondary channel for the ``SIDE_max_len`` channel, only the last one will be recorded, and all receiving processes will get the channel from the latest process. Secondary channels can only link if the source process if downstream of the sink process in its "forking" path. """ logger.debug("=====================") logger.debug("Setting main channels") logger.debug("=====================") for i, p in enumerate(self.processes): # Set main channels for the process logger.debug("[{}] Setting main channels with pid: {}".format( p.template, i)) p.set_channels(pid=i) # If there is no parent lane, set the raw input channel from user if not p.parent_lane and p.input_type: self._update_raw_input(p) self._update_secondary_inputs(p) self._update_secondary_channels(p) logger.info( colored_print("\tChannels set for {} \u2713".format( p.template)))
def build(args): welcome = [ "========= F L O W C R A F T =========", "Build mode\n" "version: {}".format(__version__), "build: {}".format(__build__), "=====================================" ] parsed_output_nf = validate_build_arguments(args) logger.info(colored_print("\n".join(welcome), "green_bold")) # If a recipe is specified, build pipeline based on the # appropriate recipe if args.recipe: pipeline_string, list_processes = brew_recipe(args) else: pipeline_string = args.tasks list_processes = None # used for lists print proc_collector(process_map, args, list_processes) logger.info(colored_print("Resulting pipeline string:\n")) logger.info(colored_print(pipeline_string + "\n")) try: logger.info(colored_print("Checking pipeline for errors...")) pipeline_list = parse_pipeline(pipeline_string) except SanityError as e: logger.error(colored_print(e.value, "red_bold")) sys.exit(1) logger.debug("Pipeline successfully parsed: {}".format(pipeline_list)) # Exit if only the pipeline parser needs to be checked if args.check_only: sys.exit() nfg = NextflowGenerator(process_connections=pipeline_list, nextflow_file=parsed_output_nf, pipeline_name=args.pipeline_name, auto_dependency=args.no_dep) logger.info(colored_print("Building your awesome pipeline...")) # building the actual pipeline nf file nfg.build() # copy template to cwd, to allow for immediate execution if not args.pipeline_only: copy_project(parsed_output_nf) logger.info(colored_print("DONE!", "green_bold"))
def brew_recipe(args): """Brews a given list of processes according to the recipe Parameters ---------- args : argparse.Namespace The arguments passed through argparser that will be used to check the the recipe, tasks and brew the process Returns ------- str The final pipeline string, ready for the engine. list List of process strings. """ # Exit if recipe does not exist if args.recipe not in available_recipes: logger.error( colored_print( "Please provide a recipe to use in automatic " "mode.", "red_bold")) sys.exit(1) # Create recipe class instance automatic_pipeline = available_recipes[args.recipe]() if not args.tasks: input_processes = " ".join( automatic_pipeline.process_descriptions.keys()) else: input_processes = args.tasks # Get the list of processes for that recipe list_processes = automatic_pipeline.get_process_info() # Validate the provided pipeline processes validated = automatic_pipeline.validate_pipeline(input_processes) if not validated: sys.exit(1) # Get the final pipeline string pipeline_string = automatic_pipeline.run_auto_pipeline(input_processes) return pipeline_string, list_processes
def _send_report(self, report_id): with open(self.report_file) as fh: report_json = json.loads(fh.read()) logger.debug("Unique payload sent with size: {}".format( asizeof(json.dumps(report_json)) )) try: requests.post( self.broadcast_address, json={"run_id": report_id, "report_json": report_json} ) except requests.exceptions.ConnectionError: logger.error(colored_print( "ERROR: Could not establish connection with server. The server" " may be down or there is a problem with your internet " "connection.", "red_bold")) sys.exit(1)
def validate_pipeline(pipeline_string): """Validate pipeline string Validates the pipeline string by searching for forbidden characters Parameters ---------- pipeline_string : str STring with the processes provided Returns ------- """ if "(" in pipeline_string or ")" in pipeline_string or "|" in \ pipeline_string: logger.error( colored_print("Please provide a valid task list!", "red_bold")) return False return True
def _dag_file_to_dict(self): """Function that opens the dotfile named .treeDag.json in the current working directory Returns ------- Returns a dictionary with the dag object to be used in the post instance available through the method _establish_connection """ try: dag_file = open(os.path.join(self.workdir, ".treeDag.json")) dag_json = json.load(dag_file) except (FileNotFoundError, json.decoder.JSONDecodeError): logger.warning( colored_print( "WARNING: dotfile named .treeDag.json not found or corrupted", "red_bold")) dag_json = {} return dag_json
def brew_recipe(recipe_name): """Returns a pipeline string from a recipe name. Parameters ---------- recipe_name : str Name of the recipe. Must match the name attribute in one of the classes defined in :mod:`flowcraft.generator.recipes` Returns ------- str Pipeline string ready for parsing and processing by flowcraft engine """ # This will iterate over all modules included in the recipes subpackage # It will return the import class and the module name, algon with the # correct prefix prefix = "{}.".format(recipes.__name__) for importer, modname, _ in pkgutil.iter_modules(recipes.__path__, prefix): # Import the current module _module = importer.find_module(modname).load_module(modname) # Fetch all available classes in module _recipe_classes = [ cls for cls in _module.__dict__.values() if isinstance(cls, type) ] # Iterate over each Recipe class, and check for a match with the # provided recipe name. for cls in _recipe_classes: # Create instance of class to allow fetching the name attribute recipe_cls = cls() if getattr(recipe_cls, "name", None) == recipe_name: return recipe_cls.brew() logger.error( colored_print("Recipe name '{}' does not exist.".format(recipe_name))) sys.exit(1)
def display_overview(self): """Displays the default pipeline inspection overview """ stay_alive = True self.screen = curses.initscr() self.screen.keypad(True) self.screen.nodelay(-1) curses.cbreak() curses.noecho() curses.start_color() self.screen_lines = self.screen.getmaxyx()[0] # self.screen_width = self.screen.getmaxyx()[1] try: while stay_alive: # Provide functionality to certain keybindings self._curses_keybindings() # Updates main inspector attributes self.update_inspection() # Display curses interface self.flush_overview() sleep(self.refresh_rate) except FileNotFoundError: sys.stderr.write( colored_print( "ERROR: nextflow log and/or trace files are no longer " "reachable!", "red_bold")) except Exception as e: sys.stderr.write(str(e)) finally: curses.nocbreak() self.screen.keypad(0) curses.echo() curses.endwin()
def _send_status_info(self, run_id): mappings, data = self._prepare_table_data() overview_data = self._prepare_overview_data() general_details = self._prepare_general_details() status_data = self._prepare_run_status_data() status_json = { "generalOverview": overview_data, "generalDetails": general_details, "tableData": data, "tableMappings": mappings, "processInfo": self._convert_process_dict(), "processTags": self.process_tags, "runStatus": status_data, "timeStart": str(self.time_start), "timeStop": str(self.time_stop) if self.time_stop else "-", "processes": list(self.processes) } self._c += 1 logger.debug("Payload [{}] sent with size: {}".format( self._c, asizeof.asizeof(json.dumps(status_json)))) try: requests.put(self.broadcast_address, json={ "run_id": run_id, "status_json": status_json }) except requests.exceptions.ConnectionError: logger.error( colored_print( "ERROR: Could not establish connection with server. The server" " may be down or there is a problem with your internet " "connection.", "red_bold")) sys.exit(1)
def build(args): # Disable standard logging for stdout when the following modes are # executed: if args.export_params or args.export_directives: logger.setLevel(logging.ERROR) welcome = [ "========= F L O W C R A F T =========", "Build mode\n" "version: {}".format(__version__), "build: {}".format(__build__), "=====================================" ] parsed_output_nf = validate_build_arguments(args) logger.info(colored_print("\n".join(welcome), "green_bold")) # If a recipe is specified, build pipeline based on the # appropriate recipe if args.recipe: if args.recipe == "innuendo": pipeline_string = brew_recipe(args, available_recipes) else: pipeline_string = available_recipes[args.recipe] if args.tasks: logger.warning(colored_print( "-t parameter will be ignored for recipe: {}\n" .format(args.recipe), "yellow_bold") ) if args.check_recipe: logger.info(colored_print("Pipeline string for recipe: {}" .format(args.recipe), "purple_bold")) logger.info(pipeline_string) sys.exit(0) else: pipeline_string = args.tasks # used for lists print proc_collector(process_map, args, pipeline_string) try: logger.info(colored_print("Checking pipeline for errors...")) pipeline_list = parse_pipeline(pipeline_string) except SanityError as e: logger.error(colored_print(e.value, "red_bold")) sys.exit(1) logger.debug("Pipeline successfully parsed: {}".format(pipeline_list)) # Exit if only the pipeline parser needs to be checked if args.check_only: sys.exit() nfg = NextflowGenerator(process_connections=pipeline_list, nextflow_file=parsed_output_nf, pipeline_name=args.pipeline_name, auto_dependency=args.no_dep, merge_params=args.merge_params, export_params=args.export_params) logger.info(colored_print("Building your awesome pipeline...")) if args.export_params: nfg.export_params() sys.exit(0) elif args.export_directives: nfg.export_directives() sys.exit(0) else: # building the actual pipeline nf file nfg.build() # copy template to cwd, to allow for immediate execution if not args.pipeline_only: copy_project(parsed_output_nf) logger.info(colored_print("DONE!", "green_bold"))
def build_upstream(self, process_descriptions, task, all_tasks, task_pipeline, count_forks, total_tasks, forks): """Builds the upstream pipeline of the current process Checks for the upstream processes to the current process and adds them to the current pipeline fragment if they were provided in the process list. Parameters ---------- process_descriptions : dict Information of processes input, output and if is forkable task : str Current process all_tasks : list A list of all provided processes task_pipeline : list Current pipeline fragment count_forks : int Current number of forks total_tasks : str All space separated processes forks : list Current forks Returns ------- list : resulting pipeline fragment """ if task in process_descriptions: if process_descriptions[task][1] is not None: if len(process_descriptions[task][1].split("|")) > 1: local_forks = process_descriptions[task][1].split("|") # Produces a new pipeline fragment for each forkable # process for local_fork in local_forks: if local_fork in total_tasks: count_forks += 1 task_pipeline.insert(0, process_descriptions[task][1]) self.define_pipeline_string( process_descriptions, local_fork, False, True, count_forks, total_tasks, forks) return task_pipeline else: # Adds the process to the pipeline fragment in case it is # provided in the task list if process_descriptions[task][1] in total_tasks: task_pipeline.insert( 0, process_descriptions[task][1].split("|")[0]) # Proceeds building upstream until the input for a # process is None self.build_upstream( process_descriptions, process_descriptions[task][1].split("|")[0], all_tasks, task_pipeline, count_forks, total_tasks, forks) else: logger.error( colored_print( "{} not in provided protocols as " "input for {}".format( process_descriptions[task][1], task), "red_bold")) sys.exit() return task_pipeline else: return task_pipeline
def define_pipeline_string(self, process_descriptions, tasks, check_upstream, check_downstream, count_forks, total_tasks, forks): """Builds the possible forks and connections between the provided processes This method loops through all the provided tasks and builds the upstream and downstream pipeline if required. It then returns all possible forks than need to be merged à posteriori` Parameters ---------- process_descriptions : dict Information of processes input, output and if is forkable tasks : str Space separated processes check_upstream : bool If is to build the upstream pipeline of the current task check_downstream : bool If is to build the downstream pipeline of the current task count_forks : int Number of current forks total_tasks : str All space separated processes forks : list Current forks Returns ------- list : List with all the possible pipeline forks """ tasks_array = tasks.split() for task_unsplit in tasks_array: task = task_unsplit.split("=")[0] if task not in process_descriptions.keys(): logger.error( colored_print( "{} not in the possible processes".format(task), "red_bold")) sys.exit() else: process_split = task_unsplit.split("=") if len(process_split) > 1: self.process_to_id[process_split[0]] = process_split[1] # Only uses the process if it is not already in the possible forks if not bool([x for x in forks if task in x]) and not bool( [y for y in forks if process_descriptions[task][2] in y]): task_pipeline = [] if task in process_descriptions: if check_upstream: task_pipeline = self.build_upstream( process_descriptions, task, tasks_array, task_pipeline, count_forks, total_tasks, forks) task_pipeline.append(task) if check_downstream: task_pipeline = self.build_downstream( process_descriptions, task, tasks_array, task_pipeline, count_forks, total_tasks, forks) # Adds the pipeline fragment to the list of possible forks forks.append(list(OrderedDict.fromkeys(task_pipeline))) # Checks for task in fork. Case order of input processes is reversed elif bool([y for y in forks if process_descriptions[task][2] in y]): for fork in forks: if task not in fork: try: dependent_index = fork.index( process_descriptions[task][2]) fork.insert(dependent_index, task) except ValueError: continue for i in range(0, len(forks)): for j in range(0, len(forks[i])): try: if len(forks[i][j].split("|")) > 1: forks[i][j] = forks[i][j].split("|") tmp_fork = [] for s in forks[i][j]: if s in total_tasks: tmp_fork.append(s) forks[i][j] = tmp_fork except AttributeError as e: continue return forks
def build(self): """Main pipeline builder This method is responsible for building the :py:attr:`NextflowGenerator.template` attribute that will contain the nextflow code of the pipeline. First it builds the header, then sets the main channels, the secondary inputs, secondary channels and finally the status channels. When the pipeline is built, is writes the code to a nextflow file. """ logger.info( colored_print("\tSuccessfully connected {} process(es) with {} " "fork(s) across {} lane(s) \u2713".format( len(self.processes[1:]), len(self._fork_tree), self.lanes))) # Generate regular nextflow header that sets up the shebang, imports # and all possible initial channels self._build_header() self._set_channels() pipeline_to_json = self.render_pipeline() self._set_secondary_inputs() logger.info( colored_print( "\tSuccessfully set {} secondary input(s) \u2713".format( len(self.secondary_inputs)))) self._set_secondary_channels() logger.info( colored_print( "\tSuccessfully set {} secondary channel(s) \u2713".format( len(self.secondary_channels)))) self._set_compiler_channels() self._set_configurations() logger.info( colored_print("\tFinished configurations \u2713".format( len(self.secondary_channels)))) for p in self.processes: self.template += p.template_str self._build_footer() project_root = dirname(self.nf_file) # Write pipeline file with open(self.nf_file, "w") as fh: fh.write(self.template) # Write resources config with open(join(project_root, "resources.config"), "w") as fh: fh.write(self.resources) # Write containers config with open(join(project_root, "containers.config"), "w") as fh: fh.write(self.containers) # Write containers config with open(splitext(self.nf_file)[0] + ".html", "w") as fh: fh.write(pipeline_to_json) logger.info( colored_print("\tPipeline written into {} \u2713".format( self.nf_file)))
def _send_live_report(self, report_id): """Sends a PUT request with the report JSON files currently in the report_queue attribute. Parameters ---------- report_id : str Hash of the report JSON as retrieved from :func:`~_get_report_hash` """ # Determines the maximum number of reports sent at the same time in # the same payload buffer_size = 100 logger.debug("Report buffer size set to: {}".format(buffer_size)) for i in range(0, len(self.report_queue), buffer_size): # Reset the report compilation batch reports_compilation = [] # Iterate over report JSON batches determined by buffer_size for report in self.report_queue[i:i + buffer_size]: try: report_file = [ x for x in os.listdir(report) if x.endswith(".json") ][0] except IndexError: continue with open(join(report, report_file)) as fh: reports_compilation.append(json.loads(fh.read())) logger.debug("Payload sent with size: {}".format( asizeof(json.dumps(reports_compilation)))) logger.debug("status: {}".format(self.status_info)) try: requests.put(self.broadcast_address, json={ "run_id": report_id, "report_json": reports_compilation, "status": self.status_info }) except requests.exceptions.ConnectionError: logger.error( colored_print( "ERROR: Could not establish connection with server. The server" " may be down or there is a problem with your internet " "connection.", "red_bold")) sys.exit(1) # When there is no change in the report queue, but there is a change # in the run status of the pipeline if not self.report_queue: logger.debug("status: {}".format(self.status_info)) try: requests.put(self.broadcast_address, json={ "run_id": report_id, "report_json": [], "status": self.status_info }) except requests.exceptions.ConnectionError: logger.error( colored_print( "ERROR: Could not establish connection with server. The" " server may be down or there is a problem with your " "internet connection.", "red_bold")) sys.exit(1) # Reset the report queue after sending the request self.report_queue = []