def process_data_for_tasks_iterator(arguments): # removing the trailing slash if any if arguments.output and arguments.output[-1] == "/": arguments.output = arguments.output[:-1] ports = InputHelper._process_port(arguments.port) if arguments.port \ else None real_ports = InputHelper._process_port(arguments.realport) if \ arguments.realport else None str_targets, ipset_targets = InputHelper._process_targets( arguments=arguments, ) targets_count = len(str_targets) + ipset_targets.size if not targets_count: raise Exception("No target provided, or empty target list") if arguments.random: files = InputHelper._get_files_from_directory(arguments.random) random_file = choice(files) else: random_file = None tasks = list() if arguments.command: tasks.append(Task(arguments.command.rstrip('\n'))) else: tasks = InputHelper._pre_process_commands(arguments.command_list) if arguments.proto: protocols = arguments.proto.split(",") # if "," not in arguments.proto, [arguments.proto] is returned by # .split() else: protocols = None # Calculate the tasks count, as we will not have access to the len() of # the tasks iterator tasks_count = len(tasks) * targets_count if ports: tasks_count *= len(ports) if real_ports: tasks_count *= len(real_ports) if protocols: tasks_count *= len(protocols) return { "tasks": tasks, "str_targets": str_targets, "ipset_targets": ipset_targets, "ports": ports, "real_ports": real_ports, "random_file": random_file, "output": arguments.output, "protocols": protocols, "proxy_list": arguments.proxy_list, "tasks_count": tasks_count, }
def _pre_process_commands(command_list, task_name=None, is_global_task=True): """ :param command_list: :param task_name: all tasks have 'scope' and all scopes have unique names, global scope defaults None :param is_global_task: when True, signifies that all global tasks are meant to be run concurrently :return: list of possibly re-adjusted commands """ task_block = [] sibling = None blocker = None for command in command_list: command = str(command).strip() if len(command) == 0: continue # the start or end of a command block if (command.startswith('_block:') and command.endswith('_')) or\ command == '_block_': # if this is the end of a block, then we're done new_task_name = '' if command.startswith('_block:'): new_task_name = command.split('_block:')[1][:-1].strip() if task_name and task_name == new_task_name: return task_block # otherwise pre-process all the commands in this new `new_task_name` block tasks = InputHelper._pre_process_commands( command_list, new_task_name, False) if blocker: for task in tasks: task.wait_for(task_block) task_block += tasks if len(tasks) > 0: sibling = tasks[-1] continue else: # if a blocker is encountered, all commands following the blocker must wait until the last # command in the block is executed. All block commands are synchronous if command == '_blocker_': blocker = sibling continue task = Task(command) # if we're in the global scope and there was a previous _blocker_ encountered, we wait for the last # child of the block if is_global_task and blocker: task.wait_for(task_block) # all but the first command in a block scope wait for its predecessor elif sibling and not is_global_task: task.wait_for([sibling]) task_block.append(task) sibling = task return task_block
def process_commands(arguments): commands = list() ranges = set() targets = set() exclusions_ranges = set() exclusions = set() # removing the trailing slash if any if arguments.output and arguments.output[-1] == "/": arguments.output = arguments.output[:-1] if arguments.port: ports = InputHelper._process_port(arguments.port) if arguments.realport: real_ports = InputHelper._process_port(arguments.realport) # process targets first if arguments.target: ranges.add(arguments.target) else: target_file = arguments.target_list if not sys.stdin.isatty(): target_file = sys.stdin ranges.update( [target.strip() for target in target_file if target.strip()]) # process exclusions first if arguments.exclusions: exclusions_ranges.add(arguments.exclusions) else: if arguments.exclusions_list: for exclusion in arguments.exclusions_list: exclusion = exclusion.strip() if exclusion: exclusions.add(exclusion) # removing elements that may have spaces (helpful for easily processing comma notation) InputHelper._pre_process_hosts(ranges, targets, arguments) InputHelper._pre_process_hosts(exclusions_ranges, exclusions, arguments) # difference operation targets -= exclusions if len(targets) == 0: raise Exception("No target provided, or empty target list") if arguments.random: files = InputHelper._get_files_from_directory(arguments.random) random_file = choice(files) if arguments.command: commands.append(Task(arguments.command.rstrip('\n'))) else: commands = InputHelper._pre_process_commands( arguments.command_list) commands = InputHelper._replace_variable_with_commands( commands, "_target_", targets) commands = InputHelper._replace_variable_with_commands( commands, "_host_", targets) commands = InputHelper._process_clean_targets(commands, targets) if arguments.port: commands = InputHelper._replace_variable_with_commands( commands, "_port_", ports) if arguments.realport: commands = InputHelper._replace_variable_with_commands( commands, "_realport_", real_ports) if arguments.random: commands = InputHelper._replace_variable_with_commands( commands, "_random_", [random_file]) if arguments.output: commands = InputHelper._replace_variable_with_commands( commands, "_output_", [arguments.output]) if arguments.proto: if "," in arguments.proto: protocols = arguments.proto.split(",") else: protocols = arguments.proto commands = InputHelper._replace_variable_with_commands( commands, "_proto_", protocols) # process proxies if arguments.proxy_list: proxy_list = [ proxy for proxy in arguments.proxy_list if proxy.strip() ] if len(proxy_list) < len(commands): proxy_list = ceil(len(commands) / len(proxy_list)) * proxy_list InputHelper._replace_variable_array(commands, "_proxy_", proxy_list) return commands