def process_data_for_tasks_iterator(arguments):
# removing the trailing slash if any
if arguments.output and arguments.output[-1] == "/":
arguments.output = arguments.output[:-1]
ports = InputHelper._process_port(arguments.port) if arguments.port \
else None
real_ports = InputHelper._process_port(arguments.realport) if \
arguments.realport else None
str_targets, ipset_targets = InputHelper._process_targets(
arguments=arguments,
)
targets_count = len(str_targets) + ipset_targets.size
if not targets_count:
raise Exception("No target provided, or empty target list")
if arguments.random:
files = InputHelper._get_files_from_directory(arguments.random)
random_file = choice(files)
else:
random_file = None
tasks = list()
if arguments.command:
tasks.append(Task(arguments.command.rstrip('\n')))
else:
tasks = InputHelper._pre_process_commands(arguments.command_list)
if arguments.proto:
protocols = arguments.proto.split(",")
# if "," not in arguments.proto, [arguments.proto] is returned by
# .split()
else:
protocols = None
# Calculate the tasks count, as we will not have access to the len() of
# the tasks iterator
tasks_count = len(tasks) * targets_count
if ports:
tasks_count *= len(ports)
if real_ports:
tasks_count *= len(real_ports)
if protocols:
tasks_count *= len(protocols)
return {
"tasks": tasks,
"str_targets": str_targets,
"ipset_targets": ipset_targets,
"ports": ports,
"real_ports": real_ports,
"random_file": random_file,
"output": arguments.output,
"protocols": protocols,
"proxy_list": arguments.proxy_list,
"tasks_count": tasks_count,
}
def _pre_process_commands(command_list,
task_name=None,
is_global_task=True):
"""
:param command_list:
:param task_name: all tasks have 'scope' and all scopes have unique names, global scope defaults None
:param is_global_task: when True, signifies that all global tasks are meant to be run concurrently
:return: list of possibly re-adjusted commands
"""
task_block = []
sibling = None
blocker = None
for command in command_list:
command = str(command).strip()
if len(command) == 0:
continue
# the start or end of a command block
if (command.startswith('_block:') and command.endswith('_')) or\
command == '_block_':
# if this is the end of a block, then we're done
new_task_name = ''
if command.startswith('_block:'):
new_task_name = command.split('_block:')[1][:-1].strip()
if task_name and task_name == new_task_name:
return task_block
# otherwise pre-process all the commands in this new `new_task_name` block
tasks = InputHelper._pre_process_commands(
command_list, new_task_name, False)
if blocker:
for task in tasks:
task.wait_for(task_block)
task_block += tasks
if len(tasks) > 0:
sibling = tasks[-1]
continue
else:
# if a blocker is encountered, all commands following the blocker must wait until the last
# command in the block is executed. All block commands are synchronous
if command == '_blocker_':
blocker = sibling
continue
task = Task(command)
# if we're in the global scope and there was a previous _blocker_ encountered, we wait for the last
# child of the block
if is_global_task and blocker:
task.wait_for(task_block)
# all but the first command in a block scope wait for its predecessor
elif sibling and not is_global_task:
task.wait_for([sibling])
task_block.append(task)
sibling = task
return task_block
def process_commands(arguments):
commands = list()
ranges = set()
targets = set()
exclusions_ranges = set()
exclusions = set()
# removing the trailing slash if any
if arguments.output and arguments.output[-1] == "/":
arguments.output = arguments.output[:-1]
if arguments.port:
ports = InputHelper._process_port(arguments.port)
if arguments.realport:
real_ports = InputHelper._process_port(arguments.realport)
# process targets first
if arguments.target:
ranges.add(arguments.target)
else:
target_file = arguments.target_list
if not sys.stdin.isatty():
target_file = sys.stdin
ranges.update(
[target.strip() for target in target_file if target.strip()])
# process exclusions first
if arguments.exclusions:
exclusions_ranges.add(arguments.exclusions)
else:
if arguments.exclusions_list:
for exclusion in arguments.exclusions_list:
exclusion = exclusion.strip()
if exclusion:
exclusions.add(exclusion)
# removing elements that may have spaces (helpful for easily processing comma notation)
InputHelper._pre_process_hosts(ranges, targets, arguments)
InputHelper._pre_process_hosts(exclusions_ranges, exclusions,
arguments)
# difference operation
targets -= exclusions
if len(targets) == 0:
raise Exception("No target provided, or empty target list")
if arguments.random:
files = InputHelper._get_files_from_directory(arguments.random)
random_file = choice(files)
if arguments.command:
commands.append(Task(arguments.command.rstrip('\n')))
else:
commands = InputHelper._pre_process_commands(
arguments.command_list)
commands = InputHelper._replace_variable_with_commands(
commands, "_target_", targets)
commands = InputHelper._replace_variable_with_commands(
commands, "_host_", targets)
commands = InputHelper._process_clean_targets(commands, targets)
if arguments.port:
commands = InputHelper._replace_variable_with_commands(
commands, "_port_", ports)
if arguments.realport:
commands = InputHelper._replace_variable_with_commands(
commands, "_realport_", real_ports)
if arguments.random:
commands = InputHelper._replace_variable_with_commands(
commands, "_random_", [random_file])
if arguments.output:
commands = InputHelper._replace_variable_with_commands(
commands, "_output_", [arguments.output])
if arguments.proto:
if "," in arguments.proto:
protocols = arguments.proto.split(",")
else:
protocols = arguments.proto
commands = InputHelper._replace_variable_with_commands(
commands, "_proto_", protocols)
# process proxies
if arguments.proxy_list:
proxy_list = [
proxy for proxy in arguments.proxy_list if proxy.strip()
]
if len(proxy_list) < len(commands):
proxy_list = ceil(len(commands) / len(proxy_list)) * proxy_list
InputHelper._replace_variable_array(commands, "_proxy_",
proxy_list)
return commands
