class ClassifierSignature:
def __init__(self):
self.full_path = os.path.dirname(os.path.abspath(__file__))
self.signature_path = os.path.join(self.full_path, 'signature')
self.util = Utilty()
# Identify product name.
def identify_product(self, categoy, response):
prod_info_list = []
file_name = 'signature_' + categoy + '.txt'
try:
# Judge product using pattern matching.
with codecs.open(os.path.join(self.signature_path, file_name), 'r', 'utf-8') as fin:
matching_patterns = fin.readlines()
for pattern in matching_patterns:
items = pattern.replace('\r', '').replace('\n', '').split('@')
product = items[0]
signature = items[2]
list_match = re.findall(signature, response, flags=re.IGNORECASE)
if len(list_match) != 0:
# Add product name and version.
prod_info_list.append(product + str(items[1]) + ' ' + list_match[0])
except Exception as err:
self.util.print_message(WARNING, '{}'.format(err))
return list(set(prod_info_list))
# Classifier product name using signatures.
def classifier_signature(self, target_info):
product_list = []
for target in target_info:
for target_url in target[2]:
# Get HTTP response (header + body).
response = ''
http = urllib3.PoolManager(timeout=self.util.http_timeout)
try:
self.util.print_message(OK, 'Accessing: {}'.format(target_url))
res = http.request('GET', target_url)
for key in res.headers._container.keys():
response += key + ': ' + res.headers[key] + '\r\n'
response += '\r\n\r\n' + res.data.decode('utf-8')
except Exception as err:
self.util.print_message(WARNING, '{}'.format(err))
for category in ['os', 'web', 'framework', 'cms']:
prod_info = self.identify_product(category, response)
for product in prod_info:
parsed = util.parse_url(target_url)
product_list.append([product, parsed.scheme, parsed.port, parsed.path])
return product_list
class Msgrpc:
def __init__(self, option=[]):
self.utility = Utilty()
self.host = option.get('host') or "127.0.0.1"
self.port = option.get('port') or 55552
self.uri = option.get('uri') or "/api/"
self.ssl = option.get('ssl') or False
self.authenticated = False
self.token = False
self.headers = {"Content-type": "binary/message-pack"}
if self.ssl:
self.client = http.client.HTTPSConnection(self.host, self.port)
else:
self.client = http.client.HTTPConnection(self.host, self.port)
# Call RPC API.
def call(self, meth, option):
if meth != "auth.login":
if not self.authenticated:
self.utility.print_message(FAIL, 'MsfRPC: Not Authenticated')
exit(1)
if meth != "auth.login":
option.insert(0, self.token)
option.insert(0, meth)
params = msgpack.packb(option)
self.client.request("POST", self.uri, params, self.headers)
resp = self.client.getresponse()
return msgpack.unpackb(resp.read())
# Log in to RPC Server.
def login(self, user, password):
ret = self.call('auth.login', [user, password])
if ret.get(b'result') == b'success':
self.authenticated = True
self.token = ret.get(b'token')
return True
else:
self.utility.print_message(FAIL, 'MsfRPC: Not Authenticated')
exit(1)
# Send Metasploit command.
def send_command(self, console_id, command, visualization, sleep=0.1):
_ = self.call('console.write', [console_id, command])
time.sleep(sleep)
ret = self.call('console.read', [console_id])
if visualization:
try:
self.utility.print_message(
NONE, '{}'.format(ret.get(b'data').decode('utf-8')))
except Exception as e:
self.utility.print_exception(e, 'Send_command is exception.')
return ret
# Get all modules.
def get_module_list(self, module_type):
ret = {}
if module_type == 'exploit':
ret = self.call('module.exploits', [])
elif module_type == 'auxiliary':
ret = self.call('module.auxiliary', [])
elif module_type == 'post':
ret = self.call('module.post', [])
elif module_type == 'payload':
ret = self.call('module.payloads', [])
elif module_type == 'encoder':
ret = self.call('module.encoders', [])
elif module_type == 'nop':
ret = self.call('module.nops', [])
byte_list = ret[b'modules']
string_list = []
for module in byte_list:
string_list.append(module.decode('utf-8'))
return string_list
# Get module detail information.
def get_module_info(self, module_type, module_name):
return self.call('module.info', [module_type, module_name])
# Get payload that compatible module.
def get_compatible_payload_list(self, module_name):
ret = self.call('module.compatible_payloads', [module_name])
byte_list = ret[b'payloads']
string_list = []
for module in byte_list:
string_list.append(module.decode('utf-8'))
return string_list
# Get payload that compatible target.
def get_target_compatible_payload_list(self, module_name, target_num):
ret = self.call('module.target_compatible_payloads',
[module_name, target_num])
byte_list = ret[b'payloads']
string_list = []
for module in byte_list:
string_list.append(module.decode('utf-8'))
return string_list
# Get module options.
def get_module_options(self, module_type, module_name):
return self.call('module.options', [module_type, module_name])
# Execute module.
def execute_module(self, module_type, module_name, options):
ret = self.call('module.execute', [module_type, module_name, options])
job_id = ret[b'job_id']
uuid = ret[b'uuid'].decode('utf-8')
return job_id, uuid
# Get job list.
def get_job_list(self):
jobs = self.call('job.list', [])
byte_list = jobs.keys()
job_list = []
for job_id in byte_list:
job_list.append(int(job_id.decode('utf-8')))
return job_list
# Get job detail information.
def get_job_info(self, job_id):
return self.call('job.info', [job_id])
# Stop job.
def stop_job(self, job_id):
return self.call('job.stop', [job_id])
# Get session list.
def get_session_list(self):
return self.call('session.list', [])
# Stop shell session.
def stop_session(self, session_id):
_ = self.call('session.stop', [str(session_id)])
# Stop meterpreter session.
def stop_meterpreter_session_kill(self, session_id):
_ = self.call('session.meterpreter_session_kill', [str(session_id)])
# Log out from RPC Server.
def logout(self):
ret = self.call('auth.logout', [self.token])
if ret.get(b'result') == b'success':
self.authenticated = False
self.token = ''
return True
else:
self.utility.print_message(FAIL, 'MsfRPC: Not Authenticated')
exit(1)
# Disconnection.
def termination(self, console_id):
# Kill a console.
_ = self.call('console.session_kill', [console_id])
# Log out
_ = self.logout()
report.add_ss_items_to_inventory_report(
report_path, screen_shot_list, df_report)
# Explore domains and subdomains from search result of DomainTools.
elif opt_invent_org_list is not None and os.path.exists(org_list_path):
# Create DomainTools instance.
dt = DomainTools(utility)
with codecs.open(org_list_path, 'r', 'utf-8') as fin:
targets = fin.readlines()
for target in targets:
items = target.replace('\r', '').replace('\n',
'').split('\t')
if len(items) != 2:
utility.print_message(
FAIL,
'Invalid inventory target : {}'.format(target))
continue
search_word = items[0]
search_type = items[1]
# Check arguments.
if search_type not in ['Organization', 'Email', 'NS']:
utility.print_message(
FAIL,
'Invalid search type : {}'.format(search_type))
continue
# Search domain.
domain_info_dict_tmp = inventory.domain_explore(
dt, search_word=search_word, search_type=search_type)
class GeneticAlgorithm:
def __init__(self, template, browser):
self.util = Utilty()
self.template = template
self.obj_browser = browser
# Read config.ini.
full_path = os.path.dirname(os.path.abspath(__file__))
config = configparser.ConfigParser()
try:
config.read(self.util.join_path(full_path, 'config.ini'))
except FileExistsError as e:
self.util.print_message(FAIL, 'File exists error: {}'.format(e))
sys.exit(1)
# Common setting value.
self.wait_time = float(config['Common']['wait_time'])
self.html_dir = self.util.join_path(full_path,
config['Common']['html_dir'])
self.html_template = config['Common']['html_template']
self.html_template_path = self.util.join_path(self.html_dir,
self.html_template)
self.html_file = config['Common']['ga_html_file']
self.result_dir = self.util.join_path(full_path,
config['Common']['result_dir'])
# Genetic Algorithm setting value.
self.genom_length = int(config['Genetic']['genom_length'])
self.max_genom_list = int(config['Genetic']['max_genom_list'])
self.select_genom = int(config['Genetic']['select_genom'])
self.individual_mutation_rate = float(
config['Genetic']['individual_mutation_rate'])
self.genom_mutation_rate = float(
config['Genetic']['genom_mutation_rate'])
self.max_generation = int(config['Genetic']['max_generation'])
self.max_fitness = int(config['Genetic']['max_fitness'])
self.gene_dir = self.util.join_path(full_path,
config['Genetic']['gene_dir'])
self.genes_path = self.util.join_path(self.gene_dir,
config['Genetic']['gene_file'])
html_checker_dir = self.util.join_path(
full_path, config['Genetic']['html_checker_dir'])
self.html_checker = self.util.join_path(
html_checker_dir, config['Genetic']['html_checker_file'])
self.html_checker_option = config['Genetic']['html_checker_option']
self.html_checked_path = self.util.join_path(
self.html_dir, config['Genetic']['html_checked_file'])
self.html_eval_place_list = config['Genetic']['html_eval_place'].split(
'@')
self.bingo_score = float(config['Genetic']['bingo_score'])
self.warning_score = float(config['Genetic']['warning_score'])
self.error_score = float(config['Genetic']['error_score'])
self.result_file = config['Genetic']['result_file']
self.result_list = []
# Create population.
def create_genom(self, df_gene):
lst_gene = []
for _ in range(self.genom_length):
lst_gene.append(random.randint(0, len(df_gene.index) - 1))
self.util.print_message(OK,
'Created individual : {}.'.format(lst_gene))
return Gene(lst_gene, 0)
# Evaluation.
def evaluation(self, obj_ga, df_gene, eval_place, individual_idx):
# Build html syntax.
indivisual = self.util.transform_gene_num2str(df_gene,
obj_ga.genom_list)
html = self.template.render({eval_place: indivisual})
eval_html_path = self.util.join_path(
self.html_dir, self.html_file.replace('*', str(individual_idx)))
with codecs.open(eval_html_path, 'w', encoding='utf-8') as fout:
fout.write(html)
# Evaluate html syntax using tidy.
command = self.html_checker + ' ' + self.html_checker_option + ' ' + \
self.html_checked_path + ' ' + eval_html_path
enc = locale.getpreferredencoding()
env_tmp = os.environ.copy()
env_tmp['PYTHONIOENCODING'] = enc
subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env_tmp)
# Check html checked result.
str_eval_result = ''
with codecs.open(self.html_checked_path, 'r', encoding='utf-8') as fin:
str_eval_result = fin.read()
# Check warning and error number.
str_pattern = r'.*Tidy found ([0-9]+) warnings and ([0-9]+) errors.*$'
obj_match = re.match(
str_pattern,
str_eval_result.replace('\t', '').replace('\r',
'').replace('\n', ''))
warnings = 0.0
errors = 0.0
if obj_match:
warnings = int(obj_match.group(1)) * -0.1
errors = int(obj_match.group(2)) * -1.0
else:
return None, 1
# Compute score.
int_score = warnings + errors
# Evaluate running script using selenium.
selenium_score, error_flag = self.util.check_individual_selenium(
self.obj_browser, eval_html_path)
if error_flag:
return None, 1
# Check result of selenium.
if selenium_score > 0:
self.util.print_message(
OK, 'Detect running script: "{}" in {}.'.format(
indivisual, eval_place))
# compute score for running script.
int_score += self.bingo_score
self.result_list.append(
[eval_place, obj_ga.genom_list, indivisual])
# Output evaluation results.
self.util.print_message(
OK, 'Evaluation result : Browser={} {}, '
'Individual="{} ({})", '
'Score={}'.format(self.obj_browser.name,
self.obj_browser.capabilities['version'],
indivisual, obj_ga.genom_list,
str(int_score)))
return int_score, 0
# Select elite individual.
def select(self, obj_ga, elite):
# Sort in desc order of evaluation.
sort_result = sorted(obj_ga, reverse=True, key=lambda u: u.evaluation)
# Extract elite individuals.
return [sort_result.pop(0) for _ in range(elite)]
# Crossover (create offspring).
def crossover(self, ga_first, ga_second):
genom_list = []
# Setting of two-point crossover.
cross_first = random.randint(0, self.genom_length)
cross_second = random.randint(cross_first, self.genom_length)
one = ga_first.getGenom()
second = ga_second.getGenom()
# Crossover.
progeny_one = one[:cross_first] + second[
cross_first:cross_second] + one[cross_second:]
progeny_second = second[:cross_first] + one[
cross_first:cross_second] + second[cross_second:]
genom_list.append(Gene(progeny_one, 0))
genom_list.append(Gene(progeny_second, 0))
return genom_list
# Create population of next generation.
def next_generation_gene_create(self, ga, ga_elite, ga_progeny):
# Sort in asc order of evaluation.
next_generation_geno = sorted(ga,
reverse=False,
key=lambda u: u.evaluation)
# Remove sum of adding the elite group and offspring group.
for _ in range(0, len(ga_elite) + len(ga_progeny)):
next_generation_geno.pop(0)
# Add the elite group and offspring group to the next generation.
next_generation_geno.extend(ga_elite)
next_generation_geno.extend(ga_progeny)
return next_generation_geno
# Mutation.
def mutation(self, obj_ga, induvidual_mutation, genom_mutation, df_genes):
lst_ga = []
for idx in obj_ga:
# Mutation to individuals.
if induvidual_mutation > (random.randint(0, 100) / Decimal(100)):
lst_gene = []
for idx2 in idx.getGenom():
# Mutation to genes.
if genom_mutation > (random.randint(0, 100) /
Decimal(100)):
lst_gene.append(
random.randint(0,
len(df_genes.index) - 1))
else:
lst_gene.append(idx2)
idx.setGenom(lst_gene)
lst_ga.append(idx)
else:
lst_ga.append(idx)
return lst_ga
# Main control.
def main(self):
# Load gene list.
df_genes = pd.read_csv(self.genes_path, encoding='utf-8').fillna('')
# Create saving file (only header).
save_path = self.util.join_path(
self.result_dir,
self.result_file.replace('*', self.obj_browser.name))
if os.path.exists(save_path) is False:
pd.DataFrame([],
columns=['eval_place', 'sig_vector',
'sig_string']).to_csv(save_path,
mode='w',
header=True,
index=False)
# Evaluate indivisual each evaluating place in html.
for eval_place in self.html_eval_place_list:
self.util.print_message(
NOTE, 'Evaluating html place : {}'.format(eval_place))
# Generate 1st generation.
self.util.print_message(NOTE, 'Create population.')
current_generation = []
for _ in range(self.max_genom_list):
current_generation.append(self.create_genom(df_genes))
# Evaluate each generation.
for int_count in range(1, self.max_generation + 1):
self.util.print_message(
NOTE, 'Evaluate individual : {}/{} generation.'.format(
str(int_count), self.max_generation))
for indivisual, idx in enumerate(range(self.max_genom_list)):
self.util.print_message(
OK, 'Evaluation individual in {}: '
'{}/{} in {} generation'.format(
eval_place, indivisual + 1, self.max_genom_list,
str(int_count)))
evaluation_result, eval_status = self.evaluation(
current_generation[indivisual], df_genes, eval_place,
idx)
idx += 1
if eval_status == 1:
indivisual -= 1
continue
current_generation[indivisual].setEvaluation(
evaluation_result)
time.sleep(self.wait_time)
# Select elite's individual.
elite_genes = self.select(current_generation,
self.select_genom)
# Crossover of elite gene.
progeny_gene = []
for i in range(0, self.select_genom):
progeny_gene.extend(
self.crossover(elite_genes[i - 1], elite_genes[i]))
# Select elite group.
next_generation_individual_group = self.next_generation_gene_create(
current_generation, elite_genes, progeny_gene)
# Mutation
next_generation_individual_group = self.mutation(
next_generation_individual_group,
self.individual_mutation_rate, self.genom_mutation_rate,
df_genes)
# Finish evolution computing for current generation.
# Arrange fitness each individual.
fits = [_.getEvaluation() for _ in current_generation]
# evaluate evolution result.
flt_avg = sum(fits) / float(len(fits))
self.util.print_message(
NOTE, '{} generation result: '
'Min={}, Max={}, Avg={}.'.format(int_count, min(fits),
max(fits), flt_avg))
# Judge fitness.
if flt_avg > self.max_fitness:
self.util.print_message(
NOTE,
'Finish evolution: average={}'.format(str(flt_avg)))
break
# Replace current generation and next generation.
current_generation = next_generation_individual_group
# Save individuals.
pd.DataFrame(self.result_list).to_csv(save_path,
mode='a',
header=True,
index=False)
# Output final result.
str_best_individual = ''
for gene_num in elite_genes[0].getGenom():
str_best_individual += str(df_genes.loc[gene_num].values[0])
str_best_individual = str_best_individual.replace('%s', ' ').replace(
'"', '"').replace('%comma', ',')
self.util.print_message(
NOTE, 'Best individual : "{}"'.format(str_best_individual))
self.util.print_message(
NOTE, 'Done creation of injection codes using Genetic Algorithm.')
return self.result_list
default='en',
choices=['en', 'ja'],
type=str,
help='Specify language of report.')
args = parser.parse_args()
print(args)
# Read config.ini.
config = configparser.ConfigParser()
config.read(os.path.join(full_path, 'config.ini'))
# Load dataset.
ret_status, x_test_path, y_test_path, X_test, y_test = utility.load_dataset(
args.test_data_name, args.test_label_name, args.use_x_test_num)
if ret_status is False:
utility.print_message(
FAIL, 'Could not load dataset: {}'.format(args.op_type))
utility.write_log(
20, '[Out] Adversarial Threat Detector [{}].'.format(file_name))
sys.exit(0)
# Load target model.
ret_status, model_path, model = utility.load_model(args.model_name)
if ret_status is False:
utility.write_log(
20, '[Out] Adversarial Threat Detector [{}].'.format(file_name))
sys.exit(0)
# Preparation.
ret_status, classifier = utility.wrap_classifier(model, X_test)
if ret_status is False:
utility.write_log(
if __name__ == '__main__':
utility = Utilty()
show_banner(utility)
# Get target information.
ip_list, port_list, path_list = get_target_info(utility)
# Check parameters.
product_list = []
full_path = os.path.dirname(os.path.abspath(__file__))
for idx in range(len(ip_list)):
if check_arg_value(ip_list[idx], port_list[idx], path_list[idx],
utility) is False:
utility.print_message(
FAIL,
'Invalid parameter: {}, {}, {}'.format(ip_list[idx],
port_list[idx],
path_list[idx]))
# Start Spider.
scheme = ['http', 'https']
web_target_info = utility.run_spider(scheme, ip_list[idx],
port_list[idx], path_list[idx])
# Get HTTP responses.
log_file = os.path.join(
full_path + '/gyoithon/',
'get_' + ip_list[idx] + '_' + str(port_list[idx]) + '_ip.log')
create_webconf(ip_list[idx], port_list[idx], log_file)
for target in web_target_info:
for target_url in target[2]:
class GAN:
def __init__(self, template, browser):
self.util = Utilty()
self.template = template
self.obj_browser = browser
# Read config.ini.
full_path = os.path.dirname(os.path.abspath(__file__))
config = configparser.ConfigParser()
try:
config.read(self.util.join_path(full_path, 'config.ini'))
except FileExistsError as e:
self.util.print_message(FAIL, 'File exists error: {}'.format(e))
sys.exit(1)
# Common setting value.
self.wait_time = float(config['Common']['wait_time'])
self.html_dir = self.util.join_path(full_path,
config['Common']['html_dir'])
self.html_file = config['Common']['gan_html_file']
self.result_dir = self.util.join_path(full_path,
config['Common']['result_dir'])
self.eval_html_path = self.util.join_path(self.html_dir,
self.html_file)
# Genetic Algorithm setting value.
self.genom_length = int(config['Genetic']['genom_length'])
self.gene_dir = self.util.join_path(full_path,
config['Genetic']['gene_dir'])
self.genes_path = self.util.join_path(self.gene_dir,
config['Genetic']['gene_file'])
self.ga_result_file = config['Genetic']['result_file']
self.eval_place_list = config['Genetic']['html_eval_place'].split('@')
# Generative Adversarial Network setting value.
self.input_size = int(config['GAN']['input_size'])
self.batch_size = int(config['GAN']['batch_size'])
self.num_epoch = int(config['GAN']['num_epoch'])
self.max_sig_num = int(config['GAN']['max_sig_num'])
self.max_explore_codes_num = int(
config['GAN']['max_explore_codes_num'])
self.max_synthetic_num = int(config['GAN']['max_synthetic_num'])
self.weight_dir = self.util.join_path(full_path,
config['GAN']['weight_dir'])
self.gen_weight_file = config['GAN']['generator_weight_file']
self.dis_weight_file = config['GAN']['discriminator_weight_file']
self.gan_result_file = config['GAN']['result_file']
self.gan_vec_result_file = config['GAN']['vec_result_file']
self.generator = None
# Load gene list.
self.df_genes = pd.read_csv(self.genes_path,
encoding='utf-8').fillna('')
self.flt_size = len(self.df_genes) / 2.0
# Path of trained weight.
self.weight_path = self.util.join_path(
self.weight_dir,
self.gen_weight_file.replace('*', str(self.num_epoch - 1)))
# Build generator model.
def generator_model(self):
model = Sequential()
model.add(
Dense(input_dim=self.input_size,
output_dim=self.input_size * 10,
init='glorot_uniform'))
model.add(LeakyReLU(0.2))
model.add(Dropout(0.5))
model.add(Dense(self.input_size * 10, init='glorot_uniform'))
model.add(LeakyReLU(0.2))
model.add(Dropout(0.5))
model.add(Dense(self.input_size * 5, init='glorot_uniform'))
model.add(LeakyReLU(0.2))
model.add(Dropout(0.5))
model.add(Dense(output_dim=self.genom_length, init='glorot_uniform'))
model.add(Activation('tanh'))
return model
# Build discriminator model.
def discriminator_model(self):
model = Sequential()
model.add(
Dense(input_dim=self.genom_length,
output_dim=self.genom_length * 10,
init='glorot_uniform'))
model.add(LeakyReLU(0.2))
model.add(Dense(self.genom_length * 10, init='glorot_uniform'))
model.add(LeakyReLU(0.2))
model.add(Dense(1, init='glorot_uniform'))
model.add(Activation('sigmoid'))
return model
# Train GAN model (generate injection codes).
def train(self, list_sigs):
# Load train data (=ga result).
X_train = []
X_train = np.array(list_sigs)
X_train = (X_train.astype(np.float32) - self.flt_size) / self.flt_size
# Build discriminator.
discriminator = self.discriminator_model()
d_opt = SGD(lr=0.1, momentum=0.1, decay=1e-5)
discriminator.compile(loss='binary_crossentropy', optimizer=d_opt)
# Build generator and discriminator (fixed weight of discriminator).
discriminator.trainable = False
self.generator = self.generator_model()
dcgan = Sequential([self.generator, discriminator])
g_opt = SGD(lr=0.1, momentum=0.3)
dcgan.compile(loss='binary_crossentropy', optimizer=g_opt)
# Execute train.
num_batches = int(len(X_train) / self.batch_size)
lst_scripts = []
for epoch in range(self.num_epoch):
for batch in range(num_batches):
# Create noise for inputting to generator.
noise = np.array([
np.random.uniform(-1, 1, self.input_size)
for _ in range(self.batch_size)
])
# Generate new injection code using noise.
generated_codes = self.generator.predict(noise, verbose=0)
# Update weight of discriminator.
image_batch = X_train[batch * self.batch_size:(batch + 1) *
self.batch_size]
X = image_batch
y = [random.uniform(0.7, 1.2) for _ in range(self.batch_size)]
d_loss = discriminator.train_on_batch(X, y)
X = generated_codes
y = [random.uniform(0.0, 0.3) for _ in range(self.batch_size)]
d_loss = discriminator.train_on_batch(X, y)
# Update weight of generator.
noise = np.array([
np.random.uniform(-1, 1, self.input_size)
for _ in range(self.batch_size)
])
g_loss = dcgan.train_on_batch(noise, [1] * self.batch_size)
# Build HTML syntax from generated codes.
for generated_code in generated_codes:
lst_genom = []
for gene_num in generated_code:
gene_num = (gene_num * self.flt_size) + self.flt_size
gene_num = int(np.round(gene_num))
if gene_num == len(self.df_genes):
gene_num -= 1
lst_genom.append(int(gene_num))
str_html = self.util.transform_gene_num2str(
self.df_genes, lst_genom)
self.util.print_message(
OK,
'Train GAN : epoch={}, batch={}, g_loss={}, d_loss={}, {} ({})'
.format(
epoch, batch, g_loss, d_loss,
np.round((generated_code * self.flt_size) +
self.flt_size), str_html))
# Evaluate generated injection code.
for eval_place in self.eval_place_list:
# Build html syntax.
html = self.template.render({eval_place: str_html})
with codecs.open(self.eval_html_path,
'w',
encoding='utf-8') as fout:
fout.write(html)
# Evaluate individual using selenium.
selenium_score, error_flag = self.util.check_individual_selenium(
self.obj_browser, self.eval_html_path)
if error_flag:
continue
# Check generated individual using selenium.
if selenium_score > 0:
self.util.print_message(
WARNING,
'Detect running script: "{}" in {}.'.format(
str_html, eval_place))
# Save running script.
lst_scripts.append([eval_place, str_html])
# Save weights of network each epoch.
self.generator.save_weights(
self.util.join_path(
self.weight_dir,
self.gen_weight_file.replace('*', str(epoch))))
discriminator.save_weights(
self.util.join_path(
self.weight_dir,
self.dis_weight_file.replace('*', str(epoch))))
return lst_scripts
# Transform from generated codes to gene list.
def transform_code2gene(self, generated_code):
lst_genom = []
for gene_num in generated_code:
gene_num = (gene_num * self.flt_size) + self.flt_size
gene_num = int(np.round(gene_num))
if gene_num == len(self.df_genes):
gene_num -= 1
lst_genom.append(int(gene_num))
return lst_genom
# Mean of two vectors.
def vector_mean(self, vector1, vector2):
return (vector1 + vector2) / 2
# Main control.
def main(self):
# Define saving path.
gan_save_path = self.util.join_path(
self.result_dir,
self.gan_result_file.replace('*', self.obj_browser.name))
vec_save_path = self.util.join_path(
self.result_dir,
self.gan_vec_result_file.replace('*', self.obj_browser.name))
# Start generating injection code.
if os.path.exists(self.weight_path):
# Load trained model.
self.generator = self.generator_model()
self.generator.load_weights('{}'.format(self.weight_path))
# Explore the valid injection codes.
valid_code_list = []
result_list = []
for idx in range(self.max_explore_codes_num):
self.util.print_message(
NOTE, '{}/{} Explore valid injection code.'.format(
idx + 1, self.max_explore_codes_num))
# Generate injection codes.
noise = np.array([
np.random.uniform(-1, 1, self.input_size) for _ in range(1)
])
generated_codes = self.generator.predict(noise, verbose=0)
str_html = self.util.transform_gene_num2str(
self.df_genes,
self.transform_code2gene(generated_codes[0]))
# Evaluate injection code using selenium.
for eval_place in self.eval_place_list:
html = self.template.render({eval_place: str_html})
with codecs.open(self.eval_html_path,
'w',
encoding='utf-8') as fout:
fout.write(html)
selenium_score, error_flag = self.util.check_individual_selenium(
self.obj_browser, self.eval_html_path)
if error_flag:
continue
# Check generated injection code.
if selenium_score > 0:
self.util.print_message(
WARNING,
'Find valid injection code: "{}" in {}.'.format(
str_html, eval_place))
valid_code_list.append([str_html, noise])
result_list.append([eval_place, str_html])
# Save generated injection codes.
if os.path.exists(gan_save_path) is False:
pd.DataFrame(result_list,
columns=['eval_place',
'injection_code']).to_csv(gan_save_path,
mode='w',
header=True,
index=False)
else:
pd.DataFrame(result_list).to_csv(gan_save_path,
mode='a',
header=False,
index=False)
# Synthesize injection codes.
vector_result_list = []
for idx in range(self.max_synthetic_num):
noise_idx1 = np.random.randint(0, len(valid_code_list))
noise_idx2 = np.random.randint(0, len(valid_code_list))
self.util.print_message(
NOTE, '{}/{} Synthesize injection codes.'.format(
idx + 1, self.max_synthetic_num))
self.util.print_message(
OK, 'Use two injection codes : ({}) + ({}).'.format(
valid_code_list[noise_idx1][0],
valid_code_list[noise_idx2][0]))
# Generate injection codes.
synthesized_noise = self.vector_mean(
valid_code_list[noise_idx1][1],
valid_code_list[noise_idx2][1])
generated_codes = self.generator.predict(synthesized_noise,
verbose=0)
str_html = self.util.transform_gene_num2str(
self.df_genes,
self.transform_code2gene(generated_codes[0]))
# Evaluate synthesized injection code using selenium.
for eval_place in self.eval_place_list:
hit_flag = 'Failure'
html = self.template.render({eval_place: str_html})
with codecs.open(self.eval_html_path,
'w',
encoding='utf-8') as fout:
fout.write(html)
selenium_score, error_flag = self.util.check_individual_selenium(
self.obj_browser, self.eval_html_path)
if error_flag:
continue
# Check synthesized injection code using selenium.
if selenium_score > 0:
self.util.print_message(
WARNING,
'Find running script: "{}".'.format(str_html))
hit_flag = 'Bingo'
# Save running script.
vector_result_list.append([
eval_place, str_html, valid_code_list[noise_idx1][0],
valid_code_list[noise_idx2][0], hit_flag
])
# Save synthesized injection codes.
if os.path.exists(vec_save_path) is False:
pd.DataFrame(vector_result_list,
columns=[
'eval_place', 'synthesized_code',
'origin_code1', 'origin_code2', 'bingo'
]).to_csv(vec_save_path,
mode='w',
header=True,
index=False)
else:
pd.DataFrame(vector_result_list).to_csv(vec_save_path,
mode='a',
header=False,
index=False)
else:
# Load created individuals by Genetic Algorithm.
sig_path = self.util.join_path(
self.result_dir,
self.ga_result_file.replace('*', self.obj_browser.name))
df_temp = pd.read_csv(sig_path, encoding='utf-8').fillna('')
df_sigs = df_temp[~df_temp.duplicated()]
list_sigs = []
# Extract genom list from ga result.
for idx in range(len(df_sigs)):
list_temp = df_sigs['sig_vector'].values[idx].replace(
'[', '').replace(']', '').split(',')
list_sigs.append([int(s) for s in list_temp])
# Generate individuals (=injection codes).
lst_scripts = []
target_sig_list = []
for target_sig in list_sigs:
self.util.print_message(
NOTE, 'Start generating injection codes using {}'.format(
target_sig))
target_sig_list.extend(
[target_sig for _ in range(self.max_sig_num)])
lst_scripts.extend(self.train(target_sig_list))
# Save generated injection codes.
if os.path.exists(gan_save_path) is False:
pd.DataFrame(lst_scripts,
columns=['eval_place',
'injection_code']).to_csv(gan_save_path,
mode='w',
header=True,
index=False)
else:
pd.DataFrame(lst_scripts).to_csv(gan_save_path,
mode='a',
header=False,
index=False)
self.util.print_message(
NOTE,
'Done generation of injection codes using Generative Adversarial Networks.'
)
full_path, utility)
# Start investigation.
for idx in range(len(fqdn_list)):
# Check parameters.
msg = 'investigation : {}, {}, {}, {}'.format(protocol_list[idx],
fqdn_list[idx],
port_list[idx],
path_list[idx])
utility.write_log(20, 'Start ' + msg)
if utility.check_arg_value(protocol_list[idx], fqdn_list[idx],
port_list[idx], path_list[idx]) is False:
msg = 'Invalid parameter : {}, {}, {}, {}'.format(
protocol_list[idx], fqdn_list[idx], port_list[idx],
path_list[idx])
utility.print_message(FAIL, msg)
utility.write_log(30, msg)
continue
# Create report header.
report.create_report_header(fqdn_list[idx], port_list[idx])
# Check encoding.
test_url = ''
if int(port_list[idx]) in [80, 443]:
test_url = protocol_list[idx] + '://' + fqdn_list[idx] + path_list[
idx]
else:
test_url = protocol_list[idx] + '://' + fqdn_list[
idx] + ':' + port_list[idx] + path_list[idx]
_, server_header, res_header, res_body, encoding = utility.send_request(
full_path, utility)
# Start investigation.
for idx in range(len(fqdn_list)):
# Check parameters.
msg = 'investigation : {}, {}, {}, {}'.format(protocol_list[idx],
fqdn_list[idx],
port_list[idx],
path_list[idx])
utility.write_log(20, 'Start ' + msg)
if utility.check_arg_value(protocol_list[idx], fqdn_list[idx],
port_list[idx], path_list[idx]) is False:
msg = 'Invalid parameter : {}, {}, {}, {}'.format(
protocol_list[idx], fqdn_list[idx], port_list[idx],
path_list[idx])
utility.print_message(FAIL, msg)
utility.write_log(30, msg)
continue
# Create report header.
report.create_report_header(fqdn_list[idx], port_list[idx],
path_list[idx].replace('/', ''))
# Check cloud service.
cloud_type = 'Unknown'
if opt_cloud:
cloud_type = cloud_checker.get_cloud_service(fqdn_list[idx])
# Search Censys.
if opt_censys:
date = utility.get_current_date('%Y%m%d%H%M%S%f')[:-3]
# Load existing Keras model.
keras.backend.set_learning_phase(0)
keras_model = adv.prepare_test()
# Exchange Keras model to foolbox model.
fool_model = foolbox.models.KerasModel(keras_model, bounds=(0, 255))
# Generate adversarial examples.
target_list = os.listdir(adv.origin_image_path)
for idx1, target_origin_image in enumerate(target_list):
# Extract label of target image.
label = int(target_origin_image.split('.')[0])
# Load target image.
utility.print_message(
OK, '{}/{} Load original image: {} = {}'.format(
label + 1, len(target_list), target_origin_image,
adv.classes[label]))
origin_image = image.img_to_array(
image.load_img(os.path.join(adv.origin_image_path,
target_origin_image),
target_size=(adv.pixel_size, adv.pixel_size)))
# Specify the target label.
for idx2, target_class in enumerate(reversed(range(adv.nb_classes))):
# Indicate target label.
criterion = TargetClassProbability(label, p=0.9)
attack = foolbox.attacks.LBFGSAttack(model=fool_model,
criterion=criterion)
utility.print_message(
OK,
'Run the attack: target={}.{}'.format(label,
full_path, utility)
# Start investigation.
for idx in range(len(fqdn_list)):
# Check parameters.
msg = 'investigation : {}, {}, {}, {}'.format(protocol_list[idx],
fqdn_list[idx],
port_list[idx],
path_list[idx])
utility.write_log(20, 'Start ' + msg)
if utility.check_arg_value(protocol_list[idx], fqdn_list[idx],
port_list[idx], path_list[idx]) is False:
msg = 'Invalid parameter : {}, {}, {}, {}'.format(
protocol_list[idx], fqdn_list[idx], port_list[idx],
path_list[idx])
utility.print_message(FAIL, msg)
utility.write_log(30, msg)
continue
# Create report header.
report.create_report_header(fqdn_list[idx])
# Check cloud service.
cloud_type = cloud_checker.get_cloud_service(fqdn_list[idx])
# Gather target url using Spider.
web_target_info = spider.run_spider(protocol_list[idx], fqdn_list[idx],
port_list[idx], path_list[idx])
# Get HTTP responses.
for target in web_target_info:
class CreateReport:
def __init__(self):
self.util = Utilty()
# Read config file.
full_path = os.path.dirname(os.path.abspath(__file__))
config = configparser.ConfigParser()
try:
config.read(os.path.join(full_path, 'config.ini'))
except Exception as err:
self.util.print_exception(err, 'File exists error')
sys.exit(1)
self.report_path = os.path.join(full_path,
config['Report']['report_path'])
self.report_file = os.path.join(self.report_path,
config['Report']['report_file'])
self.template_file = config['Report']['template_file']
self.header = str(config['Report']['header']).split('@')
def create_report(self):
self.util.print_message(NOTE, 'Creating report.')
# Gather reporting items.
csv_file_list = glob.glob(os.path.join(self.report_path, '*.csv'))
# Create DataFrame.
content_list = []
for file in csv_file_list:
content_list.append(pd.read_csv(file, names=self.header, sep=','))
df_csv = pd.concat(content_list).drop_duplicates().sort_values(
by=['ip', 'port'], ascending=True).reset_index(drop=True,
col_level=1)
items = []
for idx in range(len(df_csv)):
items.append({
'ip_addr':
df_csv.loc[idx, 'ip'],
'port':
df_csv.loc[idx, 'port'],
'prod_name':
df_csv.loc[idx, 'service'],
'vuln_name':
df_csv.loc[idx, 'vuln_name'],
'type':
df_csv.loc[idx, 'type'],
'description':
df_csv.loc[idx, 'description'],
'exploit':
df_csv.loc[idx, 'exploit'],
'target':
df_csv.loc[idx, 'target'],
'payload':
df_csv.loc[idx, 'payload'],
'ref':
str(df_csv.loc[idx, 'reference']).replace('@', '<br>')
})
try:
# Setting template.
env = Environment(loader=FileSystemLoader(self.report_path))
template = env.get_template(self.template_file)
pd.set_option('display.max_colwidth', -1)
html = template.render({
'title': 'Deep Exploit Scan Report',
'items': items
})
# Write report.
with codecs.open(self.report_file, 'w', 'utf-8') as fout:
fout.write(html)
except Exception as err:
self.util.print_exception(err, 'Creating report error.')
self.util.print_message(OK, 'Created report.')
if __name__ == "__main__":
file_name = os.path.basename(__file__)
full_path = os.path.dirname(os.path.abspath(__file__))
utility = Utilty()
# Read config.ini.
config = configparser.ConfigParser()
config.read(os.path.join(full_path, 'config.ini'))
# Show banner.
show_banner(utility)
if len(sys.argv) == 1:
utility.print_message(FAIL, 'Invalid parameter "{}"'.format(sys.argv))
exit(1)
if sys.argv[1] == 'TRAIN':
utility.print_message(NOTE, 'Start {} mode.'.format(sys.argv[1]))
obj_recommend = Recommend(utility)
obj_recommend.training_model()
utility.print_message(NOTE, 'End {} mode.'.format(sys.argv[1]))
elif sys.argv[1] == 'TEST' and len(sys.argv) == 3:
utility.print_message(NOTE, 'Start {} mode.'.format(sys.argv[1]))
target_info = sys.argv[2]
obj_parsed = urlparse(target_info)
if utility.check_arg_value(obj_parsed.scheme, obj_parsed.netloc, str(obj_parsed.port), obj_parsed.path) is False:
utility.print_message(FAIL, 'Invalid target: {}.'.format(target_info))
else:
# Generates feature vector
OK = 'ok' # [*]
NOTE = 'note' # [+]
FAIL = 'fail' # [-]
WARNING = 'warn' # [!]
NONE = 'none' # No label.
if __name__ == "__main__":
util = Utilty()
# Read config.ini.
full_path = os.path.dirname(os.path.abspath(__file__))
config = configparser.ConfigParser()
try:
config.read(util.join_path(full_path, 'config.ini'))
except FileExistsError as e:
util.print_message(FAIL, 'File exists error: {}'.format(e))
sys.exit(1)
# Common setting value.
html_dir = util.join_path(full_path, config['Common']['html_dir'])
html_template = config['Common']['html_template']
# Genetic Algorithm setting value.
html_eval_place_list = config['Genetic']['html_eval_place'].split('@')
max_try_num = int(config['Genetic']['max_try_num'])
# Selenium setting value.
driver_dir = util.join_path(full_path, config['Selenium']['driver_dir'])
driver_list = config['Selenium']['driver_list'].split('@')
window_width = int(config['Selenium']['window_width'])
window_height = int(config['Selenium']['window_height'])
class CreateReport:
def __init__(self):
self.util = Utilty()
# Read config file.
full_path = os.path.dirname(os.path.abspath(__file__))
config = configparser.ConfigParser()
try:
config.read(os.path.join(full_path, 'config.ini'))
except Exception as err:
self.util.print_exception(err, 'File exists error')
sys.exit(1)
self.report_date_format = config['Report']['date_format']
self.report_test_path = os.path.join(full_path,
config['Report']['report_test'])
self.report_test_file = os.path.join(
self.report_test_path, config['Report']['report_test_file'])
self.template_test = config['Report']['template_test']
self.report_train_path = os.path.join(self.report_test_path,
config['Report']['report_train'])
self.report_train_file = os.path.join(
self.report_train_path, config['Report']['report_train_file'])
self.template_train = config['Report']['template_train']
self.header_train = str(config['Report']['header_train']).split('@')
self.header_test = str(config['Report']['header_test']).split('@')
def create_report(self, mode='train', start_date=None):
# Check mode.
if mode not in ['train', 'test']:
self.util.print_message(FAIL, 'Invalid mode: {}'.format(mode))
exit(1)
# Gather reporting items.
if mode == 'train':
self.util.print_message(NOTE, 'Creating training report.')
csv_file_list = glob.glob(
os.path.join(self.report_train_path, '*.csv'))
# Create DataFrame.
content_list = []
for file in csv_file_list:
df = pd.read_csv(file, names=self.header_train, sep=',')
df['date'] = pd.to_datetime(df['date'])
selected_df = df[(start_date < df['date'])]
content_list.append(selected_df)
if len(content_list) != 0:
df_csv = pd.concat(content_list).drop_duplicates().sort_values(
by=['ip', 'port'], ascending=True).reset_index(drop=True,
col_level=1)
items = []
for idx in range(len(df_csv)):
items.append({
'ip_addr':
df_csv.loc[idx, 'ip'],
'port':
df_csv.loc[idx, 'port'],
'prod_name':
df_csv.loc[idx, 'service'],
'vuln_name':
df_csv.loc[idx, 'vuln_name'],
'description':
df_csv.loc[idx, 'description'],
'type':
df_csv.loc[idx, 'type'],
'exploit':
df_csv.loc[idx, 'exploit'],
'target':
df_csv.loc[idx, 'target'],
'payload':
df_csv.loc[idx, 'payload'],
'ref':
str(df_csv.loc[idx, 'reference']).replace('@', '<br>')
})
try:
# Setting template.
env = Environment(
loader=FileSystemLoader(self.report_train_path))
template = env.get_template(self.template_train)
pd.set_option('display.max_colwidth', -1)
html = template.render({
'title': 'Deep Exploit Scan Report',
'items': items
})
# Write report.
with codecs.open(self.report_train_file, 'w',
'utf-8') as fout:
fout.write(html)
except Exception as err:
self.util.print_exception(err, 'Creating report error.')
else:
self.util.print_message(WARNING,
'Exploitation result is not found.')
self.util.print_message(OK, 'Creating training report done.')
else:
self.util.print_message(NOTE, 'Creating testing report.')
csv_file_list = glob.glob(
os.path.join(self.report_test_path, '*.csv'))
# Create DataFrame.
content_list = []
for file in csv_file_list:
df = pd.read_csv(file, names=self.header_test, sep=',')
df['date'] = pd.to_datetime(df['date'])
selected_df = df[(start_date < df['date'])]
content_list.append(selected_df)
if len(content_list) != 0:
df_csv = pd.concat(content_list).drop_duplicates().sort_values(
by=['ip', 'port'], ascending=True).reset_index(drop=True,
col_level=1)
items = []
for idx in range(len(df_csv)):
items.append({
'ip_addr':
df_csv.loc[idx, 'ip'],
'port':
df_csv.loc[idx, 'port'],
'source_ip_addr':
df_csv.loc[idx, 'src_ip'],
'prod_name':
df_csv.loc[idx, 'service'],
'vuln_name':
df_csv.loc[idx, 'vuln_name'],
'description':
df_csv.loc[idx, 'description'],
'type':
df_csv.loc[idx, 'type'],
'exploit':
df_csv.loc[idx, 'exploit'],
'target':
df_csv.loc[idx, 'target'],
'payload':
df_csv.loc[idx, 'payload'],
'ref':
str(df_csv.loc[idx, 'reference']).replace('@', '<br>')
})
try:
# Setting template.
env = Environment(
loader=FileSystemLoader(self.report_test_path))
template = env.get_template(self.template_test)
pd.set_option('display.max_colwidth', -1)
html = template.render({
'title': 'Deep Exploit Scan Report',
'items': items
})
# Write report.
with codecs.open(self.report_test_file, 'w',
'utf-8') as fout:
fout.write(html)
except Exception as err:
self.util.print_exception(err, 'Creating report error.')
else:
self.util.print_message(WARNING,
'Exploitation result is not found.')
self.util.print_message(OK, 'Creating testing report done.')
class DeepClassifier:
def __init__(self):
# Read config.ini.
self.utility = Utilty()
config = configparser.ConfigParser()
self.full_path = os.path.dirname(os.path.abspath(__file__))
self.root_path = os.path.join(self.full_path, '../')
try:
config.read(os.path.join(self.full_path, 'config.ini'))
except FileExistsError as err:
self.utility.print_exception(err,
'File exists error: {0}'.format(err))
sys.exit(1)
self.category_type = config['Common']['category']
self.train_path = os.path.join(self.full_path,
config['GyoiClassifier']['train_path'])
self.trained_path = os.path.join(
self.full_path, config['GyoiClassifier']['trained_path'])
self.train_os_in = os.path.join(
self.train_path, config['GyoiClassifier']['train_os_in'])
self.train_os_out = os.path.join(
self.trained_path, config['GyoiClassifier']['train_os_out'])
self.train_web_in = os.path.join(
self.train_path, config['GyoiClassifier']['train_web_in'])
self.train_web_out = os.path.join(
self.trained_path, config['GyoiClassifier']['train_web_out'])
self.train_framework_in = os.path.join(
self.train_path, config['GyoiClassifier']['train_framework_in'])
self.train_framework_out = os.path.join(
self.trained_path, config['GyoiClassifier']['train_framework_out'])
self.train_cms_in = os.path.join(
self.train_path, config['GyoiClassifier']['train_cms_in'])
self.train_cms_out = os.path.join(
self.trained_path, config['GyoiClassifier']['train_cms_out'])
self.wait_for_banner = float(
config['GyoiClassifier']['wait_for_banner'])
self.maximum_display_num = int(
config['GyoiClassifier']['maximum_display_num'])
self.summary_path = os.path.join(self.full_path,
config['GyoiThon']['summary_path'])
self.summary_file = os.path.join(self.summary_path,
config['GyoiThon']['summary_file'])
return
# Analysis using ML.
def analyzer(self,
target_ip='',
target_port=0,
target_vhost='',
silent=False,
target_url='',
target_response=''):
self.utility.print_message(
NOTE, 'Analyzing gathered HTTP response using Machine Learning.')
identified_list = []
target_info = ''
target_log = ''
analyzing_text = ''
if target_response == '':
# Get GyoiThon's summary.
df_origin = pd.read_csv(self.summary_file,
encoding='utf-8').fillna('')
df_selected_summary = df_origin[
(df_origin['ip'] == target_ip)
& (df_origin['port'] == target_port) &
(df_origin['vhost'] == target_vhost)]
# Get log file (webconf.csv)
logfile_path = os.path.join(self.root_path,
df_selected_summary.at[0, 'log'])
fin = codecs.open(logfile_path, 'r', encoding='utf-8')
analyzing_text = fin.read()
fin.close()
target_info = target_vhost + '(' + target_ip + '):' + str(
target_port)
target_log = logfile_path
else:
target_info = target_url
target_log = 'not use'
analyzing_text = target_response
# Output result (header)
# If silent mode is True, hidden target information.
if silent is True:
self.utility.print_message(
WARNING, 'target host : *** hidden for silent mode. ***')
self.utility.print_message(
WARNING, 'target url : *** hidden for silent mode. ***')
self.utility.print_message(
WARNING, 'target log : *** hidden for silent mode. ***')
else:
self.utility.print_message(WARNING,
'target host : {}'.format(target_info))
self.utility.print_message(WARNING,
'target url : {}'.format(target_url))
self.utility.print_message(WARNING,
'target log : {}'.format(target_log))
self.utility.print_message(OK, 'judge :')
# Predict product name each category (OS, Middleware, CMS..).
list_category = self.category_type.split('@')
print('-' * 42)
for category in list_category:
# Learning.
if category == 'os':
nb = self.train(self.train_os_in, self.train_os_out)
elif category == 'web server':
nb = self.train(self.train_web_in, self.train_web_out)
elif category == 'framework':
nb = self.train(self.train_framework_in,
self.train_framework_out)
elif category == 'cms':
nb = self.train(self.train_cms_in, self.train_cms_out)
else:
self.utility.print_message(FAIL,
'Choose category is not found.')
exit(1)
# Predict product name.
product, prob, keyword_list, classified_list = nb.classify(
analyzing_text)
# Output result of prediction (body).
# If no feature, result is unknown.
if len(keyword_list) == 0:
self.utility.print_message(NOTE,
'category : {}'.format(category))
self.utility.print_message(WARNING, 'product : unknown')
self.utility.print_message(WARNING, 'too low probability.')
else:
sorted_classified_list = sorted(classified_list,
key=lambda x: x[1],
reverse=True)
self.utility.print_message(NOTE,
'category : {}'.format(category))
for idx, item in enumerate(sorted_classified_list):
add_flag = True
if idx >= self.maximum_display_num:
break
# Delete duplicated result.
reason_list = []
for reason in item[2]:
reason_list.append(list(set(reason)))
# # If no feature, reason is "too few features".
if len(item[2]) == 0:
reason_list = 'too few features..'
add_flag = False
self.utility.print_message(NOTE,
'ranking {}'.format(idx + 1))
self.utility.print_message(
OK, 'product : {}'.format(item[0]))
self.utility.print_message(
OK,
'probability : {}'.format(round(item[1] * 100.0, 4)))
self.utility.print_message(
OK, 'reason : {}'.format(reason_list))
# Add product for Exploit.
identified_list.append(item[0])
self.utility.print_message(NONE, '-' * 42)
# Output result of prediction (footer).
self.utility.print_message(
NOTE, 'done {}'.format(os.path.basename(__file__)))
return list(set(identified_list))
# Execute learning / Get learned data.
def train(self, in_file, out_file):
# If existing learned data (pkl), load learned data.
nb = None
if os.path.exists(out_file):
with open(out_file, 'rb') as f:
nb = pickle.load(f)
# If no learned data, execute learning.
else:
# Read learning data.
nb = NaiveBayes()
fin = codecs.open(in_file, 'r', 'utf-8')
lines = fin.readlines()
fin.close()
items = []
for line in lines:
words = line[:-2]
train_words = words.split('@')
items.append(train_words[1])
nb.train(train_words[1], train_words[0])
# Save learned data to pkl file.
with open(out_file, 'wb') as f:
pickle.dump(nb, f)
return nb
# Explore relevant FQDN with the target FQDN.
if opt_invent:
inventory_list_path = os.path.join(full_path, 'inventory_list.txt')
if os.path.exists(inventory_list_path):
inventory = Inventory(utility)
spider = SpiderControl(utility)
google_hack = GoogleCustomSearch(utility)
report = CreateReport(utility)
with codecs.open(inventory_list_path, 'r', 'utf-8') as fin:
targets = fin.readlines()
for target in targets:
items = target.replace('\r', '').replace('\n', '').split('\t')
if len(items) != 2:
utility.print_message(FAIL, 'Invalid inventory target : {}'.format(target))
continue
# Check target URL.
port_num = ''
invent_url = ''
keyword = ''
try:
invent_url = items[0]
keyword = items[1]
parsed = util.parse_url(invent_url)
# Judge port number.
if parsed.port is None and parsed.scheme == 'https':
port_num = '443'
elif parsed.port is None and parsed.scheme == 'http':
continue
elif image.shape[0] > preparation.pixel_size:
image = cv2.resize(
image, (preparation.pixel_size, preparation.pixel_size))
# Save image.
file_name = os.path.join(adversarial_result, 'tmp_face.jpg')
cv2.imwrite(file_name, image)
# Predict face.
results = recognition.execute_test(model, file_name)
prob = results[0][1] * 100
msg = '{}/{} {} ({:.1f}%).'.format(idx + 1, len(target_list),
results[0][0], prob)
utility.print_message(OK, msg + ': {}'.format(adversarial_example))
elif opt_record:
# Recording.
record.record(opt_rec_file_name)
else:
# Execute face recognition.
model = recognition.prepare_test()
capture = preparation.create_camera_instance()
for idx in range(max_retry):
# Read 1 frame from VideoCapture.
ret, image = capture.read()
# Execute detecting face.
gray_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
cascade = cv2.CascadeClassifier(preparation.haarcascade_path)
faces = cascade.detectMultiScale(gray_image,
class ClassifierSignature:
def __init__(self):
self.full_path = os.path.dirname(os.path.abspath(__file__))
self.signature_path = os.path.join(self.full_path, 'signature')
self.util = Utilty()
# Identify product name.
def identify_product(self, categoy, response):
prod_info_list = []
file_name = 'signature_' + categoy + '.txt'
try:
# Judge product using pattern matching.
with codecs.open(os.path.join(self.signature_path, file_name), 'r',
'utf-8') as fin:
matching_patterns = fin.readlines()
for pattern in matching_patterns:
items = pattern.replace('\r', '').replace('\n',
'').split('@')
product = items[0].lower()
signature = items[2]
list_match = re.findall(signature,
response,
flags=re.IGNORECASE)
if len(list_match) != 0:
# Check version.
version_info = ''
for target_string in list_match:
version_info = self.extract_version(
target_string).lower()
if version_info != '':
break
# Add product name and version.
if str(items[1]) != '':
prod_info_list.append(product + '@' +
str(items[1]))
elif version_info != '':
prod_info_list.append(product + '@' + version_info)
else:
prod_info_list.append(product + '@-')
except Exception as err:
self.util.print_message(WARNING, '{}'.format(err))
return prod_info_list
# Extract version.
def extract_version(self, target_string):
# Regression list for cutting version.
regex_list = [
r'(\d{1,3}\.\d{1,3}\.\d{1,3}).*', r'(\d{1,3}\.\d{1,3}).*',
r'(\d{1,3}).*', r'(\d{1,3}\.\d{1,3}[a-z]\d{1,3}).*',
r'(\d{1,3}\.\d{1,3}\.\d[a-z]{1,3}).*', r'(\d\.[xX|\*]).*'
]
version_info = ''
for regex_pattern in regex_list:
version_list = re.findall(regex_pattern, target_string)
if len(version_list) != 0:
version_info = str(version_list[0])
break
return version_info
# Classifier product name using signatures.
def classifier_signature(self, target_info, client):
product_list = []
for target in target_info:
for target_url in target[2]:
# Get HTTP response (header + body).
response = ''
http = urllib3.PoolManager(timeout=self.util.http_timeout)
try:
client.keep_alive()
self.util.print_message(
OK, '{} {}'.format(
self.util.get_current_date('%Y-%m-%d %H:%M:%S'),
target_url))
res = http.request('GET', target_url)
for header in res.headers.items():
response += header[0] + ': ' + header[1] + '\r\n'
response += '\r\n\r\n' + res.data.decode('utf-8')
except Exception as err:
self.util.print_message(WARNING, '{}'.format(err))
for category in ['os', 'web', 'framework', 'cms']:
prod_info = self.identify_product(
category, self.util.delete_ctrl_char(response))
for product in prod_info:
parsed = util.parse_url(target_url)
path_item = os.path.split(parsed.path)
if path_item[0].endswith('/') is False:
product_list.append(product + '@' +
str(parsed.port) + '@' +
path_item[0] + '/')
else:
product_list.append(product + '@' +
str(parsed.port) + '@' +
path_item[0])
time.sleep(1.0)
# Delete duplication.
uniq_product = []
tmp_list = []
for item in list(set(product_list)):
tmp_item = item.split('@')
tmp = tmp_item[0] + tmp_item[2]
if tmp not in tmp_list:
tmp_list.append(tmp)
uniq_product.append(item)
return uniq_product
ret_status, classifier = utility.wrap_classifier(model, X_test)
if ret_status is False:
utility.write_log(
20, '[Out] Adversarial Threat Detector [{}].'.format(file_name))
sys.exit(0)
# Accuracy on Benign Examples.
ret_status, acc_benign = utility.evaluate(classifier,
X_test=X_test,
y_test=y_test)
if ret_status is False:
utility.write_log(
20, '[Out] Adversarial Threat Detector [{}].'.format(file_name))
sys.exit(0)
else:
utility.print_message(
OK, 'Accuracy on Benign Examples : {}%'.format(acc_benign * 100))
report_util.template_target['accuracy'] = '{}%'.format(acc_benign *
100)
# Evaluate all attacks.
if args.attack_type == 'all':
utility.print_message(
WARNING, 'Not implementation: {}'.format(args.attack_type))
# Evaluate Data Poisoning Attacks.
elif args.attack_type == 'data_poisoning':
utility.print_message(
WARNING, 'Not implementation: {}'.format(args.attack_type))
# Evaluate Model Poisoning Attacks.
elif args.attack_type == 'model_poisoning':
utility.print_message(
WARNING, 'Not implementation: {}'.format(args.attack_type))