max_target_url = int(config['Common']['max_target_url'])
max_target_byte = int(config['Common']['max_target_byte'])
clipping_regex = config['Common']['clipping_regex']
clipping_size = int(config['Common']['clipping_size'])
if clipping_size <= 0:
clipping_size = 10000
clipping_buff = int(config['Common']['clipping_buff'])
if clipping_buff <= 0:
clipping_buff = 200
clipping_mark = config['Common']['clipping_mark']
if int(config['Common']['scramble']) == 1:
is_scramble = True
except Exception as e:
msg = 'Reading config.ini is failure : {}'.format(e)
utility.print_exception(e, msg)
utility.write_log(40, msg)
utility.write_log(20, '[Out] GyoiThon [{}].'.format(file_name))
exit(1)
# Show banner.
show_banner(utility)
# Create signature and train data.
if opt_develop:
creator = Creator(utility)
creator.extract_file_structure(opt_develop_category, opt_develop_vendor, opt_develop_package)
print(os.path.basename(__file__) + ' finish!!')
utility.write_log(20, '[Out] GyoiThon [{}].'.format(file_name))
sys.exit(0)
max_target_url = 0
max_target_byte = 0
is_scramble = False
try:
log_dir = config['Common']['log_path']
log_path = os.path.join(full_path, log_dir)
method_crawl = config['Common']['method_crawl']
method_log = config['Common']['method_log']
max_target_url = int(config['Common']['max_target_url'])
max_target_byte = int(config['Common']['max_target_byte'])
if int(config['Common']['scramble']) == 1:
is_scramble = True
except Exception as e:
msg = 'Reading config.ini is failure : {}'.format(e)
utility.print_exception(e, msg)
utility.write_log(40, msg)
utility.write_log(20, '[Out] GyoiThon [{}].'.format(file_name))
exit(1)
# Show banner.
show_banner(utility)
# Create instances.
cloud_checker = CloudChecker(utility)
version_checker = VersionChecker(utility)
version_checker_ml = VersionCheckerML(utility)
comment_checker = CommentChecker(utility)
error_checker = ErrorChecker(utility)
page_checker = PageChecker(utility)
google_hack = GoogleCustomSearch(utility)
class Msgrpc:
def __init__(self, option=[]):
self.utility = Utilty()
self.host = option.get('host') or "127.0.0.1"
self.port = option.get('port') or 55552
self.uri = option.get('uri') or "/api/"
self.ssl = option.get('ssl') or False
self.authenticated = False
self.token = False
self.headers = {"Content-type": "binary/message-pack"}
if self.ssl:
self.client = http.client.HTTPSConnection(self.host, self.port)
else:
self.client = http.client.HTTPConnection(self.host, self.port)
# Call RPC API.
def call(self, meth, option):
if meth != "auth.login":
if not self.authenticated:
self.utility.print_message(FAIL, 'MsfRPC: Not Authenticated')
exit(1)
if meth != "auth.login":
option.insert(0, self.token)
option.insert(0, meth)
params = msgpack.packb(option)
self.client.request("POST", self.uri, params, self.headers)
resp = self.client.getresponse()
return msgpack.unpackb(resp.read())
# Log in to RPC Server.
def login(self, user, password):
ret = self.call('auth.login', [user, password])
if ret.get(b'result') == b'success':
self.authenticated = True
self.token = ret.get(b'token')
return True
else:
self.utility.print_message(FAIL, 'MsfRPC: Not Authenticated')
exit(1)
# Send Metasploit command.
def send_command(self, console_id, command, visualization, sleep=0.1):
_ = self.call('console.write', [console_id, command])
time.sleep(sleep)
ret = self.call('console.read', [console_id])
if visualization:
try:
self.utility.print_message(
NONE, '{}'.format(ret.get(b'data').decode('utf-8')))
except Exception as e:
self.utility.print_exception(e, 'Send_command is exception.')
return ret
# Get all modules.
def get_module_list(self, module_type):
ret = {}
if module_type == 'exploit':
ret = self.call('module.exploits', [])
elif module_type == 'auxiliary':
ret = self.call('module.auxiliary', [])
elif module_type == 'post':
ret = self.call('module.post', [])
elif module_type == 'payload':
ret = self.call('module.payloads', [])
elif module_type == 'encoder':
ret = self.call('module.encoders', [])
elif module_type == 'nop':
ret = self.call('module.nops', [])
byte_list = ret[b'modules']
string_list = []
for module in byte_list:
string_list.append(module.decode('utf-8'))
return string_list
# Get module detail information.
def get_module_info(self, module_type, module_name):
return self.call('module.info', [module_type, module_name])
# Get payload that compatible module.
def get_compatible_payload_list(self, module_name):
ret = self.call('module.compatible_payloads', [module_name])
byte_list = ret[b'payloads']
string_list = []
for module in byte_list:
string_list.append(module.decode('utf-8'))
return string_list
# Get payload that compatible target.
def get_target_compatible_payload_list(self, module_name, target_num):
ret = self.call('module.target_compatible_payloads',
[module_name, target_num])
byte_list = ret[b'payloads']
string_list = []
for module in byte_list:
string_list.append(module.decode('utf-8'))
return string_list
# Get module options.
def get_module_options(self, module_type, module_name):
return self.call('module.options', [module_type, module_name])
# Execute module.
def execute_module(self, module_type, module_name, options):
ret = self.call('module.execute', [module_type, module_name, options])
job_id = ret[b'job_id']
uuid = ret[b'uuid'].decode('utf-8')
return job_id, uuid
# Get job list.
def get_job_list(self):
jobs = self.call('job.list', [])
byte_list = jobs.keys()
job_list = []
for job_id in byte_list:
job_list.append(int(job_id.decode('utf-8')))
return job_list
# Get job detail information.
def get_job_info(self, job_id):
return self.call('job.info', [job_id])
# Stop job.
def stop_job(self, job_id):
return self.call('job.stop', [job_id])
# Get session list.
def get_session_list(self):
return self.call('session.list', [])
# Stop shell session.
def stop_session(self, session_id):
_ = self.call('session.stop', [str(session_id)])
# Stop meterpreter session.
def stop_meterpreter_session_kill(self, session_id):
_ = self.call('session.meterpreter_session_kill', [str(session_id)])
# Log out from RPC Server.
def logout(self):
ret = self.call('auth.logout', [self.token])
if ret.get(b'result') == b'success':
self.authenticated = False
self.token = ''
return True
else:
self.utility.print_message(FAIL, 'MsfRPC: Not Authenticated')
exit(1)
# Disconnection.
def termination(self, console_id):
# Kill a console.
_ = self.call('console.session_kill', [console_id])
# Log out
_ = self.logout()
class CreateReport:
def __init__(self):
self.util = Utilty()
# Read config file.
full_path = os.path.dirname(os.path.abspath(__file__))
config = configparser.ConfigParser()
try:
config.read(os.path.join(full_path, 'config.ini'))
except Exception as err:
self.util.print_exception(err, 'File exists error')
sys.exit(1)
self.report_date_format = config['Report']['date_format']
self.report_test_path = os.path.join(full_path,
config['Report']['report_test'])
self.report_test_file = os.path.join(
self.report_test_path, config['Report']['report_test_file'])
self.template_test = config['Report']['template_test']
self.report_train_path = os.path.join(self.report_test_path,
config['Report']['report_train'])
self.report_train_file = os.path.join(
self.report_train_path, config['Report']['report_train_file'])
self.template_train = config['Report']['template_train']
self.header_train = str(config['Report']['header_train']).split('@')
self.header_test = str(config['Report']['header_test']).split('@')
def create_report(self, mode='train', start_date=None):
# Check mode.
if mode not in ['train', 'test']:
self.util.print_message(FAIL, 'Invalid mode: {}'.format(mode))
exit(1)
# Gather reporting items.
if mode == 'train':
self.util.print_message(NOTE, 'Creating training report.')
csv_file_list = glob.glob(
os.path.join(self.report_train_path, '*.csv'))
# Create DataFrame.
content_list = []
for file in csv_file_list:
df = pd.read_csv(file, names=self.header_train, sep=',')
df['date'] = pd.to_datetime(df['date'])
selected_df = df[(start_date < df['date'])]
content_list.append(selected_df)
if len(content_list) != 0:
df_csv = pd.concat(content_list).drop_duplicates().sort_values(
by=['ip', 'port'], ascending=True).reset_index(drop=True,
col_level=1)
items = []
for idx in range(len(df_csv)):
items.append({
'ip_addr':
df_csv.loc[idx, 'ip'],
'port':
df_csv.loc[idx, 'port'],
'prod_name':
df_csv.loc[idx, 'service'],
'vuln_name':
df_csv.loc[idx, 'vuln_name'],
'description':
df_csv.loc[idx, 'description'],
'type':
df_csv.loc[idx, 'type'],
'exploit':
df_csv.loc[idx, 'exploit'],
'target':
df_csv.loc[idx, 'target'],
'payload':
df_csv.loc[idx, 'payload'],
'ref':
str(df_csv.loc[idx, 'reference']).replace('@', '<br>')
})
try:
# Setting template.
env = Environment(
loader=FileSystemLoader(self.report_train_path))
template = env.get_template(self.template_train)
pd.set_option('display.max_colwidth', -1)
html = template.render({
'title': 'Deep Exploit Scan Report',
'items': items
})
# Write report.
with codecs.open(self.report_train_file, 'w',
'utf-8') as fout:
fout.write(html)
except Exception as err:
self.util.print_exception(err, 'Creating report error.')
else:
self.util.print_message(WARNING,
'Exploitation result is not found.')
self.util.print_message(OK, 'Creating training report done.')
else:
self.util.print_message(NOTE, 'Creating testing report.')
csv_file_list = glob.glob(
os.path.join(self.report_test_path, '*.csv'))
# Create DataFrame.
content_list = []
for file in csv_file_list:
df = pd.read_csv(file, names=self.header_test, sep=',')
df['date'] = pd.to_datetime(df['date'])
selected_df = df[(start_date < df['date'])]
content_list.append(selected_df)
if len(content_list) != 0:
df_csv = pd.concat(content_list).drop_duplicates().sort_values(
by=['ip', 'port'], ascending=True).reset_index(drop=True,
col_level=1)
items = []
for idx in range(len(df_csv)):
items.append({
'ip_addr':
df_csv.loc[idx, 'ip'],
'port':
df_csv.loc[idx, 'port'],
'source_ip_addr':
df_csv.loc[idx, 'src_ip'],
'prod_name':
df_csv.loc[idx, 'service'],
'vuln_name':
df_csv.loc[idx, 'vuln_name'],
'description':
df_csv.loc[idx, 'description'],
'type':
df_csv.loc[idx, 'type'],
'exploit':
df_csv.loc[idx, 'exploit'],
'target':
df_csv.loc[idx, 'target'],
'payload':
df_csv.loc[idx, 'payload'],
'ref':
str(df_csv.loc[idx, 'reference']).replace('@', '<br>')
})
try:
# Setting template.
env = Environment(
loader=FileSystemLoader(self.report_test_path))
template = env.get_template(self.template_test)
pd.set_option('display.max_colwidth', -1)
html = template.render({
'title': 'Deep Exploit Scan Report',
'items': items
})
# Write report.
with codecs.open(self.report_test_file, 'w',
'utf-8') as fout:
fout.write(html)
except Exception as err:
self.util.print_exception(err, 'Creating report error.')
else:
self.util.print_message(WARNING,
'Exploitation result is not found.')
self.util.print_message(OK, 'Creating testing report done.')
web_target_info = utility.run_spider(scheme, ip_list[idx],
port_list[idx], path_list[idx])
# Get HTTP responses.
log_file = os.path.join(
full_path + '/gyoithon/',
'get_' + ip_list[idx] + '_' + str(port_list[idx]) + '_ip.log')
create_webconf(ip_list[idx], port_list[idx], log_file)
for target in web_target_info:
for target_url in target[2]:
# Check target url.
parsed = None
try:
parsed = util.parse_url(target_url)
except Exception as err:
utility.print_exception(
err, 'Parsed error: {}'.format(target_url))
continue
# Get HTTP response (header + body).
response = ''
http = urllib3.PoolManager(timeout=utility.http_timeout)
try:
utility.print_message(
OK, '{} {}'.format(
utility.get_current_date('%Y-%m-%d %H:%M:%S'),
target_url))
res = http.request('GET', target_url)
for header in res.headers.items():
response += header[0] + ': ' + header[1] + '\r\n'
response += '\r\n\r\n' + res.data.decode('utf-8')
class DeepClassifier:
def __init__(self):
# Read config.ini.
self.utility = Utilty()
config = configparser.ConfigParser()
self.full_path = os.path.dirname(os.path.abspath(__file__))
self.root_path = os.path.join(self.full_path, '../')
try:
config.read(os.path.join(self.full_path, 'config.ini'))
except FileExistsError as err:
self.utility.print_exception(err,
'File exists error: {0}'.format(err))
sys.exit(1)
self.category_type = config['Common']['category']
self.train_path = os.path.join(self.full_path,
config['GyoiClassifier']['train_path'])
self.trained_path = os.path.join(
self.full_path, config['GyoiClassifier']['trained_path'])
self.train_os_in = os.path.join(
self.train_path, config['GyoiClassifier']['train_os_in'])
self.train_os_out = os.path.join(
self.trained_path, config['GyoiClassifier']['train_os_out'])
self.train_web_in = os.path.join(
self.train_path, config['GyoiClassifier']['train_web_in'])
self.train_web_out = os.path.join(
self.trained_path, config['GyoiClassifier']['train_web_out'])
self.train_framework_in = os.path.join(
self.train_path, config['GyoiClassifier']['train_framework_in'])
self.train_framework_out = os.path.join(
self.trained_path, config['GyoiClassifier']['train_framework_out'])
self.train_cms_in = os.path.join(
self.train_path, config['GyoiClassifier']['train_cms_in'])
self.train_cms_out = os.path.join(
self.trained_path, config['GyoiClassifier']['train_cms_out'])
self.wait_for_banner = float(
config['GyoiClassifier']['wait_for_banner'])
self.maximum_display_num = int(
config['GyoiClassifier']['maximum_display_num'])
self.summary_path = os.path.join(self.full_path,
config['GyoiThon']['summary_path'])
self.summary_file = os.path.join(self.summary_path,
config['GyoiThon']['summary_file'])
return
# Analysis using ML.
def analyzer(self,
target_ip='',
target_port=0,
target_vhost='',
silent=False,
target_url='',
target_response=''):
self.utility.print_message(
NOTE, 'Analyzing gathered HTTP response using Machine Learning.')
identified_list = []
target_info = ''
target_log = ''
analyzing_text = ''
if target_response == '':
# Get GyoiThon's summary.
df_origin = pd.read_csv(self.summary_file,
encoding='utf-8').fillna('')
df_selected_summary = df_origin[
(df_origin['ip'] == target_ip)
& (df_origin['port'] == target_port) &
(df_origin['vhost'] == target_vhost)]
# Get log file (webconf.csv)
logfile_path = os.path.join(self.root_path,
df_selected_summary.at[0, 'log'])
fin = codecs.open(logfile_path, 'r', encoding='utf-8')
analyzing_text = fin.read()
fin.close()
target_info = target_vhost + '(' + target_ip + '):' + str(
target_port)
target_log = logfile_path
else:
target_info = target_url
target_log = 'not use'
analyzing_text = target_response
# Output result (header)
# If silent mode is True, hidden target information.
if silent is True:
self.utility.print_message(
WARNING, 'target host : *** hidden for silent mode. ***')
self.utility.print_message(
WARNING, 'target url : *** hidden for silent mode. ***')
self.utility.print_message(
WARNING, 'target log : *** hidden for silent mode. ***')
else:
self.utility.print_message(WARNING,
'target host : {}'.format(target_info))
self.utility.print_message(WARNING,
'target url : {}'.format(target_url))
self.utility.print_message(WARNING,
'target log : {}'.format(target_log))
self.utility.print_message(OK, 'judge :')
# Predict product name each category (OS, Middleware, CMS..).
list_category = self.category_type.split('@')
print('-' * 42)
for category in list_category:
# Learning.
if category == 'os':
nb = self.train(self.train_os_in, self.train_os_out)
elif category == 'web server':
nb = self.train(self.train_web_in, self.train_web_out)
elif category == 'framework':
nb = self.train(self.train_framework_in,
self.train_framework_out)
elif category == 'cms':
nb = self.train(self.train_cms_in, self.train_cms_out)
else:
self.utility.print_message(FAIL,
'Choose category is not found.')
exit(1)
# Predict product name.
product, prob, keyword_list, classified_list = nb.classify(
analyzing_text)
# Output result of prediction (body).
# If no feature, result is unknown.
if len(keyword_list) == 0:
self.utility.print_message(NOTE,
'category : {}'.format(category))
self.utility.print_message(WARNING, 'product : unknown')
self.utility.print_message(WARNING, 'too low probability.')
else:
sorted_classified_list = sorted(classified_list,
key=lambda x: x[1],
reverse=True)
self.utility.print_message(NOTE,
'category : {}'.format(category))
for idx, item in enumerate(sorted_classified_list):
add_flag = True
if idx >= self.maximum_display_num:
break
# Delete duplicated result.
reason_list = []
for reason in item[2]:
reason_list.append(list(set(reason)))
# # If no feature, reason is "too few features".
if len(item[2]) == 0:
reason_list = 'too few features..'
add_flag = False
self.utility.print_message(NOTE,
'ranking {}'.format(idx + 1))
self.utility.print_message(
OK, 'product : {}'.format(item[0]))
self.utility.print_message(
OK,
'probability : {}'.format(round(item[1] * 100.0, 4)))
self.utility.print_message(
OK, 'reason : {}'.format(reason_list))
# Add product for Exploit.
identified_list.append(item[0])
self.utility.print_message(NONE, '-' * 42)
# Output result of prediction (footer).
self.utility.print_message(
NOTE, 'done {}'.format(os.path.basename(__file__)))
return list(set(identified_list))
# Execute learning / Get learned data.
def train(self, in_file, out_file):
# If existing learned data (pkl), load learned data.
nb = None
if os.path.exists(out_file):
with open(out_file, 'rb') as f:
nb = pickle.load(f)
# If no learned data, execute learning.
else:
# Read learning data.
nb = NaiveBayes()
fin = codecs.open(in_file, 'r', 'utf-8')
lines = fin.readlines()
fin.close()
items = []
for line in lines:
words = line[:-2]
train_words = words.split('@')
items.append(train_words[1])
nb.train(train_words[1], train_words[0])
# Save learned data to pkl file.
with open(out_file, 'wb') as f:
pickle.dump(nb, f)
return nb
utility.write_log(20, '[In] GyoiThon [{}].'.format(file_name))
# Read config.ini.
config = configparser.ConfigParser()
config.read(os.path.join(full_path, 'config.ini'))
# Common setting value.
log_path = ''
method_crawl = ''
try:
log_dir = config['Common']['log_path']
log_path = os.path.join(full_path, log_dir)
method_crawl = config['Common']['method_crawl']
except Exception as e:
msg = 'Reading config.ini is failure : {}'.format(e)
utility.print_exception(e, msg)
utility.write_log(40, msg)
utility.write_log(20, '[Out] GyoiThon [{}].'.format(file_name))
exit(1)
# Show banner.
show_banner(utility)
# Create instances.
cloud_checker = CloudChecker(utility)
version_checker = VersionChecker(utility)
comment_checker = CommentChecker(utility)
error_checker = ErrorChecker(utility)
page_checker = PageChecker(utility)
google_hack = GoogleCustomSearch(utility)
content_explorer = ContentExplorer(utility)
class CreateReport:
def __init__(self):
self.util = Utilty()
# Read config file.
full_path = os.path.dirname(os.path.abspath(__file__))
config = configparser.ConfigParser()
try:
config.read(os.path.join(full_path, 'config.ini'))
except Exception as err:
self.util.print_exception(err, 'File exists error')
sys.exit(1)
self.report_path = os.path.join(full_path,
config['Report']['report_path'])
self.report_file = os.path.join(self.report_path,
config['Report']['report_file'])
self.template_file = config['Report']['template_file']
self.header = str(config['Report']['header']).split('@')
def create_report(self):
self.util.print_message(NOTE, 'Creating report.')
# Gather reporting items.
csv_file_list = glob.glob(os.path.join(self.report_path, '*.csv'))
# Create DataFrame.
content_list = []
for file in csv_file_list:
content_list.append(pd.read_csv(file, names=self.header, sep=','))
df_csv = pd.concat(content_list).drop_duplicates().sort_values(
by=['ip', 'port'], ascending=True).reset_index(drop=True,
col_level=1)
items = []
for idx in range(len(df_csv)):
items.append({
'ip_addr':
df_csv.loc[idx, 'ip'],
'port':
df_csv.loc[idx, 'port'],
'prod_name':
df_csv.loc[idx, 'service'],
'vuln_name':
df_csv.loc[idx, 'vuln_name'],
'type':
df_csv.loc[idx, 'type'],
'description':
df_csv.loc[idx, 'description'],
'exploit':
df_csv.loc[idx, 'exploit'],
'target':
df_csv.loc[idx, 'target'],
'payload':
df_csv.loc[idx, 'payload'],
'ref':
str(df_csv.loc[idx, 'reference']).replace('@', '<br>')
})
try:
# Setting template.
env = Environment(loader=FileSystemLoader(self.report_path))
template = env.get_template(self.template_file)
pd.set_option('display.max_colwidth', -1)
html = template.render({
'title': 'Deep Exploit Scan Report',
'items': items
})
# Write report.
with codecs.open(self.report_file, 'w', 'utf-8') as fout:
fout.write(html)
except Exception as err:
self.util.print_exception(err, 'Creating report error.')
self.util.print_message(OK, 'Created report.')
class ClassifierSignature:
def __init__(self):
self.full_path = os.path.dirname(os.path.abspath(__file__))
self.signature_path = os.path.join(self.full_path, 'signature')
self.util = Utilty()
# Identify product name.
def identify_product(self, categoy, response):
prod_info_list = []
file_name = 'signature_' + categoy + '.txt'
try:
# Judge product using pattern matching.
with codecs.open(os.path.join(self.signature_path, file_name), 'r', 'utf-8') as fin:
matching_patterns = fin.readlines()
for pattern in matching_patterns:
items = pattern.replace('\r', '').replace('\n', '').split('@')
product = items[0].lower()
signature = items[2]
list_match = re.findall(signature, response, flags=re.IGNORECASE)
if len(list_match) != 0:
# Check version.
version_info = ''
for target_string in list_match:
version_info = self.extract_version(target_string).lower()
if version_info != '':
break
# Add product name and version.
if str(items[1]) != '':
prod_info_list.append(product + '@' + str(items[1]))
elif version_info != '':
prod_info_list.append(product + '@' + version_info)
else:
prod_info_list.append(product + '@-')
except Exception as err:
self.util.print_message(WARNING, '{}'.format(err))
return prod_info_list
# Extract version.
def extract_version(self, target_string):
# Regression list for cutting version.
regex_list = [r'(\d{1,3}\.\d{1,3}\.\d{1,3}).*',
r'(\d{1,3}\.\d{1,3}).*',
r'(\d{1,3}).*',
r'(\d{1,3}\.\d{1,3}[a-z]\d{1,3}).*',
r'(\d{1,3}\.\d{1,3}\.\d[a-z]{1,3}).*',
r'(\d\.[xX|\*]).*']
version_info = ''
for regex_pattern in regex_list:
version_list = re.findall(regex_pattern, target_string)
if len(version_list) != 0:
version_info = str(version_list[0])
break
return version_info
# Classifier product name using signatures.
def classifier_signature(self, target_info, client):
product_list = []
for target in target_info:
for target_url in target[2]:
# Check target url.
parsed = None
try:
parsed = util.parse_url(target_url)
except Exception as err:
self.util.print_exception(err, 'Parsed error: {}'.format(target_url))
continue
# Get HTTP response (header + body).
response = ''
http = urllib3.PoolManager(timeout=self.util.http_timeout)
try:
client.keep_alive()
self.util.print_message(OK, '{} {}'.format(self.util.get_current_date('%Y-%m-%d %H:%M:%S'),
target_url))
res = http.request('GET', target_url)
for header in res.headers.items():
response += header[0] + ': ' + header[1] + '\r\n'
response += '\r\n\r\n' + res.data.decode('utf-8')
except Exception as err:
self.util.print_exception(err, 'Target URL: {}'.format(target_url))
continue
for category in ['os', 'web', 'framework', 'cms']:
prod_info = self.identify_product(category, self.util.delete_ctrl_char(response))
for product in prod_info:
port_num = 80
path_item = os.path.split(parsed.path)
if parsed.port is not None:
port_num = parsed.port
if path_item[0].endswith('/') is False:
product_list.append(product + '@' + str(port_num) + '@' + path_item[0] + '/')
else:
product_list.append(product + '@' + str(port_num) + '@' + path_item[0])
time.sleep(1.0)
# Delete duplication.
uniq_product = []
tmp_list = []
for item in list(set(product_list)):
tmp_item = item.split('@')
tmp = tmp_item[0] + tmp_item[2]
if tmp not in tmp_list:
tmp_list.append(tmp)
uniq_product.append(item)
return uniq_product
