def home_pipeline():
home_template = generate_template('home')
write_rendered_template_to_file(
home_template,
filename='index',
projects_list=get_config()['projects'],
**get_config()['home']
)
def make_dataset(self):
dataset = tf.data.Dataset.from_tensor_slices(
(self.people_list, self.img_list))
dataset = dataset.map(map_func=self._parse_img,
num_parallel_calls=get_config("threads"))
dataset = dataset.shuffle(buffer_size=get_config("train.shuffle"))
dataset = dataset.batch(batch_size=get_config("train.batch_size"))
self.dataset = dataset.prefetch(
buffer_size=get_config("train.prefetch"))
def media_pipeline():
dst_path = os.path.join(get_config()['content_path'], 'media')
if os.path.exists(dst_path):
shutil.rmtree(dst_path)
shutil.copytree(
src=get_config()['media_path'],
dst=dst_path
)
def _parse_img(people, img_path):
image = tf.io.read_file(img_path)
image = tf.image.decode_image(image, get_config("model.channel"))
image.set_shape([
get_config("model.height"),
get_config("model.width"),
get_config("model.channel")
])
image = tf.cast(image, tf.float32)
image = data_rescale(image)
return people, image
def main():
"""These are the main training settings. Set each before running
this file."""
cf = get_config()
# model can be one of lstm, lrcn, mlp, conv_3d, c3d
model = cf.get('sequence', 'model')
saved_model = None # None or weights file
class_limit = cf.get('sequence',
'class_limit') # int, can be 1-101 or None
class_limit = int(class_limit) if class_limit != 'None' else None
seq_length = cf.getint('sequence', 'seq_length')
load_to_memory = cf.getboolean(
'sequence', 'load_to_memory') # pre-load the sequences into memory
batch_size = cf.getint('sequence', 'batch_size')
nb_epoch = cf.getint('sequence', 'nb_epoch')
# Chose images or features and image shape based on network.
if model in ['conv_3d', 'c3d', 'lrcn']:
data_type = 'images'
image_shape = (80, 80, 3)
elif model in ['lstm', 'gru', 'mlp']:
data_type = 'features'
image_shape = None
else:
raise ValueError("Invalid model. See train.py for options.")
train(data_type,
seq_length,
model,
saved_model=saved_model,
class_limit=class_limit,
image_shape=image_shape,
load_to_memory=load_to_memory,
batch_size=batch_size,
nb_epoch=nb_epoch)
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'--model_type',
help='Model type to validate. Could be mlp/lstm/gru/conv_3d/lrcn',
type=str,
default='mlp')
parser.add_argument(
'--saved_model',
help='Model file name with path. Should be under data/checkpoints/ dir',
type=str,
default=os.path.join(
os.path.dirname(__file__),
'data/checkpoints/mlp-features.523-0.346-0.92.hdf5'))
args = parser.parse_args()
cf = get_config()
#model = 'mlp'
#saved_model = 'data/checkpoints/mlp-features.316-0.459-0.88.hdf5'
seq_length = cf.getint('sequence', 'seq_length')
if args.model_type == 'conv_3d' or args.model_type == 'lrcn':
data_type = 'images'
image_shape = (80, 80, 3)
else:
data_type = 'features'
image_shape = None
validate(data_type,
args.model_type,
seq_length=seq_length,
saved_model=args.saved_model,
image_shape=image_shape,
class_limit=None)
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'--model_file',
help='Model file name with path. Should be under data/checkpoints/ dir',
type=str,
default=os.path.join(
os.path.dirname(__file__),
'data/checkpoints/mlp-features.316-0.459-0.88.tflite'))
parser.add_argument(
'--extractor_model',
help='Model file name with path. Should be under data/checkpoints/ dir',
type=str)
parser.add_argument(
'--video_name',
help=
'Inferenced video file in data/data_file.csv. Do not include the extension ',
type=str,
default='restRoom_001')
args = parser.parse_args()
cf = get_config()
# Sequence length must match the lengh used during training.
seq_length = cf.getint('sequence', 'seq_length')
# Limit must match that used during training.
class_limit = cf.get('sequence', 'class_limit')
class_limit = int(class_limit) if class_limit != 'None' else None
# Get the dataset.
data = DataSet(seq_length=seq_length, class_limit=class_limit)
sequence = extract(data, seq_length, args.extractor_model, args.video_name)
predict(data, sequence, args.model_file)
def login(self, request, *args, **kwargs):
"""
登录
:param request:
:param args:
:param kwargs:
:return:
"""
# 获取配置信息
res_dict = get_config()
if not res_dict:
return JsonResponse({'status': status.HTTP_500_INTERNAL_SERVER_ERROR, 'msg': 'Failed to get configuration'},
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
# 获取表单数据
user_post = request.POST.get("username")
pwd_post = request.POST.get("password")
print("post",user_post, pwd_post)
# 获取mysql连接信息
host = res_dict['data']['mysql'].get('host')
user = res_dict['data']['mysql'].get('username')
password = res_dict['data']['mysql'].get('password')
port = res_dict['data']['mysql'].get('port')
print("host",host,port)
# 判断mysql端口
if not check_tcp(host,port):
return JsonResponse({'status': status.HTTP_500_INTERNAL_SERVER_ERROR, 'msg': 'MySQL port cannot connect'},
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
print("查询表记录。。。。")
# 查询表记录
sql = "select username,password from usercenter.users where username='******' and password=md5('%s')" % (
user_post, pwd_post)
print(sql)
mysql_obj = ExecutionSql(host, user, password, port)
auth_ret = mysql_obj.select(sql)
# 判断执行结果
if not auth_ret:
# 返回 http 401
return JsonResponse({'status': status.HTTP_401_UNAUTHORIZED, 'msg': 'Authentication failure'},
status=status.HTTP_401_UNAUTHORIZED)
# 更新登录时间
last_time = time.strftime('%Y-%m-%d %H:%M:%S')
# UPDATE table_name SET field1=new-value1, field2=new-value2
sql = "update usercenter.users set last_time='%s' where username='******'" % (last_time, user_post)
print(sql)
update_ret = mysql_obj.update(sql)
if not update_ret:
return JsonResponse({'status': status.HTTP_500_INTERNAL_SERVER_ERROR, 'msg': 'Failed to update login time'},
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
# 登录成功
# 返回 http 200
return JsonResponse({'status': status.HTTP_200_OK, 'data': []}, status=status.HTTP_200_OK)
def static_pipeline():
for static_site in get_config()['static_sites']:
static_file_dir = f'static/{static_site["filename_with_ext"]}'
static_template = generate_template(static_file_dir, html_ext=False)
write_rendered_template_to_file(
template=static_template,
filename=static_site['slug'],
**static_site
)
def projects_pipeline():
project_template = generate_template('project')
for project in get_config()['projects']:
write_rendered_template_to_file(
template=project_template,
filename=project['slug'],
file_path='proyecto',
**project
)
def get_model():
img_size = [get_config("model.height"), get_config("model.width"), get_config("model.channel")]
img1 = x1 = keras.Input(img_size)
img2 = x2 = keras.Input(img_size)
for _filter in get_config("model.conv_filters"):
x1 = keras.layers.Conv2D(_filter, get_config("model.kernel"), padding="same")(x1)
x1 = keras.layers.BatchNormalization()(x1)
x1 = keras.layers.Dropout(get_config("model.dropout_rate"))(x1)
x2 = keras.layers.Conv2D(_filter, get_config("model.kernel"), padding="same")(x2)
x2 = keras.layers.BatchNormalization()(x2)
x2 = keras.layers.Dropout(get_config("model.dropout_rate"))(x2)
x1 = keras.layers.Flatten()(x1)
x1 = keras.layers.Dense(get_config("model.dense_out"))(x1)
x2 = keras.layers.Flatten()(x2)
x2 = keras.layers.Dense(get_config("model.dense_out"))(x2)
y = x2 - x1
y = keras.layers.Dense(1, activation="tanh")(y)
return keras.Model(inputs=[img1, img2], outputs=y)
def write_rendered_template_to_file(template: Template,
filename: str,
file_path='',
*args,
**kwargs):
file_dir = os.path.join(get_config()['content_path'], file_path,
f'{filename}.html')
if not os.path.exists(os.path.dirname(file_dir)):
os.makedirs(os.path.dirname(file_dir))
with open(file_dir, mode='w') as f:
f.write(custom_render_template(template, *args, **kwargs))
def get_data(data_type):
cf = get_config()
seq_length = cf.getint('sequence', 'seq_length')
data = DataSet(seq_length=seq_length, class_limit=None)
X_train, y_train = data.get_all_sequences_in_memory(data_type,
data_type='features')
# Flatten feature vectors
X_train = X_train.reshape(X_train.shape[0], -1)
y_train = np.array([np.argmax(y_train[i]) for i, _ in enumerate(y_train)])
n_samples, n_features = X_train.shape
return X_train, y_train, n_samples, n_features
def check(self, request, *args, **kwargs):
"""
服务自检
:param request:
:return:
"""
# 判断 eureka 注册中心是否启动
port_ret = check_tcp('127.0.0.1', '8001')
if not port_ret:
return JsonResponse(
{'status': status.HTTP_500_INTERNAL_SERVER_ERROR, 'msg': 'Eureka Micro service Not started'},
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
# 判断config 微服务是否启动
port_ret = check_tcp('127.0.0.1', '8002')
if not port_ret:
return JsonResponse(
{'status': status.HTTP_500_INTERNAL_SERVER_ERROR, 'msg': 'Config Micro service Not started'},
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
# 判断获取配置信息
res_dict = get_config()
if not res_dict:
return JsonResponse({'status': status.HTTP_500_INTERNAL_SERVER_ERROR, 'msg': 'Failed to get configuration'},
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
# 判断mysql连接
# 获取mysql连接信息
host = res_dict['data']['mysql'].get('host')
user = res_dict['data']['mysql'].get('username')
password = res_dict['data']['mysql'].get('password')
port = res_dict['data']['mysql'].get('port')
# 判断mysql端口
if not check_tcp(host, port):
return JsonResponse({'status': status.HTTP_500_INTERNAL_SERVER_ERROR, 'msg': 'MySQL port cannot connect'},
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
# 查询表记录
sql = "show databases"
print(sql)
mysql_obj = ExecutionSql(host, user, password, port)
select_ret = mysql_obj.select(sql)
if not select_ret:
return JsonResponse(
{'status': status.HTTP_500_INTERNAL_SERVER_ERROR, 'msg': 'Failed to detect MySQL connection'},
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
# 返回 http 200
return JsonResponse({'status': status.HTTP_200_OK, 'data': []}, status=status.HTTP_200_OK)
def main():
cf = get_config()
parser = argparse.ArgumentParser()
parser.add_argument('--target_seq_length',
help='Sequence length you want to split to',
type=int,
default=cf.getint('sequence', 'seq_length'))
args = parser.parse_args()
os.chdir(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..'))
# Get the dataset.
data = DataSet.get_data()
data_file = []
for sample in data:
# check current target_seq_length
if int(sample[3]) >= args.target_seq_length:
images = DataSet.get_frames_for_sample(sample)
label = ord('A')
count = 0
for image in images:
path_suffix = image.split('-')[-1]
# move image to "data/output" dir to avoid file name chaos
target_path = os.path.join('data', 'output', sample[0],
sample[1])
touchdir(target_path)
path_prefix = os.path.join(target_path, sample[2]) + str(
chr(label)) + '-'
# move the image
os.rename(image, path_prefix + path_suffix)
count = count + 1
if (count == args.target_seq_length):
data_file.append([
sample[0], sample[1], sample[2] + str(chr(label)),
count
])
label = label + 1
# assume A~Z + a~z is enough
if label == ord('Z') + 1:
label = ord('a')
count = 0
with open(os.path.join('data', 'output', 'data_file.csv'), 'w') as fout:
writer = csv.writer(fout)
writer.writerows(data_file)
print(
'Done. Split data at <Project>/data/output. Pls use it for your train&test'
)
def sard_generate_all(approach):
sard_generate_models = {
"vuldeepecker": generate_CDG,
"vgdetector": generate_CFG,
"sysevr": generate_SYS,
"mulvuldeepecker": generate_MULVDP
}
if approach not in sard_generate_models:
print(
f"Unknown model: {approach}, try on of {sard_generate_models.keys()}"
)
return
for cweid in CWEID_AVA:
_config = get_config(approach, cweid)
sard_generate_models[approach](_config)
def sard_preprocess_all(approach):
sard_preprocess_models = {
"token": token_preprocess.preprocess,
"vuldeepecker": cdg_preprocess.preprocess,
"sysevr": sys_preprocess.preprocess,
"mulvuldeepecker": mulvdp_preprocess.preprocess,
"code2seq": c2s_preprocess.preprocess,
"code2vec": c2v_preprocess.preprocess
}
if approach not in sard_preprocess_models:
print(
f"Unknown model: {approach}, try on of {sard_preprocess_models.keys()}"
)
return
for cweid in CWEID_AVA:
print(f"processing {cweid}")
_config = get_config(approach, cweid)
sard_preprocess_models[approach](_config)
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'--model_file',
help='Model file name with path. Should be under data/checkpoints/ dir',
type=str,
default=os.path.join(
os.path.dirname(__file__),
'data/checkpoints/mlp-features.523-0.346-0.92.hdf5'))
parser.add_argument(
'--video_name',
help=
'Inferenced video file in data/data_file.csv. Do not include the extension ',
type=str,
default='restRoom_001')
args = parser.parse_args()
cf = get_config()
# Sequence length must match the lengh used during training.
seq_length = cf.getint('sequence', 'seq_length')
# Limit must match that used during training.
class_limit = cf.get('sequence', 'class_limit')
class_limit = int(class_limit) if class_limit != 'None' else None
# Get the dataset.
data = DataSet(seq_length=seq_length, class_limit=class_limit)
# Get feature sequence and conv sequence.
frames, sequence, conv_sequence = extract_and_conv(data, seq_length,
args.video_name)
sequence = np.asarray(sequence)
# Do the predict and get feature gradient sequence.
feature_grads_sequence = get_feature_grads(data, sequence, seq_length,
args.model_file)
# GradCAM: use feature gradient sequence and conv sequence to generate heatmap.
generate_heatmap(frames, conv_sequence, feature_grads_sequence, seq_length)
s3_preventAccessToPublic.py lambda module for preventing public access to S3 Buckets and Objects
Description:
Sample event is on this module directory(event.json). This module is designed to handle the following events:
-CreateBucket
"""
import os
import botocore
from utils.common import get_config, notify_email, get_aws_client
from utils.logger import LoggerUtils as logger
# global variables
s3Bucket = os.environ['CONF_S3BUCKET']
s3Key = os.environ['CONF_DenyPolicy']
defaultDenyPolicy = get_config(s3Bucket, s3Key)
bucketList = get_config(s3Bucket, "095139704753.json")
toEmail = os.environ['ToEmail']
fromEmail = os.environ['FromEmail']
roleName = os.environ['ROLE_NAME']
notification = True if os.environ['Notifications'] == 'True' else False
logger.setLevel()
def existing_bucket(bucketName, s3, subscriberAccountId, awsRegion):
""" This function parses the bucketpolicy on an existing bucket and also identifies exceptional buckets """
try:
exceptionalBucket = bucketList['publiclyAccessibleBuckets']
if bucketName in exceptionalBucket:
logger.debug(
def info(self, request, *args, **kwargs):
"""
用户信息
:param request:
:param args:
:param kwargs:
:return:
"""
# 获取配置信息
res_dict = get_config()
if not res_dict:
return JsonResponse(
{
'status': status.HTTP_500_INTERNAL_SERVER_ERROR,
'msg': 'Failed to get configuration'
},
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
# 获取表单数据
user_post = request.POST.get("username")
# pwd_post = request.POST.get("password")
# print(user_post, pwd_post)
# 获取mysql连接信息
host = res_dict['data']['mysql'].get('host')
user = res_dict['data']['mysql'].get('username')
password = res_dict['data']['mysql'].get('password')
port = res_dict['data']['mysql'].get('port')
# 判断mysql端口
if not check_tcp(host, port):
return JsonResponse(
{
'status': status.HTTP_500_INTERNAL_SERVER_ERROR,
'msg': 'MySQL port cannot connect'
},
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
# 查询表记录
sql = "select * from usercenter.users where username='******'" % (
user_post)
print(sql)
mysql_obj = ExecutionSql(host, user, password, port)
user_info = mysql_obj.select(sql)
# 判断执行结果
if not user_info:
# 返回 http 500
return JsonResponse(
{
'status': status.HTTP_500_INTERNAL_SERVER_ERROR,
'msg': 'Failed to query user information'
},
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
# 返回 http 200
return JsonResponse({
'status': status.HTTP_200_OK,
'data': [user_info]
},
status=status.HTTP_200_OK)
"""Requested record in database was not found"""
def create_tables(engine):
meta = MetaData()
meta.create_all(bind=engine, tables=[users, images])
def drop_tables(engine):
meta = MetaData()
meta.drop_all(bind=engine, tables=[users])
def sample_user_data(engine):
conn = engine.connect()
conn.execute(users.insert(),
[{
'username': '******',
'email': '*****@*****.**',
'name': 'Borys',
'last_name': 'Oliinyk',
'password': generate_password_hash('q1w2e3r4'),
}])
conn.close()
if __name__ == '__main__':
db_url = DSN.format(**get_config()['postgres'])
engine = create_engine(db_url)
create_tables(engine)
from jinja2 import Template, Environment, PackageLoader, select_autoescape, FileSystemLoader
from utils.common import get_config
import os
from css_html_js_minify import html_minify
env = Environment(
loader=FileSystemLoader(get_config()['templates_path']),
extensions=['jinja2.ext.autoescape'],
autoescape=True,
)
env.globals['MEDIA_PREFIX'] = '/media/'
def generate_template(template_name, html_ext=True):
return env.get_template(f'{template_name}{".html" if html_ext else ""}')
def custom_render_template(template: Template, *args, **kwargs):
rendered = template.render(*args, **kwargs)
minified = html_minify(rendered)
return minified
def write_rendered_template_to_file(template: Template,
filename: str,
file_path='',
*args,
**kwargs):
file_dir = os.path.join(get_config()['content_path'], file_path,
f'{filename}.html')
if not os.path.exists(os.path.dirname(file_dir)):
def __init__(self):
logging.info(" - Initializing Dataset...")
self.base_dir = get_config("dataset.cfp")
self.load_filename()
self.make_dataset()
test_data_path = join(config.data_folder, "token", cweid, "test.txt")
val_data_path = join(config.data_folder, "token", cweid, "val.txt")
if (os.path.exists(train_data_path)):
os.system(f"rm {train_data_path}")
if (os.path.exists(test_data_path)):
os.system(f"rm {test_data_path}")
if (os.path.exists(val_data_path)):
os.system(f"rm {val_data_path}")
with open(data_path, "r", encoding="utf-8") as f:
lines = f.readlines()
X_train, X_test = train_test_split(lines, test_size=0.2)
X_test, X_val = train_test_split(
X_test,
test_size=0.5,
)
with open(train_data_path, "a") as f:
for line in X_train:
f.write(line)
with open(test_data_path, "a") as f:
for line in X_test:
f.write(line)
with open(val_data_path, "a") as f:
for line in X_val:
f.write(line)
if __name__ == "__main__":
_config = get_config("token", "example")
collect_vocab(_config)
from argparse import ArgumentParser
from utils.common import get_config
from preprocessing import generate_bc_VFG, generate_CDG
from preprocessing.cdg_preprocess import preprocess
from preprocessing.cdg_generator import generate_CDG, generate_CDG_osp
from preprocessing.cfg_generator import generate_CFG, generate_CFG_osp
from preprocessing.dot_generator import generate_bc_VFG, generate_bc_VFG_osp, generate_VFG_osp
import os
from preprocessing.token_preprocess import preprocess, split_dataset
from utils.common import CWEID_AVA
if __name__ == "__main__":
# python preprocess.py --cweid 119
arg_parser = ArgumentParser()
# arg_parser.add_argument("--cweid", type=str, default=None)
arg_parser.add_argument("--project", type=str, default=None)
args = arg_parser.parse_args()
_config = get_config("vuldeepecker", args.project)
# _config = get_config("vuldeepecker", "CWE20")
# preprocess(_config)
# split_dataset(_config)
# generateCDG(_config, "CWE20")
# for cweid in CWEID_AVA:
# _config = get_config("token", cweid)
# generate_bc_VFG_osp(_config, args.project)
generate_CDG_osp(_config)
# generate_VFG_osp(os.path.join(_config.raw_data_folder,"CVE"), 'openssl')
# generate_bc_VFG_osp(_config, 'openssl')
)
trainer.fit(model=model, datamodule=data_module)
trainer.test()
if __name__ == "__main__":
# python train.py token --dataset CWE119
arg_parser = ArgumentParser()
arg_parser.add_argument("model", type=str)
arg_parser.add_argument("--dataset", type=str, default=None)
arg_parser.add_argument("--offline", action="store_true")
arg_parser.add_argument("--resume", type=str, default=None)
args = arg_parser.parse_args()
_config = get_config(args.model, args.dataset, log_offline=args.offline)
train(_config, args.resume)
# hasdone_ids = list()
# if not os.path.exists('hasdone_id.txt'):
# with open('hasdone_id.txt','w') as f:
# f.write('')
# else:
# with open('hasdone_id.txt','r') as f:
# text = f.read()
# hasdone_ids = text.split('\n')
# print(hasdone_ids)
# for cwe in CWEID_AVA:
# if cwe not in hasdone_ids:
# try:
# _config = get_config(args.model, cwe, log_offline=args.offline)
# train(_config, args.resume)
#flatten the sequence for save
flat_sequence = sequence.reshape(-1)
with open(txt_path, 'w') as f:
#save sequence length and feature length
#in 1st two lines
f.write(str(sequence.shape[0]) + '\n')
f.write(str(sequence.shape[1]) + '\n')
for item in flat_sequence:
f.write(str(item) + '\n')
# Set defaults.
cf = get_config()
seq_length = cf.getint('sequence', 'seq_length')
class_limit = cf.get(
'sequence', 'class_limit'
) # Number of classes to extract. Can be 1-101 or None for all.
class_limit = int(class_limit) if class_limit != 'None' else None
# Get the dataset.
data = DataSet(seq_length=seq_length, class_limit=class_limit)
# get the model.
model = Extractor()
# Loop through data.
pbar = tqdm(total=len(data.data))
for video in data.data:
from logging.config import fileConfig
from sqlalchemy import engine_from_config
from sqlalchemy import pool
from alembic import context
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
from api.db.tables import metadata
from utils.common import get_config
config = context.config
project_config = get_config()['postgres']
config.set_main_option(
'sqlalchemy.url', 'postgresql://{user}:{password}@{host}/{database}'.format(**project_config))
# Interpret the config file for Python logging.
# This line sets up loggers basically.
fileConfig(config.config_file_name)
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
target_metadata = metadata
# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
