def handle_my_custom_event(audio):
#session['audio'] += audio
#session['audio'] += audio.values()
values = OrderedDict(sorted(audio.items(),
key=lambda t: int(t[0]))).values()
session['audio'] += values
pcm_audio = float_to_16_bit_pcm(session['audio'])
main(audio=pcm_audio, folder=app.config['UPLOAD_FOLDER'])
def main():
#读取配置文件
import config_load, logger
conf = config_load.load_conf()
encoding = conf.get('DEFAULT', 'encoding')
import filter
import csv
pattern_group = conf.sections()
for pattern_group_name in pattern_group:
csv_write_file = './output/' + pattern_group_name + '筛选结果.csv'
csv_junk_file = './output/' + pattern_group_name + '无用信息.csv'
csv_write = open(csv_write_file, 'w', newline='', encoding=encoding)
csv_junk = open(csv_junk_file, 'w', newline='', encoding=encoding)
writer = csv.writer(csv_write)
writer.writerow(['网站', '时间', '标题', '链接', '内容'])
junk = csv.writer(csv_junk)
junk.writerow(['网站', '时间', '标题', '链接', '内容'])
csv_read = open('./output/All.csv', 'r', newline='', encoding=encoding)
reader = csv.DictReader(csv_read)
list = []
for row in reader:
list.append(row)
for i in list:
city = i['网站']
date = i['时间']
title = i['标题']
url = i['链接']
info = i['内容']
if filter.main(pattern_group_name, info) == 1:
writer.writerow([city, date, title, url, info])
else:
junk.writerow([city, date, title, url, info])
logger.debug(f'{pattern_group_name} 关键词筛选完成！')
def handle(self):
ppm = Pixmap()
color, escala = 1, 1
dic = {"txt": " text/plain", "jpg": " image/jpeg", "ppm": " image/x-portable-pixmap", "html": " text/html", "pdf": " application/pdf", "ico": "image/vnd.microsoft.icon"}
self.data = self.request.recv(1024)
encabezado = self.data.decode().splitlines()[0]
archivo = args.documentroot + encabezado.split()[1].split("?")[0]
if (len(encabezado.split("?")) != 1) and (archivo.split('.')[1] == "ppm"):
color, escala = ppm.parseo(encabezado.split("?"))
if archivo == (args.documentroot + "/"):
archivo = args.documentroot + '/index.html'
print(self.client_address)
if os.path.isfile(archivo) is False:
archivo = args.documentroot + '/400error.html'
codigo = "HTTP/1.1 400 File Not Found"
extension = "html"
elif color == 0 and escala == 0:
archivo = args.documentroot + '/500error.html'
codigo = "HTTP/1.1 500 Internal Server Error"
extension = "html"
else:
extension = archivo.split('.')[1]
codigo = "HTTP/1.1 200 OK"
if (extension != "ppm") or (extension == "ppm" and len(encabezado.split("?")) == 1):
fd = os.open(archivo, os.O_RDONLY)
body = os.read(fd, os.path.getsize(archivo))
os.close(fd)
header = bytearray(codigo + "\r\nContent-type:" + dic[extension] + "\r\nContent-length:"+str(len(body))+"\r\n\r\n",'utf8')
self.request.sendall(header)
self.request.sendall(body)
if extension == "ppm" and color != 1:
enca, body = main(color, escala, archivo, args.size)
header = bytearray(codigo + "\r\nContent-type:" + dic[extension] + "\r\nContent-length:"+str((os.path.getsize(archivo))*4)+"\r\n\r\n",'utf8')
self.request.sendall(header)
self.request.sendall(enca)
for lista in (list(body)):
self.request.sendall(bytearray(lista, "utf-8"))
def gaussian_filtering():
filter.main(setSigmaValue())
# License for the specific language governing permissions and limitations
# under the License.
'''
'''
import filter
class ShippableFilter(filter.Filter):
'Usage: shippable_filter.py'
def __init__(self, args):
pass
def validate_data(self, data):
if 'statuses' not in data or 'sha' not in data:
return False
sha = data['sha']
if sha not in data['statuses'] or len(data['statuses'][sha]) == 0:
return False
status = data['statuses'][sha][0]
if 'state' not in status or 'context' not in status:
return False
if status['state'] == 'success' and status['context'] == 'Shippable':
return True
return False
if __name__ == "__main__":
filter.main(ShippableFilter)
# shippable_filter.py ends here
import matplotlib as mpl
from scipy.fftpack import fft, ifft
import FFT
raw_data = edfparser.main('data\lkf-right.edf')
#print (raw_data.shape)
pre_data_list = [[0 for col in range(128)] for row in range(14)]
for i in range(len(raw_data)):
if (i >= 2 and i <= 15):
#for j in range(len (raw_data[i])):
for j in range(128):
pre_data_list[i - 2][j] = raw_data[i][j]
pre_data = np.asarray(pre_data_list)
print(pre_data.shape)
#plt.subplot(221)
#plt.plot(pre_data,'o')
fft_pre_x, fft_pre_y = FFT.fft_transform(pre_data[10])
#plt.subplot(222)
plt.plot(fft_pre_x, fft_pre_y, 'b')
filter_data = filter.main(pre_data)
#plt.plot(filter_data,'p')
#print(type(filter_data))
print(filter_data.shape)
#fft_data = pyfftw.FFTW(filter_data)
#plt.plot(fft_data,'o')
def main():
usage = "Usage: vcfPytools.py [tool] [options]\n\n" + \
"Available tools:\n" + \
" annotate:\n\tAnnotate the vcf file with membership in other vcf files.\n" + \
" extract:\n\tExtract vcf records from a region.\n" + \
" filter:\n\tFilter the vcf file.\n" + \
" indel:\n\tIndel manipulation tools.\n" + \
" intersect:\n\tGenerate the intersection of two vcf files.\n" + \
" merge:\n\tMerge a list of vcf files.\n" + \
" multi:\n\tFind the intersections and unique fractions of multiple vcf files.\n" + \
" sort:\n\tSort a vcf file.\n" + \
" stats:\n\tGenerate statistics from a vcf file.\n" + \
" union:\n\tGenerate the union of two vcf files.\n" + \
" unique:\n\tGenerate the unique fraction from two vcf files.\n" + \
" validate:\n\tValidate the input vcf file.\n\n" + \
"vcfPytools.py [tool] --help for information on a specific tool."
# Determine the requested tool.
if len(sys.argv) > 1:
tool = sys.argv[1]
else:
print >> sys.stderr, usage
exit(1)
if tool == "annotate":
import annotate
success = annotate.main()
elif tool == "extract":
import extract
success = extract.main()
elif tool == "filter":
import filter
success = filter.main()
elif tool == "intersect":
import intersect
success = intersect.main()
elif tool == "indel":
import indel
success = indel.main()
elif tool == "multi":
import multi
success = multi.main()
elif tool == "merge":
import merge
success = merge.main()
elif tool == "sort":
import sort
success = sort.main()
elif tool == "stats":
import stats
success = stats.main()
elif tool == "union":
import union
success = union.main()
elif tool == "unique":
import unique
success = unique.main()
elif tool == "test":
import test
success = test.main()
elif tool == "validate":
import validate
success = validate.main()
elif tool == "--help" or tool == "-h" or tool == "?":
print >> sys.stderr, usage
else:
print >> sys.stderr, "Unknown tool: ", tool
print >> sys.stderr, "\n", usage
exit(1)
# If program completed properly, terminate.
if success == 0: exit(0)
def analyse():
global filename
global outputmap
global pixel_mm
global color_min
global color_max
global blur_img
global blur_mask
global size_point
global space_point
global frames_sec
global data
global mouse_blob
global speed
#calculate time per frame (ms):
time = float(1000.0000/float(frames_sec))
if filename == "Camera":
cap = cv2.VideoCapture(0)
else:
cap = cv2.VideoCapture(filename)
low_array = np.array([color_min[0], color_min[1], color_min[2]])
high_array = np.array([color_max[0], color_max[1], color_max[2]])
data = [] # [[x1,y1,time-1],[x2,y2,time-2]]
while True:
flag, frame = cap.read()
if flag:
framenummer = cap.get(cv2.cv.CV_CAP_PROP_POS_FRAMES)
frame = cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
frame = cv2.blur(frame,blur_img)
# generate color mask
mask = cv2.inRange(frame, low_array, high_array)
mask = cv2.blur(mask, blur_mask)
# make new image
frame_mask = cv2.bitwise_and(frame, frame, mask=mask)
frame_blob,keypoints = blob_detect(frame_mask)
#cv2.imshow('video', cv2.cvtColor(frame_blob, cv2.COLOR_RGB2BGR))
if len(data) == 0:
x = mouse_blob.pt[0]
y = mouse_blob.pt[1]
point = color.find_mouse_blob(keypoints,x,y)
print point.pt[0], point.pt[1]
data = [[point.pt[0] * pixel_mm, point.pt[1] * pixel_mm, framenummer * time]]
else:
if len(keypoints) == 1:
change = change_place(keypoints[0], data[len(data) - 1])
print 'first:\t'+str(change)
if change < (speed*pixel_mm):
data = data + [[keypoints[0].pt[0]*pixel_mm,keypoints[0].pt[1]*pixel_mm,framenummer * time]]
elif len(keypoints) == 0:
print "no points found"
else:
point = find_next_blob(keypoints, data)
change = change_place(point, data[len(data) - 1])
print 'second:\t' + str(change)
if change < (speed*pixel_mm):
data = data + [[point.pt[0] * pixel_mm, point.pt[1] * pixel_mm, framenummer * time]]
# if len(keypoints) != 0:
# if len(keypoints) == 1:
# if len(data) != 0:
# change = change_place(keypoints[0],data[len(data)-1])
# print 'first:\t'+str(change)
# if change < (speed*pixel_mm):
# data = data + [[keypoints[0].pt[0]*pixel_mm,keypoints[0].pt[1]*pixel_mm,framenummer * time]]
# else:
# x = mouse_blob.pt[0]
# y = mouse_blob.pt[1]
# print x, y
# keypoint = find_next_blob(keypoints, [x, y, 0])
# print keypoint[0].pt[0], keypoint[0].pt[1]
# data = [[keypoint[0].pt[0] * pixel_mm, keypoint[0].pt[1] * pixel_mm, framenummer * time]]
# else:
# if len(data) == 0:
# x = mouse_blob.pt[0]
# y = mouse_blob.pt[1]
# print x,y
# keypoint = find_next_blob(keypoints,[x,y,0])
# print keypoint[0].pt[0],keypoint[0].pt[1]
# data = data + [[keypoint[0].pt[0] * pixel_mm, keypoint[0].pt[1] * pixel_mm, framenummer * time]]
# else:
# keypoints = find_next_blob(keypoints,data[len(data)-1])
# change = change_place(keypoints[0], data[len(data) - 1])
# 'second:\t' + str(change)
# if change < (speed*pixel_mm):
# data = data + [[keypoints[0].pt[0] * pixel_mm, keypoints[0].pt[1] * pixel_mm, framenummer * time]]
else:
cv2.destroyAllWindows()
break
if len(data) >0:
last_point = data[len(data)-1]
X_last = int(last_point[0]/pixel_mm)
Y_last = int(last_point[1]/pixel_mm)
cv2.circle(frame_blob,(X_last,Y_last),20,(0,255,0),4)
cv2.imshow('video', cv2.cvtColor(frame_blob, cv2.COLOR_RGB2BGR))
if cv2.waitKey(10) == 27:
cv2.destroyAllWindows()
break
plot_ask = tkMessageBox.askyesno("Plot","Do you want to plot the results")
if plot_ask:
#plot gegevens
print data
filter.main(data, outputmap, ".png", 400)
def main():
usage = "Usage: vcfPytools.py [tool] [options]\n\n" + \
"Available tools:\n" + \
" annotate:\n\tAnnotate the vcf file with membership in other vcf files.\n" + \
" extract:\n\tExtract vcf records from a region.\n" + \
" filter:\n\tFilter the vcf file.\n" + \
" indel:\n\tIndel manipulation tools.\n" + \
" intersect:\n\tGenerate the intersection of two vcf files.\n" + \
" merge:\n\tMerge a list of vcf files.\n" + \
" multi:\n\tFind the intersections and unique fractions of multiple vcf files.\n" + \
" sort:\n\tSort a vcf file.\n" + \
" stats:\n\tGenerate statistics from a vcf file.\n" + \
" union:\n\tGenerate the union of two vcf files.\n" + \
" unique:\n\tGenerate the unique fraction from two vcf files.\n" + \
" validate:\n\tValidate the input vcf file.\n\n" + \
"vcfPytools.py [tool] --help for information on a specific tool."
# Determine the requested tool.
if len(sys.argv) > 1:
tool = sys.argv[1]
else:
print >> sys.stderr, usage
exit(1)
if tool == "annotate":
import annotate
success = annotate.main()
elif tool == "extract":
import extract
success = extract.main()
elif tool == "filter":
import filter
success = filter.main()
elif tool == "intersect":
import intersect
success = intersect.main()
elif tool == "indel":
import indel
success = indel.main()
elif tool == "multi":
import multi
success = multi.main()
elif tool == "merge":
import merge
success = merge.main()
elif tool == "sort":
import sort
success = sort.main()
elif tool == "stats":
import stats
success = stats.main()
elif tool == "union":
import union
success = union.main()
elif tool == "unique":
import unique
success = unique.main()
elif tool == "test":
import test
success = test.main()
elif tool == "validate":
import validate
success = validate.main()
elif tool == "--help" or tool == "-h" or tool == "?":
print >> sys.stderr, usage
else:
print >> sys.stderr, "Unknown tool: ",tool
print >> sys.stderr, "\n", usage
exit(1)
# If program completed properly, terminate.
if success == 0: exit(0)
def main():
filter.main()
twitter2foursquare.main()
sqliteModels.main()
sqliteQueries.main()
import filter
class RegexpFilter(filter.Filter):
'Usage: regexp_filter.py <key> <regexp>'
def __init__(self, args):
self.selector = args[0]
self.regexp = re.compile(args[1])
def validate_data(self, data):
data = self.select(self.selector, data)
#print(data)
if type(data) is list or type(data) is tuple:
# at least one item matching regexp
for filename in data:
if self.regexp.search(filename):
sys.stderr.write('%s\n' % filename)
return True
else:
if not isinstance(data, basestring):
data = str(data)
return self.regexp.search(data)
return False
if __name__ == "__main__":
filter.main(RegexpFilter)
# regexp_filter.py ends here
def main(TOPIC_NUM, limit, target):
"""
main
:return:
"""
# The read file is inside the loop
# File input path, permission file needs to be changed, the unchanged file directly copied to the new file, the need to change the file first read the original path, and then copied to the new path output
path = 'Iterativedata_Output/' + target + 'Target'
common.mkdir(path)
path = path + '/' + str(TOPIC_NUM) + 'Topics' + str(limit) + 'Filtering'
common.mkdir(path)
inPath = 'Data_Origin/' + target + 'Target/' + str(TOPIC_NUM) + 'topic_data_output'
# # Cutting file
operationfile.operation_file(TOPIC_NUM, limit, target)
# use SQL
drop_permissions = "DROP TABLE IF EXISTS permissions; "
common.sqlExecute(drop_permissions)
drop_permissionslog = "DROP TABLE IF EXISTS permissionlog;"
common.sqlExecute(drop_permissionslog)
create_permissions = "create table permissions ( id int auto_increment primary key, times int null comment '迭代次数', appid int null comment 'Appid', appname varchar(255) null comment 'app名称', permission varchar(5000) null comment '权限字符串', mal int(4) null comment '恶性App');"
common.sqlExecute(create_permissions)
create_permissionslog = "create table permissionlog ( id int auto_increment primary key, appid int null comment 'appid', times int null comment '循环次数', old varchar(5000) null comment '原始权限', new varchar(5000) null comment '新的权限', remove varchar(255) null comment '移除的权限', `create` datetime null comment '时间');"
common.sqlExecute(create_permissionslog)
insert_data = "insert into permissions SELECT null , 0 , id , docid , lessper , maliciousFlag from normal_test_permission;"
common.sqlExecute(insert_data)
for iterations in range(1):
outPath = path + '/' + str(iterations)
common.mkdir(outPath)
for index in range(5):
newPath = outPath + '/' + str(index)
common.mkdir(newPath)
# Divide the topic file and the permission file into 5 parts and take one of them in a loop. The permission file USES the database for easy modification
# Appid
Benign_testing_topic, Benign_training_topic, Benign_testing_appid, Benign_training_appid = getTrainAndTarget(inPath + '/Benign topic four-fifths.txt', index)
Benign_testing_permission = getTrainAndTargetPermissionFromDb(Benign_testing_appid)
Benign_training_permission = getTrainAndTargetPermissionFromDb(Benign_training_appid)
Malicious_training_topic = open(inPath + '/Malicious topic four-fifths.txt', 'r').readlines()
Malicious_training_permission = open(inPath + '/Malicious permission four-fifths.txt', 'r').readlines()
topic.mainInList(Benign_testing_topic, newPath + '/Testing_benign_topic_matrix.mat', TOPIC_NUM)
permission.mainInList(Benign_testing_permission, newPath + '/Testing_benign_permission_matrix.mat', 1)
topic.mainInList(Benign_training_topic, newPath + '/Training_benign_topic_matrix.mat', TOPIC_NUM)
permission.mainInList(Benign_training_permission, newPath + '/Training_benign_permission_matrix.mat', 1)
topic.mainInList(Malicious_training_topic, newPath + '/Training_malicious_topic_matrix.mat', TOPIC_NUM)
permission.mainInList(Malicious_training_permission, newPath + '/Training_malicious_permission_matrix.mat', False)
similar.main(newPath + '/Training_benign_topic_matrix.mat'
, newPath + '/Testing_benign_topic_matrix.mat'
, newPath + '/Similarity-training_benign.txt'
, TOPIC_NUM)
similar.main(newPath + '/Training_malicious_topic_matrix.mat'
, newPath + '/Testing_benign_topic_matrix.mat'
, newPath + '/Similarity-training_malicious.txt'
, TOPIC_NUM)
recommend.main(newPath + '/Training_benign_permission_matrix.mat'
, newPath + '/Similarity-training_benign.txt'
, newPath + '/Recommend-training_benign.txt'
, 0.6)
recommend.main(newPath + '/Training_malicious_permission_matrix.mat'
, newPath + '/Similarity-training_malicious.txt'
, newPath + '/Recommend-training_malicious.txt'
, 0.4)
print("The output is complete, and the difference filtering is performed.")
reSetPermission(newPath + '/Recommend-training_benign.txt', newPath + '/Recommend-training_malicious.txt', iterations)
print('Done,start filtering')
filter.main(limit)
print('Filtering is complete,please back up the database data')
batlogs = "mysqldump -u root -p%s analyzedata permissions permissionlog > %s/%sTopics%sFiltering-including_log.sql" % (config.sql_password, path, str(TOPIC_NUM), str(limit))
os.system(batlogs)
sqlPath = path + "/" + str(TOPIC_NUM) + 'Topics' + str(limit) + 'Filtering' + ".sql"
bat = "mysqldump -u root -p%s analyzedata permissions > %s" % (config.sql_password, sqlPath)
os.system(bat)
print('Filtering completed,database data backup complete')
return sqlPath
