import os
from tools import set_logger
import ConfigParser
from datetime import datetime
config_file = os.getcwd() + '/config.ini'
cf = ConfigParser.ConfigParser()
cf.read(config_file)
DB_ADDR = cf.get('mongodb','addr')
DB_PORT = int(cf.get('mongodb','port'),10)
READONLY = cf.get('mongodb','readonly_user')
READONLY_PASSWD = cf.get('mongodb','readonly_password')
READWRITE = cf.get('mongodb','readwrite_user')
READWRITE_PASSWD = cf.get('mongodb','readwrite_password')
LOG_FILE = os.path.join(os.getcwd(), cf.get('support','log_path'))
logger = set_logger('mongodb.py', LOG_FILE)
'==数据库连接操作=='
def connect_readwrite():
try:
con = pymongo.Connection(DB_ADDR,DB_PORT)
db = con.mydb
db.authenticate(READWRITE,READWRITE_PASSWD)
except:
traceback.print_exc()
logger.critical('Database connect error, exit.')
exit()
return db
def connect_readonly():
try:
from load_embeddings import load_word_vectors
from my_neural import CNNClassifier
import matplotlib.pyplot as plt
import copy
torch.manual_seed(1)
BATCH_SIZE = 15
EPOCHS = 20
vec_size = 300
datasets = "laptop"
default_embed_path = "word_embeds/amazon%s.txt" % vec_size
default_train_path = "train_data/reviews_Electronics_5.json.gz"
word2idx, idx2word, embeddings = load_word_vectors(default_embed_path,
vec_size)
logging = set_logger("slot3.csv")
logging.debug(
"Epoch,Train acuracy, train f1, train loss, test accuracy, test f1, test Loss"
)
###############################################################################################
train_sentences, emotion_for_sentence, all_categories_train = [], [], []
quota = [0.4, 0.2, 0.4]
max_sentences = 50000
sentences = MySentences(default_train_path, quota, max_sentences)
train_sentences, emotion_for_sentence, all_categories_train = sentences.get_sentiment(
)
import pickle
with open('dict.pickle', 'rb') as handle:
unserialized_data = pickle.load(handle)
a_e = unserialized_data
import pymongo
import ConfigParser
import traceback
import tools
from apk_spider.items import APKItem
config_file = os.getcwd()+'/apk_spider/spiders/spider_config.ini'
cf = ConfigParser.ConfigParser()
cf.read(config_file)
DB_ADDR = cf.get('mongodb','addr')
DB_PORT = int(cf.get('mongodb','port'),10)
DB_READWRITE_USER = cf.get('mongodb','readwrite_user')
DB_READWRITE_PASSWD = cf.get('mongodb','readwrite_password')
DB_READ_USER = cf.get('mongodb','read_user')
DB_READ_PASSWORD = cf.get('mongodb','read_password')
g_logger = tools.set_logger('spider_mongo',os.path.join(os.getcwd()+'/log/',cf.get('spider','log_path')))
def connect_readwrite():
try:
con = pymongo.Connection(DB_ADDR,DB_PORT)
db = con.mydb
db.authenticate(DB_READWRITE_USER,DB_READWRITE_PASSWD)
g_logger.info('Database connected.')
except:
traceback.print_exc()
g_logger.critical('Databse connect error,exit.')
return db
def insert_to_download_candidate_list2(url,urlhash,workerid,download_status,time):
db = connect_readwrite()
collection = db.apk_url_download_candidate_list
print "fail to create: %s" %(LOG_PATH)
sys.exit(1)
if not os.path.isfile(CFG_PATH):
print "no config.ini"
sys.exit(1)
cf = ConfigParser.ConfigParser()
cf.read(CFG_PATH)
TMP_NAME = cf.get("downloader", "tmp_path")
LOG_NAME = cf.get("misc", "log_file")
TMP_FOLDER = os.path.join(STORE_PATH, TMP_NAME)
LOG_FILE = os.path.join(LOG_PATH, LOG_NAME)
logger = set_logger("check cons", LOG_FILE)
logger.info("@=====check cons start work.")
if not os.path.exists(TMP_FOLDER):
logger.warn("%s: not exist." %(TMP_FOLDER))
try:
logger.info("Try to create: %s." %(TMP_FOLDER))
os.makedirs(TMP_FOLDER)
except:
logger.critical("Fail to create temp directory. Exit.")
sys.exit(1)
def get_file_list():
res = []
db = mongodb.connect_readwrite()
datasets = "laptop"
default_embed_path = "word_embeds/amazon%s.txt" % vec_size
if datasets == "rest":
default_train_path = "train_data/ABSA16_Restaurants_Train_SB1_v2.xml"
default_test_path = "test_data/EN_REST_SB1_TEST.xml.B"
else:
default_train_path = "train_data/train_english_german1.xml"
default_test_path = "test_data/EN_LAPT_SB1_TEST_.xml.B"
# get training data and transform them to word vectors
cleared = Sanitize(default_train_path)
word2idx, idx2word, embeddings = load_word_vectors(default_embed_path,
vec_size)
logging = set_logger("%s %s embeddings.csv" % (datasets, vec_size))
logging.debug("{} , {}".format(_hparams, BATCH_SIZE))
logging.debug(
"Epoch,Train acuracy, train f1, train loss, test accuracy, test f1, test Loss, general loss"
)
###############################################################################################
train_sentences, train_opinions_sentences = cleared.sanitize(
) # critique and opinion to said critique.
all_categories_train, emotion_for_sentence = cleared.extract_from_text(
train_opinions_sentences)
test_cleared = Sanitize(default_test_path)
test_sentences, test_opinions_sentences = test_cleared.sanitize()
max_words = cleared.get_max(train_sentences + test_sentences)
# TEST DATA
if not os.path.isfile(cf_path):
print "no config.ini"
sys.exit(1)
cf = ConfigParser.ConfigParser()
cf.read(cf_path)
MAL_NAME = cf.get("store", "mal_folder")
NOR_NAME = cf.get("store", "nor_folder")
BUF_LIMIT = cf.get("store", "buf_limit")
LOG_NAME = cf.get("store", "log_file")
LOG_FILE = os.path.join(LOG_PATH, LOG_NAME)
MAL_FOLDER = os.path.join(STORE_PATH, MAL_NAME)
NOR_FOLDER = os.path.join(STORE_PATH, NOR_NAME)
logger = set_logger("store", LOG_FILE)
logger.info("@====store start work.")
def get_positive_file_list():
res = []
db = mongodb.connect_readonly()
if not db:
logger.critical("DB error. Exit.")
sys.exit(1)
collection = db.file_info_list
cur = collection.find(
{"archive_flag":"none", "type_tag":{"$ne":"none"}},
{"_id":0})
length = cur.count()
if length < BUF_LIMIT:
pass
import string
import traceback
import time
from DBOperator import DBOperator
from FtpScanner import FtpScanner
config_file = os.getcwd() + '/config.ini'
cf = ConfigParser.ConfigParser()
cf.read(config_file)
SERVER_ADDR = cf.get('client','server_addr')
SERVER_PORT = int(cf.get('client','server_port'),10)
IP_STEP = int(cf.get('client','step'),10)
TIMEOUT = int(cf.get('client','timeout'),10)
INTERVAL = int(cf.get('client','interval'),10)
LOG_FILE = os.path.join(os.getcwd(),cf.get('support','client_log'))
logger = tools.set_logger('worker.py',LOG_FILE)
def worker():
global ip_list
global result_map
global local_ip
ip_list = []
result_map = dict()
local_ip = tools.get_local_ip('eth0')
# print local_ip,SERVER_ADDR,SERVER_PORT
conn = myconnect(SERVER_ADDR,SERVER_PORT)
time.sleep(2)
tmp = {}
tmp['cmd'] = 'add_client'
cf = ConfigParser.ConfigParser()
cf.read(CFG_PATH)
BUF_LIMIT = int(cf.get("downloader", "buf_limit"), 10)
SIZE_LIMIT = int(cf.get("downloader", "size_limit"), 10)
RETRY_TIME = cf.get("downloader", "retry_time")
TMP_NAME = cf.get("downloader", "tmp_path")
LOG_NAME = cf.get("downloader", "log_file")
CONCURRENT = int(cf.get("downloader", "concurrent"))
TMP_FOLDER = os.path.join(STORE_PATH, TMP_NAME)
LOG_FILE = os.path.join(LOG_PATH, LOG_NAME)
PID_FILE = os.path.join(LOG_PATH, "pidfile")
logger = set_logger("downloader", LOG_FILE)
logger.info("@=====Downloader start work.")
if not os.path.exists(TMP_FOLDER):
logger.warn("%s: not exist." %(TMP_FOLDER))
try:
logger.info("Try to create: %s." %(TMP_FOLDER))
os.makedirs(TMP_FOLDER)
except:
logger.critical("Fail to create temp directory. Exit.")
sys.exit(1)
class UrlHandler(threading.Thread):
def __init__(self, url):
import socket
import os
import pymongo
import ConfigParser
from tools import set_logger
import traceback
config_file = os.getcwd() + '/config.ini'
cf = ConfigParser.ConfigParser()
cf.read(config_file)
DB_ADDR = cf.get('mongodb','addr')
DB_PORT = int(cf.get('mongodb','port'),10)
READWRITE_USER = cf.get('mongodb','readwrite_user')
READWRITE_PASSWD = cf.get('mongodb','readwrite_password')
LOG_FILE = os.path.join(os.getcwd(),cf.get('support','log_path'))
logger = set_logger('DBOperator.py',LOG_FILE)
class DBOperator:
def __init__(self):
try:
self.connection = pymongo.Connection(DB_ADDR,DB_PORT)
#Database
self.db = self.connection.ftp
# logger.debug('Connected to ftp')
self.db.authenticate(READWRITE_USER,READWRITE_PASSWD)
# logger.debug('Database authenticate success!')
except:
# print 'Database connect error!'
traceback.print_exc()
logger.error('Database connect error,exit.')
exit(0)
import socket
import json
import threading
import thread
import datetime
config_file = os.getcwd() + '/config.ini'
cf = ConfigParser.ConfigParser()
cf.read(config_file)
MIN_CLIENTS = int(cf.get('server','min_clients'),10)
TIMEOUT = int(cf.get('server','timeout'),10)
PORT = int(cf.get('server','port'),10)
IP_STEP = int(cf.get('server','step'),10)
INTERVAL = int(cf.get('server','interval'),10)
LOG_FILE = os.path.join(os.getcwd(),cf.get('support','log_file'))
logger = tools.set_logger('Master2.py',LOG_FILE)
def master():
str_ip_start = sys.argv[1]
str_ip_end = sys.argv[2]
client_list = []
client_num = 0
t = threading.Thread(target = serv, name = 'listener')
t.setDaemon(1)
t.start()
logger.info('Start listen thread.')
t2 = threading.Thread(target = allocate, name = 'allocate_ips')
t2.setDaemon(1)
t2.start()
cf.read(CFG_PATH)
APIKEY = cf.get("scanner", "apikey")
URL = cf.get("scanner", "url")
INTERVAl = int(cf.get("scanner", "interval"), 10)
BUF_LIMIT = int(cf.get("scanner", "buf_limit"), 10)
LOG_NAME = cf.get("scanner", "log_file")
LOG_FILE = os.path.join(LOG_PATH, LOG_NAME)
PID_FILE = os.path.join(LOG_PATH, "pidfile")
HOST = cf.get("scanner", "host")
SELECTOR = cf.get("scanner", "selector")
FIELDS = [("apikey",APIKEY)]
logger = set_logger("scanner", LOG_FILE)
logger.info("@====scanner start work.")
JUDGERS = ["Emsisoft", "ESET-NOD32", "Kaspersky", "Kingsoft"]
SIZE = 20
RESCAN = False
def get_file_list():
global RESCAN
res = []
db = mongodb.connect_readonly()
if not db:
logger.critical("DB error. Exit.")
sys.exit(1)
collection = db.file_info_list
import time
import traceback
import string
from DBOperator import DBOperator
config_file = os.getcwd() + '/config.ini'
cf = ConfigParser.ConfigParser()
cf.read(config_file)
MIN_CLIENTS = int(cf.get('server','min_clients'),10)
TIMEOUT = int(cf.get('server','timeout'),10)
HOST = cf.get('server','ip')
PORT = int(cf.get('server','port'),10)
IP_STEP = int(cf.get('server','step'),10)
INTERVAL = int(cf.get('server','interval'),10)
LOG_FILE = os.path.join(os.getcwd(),cf.get('support','log_file'))
logger = tools.set_logger('master3.py',LOG_FILE)
def master():
global str_ip_start
str_ip_start = sys.argv[1]
global str_ip_end
str_ip_end = sys.argv[2]
global client_list
client_list = []
# global client_num
# client_num = 0
global start_time
start_time = tools.get_current_time()
global end_time
class testcase(unittest.TestCase):
logger = tools.set_logger(__name__)
sdk = tools.get_api()
model = tools.get_model()
filters = tools.get_filter()
timeout = tools.read_job_config()['case_timeout']
filter_count = int(tools.read_job_config()['filter_count'])
unfilter_count = int(tools.read_job_config()['unfilter_count'])
def setUp(self):
self.timeout_flag = None
self.flag1 = 0
self.flag2 = 0
self.logger.info('测试chip0不加过虑主动扫描，chip1 filter mac 主动扫描')
self.timer = Timer(self.timeout, self.set_timeout)
self.timer.start()
def tearDown(self):
self.timer.cancel()
# 测试方法
# noinspection PyUnreachableCode,PyUnreachableCode
def test_scan_filter_mac(self):
if self.model.startswith('S') or self.model.startswith('s'):
a = threading.Thread(target=self.chip0_scan, args=(1, ))
a.setDaemon(True)
a.start()
b = threading.Thread(target=self.chip0_scan,
args=(1, self.filters['filter_mac']))
b.setDaemon(True)
b.start()
while True:
if self.flag1 == 2:
self.assertTrue(True)
self.logger.info('pass\n')
break
elif self.timeout_flag:
self.logger.info('fail\n')
self.fail('Case failed,case test timeout.')
self.logger.error("Case failed,case test timeout.")
break
else:
a = threading.Thread(target=self.chip0_scan, args=(1, ))
b = threading.Thread(target=self.chip1_scan,
args=(1, self.filters['filter_mac']))
a.setDaemon(True)
b.setDaemon(True)
a.start()
b.start()
while True:
if self.flag1 == 1 and self.flag2 == 1:
self.assertTrue(True)
self.logger.info('pass\n')
break
elif self.timeout_flag:
self.logger.info('fail\n')
self.fail('Case failed,start scan timeout.')
self.logger.error("Case failed,start scan timeout.")
break
def chip0_scan(self, active=0, filter_mac=None):
# step1:chip 1 start passive scan,then start chip0 scan.
with closing(
self.sdk.scan(chip=0, active=active,
filter_mac=filter_mac)) as self.sse1:
count = 0
for message in self.sse1:
if message.startswith('data'):
msg = json.loads(message[5:])
if filter_mac:
# 进入开启过滤的扫描结果判断流程
if count < self.filter_count:
print('chip0', count, message)
mac = msg['bdaddrs'][0]['bdaddr']
if mac != self.filters['filter_mac']:
self.fail('filter mac failed.')
self.logger.debug('filter mac failed.')
break
else:
count += 1
else:
self.flag1 += 1
self.logger.debug(
'Step 1:chip0 start scan with filter mac success.')
break
else:
# 进入不开启过滤的扫描结果判断流程
if count < self.unfilter_count:
print('chip0', count, message)
count += 1
else:
self.flag1 += 1
self.logger.debug(
'Step 1:chip0 start scan with no filter mac success.'
)
break
# noinspection PyUnreachableCode
def chip1_scan(self, active=0, filter_mac=None):
# step2:start chip0 scan.
with closing(
self.sdk.scan(chip=1, active=active,
filter_mac=filter_mac)) as self.sse2:
count = 0
for message in self.sse2:
if message.startswith('data'):
msg = json.loads(message[5:])
if filter_mac:
if count < self.filter_count:
print('chip1', count, message)
mac = msg['bdaddrs'][0]['bdaddr']
if mac != filter_mac:
self.fail('filter mac failed.')
self.logger.debug('filter mac failed.')
break
else:
count += 1
else:
self.flag2 += 1
self.logger.debug(
'Step 2:chip1 start scan with filter mac success.'
)
break
else:
# 进入不开启过滤的扫描结果判断流程
if count < self.unfilter_count:
print('chip1', count, message)
count += 1
else:
self.flag2 += 1
self.logger.debug(
'Step 2:chip1 start scan with no filter mac success.'
)
break
def set_timeout(self):
self.timeout_flag = True
sys.exit(1)
if not os.path.isfile(CFG_PATH):
print colors.red("no config.ini")
sys.exit(1)
cf = ConfigParser.ConfigParser()
cf.read(CFG_PATH)
DISK_LOW_LIMIT = int(cf.get("disk", "low_limit"), 10)
DISK_RSD_LIMIT = int(cf.get("disk", "rsd_limit"), 10)
PID_FILE = os.path.join(LOG_PATH, "pidfile")
LOG_FILE = os.path.join(LOG_PATH, "monitor.log")
logger = set_logger("monitor", LOG_FILE)
logger.info("@=====monitor start work.")
def get_freespace(dir_path):
space_avaliable = 0
if hasattr(os, "statvfs"):
dir_status = os.statvfs(dir_path)
space_avaliable = dir_status.f_bavail * dir_status.f_frsize
space_avaliable /= 1024 * 1024
return (space_avaliable, "MB")
else:
logger.critical("no statvfs supported. exit.")
sys.exit(1)
def stop_downloader():
flag = False
cf_path = os.path.join(os.getcwd(), CFG)
if not os.path.isfile(cf_path):
print "no config.ini"
sys.exit(1)
cf = ConfigParser.ConfigParser()
cf.read(cf_path)
APIKEY = cf.get("scan_read", "apikey")
URL = cf.get("scan_read", "url")
INTERVAl = int(cf.get("scan_read", "interval"), 10)
BUF_LIMIT = int(cf.get("scan_read", "buf_limit"), 10)
LOG_NAME = cf.get("scan_read", "log_file")
LOG_FILE = os.path.join(LOG_PATH, LOG_NAME)
logger = set_logger("scan_read", LOG_FILE)
logger.info("@====scan_read start work.")
def get_file_md5():
res = []
db = mongodb.connect_readonly()
if not db:
logger.critical("DB error. Exit.")
sys.exit(1)
collection = db.file_info_list
cur = collection.find(
{"scan_status":"success", "type_tag":"none"},
{"_id":0})
length = cur.count()
if not length:
import socket
import os
import pymongo
import ConfigParser
import time
import threading
#import thread
import tools
import traceback
DEBUG = 2
config_file = os.getcwd() + '/config.ini'
cf = ConfigParser.ConfigParser()
cf.read(config_file)
LOG_FILE = os.path.join(os.getcwd(),cf.get('support','log_path'))
logger = tools.set_logger('FtpScanner.py',LOG_FILE)
ISOTIMEFORMAT = '%Y-%m-%d %X'
class FtpScanner:
def __init__(self,port=21,timeout=5,thread_num=1024):
self.port = port
self.timeout = timeout
self.thread_pool = []
self.thread_num = thread_num
self.current_threads = 0
self.ip_list = []
self.result_map = []
self.lock = threading.Lock()
sys.exit(1)
cf = ConfigParser.ConfigParser()
cf.read(cf_path)
APIKEY = cf.get("scan_upload", "apikey")
HOST = cf.get("scan_upload", "host")
SELECTOR = cf.get("scan_upload", "selector")
FIELDS = [("apikey",APIKEY)]
INTERVAl = int(cf.get("scan_upload", "interval"), 10)
BUF_LIMIT = int(cf.get("scan_upload", "buf_limit"), 10)
LOG_NAME = cf.get("scan_upload", "log_file")
LOG_FILE = os.path.join(LOG_PATH, LOG_NAME)
logger = set_logger("scan_upload", LOG_FILE)
logger.info("@====scan_upload start work.")
SIZE = 30
def get_file_list():
res = []
db = mongodb.connect_readonly()
if not db:
logger.error("DB error. Exit.")
sys.exit(1)
collection = db.file_info_list
cur = collection.find({"scan_status":"none"}, {"_id":0})
length = cur.count()
if length < BUF_LIMIT:
pass
print "fail to create: %s" %(LOG_PATH)
sys.exit(1)
if not os.path.isfile(CFG_PATH):
print "no config.ini"
sys.exit(1)
cf = ConfigParser.ConfigParser()
cf.read(CFG_PATH)
TMP_NAME = cf.get("downloader", "tmp_path")
LOG_NAME = cf.get("cleaner", "log_file")
TMP_FOLDER = os.path.join(STORE_PATH, TMP_NAME)
LOG_FILE = os.path.join(LOG_PATH, LOG_NAME)
logger = set_logger("cleaner", LOG_FILE)
logger.info("@=====cleaner start work.")
if not os.path.exists(TMP_FOLDER):
logger.warn("%s: not exist." %(TMP_FOLDER))
try:
logger.info("Try to create: %s." %(TMP_FOLDER))
os.makedirs(TMP_FOLDER)
except:
logger.critical("Fail to create temp directory. Exit.")
sys.exit(1)
def walk_dir(root_path):
length = len(root_path)
file_list = []
list_dir = os.walk(root_path)
import sys
import ConfigParser
from mongodb import get_static_info_key
from tools import set_logger
config_file = os.getcwd() + '/config.ini'
cf = ConfigParser.ConfigParser()
cf.read(config_file)
DB_ADDR = cf.get('mongodb','addr')
DB_PORT = int(cf.get('mongodb','port'),10)
READONLY = cf.get('mongodb','readonly_user')
READONLY_PASSWD = cf.get('mongodb','readonly_password')
READWRITE = cf.get('mongodb','readwrite_user')
READWRITE_PASSWD = cf.get('mongodb','readwrite_password')
LOG_FILE = os.path.join(os.getcwd(), cf.get('support','log_path'))
logger = set_logger('search.py', LOG_FILE)
def get_compiler(hashvalue):
res = get_static_info_key("hashvalue", hashvalue, "compiler")
if not res:
return "NULL"
return res["compiler"]
if __name__ == "__main__":
if len(sys.argv) != 2:
logger.info("usage: program <hashvalue>")
sys.exit(1)
else:
hashvalue = sys.argv[1]
res = get_compiler(hashvalue)
logger.info("compiler: {0}".format(res))
#!/usr/bin/python
import os
from tools import set_logger
import socket
import json
import threading
import datetime
import string
config_file = os.getcwd() + '/config.ini'
cf = ConfigParser.ConfigParser()
cf.read(config_filei)
MIN_CLIENTS = int(cf.get('server','min_clients'),10)
LOG_FILE = os.path.join(os.getcwd(),cf.get('support','log_file'))
logger = set_logger('Master.py',LOG_FILE)
class Client;
def __init__(self, ip, level = 1, status = True):
self.ip = ip
self.level = level
sefl.status = status
def set_status(self,status):
self.status = status
def get_status(self,status):
return self.status
class Master:
def __init__(self):
self.clients = get_clients()
self.client_num = self.clients.length()
