import os from tools import set_logger import ConfigParser from datetime import datetime config_file = os.getcwd() + '/config.ini' cf = ConfigParser.ConfigParser() cf.read(config_file) DB_ADDR = cf.get('mongodb','addr') DB_PORT = int(cf.get('mongodb','port'),10) READONLY = cf.get('mongodb','readonly_user') READONLY_PASSWD = cf.get('mongodb','readonly_password') READWRITE = cf.get('mongodb','readwrite_user') READWRITE_PASSWD = cf.get('mongodb','readwrite_password') LOG_FILE = os.path.join(os.getcwd(), cf.get('support','log_path')) logger = set_logger('mongodb.py', LOG_FILE) '==数据库连接操作==' def connect_readwrite(): try: con = pymongo.Connection(DB_ADDR,DB_PORT) db = con.mydb db.authenticate(READWRITE,READWRITE_PASSWD) except: traceback.print_exc() logger.critical('Database connect error, exit.') exit() return db def connect_readonly(): try:
from load_embeddings import load_word_vectors from my_neural import CNNClassifier import matplotlib.pyplot as plt import copy torch.manual_seed(1) BATCH_SIZE = 15 EPOCHS = 20 vec_size = 300 datasets = "laptop" default_embed_path = "word_embeds/amazon%s.txt" % vec_size default_train_path = "train_data/reviews_Electronics_5.json.gz" word2idx, idx2word, embeddings = load_word_vectors(default_embed_path, vec_size) logging = set_logger("slot3.csv") logging.debug( "Epoch,Train acuracy, train f1, train loss, test accuracy, test f1, test Loss" ) ############################################################################################### train_sentences, emotion_for_sentence, all_categories_train = [], [], [] quota = [0.4, 0.2, 0.4] max_sentences = 50000 sentences = MySentences(default_train_path, quota, max_sentences) train_sentences, emotion_for_sentence, all_categories_train = sentences.get_sentiment( ) import pickle with open('dict.pickle', 'rb') as handle: unserialized_data = pickle.load(handle) a_e = unserialized_data
import pymongo import ConfigParser import traceback import tools from apk_spider.items import APKItem config_file = os.getcwd()+'/apk_spider/spiders/spider_config.ini' cf = ConfigParser.ConfigParser() cf.read(config_file) DB_ADDR = cf.get('mongodb','addr') DB_PORT = int(cf.get('mongodb','port'),10) DB_READWRITE_USER = cf.get('mongodb','readwrite_user') DB_READWRITE_PASSWD = cf.get('mongodb','readwrite_password') DB_READ_USER = cf.get('mongodb','read_user') DB_READ_PASSWORD = cf.get('mongodb','read_password') g_logger = tools.set_logger('spider_mongo',os.path.join(os.getcwd()+'/log/',cf.get('spider','log_path'))) def connect_readwrite(): try: con = pymongo.Connection(DB_ADDR,DB_PORT) db = con.mydb db.authenticate(DB_READWRITE_USER,DB_READWRITE_PASSWD) g_logger.info('Database connected.') except: traceback.print_exc() g_logger.critical('Databse connect error,exit.') return db def insert_to_download_candidate_list2(url,urlhash,workerid,download_status,time): db = connect_readwrite() collection = db.apk_url_download_candidate_list
print "fail to create: %s" %(LOG_PATH) sys.exit(1) if not os.path.isfile(CFG_PATH): print "no config.ini" sys.exit(1) cf = ConfigParser.ConfigParser() cf.read(CFG_PATH) TMP_NAME = cf.get("downloader", "tmp_path") LOG_NAME = cf.get("misc", "log_file") TMP_FOLDER = os.path.join(STORE_PATH, TMP_NAME) LOG_FILE = os.path.join(LOG_PATH, LOG_NAME) logger = set_logger("check cons", LOG_FILE) logger.info("@=====check cons start work.") if not os.path.exists(TMP_FOLDER): logger.warn("%s: not exist." %(TMP_FOLDER)) try: logger.info("Try to create: %s." %(TMP_FOLDER)) os.makedirs(TMP_FOLDER) except: logger.critical("Fail to create temp directory. Exit.") sys.exit(1) def get_file_list(): res = [] db = mongodb.connect_readwrite()
datasets = "laptop" default_embed_path = "word_embeds/amazon%s.txt" % vec_size if datasets == "rest": default_train_path = "train_data/ABSA16_Restaurants_Train_SB1_v2.xml" default_test_path = "test_data/EN_REST_SB1_TEST.xml.B" else: default_train_path = "train_data/train_english_german1.xml" default_test_path = "test_data/EN_LAPT_SB1_TEST_.xml.B" # get training data and transform them to word vectors cleared = Sanitize(default_train_path) word2idx, idx2word, embeddings = load_word_vectors(default_embed_path, vec_size) logging = set_logger("%s %s embeddings.csv" % (datasets, vec_size)) logging.debug("{} , {}".format(_hparams, BATCH_SIZE)) logging.debug( "Epoch,Train acuracy, train f1, train loss, test accuracy, test f1, test Loss, general loss" ) ############################################################################################### train_sentences, train_opinions_sentences = cleared.sanitize( ) # critique and opinion to said critique. all_categories_train, emotion_for_sentence = cleared.extract_from_text( train_opinions_sentences) test_cleared = Sanitize(default_test_path) test_sentences, test_opinions_sentences = test_cleared.sanitize() max_words = cleared.get_max(train_sentences + test_sentences) # TEST DATA
if not os.path.isfile(cf_path): print "no config.ini" sys.exit(1) cf = ConfigParser.ConfigParser() cf.read(cf_path) MAL_NAME = cf.get("store", "mal_folder") NOR_NAME = cf.get("store", "nor_folder") BUF_LIMIT = cf.get("store", "buf_limit") LOG_NAME = cf.get("store", "log_file") LOG_FILE = os.path.join(LOG_PATH, LOG_NAME) MAL_FOLDER = os.path.join(STORE_PATH, MAL_NAME) NOR_FOLDER = os.path.join(STORE_PATH, NOR_NAME) logger = set_logger("store", LOG_FILE) logger.info("@====store start work.") def get_positive_file_list(): res = [] db = mongodb.connect_readonly() if not db: logger.critical("DB error. Exit.") sys.exit(1) collection = db.file_info_list cur = collection.find( {"archive_flag":"none", "type_tag":{"$ne":"none"}}, {"_id":0}) length = cur.count() if length < BUF_LIMIT: pass
import string import traceback import time from DBOperator import DBOperator from FtpScanner import FtpScanner config_file = os.getcwd() + '/config.ini' cf = ConfigParser.ConfigParser() cf.read(config_file) SERVER_ADDR = cf.get('client','server_addr') SERVER_PORT = int(cf.get('client','server_port'),10) IP_STEP = int(cf.get('client','step'),10) TIMEOUT = int(cf.get('client','timeout'),10) INTERVAL = int(cf.get('client','interval'),10) LOG_FILE = os.path.join(os.getcwd(),cf.get('support','client_log')) logger = tools.set_logger('worker.py',LOG_FILE) def worker(): global ip_list global result_map global local_ip ip_list = [] result_map = dict() local_ip = tools.get_local_ip('eth0') # print local_ip,SERVER_ADDR,SERVER_PORT conn = myconnect(SERVER_ADDR,SERVER_PORT) time.sleep(2) tmp = {} tmp['cmd'] = 'add_client'
cf = ConfigParser.ConfigParser() cf.read(CFG_PATH) BUF_LIMIT = int(cf.get("downloader", "buf_limit"), 10) SIZE_LIMIT = int(cf.get("downloader", "size_limit"), 10) RETRY_TIME = cf.get("downloader", "retry_time") TMP_NAME = cf.get("downloader", "tmp_path") LOG_NAME = cf.get("downloader", "log_file") CONCURRENT = int(cf.get("downloader", "concurrent")) TMP_FOLDER = os.path.join(STORE_PATH, TMP_NAME) LOG_FILE = os.path.join(LOG_PATH, LOG_NAME) PID_FILE = os.path.join(LOG_PATH, "pidfile") logger = set_logger("downloader", LOG_FILE) logger.info("@=====Downloader start work.") if not os.path.exists(TMP_FOLDER): logger.warn("%s: not exist." %(TMP_FOLDER)) try: logger.info("Try to create: %s." %(TMP_FOLDER)) os.makedirs(TMP_FOLDER) except: logger.critical("Fail to create temp directory. Exit.") sys.exit(1) class UrlHandler(threading.Thread): def __init__(self, url):
import socket import os import pymongo import ConfigParser from tools import set_logger import traceback config_file = os.getcwd() + '/config.ini' cf = ConfigParser.ConfigParser() cf.read(config_file) DB_ADDR = cf.get('mongodb','addr') DB_PORT = int(cf.get('mongodb','port'),10) READWRITE_USER = cf.get('mongodb','readwrite_user') READWRITE_PASSWD = cf.get('mongodb','readwrite_password') LOG_FILE = os.path.join(os.getcwd(),cf.get('support','log_path')) logger = set_logger('DBOperator.py',LOG_FILE) class DBOperator: def __init__(self): try: self.connection = pymongo.Connection(DB_ADDR,DB_PORT) #Database self.db = self.connection.ftp # logger.debug('Connected to ftp') self.db.authenticate(READWRITE_USER,READWRITE_PASSWD) # logger.debug('Database authenticate success!') except: # print 'Database connect error!' traceback.print_exc() logger.error('Database connect error,exit.') exit(0)
import socket import json import threading import thread import datetime config_file = os.getcwd() + '/config.ini' cf = ConfigParser.ConfigParser() cf.read(config_file) MIN_CLIENTS = int(cf.get('server','min_clients'),10) TIMEOUT = int(cf.get('server','timeout'),10) PORT = int(cf.get('server','port'),10) IP_STEP = int(cf.get('server','step'),10) INTERVAL = int(cf.get('server','interval'),10) LOG_FILE = os.path.join(os.getcwd(),cf.get('support','log_file')) logger = tools.set_logger('Master2.py',LOG_FILE) def master(): str_ip_start = sys.argv[1] str_ip_end = sys.argv[2] client_list = [] client_num = 0 t = threading.Thread(target = serv, name = 'listener') t.setDaemon(1) t.start() logger.info('Start listen thread.') t2 = threading.Thread(target = allocate, name = 'allocate_ips') t2.setDaemon(1) t2.start()
cf.read(CFG_PATH) APIKEY = cf.get("scanner", "apikey") URL = cf.get("scanner", "url") INTERVAl = int(cf.get("scanner", "interval"), 10) BUF_LIMIT = int(cf.get("scanner", "buf_limit"), 10) LOG_NAME = cf.get("scanner", "log_file") LOG_FILE = os.path.join(LOG_PATH, LOG_NAME) PID_FILE = os.path.join(LOG_PATH, "pidfile") HOST = cf.get("scanner", "host") SELECTOR = cf.get("scanner", "selector") FIELDS = [("apikey",APIKEY)] logger = set_logger("scanner", LOG_FILE) logger.info("@====scanner start work.") JUDGERS = ["Emsisoft", "ESET-NOD32", "Kaspersky", "Kingsoft"] SIZE = 20 RESCAN = False def get_file_list(): global RESCAN res = [] db = mongodb.connect_readonly() if not db: logger.critical("DB error. Exit.") sys.exit(1) collection = db.file_info_list
import time import traceback import string from DBOperator import DBOperator config_file = os.getcwd() + '/config.ini' cf = ConfigParser.ConfigParser() cf.read(config_file) MIN_CLIENTS = int(cf.get('server','min_clients'),10) TIMEOUT = int(cf.get('server','timeout'),10) HOST = cf.get('server','ip') PORT = int(cf.get('server','port'),10) IP_STEP = int(cf.get('server','step'),10) INTERVAL = int(cf.get('server','interval'),10) LOG_FILE = os.path.join(os.getcwd(),cf.get('support','log_file')) logger = tools.set_logger('master3.py',LOG_FILE) def master(): global str_ip_start str_ip_start = sys.argv[1] global str_ip_end str_ip_end = sys.argv[2] global client_list client_list = [] # global client_num # client_num = 0 global start_time start_time = tools.get_current_time() global end_time
class testcase(unittest.TestCase): logger = tools.set_logger(__name__) sdk = tools.get_api() model = tools.get_model() filters = tools.get_filter() timeout = tools.read_job_config()['case_timeout'] filter_count = int(tools.read_job_config()['filter_count']) unfilter_count = int(tools.read_job_config()['unfilter_count']) def setUp(self): self.timeout_flag = None self.flag1 = 0 self.flag2 = 0 self.logger.info('测试chip0不加过虑主动扫描,chip1 filter mac 主动扫描') self.timer = Timer(self.timeout, self.set_timeout) self.timer.start() def tearDown(self): self.timer.cancel() # 测试方法 # noinspection PyUnreachableCode,PyUnreachableCode def test_scan_filter_mac(self): if self.model.startswith('S') or self.model.startswith('s'): a = threading.Thread(target=self.chip0_scan, args=(1, )) a.setDaemon(True) a.start() b = threading.Thread(target=self.chip0_scan, args=(1, self.filters['filter_mac'])) b.setDaemon(True) b.start() while True: if self.flag1 == 2: self.assertTrue(True) self.logger.info('pass\n') break elif self.timeout_flag: self.logger.info('fail\n') self.fail('Case failed,case test timeout.') self.logger.error("Case failed,case test timeout.") break else: a = threading.Thread(target=self.chip0_scan, args=(1, )) b = threading.Thread(target=self.chip1_scan, args=(1, self.filters['filter_mac'])) a.setDaemon(True) b.setDaemon(True) a.start() b.start() while True: if self.flag1 == 1 and self.flag2 == 1: self.assertTrue(True) self.logger.info('pass\n') break elif self.timeout_flag: self.logger.info('fail\n') self.fail('Case failed,start scan timeout.') self.logger.error("Case failed,start scan timeout.") break def chip0_scan(self, active=0, filter_mac=None): # step1:chip 1 start passive scan,then start chip0 scan. with closing( self.sdk.scan(chip=0, active=active, filter_mac=filter_mac)) as self.sse1: count = 0 for message in self.sse1: if message.startswith('data'): msg = json.loads(message[5:]) if filter_mac: # 进入开启过滤的扫描结果判断流程 if count < self.filter_count: print('chip0', count, message) mac = msg['bdaddrs'][0]['bdaddr'] if mac != self.filters['filter_mac']: self.fail('filter mac failed.') self.logger.debug('filter mac failed.') break else: count += 1 else: self.flag1 += 1 self.logger.debug( 'Step 1:chip0 start scan with filter mac success.') break else: # 进入不开启过滤的扫描结果判断流程 if count < self.unfilter_count: print('chip0', count, message) count += 1 else: self.flag1 += 1 self.logger.debug( 'Step 1:chip0 start scan with no filter mac success.' ) break # noinspection PyUnreachableCode def chip1_scan(self, active=0, filter_mac=None): # step2:start chip0 scan. with closing( self.sdk.scan(chip=1, active=active, filter_mac=filter_mac)) as self.sse2: count = 0 for message in self.sse2: if message.startswith('data'): msg = json.loads(message[5:]) if filter_mac: if count < self.filter_count: print('chip1', count, message) mac = msg['bdaddrs'][0]['bdaddr'] if mac != filter_mac: self.fail('filter mac failed.') self.logger.debug('filter mac failed.') break else: count += 1 else: self.flag2 += 1 self.logger.debug( 'Step 2:chip1 start scan with filter mac success.' ) break else: # 进入不开启过滤的扫描结果判断流程 if count < self.unfilter_count: print('chip1', count, message) count += 1 else: self.flag2 += 1 self.logger.debug( 'Step 2:chip1 start scan with no filter mac success.' ) break def set_timeout(self): self.timeout_flag = True
sys.exit(1) if not os.path.isfile(CFG_PATH): print colors.red("no config.ini") sys.exit(1) cf = ConfigParser.ConfigParser() cf.read(CFG_PATH) DISK_LOW_LIMIT = int(cf.get("disk", "low_limit"), 10) DISK_RSD_LIMIT = int(cf.get("disk", "rsd_limit"), 10) PID_FILE = os.path.join(LOG_PATH, "pidfile") LOG_FILE = os.path.join(LOG_PATH, "monitor.log") logger = set_logger("monitor", LOG_FILE) logger.info("@=====monitor start work.") def get_freespace(dir_path): space_avaliable = 0 if hasattr(os, "statvfs"): dir_status = os.statvfs(dir_path) space_avaliable = dir_status.f_bavail * dir_status.f_frsize space_avaliable /= 1024 * 1024 return (space_avaliable, "MB") else: logger.critical("no statvfs supported. exit.") sys.exit(1) def stop_downloader(): flag = False
cf_path = os.path.join(os.getcwd(), CFG) if not os.path.isfile(cf_path): print "no config.ini" sys.exit(1) cf = ConfigParser.ConfigParser() cf.read(cf_path) APIKEY = cf.get("scan_read", "apikey") URL = cf.get("scan_read", "url") INTERVAl = int(cf.get("scan_read", "interval"), 10) BUF_LIMIT = int(cf.get("scan_read", "buf_limit"), 10) LOG_NAME = cf.get("scan_read", "log_file") LOG_FILE = os.path.join(LOG_PATH, LOG_NAME) logger = set_logger("scan_read", LOG_FILE) logger.info("@====scan_read start work.") def get_file_md5(): res = [] db = mongodb.connect_readonly() if not db: logger.critical("DB error. Exit.") sys.exit(1) collection = db.file_info_list cur = collection.find( {"scan_status":"success", "type_tag":"none"}, {"_id":0}) length = cur.count() if not length:
import socket import os import pymongo import ConfigParser import time import threading #import thread import tools import traceback DEBUG = 2 config_file = os.getcwd() + '/config.ini' cf = ConfigParser.ConfigParser() cf.read(config_file) LOG_FILE = os.path.join(os.getcwd(),cf.get('support','log_path')) logger = tools.set_logger('FtpScanner.py',LOG_FILE) ISOTIMEFORMAT = '%Y-%m-%d %X' class FtpScanner: def __init__(self,port=21,timeout=5,thread_num=1024): self.port = port self.timeout = timeout self.thread_pool = [] self.thread_num = thread_num self.current_threads = 0 self.ip_list = [] self.result_map = [] self.lock = threading.Lock()
sys.exit(1) cf = ConfigParser.ConfigParser() cf.read(cf_path) APIKEY = cf.get("scan_upload", "apikey") HOST = cf.get("scan_upload", "host") SELECTOR = cf.get("scan_upload", "selector") FIELDS = [("apikey",APIKEY)] INTERVAl = int(cf.get("scan_upload", "interval"), 10) BUF_LIMIT = int(cf.get("scan_upload", "buf_limit"), 10) LOG_NAME = cf.get("scan_upload", "log_file") LOG_FILE = os.path.join(LOG_PATH, LOG_NAME) logger = set_logger("scan_upload", LOG_FILE) logger.info("@====scan_upload start work.") SIZE = 30 def get_file_list(): res = [] db = mongodb.connect_readonly() if not db: logger.error("DB error. Exit.") sys.exit(1) collection = db.file_info_list cur = collection.find({"scan_status":"none"}, {"_id":0}) length = cur.count() if length < BUF_LIMIT: pass
print "fail to create: %s" %(LOG_PATH) sys.exit(1) if not os.path.isfile(CFG_PATH): print "no config.ini" sys.exit(1) cf = ConfigParser.ConfigParser() cf.read(CFG_PATH) TMP_NAME = cf.get("downloader", "tmp_path") LOG_NAME = cf.get("cleaner", "log_file") TMP_FOLDER = os.path.join(STORE_PATH, TMP_NAME) LOG_FILE = os.path.join(LOG_PATH, LOG_NAME) logger = set_logger("cleaner", LOG_FILE) logger.info("@=====cleaner start work.") if not os.path.exists(TMP_FOLDER): logger.warn("%s: not exist." %(TMP_FOLDER)) try: logger.info("Try to create: %s." %(TMP_FOLDER)) os.makedirs(TMP_FOLDER) except: logger.critical("Fail to create temp directory. Exit.") sys.exit(1) def walk_dir(root_path): length = len(root_path) file_list = [] list_dir = os.walk(root_path)
import sys import ConfigParser from mongodb import get_static_info_key from tools import set_logger config_file = os.getcwd() + '/config.ini' cf = ConfigParser.ConfigParser() cf.read(config_file) DB_ADDR = cf.get('mongodb','addr') DB_PORT = int(cf.get('mongodb','port'),10) READONLY = cf.get('mongodb','readonly_user') READONLY_PASSWD = cf.get('mongodb','readonly_password') READWRITE = cf.get('mongodb','readwrite_user') READWRITE_PASSWD = cf.get('mongodb','readwrite_password') LOG_FILE = os.path.join(os.getcwd(), cf.get('support','log_path')) logger = set_logger('search.py', LOG_FILE) def get_compiler(hashvalue): res = get_static_info_key("hashvalue", hashvalue, "compiler") if not res: return "NULL" return res["compiler"] if __name__ == "__main__": if len(sys.argv) != 2: logger.info("usage: program <hashvalue>") sys.exit(1) else: hashvalue = sys.argv[1] res = get_compiler(hashvalue) logger.info("compiler: {0}".format(res))
#!/usr/bin/python import os from tools import set_logger import socket import json import threading import datetime import string config_file = os.getcwd() + '/config.ini' cf = ConfigParser.ConfigParser() cf.read(config_filei) MIN_CLIENTS = int(cf.get('server','min_clients'),10) LOG_FILE = os.path.join(os.getcwd(),cf.get('support','log_file')) logger = set_logger('Master.py',LOG_FILE) class Client; def __init__(self, ip, level = 1, status = True): self.ip = ip self.level = level sefl.status = status def set_status(self,status): self.status = status def get_status(self,status): return self.status class Master: def __init__(self): self.clients = get_clients() self.client_num = self.clients.length()