class TestDict(unittest.TestCase):
def setUp(self):
self.keys = {'a', 'b', 'c', 'd'}
self.values = {'1', '2', '3', '4'}
self.dict = MyDict()
def testDict(self):
for i in range(0, len(self.keys)):
self.dict.put(self.keys.pop(), self.values.pop())
for i in range(0, len(self.keys)):
self.assertEqual(self.dict.get[self.keys[i]], self.values[i])
def testKey(self):
myDictTest = MyDict()
myDictTest['key'] = 'value'
self.assertEqual(myDictTest.key, 'value')
def testInit(self):
myDictTest = MyDict(a=1, b='test')
self.assertEqual(myDictTest.a, 1)
self.assertEqual(myDictTest.b, 'test')
self.assertIsInstance(myDictTest, dict)
f.close()
return data
def fsave(filename, data):
f = open(filename, "w")
pickle.dump(data, f)
f.close()
return
## Test Main
if __name__ == "__main__":
dictfile = 'dict.dat'
configFile = 'analog.ini'
acceptFile = 'accept.dat' ## 採用するメッセージid一覧
dict = MyDict(dictfile)
dict.loadict()
print 'dict count: %d' % dict.countDictWords()
## RNNメッセージID化
vocab=10000
dim=100
y=94
train = Trainer(vocab, dim, y)
train.load('train')
labellist = [] ## ラベル一覧(出力)
#msglist = [] ## メッセージ一覧（無視リスト作成のため使用メッセージを保存）
analizeRnnInputList = []
analizeRnnLabelList = []
def test_attrerror(self):
d = MyDict()
with self.assertRaises(AttributeError):
value = d.empty
def test_attr(self):
d = MyDict()
d.key = 'value'
self.assertTrue('key' in d)
def setUp(self):
self.keys = {'a', 'b', 'c', 'd'}
self.values = {'1', '2', '3', '4'}
self.dict = MyDict()
def testAttrError(self):
myDictTest = MyDict()
with self.assertRaises(AttributeError):
value = myDictTest.empty
source:
https://github.com/DQinYuan/AndroidMalwareWithN-gram
modified by Charlesdu
'''
from mydict import MyDict
import pandas as pd
#n-gram
n = 3
origin = pd.read_csv("gram_data.csv")
mdict = MyDict()
feature = origin["Feature"].str.split("|")
total = len(feature)
print(total)
for i, code in enumerate(feature):
mdict.newLayer()
if not type(code) == list:
continue
for method in code:
length = len(method)
if length < n:
continue
def test_keyerror(self):
d = MyDict()
with self.assertRaises(KeyError):
value = d['empty']
def test_attr(self):
d = MyDict()
d.key = 'value'
self.assertTrue('key' in d)
self.assertEqual(d['key'], 'value')
return self.fit_obj.transform(data)
## Test Main
if __name__ == "__main__":
from prettyprint import pp
from mydict import MyDict
import sys
arg1 = sys.argv[1] if len(sys.argv) == 2 else None
## 学習
if arg1 == 'train':
print "training..."
dictfile = 'dict.dat'
dict = MyDict(dictfile)
dict.loadict()
lids = dict.loadtext('jp.txt')
dict.savedict()
#vocab = dict.countDictWords()
vocab = 10000
dim = 100
y = len(lids)
print 'vocab=%d, dim=%d, y=%d' % (vocab, dim, y)
train = Trainer(vocab, dim, y)
epoch = 1000
for j in range(epoch):
i = 0
for ids in lids:
def __init__(self):
import pickle
self.pos = MyDict()
self.neg = MyDict()
self.tot_pos, self.pos = pickle.load(open('pos_file', 'rb'))
self.tot_neg, self.neg = pickle.load(open('neg_file', 'rb'))
def test_setattr2(self):
d = MyDict()
d.key = "value"
self.assertTrue('key' in d)
self.assertEqual(d['key'],"value")
def test_setattr(self):
d = MyDict()
d["key"]="value"
self.assertEqual(d.key,"value")
def testValue(self):
myDictTest = MyDict()
myDictTest.key = 'value'
self.assertTrue('key' in myDictTest)
self.assertEqual(myDictTest['key'], 'value')
def testKeyError(self):
myDictTest = MyDict()
with self.assertRaises(KeyError):
value = myDictTest['empty']
def test_init(self):
d = MyDict(a=1, b='test')
self.assertEqual(d.a, 1)
self.assertEqual(d.b, 'test')
self.assertTrue(isinstance(d, dict))
def predict(self, data):
inmsg = data.split('\n')
#inmsg = ['エージェントが可能な変更を行った後動作を開始しました']
log.debug('predict: predict data=%s', inmsg)
dictfile = self.model_path + '/dict.dat'
configFile = self.model_path + '/analog.ini'
dict = MyDict(dictfile)
dict.loadict()
print 'dict count: %d' % dict.countDictWords()
## 採用メッセージID
acceptid = self.fload(self.model_path + '/accept.dat')
print '**accept id:'
print acceptid
## エラー定義の読み込み
with open(self.model_path + '/error.json', 'r') as f:
errjson = json.load(f, "utf-8")
## RNNメッセージID化
vocab = 10000
dim = 100
y = 94
train = Trainer(vocab, dim, y)
train.load(self.model_path + '/train')
labellist = [] ## ラベル一覧(出力)
#msglist = [] ## メッセージ一覧（無視リスト作成のため使用メッセージを保存）
analizeRnnInputList = []
analizeRnnLabelList = []
msgidlist = []
for msgtxt in inmsg:
## wakachiで処理するため明示的にunicodeにする
## ex) cmapeerdが停止したためプロセスを再起動します
#w = u'' + msgtxt
#words = dict.wakachi(w.encode('utf-8'))
words = dict.wakachi(msgtxt)
ids = dict.words2ids(words)
## 改行のみはスキップ
if ids == []:
continue
## ex) [113, 63, 11, 9, 13, 104, 6, 7, 66, 8, 9, 10]
y = train.predict(ids[::-1])
wordid = y.data.argmax(1)[0] ## ex) 84
## エラーリストにあるメッセージ以外は処理にしない
if wordid in acceptid:
msgidlist.append(wordid)
else:
print 'not accept id:%d, msg:%s' % (wordid, msgtxt)
continue
print '** msg prdict %d, %2f' % (wordid, y.data[0][wordid]
) ## 選択されたidの確率表示
pp(inmsg)
pp(msgidlist)
## 対象メッセージが無い
if msgidlist == []:
result = [{'score': 100, 'id': None, 'label': None}]
return result
## エラーケースの予測
config = ConfigParser.SafeConfigParser()
config.read(configFile)
dim_in = config.getint('analize', 'dim_in')
dim_mid = config.getint('analize', 'dim_mid')
dim_out = config.getint('analize', 'dim_out')
"""
train = AnazlizeTrainer(dim_in, dim_mid, dim_out)
train.load(self.model_path + '/train_analize')
y = train.predict(msgidlist[::-1])
print y.data.argmax(1)[0]
rank = y.data.argsort()[0]
uprank = map(int, rank[::-1])
print uprank
#print y.data[0]
"""
train = AnazlizeTrainer(dim_in, dim_mid, dim_out)
train.load(self.model_path + '/train_analize')
#y = train.predict(msgidlist[::-1])
targetlist = msgidlist
resultlist = np.zeros(dim_out) ## 確率結果の最大値を格納する配列
print resultlist
for i in range(len(targetlist)):
target = targetlist[i:]
y = train.predict(target[::-1])
print target
print y.data[0]
for i in range(len(y.data[0])):
if y.data[0][i] > resultlist[i]:
resultlist[i] = y.data[0][i]
print resultlist
#print y.data.argmax(1)[0]
#rank = y.data.argsort()[0]
rank = resultlist.argsort()
uprank = map(int, rank[::-1])
print uprank
#print y.data[0]
result = []
for i in uprank:
print '%d, %2f' % (i, resultlist[i])
item = {
'score': round(float(resultlist[i]) * 100, 2),
'id': i,
'label': errjson[i]['label']
}
result.append(item)
return result
def test_key(self):
d = MyDict()
d['key'] = 'value'
self.assertEqual(d.key, 'value')