def save_lime_coef(filename,
model_name,
SAVE_DIR,
train_dev_tokens,
test_tokens,
d_file=None):
model = 'models/{}.pkl'.format(filename)
path = utils.get_abs_path(SAVE_DIR, model)
if 'svm' in model_name:
model = utils.load_pickle(path, encoding=False)
else:
if model_name == 'lstm_att':
hp_d = 'models/{}.pkl'.format(d_file)
hp_path = utils.get_abs_path(SAVE_DIR, hp_d)
d = utils.load_pickle(hp_path)
model = init_model(train_dev_tokens, d, path)
else:
model = utils.load_pickle(path)
features_l, importance_l = get_lime(model, test_tokens, model_name)
features = 'features/{}_lime_all_features.pkl'.format(model_name)
path = utils.get_abs_path(SAVE_DIR, features)
utils.save_pickle(features_l, path)
scores = 'feature_importance/{}_lime_all_scores.pkl'.format(model_name)
path = utils.get_abs_path(SAVE_DIR, scores)
utils.save_pickle(importance_l, path)
def save_shap_val(hp_filename,
filename,
name,
SAVE_DIR,
train_data,
test_data,
test_labels,
use_gpu=True,
background_length=100,
padding_length=512):
hp_d = 'models/{}.pkl'.format(hp_filename)
hp_path = utils.get_abs_path(SAVE_DIR, hp_d)
d = utils.load_pickle(hp_path)
model_d = 'models/{}.pkl'.format(filename)
model_path = utils.get_abs_path(SAVE_DIR, model_d)
model = init_model(train_data, d, model_path, use_gpu=use_gpu)
features_l, importance_l = [], []
features = 'features/{}_shap_all_features.pkl'.format(name)
feature_path = utils.get_abs_path(SAVE_DIR, features)
scores = 'feature_importance/{}_shap_all_scores.pkl'.format(name)
model_path = utils.get_abs_path(SAVE_DIR, scores)
features_l, importance_l = get_lstm_shap(
model,
train_data,
test_data,
background_length=background_length,
padding_length=padding_length,
feature_path=feature_path,
model_path=model_path)
utils.save_pickle(features_l, feature_path)
utils.save_pickle(importance_l, model_path)
def get_prediction(test_tokens, model_name, save_dir, train_dev_tokens, test_labels, hp_name):
hp_path, pipeline, model, predictions = None, None, None, None
model_path = utils.get_abs_path(save_dir, 'models/{}.pkl'.format(model_name))
if 'deception' in save_dir and model_name == 'svm':
pipeline = utils.load_pickle(model_path, encoding=False)
elif model_name == 'bert':
pass
else:
pipeline = utils.load_pickle(model_path)
if model_name == 'bert':
if 'deception' in save_dir:
dataset_name = 'deception'
elif 'yelp' in save_dir:
dataset_name = 'yelp'
elif 'sst' in save_dir:
dataset_name = 'sst'
path = '/data/BERT_att_weights/{}-bert-preds.npy'.format(dataset_name)
predictions = np.load(path)
elif model_name == 'lstm_att':
hp_path = utils.get_abs_path(save_dir, 'models/{}.pkl'.format(hp_name))
d = utils.load_pickle(hp_path)
model = init_model(train_dev_tokens, d, model_path)
tokens = split_tokens(test_tokens)
mapping = [model.get_words_to_ids(l) for l in tokens]
predictions = model.predict(tokens, mapping)
else:
predictions, accuracy = sc.heldout_test(pipeline, test_tokens, test_labels)
assert len(predictions) == len(test_tokens)
return predictions
def save_att_weights(word_score_ds, save_dir):
features_l, importance_l = get_att_weights(word_score_ds)
features_file_name = 'features/lstm_att_weights_all_features.pkl'
path = utils.get_abs_path(save_dir, features_file_name)
utils.save_pickle(features_l, path)
scores_file_name = 'feature_importance/lstm_att_weights_all_scores.pkl'
path = utils.get_abs_path(save_dir, scores_file_name)
utils.save_pickle(importance_l, path)
def save_xgb_impt(file, name, SAVE_DIR):
model = 'models/{}.pkl'.format(file)
path = utils.get_abs_path(SAVE_DIR, model)
print('model path: {}'.format(path))
pipeline = utils.load_pickle(path)
xgb_impt_d = get_xgb_impt_d(pipeline)
features = 'features/{}_impt_all_features.pkl'.format(name)
path = utils.get_abs_path(SAVE_DIR, features)
utils.save_pickle(xgb_impt_d, path)
def save_files(save_dir, model_name, features_l, importance_l, explainer):
features = 'features/{}_{}_all_features.pkl'.format(model_name, explainer)
path = utils.get_abs_path(save_dir, features)
utils.save_pickle(features_l, path)
print('saved features at {}'.format(path))
scores = 'feature_importance/{}_{}_all_scores.pkl'.format(
model_name, explainer)
path = utils.get_abs_path(save_dir, scores)
utils.save_pickle(importance_l, path)
print('saved scores at {}'.format(scores))
def get_word_score_ds(save_dir, model_name, explainer, test_labels=None):
path = None
path = utils.get_abs_path(save_dir, 'features/{}_{}_all_features.pkl'.format(model_name, explainer))
tmp_features = utils.load_pickle(path)
if model_name == 'lstm_att' and explainer == 'shap':
tmp_features = [' '.join(l) for l in tmp_features]
path = utils.get_abs_path(save_dir, 'feature_importance/{}_{}_all_scores.pkl'.format(model_name, explainer))
tmp_scores = utils.load_pickle(path)
ret = create_word_score_ds(tmp_features, tmp_scores, model_name, explainer, labels=test_labels)
return ret
def get_built_in(save_dir, model_name, total):
path = None
if model_name == 'svm':
path = utils.get_abs_path(save_dir, 'features/svm_coef_all_features.pkl')
elif model_name == 'svm_l1':
path = utils.get_abs_path(save_dir, 'features/svm_l1_coef_all_features.pkl')
elif model_name == 'xgb':
path = utils.get_abs_path(save_dir, 'features/xgb_impt_all_features.pkl')
tmp = utils.load_pickle(path)
tmp = convert_to_absolute(tmp) # take absoluate values
ret = create_copies(tmp, total)
return ret
def save_svm_coef(file, name, SAVE_DIR):
model = 'models/{}.pkl'.format(file)
path = utils.get_abs_path(SAVE_DIR, model)
print('model path: {}'.format(path))
pipeline = None
if file == 'svm':
pipeline = utils.load_pickle(path, encoding=False)
else:
pipeline = utils.load_pickle(path)
svm_coef_d = get_svm_coef_d(pipeline)
features = 'features/{}_coef_all_features.pkl'.format(name)
path = utils.get_abs_path(SAVE_DIR, features)
utils.save_pickle(svm_coef_d, path)
def choose_font(self, index: int = None, name: str = None):
fonts = self.font_name_list
if self.font_name_list is None:
fonts = read_from_file(get_abs_path('/fonts/font.txt'))
all_fonts = self.fonts
if index is not None and len(all_fonts) > 0:
return all_fonts[fonts[index]]
elif name is not None and len(all_fonts) > 0:
return all_fonts[fonts[index]]
elif not name and not index:
return get_abs_path('/pacifico/Pacifico.ttf')
else:
raise Exception('fonts are missing')
def parse_docs():
"""
Parse all documentation files and store information in data.
"""
data = {}
doc_folders = map(lambda x: get_abs_path(x),
['coala/docs/Developers', 'documentation/Users',
'documentation/Help'])
for folder in doc_folders:
for files in os.listdir(folder):
rst = parse_rst(os.path.join(get_abs_path(folder), files))
extractor = Extractor(rst, data, folder + '/' + files)
rst.walk(extractor)
return data
def run(train_dev_tokens, test_tokens, save_dir):
file = 'lstm_att'
model = 'models/{}.pkl'.format(file)
path = utils.get_abs_path(save_dir, model)
d_file = 'lstm_att_hp'
hp_d = 'models/{}.pkl'.format(d_file)
hp_path = utils.get_abs_path(save_dir, hp_d)
d = utils.load_pickle(hp_path)
model = init_model(train_dev_tokens, d, path)
test_split_tokens = split_tokens(test_tokens)
mapping = [model.get_words_to_ids(l) for l in test_split_tokens]
predictions, word_score_ds = model.predict(test_split_tokens, mapping,
True)
accuracy = accuracy_score(test_labels, predictions)
return word_score_ds
def authorize():
data = {
"amount": 0.10,
"currency": "RUB",
"description": "test pay",
"merchant_order_id": uuid.uuid4().hex,
"client_ip": "173.65.34.137",
"card_number": "4444444444444448",
"expiration_year": "2019",
"expiration_month": "06",
"cvn": "000",
"cardholder": "JOHN SMITH",
"force_3ds": 1,
"auto_clearing": 0,
"authorized_timeout": 360,
"authorize_timeout_action": "cancel",
"notify_url": "https://hello.com/payment/notify",
"success_url_3ds": "https://hello.com/payment/success",
"fail_url_3ds": "https://hello.com/payment/failed",
}
data["payment_params"] = {
"paymentLinkParams": {
"shopId": 531189,
"shopArticleId": 579836
}
}
url = "https://demo-scrat.yamoney.ru:9094/gates/system/authorize"
headers = {"Content-Type": "application/x-www-form-urlencoded"}
timeout = 5
certs = (get_abs_path(CERTIFICATE_PATH, acquirer_cert),
get_abs_path(CERTIFICATE_PATH, acquirer_cert_key))
try:
response = requests.post(url,
data=data,
headers=headers,
timeout=timeout,
cert=certs)
except Exception as ex:
print(ex)
print('URL: ', url)
print('Headers: ', headers)
pprint(data)
print("=" * 150)
print(response.content)
print("=" * 150)
print(response.text)
def write_fonts_on_image(self):
# all_fonts = read_from_file(file_path=get_abs_path('/fonts/fonts.txt'))
all_fonts = self.fonts
font_size = 20
pin_width = 1500
text_in_one_column = 70
x = 20
y = 20
pin_height = int(20 * len(all_fonts) / 2.5)
img = Img.make_blank_img(img_size=(pin_width, pin_height + 10),
folder_path='/tmp/',
color=(255, 255, 255))
arr = np.arange(0, len(all_fonts), text_in_one_column)
x_w = int(pin_width / len(arr))
r = 1
for i, font in enumerate(all_fonts):
txt_obj = Text(txt=str('{}. {}'.format(i, font)),
font_path=all_fonts[font],
font_size=font_size,
max_width=pin_width)
y += (font_size + 8)
if i / text_in_one_column >= r:
r += 1
x += x_w
y = 20
txt_obj.draw_text(img=img, loc=(x, y))
img.write_img(file_path=get_abs_path('/fonts/fonts.jpg'))
return True
def read_nhd_categories():
"""
read the nhd categories to include
"""
nhd_category_file = get_abs_path(
'../data/tables/nhd_categories_filtered.csv')
df = pd.read_csv(nhd_category_file)
nhd_cats = df['ID'].to_list()
return nhd_cats
def retrieve_test(self, line_no):
test_file_path = get_abs_path(self.test_file, __file__)
with open(test_file_path) as f:
source_code = f.read()
tree = ast.parse(source_code)
line_finder = test_finder.LineFinder(line_no=line_no)
line_finder.visit(tree)
return line_finder.path
def make_all_fonts(self, zip_file_path):
dir = get_abs_path('/fonts')
if not os.path.exists(dir):
os.mkdir(dir)
dirContents = os.listdir(dir)
if len(dirContents) == 0:
if not os.path.exists(dir):
os.mkdir(dir)
with zipfile.ZipFile(zip_file_path, "r") as zip_ref:
zip_ref.extractall(dir)
mylist = [f for f in glob.glob(dir + "/**/*.ttf", recursive=True)]
# rmtree(tmp, ignore_errors=True)
font_dict = OrderedDict(
(k.split('.')[0].split('/')[-1], k) for k in mylist)
f_path = get_abs_path("/fonts/font.txt")
setattr(self, 'font_name_list', list(font_dict.keys()))
assert write_to_file(f_path, list(font_dict.keys()))
return font_dict
def save_db(temp):
timestamp = int(time.time())
datetime = get_current_time()
pushed = 0
query = 'insert into measure values (null, %s, "%s", "%s", %s, %s)' % (timestamp, datetime, ROOM, pushed, temp)
try:
conn = sqlite3.connect(get_abs_path() + '/pi-temp.db')
conn.execute(query)
conn.commit()
conn.close()
except:
logging.error('Error when saving measure in database')
pass
def save_shap_val(file, name, SAVE_DIR, train_data, test_data):
model = 'models/{}.pkl'.format(file)
path = utils.get_abs_path(SAVE_DIR, model)
print('model path: {}'.format(path))
model = None
if file == 'svm':
model = utils.load_pickle(path, encoding=False)
else:
model = utils.load_pickle(path)
features_l, importance_l = [], []
if 'svm' in name:
features_l, importance_l = get_shap('svm', model, train_dev_tokens,
test_tokens)
elif 'xgb' in name:
features_l, importance_l = get_shap('xgb', model, train_dev_tokens,
test_tokens)
features = 'features/{}_shap_all_features.pkl'.format(name)
path = utils.get_abs_path(SAVE_DIR, features)
utils.save_pickle(features_l, path)
scores = 'feature_importance/{}_shap_all_scores.pkl'.format(name)
path = utils.get_abs_path(SAVE_DIR, scores)
utils.save_pickle(importance_l, path)
def spell_check(text: str) -> Tuple[int, int]:
"""Run spell_check.sh on the string given as argument.
spell_check.sh by default returns two-element array
[# incorrect words, # all words] for English documents.
[-1, 0] otherwise.
Elements are separated by \n.
:param text: input string to the script
:return: tuple of exactly two numbers returned by spell_check.sh"""
p: sp.Popen \
= sp.Popen([utils.get_abs_path('denormalization/spell_check.sh')],
stdout=PIPE, stdin=PIPE, stderr=sys.stderr)
stdout_data: List[str] = p.communicate(input=text.encode('utf-8'))[0] \
.decode('utf-8') \
.strip() \
.split('\n')
return (int(stdout_data[0]), int(stdout_data[1]))
def send(user, password, receiver, host='smtp.qq.com', port=25):
sender = '*****@*****.**' % user
content = MIMEMultipart('related')
content['Subject'] = u'send embedded image'
content['From'] = sender
content['To'] = receiver
body = '<html><img src="cid:logo.png"></html>'
html = MIMEText(body, 'html', 'utf-8')
content.attach(html)
logo_filename = 'logo.png'
mail_logo_path = get_abs_path(['static', 'img', logo_filename])
fp = open(mail_logo_path, 'rb')
img = MIMEImage(fp.read())
fp.close()
img.add_header('Content-ID', logo_filename)
content.attach(img)
smtp = smtplib.SMTP()
smtp.connect(host=host, port=port)
smtp.login(user=user, password=password)
smtp.sendmail(sender, receiver, content.as_string())
smtp.quit()
def send(user, password, receiver, host='smtp.qq.com', port=25):
sender = '*****@*****.**' % user
content = MIMEMultipart('related')
content['Subject'] = 'send embedded image'
content['From'] = sender
content['To'] = receiver
body = '<html><img src="cid:logo.png"></html>'
html = MIMEText(body, 'html', 'utf-8')
content.attach(html)
logo_filename = 'logo.png'
mail_logo_path = get_abs_path(['static', 'img', logo_filename])
fp = open(mail_logo_path, 'rb')
img = MIMEImage(fp.read())
fp.close()
img.add_header('Content-ID', logo_filename)
content.attach(img)
smtp = smtplib.SMTP()
smtp.connect(host=host, port=port)
smtp.login(user=user, password=password)
smtp.sendmail(sender, receiver, content.as_string())
smtp.quit()
def sheets_test():
filepath = get_abs_path(['utils', 'sheets.xlsx'])
print read_excel(filepath)
'na': args.na,
'nb': args.nb,
'nc': args.nc,
'batch_norm': batch_norm,
'drop_prob': args.drop_prob,
'augmentation': True
}
cnn = CNN(layers,
n_classes=n_classes,
batch_size=128,
l2_lambda=args.l2_lambda,
learning_rate=args.learning_rate,
add_scaling=args.scaling,
data_params=data_params,
models_dir=get_abs_path('../miniplaces/models'),
config=config,
img_width=100,
img_height=100)
train(cnn,
train_inputs,
train_labels,
val_inputs,
val_labels,
verbose=1,
train_generator=train_img_generator,
test_generator=test_img_generator)
# can't do this part; images are the wrong size (need to be 100 x 100 now)
# preds = cnn.predict_proba(val_inputs)
# acc1 = acc_at_k(1, preds, val_labels)
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'
SITE_ID = 1
# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True
# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale
USE_L10N = True
# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/home/media/media.lawrence.com/media/"
MEDIA_ROOT = utils.get_abs_path('../lmsite/media')
# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://media.lawrence.com/media/", "http://example.com/media/"
MEDIA_URL = '/media/'
# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/home/media/media.lawrence.com/static/"
STATIC_ROOT = ''
# URL prefix for static files.
# Example: "http://media.lawrence.com/static/"
STATIC_URL = '/static/'
def test_python_file_in_non_package_directory(self):
test_file_path = get_abs_path("source_code.py", __file__)
module_path = ModulePathFinder().find_path(test_file_path)
self.assertEqual("source_code", module_path)
def test_find_method_in_python_file_in_package_directory(self):
test_file_path = get_abs_path("package/source_code.py", __file__)
test_path = test_finder.get_path(test_file_path, 7)
self.assertEqual("package.source_code.TestMe.test_a", test_path)
aligned_att_weights, aligned_att_weights_len = get_aligned_att_weights(test_tokens, 'sst', 128, \
'avg', layer_num, impt=True)
relevant_tokens = get_relevant_features('sst', aligned_att_weights_len)
save_files(SAVE_SST_DIR, 'bert', relevant_tokens, aligned_att_weights,
explainer)
print(len(aligned_att_weights), len(aligned_att_weights_len))
'''
BERT LIME
'''
fp = 'data/LIME_features/deception/new_feature_l.npy'
sp = 'data/LIME_features/deception/new_scores_l.npy'
features = np.load(fp)
scores = np.load(sp)
print(len(features), len(scores))
tmp = 'features/bert_lime_all_features.pkl'
path = utils.get_abs_path(SAVE_DECEPTION_DIR, tmp)
utils.save_pickle(features, path)
tmp = 'feature_importance/bert_lime_all_scores.pkl'
path = utils.get_abs_path(SAVE_DECEPTION_DIR, tmp)
utils.save_pickle(scores, path)
fp = 'data/LIME_features/deception/new_feature_l.npy'
sp = 'data/LIME_features/deception/new_scores_l.npy'
features = np.load(fp)
scores = np.load(sp)
print(len(features), len(scores))
tmp = 'features/bert_lime_all_features.pkl'
path = utils.get_abs_path(SAVE_YELP_DIR, tmp)
utils.save_pickle(features, path)
tmp = 'feature_importance/bert_lime_all_scores.pkl'
path = utils.get_abs_path(SAVE_YELP_DIR, tmp)
def test_find_module_in_python_file(self):
test_file_path = get_abs_path("source_code.py", __file__)
test_path = test_finder.get_path(test_file_path, 0)
self.assertEqual("source_code", test_path)
import utils # pppoe pppoe_service = 'isp' pppoe_username = '******' pppoe_password = '******' # vpn-l2tp vpn_l2tp_username = '******' vpn_l2tp_password = '******' # vpn-l2tp-ipsec vpn_l2tp_ipsec_key = 'test' # vpn-openconnect-cert vpn_openconnect_cert_cacert = utils.get_abs_path( ) + '/../dockerfiles/vpn-openconnect-cert/etc/ocserv/ca.crt' vpn_openconnect_cert_clientcert = utils.get_abs_path( ) + '/../dockerfiles/vpn-openconnect-cert/etc/ocserv/client.crt' vpn_openconnect_cert_clientkey = utils.get_abs_path( ) + '/../dockerfiles/vpn-openconnect-cert/etc/ocserv/client.key' # vpn-openvpn-password vpn_openvpn_password_cacert = utils.get_abs_path( ) + '/../dockerfiles/vpn-openvpn-password/etc/openvpn/easy-rsa/pki/ca.crt' vpn_openvpn_password_username = '******' vpn_openvpn_password_password = '******' # vpn-openvpn-tls vpn_openvpn_tls_cacert = utils.get_abs_path( ) + '/../dockerfiles/vpn-openvpn-tls/etc/openvpn/easy-rsa/pki/ca.crt' vpn_openvpn_tls_clientcert = utils.get_abs_path(
return pin_width, pin_height
def read_img_files(file_path):
imgs = []
valid_images = [".jpg", ".gif", ".png", ".tga"]
for f in os.listdir(file_path):
ext = os.path.splitext(f)[1]
if ext.lower() not in valid_images:
continue
imgs.append(os.path.join(file_path, f))
return imgs
if __name__ == '__main__':
imgs_folder = get_abs_path('/pics')
imgs_loaded = read_img_files(imgs_folder)
matrix_dim = (1, 3)
imgs_loaded = imgs_loaded[:matrix_dim[0] * matrix_dim[1]]
# all_fonts = Text(zip_file_path=get_abs_path('/Font Pack.zip'),font_index=1)
# Text.write_fonts_on_image(all_fonts)
# imags_folder where all intermediate files get saved
pin = Pin(imgs=imgs_loaded,
folder_path=imgs_folder,
product_header=[
'TOP FOUR PRODUCTS', 'CHECK THEM', 'TOP FOUR PRODUCTS',
'CHECK THEM'
],
matrix_dim=matrix_dim,
font_index=20)
res = pin.make_collage()
def sheets_test():
filepath = get_abs_path(['utils', 'sheets.xlsx'])
print(read_excel(filepath))
def __init__(self, zip_file_path=None):
self.font_name_list = None
if zip_file_path is None:
zip_file_path = get_abs_path('/Font Pack.zip')
self.zip_path = zip_file_path
self.fonts = self.make_all_fonts(self.zip_path)
