def generate_file(context, output_path, template):
if "$file_name$" in context:
file_name = context["$file_name$"]
else:
file_name = get_timestamp()
output_path_with_filename = output_path / file_name
template.build_text_file(context, output_path_with_filename)
return output_path_with_filename
def _get_default_bindings(self):
return {
'cve_id': self._cve_dict.get('cve_id'),
'description': self._cve_dict.get('description'),
'cvss_v2': self._cve_dict.get('cvss_v2') or 10.0, # assume the worst
'ecosystem': self._cve_dict.get('ecosystem'),
'modified_date': get_timestamp()
}
def cross_validate(train_files, test_files, model_name, exp, kfolds):
# Complete args
model_name = model_name.lower() if model_name else input(
'model_name=?').lower()
exp_num = 'exp_'
n = str(exp) if exp else input('exp_?')
exp_num += n
logger.info('---------- Cross validation {} on {} start ----------'.format(
model_name, exp_num))
# Load data
logger.info('Loading training set: {}'.format(list(train_files)))
train_df = load_corpus(list(train_files))
if test_files != ():
logger.info('Loading test set: {}'.format(list(test_files)))
test_df = load_corpus(list(test_files))
else:
test_df = None
# Load configs
cfg_path = get_envar('CONFIG_PATH') + '/' + get_envar('BASE_CONFIG')
logger.info('Loading base_configs from {}'.format(cfg_path))
base_configs = read_config(cfg_path, obj_view=True)
logger.info('Loading exp_configs on {} from {}'.format(
exp_num, base_configs.exp_configs.path))
exp_configs = read_config(base_configs.exp_configs.path,
obj_view=False)[exp_num]
description = exp_configs['description']
hyparams = exp_configs['hyperparams']
logger.info('Experiment description: {}'.format(description.strip()))
logger.info('Hyperparams: {}'.format(hyparams))
wdir = base_configs.model.savepath + get_timestamp() + '/'
# CV
kf = KFold(n_splits=kfolds, shuffle=True, random_state=42)
cv = {}
for k, (train_idx, val_idx) in enumerate(kf.split(train_df)):
logger.info(f'-- Cross validation split {k+1} --')
rec = train_validate(model_name, hyparams, train_df.iloc[train_idx],
train_df.iloc[val_idx], test_df)
cv.update({f'CV_{k+1}': rec})
(_, cv_val, cv_test), df = best_scores(cv, complete=False)
logger.info(
f'**CV RESULTS** val_acc3={cv_val:.2%} test_acc3={cv_test:.2%}')
df.to_clipboard()
logger.info(f'CV details copied to clipboard \n{df}')
logger.info('---------- Cross validation {} on {} end ----------'.format(
model_name, exp_num))
def prepare_payload(self):
"""Prepare payload for Gremlin."""
timestamp = get_timestamp()
payload = {
'gremlin': cve_node_delete_script_template,
'bindings': {
'cve_id': self._cve_id_dict.get('cve_id'),
'timestamp': timestamp
}
}
return payload
def predict(self, path, plot=False):
y = self._load(path, mfcc=False)
activity = zcr_vad(y)
spans = get_timestamp(activity)
embed = [self._encode_segment(y, span) for span in spans]
embed = torch.cat(embed).cpu().numpy()
speakers = OptimizedAgglomerativeClustering().fit_predict(embed)
if plot:
self._plot_diarization(y, spans, speakers)
timestamp = np.array(spans) / self.sr
return timestamp, speakers
def _get_bindings(self, vulnerability):
return {
'snyk_vuln_id': vulnerability.get('id'),
'description': vulnerability.get('description'),
'cvss_score': vulnerability.get('cvssScore') or 10.0, # assume the worst
'ecosystem': vulnerability.get('ecosystem'),
'modified_date': get_timestamp(),
'severity': vulnerability.get('severity'),
'title': vulnerability.get('title') or "",
'url': vulnerability.get('url') or "",
'cvssV3': vulnerability.get('cvssV3') or "",
'exploit': vulnerability.get('exploit') or "",
'fixable': vulnerability.get('fixable') or "",
'malicious': vulnerability.get('malicious'),
'patch_exists': vulnerability.get('patchExists') or "",
'snyk_pvt_vul': vulnerability.get('pvtVuln') or False
}
def train(train_files, val_files, test_files, model_name, exp):
# Complete args
model_name = model_name.lower() if model_name else input(
'model_name=?').lower()
exp_num = 'exp_'
n = str(exp) if exp else input('exp_?')
exp_num += n
logger.info('---------- Training {} on {} start ----------'.format(
model_name, exp_num))
# Load data
logger.info('Loading training set: {}'.format(list(train_files)))
train_df = load_corpus(list(train_files))
if val_files != ():
logger.info('Loading validation set: {}'.format(list(val_files)))
val_df = load_corpus(list(val_files))
else:
val_df = None
if test_files != ():
logger.info('Loading test set: {}'.format(list(test_files)))
test_df = load_corpus(list(test_files))
else:
test_df = None
# Load configs
cfg_path = get_envar('CONFIG_PATH') + '/' + get_envar('BASE_CONFIG')
logger.info('Loading base_configs from {}'.format(cfg_path))
base_configs = read_config(cfg_path, obj_view=True)
logger.info('Loading exp_configs on {} from {}'.format(
exp_num, base_configs.exp_configs.path))
exp_configs = read_config(base_configs.exp_configs.path,
obj_view=False)[exp_num]
description = exp_configs['description']
hyparams = exp_configs['hyperparams']
logger.info('Experiment description: {}'.format(description.strip()))
logger.info('Hyperparams: {}'.format(hyparams))
wdir = base_configs.model.savepath + get_timestamp() + '/'
# Build Model
lm = LexiconManager()
dm = AbsaDataManager(lexicon_manager=lm)
model = VALID_MODELS[model_name.lower()]
model = model(datamanager=dm, parameters=hyparams)
# Train
model.train(train_df, val_df, test_df)
# Predict and score on test
if test_df is not None:
_, _, loss_, acc3_ = model.score(test_df)
logger.info('Final score on test set: '
'test_loss={loss:.4f} ' \
'test_acc3={acc:.2%}'\
.format(loss=loss_, acc=acc3_))
# Save model
model.save(wdir)
# Close tf.Session, not really necessary but... anyway
model.close_session()
logger.info('---------- Training {} on {} end ----------'.format(
model_name, exp_num))
def test_get_timestamp():
"""Test utils.get_timestamp()."""
timestamp = get_timestamp()
result = (datetime.datetime.utcnow()).strftime('%Y%m%d')
assert result == timestamp
Firstly we loop through all the Properties within the main account (Bestseller (Universal)) to get information related with all the views under each Property.
Secondly we filter for the Views' profiles we want -- we only need brand-country level Views to pull data from.
Thirdly we add more columns with information we need -- sitebrand, sitecountry, table_updated_time and etc..
"""
import pandas as pd
from datetime import datetime
from src.configure_logging import configure_logging
from src.ga_connector import GoogleAnalytics
from src.s3_toolkit import s3Toolkit
from src.utils import get_timestamp
# to avoid printing out logs from GA's module we need to setup our own logger
logger = configure_logging(logger_name=__name__)
ACCOUNT = 66188758
TIMESTAMP = get_timestamp()
def get_bucket_name(env):
""""
Gets Bucket Name according to the chosen environment
Parameters:
----------
env : string
dev or prod
Returns:
----------
bucket_name: string
"""
X_test = test.drop(drop_cols + [target_col], axis=1)
# fill inf/nan
X_train.replace(np.inf, np.nan, inplace=True)
X_test.replace(np.inf, np.nan, inplace=True)
X_train.fillna(X_train.mean(), inplace=True)
X_test.fillna(X_train.mean(), inplace=True)
# rankgauss transform
# https://www.kaggle.com/c/porto-seguro-safe-driver-prediction/discussion/44629
prep = QuantileTransformer(output_distribution="normal")
X_cont_train = prep.fit_transform(X_train)
X_cont_test = prep.transform(X_test)
# train and predict
timestamp = get_timestamp()
oof_preds, test_preds, cv_scores = run(
X_seq_train,
X_cont_train,
y_train,
X_seq_test,
X_cont_test,
timestamp,
random_state=0,
)
# save results
print(cv_scores)
output_dir = Path(f"./output/{timestamp}")
output_dir.mkdir(parents=True)
pd.DataFrame(oof_preds).to_csv(output_dir / f"{timestamp}_oof.csv",
static_template_path = None
compile_output_path = None
if args.compile is not None:
static_template_path = Path(args.compile).resolve()
if not static_template_path.is_file():
exit_on_arg_error("ERROR - Invalid path for static template file"
" (--compile).")
if args.compileoutput is not None:
compile_output_path = Path(args.compileoutput).resolve()
if not compile_output_path.parent.exists():
exit_on_arg_error("ERROR - Invalid output path for compilation"
" - The output path directory does not exist."
" (--compileoutput).")
else:
compile_file_name = "tmplt" + get_timestamp() + ".py"
compile_output_path = static_template_path.parent / compile_file_name
compile_template(static_template_path, compile_output_path)
if args.execute:
template_path = None
template_path_parent_directory = None
template_module = None
if compile_output_path is None:
if args.execute is "$compiled$":
exit_on_arg_error("ERROR - A path must be passed to the execute"
" argument when execution does not occur along "
"with compilation (--execute <path>).")
template_path = Path(args.execute).resolve()
if not template_path.is_file():
