def get_file_path():
out_dir = utils.get_script_dir()
fpath = '%s/outputs/%s.csv' % (out_dir, utils.get_timestamp())
# Create outputs dir if it does not exist yet
base_dir = os.path.dirname(fpath)
if not os.path.exists(base_dir):
os.makedirs(base_dir)
return fpath
def collect_profiling_data(self):
self.adb.check_run_and_return_output(['pull', '/data/local/tmp/perf.data',
self.args.perf_data_path])
if not self.args.skip_collect_binaries:
binary_cache_args = [sys.executable,
os.path.join(get_script_dir(), 'binary_cache_builder.py')]
binary_cache_args += ['-i', self.args.perf_data_path]
if self.args.native_lib_dir:
binary_cache_args += ['-lib', self.args.native_lib_dir]
if self.args.disable_adb_root:
binary_cache_args += ['--disable_adb_root']
if self.args.ndk_path:
binary_cache_args += ['--ndk_path', self.args.ndk_path]
subprocess.check_call(binary_cache_args)
def collect_profiling_data(self):
self.adb.check_run_and_return_output(['pull', '/data/local/tmp/perf.data',
self.args.perf_data_path])
if not self.args.skip_collect_binaries:
binary_cache_args = [sys.executable,
os.path.join(get_script_dir(), 'binary_cache_builder.py')]
binary_cache_args += ['-i', self.args.perf_data_path, '--log', self.args.log]
if self.args.native_lib_dir:
binary_cache_args += ['-lib', self.args.native_lib_dir]
if self.args.disable_adb_root:
binary_cache_args += ['--disable_adb_root']
if self.args.ndk_path:
binary_cache_args += ['--ndk_path', self.args.ndk_path]
subprocess.check_call(binary_cache_args)
def initialize():
data_folder = get_script_dir() + '/preprocessing'
if not os.path.exists(data_folder):
os.mkdir(data_folder)
for url, file_name in STUFF_TO_DOWNLOAD:
save_path = data_folder + '/' + file_name
if not os.path.exists(save_path):
print '...downloading', url
subprocess.call(['wget', '-O', save_path, url])
if file_name.endswith('.tar.gz') and os.path.exists(save_path):
print '...unpacking'
subprocess.call((
'tar -zxf /home/alex/chatbot/preprocessing/names_postcodes.tar.gz -C %s'
% data_folder).split())
os.remove(save_path)
def parse_args():
ap = argparse.ArgumentParser()
ap.add_argument("-o",
"--output",
dest="output",
type=utils.to_path,
help="output file path")
ap.add_argument("-t",
"--type",
dest="type",
type=str,
help=f"script type ({TEMPLATE_OPTIONS.keys()})",
choices=TEMPLATE_OPTIONS.keys())
ap.add_argument("-a",
"--author",
dest="author",
type=str,
help=f"author name",
nargs="?",
default=utils.get_user_name())
ap.add_argument("-d",
"--description",
dest="description",
type=str,
help="script description")
ap.add_argument(
"-c",
"--copy-utils",
dest="copy_utils",
action="store_true", # default: False
help="copy utils to output folder",
)
args = ap.parse_args()
args.script_dir = utils.get_script_dir()
args.current_dir = utils.get_current_dir()
return args
def save_predictions(self, preds):
out_dir = utils.get_script_dir()
fpath = '%s/outputs/predictions/%s.csv' % (out_dir, self.model_name)
preds.set_index(MONTH_INT_LABEL).to_csv(fpath, header=True)
logging.info('predictions saved to %s' % fpath)
def save_model(self):
logging.info('saving model')
filename = 'model.sav' if self.model_name is None else self.model_name
fpath = utils.get_script_dir()
joblib.dump(self.model, fpath + filename)
logging.info('done')
def add_file(self, file_path):
file_path = os.path.join(get_script_dir(), file_path)
with open(file_path, 'r') as f:
self.add(f.read())
return self
def add_file(self, file_path):
file_path = os.path.join(get_script_dir(), file_path)
with open(file_path, 'r') as f:
self.add(f.read())
return self
import core
import logging
import sys
from sklearn.linear_model import Ridge, LinearRegression, Lasso
from sklearn.svm import LinearSVR
from core import MONTH_INT_LABEL, TARGET_LABEL, INDEX_COLUMNS
import utils
sys.path.append(utils.up_dir(utils.get_script_dir(), 1))
from features import build_features
import numpy as np
import tuning
class LinearModel(core.Model):
def __init__(self,
model,
drop_first_ohe,
standardize,
model_class,
model_name,
n_evals=2,
n_eval_months=2,
sample=None):
super().__init__(model,
sample=sample,
standardize=standardize,
training_range=(0., 30.),
n_eval_months=n_eval_months,
model_name=model_name)
self.drop_first_ohe = drop_first_ohe