def main(targets):
if "download" in targets:
os.system("src/download.sh")
if "data" in targets:
data.main()
if "train" in targets:
train.main()
def test_main(
mock_dump,
mock_cross_validate,
mock_load_data,
mock_register_model,
input_df,
cv_results,
):
# Mock retuirn values
mock_cross_validate.return_value = cv_results
mock_load_data.return_value = input_df
# Execute main
main()
# Should have made a call to write model
mock_dump.assert_called_once()
# Should have made a call to register model
mock_register_model.assert_called_once()
def train(test=False):
main(test)
# all arguments needed in the pipeline
parser.add_argument("--start_point", default="acquire", help="available start point: 'acquire', 'download'")
parser.add_argument("--bucket", default=config.BUCKET,help="S3 bucket name")
parser.add_argument("--url", default=os.path.join(config.BASE_URL, config.FILE_NAME),help="url to raw data")
parser.add_argument("--file_name", default=config.FILE_NAME, help="file name")
parser.add_argument("--raw_data", default=config.RAW_DATA,help="path to raw data")
parser.add_argument("--clean_file", default=config.CLEAN_FILE_PATH, help="path to cleaned data")
parser.add_argument("--model_type", default="exp",help="available models: exp, log or lstm")
parser.add_argument("--train", default=config.TRAIN, help="train directory")
parser.add_argument("--test", default=config.TEST, help="test directory")
parser.add_argument("--model_dir", default=config.MODEL_DIR, help="model directory")
parser.add_argument("--pred", default=config.PRED, help="prediction directory")
args = parser.parse_args()
assert args.start_point in ["download", "acquire"], "Invalid input for start point"
if args.start_point == "download":
download_data.main(args)
upload_data.main(args)
acquire_data.main(args)
clean_data.main(args)
train.main(args)
score_model.main(args)
evaluate.main(args)
# add_cases.main(config.SQLALCHEMY_DATABASE_URI)
# -*- coding: UTF-8 -*-
from src.train import main
if __name__ == '__main__':
main()
import sys
from src import runtime, train
from src.common.config.global_config import cfg
if __name__ == "__main__":
# do some basic cfg validation and call runtime or train according to mode
if not cfg.data_root or not cfg.work_dir:
raise Exception('data_root and work_dir have to be specified')
if cfg.mode == 'runtime':
if not cfg.trained_model:
raise Exception('define your trained_model')
try:
runtime.main()
except KeyboardInterrupt:
print('quitting because of keyboard interrupt (probably ctrl + c)')
sys.exit(0)
elif cfg.mode == 'train':
train.main()
else:
raise Exception('invalid mode')