def load_model(model_name: str): pipeline = jl_load( Path(dirname(__file__)) / 'model' / f"{model_name}.joblib") pipeline.steps[0][1].init() return pipeline
def main(argv): """メイン関数。入力を受け取り、並列処理を実行。""" assert len( argv ) > 2, "SYNTAX: python src/10.3-screening.py MODEL_FILE.joblib SMILES.smi" modelfile, smilesfile = argv[1:] model = jl_load(modelfile) n_counts = count_lines(smilesfile) cs = 1000 model_sampler = (model for _ in range(n_counts)) outfile = open(smilesfile.replace('.smi', '.out'), 'w') with open(smilesfile, 'r') as f: with Pool(cpu_count()) as pool: "並列処理" for ret in pool.imap(processor, zip(f, model_sampler), chunksize=cs): outfile.write(ret + '\n') outfile.close() # 開いたファイルオブジェクトは必ず閉じる。
def load(filename): """Load an object that has been saved with dump. We try to open it using the pickle protocol. As a fallback, we use joblib.load. Joblib was the default prior to msmbuilder v3.2 Parameters ---------- filename : string The name of the file to load. """ try: with open(filename, 'rb') as f: return pickle.load(f) except Exception as e1: try: return jl_load(filename) except Exception as e2: raise IOError( "Unable to load {} using the pickle or joblib protocol.\n" "Pickle: {}\n" "Joblib: {}".format(filename, e1, e2) )
def load(model_name="news_classification"): return jl_load(Path(dirname(__file__)) / 'model' / f"{model_name}.joblib")
def load(model_name="tweet_sentiment"): return jl_load(Path(dirname(__file__)) / 'model' / f"{model_name}.joblib")
def load(): return jl_load( Path(dirname(__file__)) / 'model' / 'tweet_profanity_classification.joblib')