def load_model(model_name: str):
pipeline = jl_load(
Path(dirname(__file__)) / 'model' / f"{model_name}.joblib")
pipeline.steps[0][1].init()
return pipeline
def main(argv):
"""メイン関数。入力を受け取り、並列処理を実行。"""
assert len(
argv
) > 2, "SYNTAX: python src/10.3-screening.py MODEL_FILE.joblib SMILES.smi"
modelfile, smilesfile = argv[1:]
model = jl_load(modelfile)
n_counts = count_lines(smilesfile)
cs = 1000
model_sampler = (model for _ in range(n_counts))
outfile = open(smilesfile.replace('.smi', '.out'), 'w')
with open(smilesfile, 'r') as f:
with Pool(cpu_count()) as pool:
"並列処理"
for ret in pool.imap(processor,
zip(f, model_sampler),
chunksize=cs):
outfile.write(ret + '\n')
outfile.close() # 開いたファイルオブジェクトは必ず閉じる。
def load(filename):
"""Load an object that has been saved with dump.
We try to open it using the pickle protocol. As a fallback, we
use joblib.load. Joblib was the default prior to msmbuilder v3.2
Parameters
----------
filename : string
The name of the file to load.
"""
try:
with open(filename, 'rb') as f:
return pickle.load(f)
except Exception as e1:
try:
return jl_load(filename)
except Exception as e2:
raise IOError(
"Unable to load {} using the pickle or joblib protocol.\n"
"Pickle: {}\n"
"Joblib: {}".format(filename, e1, e2)
)
def load(model_name="news_classification"):
return jl_load(Path(dirname(__file__)) / 'model' / f"{model_name}.joblib")
def load(model_name="tweet_sentiment"):
return jl_load(Path(dirname(__file__)) / 'model' / f"{model_name}.joblib")
def load():
return jl_load(
Path(dirname(__file__)) / 'model' /
'tweet_profanity_classification.joblib')