def _ftp_stage_in_app(dm, executor):
return python_app(executors=[executor],
data_flow_kernel=dm.dfk)(_ftp_stage_in)
from parsl.configs.htex_local import config
config.executors[0].label = "Foo"
config.executors[0].max_workers = 1
elif args.config == "theta":
from theta import config
elif args.config == "theta_test":
from theta_test import config
elif args.config == "comet":
from comet import config
# Most of the app that hit the timeout will complete if retried.
# but for this demo, I'm not setting retries.
# config.retries = 2
parsl.load(config)
parsl_runner = parsl.python_app(process_files)
if args.debug:
parsl.set_stream_logger()
os.makedirs(args.outdir, exist_ok=True)
all_smile_files = glob.glob(args.smile_glob)
counter = 0
batch_futures = {}
chunksize = int(args.batch_size)
for smile_file in all_smile_files:
if not smile_file.endswith('smi'):
print(f"Ignoring {smile_file} not smile file")
continue
config.executors[0].max_workers = 1
elif args.config == "theta":
from theta import config
print("Loading theta config")
elif args.config == "theta_test":
from theta_test import config
elif args.config == "comet":
from comet import config
# Most of the app that hit the timeout will complete if retried.
# but for this demo, I'm not setting retries.
# config.retries = 2
parsl.load(config)
parsl_runner = {}
parsl_runner['pkl'] = parsl.python_app(reg_go_infer)
parsl_runner['csv'] = parsl.python_app(reg_go_infer_csv)
if args.debug:
parsl.set_stream_logger()
os.makedirs(args.outdir, exist_ok=True)
all_smile_dirs = glob.glob(args.smile_dir)
counter = 0
batch_futures = {}
for smile_dir in all_smile_dirs:
print("Processing smile_dir: {} {}/{}".format(smile_dir, counter,
len(all_smile_dirs)))
counter += 1
if args.config == "local":
from parsl.configs.htex_local import config
from parsl.configs.htex_local import config
config.executors[0].label = "Foo"
config.executors[0].max_workers = 1
elif args.config == "theta":
from theta import config
elif args.config == "comet":
from comet import config
# Most of the app that hit the timeout will complete if retried.
# but for this demo, I'm not setting retries.
# config.retries = 2
parsl.load(config)
parsl_runner = parsl.python_app(run_local)
if args.debug:
parsl.set_stream_logger()
"""
if args.num_smiles == "0":
#print("[Main] Loading all data available")
# smiles = pd.read_csv("train.csv").iloc[:,0].tolist()
smiles = pd.read_csv(args.smile_file, error_bad_lines=False) # .iloc[:,0].tolist()
else:
print(f"[Main] Loading {args.num_smiles} smiles from file")
smiles = pd.read_csv(args.smile_file, error_bad_lines=False, nrows=int(args.num_smiles)) # .iloc[:,0].tolist()
"""
def make_stage_in_app(executor, dfk):
return python_app(executors=[executor], data_flow_kernel=dfk)(stage_in_noop)