def run_mpi_blast(config):
size = MPI.COMM_WORLD.Get_size()
rank = MPI.COMM_WORLD.Get_rank()
name = MPI.Get_processor_name()
print("Hello World! \n I am process %d of %d on %s." % (rank, size, name))
uniprot_db = config["data"]["mixed-method"]["preprocess"]["uniprot_db"]
if "tmpdir" in config["input"]:
tmpdir = config["input"]["tmpdir"]
else:
tmpdir = "/tmpdir"
src = os.path.dirname(uniprot_db)
dest = tmpdir + "/blastdb"
if rank == 0:
workdir = config["input"]["gomap_dir"] + "/"
print(workdir)
tmp_fa_dir = workdir + config["input"]["split_path"] + "/"
dest = workdir + config["data"]["mixed-method"]["preprocess"][
"blast_out"] + "/temp/"
results = pyrocopy.copy(tmp_fa_dir, dest)
print(results)
fa_pattern = dest + config["input"]["basename"] + "*.fa"
fa_files = glob(fa_pattern)
work_list = natsorted(fa_files)
all_dat = master(work_list)
else:
results = pyrocopy.copy(src, dest)
#pprint(results)
slave(dest, config)
def run_uniprot_blast(config):
if config["input"]["mpi"] is True:
from code.utils.run_mpi_blast import run_mpi_blast
run_mpi_blast(config)
else:
from code.utils.run_single_blast import run_single_blast
print("Running the single blast step")
workdir = config["input"]["gomap_dir"] + "/"
#print(workdir)
tmp_fa_dir = workdir + config["input"]["split_path"] + "/"
dest = workdir + config["data"]["mixed-method"]["preprocess"][
"blast_out"] + "/temp/"
results = pyrocopy.copy(tmp_fa_dir, dest)
#print(results)
fa_pattern = dest + config["input"]["basename"] + "*.fa"
fa_files = sorted(glob(fa_pattern))
uniprot_db = config["pipeline"]["pipeline_loc"] + "/" + config["data"][
"mixed-method"]["preprocess"]["uniprot_db"]
src_db = os.path.dirname(uniprot_db)
# print(src_db)
dest_db = "/tmpdir/blastdb"
results = pyrocopy.copy(src_db, dest_db)
uniprot_db = dest_db + "/" + os.path.basename(
config["data"]["mixed-method"]["preprocess"]["uniprot_db"])
run_single_blast(fa_files, uniprot_db, config)
def run_mpi_iprs(fa_files, config):
size = MPI.COMM_WORLD.Get_size()
rank = MPI.COMM_WORLD.Get_rank()
name = MPI.Get_processor_name()
print("Hello World! \n I am process %d of %d on %s." % (rank, size, name))
iprs_src = config["software"]["iprs"]["path"]
if "tmpdir" in config["input"]:
tmpdir = config["input"]["tmpdir"]
else:
tmpdir = "/tmpdir"
src = iprs_src
dest = tmpdir + "/"
print(src + " " + dest)
work_list = natsorted(fa_files)
if rank == 0:
all_dat = master(work_list)
else:
results = pyrocopy.copy(src, dest)
pprint(results)
slave(config)
def copy_blast(config):
workdir = config["input"]["gomap_dir"] + "/"
tmp_blast_dir = workdir + config["data"]["mixed-method"]["preprocess"][
"blast_out"]
pannzer_blast_dir = workdir + config["data"]["mixed-method"]["pannzer"][
"preprocess"]["blast"]
results = pyrocopy.copy(tmp_blast_dir,
pannzer_blast_dir,
excludeDirs=['temp'])
def run_uniprot_blast(config):
if config["input"]["mpi"] is True:
from code.utils.run_mpi_blast import run_mpi_blast
run_mpi_blast(config)
else:
from code.utils.run_single_blast import run_single_blast
workdir = config["input"]["gomap_dir"] + "/"
print(workdir)
tmp_fa_dir = workdir + config["input"]["split_path"] + "/"
dest = workdir + config["data"]["mixed-method"]["preprocess"][
"blast_out"] + "/temp/"
results = pyrocopy.copy(tmp_fa_dir, dest)
print(results)
fa_pattern = dest + config["input"]["basename"] + "*.fa"
fa_files = sorted(glob(fa_pattern))
run_single_blast(fa_files, config)
def init_dirs(config):
gomap_dir = config["input"]["workdir"] + "/GOMAP-" + config["input"][
"basename"]
config["input"]["gomap_dir"] = gomap_dir
if not os.path.exists(gomap_dir):
os.makedirs(gomap_dir, mode=0777)
excl_files = ['a']
for root, dir, files in os.walk(gomap_dir):
excl_files = excl_files + files
with open(config["pipeline"]["dir_struct"]) as tmp_file:
dir_struct = tmp_file.read()
with create_files(dir_struct) as workdir:
results = pyrocopy.copy(workdir,
gomap_dir,
excludeFiles=excl_files,
detailedResults=True)
setlogging(config, "init_dirs")
logging.info("Initializing directory structure")
logging.info(pformat(results))
return (config)
def run_iprs(config):
dom_config = config["data"]["domain"]
iprs_config = config["software"]["iprs"]
workdir = config["input"]["gomap_dir"] + "/"
fa_pattern = workdir + config["input"]["split_path"] + "/" + config[
"input"]["basename"] + "*fa"
fa_files = sorted(glob(fa_pattern))
#print(fa_files)
iprs_src = config["software"]["iprs"]["path"]
# print(src_db)
iprs_loc = "/tmpdir/iprs"
results = pyrocopy.copy(iprs_src, iprs_loc)
if config["input"]["mpi"] is True:
from code.utils.run_mpi_iprs import run_mpi_iprs
run_mpi_iprs(fa_files, config)
else:
from code.utils.run_single_iprs import run_single_iprs
run_single_iprs(fa_files, config, iprs_loc)
raise Exception("Failed _isSamePath test with same folder")
# _getTreeDepth test
depth = pyrocopy._getTreeDepth("Level1")
if (depth != 3):
raise Exception("Failed to get maximum depth of tree: Level1")
# copy test
logger.info("Testing pyrocopy.copy() ...")
numFiles = 30
src = os.path.relpath(genRandomTree(tmpdir, 4, numFiles, MAX_FILE_SIZE),
tmpdir)
dst = os.path.basename(src) + "Copy"
# check initial copy
results = pyrocopy.copy(src, dst, preserveStats=PRESERVE_TIMESTAMPS)
if (results['filesCopied'] != numFiles):
raise Exception("Failed to copy all files.")
if (results['filesFailed'] > 0 or results['dirsFailed'] > 0):
raise Exception("Failed to copy some files or directories.")
# TODO Diff src and dst
# check second copy (should skip all files)
results = pyrocopy.copy(src, dst, preserveStats=PRESERVE_TIMESTAMPS)
if (results['filesSkipped'] != numFiles):
raise Exception("Failed to skip all files.")
if (results['filesFailed'] > 0 or results['dirsFailed'] > 0):
raise Exception("Failed to process some files or directories.")
# TODO Diff src and dst
# check overwrite copy
if (not pyrocopy._isSamePath("Level1", "Level1")):
raise Exception("Failed _isSamePath test with same folder")
# _getTreeDepth test
depth = pyrocopy._getTreeDepth("Level1")
if (depth != 3):
raise Exception("Failed to get maximum depth of tree: Level1")
# copy test
logger.info("Testing pyrocopy.copy() ...")
numFiles = 30
src = os.path.relpath(genRandomTree(tmpdir, 4, numFiles, MAX_FILE_SIZE), tmpdir)
dst = os.path.basename(src) + "Copy"
# check initial copy
results = pyrocopy.copy(src, dst, preserveStats=PRESERVE_TIMESTAMPS)
if (results['filesCopied'] != numFiles):
raise Exception("Failed to copy all files.")
if (results['filesFailed'] > 0 or results['dirsFailed'] > 0):
raise Exception("Failed to copy some files or directories.")
# TODO Diff src and dst
# check second copy (should skip all files)
results = pyrocopy.copy(src, dst, preserveStats=PRESERVE_TIMESTAMPS)
if (results['filesSkipped'] != numFiles):
raise Exception("Failed to skip all files.")
if (results['filesFailed'] > 0 or results['dirsFailed'] > 0):
raise Exception("Failed to process some files or directories.")
# TODO Diff src and dst
# check overwrite copy
