def _actually_du():
srv, remote_path = parse_fasp_url(url)
bn = basename(remote_path)
local_file = join(local_dir, bn)
skip = (bn in local_cached
and os.stat(local_file).st_size == remote_size)
if skip == False:
ret = asp.download_file(srv, dcc_user, dcc_pw, remote_path,
local_dir)
if not ret:
raise Exception("Download failed: " + url)
to_rm, files_to_upload = untar(local_file)
for i, f in enumerate(files_to_upload):
new_f = addtag(f, namespace)
os.rename(f, new_f)
files_to_upload[i] = new_f
names_sizes = [(basename(f), os.stat(f).st_size)
for f in files_to_upload]
for f in files_to_upload:
ret = asp.upload_file(ncbi_srv,
ncbi_user,
None,
f,
ncbi_path,
keyfile=ncbi_keyfile)
with open(local_file + "." + namespace + ".complete", 'w') as f:
for name_size in names_sizes:
print >> f, "\t".join(map(str, name_size))
for f in reversed(to_rm):
try:
os.rmdir(f) if os.path.isdir(f) else os.remove(f)
except:
print >> sys.stderr, "Unable to remove " + f
def _upload_data(self):
self.logger.debug("In _upload_data.")
session = iHMPSession.get_session()
study = self._study
study2dir = {"ibd": "ibd", "preg_preterm": "ptb", "prediabetes": "t2d"}
if study not in study2dir:
raise ValueError("Invalid study. No directory mapping for %s" %
study)
study_dir = study2dir[study]
remote_base = os.path.basename(self._local_file)
valid_chars = "-_.%s%s" % (string.ascii_letters, string.digits)
remote_base = ''.join(c for c in remote_base if c in valid_chars)
remote_base = remote_base.replace(' ', '_') # No spaces in filenames
remote_path = "/".join(
["/" + study_dir, "cytokine", "host", remote_base])
self.logger.debug("Remote path for this file will be %s.", remote_path)
upload_result = aspera.upload_file(Cytokine.aspera_server,
session.username, session.password,
self._local_file, remote_path)
if not upload_result:
self.logger.error("Experienced an error uploading the data. " + \
"Aborting save.")
raise Exception("Unable to upload cytokine.")
else:
self._urls = ["fasp://" + Cytokine.aspera_server + remote_path]
def _u():
ret = asp.upload_file(remote_srv,
user,
None,
local_fname,
remote_path,
keyfile=keyfile)
if blithely or ret:
open(complete_fname, 'w').close()
return blithely or ret # return True if blithely is True
def _upload_files(self, file_map):
self.logger.debug("In _upload_files.")
study2dir = {"ibd": "ibd", "preg_preterm": "ptb", "prediabetes": "t2d"}
study = self.study
subtype = self.subtype
if study not in study2dir:
raise ValueError("Invalid study. No directory mapping for %s" %
study)
study_dir = study2dir[study]
remote_paths = {}
# Get the session so we can get the username and password
session = iHMPSession.get_session()
username = session.username
password = session.password
# For each of the Proteome data files (there are 4), transmit them
# to the Aspera server and return a dictionary with the computed remote
# paths...
for file_type, local_file in file_map.iteritems():
self.logger.debug("Uploading %s of %s type %s", __name__,
local_file, file_type)
remote_base = os.path.basename(local_file)
valid_chars = "-_.%s%s" % (string.ascii_letters, string.digits)
remote_base = ''.join(c for c in remote_base if c in valid_chars)
remote_base = remote_base.replace(' ',
'_') # No spaces in filenames
remote_path = "/".join([
"/" + study_dir, "proteome_nonpride", subtype, file_type,
remote_base
])
self.logger.debug("Remote path for this file will be %s.",
remote_path)
# Upload the file to the iHMP aspera server
upload_success = aspera.upload_file(ProteomeNonPride.aspera_server,
username, password, local_file,
remote_path)
if not upload_success:
self.logger.error("Experienced an error uploading file %s.",
local_file)
raise Exception("Unable to upload " + local_file)
else:
remote_paths[
file_type] = "fasp://" + ProteomeNonPride.aspera_server + remote_path
return remote_paths
def _u():
with open(sub_fname, 'r') as f:
b = basename(local_fname)
if b not in ("submission.xml", "submit.ready") \
and b not in f.read():
return
ret = asp.upload_file(remote_srv,
user,
None,
local_fname,
remote_path,
keyfile=keyfile)
if blithely or ret:
open(complete_fname, 'w').close()
return blithely or ret # return True if blithely is True
def _upload_data(self):
self.logger.debug("In _upload_data.")
session = iHMPSession.get_session()
study = self._study
study2dir = {
"ibd": "ibd",
"preg_preterm": "ptb",
"prediabetes": "t2d",
"risk": "risk",
"protect": "protect",
"mucosal_ibd": "mucosal_ibd",
"herfarth_ccfa": "herfarth_ccfa",
"jl_crohns": "jl_crohns"
}
if study not in study2dir:
raise ValueError("Invalid study. No directory mapping for %s" %
study)
study_dir = study2dir[study]
remote_base = os.path.basename(self._local_file)
valid_chars = "-_.%s%s" % (string.ascii_letters, string.digits)
remote_base = ''.join(c for c in remote_base if c in valid_chars)
remote_base = remote_base.replace(' ', '_') # No spaces in filenames
remote_path = "/".join([
"/" + study_dir, "genome", "microbiome", "16s", "raw", remote_base
])
self.logger.debug("Remote path for this file will be %s.", remote_path)
# Upload the file to the iHMP aspera server
upload_result = aspera.upload_file(SixteenSRawSeqSet.aspera_server,
session.username, session.password,
self._local_file, remote_path)
if not upload_result:
self.logger.error("Experienced an error uploading the data. " + \
"Aborting save.")
raise Exception("Unable to load 16S raw sequence set.")
else:
self._urls = [
"fasp://" + SixteenSRawSeqSet.aspera_server + remote_path
]
def _upload_data(self):
self.logger.debug("In _upload_data.")
session = iHMPSession.get_session()
study = self._study
study2dir = {
"ibd": "ibd",
"preg_preterm": "ptb",
"prediabetes": "t2d"
}
if study not in study2dir:
raise ValueError("Invalid study. No directory mapping for %s" % study)
study_dir = study2dir[study]
remote_base = os.path.basename(self._local_file)
valid_chars = "-_.%s%s" % (string.ascii_letters, string.digits)
remote_base = ''.join(c for c in remote_base if c in valid_chars)
remote_base = remote_base.replace(' ', '_') # No spaces in filenames
remote_map = {
"16s_community": ["genome", "microbiome", "16s", "analysis", "hmqcp"],
"wgs_community": ["genome", "microbiome", "wgs", "analysis", "hmscp"],
"wgs_functional": ["genome", "microbiome", "wgs", "analysis", "hmmrc"],
"microb_proteomic": ["proteome", "microbiome", "analysis"],
"microb_lipidomic": ["lipidome", "microbiome", "analysis"],
"microb_cytokine": ["cytokine", "microbiome", "analysis"],
"microb_metabolome": ["metabolome", "microbiome", "analysis"],
"microb_metatranscriptome": ["metatranscriptome", "microbiome", "analysis"],
"host_proteomic": ["proteome", "host", "analysis"],
"host_lipidomic": ["lipidome", "host", "analysis"],
"host_cytokine": ["cytokine", "host", "analysis"],
"host_metabolome": ["metabolome", "host", "analysis"],
"host_transcriptome": ["transcriptome", "host", "analysis"]
}
matrix_type = self._matrix_type
if matrix_type not in remote_map:
raise ValueError("Invalid matrix type. No mapping for %s" % matrix_type)
remote_elements = [study_dir]
remote_elements.extend(remote_map[matrix_type])
remote_elements.append(remote_base)
remote_path = "/" + "/".join(remote_elements)
self.logger.debug("Remote path for this abundance matrix will be %s.",
remote_path
)
# Upload the file to the iHMP aspera server
upload_result = aspera.upload_file(AbundanceMatrix.aspera_server,
session.username,
session.password,
self._local_file,
remote_path)
if not upload_result:
self.logger.error("Experienced an error uploading the data. " + \
"Aborting save.")
raise Exception("Unable to upload abundance matrix.")
else:
self._urls = ["fasp://" + AbundanceMatrix.aspera_server + remote_path]
