def _actually_du(): srv, remote_path = parse_fasp_url(url) bn = basename(remote_path) local_file = join(local_dir, bn) skip = (bn in local_cached and os.stat(local_file).st_size == remote_size) if skip == False: ret = asp.download_file(srv, dcc_user, dcc_pw, remote_path, local_dir) if not ret: raise Exception("Download failed: " + url) to_rm, files_to_upload = untar(local_file) for i, f in enumerate(files_to_upload): new_f = addtag(f, namespace) os.rename(f, new_f) files_to_upload[i] = new_f names_sizes = [(basename(f), os.stat(f).st_size) for f in files_to_upload] for f in files_to_upload: ret = asp.upload_file(ncbi_srv, ncbi_user, None, f, ncbi_path, keyfile=ncbi_keyfile) with open(local_file + "." + namespace + ".complete", 'w') as f: for name_size in names_sizes: print >> f, "\t".join(map(str, name_size)) for f in reversed(to_rm): try: os.rmdir(f) if os.path.isdir(f) else os.remove(f) except: print >> sys.stderr, "Unable to remove " + f
def _upload_data(self): self.logger.debug("In _upload_data.") session = iHMPSession.get_session() study = self._study study2dir = {"ibd": "ibd", "preg_preterm": "ptb", "prediabetes": "t2d"} if study not in study2dir: raise ValueError("Invalid study. No directory mapping for %s" % study) study_dir = study2dir[study] remote_base = os.path.basename(self._local_file) valid_chars = "-_.%s%s" % (string.ascii_letters, string.digits) remote_base = ''.join(c for c in remote_base if c in valid_chars) remote_base = remote_base.replace(' ', '_') # No spaces in filenames remote_path = "/".join( ["/" + study_dir, "cytokine", "host", remote_base]) self.logger.debug("Remote path for this file will be %s.", remote_path) upload_result = aspera.upload_file(Cytokine.aspera_server, session.username, session.password, self._local_file, remote_path) if not upload_result: self.logger.error("Experienced an error uploading the data. " + \ "Aborting save.") raise Exception("Unable to upload cytokine.") else: self._urls = ["fasp://" + Cytokine.aspera_server + remote_path]
def _u(): ret = asp.upload_file(remote_srv, user, None, local_fname, remote_path, keyfile=keyfile) if blithely or ret: open(complete_fname, 'w').close() return blithely or ret # return True if blithely is True
def _upload_files(self, file_map): self.logger.debug("In _upload_files.") study2dir = {"ibd": "ibd", "preg_preterm": "ptb", "prediabetes": "t2d"} study = self.study subtype = self.subtype if study not in study2dir: raise ValueError("Invalid study. No directory mapping for %s" % study) study_dir = study2dir[study] remote_paths = {} # Get the session so we can get the username and password session = iHMPSession.get_session() username = session.username password = session.password # For each of the Proteome data files (there are 4), transmit them # to the Aspera server and return a dictionary with the computed remote # paths... for file_type, local_file in file_map.iteritems(): self.logger.debug("Uploading %s of %s type %s", __name__, local_file, file_type) remote_base = os.path.basename(local_file) valid_chars = "-_.%s%s" % (string.ascii_letters, string.digits) remote_base = ''.join(c for c in remote_base if c in valid_chars) remote_base = remote_base.replace(' ', '_') # No spaces in filenames remote_path = "/".join([ "/" + study_dir, "proteome_nonpride", subtype, file_type, remote_base ]) self.logger.debug("Remote path for this file will be %s.", remote_path) # Upload the file to the iHMP aspera server upload_success = aspera.upload_file(ProteomeNonPride.aspera_server, username, password, local_file, remote_path) if not upload_success: self.logger.error("Experienced an error uploading file %s.", local_file) raise Exception("Unable to upload " + local_file) else: remote_paths[ file_type] = "fasp://" + ProteomeNonPride.aspera_server + remote_path return remote_paths
def _u(): with open(sub_fname, 'r') as f: b = basename(local_fname) if b not in ("submission.xml", "submit.ready") \ and b not in f.read(): return ret = asp.upload_file(remote_srv, user, None, local_fname, remote_path, keyfile=keyfile) if blithely or ret: open(complete_fname, 'w').close() return blithely or ret # return True if blithely is True
def _upload_data(self): self.logger.debug("In _upload_data.") session = iHMPSession.get_session() study = self._study study2dir = { "ibd": "ibd", "preg_preterm": "ptb", "prediabetes": "t2d", "risk": "risk", "protect": "protect", "mucosal_ibd": "mucosal_ibd", "herfarth_ccfa": "herfarth_ccfa", "jl_crohns": "jl_crohns" } if study not in study2dir: raise ValueError("Invalid study. No directory mapping for %s" % study) study_dir = study2dir[study] remote_base = os.path.basename(self._local_file) valid_chars = "-_.%s%s" % (string.ascii_letters, string.digits) remote_base = ''.join(c for c in remote_base if c in valid_chars) remote_base = remote_base.replace(' ', '_') # No spaces in filenames remote_path = "/".join([ "/" + study_dir, "genome", "microbiome", "16s", "raw", remote_base ]) self.logger.debug("Remote path for this file will be %s.", remote_path) # Upload the file to the iHMP aspera server upload_result = aspera.upload_file(SixteenSRawSeqSet.aspera_server, session.username, session.password, self._local_file, remote_path) if not upload_result: self.logger.error("Experienced an error uploading the data. " + \ "Aborting save.") raise Exception("Unable to load 16S raw sequence set.") else: self._urls = [ "fasp://" + SixteenSRawSeqSet.aspera_server + remote_path ]
def _upload_data(self): self.logger.debug("In _upload_data.") session = iHMPSession.get_session() study = self._study study2dir = { "ibd": "ibd", "preg_preterm": "ptb", "prediabetes": "t2d" } if study not in study2dir: raise ValueError("Invalid study. No directory mapping for %s" % study) study_dir = study2dir[study] remote_base = os.path.basename(self._local_file) valid_chars = "-_.%s%s" % (string.ascii_letters, string.digits) remote_base = ''.join(c for c in remote_base if c in valid_chars) remote_base = remote_base.replace(' ', '_') # No spaces in filenames remote_map = { "16s_community": ["genome", "microbiome", "16s", "analysis", "hmqcp"], "wgs_community": ["genome", "microbiome", "wgs", "analysis", "hmscp"], "wgs_functional": ["genome", "microbiome", "wgs", "analysis", "hmmrc"], "microb_proteomic": ["proteome", "microbiome", "analysis"], "microb_lipidomic": ["lipidome", "microbiome", "analysis"], "microb_cytokine": ["cytokine", "microbiome", "analysis"], "microb_metabolome": ["metabolome", "microbiome", "analysis"], "microb_metatranscriptome": ["metatranscriptome", "microbiome", "analysis"], "host_proteomic": ["proteome", "host", "analysis"], "host_lipidomic": ["lipidome", "host", "analysis"], "host_cytokine": ["cytokine", "host", "analysis"], "host_metabolome": ["metabolome", "host", "analysis"], "host_transcriptome": ["transcriptome", "host", "analysis"] } matrix_type = self._matrix_type if matrix_type not in remote_map: raise ValueError("Invalid matrix type. No mapping for %s" % matrix_type) remote_elements = [study_dir] remote_elements.extend(remote_map[matrix_type]) remote_elements.append(remote_base) remote_path = "/" + "/".join(remote_elements) self.logger.debug("Remote path for this abundance matrix will be %s.", remote_path ) # Upload the file to the iHMP aspera server upload_result = aspera.upload_file(AbundanceMatrix.aspera_server, session.username, session.password, self._local_file, remote_path) if not upload_result: self.logger.error("Experienced an error uploading the data. " + \ "Aborting save.") raise Exception("Unable to upload abundance matrix.") else: self._urls = ["fasp://" + AbundanceMatrix.aspera_server + remote_path]