def check_all_input_data(self, protocal=None, address=None, input_data_root=None, data_list_dir="Buildconf", download=True): ############################################################################### success = False if protocal is not None and address is not None: success = self.check_input_data(protocal=protocal, address=address, download=download, input_data_root=input_data_root, data_list_dir=data_list_dir) else: inputdata = Inputdata() while not success: if download: protocal, address = inputdata.get_next_server() expect(protocal is not None, "Failed to find input data") logger.info("Checking server {} with protocal {}".format( address, protocal)) success = self.check_input_data(protocal=protocal, address=address, download=download, input_data_root=input_data_root, data_list_dir=data_list_dir) self.stage_refcase()
def download_file(input_root, the_file): ############################################################################### inputdata = Inputdata() protocol = "wget" success = False while not success and protocol is not None: protocol, address, user, passwd, _, ic_filepath, _ = inputdata.get_next_server( ) if protocol is not None: if protocol == "svn": from CIME.Servers import SVN server = SVN(address, user, passwd) elif protocol == "gftp": from CIME.Servers import GridFTP server = GridFTP(address, user, passwd) elif protocol == "ftp": from CIME.Servers import FTP server = FTP.ftp_login(address, user, passwd) elif protocol == "wget": from CIME.Servers import WGET server = WGET.wget_login(address, user, passwd) else: expect(False, "Unsupported inputdata protocol: {}".format(protocol)) print( " Attempting to download {} from {} with protocol {}".format( the_file, address, protocol)) success = _download_if_in_repo(server, input_root, the_file.strip(os.sep)) print(" {}".format("SUCCESS" if success else "FAILED")) return success
def _download_checksum_file(rundir): """ Download the checksum files from each server and merge them into rundir. """ inputdata = Inputdata() protocol = "svn" # download and merge all available chksum files. while protocol is not None: protocol, address, user, passwd, chksum_file = inputdata.get_next_server( ) if protocol not in vars(CIME.Servers): logger.warning("Client protocol {} not enabled".format(protocol)) continue logger.info("Using protocol {} with user {} and passwd {}".format( protocol, user, passwd)) if protocol == "svn": server = CIME.Servers.SVN(address, user, passwd) elif protocol == "gftp": server = CIME.Servers.GridFTP(address, user, passwd) elif protocol == "ftp": server = CIME.Servers.FTP(address, user, passwd) elif protocol == "wget": server = CIME.Servers.WGET(address, user, passwd) else: expect(False, "Unsupported inputdata protocol: {}".format(protocol)) if not chksum_file: continue success = False rel_path = chksum_file full_path = os.path.join(rundir, local_chksum_file) new_file = full_path + '.raw' protocol = type(server).__name__ logging.info( "Trying to download file: '{}' to path '{}' using {} protocol.". format(rel_path, new_file, protocol)) tmpfile = None if os.path.isfile(full_path): tmpfile = full_path + ".tmp" os.rename(full_path, tmpfile) # Use umask to make sure files are group read/writable. As long as parent directories # have +s, then everything should work. with SharedArea(): success = server.getfile(rel_path, new_file) if success: _reformat_chksum_file(full_path, new_file) if tmpfile: _merge_chksum_files(full_path, tmpfile) chksum_hash.clear() else: if tmpfile and os.path.isfile(tmpfile): os.rename(tmpfile, full_path) logger.warning("Could not automatically download file " + full_path + " Restoring existing version.") else: logger.warning( "Could not automatically download file {}".format( full_path))
def _download_checksum_file(rundir): """ Download the checksum files from each server and merge them into rundir. """ inputdata = Inputdata() protocol = "svn" # download and merge all available chksum files. while protocol is not None: protocol, address, user, passwd, chksum_file = inputdata.get_next_server() if protocol not in vars(CIME.Servers): logger.warning("Client protocol {} not enabled".format(protocol)) continue logger.info("Using protocol {} with user {} and passwd {}".format(protocol, user, passwd)) if protocol == "svn": server = CIME.Servers.SVN(address, user, passwd) elif protocol == "gftp": server = CIME.Servers.GridFTP(address, user, passwd) elif protocol == "ftp": server = CIME.Servers.FTP(address, user, passwd) elif protocol == "wget": server = CIME.Servers.WGET(address, user, passwd) else: expect(False, "Unsupported inputdata protocol: {}".format(protocol)) if not chksum_file: continue success = False rel_path = chksum_file full_path = os.path.join(rundir, local_chksum_file) new_file = full_path + '.raw' protocol = type(server).__name__ logging.info("Trying to download file: '{}' to path '{}' using {} protocol.".format(rel_path, new_file, protocol)) tmpfile = None if os.path.isfile(full_path): tmpfile = full_path+".tmp" os.rename(full_path, tmpfile) # Use umask to make sure files are group read/writable. As long as parent directories # have +s, then everything should work. with SharedArea(): success = server.getfile(rel_path, new_file) if success: _reformat_chksum_file(full_path, new_file) if tmpfile: _merge_chksum_files(full_path, tmpfile) chksum_hash.clear() else: if tmpfile and os.path.isfile(tmpfile): os.rename(tmpfile, full_path) logger.warning("Could not automatically download file "+full_path+ " Restoring existing version.") else: logger.warning("Could not automatically download file {}". format(full_path))
def _downloadfromserver(case, input_data_root, data_list_dir): # needs to be downloaded success = False protocol = 'svn' inputdata = Inputdata() while not success and protocol is not None: protocol, address = inputdata.get_next_server() logger.info("Checking server {} with protocol {}".format(address, protocol)) success = case.check_input_data(protocol=protocol, address=address, download=True, input_data_root=input_data_root, data_list_dir=data_list_dir) return success
def _downloadfromserver(case, input_data_root, data_list_dir): """ Download files """ success = False protocol = 'svn' inputdata = Inputdata() if not input_data_root: input_data_root = case.get_value('DIN_LOC_ROOT') while not success and protocol is not None: protocol, address, user, passwd, _ = inputdata.get_next_server() logger.info("Checking server {} with protocol {}".format(address, protocol)) success = case.check_input_data(protocol=protocol, address=address, download=True, input_data_root=input_data_root, data_list_dir=data_list_dir, user=user, passwd=passwd) return success
def check_all_input_data(self, protocal=None, address=None, input_data_root=None, data_list_dir="Buildconf", download=True): ############################################################################### success = False if protocal is not None and address is not None: success = self.check_input_data(protocal=protocal, address=address, download=download, input_data_root=input_data_root, data_list_dir=data_list_dir) else: inputdata = Inputdata() while not success: if download: protocal, address = inputdata.get_next_server() expect(protocal is not None, "Failed to find input data") logger.info("Checking server {} with protocal {}".format(address, protocal)) success = self.check_input_data(protocal=protocal, address=address, download=download, input_data_root=input_data_root, data_list_dir=data_list_dir) self.stage_refcase()
data_list_dir=data_list_dir) return success def _downloadfromserver(case, input_data_root, data_list_dir, attributes=None): """ Download files """ success = False protocol = 'svn' inputdata = Inputdata() if not input_data_root: input_data_root = case.get_value('DIN_LOC_ROOT') while not success and protocol is not None: protocol, address, user, passwd, _, ic_filepath = inputdata.get_next_server( attributes=attributes) logger.info("Checking server {} with protocol {}".format( address, protocol)) success = case.check_input_data(protocol=protocol, address=address, download=True, input_data_root=input_data_root, data_list_dir=data_list_dir, user=user, passwd=passwd, ic_filepath=ic_filepath) return success def stage_refcase(self, input_data_root=None, data_list_dir=None): """
data_list_dir=data_list_dir) return success def _downloadfromserver(case, input_data_root, data_list_dir): """ Download files """ success = False protocol = 'svn' inputdata = Inputdata() if not input_data_root: input_data_root = case.get_value('DIN_LOC_ROOT') while not success and protocol is not None: protocol, address, user, passwd, _ = inputdata.get_next_server() logger.info("Checking server {} with protocol {}".format( address, protocol)) success = case.check_input_data(protocol=protocol, address=address, download=True, input_data_root=input_data_root, data_list_dir=data_list_dir, user=user, passwd=passwd) return success def stage_refcase(self, input_data_root=None, data_list_dir=None): """ Get a REFCASE for a hybrid or branch run
success = _downloadfromserver(self, input_data_root, data_list_dir) expect(not download or (download and success), "Could not find all inputdata on any server") self.stage_refcase(input_data_root=input_data_root, data_list_dir=data_list_dir) return success def _downloadfromserver(case, input_data_root, data_list_dir): # needs to be downloaded success = False protocol = 'svn' inputdata = Inputdata() while not success and protocol is not None: protocol, address = inputdata.get_next_server() logger.info("Checking server {} with protocol {}".format( address, protocol)) success = case.check_input_data(protocol=protocol, address=address, download=True, input_data_root=input_data_root, data_list_dir=data_list_dir) return success def stage_refcase(self, input_data_root=None, data_list_dir=None): get_refcase = self.get_value("GET_REFCASE") run_type = self.get_value("RUN_TYPE") continue_run = self.get_value("CONTINUE_RUN")