def check_all_input_data(self,
protocal=None,
address=None,
input_data_root=None,
data_list_dir="Buildconf",
download=True):
###############################################################################
success = False
if protocal is not None and address is not None:
success = self.check_input_data(protocal=protocal,
address=address,
download=download,
input_data_root=input_data_root,
data_list_dir=data_list_dir)
else:
inputdata = Inputdata()
while not success:
if download:
protocal, address = inputdata.get_next_server()
expect(protocal is not None, "Failed to find input data")
logger.info("Checking server {} with protocal {}".format(
address, protocal))
success = self.check_input_data(protocal=protocal,
address=address,
download=download,
input_data_root=input_data_root,
data_list_dir=data_list_dir)
self.stage_refcase()
def download_file(input_root, the_file):
###############################################################################
inputdata = Inputdata()
protocol = "wget"
success = False
while not success and protocol is not None:
protocol, address, user, passwd, _, ic_filepath, _ = inputdata.get_next_server(
)
if protocol is not None:
if protocol == "svn":
from CIME.Servers import SVN
server = SVN(address, user, passwd)
elif protocol == "gftp":
from CIME.Servers import GridFTP
server = GridFTP(address, user, passwd)
elif protocol == "ftp":
from CIME.Servers import FTP
server = FTP.ftp_login(address, user, passwd)
elif protocol == "wget":
from CIME.Servers import WGET
server = WGET.wget_login(address, user, passwd)
else:
expect(False,
"Unsupported inputdata protocol: {}".format(protocol))
print(
" Attempting to download {} from {} with protocol {}".format(
the_file, address, protocol))
success = _download_if_in_repo(server, input_root,
the_file.strip(os.sep))
print(" {}".format("SUCCESS" if success else "FAILED"))
return success
def _download_checksum_file(rundir):
"""
Download the checksum files from each server and merge them into rundir.
"""
inputdata = Inputdata()
protocol = "svn"
# download and merge all available chksum files.
while protocol is not None:
protocol, address, user, passwd, chksum_file = inputdata.get_next_server(
)
if protocol not in vars(CIME.Servers):
logger.warning("Client protocol {} not enabled".format(protocol))
continue
logger.info("Using protocol {} with user {} and passwd {}".format(
protocol, user, passwd))
if protocol == "svn":
server = CIME.Servers.SVN(address, user, passwd)
elif protocol == "gftp":
server = CIME.Servers.GridFTP(address, user, passwd)
elif protocol == "ftp":
server = CIME.Servers.FTP(address, user, passwd)
elif protocol == "wget":
server = CIME.Servers.WGET(address, user, passwd)
else:
expect(False,
"Unsupported inputdata protocol: {}".format(protocol))
if not chksum_file:
continue
success = False
rel_path = chksum_file
full_path = os.path.join(rundir, local_chksum_file)
new_file = full_path + '.raw'
protocol = type(server).__name__
logging.info(
"Trying to download file: '{}' to path '{}' using {} protocol.".
format(rel_path, new_file, protocol))
tmpfile = None
if os.path.isfile(full_path):
tmpfile = full_path + ".tmp"
os.rename(full_path, tmpfile)
# Use umask to make sure files are group read/writable. As long as parent directories
# have +s, then everything should work.
with SharedArea():
success = server.getfile(rel_path, new_file)
if success:
_reformat_chksum_file(full_path, new_file)
if tmpfile:
_merge_chksum_files(full_path, tmpfile)
chksum_hash.clear()
else:
if tmpfile and os.path.isfile(tmpfile):
os.rename(tmpfile, full_path)
logger.warning("Could not automatically download file " +
full_path + " Restoring existing version.")
else:
logger.warning(
"Could not automatically download file {}".format(
full_path))
def _download_checksum_file(rundir):
"""
Download the checksum files from each server and merge them into rundir.
"""
inputdata = Inputdata()
protocol = "svn"
# download and merge all available chksum files.
while protocol is not None:
protocol, address, user, passwd, chksum_file = inputdata.get_next_server()
if protocol not in vars(CIME.Servers):
logger.warning("Client protocol {} not enabled".format(protocol))
continue
logger.info("Using protocol {} with user {} and passwd {}".format(protocol, user, passwd))
if protocol == "svn":
server = CIME.Servers.SVN(address, user, passwd)
elif protocol == "gftp":
server = CIME.Servers.GridFTP(address, user, passwd)
elif protocol == "ftp":
server = CIME.Servers.FTP(address, user, passwd)
elif protocol == "wget":
server = CIME.Servers.WGET(address, user, passwd)
else:
expect(False, "Unsupported inputdata protocol: {}".format(protocol))
if not chksum_file:
continue
success = False
rel_path = chksum_file
full_path = os.path.join(rundir, local_chksum_file)
new_file = full_path + '.raw'
protocol = type(server).__name__
logging.info("Trying to download file: '{}' to path '{}' using {} protocol.".format(rel_path, new_file, protocol))
tmpfile = None
if os.path.isfile(full_path):
tmpfile = full_path+".tmp"
os.rename(full_path, tmpfile)
# Use umask to make sure files are group read/writable. As long as parent directories
# have +s, then everything should work.
with SharedArea():
success = server.getfile(rel_path, new_file)
if success:
_reformat_chksum_file(full_path, new_file)
if tmpfile:
_merge_chksum_files(full_path, tmpfile)
chksum_hash.clear()
else:
if tmpfile and os.path.isfile(tmpfile):
os.rename(tmpfile, full_path)
logger.warning("Could not automatically download file "+full_path+
" Restoring existing version.")
else:
logger.warning("Could not automatically download file {}".
format(full_path))
def _downloadfromserver(case, input_data_root, data_list_dir):
# needs to be downloaded
success = False
protocol = 'svn'
inputdata = Inputdata()
while not success and protocol is not None:
protocol, address = inputdata.get_next_server()
logger.info("Checking server {} with protocol {}".format(address, protocol))
success = case.check_input_data(protocol=protocol, address=address, download=True,
input_data_root=input_data_root, data_list_dir=data_list_dir)
return success
def _downloadfromserver(case, input_data_root, data_list_dir):
"""
Download files
"""
success = False
protocol = 'svn'
inputdata = Inputdata()
if not input_data_root:
input_data_root = case.get_value('DIN_LOC_ROOT')
while not success and protocol is not None:
protocol, address, user, passwd, _ = inputdata.get_next_server()
logger.info("Checking server {} with protocol {}".format(address, protocol))
success = case.check_input_data(protocol=protocol, address=address, download=True,
input_data_root=input_data_root,
data_list_dir=data_list_dir,
user=user, passwd=passwd)
return success
def check_all_input_data(self, protocal=None, address=None, input_data_root=None, data_list_dir="Buildconf", download=True):
###############################################################################
success = False
if protocal is not None and address is not None:
success = self.check_input_data(protocal=protocal, address=address, download=download,
input_data_root=input_data_root, data_list_dir=data_list_dir)
else:
inputdata = Inputdata()
while not success:
if download:
protocal, address = inputdata.get_next_server()
expect(protocal is not None, "Failed to find input data")
logger.info("Checking server {} with protocal {}".format(address, protocal))
success = self.check_input_data(protocal=protocal, address=address, download=download,
input_data_root=input_data_root, data_list_dir=data_list_dir)
self.stage_refcase()
data_list_dir=data_list_dir)
return success
def _downloadfromserver(case, input_data_root, data_list_dir, attributes=None):
"""
Download files
"""
success = False
protocol = 'svn'
inputdata = Inputdata()
if not input_data_root:
input_data_root = case.get_value('DIN_LOC_ROOT')
while not success and protocol is not None:
protocol, address, user, passwd, _, ic_filepath = inputdata.get_next_server(
attributes=attributes)
logger.info("Checking server {} with protocol {}".format(
address, protocol))
success = case.check_input_data(protocol=protocol,
address=address,
download=True,
input_data_root=input_data_root,
data_list_dir=data_list_dir,
user=user,
passwd=passwd,
ic_filepath=ic_filepath)
return success
def stage_refcase(self, input_data_root=None, data_list_dir=None):
"""
data_list_dir=data_list_dir)
return success
def _downloadfromserver(case, input_data_root, data_list_dir):
"""
Download files
"""
success = False
protocol = 'svn'
inputdata = Inputdata()
if not input_data_root:
input_data_root = case.get_value('DIN_LOC_ROOT')
while not success and protocol is not None:
protocol, address, user, passwd, _ = inputdata.get_next_server()
logger.info("Checking server {} with protocol {}".format(
address, protocol))
success = case.check_input_data(protocol=protocol,
address=address,
download=True,
input_data_root=input_data_root,
data_list_dir=data_list_dir,
user=user,
passwd=passwd)
return success
def stage_refcase(self, input_data_root=None, data_list_dir=None):
"""
Get a REFCASE for a hybrid or branch run
success = _downloadfromserver(self, input_data_root, data_list_dir)
expect(not download or (download and success),
"Could not find all inputdata on any server")
self.stage_refcase(input_data_root=input_data_root,
data_list_dir=data_list_dir)
return success
def _downloadfromserver(case, input_data_root, data_list_dir):
# needs to be downloaded
success = False
protocol = 'svn'
inputdata = Inputdata()
while not success and protocol is not None:
protocol, address = inputdata.get_next_server()
logger.info("Checking server {} with protocol {}".format(
address, protocol))
success = case.check_input_data(protocol=protocol,
address=address,
download=True,
input_data_root=input_data_root,
data_list_dir=data_list_dir)
return success
def stage_refcase(self, input_data_root=None, data_list_dir=None):
get_refcase = self.get_value("GET_REFCASE")
run_type = self.get_value("RUN_TYPE")
continue_run = self.get_value("CONTINUE_RUN")
