def _check_disposition(testdir):
copyfilelist = []
for root, _, files in os.walk(testdir):
for _file in files:
with open(os.path.join(root, _file), "r") as fd:
disposition = fd.readline()
logger.info("Checking testfile {} with disposition {}".format(
_file, disposition))
if root == testdir:
if "move" in disposition:
if find_files(os.path.join(testdir, "archive"), _file):
expect(
False,
"Copied file {} to archive with disposition move".
format(_file))
else:
expect(
False,
"Failed to move file {} to archive".format(_file))
if "copy" in disposition:
copyfilelist.append(_file)
elif "ignore" in disposition:
expect(
False,
"Moved file {} with dispostion ignore to directory {}".
format(_file, root))
elif "copy" in disposition:
expect(
_file in copyfilelist,
"File {} with disposition copy was moved to directory {}".
format(_file, root))
for _file in copyfilelist:
expect(
find_files(os.path.join(testdir, "archive"), _file) != [],
"File {} was not copied to archive.".format(_file))
def _check_disposition(testdir):
copyfilelist = []
for root, _, files in os.walk(testdir):
for _file in files:
with open(os.path.join(root, _file), "r") as fd:
disposition = fd.readline()
logger.info("Checking testfile {} with disposition {}".format(_file, disposition))
if root == testdir:
if "move" in disposition:
if find_files(os.path.join(testdir, "archive"), _file):
expect(False,
"Copied file {} to archive with disposition move".format(_file))
else:
expect(False,
"Failed to move file {} to archive".format(_file))
if "copy" in disposition:
copyfilelist.append(_file)
elif "ignore" in disposition:
expect(False, "Moved file {} with dispostion ignore to directory {}".format(_file, root))
elif "copy" in disposition:
expect(_file in copyfilelist, "File {} with disposition copy was moved to directory {}"
.format(_file, root))
for _file in copyfilelist:
expect(find_files(os.path.join(testdir,"archive"), _file) != [],
"File {} was not copied to archive.".format(_file))
For a given case check for the relevant input data as specified in data_list_dir/*.input_data_list
in the directory input_data_root, if not found optionally download it using the servers specified
in config_inputdata.xml. If a chksum file is available compute the chksum and compare it to that
in the file.
Return True if no files missing
"""
case.load_env(reset=True)
rundir = case.get_value("RUNDIR")
# Fill in defaults as needed
input_data_root = case.get_value(
"DIN_LOC_ROOT") if input_data_root is None else input_data_root
input_ic_root = case.get_value("DIN_LOC_IC", resolved=True)
expect(os.path.isdir(data_list_dir),
"Invalid data_list_dir directory: '{}'".format(data_list_dir))
data_list_files = find_files(data_list_dir, "*.input_data_list")
if not data_list_files:
logger.warning(
"WARNING: No .input_data_list files found in dir '{}'".format(
data_list_dir))
no_files_missing = True
if download:
if protocol not in vars(CIME.Servers):
logger.info("Client protocol {} not enabled".format(protocol))
return False
logger.info("Using protocol {} with user {} and passwd {}".format(
protocol, user, passwd))
if protocol == "svn":
server = CIME.Servers.SVN(address, user, passwd)
elif protocol == "gftp":
def check_input_data(case, protocol="svn", address=None, input_data_root=None, data_list_dir="Buildconf",
download=False, user=None, passwd=None, chksum=False):
"""
For a given case check for the relevant input data as specified in data_list_dir/*.input_data_list
in the directory input_data_root, if not found optionally download it using the servers specified
in config_inputdata.xml. If a chksum file is available compute the chksum and compare it to that
in the file.
Return True if no files missing
"""
case.load_env(reset=True)
rundir = case.get_value("RUNDIR")
# Fill in defaults as needed
input_data_root = case.get_value("DIN_LOC_ROOT") if input_data_root is None else input_data_root
expect(os.path.isdir(data_list_dir), "Invalid data_list_dir directory: '{}'".format(data_list_dir))
data_list_files = find_files(data_list_dir, "*.input_data_list")
expect(data_list_files, "No .input_data_list files found in dir '{}'".format(data_list_dir))
no_files_missing = True
if download:
if protocol not in vars(CIME.Servers):
logger.warning("Client protocol {} not enabled".format(protocol))
return False
logger.info("Using protocol {} with user {} and passwd {}".format(protocol, user, passwd))
if protocol == "svn":
server = CIME.Servers.SVN(address, user, passwd)
elif protocol == "gftp":
server = CIME.Servers.GridFTP(address, user, passwd)
elif protocol == "ftp":
server = CIME.Servers.FTP(address, user, passwd)
elif protocol == "wget":
server = CIME.Servers.WGET(address, user, passwd)
else:
expect(False, "Unsupported inputdata protocol: {}".format(protocol))
for data_list_file in data_list_files:
logging.info("Loading input file list: '{}'".format(data_list_file))
with open(data_list_file, "r") as fd:
lines = fd.readlines()
for line in lines:
line = line.strip()
if (line and not line.startswith("#")):
tokens = line.split('=')
description, full_path = tokens[0].strip(), tokens[1].strip()
if description.endswith('datapath'):
continue
if(full_path):
# expand xml variables
full_path = case.get_resolved_value(full_path)
rel_path = full_path.replace(input_data_root, "")
model = os.path.basename(data_list_file).split('.')[0]
if ("/" in rel_path and rel_path == full_path):
# User pointing to a file outside of input_data_root, we cannot determine
# rel_path, and so cannot download the file. If it already exists, we can
# proceed
if not os.path.exists(full_path):
logging.warning("Model {} missing file {} = '{}'".format(model, description, full_path))
if download:
logging.warning(" Cannot download file since it lives outside of the input_data_root '{}'".format(input_data_root))
no_files_missing = False
else:
logging.debug(" Found input file: '{}'".format(full_path))
else:
# There are some special values of rel_path that
# we need to ignore - some of the component models
# set things like 'NULL' or 'same_as_TS' -
# basically if rel_path does not contain '/' (a
# directory tree) you can assume it's a special
# value and ignore it (perhaps with a warning)
isdirectory=rel_path.endswith(os.sep)
if ("/" in rel_path and not os.path.exists(full_path)):
logger.warning(" Model {} missing file {} = '{}'".format(model, description, full_path))
no_files_missing = False
if (download):
no_files_missing = _download_if_in_repo(server,
input_data_root, rel_path.strip(os.sep),
isdirectory=isdirectory)
if no_files_missing:
verify_chksum(input_data_root, rundir, rel_path.strip(os.sep), isdirectory)
else:
if chksum:
verify_chksum(input_data_root, rundir, rel_path.strip(os.sep), isdirectory)
logger.info("Chksum passed for file {}".format(os.path.join(input_data_root,rel_path)))
logging.debug(" Already had input file: '{}'".format(full_path))
else:
model = os.path.basename(data_list_file).split('.')[0]
logging.warning("Model {} no file specified for {}".format(model, description))
return no_files_missing
def check_input_data(case, protocal="svn", address=None, input_data_root=None, data_list_dir="Buildconf", download=False):
"""
Return True if no files missing
"""
# Fill in defaults as needed
input_data_root = case.get_value("DIN_LOC_ROOT") if input_data_root is None else input_data_root
expect(os.path.isdir(input_data_root), "Invalid input_data_root directory: '{}'".format(input_data_root))
expect(os.path.isdir(data_list_dir), "Invalid data_list_dir directory: '{}'".format(data_list_dir))
data_list_files = find_files(data_list_dir, "*.input_data_list")
expect(data_list_files, "No .input_data_list files found in dir '{}'".format(data_list_dir))
no_files_missing = True
if download:
if protocal not in vars(CIME.Servers):
logger.warning("Client protocal {} not enabled".format(protocal))
return False
if protocal == "svn":
server = CIME.Servers.SVN(address)
elif protocal == "gftp":
server = CIME.Servers.GridFTP(address)
elif protocal == "ftp":
server = CIME.Servers.FTP(address)
elif protocal == "wget":
server = CIME.Servers.WGET(address)
else:
expect(False, "Unsupported inputdata protocal: {}".format(protocal))
for data_list_file in data_list_files:
logging.info("Loading input file list: '{}'".format(data_list_file))
with open(data_list_file, "r") as fd:
lines = fd.readlines()
for line in lines:
line = line.strip()
if (line and not line.startswith("#")):
tokens = line.split('=')
description, full_path = tokens[0].strip(), tokens[1].strip()
if(full_path):
# expand xml variables
full_path = case.get_resolved_value(full_path)
rel_path = full_path.replace(input_data_root, "")
model = os.path.basename(data_list_file).split('.')[0]
if ("/" in rel_path and rel_path == full_path):
# User pointing to a file outside of input_data_root, we cannot determine
# rel_path, and so cannot download the file. If it already exists, we can
# proceed
if not os.path.exists(full_path):
logging.warning(" Model {} missing file {} = '{}'".format(model, description, full_path))
if download:
logging.warning(" Cannot download file since it lives outside of the input_data_root '{}'".format(input_data_root))
no_files_missing = False
else:
logging.debug(" Found input file: '{}'".format(full_path))
else:
# There are some special values of rel_path that
# we need to ignore - some of the component models
# set things like 'NULL' or 'same_as_TS' -
# basically if rel_path does not contain '/' (a
# directory tree) you can assume it's a special
# value and ignore it (perhaps with a warning)
if ("/" in rel_path and not os.path.exists(full_path)):
logging.warning(" Model {} missing file {} = '{}'".format(model, description, full_path))
if (download):
success = _download_if_in_repo(server, input_data_root, rel_path.strip('/'))
if not success:
no_files_missing = False
else:
logging.debug(" Already had input file: '{}'".format(full_path))
else:
model = os.path.basename(data_list_file).split('.')[0]
logging.warning("Model {} no file specified for {}".format(model, description))
return no_files_missing
