def _check_disposition(testdir): copyfilelist = [] for root, _, files in os.walk(testdir): for _file in files: with open(os.path.join(root, _file), "r") as fd: disposition = fd.readline() logger.info("Checking testfile {} with disposition {}".format( _file, disposition)) if root == testdir: if "move" in disposition: if find_files(os.path.join(testdir, "archive"), _file): expect( False, "Copied file {} to archive with disposition move". format(_file)) else: expect( False, "Failed to move file {} to archive".format(_file)) if "copy" in disposition: copyfilelist.append(_file) elif "ignore" in disposition: expect( False, "Moved file {} with dispostion ignore to directory {}". format(_file, root)) elif "copy" in disposition: expect( _file in copyfilelist, "File {} with disposition copy was moved to directory {}". format(_file, root)) for _file in copyfilelist: expect( find_files(os.path.join(testdir, "archive"), _file) != [], "File {} was not copied to archive.".format(_file))
def _check_disposition(testdir): copyfilelist = [] for root, _, files in os.walk(testdir): for _file in files: with open(os.path.join(root, _file), "r") as fd: disposition = fd.readline() logger.info("Checking testfile {} with disposition {}".format(_file, disposition)) if root == testdir: if "move" in disposition: if find_files(os.path.join(testdir, "archive"), _file): expect(False, "Copied file {} to archive with disposition move".format(_file)) else: expect(False, "Failed to move file {} to archive".format(_file)) if "copy" in disposition: copyfilelist.append(_file) elif "ignore" in disposition: expect(False, "Moved file {} with dispostion ignore to directory {}".format(_file, root)) elif "copy" in disposition: expect(_file in copyfilelist, "File {} with disposition copy was moved to directory {}" .format(_file, root)) for _file in copyfilelist: expect(find_files(os.path.join(testdir,"archive"), _file) != [], "File {} was not copied to archive.".format(_file))
For a given case check for the relevant input data as specified in data_list_dir/*.input_data_list in the directory input_data_root, if not found optionally download it using the servers specified in config_inputdata.xml. If a chksum file is available compute the chksum and compare it to that in the file. Return True if no files missing """ case.load_env(reset=True) rundir = case.get_value("RUNDIR") # Fill in defaults as needed input_data_root = case.get_value( "DIN_LOC_ROOT") if input_data_root is None else input_data_root input_ic_root = case.get_value("DIN_LOC_IC", resolved=True) expect(os.path.isdir(data_list_dir), "Invalid data_list_dir directory: '{}'".format(data_list_dir)) data_list_files = find_files(data_list_dir, "*.input_data_list") if not data_list_files: logger.warning( "WARNING: No .input_data_list files found in dir '{}'".format( data_list_dir)) no_files_missing = True if download: if protocol not in vars(CIME.Servers): logger.info("Client protocol {} not enabled".format(protocol)) return False logger.info("Using protocol {} with user {} and passwd {}".format( protocol, user, passwd)) if protocol == "svn": server = CIME.Servers.SVN(address, user, passwd) elif protocol == "gftp":
def check_input_data(case, protocol="svn", address=None, input_data_root=None, data_list_dir="Buildconf", download=False, user=None, passwd=None, chksum=False): """ For a given case check for the relevant input data as specified in data_list_dir/*.input_data_list in the directory input_data_root, if not found optionally download it using the servers specified in config_inputdata.xml. If a chksum file is available compute the chksum and compare it to that in the file. Return True if no files missing """ case.load_env(reset=True) rundir = case.get_value("RUNDIR") # Fill in defaults as needed input_data_root = case.get_value("DIN_LOC_ROOT") if input_data_root is None else input_data_root expect(os.path.isdir(data_list_dir), "Invalid data_list_dir directory: '{}'".format(data_list_dir)) data_list_files = find_files(data_list_dir, "*.input_data_list") expect(data_list_files, "No .input_data_list files found in dir '{}'".format(data_list_dir)) no_files_missing = True if download: if protocol not in vars(CIME.Servers): logger.warning("Client protocol {} not enabled".format(protocol)) return False logger.info("Using protocol {} with user {} and passwd {}".format(protocol, user, passwd)) if protocol == "svn": server = CIME.Servers.SVN(address, user, passwd) elif protocol == "gftp": server = CIME.Servers.GridFTP(address, user, passwd) elif protocol == "ftp": server = CIME.Servers.FTP(address, user, passwd) elif protocol == "wget": server = CIME.Servers.WGET(address, user, passwd) else: expect(False, "Unsupported inputdata protocol: {}".format(protocol)) for data_list_file in data_list_files: logging.info("Loading input file list: '{}'".format(data_list_file)) with open(data_list_file, "r") as fd: lines = fd.readlines() for line in lines: line = line.strip() if (line and not line.startswith("#")): tokens = line.split('=') description, full_path = tokens[0].strip(), tokens[1].strip() if description.endswith('datapath'): continue if(full_path): # expand xml variables full_path = case.get_resolved_value(full_path) rel_path = full_path.replace(input_data_root, "") model = os.path.basename(data_list_file).split('.')[0] if ("/" in rel_path and rel_path == full_path): # User pointing to a file outside of input_data_root, we cannot determine # rel_path, and so cannot download the file. If it already exists, we can # proceed if not os.path.exists(full_path): logging.warning("Model {} missing file {} = '{}'".format(model, description, full_path)) if download: logging.warning(" Cannot download file since it lives outside of the input_data_root '{}'".format(input_data_root)) no_files_missing = False else: logging.debug(" Found input file: '{}'".format(full_path)) else: # There are some special values of rel_path that # we need to ignore - some of the component models # set things like 'NULL' or 'same_as_TS' - # basically if rel_path does not contain '/' (a # directory tree) you can assume it's a special # value and ignore it (perhaps with a warning) isdirectory=rel_path.endswith(os.sep) if ("/" in rel_path and not os.path.exists(full_path)): logger.warning(" Model {} missing file {} = '{}'".format(model, description, full_path)) no_files_missing = False if (download): no_files_missing = _download_if_in_repo(server, input_data_root, rel_path.strip(os.sep), isdirectory=isdirectory) if no_files_missing: verify_chksum(input_data_root, rundir, rel_path.strip(os.sep), isdirectory) else: if chksum: verify_chksum(input_data_root, rundir, rel_path.strip(os.sep), isdirectory) logger.info("Chksum passed for file {}".format(os.path.join(input_data_root,rel_path))) logging.debug(" Already had input file: '{}'".format(full_path)) else: model = os.path.basename(data_list_file).split('.')[0] logging.warning("Model {} no file specified for {}".format(model, description)) return no_files_missing
def check_input_data(case, protocal="svn", address=None, input_data_root=None, data_list_dir="Buildconf", download=False): """ Return True if no files missing """ # Fill in defaults as needed input_data_root = case.get_value("DIN_LOC_ROOT") if input_data_root is None else input_data_root expect(os.path.isdir(input_data_root), "Invalid input_data_root directory: '{}'".format(input_data_root)) expect(os.path.isdir(data_list_dir), "Invalid data_list_dir directory: '{}'".format(data_list_dir)) data_list_files = find_files(data_list_dir, "*.input_data_list") expect(data_list_files, "No .input_data_list files found in dir '{}'".format(data_list_dir)) no_files_missing = True if download: if protocal not in vars(CIME.Servers): logger.warning("Client protocal {} not enabled".format(protocal)) return False if protocal == "svn": server = CIME.Servers.SVN(address) elif protocal == "gftp": server = CIME.Servers.GridFTP(address) elif protocal == "ftp": server = CIME.Servers.FTP(address) elif protocal == "wget": server = CIME.Servers.WGET(address) else: expect(False, "Unsupported inputdata protocal: {}".format(protocal)) for data_list_file in data_list_files: logging.info("Loading input file list: '{}'".format(data_list_file)) with open(data_list_file, "r") as fd: lines = fd.readlines() for line in lines: line = line.strip() if (line and not line.startswith("#")): tokens = line.split('=') description, full_path = tokens[0].strip(), tokens[1].strip() if(full_path): # expand xml variables full_path = case.get_resolved_value(full_path) rel_path = full_path.replace(input_data_root, "") model = os.path.basename(data_list_file).split('.')[0] if ("/" in rel_path and rel_path == full_path): # User pointing to a file outside of input_data_root, we cannot determine # rel_path, and so cannot download the file. If it already exists, we can # proceed if not os.path.exists(full_path): logging.warning(" Model {} missing file {} = '{}'".format(model, description, full_path)) if download: logging.warning(" Cannot download file since it lives outside of the input_data_root '{}'".format(input_data_root)) no_files_missing = False else: logging.debug(" Found input file: '{}'".format(full_path)) else: # There are some special values of rel_path that # we need to ignore - some of the component models # set things like 'NULL' or 'same_as_TS' - # basically if rel_path does not contain '/' (a # directory tree) you can assume it's a special # value and ignore it (perhaps with a warning) if ("/" in rel_path and not os.path.exists(full_path)): logging.warning(" Model {} missing file {} = '{}'".format(model, description, full_path)) if (download): success = _download_if_in_repo(server, input_data_root, rel_path.strip('/')) if not success: no_files_missing = False else: logging.debug(" Already had input file: '{}'".format(full_path)) else: model = os.path.basename(data_list_file).split('.')[0] logging.warning("Model {} no file specified for {}".format(model, description)) return no_files_missing