def get_fingerprinters(tag): """Return a sequence of fingerprint functors for a specific tag. """ if not len(_tag2fx): # charge from testkraut import cfg tags = set(cfg.options('system fingerprints')).union(cfg.options('fingerprints')) for tag_ in tags: fp_tag = set() for fps_str in cfg.get('system fingerprints', tag_, default="").split() \ + cfg.get('fingerprints', tag_, default="").split(): fps_comp = fps_str.split('.') try: mod = __import__('.'.join(fps_comp[:-1]), globals(), locals(), fps_comp[-1:], -1) fps = getattr(mod, fps_comp[-1]) except: lgr.warning( "ignoring invalid fingerprinting function '%s' for tag '%s'" % (fps_str, tag_)) fp_tag.add(fps) _tag2fx[tag_] = fp_tag fprinters = _tag2fx.get(tag, set()) fprinters.add(_fp_file) return fprinters
def get_fingerprinters(tag): """Return a sequence of fingerprint functors for a specific tag. """ if not len(_tag2fx): # charge from testkraut import cfg tags = set(cfg.options('system fingerprints')).union( cfg.options('fingerprints')) for tag_ in tags: fp_tag = set() for fps_str in cfg.get('system fingerprints', tag_, default="").split() \ + cfg.get('fingerprints', tag_, default="").split(): fps_comp = fps_str.split('.') try: mod = __import__('.'.join(fps_comp[:-1]), globals(), locals(), fps_comp[-1:], -1) fps = getattr(mod, fps_comp[-1]) except: lgr.warning( "ignoring invalid fingerprinting function '%s' for tag '%s'" % (fps_str, tag_)) fp_tag.add(fps) _tag2fx[tag_] = fp_tag fprinters = _tag2fx.get(tag, set()) fprinters.add(_fp_file) return fprinters
def place_file_into_dir(filespec, dest_dir, search_dirs=None, cache=None, force_overwrite=True, symlink_to_cache=True): """Search for a file given a SPEC and place it into a destination directory Parameters ---------- filespec : SPEC dict Dictionary with information on the file, keys could be, e.g., 'value', 'type', or 'sha1sum'. dest_dir : path Path of the destination/target directory search_dirs : None or sequence If not None, a sequence of additional local directories to be searched for the desired file (tetskraut configuration might provide more locations that will also be searched afterwards) cache : None or path If not None, a path to a file cache directory where the desired file is searched by its sha1sum (if present in the SPEC) """ # TODO refactor from simple cachedir to cache object that could also obtain # a file from remote locations # sanity if not 'type' in filespec or filespec['type'] != 'file': raise ValueError("expected SPEC is not a file SPEC, got : '%s'" % filespec) # have a default cache if cache is None: cache = get_filecache_dir() if not os.path.exists(cache): os.makedirs(cache) # search path if search_dirs is None: search_dirs = [] search_dirs += cfg.get('data sources', 'local dirs', default='').split() fname = filespec['value'] # where the file needs to end up in the testbed dest_fname = opj(dest_dir, fname) # this will be the discovered file path fpath = None # first try the cache fpath = locate_file_in_cache(filespec, cache) # do a local search if fpath is None and len(search_dirs): lgr.debug("cache lookup for '%s' unsuccessful, trying local search" % fname) # do a two-pass scan: first try locating the file by name to avoid # sha1-summing all files for search_dir in search_dirs: for root, dirnames, filenames in os.walk(search_dir): cand_path = opj(root, fname) if not os.path.isfile(cand_path): lgr.debug("could not find file '%s' at '%s'" % (fname, cand_path)) continue hashmatch = check_file_hash(filespec, cand_path) if hashmatch in (True, None): lgr.debug("found matching file '%s' at '%s'" % (fname, cand_path)) # run with the file if there is no hash or it matches fpath = cand_path break if not fpath is None: break if fpath is None and ('sha1sum' in filespec or 'md5sum' in filespec): lgr.debug( "could not find file '%s' by its name, doing hash lookup" % fname) # 2nd pass if we have a hash try locating by hash for search_dir in search_dirs: for root, dirnames, filenames in os.walk(search_dir): for cand_name in filenames: cand_path = opj(root, cand_name) if check_file_hash(filespec, cand_path) is True: lgr.debug("found matching file '%s' at '%s'" % (fname, cand_path)) fpath = cand_path break if not fpath is None: break if not fpath is None: break if not fpath is None and ('md5sum' in filespec or 'sha1sum' in filespec): # place in cache -- but only if any hash is given in the file spec # if no hash is given, this file is volatile and it makes no sense # to cache it if not 'sha1sum' in filespec: sha1 = sha1sum(fpath) else: sha1 = filespec['sha1sum'] dst_path = opj(cache, sha1) if os.path.exists(dst_path) or os.path.lexists(dst_path): os.remove(dst_path) lgr.debug("removing existing cache entry '%s'" % dst_path) if symlink_to_cache: os.symlink(fpath, dst_path) lgr.debug("symlink to cache '%s'->'%s'" % (fpath, dst_path)) elif hasattr(os, 'link'): # be nice and try hard-linking try: os.link(fpath, dst_path) lgr.debug("hardlink to cache '%s'->'%s'" % (fpath, dst_path)) except OSError: # silently fail if linking doesn't work (e.g. # cross-device link ... will recover later shutil.copy(fpath, dst_path) lgr.debug("copy to cache '%s'->'%s'" % (fpath, dst_path)) else: shutil.copy(fpath, dst_path) lgr.debug("copy to cache '%s'->'%s'" % (fpath, dst_path)) # trying external data sources if fpath is None and 'url' in filespec: # url is given fpath = download_file(filespec['url'], dest_fname) if fpath is None and 'sha1sum' in filespec: # lookup in any configured hash store hashpots = cfg.get('data sources', 'hash stores').split() sha1 = filespec['sha1sum'] lgr.debug("local search '%s' unsuccessful, trying hash stores" % fname) dst_path = opj(cache, sha1) for hp in hashpots: fpath = download_file('%s%s' % (hp, sha1), dst_path) if not fpath is None: break if fpath is None: # out of ideas raise LookupError("cannot find file matching spec %s" % filespec) # get the file into the dest_dir if not fpath == dest_fname \ and (force_overwrite or not os.path.isfile(dest_fname)): if not os.path.exists(dest_dir): os.makedirs(dest_dir) shutil.copy(fpath, dest_fname) else: lgr.debug("skip copying already present file '%s'" % fname)
def place_file_into_dir(filespec, dest_dir, search_dirs=None, cache=None, force_overwrite=True, symlink_to_cache=True): """Search for a file given a SPEC and place it into a destination directory Parameters ---------- filespec : SPEC dict Dictionary with information on the file, keys could be, e.g., 'value', 'type', or 'sha1sum'. dest_dir : path Path of the destination/target directory search_dirs : None or sequence If not None, a sequence of additional local directories to be searched for the desired file (tetskraut configuration might provide more locations that will also be searched afterwards) cache : None or path If not None, a path to a file cache directory where the desired file is searched by its sha1sum (if present in the SPEC) """ # TODO refactor from simple cachedir to cache object that could also obtain # a file from remote locations # sanity if not 'type' in filespec or filespec['type'] != 'file': raise ValueError("expected SPEC is not a file SPEC, got : '%s'" % filespec) # have a default cache if cache is None: cache = get_filecache_dir() if not os.path.exists(cache): os.makedirs(cache) # search path if search_dirs is None: search_dirs = [] search_dirs += cfg.get('data sources', 'local dirs', default='').split() fname = filespec['value'] # where the file needs to end up in the testbed dest_fname = opj(dest_dir, fname) # this will be the discovered file path fpath = None # first try the cache fpath = locate_file_in_cache(filespec, cache) # do a local search if fpath is None and len(search_dirs): lgr.debug("cache lookup for '%s' unsuccessful, trying local search" % fname) # do a two-pass scan: first try locating the file by name to avoid # sha1-summing all files for search_dir in search_dirs: for root, dirnames, filenames in os.walk(search_dir): cand_path = opj(root, fname) if not os.path.isfile(cand_path): lgr.debug("could not find file '%s' at '%s'" % (fname, cand_path)) continue hashmatch = check_file_hash(filespec, cand_path) if hashmatch in (True, None): lgr.debug("found matching file '%s' at '%s'" % (fname, cand_path)) # run with the file if there is no hash or it matches fpath = cand_path break if not fpath is None: break if fpath is None and ('sha1sum' in filespec or 'md5sum' in filespec): lgr.debug("could not find file '%s' by its name, doing hash lookup" % fname) # 2nd pass if we have a hash try locating by hash for search_dir in search_dirs: for root, dirnames, filenames in os.walk(search_dir): for cand_name in filenames: cand_path = opj(root, cand_name) if check_file_hash(filespec, cand_path) is True: lgr.debug("found matching file '%s' at '%s'" % (fname, cand_path)) fpath = cand_path break if not fpath is None: break if not fpath is None: break if not fpath is None and ('md5sum' in filespec or 'sha1sum' in filespec): # place in cache -- but only if any hash is given in the file spec # if no hash is given, this file is volatile and it makes no sense # to cache it if not 'sha1sum' in filespec: sha1 = sha1sum(fpath) else: sha1 = filespec['sha1sum'] dst_path = opj(cache, sha1) if os.path.exists(dst_path) or os.path.lexists(dst_path): os.remove(dst_path) lgr.debug("removing existing cache entry '%s'" % dst_path) if symlink_to_cache: os.symlink(fpath, dst_path) lgr.debug("symlink to cache '%s'->'%s'" % (fpath, dst_path)) elif hasattr(os, 'link'): # be nice and try hard-linking try: os.link(fpath, dst_path) lgr.debug("hardlink to cache '%s'->'%s'" % (fpath, dst_path)) except OSError: # silently fail if linking doesn't work (e.g. # cross-device link ... will recover later shutil.copy(fpath, dst_path) lgr.debug("copy to cache '%s'->'%s'" % (fpath, dst_path)) else: shutil.copy(fpath, dst_path) lgr.debug("copy to cache '%s'->'%s'" % (fpath, dst_path)) # trying external data sources if fpath is None and 'url' in filespec: # url is given fpath = download_file(filespec['url'], dest_fname) if fpath is None and 'sha1sum' in filespec: # lookup in any configured hash store hashpots = cfg.get('data sources', 'hash stores').split() sha1 = filespec['sha1sum'] lgr.debug("local search '%s' unsuccessful, trying hash stores" % fname) dst_path = opj(cache, sha1) for hp in hashpots: fpath = download_file('%s%s' % (hp, sha1), dst_path) if not fpath is None: break if fpath is None: # out of ideas raise LookupError("cannot find file matching spec %s" % filespec) # get the file into the dest_dir if not fpath == dest_fname \ and (force_overwrite or not os.path.isfile(dest_fname)): if not os.path.exists(dest_dir): os.makedirs(dest_dir) shutil.copy(fpath, dest_fname) else: lgr.debug("skip copying already present file '%s'" % fname)
def run(args): lgr = args.logger lgr.debug("using file cache at '%s'" % args.filecache) if not len(args.ids): # if none specified go through all the SPECs in the lib args.ids = [] for tld in get_test_library_paths(args.library): args.ids.extend([os.path.basename(d) for d in glob(opj(tld, '*')) if os.path.isdir(d)]) wanted_files = set() hash_lookup = {} # scan the SPECs of all tests for needed files and their sha1sums for test_id in args.ids: lgr.debug("scan required files for test '%s'" % test_id) spec = get_spec(test_id, args.library) for _, input in spec.get_inputs('file').iteritems(): if 'sha1sum' in input: wanted_files.add(input['sha1sum']) hash_lookup[input['sha1sum']] = (test_id, input.get('value', '')) lgr.debug("add '%s' (%s) to the list of files to look for" % (input.get('value', ''), input['sha1sum'])) # what do we have in the cache? have_files = [os.path.basename(f) for f in glob(opj(args.filecache, '*')) if os.path.isfile(f)] # what is missing missing_files = wanted_files.difference(have_files) search_cache = {} # search in all locale dirs for search_dir in (args.search + args.library): for root, dirnames, filenames in os.walk(search_dir): for fname in filenames: fpath = opj(root, fname) sha1 = sha1sum(fpath) if sha1 in missing_files: # make path relative to cache dir search_cache[sha1] = os.path.relpath(fpath, args.filecache) lgr.debug("found missing '%s' at '%s'" % (sha1, fpath)) missing_files.remove(sha1) # ensure the cache is there if not os.path.exists(args.filecache): os.makedirs(args.filecache) # try downloading missing files from the web hashpots = cfg.get('hash stores', 'http').split() for sha1 in missing_files.copy(): for hp in hashpots: try: urip = urllib2.urlopen('%s%s' % (hp, sha1)) dst_path = opj(args.filecache, sha1) fp = open(dst_path, 'wb') lgr.debug("download '%s%s'->'%s'" % (hp, sha1, dst_path)) fp.write(urip.read()) fp.close() missing_files.remove(sha1) break except urllib2.HTTPError: lgr.debug("cannot find '%s' at '%s'" % (sha1, hp)) except urllib2.URLError: lgr.debug("cannot connect to at '%s'" % hp) # copy/link them into the cache for sha1, fpath in search_cache.iteritems(): dst_path = opj(args.filecache, sha1) if os.path.lexists(dst_path): if os.path.islink(dst_path): # remove existing symlink os.remove(dst_path) lgr.debug("removing existing symlink '%s' in filecache" % dst_path) else: lgr.warning( "Will not replace existing non-symlink cache content: [%s: %s (%s)]" % (hash_lookup[sha1] + (sha1,))) if not args.copy: os.symlink(fpath, dst_path) lgr.debug("symlink '%s'->'%s'" % (fpath, dst_path)) elif hasattr(os, 'link'): # be nice and try hard-linking try: os.link(fpath, dst_path) lgr.debug("hardlink '%s'->'%s'" % (fpath, dst_path)) except OSError: # silently fail if linking doesn't work (e.g. # cross-device link ... will recover later shutil.copy(fpath, dst_path) lgr.debug("copylink '%s'->'%s'" % (fpath, dst_path)) else: shutil.copy(fpath, dst_path) lgr.debug("copylink '%s'->'%s'" % (fpath, dst_path)) if len(missing_files): lgr.warning('cannot find needed file(s):') for mf in missing_files: lgr.warning(' %s: %s (%s)' % (hash_lookup[mf] + (mf,)))
def run(args): lgr = args.logger lgr.debug("using file cache at '%s'" % args.filecache) if not len(args.ids): # if none specified go through all the SPECs in the lib args.ids = [] for tld in get_test_library_paths(args.library): args.ids.extend([ os.path.basename(d) for d in glob(opj(tld, '*')) if os.path.isdir(d) ]) wanted_files = set() hash_lookup = {} # scan the SPECs of all tests for needed files and their sha1sums for test_id in args.ids: lgr.debug("scan required files for test '%s'" % test_id) spec = get_spec(test_id, args.library) for _, input in spec.get_inputs('file').iteritems(): if 'sha1sum' in input: wanted_files.add(input['sha1sum']) hash_lookup[input['sha1sum']] = (test_id, input.get('value', '')) lgr.debug("add '%s' (%s) to the list of files to look for" % (input.get('value', ''), input['sha1sum'])) # what do we have in the cache? have_files = [ os.path.basename(f) for f in glob(opj(args.filecache, '*')) if os.path.isfile(f) ] # what is missing missing_files = wanted_files.difference(have_files) search_cache = {} # search in all locale dirs for search_dir in (args.search + args.library): for root, dirnames, filenames in os.walk(search_dir): for fname in filenames: fpath = opj(root, fname) sha1 = sha1sum(fpath) if sha1 in missing_files: # make path relative to cache dir search_cache[sha1] = os.path.relpath(fpath, args.filecache) lgr.debug("found missing '%s' at '%s'" % (sha1, fpath)) missing_files.remove(sha1) # ensure the cache is there if not os.path.exists(args.filecache): os.makedirs(args.filecache) # try downloading missing files from the web hashpots = cfg.get('hash stores', 'http').split() for sha1 in missing_files.copy(): for hp in hashpots: try: urip = urllib2.urlopen('%s%s' % (hp, sha1)) dst_path = opj(args.filecache, sha1) fp = open(dst_path, 'wb') lgr.debug("download '%s%s'->'%s'" % (hp, sha1, dst_path)) fp.write(urip.read()) fp.close() missing_files.remove(sha1) break except urllib2.HTTPError: lgr.debug("cannot find '%s' at '%s'" % (sha1, hp)) except urllib2.URLError: lgr.debug("cannot connect to at '%s'" % hp) # copy/link them into the cache for sha1, fpath in search_cache.iteritems(): dst_path = opj(args.filecache, sha1) if os.path.lexists(dst_path): if os.path.islink(dst_path): # remove existing symlink os.remove(dst_path) lgr.debug("removing existing symlink '%s' in filecache" % dst_path) else: lgr.warning( "Will not replace existing non-symlink cache content: [%s: %s (%s)]" % (hash_lookup[sha1] + (sha1, ))) if not args.copy: os.symlink(fpath, dst_path) lgr.debug("symlink '%s'->'%s'" % (fpath, dst_path)) elif hasattr(os, 'link'): # be nice and try hard-linking try: os.link(fpath, dst_path) lgr.debug("hardlink '%s'->'%s'" % (fpath, dst_path)) except OSError: # silently fail if linking doesn't work (e.g. # cross-device link ... will recover later shutil.copy(fpath, dst_path) lgr.debug("copylink '%s'->'%s'" % (fpath, dst_path)) else: shutil.copy(fpath, dst_path) lgr.debug("copylink '%s'->'%s'" % (fpath, dst_path)) if len(missing_files): lgr.warning('cannot find needed file(s):') for mf in missing_files: lgr.warning(' %s: %s (%s)' % (hash_lookup[mf] + (mf, )))