示例#1
0
def setup_module():
    """Put raw data and scripts in appropriate .retriever directories"""
    for test in tests:
        if not os.path.exists(os.path.join(HOME_DIR, "raw_data", test['name'])):
            os.makedirs(os.path.join(HOME_DIR, "raw_data", test['name']))
        create_file(test['raw_data'], os.path.join(HOME_DIR, "raw_data", test['name'], test['name'] + '.txt'))
        create_file(test['script'], os.path.join(HOME_DIR, "scripts", test['name'] + '.json'))
        compile_json(os.path.join(HOME_DIR, "scripts", test['name']))
示例#2
0
def setup_module():
    """Put raw data and scripts in appropriate .retriever directories"""
    for test in tests:
        if not os.path.exists(os.path.join(HOME_DIR, "raw_data", test['name'])):
            os.makedirs(os.path.join(HOME_DIR, "raw_data", test['name']))
        create_file(test['raw_data'], os.path.join(HOME_DIR, "raw_data", test['name'], test['name'] + '.txt'))
        create_file(test['script'], os.path.join(HOME_DIR, "scripts", test['name'] + '.json'))
        compile_json(os.path.join(HOME_DIR, "scripts", test['name']))
示例#3
0
def MODULE_LIST(force_compile=False):
    """Load scripts from scripts directory and return list of modules."""
    modules = []
    loaded_scripts = []

    for search_path in [
            search_path for search_path in SCRIPT_SEARCH_PATHS
            if exists(search_path)
    ]:
        to_compile = [
            file for file in os.listdir(search_path)
            if file[-5:] == ".json" and file[0] != "_" and (
                (not isfile(join(search_path, file[:-5] + '.py'))) or
                (isfile(join(search_path, file[:-5] + '.py')) and
                 (getmtime(join(search_path, file[:-5] + '.py')) < getmtime(
                     join(search_path, file)))) or force_compile)
        ]
        for script in to_compile:
            script_name = '.'.join(script.split('.')[:-1])
            compile_json(join(search_path, script_name))

        files = [
            file for file in os.listdir(search_path)
            if file[-3:] == ".py" and file[0] != "_" and '#retriever' in ' '.
            join(open(join(search_path, file), 'r').readlines()[:2]).lower()
        ]

        for script in files:
            script_name = '.'.join(script.split('.')[:-1])
            if script_name not in loaded_scripts:
                loaded_scripts.append(script_name)
                file, pathname, desc = imp.find_module(script_name,
                                                       [search_path])
                try:
                    new_module = imp.load_module(script_name, file, pathname,
                                                 desc)
                    if hasattr(new_module.SCRIPT, "retriever_minimum_version"):
                        # a script with retriever_minimum_version should be loaded
                        # only if its compliant with the version of the retriever
                        if not parse_version(VERSION) >= parse_version(
                                "{}".format(new_module.SCRIPT.
                                            retriever_minimum_version)):
                            print("{} is supported by Retriever version {}".
                                  format(
                                      script_name, new_module.SCRIPT.
                                      retriever_minimum_version))
                            print("Current version is {}".format(VERSION))
                            continue
                    # if the script wasn't found in an early search path
                    # make sure it works and then add it
                    new_module.SCRIPT.download
                    modules.append(new_module)
                except Exception as e:
                    sys.stderr.write(
                        "Failed to load script: %s (%s)\nException: %s \n" %
                        (script_name, search_path, str(e)))
    return modules
示例#4
0
def setup_module():
    """Put raw data and scripts in appropriate .retriever directories."""
    for test in tests:
        if not os.path.exists(os.path.join(HOME_DIR, "raw_data", test['name'])):
            os.makedirs(os.path.join(HOME_DIR, "raw_data", test['name']))
        rd_path = os.path.join(HOME_DIR,
                               "raw_data", test['name'], test['name'] + '.txt')
        create_file(test['raw_data'], rd_path)

        path_js = os.path.join(HOME_DIR, "scripts", test['name'] + '.json')
        with open(path_js, 'w') as js:
            json.dump(test['script'], js, indent=2)
        compile_json(os.path.join(HOME_DIR, "scripts", test['name']))
示例#5
0
def setup_module():
    """Put raw data and scripts in appropriate .retriever directories."""
    for test in tests:
        if not os.path.exists(os.path.join(HOME_DIR, "raw_data",
                                           test['name'])):
            os.makedirs(os.path.join(HOME_DIR, "raw_data", test['name']))
        rd_path = os.path.join(HOME_DIR, "raw_data", test['name'],
                               test['name'] + '.txt')
        create_file(test['raw_data'], rd_path)

        path_js = os.path.join(HOME_DIR, "scripts", test['name'] + '.json')
        with open(path_js, 'w') as js:
            json.dump(test['script'], js, indent=2)
        compile_json(os.path.join(HOME_DIR, "scripts", test['name']))
示例#6
0
def MODULE_LIST(force_compile=False):
    """Load scripts from scripts directory and return list of modules."""
    modules = []
    loaded_scripts = []

    for search_path in [search_path for search_path in SCRIPT_SEARCH_PATHS if exists(search_path)]:
        to_compile = [
            file for file in os.listdir(search_path) if file[-5:] == ".json" and
            file[0] != "_" and (
                (not isfile(join(search_path, file[:-5] + '.py'))) or (
                    isfile(join(search_path, file[:-5] + '.py')) and (
                        getmtime(join(search_path, file[:-5] + '.py')) < getmtime(
                            join(search_path, file)))) or force_compile)]
        for script in to_compile:
            script_name = '.'.join(script.split('.')[:-1])
            compile_json(join(search_path, script_name))

        files = [file for file in os.listdir(search_path)
                 if file[-3:] == ".py" and file[0] != "_" and
                 '#retriever' in ' '.join(open(join(search_path, file), 'r').readlines()[:2]).lower()]

        for script in files:
            script_name = '.'.join(script.split('.')[:-1])
            if script_name not in loaded_scripts:
                loaded_scripts.append(script_name)
                file, pathname, desc = imp.find_module(script_name, [search_path])
                try:
                    new_module = imp.load_module(script_name, file, pathname, desc)
                    if hasattr(new_module.SCRIPT, "retriever_minimum_version"):
                        # a script with retriever_minimum_version should be loaded
                        # only if its compliant with the version of the retriever
                        if not parse_version(VERSION) >=  parse_version("{}".format(
                                new_module.SCRIPT.retriever_minimum_version)):
                            print("{} is supported by Retriever version {}".format(script_name,
                                                                                   new_module.SCRIPT.retriever_minimum_version))
                            print("Current version is {}".format(VERSION))
                            continue
                    # if the script wasn't found in an early search path
                    # make sure it works and then add it
                    new_module.SCRIPT.download
                    modules.append(new_module)
                except Exception as e:
                    sys.stderr.write("Failed to load script: %s (%s)\nException: %s \n" % (
                        script_name, search_path, str(e)))
    return modules