def check_pgpass_credentials(config): pgpass_file_path = environ['HOME'] + '/.pgpass' if not path.isfile(pgpass_file_path): logger.debug("~/.pgpass does not exist, creating.") open(pgpass_file_path, 'w+') chmod(pgpass_file_path, stat.S_IWRITE | stat.S_IREAD) pgpass_file_mask = oct(os.stat(pgpass_file_path).st_mode & 0o777) if pgpass_file_mask != '0o600': logger.debug("Updating ~/.pgpass file permissions.") chmod(pgpass_file_path, stat.S_IWRITE | stat.S_IREAD) with open(pgpass_file_path, 'a+') as pgpass_file: end = pgpass_file.tell() pgpass_file.seek(0) credentials_string = str(config['Database']['host']) \ + ':' + str(config['Database']['port']) \ + ':' + str(config['Database']['name']) \ + ':' + str(config['Database']['user']) \ + ':' + str(config['Database']['password']) if credentials_string.lower() not in [ ''.join(line.split()).lower() for line in pgpass_file.readlines() ]: logger.info("Adding credentials to $HOME/.pgpass") pgpass_file.seek(end) pgpass_file.write(credentials_string + '\n') else: logger.info("Credentials found in $HOME/.pgpass")
def update_all(self): """ Updates all plugins """ for updatable in self.__updatable: logger.info('Updating {}...'.format(updatable['name'])) updatable['update']()
def list_processes(): """ Outputs the name and process ID (PID) of all currently running backend Augur processes, including any workers. Will only work in a virtual environment. """ processes = get_augur_processes() for process in processes: logger.info(f"Found process {process.pid}")
def add_repo_groups(ctx, filename): """ Create new repo groups in Augur's database """ app = ctx.obj df = pd.read_sql( s.sql.text("SELECT repo_group_id FROM augur_data.repo_groups"), app.database) repo_group_IDs = df['repo_group_id'].values.tolist() insert_repo_group_sql = s.sql.text(""" INSERT INTO "augur_data"."repo_groups"("repo_group_id", "rg_name", "rg_description", "rg_website", "rg_recache", "rg_last_modified", "rg_type", "tool_source", "tool_version", "data_source", "data_collection_date") VALUES (:repo_group_id, :repo_group_name, '', '', 0, CURRENT_TIMESTAMP, 'Unknown', 'Loaded by user', '1.0', 'Git', CURRENT_TIMESTAMP); """) with open(filename) as create_repo_groups_file: data = csv.reader(create_repo_groups_file, delimiter=',') for row in data: logger.info( f"Inserting repo group with name {row[1]} and ID {row[0]}...") if int(row[0]) not in repo_group_IDs: repo_group_IDs.append(int(row[0])) app.database.execute(insert_repo_group_sql, repo_group_id=int(row[0]), repo_group_name=row[1]) else: logger.info( f"Repo group with ID {row[1]} for repo group {row[1]} already exists, skipping..." )
def add_repos(ctx, filename): """ Add repositories to Augur's database """ app = ctx.obj df = app.database.execute( s.sql.text("SELECT repo_group_id FROM augur_data.repo_groups")) repo_group_IDs = [group[0] for group in df.fetchall()] insertSQL = s.sql.text(""" INSERT INTO augur_data.repo(repo_group_id, repo_git, repo_status, tool_source, tool_version, data_source, data_collection_date) VALUES (:repo_group_id, :repo_git, 'New', 'CLI', 1.0, 'Git', CURRENT_TIMESTAMP) """) with open(filename) as upload_repos_file: data = csv.reader(upload_repos_file, delimiter=',') for row in data: logger.info( f"Inserting repo with Git URL `{row[1]}` into repo group {row[0]}" ) if int(row[0]) in repo_group_IDs: result = app.database.execute(insertSQL, repo_group_id=int(row[0]), repo_git=row[1]) else: logger.warn( f"Invalid repo group id specified for {row[1]}, skipping.")
def check_for_upgrade(ctx): """ Upgrade the configured database to the latest version """ app = ctx.obj check_pgpass_credentials(app.config) current_db_version = get_db_version(app) update_scripts_filenames = [] for (_, _, filenames) in walk('schema/generate'): update_scripts_filenames.extend( [file for file in filenames if 'update' in file]) # files_temp.extend([file.split("-")[1][14:].split(".")[0] for file in filenames if 'update' in file]) break target_version_script_map = {} for script in update_scripts_filenames: upgrades_to = int(script.split("-")[1][14:].split(".")[0]) target_version_script_map[upgrades_to] = str(script) target_version_script_map = OrderedDict( sorted(target_version_script_map.items())) most_recent_version = list(target_version_script_map.keys())[-1] if current_db_version == most_recent_version: logger.info("Database is already up to date.") elif current_db_version < most_recent_version: logger.info( f"Current database version: v{current_db_version}\nPlease upgrade to the most recent version (v{most_recent_version}) with augur db upgrade-db-version." ) elif current_db_version > most_recent_version: logger.warn( f"Unrecognized version: {current_db_version}\nThe most recent version is {most_recent_version}. Please contact your system administrator to resolve this error." )
def __init__(self, user, password, host, port, dbname, ghtorrent, buildMode="auto"): """ Connect to the database :param dbstr: The [database string](http://docs.sqlalchemy.org/en/latest/core/engines.html) to connect to the GHTorrent database """ char = "charset=utf8" self.DB_STR = 'mysql+pymysql://{}:{}@{}:{}/{}?{}'.format( user, password, host, port, dbname, char) #print('GHTorrentPlus: Connecting to {}:{}/{}?{} as {}'.format(host, port, dbname, char,user)) self.db = s.create_engine(self.DB_STR, poolclass=s.pool.NullPool) self.ghtorrent = ghtorrent try: # Table creation if (buildMode == 'rebuild') or ((not self.db.dialect.has_table( self.db.connect(), 'issue_response_time')) and buildMode == "auto"): logger.info( "[GHTorrentPlus] Creating Issue Response Time table...") self.build_issue_response_time() except Exception as e: logger.error( "Could not connect to GHTorrentPlus database. Error: " + str(e))
def batch_metadata(): """ Returns endpoint metadata in batch format """ server.show_metadata = True if request.method == 'GET': """this will return sensible defaults in the future""" return server.app.make_response(json.dumps(metric_metadata)) try: requests = json.loads(request.data.decode('utf-8')) except ValueError as e: request.abort(400) responses = [] for index, req in enumerate(requests): method = req['method'] path = req['path'] body = req.get('body', None) try: logger.info('batch endpoint: ' + path) with server.app.server.app.context(): with server.app.test_request_context(path, method=method, data=body): try: rv = server.app.preprocess_request() if rv is None: rv = server.app.dispatch_request() except Exception as e: rv = server.app.handle_user_exception(e) response = server.app.make_response(rv) response = server.app.process_response(response) responses.append({ "path": path, "status": response.status_code, "response": str(response.get_data(), 'utf8'), }) except Exception as e: responses.append({ "path": path, "status": 500, "response": str(e) }) server.show_metadata = False return Response(response=json.dumps(responses), status=207, mimetype="server.app.ication/json")
def repo_reset(ctx): """ Refresh repo collection to force data collection """ app = ctx.obj app.database.execute( "UPDATE augur_data.repo SET repo_path = NULL, repo_name = NULL, repo_status = 'New'; TRUNCATE augur_data.commits CASCADE; " ) logger.info("Repos successfully reset")
def update_api_key(ctx, api_key): """ Update the API key in the database to the given key """ app = ctx.obj update_api_key_sql = s.sql.text(""" UPDATE augur_operations.augur_settings SET VALUE = :api_key WHERE setting='augur_api_key'; """) app.database.execute(update_api_key_sql, api_key=api_key) logger.info(f"Update Augur API key to: {api_key}")
def update_repo_directory(ctx, repo_directory): """ Update Facade worker repo cloning directory """ app = ctx.obj updateRepoDirectorySQL = s.sql.text(""" UPDATE augur_data.settings SET VALUE = :repo_directory WHERE setting='repo_directory'; """) app.database.execute(updateRepoDirectorySQL, repo_directory=repo_directory) logger.info(f"Updated Facade repo directory to: {repo_directory}")
def finalize_config(self): """ Parse args and generates a valid config if the given one is bad """ # Close files and save config if self.__config_bad: logger.info('Regenerating config with missing values...') self.__config_file.close() self.__config_file = open(self.__config_file_path, 'w') config_text = json.dumps(self.__config, sort_keys=True, indent=4) config_text = config_text.replace(self.__config_location, '$(AUGUR)') self.__config_file.write(config_text) self.__config_file.close()
def update(self): try: # Table creation if (buildMode == 'rebuild') or ((not self.db.dialect.has_table( self.db.connect(), 'issue_response_time')) and buildMode == "auto"): logger.info( "[GHTorrentPlus] Creating Issue Response Time table...") self.build_issue_response_time() except Exception as e: logger.error( "Could not connect to GHTorrentPlus database. Error: " + str(e))
def updater_process(name, delay): logger.info('Spawned {} updater process with PID {}'.format( name, os.getpid())) app = Application() datasource = getattr(app, name)() try: while True: logger.info('Updating {}...'.format(name)) datasource.update() time.sleep(delay) except KeyboardInterrupt: os._exit(0) except: raise
def kill_processes(ctx): """ Terminates all currently running backend Augur processes, including any workers. Will only work in a virtual environment. """ processes = get_augur_processes() if processes != []: for process in processes: if process.pid != os.getpid(): # logger.info(f"Killing {process.pid}: {' '.join(process.info['cmdline'][1:])}") logger.info(f"Killing process {process.pid}") try: process.send_signal(signal.SIGTERM) except psutil.NoSuchProcess as e: pass
def exit(): try: for pid in worker_pids: os.kill(pid, 9) except: logger.info("Worker process {} already killed".format(pid)) for process in worker_processes: logger.info("Shutting down worker process with pid: {} ...".format( process.pid)) process.terminate() if master is not None: master.halt() logger.info("Shutting down housekeeper updates...") if housekeeper is not None: housekeeper.shutdown_updates() # if hasattr(manager, "shutdown"): # wait for the spawner and the worker threads to go down # if manager is not None: manager.shutdown() # check if it is still alive and kill it if necessary # if manager._process.is_alive(): manager._process.terminate() # Prevent multiprocessing's atexit from conflicting with gunicorn logger.info("Killing main augur process with PID: {}".format( os.getpid())) os.kill(os.getpid(), 9) os._exit(0)
def finalize_config(self): # Parse args with help self.arg_parser.parse_known_args() # Close files and save config if self.__config_bad: logger.info('Regenerating config with missing values...') self.__config_file.close() self.__config_file = open(self.__config_file_path, 'w') config_text = json.dumps(self.__config, sort_keys=True, indent=4) config_text = config_text.replace(self.__config_location, '$(AUGUR)') self.__config_file.write(config_text) self.__config_file.close() if (self.__export_env): self.__export_file.close()
def worker_start(worker_name=None, instance_number=0, worker_port=None): time.sleep(120 * instance_number) destination = subprocess.DEVNULL try: destination = open( "workers/{}/worker_{}.log".format(worker_name, worker_port), "a+") except IOError as e: logger.error( "Error opening log file for auto-started worker {}: {}".format( worker_name, e)) process = subprocess.Popen("cd workers/{} && {}_start".format( worker_name, worker_name), shell=True, stdout=destination, stderr=subprocess.STDOUT) logger.info("{} booted.".format(worker_name))
def updater_process(name, delay, shared): """ Controls a given plugin's update process :param name: name of object to be updated :param delay: time needed to update :param shared: shared object that is to also be updated """ logger.info('Spawned {} updater process with PID {}'.format(name, os.getpid())) app = Application() datasource = getattr(app, name)() try: while True: logger.info('Updating {}...'.format(name)) datasource.update(shared) time.sleep(delay) except KeyboardInterrupt: os._exit(0) except: raise
def upgrade_db_version(ctx): """ Upgrade the configured database to the latest version """ app = ctx.obj check_pgpass_credentials(app.config) current_db_version = get_db_version(app) update_scripts_filenames = [] for (_, _, filenames) in walk('schema/generate'): update_scripts_filenames.extend( [file for file in filenames if 'update' in file]) # files_temp.extend([file.split("-")[1][14:].split(".")[0] for file in filenames if 'update' in file]) break target_version_script_map = {} for script in update_scripts_filenames: upgrades_to = int(script.split("-")[1][14:].split(".")[0]) target_version_script_map[upgrades_to] = str(script) target_version_script_map = OrderedDict( sorted(target_version_script_map.items())) most_recent_version = list(target_version_script_map.keys())[-1] if current_db_version == most_recent_version: logger.info("Your database is already up to date. ") elif current_db_version > most_recent_version: logger.info( f"Unrecognized version: {current_db_version}\nThe most recent version is {most_recent_version}. Please contact your system administrator to resolve this error." ) for target_version, script_location in target_version_script_map.items(): if target_version == current_db_version + 1: logger.info( f"Upgrading from {current_db_version} to {target_version}") run_psql_command_in_database(app, '-f', f"schema/generate/{script_location}") current_db_version += 1
def __init__(self, default_config_file_path='augur.config.json', no_config_file=0, config=None): """ Reads config, creates DB session, and initializes cache """ # Open the config file self.__config_file_name = 'augur.config.json' self.__already_exported = {} self.__default_config = { "Cache": { "config": { "cache.data_dir": "runtime/cache/", "cache.lock_dir": "runtime/cache/", "cache.type": "file" } }, "Database": { "connection_string": "sqlite:///:memory:", "database": "augur", "host": "localhost", "key": "key", "password": "******", "port": 5432, "schema": "augur_data", "user": "******" }, "Development": { "developer": "0", "interactive": "0" }, "Facade": { "check_updates": 1, "clone_repos": 1, "create_xlsx_summary_files": 1, "delete_marked_repos": 0, "fix_affiliations": 1, "force_analysis": 1, "force_invalidate_caches": 1, "force_updates": 1, "limited_run": 0, "multithreaded": 0, "nuke_stored_affiliations": 0, "pull_repos": 1, "rebuild_caches": 1, "run_analysis": 1 }, "Housekeeper": { "jobs": [] }, "Plugins": [], "Server": { "cache_expire": "3600", "host": "0.0.0.0", "port": "5000", "workers": "1" }, "Workers": {} } _root_augur_dir_path = os.path.dirname( os.path.dirname(os.path.realpath(__file__))) _possible_config_paths = [ self.__config_file_name, _root_augur_dir_path + "/" + self.__config_file_name, f"/opt/augur/{self.__config_file_name}" ] _config_file_path = default_config_file_path for location in _possible_config_paths: try: f = open(location, "r+") _config_file_path = os.path.abspath(location) f.close() logging.info("Using config file at " + os.path.abspath(location)) break except FileNotFoundError: pass self.__using_config_file = True self.__config_bad = False self.__config_file_path = os.path.abspath( os.getenv('AUGUR_CONFIG_FILE', _config_file_path)) self.__config_location = os.path.dirname(self.__config_file_path) self.__runtime_location = 'runtime/' self.__export_env = os.getenv('AUGUR_ENV_EXPORT', '0') == '1' self.__shell_config = None if os.getenv('AUGUR_ENV_ONLY', '0') != '1' and no_config_file == 0: try: self.__config_file = open(self.__config_file_path, 'r+') except: logger.info('Couldn\'t open {}, attempting to create.'.format( self.__config_file_name)) if not os.path.exists(self.__config_location): os.makedirs(self.__config_location) self.__config_file = open(self.__config_file_path, 'w+') self.__config_bad = True if self.__export_env: export_filename = os.getenv('AUGUR_ENV_EXPORT_FILE', 'augur.config.json.sh') self.__export_file = open(export_filename, 'w+') logger.info( 'Exporting {} to environment variable export statements in {}' .format(self.__config_file_name, export_filename)) self.__export_file.write('#!/bin/bash\n') # Load the config file try: config_text = self.__config_file.read() self.__config = json.loads(config_text) except json.decoder.JSONDecodeError as e: if not self.__config_bad: self.__using_config_file = False logger.error( '%s could not be parsed, using defaults. Fix that file, or delete it and run this again to regenerate it. Error: %s', self.__config_file_path, str(e)) self.__config = self.__default_config else: self.__using_config_file = False self.__config = self.__default_config if isinstance(config, dict): self.__config.update(config) # List of data sources that can do periodic updates self.__updatable = [] self.__processes = [] # Create cache cache_config = { 'cache.type': 'file', 'cache.data_dir': self.path('$(RUNTIME)/cache/'), 'cache.lock_dir': self.path('$(RUNTIME)/cache/') } cache_config.update( self.read_config('Cache', 'config', None, cache_config)) cache_config['cache.data_dir'] = self.path( cache_config['cache.data_dir']) cache_config['cache.lock_dir'] = self.path( cache_config['cache.lock_dir']) if not os.path.exists(cache_config['cache.data_dir']): os.makedirs(cache_config['cache.data_dir']) if not os.path.exists(cache_config['cache.lock_dir']): os.makedirs(cache_config['cache.lock_dir']) cache_parsed = parse_cache_config_options(cache_config) self.cache = CacheManager(**cache_parsed) self.metrics = MetricDefinitions(self) # # Initalize all objects to None self._loaded_plugins = {}
def __init__(self, config_file='augur.config.json', no_config_file=0, description='Augur application', db_str='sqlite:///:memory:'): """ Reads config, creates DB session, and initializes cache """ # Command line arguments # TODO: make this useful self.arg_parser = argparse.ArgumentParser(description=description) self.arg_parser.parse_known_args() # Open the config file self.__already_exported = {} self.__default_config = {'Plugins': []} self.__using_config_file = True self.__config_bad = False self.__config_file_path = os.path.abspath( os.getenv('AUGUR_CONFIG_FILE', config_file)) self.__config_location = os.path.dirname(self.__config_file_path) self.__runtime_location = 'runtime/' self.__export_env = os.getenv('AUGUR_ENV_EXPORT', '0') == '1' if os.getenv('AUGUR_ENV_ONLY', '0') != '1' and no_config_file == 0: try: self.__config_file = open(self.__config_file_path, 'r+') except: logger.info( 'Couldn\'t open {}, attempting to create. If you have a augur.cfg, you can convert it to a json file using "make to-json"' .format(config_file)) if not os.path.exists(self.__config_location): os.makedirs(self.__config_location) self.__config_file = open(self.__config_file_path, 'w+') self.__config_bad = True # Options to export the loaded configuration as environment variables for Docker if self.__export_env: export_filename = os.getenv('AUGUR_ENV_EXPORT_FILE', 'augur.cfg.sh') self.__export_file = open(export_filename, 'w+') logger.info( 'Exporting {} to environment variable export statements in {}' .format(config_file, export_filename)) self.__export_file.write('#!/bin/bash\n') # Load the config file try: config_text = self.__config_file.read() self.__config = json.loads(config_text) except json.decoder.JSONDecodeError as e: if not self.__config_bad: self.__using_config_file = False logger.error( '%s could not be parsed, using defaults. Fix that file, or delete it and run this again to regenerate it. Error: %s', self.__config_file_path, str(e)) self.__config = self.__default_config else: self.__using_config_file = False self.__config = self.__default_config # List of data sources that can do periodic updates self.__updatable = [] self.__processes = [] # Create cache cache_config = { 'cache.type': 'file', 'cache.data_dir': self.path('$(RUNTIME)/cache/'), 'cache.lock_dir': self.path('$(RUNTIME)/cache/') } cache_config.update( self.read_config('Cache', 'config', None, cache_config)) cache_config['cache.data_dir'] = self.path( cache_config['cache.data_dir']) cache_config['cache.lock_dir'] = self.path( cache_config['cache.lock_dir']) if not os.path.exists(cache_config['cache.data_dir']): os.makedirs(cache_config['cache.data_dir']) if not os.path.exists(cache_config['cache.lock_dir']): os.makedirs(cache_config['cache.lock_dir']) cache_parsed = parse_cache_config_options(cache_config) self.cache = CacheManager(**cache_parsed) # Create DB Session self.db = None self.session = None if db_str: self.db = create_engine(db_str) self.__Session = sessionmaker(bind=self.db) self.session = self.__Session() # Initalize all objects to None self.__metrics_status = None self._loaded_plugins = {} for plugin_name in Application.default_plugins: self[plugin_name]
def __init__(self): logger.info('example-plugin loaded') return
def cli(ctx, disable_housekeeper, skip_cleanup): """ Start Augur's backend server """ if not skip_cleanup: logger.info("Cleaning up old Augur processes. Just a moment please...") ctx.invoke(kill_processes) time.sleep(2) else: logger.info("Skipping cleanup processes.") def get_process_id(name): """Return process ids found by name or command """ child = subprocess.Popen(['pgrep', '-f', name], stdout=subprocess.PIPE, shell=False) response = child.communicate()[0] return [int(pid) for pid in response.split()] app = ctx.obj mp.set_start_method('forkserver', force=True) master = None manager = None broker = None housekeeper = None logger.info("Booting broker and its manager...") manager = mp.Manager() broker = manager.dict() controller = app.read_config('Workers') worker_pids = [] worker_processes = [] if not disable_housekeeper: if not controller: return for worker in controller.keys(): if not controller[worker]['switch']: continue logger.info( "Your config has the option set to automatically boot {} instances of the {}" .format(controller[worker]['workers'], worker)) pids = get_process_id( "/bin/sh -c cd workers/{} && {}_start".format(worker, worker)) worker_pids += pids if len(pids) > 0: worker_pids.append(pids[0] + 1) pids.append(pids[0] + 1) logger.info( "Found and preparing to kill previous {} worker pids: {}". format(worker, pids)) for pid in pids: try: os.kill(pid, 9) except: logger.info( "Worker process {} already killed".format(pid)) @atexit.register def exit(): try: for pid in worker_pids: os.kill(pid, 9) except: logger.info("Worker process {} already killed".format(pid)) for process in worker_processes: logger.info("Shutting down worker process with pid: {} ...".format( process.pid)) process.terminate() if master is not None: master.halt() logger.info("Shutting down housekeeper updates...") if housekeeper is not None: housekeeper.shutdown_updates() # if hasattr(manager, "shutdown"): # wait for the spawner and the worker threads to go down # if manager is not None: manager.shutdown() # check if it is still alive and kill it if necessary # if manager._process.is_alive(): manager._process.terminate() # Prevent multiprocessing's atexit from conflicting with gunicorn logger.info("Killing main augur process with PID: {}".format( os.getpid())) os.kill(os.getpid(), 9) os._exit(0) if not disable_housekeeper: logger.info("Booting housekeeper...") jobs = deepcopy(app.read_config('Housekeeper', 'jobs')) try: housekeeper = Housekeeper( jobs, broker, broker_host=app.read_config('Server', 'host'), broker_port=app.read_config('Server', 'port'), user=app.read_config('Database', 'user'), password=app.read_config('Database', 'password'), host=app.read_config('Database', 'host'), port=app.read_config('Database', 'port'), dbname=app.read_config('Database', 'name')) except KeyboardInterrupt as e: exit() logger.info("Housekeeper has finished booting.") if controller: for worker in controller.keys(): if controller[worker]['switch']: for i in range(controller[worker]['workers']): logger.info("Booting {} #{}".format(worker, i + 1)) worker_process = mp.Process( target=worker_start, kwargs={ 'worker_name': worker, 'instance_number': i, 'worker_port': controller[worker]['port'] }, daemon=True) worker_process.start() worker_processes.append(worker_process) host = app.read_config('Server', 'host') port = app.read_config('Server', 'port') workers = int(app.read_config('Server', 'workers')) timeout = int(app.read_config('Server', 'timeout')) options = { 'bind': '%s:%s' % (host, port), 'workers': workers, 'accesslog': '-', 'access_log_format': '%(h)s - %(t)s - %(r)s', 'timeout': timeout } logger.info('Starting server...') master = Arbiter( AugurGunicornApp(options, manager=manager, broker=broker, housekeeper=housekeeper)).run()
def update_all(self): print(self.__updatable) for updatable in self.__updatable: logger.info('Updating {}...'.format(updatable['name'])) updatable['update']()
def __init__(self, config_file='augur.config.json', no_config_file=0, description='Augur application'): # Command line arguments # TODO: make this useful self.arg_parser = argparse.ArgumentParser(description=description) self.arg_parser.parse_known_args() # Open the config file self.__already_exported = {} self.__default_config = {'Plugins': []} self.__using_config_file = True self.__config_bad = False self.__config_file_path = os.path.abspath( os.getenv('AUGUR_CONFIG_FILE', config_file)) self.__config_location = os.path.dirname(self.__config_file_path) self.__export_env = os.getenv('AUGUR_ENV_EXPORT', '0') == '1' if os.getenv('AUGUR_ENV_ONLY', '0') != '1' and no_config_file == 0: try: self.__config_file = open(self.__config_file_path, 'r+') except: logger.info( 'Couldn\'t open {}, attempting to create. If you have a augur.cfg, you can convert it to a json file using "make to-json"' .format(config_file)) if not os.path.exists(self.__config_location): os.makedirs(self.__config_location) self.__config_file = open(self.__config_file_path, 'w+') self.__config_bad = True # Options to export the loaded configuration as environment variables for Docker if self.__export_env: export_filename = os.getenv('AUGUR_ENV_EXPORT_FILE', 'augur.cfg.sh') self.__export_file = open(export_filename, 'w+') logger.info( 'Exporting {} to environment variable export statements in {}' .format(config_file, export_filename)) self.__export_file.write('#!/bin/bash\n') # Load the config file try: config_text = self.__config_file.read() config_text = config_text.replace('$(AUGUR)', self.__config_location) self.__config = json.loads(config_text) except json.decoder.JSONDecodeError as e: if not self.__config_bad: self.__using_config_file = False logger.error( '%s could not be parsed, using defaults. Fix that file, or delete it and run this again to regenerate it. Error: %s', self.__config_file_path, str(e)) self.__config = self.__default_config else: self.__using_config_file = False self.__config = self.__default_config # List of data sources that can do periodic updates self.__updatable = [] self.__processes = [] # Create cache cache_config = self.read_config( 'Cache', 'config', None, { 'cache.type': 'file', 'cache.data_dir': 'runtime/cache/', 'cache.lock_dir': 'runtime/cache/' }) if not os.path.exists(cache_config['cache.data_dir']): os.makedirs(cache_config['cache.data_dir']) if not os.path.exists(cache_config['cache.lock_dir']): os.makedirs(cache_config['cache.lock_dir']) cache_parsed = parse_cache_config_options(cache_config) self.cache = CacheManager(**cache_parsed) # Initalize all objects to None self.__ghtorrent = None self.__ghtorrentplus = None self.__githubapi = None self.__git = None self.__librariesio = None self.__downloads = None self.__publicwww = None self.__localCSV = None