def read_config(self, section, name, environment_variable=None, default=None): value = None if environment_variable is not None: value = os.getenv(environment_variable) if value is None: try: value = self.__config[section][name] except Exception as e: value = default if not section in self.__config: self.__config[section] = {} if self.__using_config_file: self.__config_bad = True self.__config[section][name] = default if (environment_variable is not None and value is not None and self.__export_env and not hasattr(self.__already_exported, environment_variable)): self.__export_file.write('export ' + environment_variable + '="' + value + '"\n') self.__already_exported[environment_variable] = True if os.getenv('AUGUR_DEBUG_LOG_ENV', '0') == '1': logger.debug('{}:{} = {}'.format(section, name, value)) return value
def publicwww(self): from augur.publicwww import PublicWWW if self.__publicwww is None: logger.debug('Initializing PublicWWW') self.__publicwww = PublicWWW(api_key=self.read_config( 'PublicWWW', 'apikey', 'AUGUR_PUBLIC_WWW_API_KEY', 'None')) return self.__publicwww
def git(self, update=False): from augur.git import Git storage = self.path_relative_to_config( self.read_config('Git', 'storage', 'AUGUR_GIT_STORAGE', 'runtime/git_repos/')) repolist = self.read_config('Git', 'repositories', None, []) if self.__git is None: logger.debug('Initializing Git') self.__git = Git(list_of_repositories=repolist, storage_folder=storage, csv=self.localcsv(), cache=self.cache) self.__updatable.append({ 'name': 'git', 'delay': int( self.read_config('Git', 'refresh', 'AUGUR_GIT_REFRESH', '3600')), 'update': self.__git.update }) if update: self.__git.update() return self.__git
def read_config(self, section, name, environment_variable=None, default=None): """ Read a variable in specified section of the config file, unless provided an environment variable :param section: location of given variable :param name: name of variable """ value = None if environment_variable is not None: value = os.getenv(environment_variable) if value is None: try: value = self.__config[section][name] except Exception as e: value = default if not section in self.__config: self.__config[section] = {} if self.__using_config_file: self.__config_bad = True self.__config[section][name] = default if (environment_variable is not None and value is not None and self.__export_env and not hasattr(self.__already_exported, environment_variable)): self.__export_file.write('export ' + environment_variable + '="' + str(value) + '"\n') self.__already_exported[environment_variable] = True if os.getenv('AUGUR_DEBUG_LOG_ENV', '0') == '1': logger.debug('{}:{} = {}'.format(section, name, value)) return value
def schedule_updates(self): """ Schedules updates """ # don't use this, logger.debug('Scheduling updates...') self.__updater()
def check_pgpass_credentials(config): pgpass_file_path = environ['HOME'] + '/.pgpass' if not path.isfile(pgpass_file_path): logger.debug("~/.pgpass does not exist, creating.") open(pgpass_file_path, 'w+') chmod(pgpass_file_path, stat.S_IWRITE | stat.S_IREAD) pgpass_file_mask = oct(os.stat(pgpass_file_path).st_mode & 0o777) if pgpass_file_mask != '0o600': logger.debug("Updating ~/.pgpass file permissions.") chmod(pgpass_file_path, stat.S_IWRITE | stat.S_IREAD) with open(pgpass_file_path, 'a+') as pgpass_file: end = pgpass_file.tell() pgpass_file.seek(0) credentials_string = str(config['Database']['host']) \ + ':' + str(config['Database']['port']) \ + ':' + str(config['Database']['name']) \ + ':' + str(config['Database']['user']) \ + ':' + str(config['Database']['password']) if credentials_string.lower() not in [ ''.join(line.split()).lower() for line in pgpass_file.readlines() ]: logger.info("Adding credentials to $HOME/.pgpass") pgpass_file.seek(end) pgpass_file.write(credentials_string + '\n') else: logger.info("Credentials found in $HOME/.pgpass")
def __init__(self, user, password, host, port, dbname, ghtorrent, buildMode="auto"): """ Connect to the database :param dbstr: The [database string](http://docs.sqlalchemy.org/en/latest/core/engines.html) to connect to the GHTorrent database """ self.DB_STR = 'mysql+pymysql://{}:{}@{}:{}/{}'.format( user, password, host, port, dbname) logger.debug('GHTorrentPlus: Connecting to {}:{}/{} as {}'.format( host, port, dbname, user)) self.db = s.create_engine(self.DB_STR, poolclass=s.pool.NullPool) self.ghtorrent = ghtorrent try: # Table creation if (buildMode == 'rebuild') or ((not self.db.dialect.has_table( self.db.connect(), 'issue_response_time')) and buildMode == "auto"): logger.info( "[GHTorrentPlus] Creating Issue Response Time table...") self.build_issue_response_time() except Exception as e: logger.error( "Could not connect to GHTorrentPlus database. Error: " + str(e))
def librariesio(self): from augur.librariesio import LibrariesIO if self.__librariesio is None: logger.debug('Initializing LibrariesIO') self.__librariesio = LibrariesIO(api_key=self.read_config( 'LibrariesIO', 'apikey', 'AUGUR_LIBRARIESIO_API_KEY', 'None'), githubapi=self.githubapi()) return self.__librariesio
def __call__(self): from .githubapi import GitHubAPI if self.__githubapi is None: logger.debug('Initializing GitHub API') api_key = self._augur.read_config('GitHub', 'apikey', 'AUGUR_GITHUB_API_KEY', 'None') self.__githubapi = GitHubAPI(api_key=api_key) return self.__githubapi
def __call__(self): from .metrics_status import MetricsStatus from augur.datasources.githubapi.githubapi import GitHubAPI if self.__metrics_status is None: logger.debug('Initializing MetricsStatus') api_key = self._augur.read_config('GitHub', 'apikey', 'AUGUR_GITHUB_API_KEY', 'None') self.__githubapi = GitHubAPI(api_key=api_key) self.__metrics_status = MetricsStatus(self.__githubapi) return self.__metrics_status
def __init__(self, user, password, host, port, dbname): """ Connect to LibrariesIO :param dbstr: The [database string](http://docs.sqlalchemy.org/en/latest/core/engines.html) to connect to the LibrariesIO database """ self.DB_STR = 'mysql+pymysql://{}:{}@{}:{}/{}'.format( user, password, host, port, dbname) logger.debug('LibrariesIO: Connecting to {}:{}/{} as {}'.format( host, port, dbname, user)) self.db = s.create_engine(self.DB_STR, poolclass=s.pool.NullPool)
def __getitem__(self, plugin_name): """ Returns plugin matching the name of the parameter 'plugin_name' :param plugin_name: name of specified plugin """ if plugin_name not in self._loaded_plugins: if plugin_name not in Application.plugins: raise ValueError('Plugin %s not found.' % plugin_name) self._loaded_plugins[plugin_name] = Application.plugins[plugin_name](self) logger.debug('{plugin_name} plugin loaded') return self._loaded_plugins[plugin_name]
def sbom_download(self, repo_group_id, repo_id=None): """REQUIRES SBOMS TO BE PRESENT IN THE DATABASE :param repo_id: The repository's repo_id, defaults to None :return: dosocs sbom """ dosocs_SQL = s.sql.text(""" select * from augur_data.repo_sbom_scans where repo_id = :repo_id; """) logger.debug(dosocs_SQL) params = {'repo_id': repo_id} return pd.read_sql(dosocs_SQL, self.database, params=params)
def __init__(self, user, password, host, port, dbname): """ Connect to GHTorrent :param dbstr: The [database string](http://docs.sqlalchemy.org/en/latest/core/engines.html) to connect to the GHTorrent database """ self.DB_STR = 'mysql+pymysql://{}:{}@{}:{}/{}'.format( user, password, host, port, dbname ) logger.debug('GHTorrent: Connecting to {}:{}/{} as {}'.format(host, port, dbname, user)) self.db = s.create_engine(self.DB_STR, poolclass=s.pool.NullPool) try: self.userid('howderek') except Exception as e: logger.error("Could not connect to GHTorrent database. Error: " + str(e))
def __call__(self): from .ghtorrent import GHTorrent if self.__ghtorrent is None: logger.debug('Initializing GHTorrent') self.__ghtorrent = GHTorrent( user=self._augur.read_config('GHTorrent', 'user', 'AUGUR_DB_USER', 'root'), password=self._augur.read_config('GHTorrent', 'pass', 'AUGUR_DB_PASS', 'password'), host=self._augur.read_config('GHTorrent', 'host', 'AUGUR_DB_HOST', '127.0.0.1'), port=self._augur.read_config('GHTorrent', 'port', 'AUGUR_DB_PORT', '3306'), dbname=self._augur.read_config('GHTorrent', 'name', 'AUGUR_DB_NAME', 'msr14')) return self.__ghtorrent
def __init__(self, user, password, host, port, dbname, schema): """ Connect to Augur :param dbstr: The [database string](http://docs.sqlalchemy.org/en/latest/core/engines.html) to connect to the Augur database """ self.DB_STR = 'postgresql://{}:{}@{}:{}/{}'.format( user, password, host, port, dbname) self.db = s.create_engine( self.DB_STR, poolclass=s.pool.NullPool, connect_args={'options': '-csearch_path={}'.format(schema)}) logger.debug( 'Augur DB: Connecting to {} schema of {}:{}/{} as {}'.format( schema, host, port, dbname, user)) self.metrics_status = []
def __init__(self, user, password, host, port, dbname, ghtorrent, buildMode="auto"): """ Connect to the database :param dbstr: The [database string](http://docs.sqlalchemy.org/en/latest/core/engines.html) to connect to the GHTorrent database """ self.DB_STR = 'mysql+pymysql://{}:{}@{}:{}/{}'.format( user, password, host, port, dbname) logger.debug('GHTorrentPlus: Connecting to {}:{}/{} as {}'.format( host, port, dbname, user)) self.db = s.create_engine(self.DB_STR, poolclass=s.pool.NullPool) self.ghtorrent = ghtorrent
def __call__(self): from .metrics_status import MetricsStatus if self.__metrics_status is None: logger.debug('Initializing MetricsStatus') self.__metrics_status = MetricsStatus( user=self._augur.read_config('Database', 'user', 'AUGUR_DB_USER', 'augur'), password=self._augur.read_config('Database', 'password', 'AUGUR_DB_PASS', 'password'), host=self._augur.read_config('Database', 'host', 'AUGUR_DB_HOST', '127.0.0.1'), port=self._augur.read_config('Database', 'port', 'AUGUR_DB_PORT', '5433'), dbname=self._augur.read_config('Database', 'name', 'AUGUR_DB_NAME', 'augur'), schema=self._augur.read_config('Database', 'schema', 'AUGUR_DB_SCHEMA', 'augur_data')) return self.__metrics_status
def ghtorrentplus(self): from augur.ghtorrentplus import GHTorrentPlus if self.__ghtorrentplus is None: logger.debug('Initializing GHTorrentPlus') self.__ghtorrentplus = GHTorrentPlus( user=self.read_config('GHTorrentPlus', 'user', 'AUGUR_GHTORRENT_PLUS_USER', 'root'), password=self.read_config('GHTorrentPlus', 'pass', 'AUGUR_GHTORRENT_PLUS_PASS', 'password'), host=self.read_config('GHTorrentPlus', 'host', 'AUGUR_GHTORRENT_PLUS_HOST', '127.0.0.1'), port=self.read_config('GHTorrentPlus', 'port', 'AUGUR_GHTORRENT_PLUS_PORT', '3306'), dbname=self.read_config('GHTorrentPlus', 'name', 'AUGUR_GHTORRENT_PLUS_NAME', 'ghtorrentplus'), ghtorrent=self.ghtorrent()) return self.__ghtorrentplus
def __call__(self): from .facade import Facade if self.__facade is None: logger.debug('Initializing Facade') self.__facade = Facade( user=self._augur.read_config('Facade', 'user', 'AUGUR_FACADE_DB_USER', 'root'), password=self._augur.read_config('Facade', 'pass', 'AUGUR_FACADE_DB_PASS', ''), host=self._augur.read_config('Facade', 'host', 'AUGUR_FACADE_DB_HOST', '127.0.0.1'), port=self._augur.read_config('Facade', 'port', 'AUGUR_FACADE_DB_PORT', '3306'), dbname=self._augur.read_config('Facade', 'name', 'AUGUR_FACADE_DB_NAME', 'facade'), projects=self._augur.read_config('Facade', 'projects', None, [])) return self.__facade
def read_config(self, section, name=None): """ Read a variable in specified section of the config file, unless provided an environment variable :param section: location of given variable :param name: name of variable """ if section is not None: value = self.config[section] if name is not None: try: value = self.config[section][name] except KeyError as e: pass else: value = None if os.getenv('AUGUR_DEBUG_LOG_ENV', '0') == '1': logger.debug('{}:{} = {}'.format(section, name, value)) return value
def __call__(self): from .librariesio import LibrariesIO if self.__librariesio is None: logger.debug('Initializing LibrariesIO') self.__librariesio = LibrariesIO( user=self._augur.read_config('Database', 'user', 'AUGUR_LIBRARIESIO_DB_USER', 'root'), password=self._augur.read_config('Database', 'pass', 'AUGUR_LIBRARIESIO_DB_PASS', 'password'), host=self._augur.read_config('Database', 'host', 'AUGUR_LIBRARIESIO_DB_HOST', '127.0.0.1'), port=self._augur.read_config('Database', 'port', 'AUGUR_LIBRARIESIO_DB_PORT', '3306'), dbname=self._augur.read_config('Database', 'name', 'AUGUR_LIBRARIESIO_DB_NAME', 'librariesio')) return self.__librariesio
def __call__(self): from .downloads import Downloads if self.__downloads is None: logger.debug('Initializing Downloads') self.__downloads = Downloads(self._augur['githubapi']()) return self.__downloads
def __call__(self): from .metrics_status import MetricsStatus if self.__metrics_status is None: logger.debug('Initializing MetricsStatus') self.__metrics_status = MetricsStatus(self._augur['githubapi']()) return self.__metrics_status
def batch(): """ Execute multiple requests, submitted as a batch. :statuscode 207: Multi status """ server.show_metadata = False if request.method == 'GET': """this will return sensible defaults in the future""" return server.app.make_response( '{"status": "501", "response": "Defaults for batch requests not implemented. Please POST a JSON array of requests to this endpoint for now."}' ) try: requests = json.loads(request.data.decode('utf-8')) except ValueError as e: request.abort(400) responses = [] for index, req in enumerate(requests): method = req['method'] path = req['path'] body = req.get('body', None) try: logger.debug('batch-internal-loop: %s %s' % (method, path)) with server.app.server.app.context(): with server.app.test_request_context(path, method=method, data=body): try: # Can modify flask.g here without affecting # flask.g of the root request for the batch # Pre process Request rv = server.app.preprocess_request() if rv is None: # Main Dispatch rv = server.app.dispatch_request() except Exception as e: rv = server.app.handle_user_exception(e) response = server.app.make_response(rv) # Post process Request response = server.app.process_response(response) # Response is a Flask response object. # _read_response(response) reads response.response # and returns a string. If your endpoints return JSON object, # this string would be the response as a JSON string. responses.append({ "path": path, "status": response.status_code, "response": str(response.get_data(), 'utf8'), }) except Exception as e: responses.append({ "path": path, "status": 500, "response": str(e) }) return Response(response=json.dumps(responses), status=207, mimetype="server.app.ication/json")
def downloads(self): from augur.downloads import Downloads if self.__downloads is None: logger.debug('Initializing Downloads') self.__downloads = Downloads(self.githubapi()) return self.__downloads
def localcsv(self): from augur.localcsv import LocalCSV if self.__localCSV is None: logger.debug('Initializing LocalCSV') self.__localCSV = LocalCSV() return self.__localCSV