def read_config(self,
section,
name,
environment_variable=None,
default=None):
value = None
if environment_variable is not None:
value = os.getenv(environment_variable)
if value is None:
try:
value = self.__config[section][name]
except Exception as e:
value = default
if not section in self.__config:
self.__config[section] = {}
if self.__using_config_file:
self.__config_bad = True
self.__config[section][name] = default
if (environment_variable is not None and value is not None
and self.__export_env and
not hasattr(self.__already_exported, environment_variable)):
self.__export_file.write('export ' + environment_variable + '="' +
value + '"\n')
self.__already_exported[environment_variable] = True
if os.getenv('AUGUR_DEBUG_LOG_ENV', '0') == '1':
logger.debug('{}:{} = {}'.format(section, name, value))
return value
def publicwww(self):
from augur.publicwww import PublicWWW
if self.__publicwww is None:
logger.debug('Initializing PublicWWW')
self.__publicwww = PublicWWW(api_key=self.read_config(
'PublicWWW', 'apikey', 'AUGUR_PUBLIC_WWW_API_KEY', 'None'))
return self.__publicwww
def git(self, update=False):
from augur.git import Git
storage = self.path_relative_to_config(
self.read_config('Git', 'storage', 'AUGUR_GIT_STORAGE',
'runtime/git_repos/'))
repolist = self.read_config('Git', 'repositories', None, [])
if self.__git is None:
logger.debug('Initializing Git')
self.__git = Git(list_of_repositories=repolist,
storage_folder=storage,
csv=self.localcsv(),
cache=self.cache)
self.__updatable.append({
'name':
'git',
'delay':
int(
self.read_config('Git', 'refresh', 'AUGUR_GIT_REFRESH',
'3600')),
'update':
self.__git.update
})
if update:
self.__git.update()
return self.__git
def read_config(self,
section,
name,
environment_variable=None,
default=None):
"""
Read a variable in specified section of the config file, unless provided an environment variable
:param section: location of given variable
:param name: name of variable
"""
value = None
if environment_variable is not None:
value = os.getenv(environment_variable)
if value is None:
try:
value = self.__config[section][name]
except Exception as e:
value = default
if not section in self.__config:
self.__config[section] = {}
if self.__using_config_file:
self.__config_bad = True
self.__config[section][name] = default
if (environment_variable is not None and value is not None
and self.__export_env and
not hasattr(self.__already_exported, environment_variable)):
self.__export_file.write('export ' + environment_variable + '="' +
str(value) + '"\n')
self.__already_exported[environment_variable] = True
if os.getenv('AUGUR_DEBUG_LOG_ENV', '0') == '1':
logger.debug('{}:{} = {}'.format(section, name, value))
return value
def schedule_updates(self):
"""
Schedules updates
"""
# don't use this,
logger.debug('Scheduling updates...')
self.__updater()
def check_pgpass_credentials(config):
pgpass_file_path = environ['HOME'] + '/.pgpass'
if not path.isfile(pgpass_file_path):
logger.debug("~/.pgpass does not exist, creating.")
open(pgpass_file_path, 'w+')
chmod(pgpass_file_path, stat.S_IWRITE | stat.S_IREAD)
pgpass_file_mask = oct(os.stat(pgpass_file_path).st_mode & 0o777)
if pgpass_file_mask != '0o600':
logger.debug("Updating ~/.pgpass file permissions.")
chmod(pgpass_file_path, stat.S_IWRITE | stat.S_IREAD)
with open(pgpass_file_path, 'a+') as pgpass_file:
end = pgpass_file.tell()
pgpass_file.seek(0)
credentials_string = str(config['Database']['host']) \
+ ':' + str(config['Database']['port']) \
+ ':' + str(config['Database']['name']) \
+ ':' + str(config['Database']['user']) \
+ ':' + str(config['Database']['password'])
if credentials_string.lower() not in [
''.join(line.split()).lower()
for line in pgpass_file.readlines()
]:
logger.info("Adding credentials to $HOME/.pgpass")
pgpass_file.seek(end)
pgpass_file.write(credentials_string + '\n')
else:
logger.info("Credentials found in $HOME/.pgpass")
def __init__(self,
user,
password,
host,
port,
dbname,
ghtorrent,
buildMode="auto"):
"""
Connect to the database
:param dbstr: The [database string](http://docs.sqlalchemy.org/en/latest/core/engines.html) to connect to the GHTorrent database
"""
self.DB_STR = 'mysql+pymysql://{}:{}@{}:{}/{}'.format(
user, password, host, port, dbname)
logger.debug('GHTorrentPlus: Connecting to {}:{}/{} as {}'.format(
host, port, dbname, user))
self.db = s.create_engine(self.DB_STR, poolclass=s.pool.NullPool)
self.ghtorrent = ghtorrent
try:
# Table creation
if (buildMode == 'rebuild') or ((not self.db.dialect.has_table(
self.db.connect(), 'issue_response_time'))
and buildMode == "auto"):
logger.info(
"[GHTorrentPlus] Creating Issue Response Time table...")
self.build_issue_response_time()
except Exception as e:
logger.error(
"Could not connect to GHTorrentPlus database. Error: " +
str(e))
def librariesio(self):
from augur.librariesio import LibrariesIO
if self.__librariesio is None:
logger.debug('Initializing LibrariesIO')
self.__librariesio = LibrariesIO(api_key=self.read_config(
'LibrariesIO', 'apikey', 'AUGUR_LIBRARIESIO_API_KEY', 'None'),
githubapi=self.githubapi())
return self.__librariesio
def __call__(self):
from .githubapi import GitHubAPI
if self.__githubapi is None:
logger.debug('Initializing GitHub API')
api_key = self._augur.read_config('GitHub', 'apikey',
'AUGUR_GITHUB_API_KEY', 'None')
self.__githubapi = GitHubAPI(api_key=api_key)
return self.__githubapi
def __call__(self):
from .metrics_status import MetricsStatus
from augur.datasources.githubapi.githubapi import GitHubAPI
if self.__metrics_status is None:
logger.debug('Initializing MetricsStatus')
api_key = self._augur.read_config('GitHub', 'apikey', 'AUGUR_GITHUB_API_KEY', 'None')
self.__githubapi = GitHubAPI(api_key=api_key)
self.__metrics_status = MetricsStatus(self.__githubapi)
return self.__metrics_status
def __init__(self, user, password, host, port, dbname):
"""
Connect to LibrariesIO
:param dbstr: The [database string](http://docs.sqlalchemy.org/en/latest/core/engines.html) to connect to the LibrariesIO database
"""
self.DB_STR = 'mysql+pymysql://{}:{}@{}:{}/{}'.format(
user, password, host, port, dbname)
logger.debug('LibrariesIO: Connecting to {}:{}/{} as {}'.format(
host, port, dbname, user))
self.db = s.create_engine(self.DB_STR, poolclass=s.pool.NullPool)
def __getitem__(self, plugin_name):
"""
Returns plugin matching the name of the parameter 'plugin_name'
:param plugin_name: name of specified plugin
"""
if plugin_name not in self._loaded_plugins:
if plugin_name not in Application.plugins:
raise ValueError('Plugin %s not found.' % plugin_name)
self._loaded_plugins[plugin_name] = Application.plugins[plugin_name](self)
logger.debug('{plugin_name} plugin loaded')
return self._loaded_plugins[plugin_name]
def sbom_download(self, repo_group_id, repo_id=None):
"""REQUIRES SBOMS TO BE PRESENT IN THE DATABASE
:param repo_id: The repository's repo_id, defaults to None
:return: dosocs sbom
"""
dosocs_SQL = s.sql.text("""
select * from augur_data.repo_sbom_scans
where repo_id = :repo_id;
""")
logger.debug(dosocs_SQL)
params = {'repo_id': repo_id}
return pd.read_sql(dosocs_SQL, self.database, params=params)
def __init__(self, user, password, host, port, dbname):
"""
Connect to GHTorrent
:param dbstr: The [database string](http://docs.sqlalchemy.org/en/latest/core/engines.html) to connect to the GHTorrent database
"""
self.DB_STR = 'mysql+pymysql://{}:{}@{}:{}/{}'.format(
user, password, host, port, dbname
)
logger.debug('GHTorrent: Connecting to {}:{}/{} as {}'.format(host, port, dbname, user))
self.db = s.create_engine(self.DB_STR, poolclass=s.pool.NullPool)
try:
self.userid('howderek')
except Exception as e:
logger.error("Could not connect to GHTorrent database. Error: " + str(e))
def __call__(self):
from .ghtorrent import GHTorrent
if self.__ghtorrent is None:
logger.debug('Initializing GHTorrent')
self.__ghtorrent = GHTorrent(
user=self._augur.read_config('GHTorrent', 'user',
'AUGUR_DB_USER', 'root'),
password=self._augur.read_config('GHTorrent', 'pass',
'AUGUR_DB_PASS', 'password'),
host=self._augur.read_config('GHTorrent', 'host',
'AUGUR_DB_HOST', '127.0.0.1'),
port=self._augur.read_config('GHTorrent', 'port',
'AUGUR_DB_PORT', '3306'),
dbname=self._augur.read_config('GHTorrent', 'name',
'AUGUR_DB_NAME', 'msr14'))
return self.__ghtorrent
def __init__(self, user, password, host, port, dbname, schema):
"""
Connect to Augur
:param dbstr: The [database string](http://docs.sqlalchemy.org/en/latest/core/engines.html) to connect to the Augur database
"""
self.DB_STR = 'postgresql://{}:{}@{}:{}/{}'.format(
user, password, host, port, dbname)
self.db = s.create_engine(
self.DB_STR,
poolclass=s.pool.NullPool,
connect_args={'options': '-csearch_path={}'.format(schema)})
logger.debug(
'Augur DB: Connecting to {} schema of {}:{}/{} as {}'.format(
schema, host, port, dbname, user))
self.metrics_status = []
def __init__(self,
user,
password,
host,
port,
dbname,
ghtorrent,
buildMode="auto"):
"""
Connect to the database
:param dbstr: The [database string](http://docs.sqlalchemy.org/en/latest/core/engines.html) to connect to the GHTorrent database
"""
self.DB_STR = 'mysql+pymysql://{}:{}@{}:{}/{}'.format(
user, password, host, port, dbname)
logger.debug('GHTorrentPlus: Connecting to {}:{}/{} as {}'.format(
host, port, dbname, user))
self.db = s.create_engine(self.DB_STR, poolclass=s.pool.NullPool)
self.ghtorrent = ghtorrent
def __call__(self):
from .metrics_status import MetricsStatus
if self.__metrics_status is None:
logger.debug('Initializing MetricsStatus')
self.__metrics_status = MetricsStatus(
user=self._augur.read_config('Database', 'user',
'AUGUR_DB_USER', 'augur'),
password=self._augur.read_config('Database', 'password',
'AUGUR_DB_PASS', 'password'),
host=self._augur.read_config('Database', 'host',
'AUGUR_DB_HOST', '127.0.0.1'),
port=self._augur.read_config('Database', 'port',
'AUGUR_DB_PORT', '5433'),
dbname=self._augur.read_config('Database', 'name',
'AUGUR_DB_NAME', 'augur'),
schema=self._augur.read_config('Database', 'schema',
'AUGUR_DB_SCHEMA',
'augur_data'))
return self.__metrics_status
def ghtorrentplus(self):
from augur.ghtorrentplus import GHTorrentPlus
if self.__ghtorrentplus is None:
logger.debug('Initializing GHTorrentPlus')
self.__ghtorrentplus = GHTorrentPlus(
user=self.read_config('GHTorrentPlus', 'user',
'AUGUR_GHTORRENT_PLUS_USER', 'root'),
password=self.read_config('GHTorrentPlus', 'pass',
'AUGUR_GHTORRENT_PLUS_PASS',
'password'),
host=self.read_config('GHTorrentPlus', 'host',
'AUGUR_GHTORRENT_PLUS_HOST',
'127.0.0.1'),
port=self.read_config('GHTorrentPlus', 'port',
'AUGUR_GHTORRENT_PLUS_PORT', '3306'),
dbname=self.read_config('GHTorrentPlus', 'name',
'AUGUR_GHTORRENT_PLUS_NAME',
'ghtorrentplus'),
ghtorrent=self.ghtorrent())
return self.__ghtorrentplus
def __call__(self):
from .facade import Facade
if self.__facade is None:
logger.debug('Initializing Facade')
self.__facade = Facade(
user=self._augur.read_config('Facade', 'user',
'AUGUR_FACADE_DB_USER', 'root'),
password=self._augur.read_config('Facade', 'pass',
'AUGUR_FACADE_DB_PASS', ''),
host=self._augur.read_config('Facade', 'host',
'AUGUR_FACADE_DB_HOST',
'127.0.0.1'),
port=self._augur.read_config('Facade', 'port',
'AUGUR_FACADE_DB_PORT', '3306'),
dbname=self._augur.read_config('Facade', 'name',
'AUGUR_FACADE_DB_NAME',
'facade'),
projects=self._augur.read_config('Facade', 'projects', None,
[]))
return self.__facade
def read_config(self, section, name=None):
"""
Read a variable in specified section of the config file, unless provided an environment variable
:param section: location of given variable
:param name: name of variable
"""
if section is not None:
value = self.config[section]
if name is not None:
try:
value = self.config[section][name]
except KeyError as e:
pass
else:
value = None
if os.getenv('AUGUR_DEBUG_LOG_ENV', '0') == '1':
logger.debug('{}:{} = {}'.format(section, name, value))
return value
def __call__(self):
from .librariesio import LibrariesIO
if self.__librariesio is None:
logger.debug('Initializing LibrariesIO')
self.__librariesio = LibrariesIO(
user=self._augur.read_config('Database', 'user',
'AUGUR_LIBRARIESIO_DB_USER',
'root'),
password=self._augur.read_config('Database', 'pass',
'AUGUR_LIBRARIESIO_DB_PASS',
'password'),
host=self._augur.read_config('Database', 'host',
'AUGUR_LIBRARIESIO_DB_HOST',
'127.0.0.1'),
port=self._augur.read_config('Database', 'port',
'AUGUR_LIBRARIESIO_DB_PORT',
'3306'),
dbname=self._augur.read_config('Database', 'name',
'AUGUR_LIBRARIESIO_DB_NAME',
'librariesio'))
return self.__librariesio
def __call__(self):
from .downloads import Downloads
if self.__downloads is None:
logger.debug('Initializing Downloads')
self.__downloads = Downloads(self._augur['githubapi']())
return self.__downloads
def __call__(self):
from .metrics_status import MetricsStatus
if self.__metrics_status is None:
logger.debug('Initializing MetricsStatus')
self.__metrics_status = MetricsStatus(self._augur['githubapi']())
return self.__metrics_status
def batch():
"""
Execute multiple requests, submitted as a batch.
:statuscode 207: Multi status
"""
server.show_metadata = False
if request.method == 'GET':
"""this will return sensible defaults in the future"""
return server.app.make_response(
'{"status": "501", "response": "Defaults for batch requests not implemented. Please POST a JSON array of requests to this endpoint for now."}'
)
try:
requests = json.loads(request.data.decode('utf-8'))
except ValueError as e:
request.abort(400)
responses = []
for index, req in enumerate(requests):
method = req['method']
path = req['path']
body = req.get('body', None)
try:
logger.debug('batch-internal-loop: %s %s' % (method, path))
with server.app.server.app.context():
with server.app.test_request_context(path,
method=method,
data=body):
try:
# Can modify flask.g here without affecting
# flask.g of the root request for the batch
# Pre process Request
rv = server.app.preprocess_request()
if rv is None:
# Main Dispatch
rv = server.app.dispatch_request()
except Exception as e:
rv = server.app.handle_user_exception(e)
response = server.app.make_response(rv)
# Post process Request
response = server.app.process_response(response)
# Response is a Flask response object.
# _read_response(response) reads response.response
# and returns a string. If your endpoints return JSON object,
# this string would be the response as a JSON string.
responses.append({
"path": path,
"status": response.status_code,
"response": str(response.get_data(), 'utf8'),
})
except Exception as e:
responses.append({
"path": path,
"status": 500,
"response": str(e)
})
return Response(response=json.dumps(responses),
status=207,
mimetype="server.app.ication/json")
def downloads(self):
from augur.downloads import Downloads
if self.__downloads is None:
logger.debug('Initializing Downloads')
self.__downloads = Downloads(self.githubapi())
return self.__downloads
def localcsv(self):
from augur.localcsv import LocalCSV
if self.__localCSV is None:
logger.debug('Initializing LocalCSV')
self.__localCSV = LocalCSV()
return self.__localCSV
