class Repo_insertion_manager():
ROOT_AUGUR_DIR = os.path.dirname(
os.path.dirname(os.path.realpath(__file__)))
def __init__(self, organization_name, database_connection):
self.org = organization_name
self.db = database_connection
## added for keys
self._root_augur_dir = Repo_insertion_manager.ROOT_AUGUR_DIR
self.augur_config = AugurConfig(self._root_augur_dir)
##########
def get_existing_repos(self, group_id):
"""returns repos belonging to repogroup in augur db"""
select_repos_query = s.sql.text("""
SELECT repo_git from augur_data.repo
WHERE repo_group_id = :repo_group_id
""")
select_repos_query = select_repos_query.bindparams(
repo_group_id=group_id)
result = self.db.execute(select_repos_query)
return result.fetchall()
## This doesn't permit importing of an individual's repo, as they don't show up under "orgs"
# def group_exists_gh(self):
# url = url = "https://api.github.com/orgs/{}".format(self.org)
# res = requests.get(url).json()
# try:
# if res['message'] == "Not Found":
# return False
# except KeyError:
# return True
## Revised Version of Method
def group_exists_gh(self):
url = url = "https://api.github.com/orgs/{}".format(self.org)
## attempting to add key due to rate limiting
gh_api_key = self.augur_config.get_value('Database', 'key')
self.headers = {'Authorization': 'token %s' % gh_api_key}
#r = requests.get(url=cntrb_url, headers=self.headers)
####### Original request code
# res = requests.get(url).json()
########
res = requests.get(url=url, headers=self.headers).json()
try:
if res['message'] == "Not Found":
url = url = "https://api.github.com/users/{}".format(self.org)
res = requests.get(url=url, headers=self.headers).json()
if res['message'] == "Not Found":
return False
except KeyError:
return True
def insert_repo(self, orgid, given_org, reponame):
"""creates a new repo record"""
insert_repo_query = s.sql.text("""
INSERT INTO augur_data.repo(repo_group_id, repo_git, repo_status,
tool_source, tool_version, data_source, data_collection_date)
VALUES (:repo_group_id, :repo_git, 'New', 'CLI', 1.0, 'Git', CURRENT_TIMESTAMP)
RETURNING repo_id
""")
repogit = self.github_urlify(given_org, reponame)
insert_repo_query = insert_repo_query.bindparams(
repo_group_id=int(orgid), repo_git=repogit)
result = self.db.execute(insert_repo_query).fetchone()
return result['repo_id']
def github_urlify(self, org, repo):
return "https://github.com/" + org + "/" + repo
def get_org_id(self):
select_group_query = s.sql.text("""
SELECT repo_group_id
FROM augur_data.repo_groups
WHERE rg_name = :group_name
""")
select_group_query = select_group_query.bindparams(group_name=self.org)
result = self.db.execute(select_group_query)
row = result.fetchone()
return row['repo_group_id']
def insert_repo_group(self):
"""creates a new repo_group record and returns its id"""
insert_group_query = s.sql.text("""
INSERT INTO augur_data.repo_groups(rg_name, rg_description, rg_website, rg_recache, rg_last_modified, rg_type,
tool_source, tool_version, data_source, data_collection_date)
VALUES (:group_name, '', '', 1, CURRENT_TIMESTAMP, 'Unknown', 'Loaded by user', 1.0, 'Git', CURRENT_TIMESTAMP)
RETURNING repo_group_id
""")
insert_group_query = insert_group_query.bindparams(group_name=self.org)
result = self.db.execute(insert_group_query)
row = result.fetchone()
return row['repo_group_id']
def fetch_repos(self):
"""uses the github api to return repos belonging to the given organization"""
gh_api_key = self.augur_config.get_value('Database', 'key')
self.headers = {'Authorization': 'token %s' % gh_api_key}
repos = []
page = 1
url = self.paginate(page)
res = requests.get(url, headers=self.headers).json()
while res:
for repo in res:
repos.append(repo['name'])
page += 1
res = requests.get(self.paginate(page)).json()
return repos
## Modified pagination to account for github orgs that look like orgs but are actually users.
def paginate(self, page):
### Modified here to incorporate the use of a GitHub API Key
gh_api_key = self.augur_config.get_value('Database', 'key')
self.headers = {'Authorization': 'token %s' % gh_api_key}
url = "https://api.github.com/orgs/{}/repos?per_page=100&page={}"
res = requests.get(url, headers=self.headers).json()
if res['message'] == "Not Found":
url = "https://api.github.com/users/{}/repos?per_page=100&page={}"
res = requests.get(url=url, headers=self.headers).json()
return url.format(self.org, str(page))
#r = requests.get(url=cntrb_url, headers=self.headers)
####### Original request code
# res = requests.get(url).json()
########
res = requests.get(url=url, headers=self.headers).json()
class Application():
"""Initalizes all classes from Augur using a config file or environment variables"""
def __init__(self,
given_config={},
disable_logs=False,
offline_mode=False):
"""
Reads config, creates DB session, and initializes cache
"""
self.logging = AugurLogging(disable_logs=disable_logs)
self.root_augur_dir = ROOT_AUGUR_DIRECTORY
self.config = AugurConfig(self.root_augur_dir, given_config)
# we need these for later
self.housekeeper = None
self.manager = None
self.gunicorn_options = {
'bind':
'%s:%s' % (self.config.get_value(
"Server", "host"), self.config.get_value("Server", "port")),
'workers':
int(self.config.get_value('Server', 'workers')),
'timeout':
int(self.config.get_value('Server', 'timeout'))
}
self.logging.configure_logging(self.config)
self.gunicorn_options.update(self.logging.gunicorn_logging_options)
self.cache_config = {
'cache.type': 'file',
'cache.data_dir': 'runtime/cache/',
'cache.lock_dir': 'runtime/cache/'
}
if not os.path.exists(self.cache_config['cache.data_dir']):
os.makedirs(self.cache_config['cache.data_dir'])
if not os.path.exists(self.cache_config['cache.lock_dir']):
os.makedirs(self.cache_config['cache.lock_dir'])
cache_parsed = parse_cache_config_options(self.cache_config)
self.cache = CacheManager(**cache_parsed)
if offline_mode is False:
logger.debug("Running in online mode")
self.database, self.operations_database, self.spdx_database = self._connect_to_database(
)
self.metrics = Metrics(self)
def _connect_to_database(self):
user = self.config.get_value('Database', 'user')
host = self.config.get_value('Database', 'host')
port = self.config.get_value('Database', 'port')
dbname = self.config.get_value('Database', 'name')
database_connection_string = 'postgresql://{}:{}@{}:{}/{}'.format(
user, self.config.get_value('Database', 'password'), host, port,
dbname)
csearch_path_options = 'augur_data'
engine = s.create_engine(
database_connection_string,
poolclass=s.pool.NullPool,
connect_args={'options': f'-csearch_path={csearch_path_options}'},
pool_pre_ping=True)
csearch_path_options += ',spdx'
spdx_engine = s.create_engine(
database_connection_string,
poolclass=s.pool.NullPool,
connect_args={'options': f'-csearch_path={csearch_path_options}'},
pool_pre_ping=True)
helper_engine = s.create_engine(
database_connection_string,
poolclass=s.pool.NullPool,
connect_args={'options': f'-csearch_path=augur_operations'},
pool_pre_ping=True)
try:
engine.connect().close()
helper_engine.connect().close()
spdx_engine.connect().close()
logger.debug("Database connection successfully established")
return engine, helper_engine, spdx_engine
except s.exc.OperationalError as e:
logger.error("Unable to connect to the database. Terminating...")
raise (e)
def shutdown(self):
if self.logging.stop_event is not None:
logger.debug("Stopping housekeeper logging listener...")
self.logging.stop_event.set()
if self.housekeeper is not None:
logger.debug("Shutting down housekeeper updates...")
self.housekeeper.shutdown_updates()
self.housekeeper = None
if self.manager is not None:
logger.debug("Shutting down manager...")
self.manager.shutdown()
self.manager = None