def git_repo_initialize(cfg, repo_group_id=None):
# Select any new git repos so we can set up their locations and git clone
if repo_group_id is None:
cfg.update_status('Fetching non-cloned repos')
cfg.log_activity('Info', 'Fetching non-cloned repos')
query = "SELECT repo_id,repo_group_id,repo_git FROM repo WHERE repo_status LIKE 'New%'"
cfg.cursor.execute(query)
new_repos = []
all_repos = list(cfg.cursor)
for repo in all_repos:
if not os.path.isdir(cfg.repo_base_directory + str(repo[0])):
new_repos.append(repo)
else:
cfg.update_status(
'Fetching repos with repo group id: {}'.format(repo_group_id))
cfg.log_activity(
'Info',
'Fetching repos with repo group id: {}'.format(repo_group_id))
query = "SELECT repo_id,repo_group_id,repo_git FROM repo WHERE repo_status LIKE 'New%'"
cfg.cursor.execute(query)
new_repos = list(cfg.cursor)
for row in new_repos:
cfg.log_activity(
'Info',
'Fetching repos with repo group id: {}'.format(repo_group_id))
cfg.log_activity(
'Info',
'Fetching repos with repo group id: {}'.format(repo_group_id))
update_repo_log(cfg, row[0], 'Cloning')
git = html.unescape(row[2])
# Strip protocol from remote URL, set a unique path on the filesystem
if git.find('://', 0) > 0:
repo_relative_path = git[git.find('://', 0) +
3:][:git[git.find('://', 0) +
3:].rfind('/', 0) + 1]
cfg.log_activity(
'Info',
'Repo Relative Path from facade05, from for row in new_repos, line 79: {}'
.format(repo_relative_path))
cfg.log_activity('Info', 'The git path used : {}'.format(git))
else:
repo_relative_path = git[:git.rfind('/', 0) + 1]
cfg.log_activity(
'Info',
'Repo Relative Path from facade05, line 80, reset at 86: {}'.
format(repo_relative_path))
# Get the full path to the directory where we'll clone the repo
repo_path = ('%s%s/%s' %
(cfg.repo_base_directory, row[1], repo_relative_path))
cfg.log_activity(
'Info', 'Repo Path from facade05, line 86: {}'.format(repo_path))
# Get the name of repo
repo_name = git[git.rfind('/', 0) + 1:]
if repo_name.find('.git', 0) > -1:
repo_name = repo_name[:repo_name.find('.git', 0)]
cfg.log_activity(
'Info',
'Repo Name from facade05, line 93: {}'.format(repo_name))
# Check if there will be a storage path collision
query = (
"SELECT NULL FROM repo WHERE CONCAT(repo_group_id,'/',repo_path,repo_name) = %s"
)
cfg.cursor.execute(
query, ('{}/{}{}'.format(row[1], repo_relative_path, repo_name), ))
cfg.db.commit()
# If there is a collision, append a slug to repo_name to yield a unique path
if cfg.cursor.rowcount:
slug = 1
is_collision = True
while is_collision:
if os.path.isdir('%s%s-%s' % (repo_path, repo_name, slug)):
slug += 1
else:
is_collision = False
repo_name = '%s-%s' % (repo_name, slug)
cfg.log_activity(
'Verbose',
'Identical repo detected, storing %s in %s' % (git, repo_name))
# Create the prerequisite directories
return_code = subprocess.Popen(['mkdir -p %s' % repo_path],
shell=True).wait()
# cfg.log_activity('Info','Return code value when making directors from facade05, line 120: {:d}'.format(return_code))
# Make sure it's ok to proceed
if return_code != 0:
print("COULD NOT CREATE REPO DIRECTORY")
update_repo_log(cfg, row[0], 'Failed (mkdir)')
cfg.update_status('Failed (mkdir %s)' % repo_path)
cfg.log_activity('Error',
'Could not create repo directory: %s' % repo_path)
sys.exit("Could not create git repo's prerequisite directories. "
" Do you have write access?")
update_repo_log(cfg, row[0], 'New (cloning)')
query = (
"UPDATE repo SET repo_status='New (Initializing)', repo_path=%s, "
"repo_name=%s WHERE repo_id=%s and repo_status != 'Empty'")
cfg.cursor.execute(query, (repo_relative_path, repo_name, row[0]))
cfg.db.commit()
cfg.log_activity('Verbose', 'Cloning: %s' % git)
cmd = "git -C %s clone '%s' %s" % (repo_path, git, repo_name)
return_code = subprocess.Popen([cmd], shell=True).wait()
if (return_code == 0):
# If cloning succeeded, repo is ready for analysis
# Mark the entire project for an update, so that under normal
# circumstances caches are rebuilt only once per waiting period.
update_project_status = (
"UPDATE repo SET repo_status='Update' WHERE "
"repo_group_id=%s AND repo_status != 'Empty'")
cfg.cursor.execute(update_project_status, (row[1], ))
cfg.db.commit()
# Since we just cloned the new repo, set it straight to analyze.
query = (
"UPDATE repo SET repo_status='Analyze',repo_path=%s, repo_name=%s "
"WHERE repo_id=%s and repo_status != 'Empty'")
cfg.cursor.execute(query, (repo_relative_path, repo_name, row[0]))
cfg.db.commit()
update_repo_log(cfg, row[0], 'Up-to-date')
cfg.log_activity('Info', 'Cloned %s' % git)
else:
# If cloning failed, log it and set the status back to new
update_repo_log(cfg, row[0], 'Failed (%s)' % return_code)
query = (
"UPDATE repo SET repo_status='New (failed)' WHERE repo_id=%s and repo_status !='Empty'"
)
cfg.cursor.execute(query, (row[0], ))
cfg.db.commit()
cfg.log_activity('Error', 'Could not clone %s' % git)
cfg.log_activity('Info', 'Fetching new repos (complete)')
def git_repo_updates(cfg):
# Update existing repos
cfg.update_status('Updating repos')
cfg.log_activity('Info', 'Updating existing repos')
query = (
"SELECT repo_id,repo_group_id,repo_git,repo_name,repo_path FROM repo WHERE "
"repo_status='Update'")
cfg.cursor.execute(query)
existing_repos = list(cfg.cursor)
for row in existing_repos:
cfg.log_activity('Verbose',
'Attempting to update %s' % row[2]) #['git'])
update_repo_log(cfg, row[0], 'Updating') #['id'],'Updating')
attempt = 0
# Try two times. If it fails the first time, reset and clean the git repo,
# as somebody may have done a rebase. No work is being done in the local
# repo, so there shouldn't be legit local changes to worry about.
while attempt < 2:
cmd = ("git -C %s%s/%s%s pull" %
(cfg.repo_base_directory, row[1], row[4], row[3])
) #['projects_id'],row['path'],row['name']))
return_code = subprocess.Popen([cmd], shell=True).wait()
# If the attempt succeeded, then don't try any further fixes. If
# the attempt to fix things failed, give up and try next time.
if return_code == 0 or attempt == 1:
break
elif attempt == 0:
cfg.log_activity(
'Verbose', 'git pull failed, attempting reset and '
'clean for %s' % row[2])
cmd_reset = ("git -C %s%s/%s%s reset --hard origin/master" %
(cfg.repo_base_directory, row[1], row[4], row[3]))
return_code_reset = subprocess.Popen([cmd_reset],
shell=True).wait()
cmd_clean = ("git -C %s%s/%s%s clean -df" %
(cfg.repo_base_directory, row[1], row[4], row[3]))
return_code_clean = subprocess.Popen([cmd_clean],
shell=True).wait()
## patch for primary branch changes to main
cmd_main_branch = (
"git -C %s%s/%s%s checkout main" %
(cfg.repo_base_directory, row[1], row[4], row[3]))
return_code_main_branch = subprocess.Popen([cmd_main_branch],
shell=True).wait()
attempt += 1
if return_code == 0:
set_to_analyze = "UPDATE repo SET repo_status='Analyze' WHERE repo_id=%s and repo_status != 'Empty'"
cfg.cursor.execute(set_to_analyze, (row[0], ))
cfg.db.commit()
update_repo_log(cfg, row[0], 'Up-to-date')
cfg.log_activity('Verbose', 'Updated %s' % row[2])
else:
update_repo_log(cfg, row[0], 'Failed (%s)' % return_code)
cfg.log_activity('Error', 'Could not update %s' % row[2])
cfg.log_activity('Info', 'Updating existing repos (complete)')