def set_status(migration_id: str, path_to_db_dir: str, status: Status, app_logger: logger.Logger) -> bool:
"""
Mark current migration as :param mark: in sqlite database. Return True on success.
:param migration_id: migration name (without extension)
:param path_to_db_dir: absolute path to sqlite database with migrations metadata
:param status: how to mark a migration. Possible values are covered by :type migration.Status: enum.
:param app_logger: pymigrate configured logger
:return: True on success, False otherwise
"""
query_template = "UPDATE migrations SET migration_id='{0}', branch='{1}', status='{2}' where migration_id='{3}'"
if not path_to_db_dir:
db = os.path.dirname(os.path.realpath(__file__)) + '/' + os.path.join(os.pardir, 'migrations.db')
if not os.path.isfile(db):
db_init(os.path.dirname(db), app_logger)
branch = git.get_branch(os.path.dirname(db))
else:
db = path_to_db_dir + '/migrations.db'
if not os.path.isfile(db):
db_init(path_to_db_dir, app_logger)
branch = git.get_branch(path_to_db_dir)
if check_status(migration_id, db, app_logger) == 'UNKNOWN':
print("Migration not found: %s" % migration_id)
return False
with sqlite3.connect(db) as conn:
c = conn.cursor()
c.execute(query_template.format(migration_id, branch, status.name, migration_id))
conn.commit()
return True
def rebase():
"""Rebases the target and temporary change branches.
Rebases the target branch (the branch from which the temporary
change branch was created) and then rebases the temporary change
branch. This can be used to pull upstream changes down to both
branches to resolve a failed Gerrit submission due to a path
conflict.
If there are conflicts with either rebase operation, the process
terminates and it is up to the user to resolve the conflicts.
"""
change_id = check_for_change_branch()
change = get_change(change_id)
target_branch = change["branch"]
change_branch = git.get_branch()
git.run_command_or_die("git checkout %s" % target_branch)
try:
git.run_command("git pull --rebase", output_on_error=False)
except git.CalledProcessError, e:
print (
"Rebase failed for branch %s. After resolving merge failure(s),\n"
'check out the change branch (%s) and run "git change rebase" again.\n'
'See "git help rebase" for help on resolving merge conflicts.' % (target_branch, change_branch)
)
sys.exit(e.returncode)
def db_init(path_to_db_dir: str, app_logger: logger.Logger) -> bool:
"""
Initialize sqlite3 database with migrations metadata at given absolute path.
:param path_to_db_dir: absolute path to migrations directory
:param app_logger: instance of configured logger
:return: True on success, False otherwise
"""
if not os.path.isdir(path_to_db_dir):
app_logger.log_with_ts('Migrations directory does not exist, creating at {0}'.format(path_to_db_dir),
logger.Levels.WARNING)
os.makedirs(path_to_db_dir, 0o775)
migration_names = [migration for migration in os.listdir(path_to_db_dir) if
os.path.isdir(os.path.join(path_to_db_dir, migration))]
# TODO: handle io, sqlite db exceptions
with sqlite3.connect(path_to_db_dir + '/migrations.db') as conn:
app_logger.log_with_ts('Initializing sqlite database', logger.Levels.DEBUG)
c = conn.cursor()
c.execute('CREATE TABLE IF NOT EXISTS migrations (migration_id, status, presence, branch)')
branch = git.get_branch(path_to_db_dir)
for migration_id in migration_names:
c.execute("INSERT INTO migrations VALUES ('{0}', 'PENDING', 'PRESENT', '{1}')".format(migration_id, branch))
conn.commit()
return True
def determine_branches():
"""Determines the current and target branches.
The current branch is the current HEAD, and the target branch is
the branch to which this change is to be merged. The current
branch may or may not be a temporary change branch but the target
branch is always a tracking branch.
Exits with a non-zero status if --chain is true and the current
branch is *not* a change branch, or if --chain is false and the
current branch *is* a change branch.
Returns:
A tuple of two strings: the name of the current branch and
that of the target branch.
"""
original_branch = git.get_branch()
if FLAGS.chain:
# Extract the target branch from the previous change with
# which we're chaining.
previous_change_id = get_change_id_from_branch()
if previous_change_id is None:
exit_error("The current branch must be a change branch when you specify --chain.")
previous_change = get_change(previous_change_id)
target_branch = previous_change["branch"]
else:
if original_branch.startswith("change-I"):
exit_error("You are in a temporary change branch. " "If you wish to chain commits, pass --chain.")
target_branch = original_branch
return original_branch, target_branch
def get_base_context(self, repo, sha, path):
d = {}
self.repo = self.get_repo(repo)
try:
if sha in self.repo: # explicit sha
d['ref_name'] = sha[:10]
d['ref_link'] = sha
self.sha = self.repo.get_object(sha)
else:
d['ref_name'] = d['ref_link'] = sha
self.sha = git.get_branch(self.repo, sha)
except KeyError:
logger.exception("Failed to find sha: {0}".format(sha))
raise web.notfound('Bad SHA: {0}'.format(sha))
d['repo'] = repo
d['sha'] = self.sha.id
d['branches'] = git.branches(self.repo)
d['tags'] = git.tags(self.repo)
d['sha_type'] = self.sha_type
d['path'] = path.strip('/')
d['breadcrumbs'] = d['path'].split('/') if path else []
return d
def print_push_command():
"""Prints the command to push a change to Gerrit."""
change_id = get_change_id_from_branch()
if change_id is not None:
change = get_change(change_id)
target_branch = change["branch"]
else:
target_branch = git.get_branch()
print build_push_command(target_branch, draft=FLAGS.draft)
def swarm(num_workers: int):
"""
Kicks off a swarm of builds in Buildkite
"""
branch = git.get_branch()
time_str = str(int(time.time()))
assert "train" in branch, "Must use a training branch"
for i in range(1, num_workers + 1):
worker_branch = f"{branch}-{i}-{time_str}"
print(f"Launching worker {worker_branch}")
git.set_new_branch(worker_branch)
git.push_new_branch(worker_branch)
git.set_branch(branch)
def db_update(config: dict, app_logger: logger.Logger) -> bool:
"""
Check migrations directory for new migrations since last run and update migrations database.
:param config: pymigrate configuration
:param app_logger: instance of configured logger
:return:
"""
app_logger.log_with_ts('Starting migration database update process', logger.Levels.DEBUG)
migrations_directory_path = os.path.join(os.pardir, config['PROJECT_DIR'] + '/' + config['MIGRATIONS_DIR'])
migration_ids = [migration_id for migration_id in os.listdir(migrations_directory_path) if
os.path.isdir(os.path.join(migrations_directory_path, migration_id))]
migrations_from_db = get_statuses(migrations_directory_path + '/migrations.db', app_logger)
branch = git.get_branch(migrations_directory_path)
app_logger.log_with_ts('Got git branch: {0}'.format(branch), logger.Levels.DEBUG)
# TODO: handle io, sqlite db exceptions
with sqlite3.connect(migrations_directory_path + '/migrations.db') as conn:
c = conn.cursor()
for migration_id, status in migrations_from_db.items():
if migration_id not in migration_ids:
app_logger.log_with_ts('Migration {0} is missing on disk, marking it ABSENT'.format(migration_id),
logger.Levels.DEBUG)
c.execute(
"UPDATE migrations SET presence='ABSENT' where migration_id='{0}'".format(migration_id))
elif migration_id not in migration_ids and status == 'ABSENT':
app_logger.log_with_ts('Migration re-appeared: {0}'.format(migration_id), logger.Levels.DEBUG)
c.execute("UPDATE migrations SET presence='PRESENT' where migration_id='{0}'".format(migration_id))
for migration_id in migration_ids:
if migration_id not in migrations_from_db:
app_logger.log_with_ts('New migration detected: {0}'.format(migration_id), logger.Levels.DEBUG)
c.execute(
"INSERT INTO migrations VALUES ('{0}', 'PENDING', 'PRESENT', '{1}')".format(migration_id, branch))
# Set migration status MANUAL if readme.* is present
check_query = "SELECT status from migrations where migration_id='{0}'"
readme_files = util.find_files('readme*', migrations_directory_path + '/' + migration_id, False)
if len(readme_files) != 0 and os.path.isfile(readme_files[0]) and \
c.execute(check_query.format(migration_id)).fetchone()[0].replace('\n', '') not in (
Status.DONE.name, Status.FAILED.name, Status.SKIP.name):
app_logger.log_with_ts('Readme file detected for migration: {0}'.format(migration_id),
logger.Levels.DEBUG)
c.execute("UPDATE migrations SET status='MANUAL' where migration_id='{0}'".format(migration_id))
return True
def get_change_id_from_branch():
"""Returns the change ID embedded in the current branch name.
Assumes the current branch is a temporary change branch created by
a previous run of git-change. Example branch name:
'change-Id06774ede265426f85d36cca50bba69d8aa54ed8'.
Returns:
A string representing the change ID embedded in the current
branch name, or None if the current branch is not a temporary
change branch.
"""
branch = git.get_branch()
if branch.startswith("change-I"):
_, change_id = branch.split("-")
return change_id
return None
def get_conf(path=None):
conf = load_conf(path)
conf['system'] = AttributeDict()
if os.path.exists('/etc/arch-release'):
conf.system.python = 'python2'
else:
conf.system.python = 'python'
conf.git['branches'] = AttributeDict()
conf.git.branches.current = get_branch()
conf.git.commit = get_commit()
conf.paths.update(render_paths(conf))
conf.system.dependency_cache = os.path.join(conf.paths.projectroot,
conf.paths.output,
'dependencies.json')
return conf
