class BIF(object):
def __init__(self, interval, length, should_notify=True):
self.cwd = CURRENT_DIRECTORY
self.start = datetime.datetime.now()
reg = r'(\d+)'
length_split = re.split(reg, length)
time_int = length_split[1]
time_type = length_split[2]
if time_type == "h":
self.until = datetime.datetime.now() + datetime.timedelta(
hours=int(time_int))
elif time_type == "m":
self.until = datetime.datetime.now() + datetime.timedelta(
minutes=int(time_int))
elif time_type == "s":
self.until = datetime.datetime.now() + datetime.timedelta(
seconds=int(time_int))
print('Running BIF until {}'.format(self.until))
interval_split = re.split(reg, interval)
interval_time = interval_split[1]
interval_type = interval_split[2]
if interval_type == "m":
self.interval = datetime.timedelta(
minutes=int(interval_time)).total_seconds()
elif interval_type == "h":
self.interval = datetime.timedelta(
minutes=int(interval_time)).total_seconds()
else:
self.interval = int(interval_time)
print('BIF Will Commit Every {} seconds'.format(self.interval))
self.g = Git(self.cwd)
self.length = length
self.should_run = True
self.should_skip = False
self.should_notify = should_notify
self.run()
def run(self):
try:
while True:
time.sleep(self.interval)
if datetime.datetime.now() >= self.until:
break
if self.should_run:
try:
print('[{}] BIF Committing'.format(
datetime.datetime.now()))
self.g.execute(command=[
'git', 'commit', '-am',
'Auto-Commit from BIF @ {} UTC'.format(
datetime.datetime.now())
])
except Exception as e:
with open(CURRENT_DIRECTORY + '/bif.log', 'a') as log:
log.write('FAILED TO COMMIT {}: ERROR {}\n'.format(
datetime.datetime.now(), e))
except KeyboardInterrupt:
print('Exiting...')
def command(self, command, freezer=None):
"""
Runs the Git command in self.repo
"""
args = split(command)
cmd = Git(self.repodir)
cmd.execute(args)
def command(self, command):
"""
Runs the Git command in self.repo
"""
args = split(command)
cmd = Git(self.repodir)
cmd.execute(args)
def get_days_since_last_update(path):
"""
:return: The days since the last update of any of the files in path. If the
path points to a filename, the last update of that filename is
returned.
"""
git = Git(".")
cmd_str = 'git log -1 --format=%%cd %s' % path
cmd = shlex.split(cmd_str)
try:
date_str = git.execute(command=cmd, with_extended_output=False)
except GitCommandError:
raise ValueError('"%s" is not in tracked by this repository.' % path)
# The date_str is in the following format: Sat Jun 21 10:20:31 2014 -0300
# We need to parse it, and then do some date math to return the result
#
# We ignore the UTC offset because it was "hard to parse" and we don't care
last_commit_time = datetime.strptime(date_str[:-6], '%a %b %d %H:%M:%S %Y')
last_commit_date = last_commit_time.date()
today_date = date.today()
time_delta = today_date - last_commit_date
return time_delta.days
def check(self):
git = Git()
output = git.execute(['git', 'ls-remote', '--heads', self.repository])
if output:
return True
return False
def get_days_since_last_update(path):
"""
:return: The days since the last update of any of the files in path. If the
path points to a filename, the last update of that filename is
returned.
"""
git = Git(".")
cmd_str = "git log -1 --format=%%cd %s" % path
cmd = shlex.split(cmd_str)
try:
date_str = git.execute(command=cmd, with_extended_output=False)
except GitCommandError:
raise ValueError('"%s" is not in tracked by this repository.' % path)
# The date_str is in the following format: Sat Jun 21 10:20:31 2014 -0300
# We need to parse it, and then do some date math to return the result
#
# We ignore the UTC offset because it was "hard to parse" and we don't care
last_commit_time = datetime.strptime(date_str[:-6], "%a %b %d %H:%M:%S %Y")
last_commit_date = last_commit_time.date()
today_date = date.today()
time_delta = today_date - last_commit_date
return time_delta.days
def update_repo(project, mirror_dir, gerrit_url):
"""Utility function to mirror git repo
Arguments:
:arg str project: Git project to mirror
:arg str mirror_dir: Path to directory containing repos.
:arg str gerrit_url: URL to the Gerrit server. Used for cloning repos.
"""
log.debug("Updating repo for {0}".format(project))
project_dir = os.path.join(mirror_dir, '{0}.git'.format(project))
if os.path.exists(project_dir):
args = ['git', 'fetch']
else:
os.makedirs(project_dir)
args = ['git', 'clone', '--mirror',
'{0}/{1}'.format(gerrit_url, project), '.']
project_repo = Git(project_dir)
project_repo.execute(args)
def update_repo(project, mirror_dir, gerrit_url):
"""Utility function to mirror git repo
Arguments:
:arg str project: Git project to mirror
:arg str mirror_dir: Path to directory containing repos.
:arg str gerrit_url: URL to the Gerrit server. Used for cloning repos.
"""
log.debug("Updating repo for {0}".format(project))
project_dir = os.path.join(mirror_dir, '{0}.git'.format(project))
if os.path.exists(project_dir):
args = ['git', 'fetch']
else:
os.makedirs(project_dir)
args = [
'git', 'clone', '--mirror', '{0}/{1}'.format(gerrit_url, project),
'.'
]
project_repo = Git(project_dir)
project_repo.execute(args)
def sync_mirror(self):
dirname = self.cache_dir + '/' + self.project
if not self.is_cached():
Workspace.mkdir(dirname)
git = Git(dirname)
git.execute(['git', 'clone', '--mirror', self.repository, dirname])
command = ['git', 'remote', 'add', 'eclogue', self.repository]
git.execute(command)
self.logger.info(' '.join(command))
command = [
'git', 'remote', 'set-url', '--push', 'origin', self.repository
]
git.execute(command)
self.logger.info(' '.join(command))
command = ['git', 'remote', 'update', '--prune', 'origin']
git.execute(command)
self.logger.info(' '.join(command))
def get_srcrev(self):
# e.g. git ls-remote https://github.com/ros2-gbp/ament_lint-release \
# release/bouncy/ament_cmake_copyright/0.5.2-0
# 48bf1aa1cb083a884fbc8520ced00523255aeaed \
# refs/tags/release/bouncy/ament_cmake_copyright/0.5.2-0
# from https://github.com/ros2-gbp/ament_lint-release/archive/ \
# release/bouncy/ament_cmake_copyright/0.5.2-0.tar.gz
from git.cmd import Git
g = Git()
for ref in g.execute([
"git", "ls-remote",
"https://%s" % self.get_repo_src_uri(),
"refs/tags/%s" % self.get_repo_tag_name()
]).split('\n'):
srcrev, tag = ref.split('\t')
if tag == "refs/tags/%s" % self.get_repo_tag_name():
return srcrev
err("Cannot map refs/tags/%s to srcrev in https://%s repository with "
"git ls-remote" %
(self.get_repo_tag_name(), self.get_repo_src_uri()))
return "INVALID"
class GitDownload(object):
def __init__(self, options, build_type='job'):
self.options = options
repository, project, version = self.parse_repository()
self.repository = repository
self.project = project
self.version = version
self.cache_dir = Workspace().get_vcs_space('git')
self.build_type = build_type
self.workspace = self.cwd
self.git = Git(working_dir=self.cwd)
self.refs = None
self.logger = get_logger('console')
def job_space(self, *args, **kwargs):
prefix = config.workspace.get('job')
return os.path.join(prefix, 'git', self.project)
def build_spasce(self, *args, **kwargs):
prefix = config.workspace.get('job')
return os.path.join(prefix, 'git', self.project)
@property
def cwd(self):
if self.build_type == 'job':
return self.job_space()
else:
return self.build_spasce()
def is_cached(self):
try:
cache_path = self.cache_dir + '/' + self.project
is_dir = os.path.isdir(cache_path)
if is_dir:
result = Git(cache_path).execute(
['git', 'rev-parse', '--git-dir'], )
if result.startswith('.'):
return True
return False
except Exception:
return False
def run_command(self, command, *args, **kwargs):
if type(command) == str:
command = command.split(' ')
self.logger.info(' '.join(command))
result = self.git.execute(command, *args, **kwargs)
# self.logger.info(result)
return result
def install(self):
url = self.repository
dest = self.cwd
cache_dir = os.path.join(self.cache_dir, self.project)
self.sync_mirror()
if not os.path.exists(dest):
command = ['git', 'clone', url, dest, '--reference', cache_dir]
Git().execute(command)
command = ['git', 'remote', 'add', 'eclogue', self.repository]
self.logger.info('execute command:{}'.format(' '.join(command)))
self.run_command(command)
command = ['git', 'fetch', 'eclogue']
self.run_command(command)
command = ['git', 'remote', 'set-url', '--push', 'origin', url]
self.run_command(command)
branch_name = self.version
sha, is_branch = self.get_revision_sha(rev=branch_name)
if is_branch:
current_branch = self.current_branch()
if current_branch != self.version:
track_branch = 'origin/{}'.format(self.version)
command = [
'git', 'checkout', '-B', branch_name, '--track',
track_branch
]
self.run_command(command)
command[3] = 'eclogue/' + branch_name
self.run_command(command)
else:
revision = self.get_revision()
if revision != self.version:
command = ['git', 'checkout', '-B', self.version]
self.run_command(command)
command = ['git', 'checkout', '-B', 'eclogue/' + self.version]
self.run_command(command)
self.update_commit()
return self.cwd
def update_commit(self):
revision = self.get_revision()
command = ['git', 'reset', '--hard', revision]
self.run_command(command)
def get_revision_sha(self, rev):
cmd = ['git', 'show-ref', rev]
output = self.run_command(cmd, kill_after_timeout=True)
refs = {}
for line in output.strip().splitlines():
try:
sha, ref = line.split()
except ValueError:
# Include the offending line to simplify troubleshooting if
# this error ever occurs.
raise ValueError('unexpected show-ref line: {!r}'.format(line))
refs[ref] = sha
branch_ref = 'refs/remotes/origin/{}'.format(rev)
tag_ref = 'refs/tags/{}'.format(rev)
sha = refs.get(branch_ref)
if sha is not None:
return [sha, True]
if not sha:
self.run_command(['git', 'fetch', self.repository, self.version])
sha = refs.get(tag_ref)
return [sha, False]
def resolve_revision(self, rev):
sha, is_branch = self.get_revision_sha(rev)
if sha:
return [sha, is_branch]
if not sha:
raise Exception('invalid sha')
def get_revision(self, rev=None):
if rev is None:
rev = 'HEAD'
commad = ['git', 'rev-parse', rev]
current_rev = self.run_command(commad)
return current_rev.strip()
def parse_repository(self):
url = self.options.get('repository')
tag = self.options.get('tag')
branch = self.options.get('branch')
sha = self.options.get('sha')
target = tag or branch or sha
if url.find('#') < 0:
project = os.path.basename(url)
project = project.replace('.git', '')
target = target or 'master'
return [url, project, target]
else:
result = url.split('#')
repository, version = result
project = os.path.basename(repository)
project = project.replace('.git', '')
target = target or version
return [repository, project, target]
def current_branch(self):
cmd = ['git', 'symbolic-ref', '-q', 'HEAD']
output = self.run_command(cmd)
ref = output.strip()
if ref.startswith('refs/heads/'):
return ref[len('refs/heads/'):]
return None
def sync_mirror(self):
dirname = self.cache_dir + '/' + self.project
if not self.is_cached():
Workspace.mkdir(dirname)
git = Git(dirname)
git.execute(['git', 'clone', '--mirror', self.repository, dirname])
command = ['git', 'remote', 'add', 'eclogue', self.repository]
git.execute(command)
self.logger.info(' '.join(command))
command = [
'git', 'remote', 'set-url', '--push', 'origin', self.repository
]
git.execute(command)
self.logger.info(' '.join(command))
command = ['git', 'remote', 'update', '--prune', 'origin']
git.execute(command)
self.logger.info(' '.join(command))
def check(self):
git = Git()
output = git.execute(['git', 'ls-remote', '--heads', self.repository])
if output:
return True
return False
