class BIF(object): def __init__(self, interval, length, should_notify=True): self.cwd = CURRENT_DIRECTORY self.start = datetime.datetime.now() reg = r'(\d+)' length_split = re.split(reg, length) time_int = length_split[1] time_type = length_split[2] if time_type == "h": self.until = datetime.datetime.now() + datetime.timedelta( hours=int(time_int)) elif time_type == "m": self.until = datetime.datetime.now() + datetime.timedelta( minutes=int(time_int)) elif time_type == "s": self.until = datetime.datetime.now() + datetime.timedelta( seconds=int(time_int)) print('Running BIF until {}'.format(self.until)) interval_split = re.split(reg, interval) interval_time = interval_split[1] interval_type = interval_split[2] if interval_type == "m": self.interval = datetime.timedelta( minutes=int(interval_time)).total_seconds() elif interval_type == "h": self.interval = datetime.timedelta( minutes=int(interval_time)).total_seconds() else: self.interval = int(interval_time) print('BIF Will Commit Every {} seconds'.format(self.interval)) self.g = Git(self.cwd) self.length = length self.should_run = True self.should_skip = False self.should_notify = should_notify self.run() def run(self): try: while True: time.sleep(self.interval) if datetime.datetime.now() >= self.until: break if self.should_run: try: print('[{}] BIF Committing'.format( datetime.datetime.now())) self.g.execute(command=[ 'git', 'commit', '-am', 'Auto-Commit from BIF @ {} UTC'.format( datetime.datetime.now()) ]) except Exception as e: with open(CURRENT_DIRECTORY + '/bif.log', 'a') as log: log.write('FAILED TO COMMIT {}: ERROR {}\n'.format( datetime.datetime.now(), e)) except KeyboardInterrupt: print('Exiting...')
def command(self, command, freezer=None): """ Runs the Git command in self.repo """ args = split(command) cmd = Git(self.repodir) cmd.execute(args)
def command(self, command): """ Runs the Git command in self.repo """ args = split(command) cmd = Git(self.repodir) cmd.execute(args)
def get_days_since_last_update(path): """ :return: The days since the last update of any of the files in path. If the path points to a filename, the last update of that filename is returned. """ git = Git(".") cmd_str = 'git log -1 --format=%%cd %s' % path cmd = shlex.split(cmd_str) try: date_str = git.execute(command=cmd, with_extended_output=False) except GitCommandError: raise ValueError('"%s" is not in tracked by this repository.' % path) # The date_str is in the following format: Sat Jun 21 10:20:31 2014 -0300 # We need to parse it, and then do some date math to return the result # # We ignore the UTC offset because it was "hard to parse" and we don't care last_commit_time = datetime.strptime(date_str[:-6], '%a %b %d %H:%M:%S %Y') last_commit_date = last_commit_time.date() today_date = date.today() time_delta = today_date - last_commit_date return time_delta.days
def check(self): git = Git() output = git.execute(['git', 'ls-remote', '--heads', self.repository]) if output: return True return False
def get_days_since_last_update(path): """ :return: The days since the last update of any of the files in path. If the path points to a filename, the last update of that filename is returned. """ git = Git(".") cmd_str = "git log -1 --format=%%cd %s" % path cmd = shlex.split(cmd_str) try: date_str = git.execute(command=cmd, with_extended_output=False) except GitCommandError: raise ValueError('"%s" is not in tracked by this repository.' % path) # The date_str is in the following format: Sat Jun 21 10:20:31 2014 -0300 # We need to parse it, and then do some date math to return the result # # We ignore the UTC offset because it was "hard to parse" and we don't care last_commit_time = datetime.strptime(date_str[:-6], "%a %b %d %H:%M:%S %Y") last_commit_date = last_commit_time.date() today_date = date.today() time_delta = today_date - last_commit_date return time_delta.days
def update_repo(project, mirror_dir, gerrit_url): """Utility function to mirror git repo Arguments: :arg str project: Git project to mirror :arg str mirror_dir: Path to directory containing repos. :arg str gerrit_url: URL to the Gerrit server. Used for cloning repos. """ log.debug("Updating repo for {0}".format(project)) project_dir = os.path.join(mirror_dir, '{0}.git'.format(project)) if os.path.exists(project_dir): args = ['git', 'fetch'] else: os.makedirs(project_dir) args = ['git', 'clone', '--mirror', '{0}/{1}'.format(gerrit_url, project), '.'] project_repo = Git(project_dir) project_repo.execute(args)
def update_repo(project, mirror_dir, gerrit_url): """Utility function to mirror git repo Arguments: :arg str project: Git project to mirror :arg str mirror_dir: Path to directory containing repos. :arg str gerrit_url: URL to the Gerrit server. Used for cloning repos. """ log.debug("Updating repo for {0}".format(project)) project_dir = os.path.join(mirror_dir, '{0}.git'.format(project)) if os.path.exists(project_dir): args = ['git', 'fetch'] else: os.makedirs(project_dir) args = [ 'git', 'clone', '--mirror', '{0}/{1}'.format(gerrit_url, project), '.' ] project_repo = Git(project_dir) project_repo.execute(args)
def sync_mirror(self): dirname = self.cache_dir + '/' + self.project if not self.is_cached(): Workspace.mkdir(dirname) git = Git(dirname) git.execute(['git', 'clone', '--mirror', self.repository, dirname]) command = ['git', 'remote', 'add', 'eclogue', self.repository] git.execute(command) self.logger.info(' '.join(command)) command = [ 'git', 'remote', 'set-url', '--push', 'origin', self.repository ] git.execute(command) self.logger.info(' '.join(command)) command = ['git', 'remote', 'update', '--prune', 'origin'] git.execute(command) self.logger.info(' '.join(command))
def get_srcrev(self): # e.g. git ls-remote https://github.com/ros2-gbp/ament_lint-release \ # release/bouncy/ament_cmake_copyright/0.5.2-0 # 48bf1aa1cb083a884fbc8520ced00523255aeaed \ # refs/tags/release/bouncy/ament_cmake_copyright/0.5.2-0 # from https://github.com/ros2-gbp/ament_lint-release/archive/ \ # release/bouncy/ament_cmake_copyright/0.5.2-0.tar.gz from git.cmd import Git g = Git() for ref in g.execute([ "git", "ls-remote", "https://%s" % self.get_repo_src_uri(), "refs/tags/%s" % self.get_repo_tag_name() ]).split('\n'): srcrev, tag = ref.split('\t') if tag == "refs/tags/%s" % self.get_repo_tag_name(): return srcrev err("Cannot map refs/tags/%s to srcrev in https://%s repository with " "git ls-remote" % (self.get_repo_tag_name(), self.get_repo_src_uri())) return "INVALID"
class GitDownload(object): def __init__(self, options, build_type='job'): self.options = options repository, project, version = self.parse_repository() self.repository = repository self.project = project self.version = version self.cache_dir = Workspace().get_vcs_space('git') self.build_type = build_type self.workspace = self.cwd self.git = Git(working_dir=self.cwd) self.refs = None self.logger = get_logger('console') def job_space(self, *args, **kwargs): prefix = config.workspace.get('job') return os.path.join(prefix, 'git', self.project) def build_spasce(self, *args, **kwargs): prefix = config.workspace.get('job') return os.path.join(prefix, 'git', self.project) @property def cwd(self): if self.build_type == 'job': return self.job_space() else: return self.build_spasce() def is_cached(self): try: cache_path = self.cache_dir + '/' + self.project is_dir = os.path.isdir(cache_path) if is_dir: result = Git(cache_path).execute( ['git', 'rev-parse', '--git-dir'], ) if result.startswith('.'): return True return False except Exception: return False def run_command(self, command, *args, **kwargs): if type(command) == str: command = command.split(' ') self.logger.info(' '.join(command)) result = self.git.execute(command, *args, **kwargs) # self.logger.info(result) return result def install(self): url = self.repository dest = self.cwd cache_dir = os.path.join(self.cache_dir, self.project) self.sync_mirror() if not os.path.exists(dest): command = ['git', 'clone', url, dest, '--reference', cache_dir] Git().execute(command) command = ['git', 'remote', 'add', 'eclogue', self.repository] self.logger.info('execute command:{}'.format(' '.join(command))) self.run_command(command) command = ['git', 'fetch', 'eclogue'] self.run_command(command) command = ['git', 'remote', 'set-url', '--push', 'origin', url] self.run_command(command) branch_name = self.version sha, is_branch = self.get_revision_sha(rev=branch_name) if is_branch: current_branch = self.current_branch() if current_branch != self.version: track_branch = 'origin/{}'.format(self.version) command = [ 'git', 'checkout', '-B', branch_name, '--track', track_branch ] self.run_command(command) command[3] = 'eclogue/' + branch_name self.run_command(command) else: revision = self.get_revision() if revision != self.version: command = ['git', 'checkout', '-B', self.version] self.run_command(command) command = ['git', 'checkout', '-B', 'eclogue/' + self.version] self.run_command(command) self.update_commit() return self.cwd def update_commit(self): revision = self.get_revision() command = ['git', 'reset', '--hard', revision] self.run_command(command) def get_revision_sha(self, rev): cmd = ['git', 'show-ref', rev] output = self.run_command(cmd, kill_after_timeout=True) refs = {} for line in output.strip().splitlines(): try: sha, ref = line.split() except ValueError: # Include the offending line to simplify troubleshooting if # this error ever occurs. raise ValueError('unexpected show-ref line: {!r}'.format(line)) refs[ref] = sha branch_ref = 'refs/remotes/origin/{}'.format(rev) tag_ref = 'refs/tags/{}'.format(rev) sha = refs.get(branch_ref) if sha is not None: return [sha, True] if not sha: self.run_command(['git', 'fetch', self.repository, self.version]) sha = refs.get(tag_ref) return [sha, False] def resolve_revision(self, rev): sha, is_branch = self.get_revision_sha(rev) if sha: return [sha, is_branch] if not sha: raise Exception('invalid sha') def get_revision(self, rev=None): if rev is None: rev = 'HEAD' commad = ['git', 'rev-parse', rev] current_rev = self.run_command(commad) return current_rev.strip() def parse_repository(self): url = self.options.get('repository') tag = self.options.get('tag') branch = self.options.get('branch') sha = self.options.get('sha') target = tag or branch or sha if url.find('#') < 0: project = os.path.basename(url) project = project.replace('.git', '') target = target or 'master' return [url, project, target] else: result = url.split('#') repository, version = result project = os.path.basename(repository) project = project.replace('.git', '') target = target or version return [repository, project, target] def current_branch(self): cmd = ['git', 'symbolic-ref', '-q', 'HEAD'] output = self.run_command(cmd) ref = output.strip() if ref.startswith('refs/heads/'): return ref[len('refs/heads/'):] return None def sync_mirror(self): dirname = self.cache_dir + '/' + self.project if not self.is_cached(): Workspace.mkdir(dirname) git = Git(dirname) git.execute(['git', 'clone', '--mirror', self.repository, dirname]) command = ['git', 'remote', 'add', 'eclogue', self.repository] git.execute(command) self.logger.info(' '.join(command)) command = [ 'git', 'remote', 'set-url', '--push', 'origin', self.repository ] git.execute(command) self.logger.info(' '.join(command)) command = ['git', 'remote', 'update', '--prune', 'origin'] git.execute(command) self.logger.info(' '.join(command)) def check(self): git = Git() output = git.execute(['git', 'ls-remote', '--heads', self.repository]) if output: return True return False