def git_pull(remote_url, local, idc_dir): """ @remote: git remote address @local: local git repo dir """ commit_msg = None if not os.path.exists(local): os.mkdir(local) logging.info('git start pull from remote repo.') if not os.path.exists( "{0}/.git".format(local)) or not git.repo.fun.is_git_dir( "{0}/.git".format(local)): repo_clone = Repo.clone_from(remote_url, local) if not repo_clone: logging.debug("clone repo from {0} to {1}failed.\n".format( remote_url, local)) return repo = Repo(local) remote = None git_commit = git.Git(idc_dir) last_commit_log = "/var/log/last_commit.log" last_commit = None if os.path.exists(last_commit_log): with file(last_commit_log) as fhand: last_log_commit = fhand.read().strip() # Warning: for performance only check 500 commit, use this write mysql for commit in repo.iter_commits('master', max_count=500): if last_commit == commit.hexsha: last_commit = last_log_commit break if not last_commit: last_commit = repo.head.commit.hexsha with FileLock(os.path.split(local)[1]) as _flock: for remote_name in repo.remotes: if remote_name.url == remote_url: remote = remote_name break if remote is None: remote_name = "%x" % (random.randint(0, 1000000) ^ int(time.time())) remote = repo.create_remote(remote_name, remote_url) ret = 0 try: info_list = remote.pull() for info in info_list: ret = info.flags & (git.FetchInfo.REJECTED | git.FetchInfo.ERROR) if ret > 0: logging.warning("[conf-pull] pull from remote error!\n") commit_msg = git_commit.log("{0}..HEAD".format(last_commit), "--pretty=%H,%ai,%s", idc_dir) except (git.GitCommandError, git.RepositoryDirtyError), e: logging.warning("git pull error, error={0}\n".format(str(e))) except Exception: logging.warning("git pull error, other exception.\n")
class GitSync(object): def __init__(self, database): logging.info("Initializing GitSync.") self.database = database self.charts = dict() self.url = DEFAULT_GITREPO try: self.repo = Repo(REPO_DIRECTORY) except InvalidGitRepositoryError: logging.info("Cloning repository in %s", REPO_DIRECTORY) self.repo = Repo.clone_from(self.url, REPO_DIRECTORY) @coroutine def update_repo(self, document): if document['o']["charts"]["repo_url"] != self.url: logging.info("Repo url updated to '%s'.", document['o']["charts"]["repo_url"]) self.url = document['o']["charts"]["repo_url"] @coroutine def sync_loop(self): logging.info("Initializing sync loop.") yield add_callback("Settings", self.update_repo) synced_head = None settings = yield Query(self.database, "Settings").find_one() or dict() if settings["charts"]: self.url = settings["charts"]["repo_url"] while True: try: if self.url != self.repo.remotes.origin.url: logging.info("Changing reference from '%s' to '%s'.", self.repo.remotes.origin.url, self.url) self.repo.delete_remote('origin') self.repo.create_remote('origin', url=self.url) synced_head = None self.repo.git.fetch('--all') self.repo.git.reset('--hard', 'origin/master') if synced_head != self.repo.head.ref.commit: yield self.sync() synced_head = self.repo.head.ref.commit except: logging.exception("Failed to pull repository.") yield sleep(5) @coroutine def sync(self): logging.info("Syncing '%s'.", REPO_DIRECTORY) charts = yield Query(self.database, "Charts", manipulate=True).find() for chart in charts: path = chart["path"] self.charts[path] = chart discovered_charts = dict() for subdir, _, files in os.walk(REPO_DIRECTORY): for chart_file in files: if chart_file == "Chart.yaml": try: discovered_charts[subdir] = yield self.import_chart( subdir) except Exception: logging.exception("Failed to import chart at '%s'", subdir) for path, existing in self.charts.iteritems(): discovered = discovered_charts.get(path, None) if discovered is None: logging.debug("Deleting chart %(name)s", existing) yield Query(self.database, 'Charts').remove(existing) else: discovered["_id"] = existing["_id"] discovered["metadata"] = existing["metadata"] if discovered["commit"] != existing["commit"]: logging.debug("Updating existing chart %(name)s", discovered) yield Query(self.database, "Charts", manipulate=True).update(discovered) for path, discovered in discovered_charts.iteritems(): if discovered and "_id" not in discovered: logging.debug("Inserting new chart %(name)s", discovered) try: yield Query(self.database, "Charts", manipulate=True).insert(discovered) except: logging.error("Failed to insert chart %(name)s", discovered) self.charts = discovered_charts @coroutine def import_chart(self, directory): chart_path = os.path.join(directory, "Chart.yaml") with open(chart_path, "r") as stream: chart = load(stream) chart["path"] = directory commit = self.repo.iter_commits(paths=chart_path).next() chart["commit"] = binascii.hexlify(commit.binsha) chart["committed_date"] = commit.committed_date chart["resources"] = [] manifests = yield self.import_manifests(directory) for _, manifest in manifests.iteritems(): if commit.committed_date < manifest["commit"].committed_date: chart["commit"] = binascii.hexlify( manifest["commit"].binsha) chart["committed_date"] = manifest["commit"].committed_date for resource in manifest["resources"]: chart["resources"].append(resource) raise Return(chart) @coroutine def import_manifests(self, directory): manifests = dict() manifests_path = os.path.join(directory, "manifests", "*.yaml") for manifest in glob.glob(manifests_path): with open(manifest, "r") as stream: manifests[manifest] = dict( resources=[resource for resource in load_all(stream)], commit=self.repo.iter_commits(paths=manifest).next()) manifests_path = os.path.join(directory, "templates", "*.yaml") for manifest in glob.glob(manifests_path): manifest_filename = ntpath.basename(manifest) rendered_manifest = check_output([ "tide", "view", "-f", "templates/" + manifest_filename, directory ]) with io.TextIOWrapper(io.BytesIO(rendered_manifest)) as stream: manifests[manifest] = dict( resources=[resource for resource in load_all(stream)], commit=self.repo.iter_commits(paths=manifest).next()) raise Return(manifests)
class GitSync(object): def __init__(self, database): logging.info("Initializing GitSync.") self.database = database self.charts = dict() self.url = DEFAULT_GITREPO try: self.repo = Repo(REPO_DIRECTORY) except InvalidGitRepositoryError: logging.info("Cloning repository in %s", REPO_DIRECTORY) self.repo = Repo.clone_from(self.url, REPO_DIRECTORY) @coroutine def update_repo(self, document): if document['o']["charts"]["repo_url"] != self.url: logging.info("Repo url updated to '%s'.", document['o']["charts"]["repo_url"]) self.url = document['o']["charts"]["repo_url"] @coroutine def sync_loop(self): logging.info("Initializing sync loop.") yield add_callback("Settings", self.update_repo) synced_head = None settings = yield Query(self.database, "Settings").find_one() or dict() if settings["charts"]: self.url = settings["charts"]["repo_url"] while True: try: if self.url != self.repo.remotes.origin.url: logging.info("Changing reference from '%s' to '%s'.", self.repo.remotes.origin.url, self.url) self.repo.delete_remote('origin') self.repo.create_remote('origin', url=self.url) synced_head = None self.repo.git.fetch('--all') self.repo.git.reset('--hard', 'origin/master') if synced_head != self.repo.head.ref.commit: yield self.sync() synced_head = self.repo.head.ref.commit except: logging.exception("Failed to pull repository.") yield sleep(5) @coroutine def sync(self): logging.info("Syncing '%s'.", REPO_DIRECTORY) charts = yield Query(self.database, "Charts", manipulate=True).find() for chart in charts: path = chart["path"] self.charts[path] = chart discovered_charts = dict() for subdir, _, files in os.walk(REPO_DIRECTORY): for chart_file in files: if chart_file == "Chart.yaml": try: discovered_charts[subdir] = yield self.import_chart(subdir) except Exception: logging.exception("Failed to import chart at '%s'", subdir) for path, existing in self.charts.iteritems(): discovered = discovered_charts.get(path, None) if discovered is None: logging.debug("Deleting chart %(name)s", existing) yield Query(self.database, 'Charts').remove(existing) else: discovered["_id"] = existing["_id"] discovered["metadata"] = existing["metadata"] if discovered["commit"] != existing["commit"]: logging.debug("Updating existing chart %(name)s", discovered) yield Query(self.database, "Charts", manipulate=True).update(discovered) for path, discovered in discovered_charts.iteritems(): if discovered and "_id" not in discovered: logging.debug("Inserting new chart %(name)s", discovered) try: yield Query(self.database, "Charts", manipulate=True).insert(discovered) except: logging.error("Failed to insert chart %(name)s", discovered) self.charts = discovered_charts @coroutine def import_chart(self, directory): chart_path = os.path.join(directory, "Chart.yaml") with open(chart_path, "r") as stream: chart = load(stream) chart["path"] = directory commit = self.repo.iter_commits(paths=chart_path).next() chart["commit"] = binascii.hexlify(commit.binsha) chart["committed_date"] = commit.committed_date chart["resources"] = [] manifests = yield self.import_manifests(directory) for _, manifest in manifests.iteritems(): if commit.committed_date < manifest["commit"].committed_date: chart["commit"] = binascii.hexlify(manifest["commit"].binsha) chart["committed_date"] = manifest["commit"].committed_date for resource in manifest["resources"]: chart["resources"].append(resource) raise Return(chart) @coroutine def import_manifests(self, directory): manifests = dict() manifests_path = os.path.join(directory, "manifests", "*.yaml") for manifest in glob.glob(manifests_path): with open(manifest, "r") as stream: manifests[manifest] = dict( resources=[resource for resource in load_all(stream)], commit=self.repo.iter_commits(paths=manifest).next() ) manifests_path = os.path.join(directory, "templates", "*.yaml") for manifest in glob.glob(manifests_path): manifest_filename = ntpath.basename(manifest) rendered_manifest = check_output(["tide", "view", "-f", "templates/" + manifest_filename, directory]) with io.TextIOWrapper(io.BytesIO(rendered_manifest)) as stream: manifests[manifest] = dict( resources=[resource for resource in load_all(stream)], commit=self.repo.iter_commits(paths=manifest).next() ) raise Return(manifests)