def git_pull(remote_url, local, idc_dir):
"""
@remote: git remote address
@local: local git repo dir
"""
commit_msg = None
if not os.path.exists(local):
os.mkdir(local)
logging.info('git start pull from remote repo.')
if not os.path.exists(
"{0}/.git".format(local)) or not git.repo.fun.is_git_dir(
"{0}/.git".format(local)):
repo_clone = Repo.clone_from(remote_url, local)
if not repo_clone:
logging.debug("clone repo from {0} to {1}failed.\n".format(
remote_url, local))
return
repo = Repo(local)
remote = None
git_commit = git.Git(idc_dir)
last_commit_log = "/var/log/last_commit.log"
last_commit = None
if os.path.exists(last_commit_log):
with file(last_commit_log) as fhand:
last_log_commit = fhand.read().strip()
# Warning: for performance only check 500 commit, use this write mysql
for commit in repo.iter_commits('master', max_count=500):
if last_commit == commit.hexsha:
last_commit = last_log_commit
break
if not last_commit:
last_commit = repo.head.commit.hexsha
with FileLock(os.path.split(local)[1]) as _flock:
for remote_name in repo.remotes:
if remote_name.url == remote_url:
remote = remote_name
break
if remote is None:
remote_name = "%x" % (random.randint(0, 1000000)
^ int(time.time()))
remote = repo.create_remote(remote_name, remote_url)
ret = 0
try:
info_list = remote.pull()
for info in info_list:
ret = info.flags & (git.FetchInfo.REJECTED
| git.FetchInfo.ERROR)
if ret > 0:
logging.warning("[conf-pull] pull from remote error!\n")
commit_msg = git_commit.log("{0}..HEAD".format(last_commit),
"--pretty=%H,%ai,%s", idc_dir)
except (git.GitCommandError, git.RepositoryDirtyError), e:
logging.warning("git pull error, error={0}\n".format(str(e)))
except Exception:
logging.warning("git pull error, other exception.\n")
class GitSync(object):
def __init__(self, database):
logging.info("Initializing GitSync.")
self.database = database
self.charts = dict()
self.url = DEFAULT_GITREPO
try:
self.repo = Repo(REPO_DIRECTORY)
except InvalidGitRepositoryError:
logging.info("Cloning repository in %s", REPO_DIRECTORY)
self.repo = Repo.clone_from(self.url, REPO_DIRECTORY)
@coroutine
def update_repo(self, document):
if document['o']["charts"]["repo_url"] != self.url:
logging.info("Repo url updated to '%s'.",
document['o']["charts"]["repo_url"])
self.url = document['o']["charts"]["repo_url"]
@coroutine
def sync_loop(self):
logging.info("Initializing sync loop.")
yield add_callback("Settings", self.update_repo)
synced_head = None
settings = yield Query(self.database, "Settings").find_one() or dict()
if settings["charts"]:
self.url = settings["charts"]["repo_url"]
while True:
try:
if self.url != self.repo.remotes.origin.url:
logging.info("Changing reference from '%s' to '%s'.",
self.repo.remotes.origin.url, self.url)
self.repo.delete_remote('origin')
self.repo.create_remote('origin', url=self.url)
synced_head = None
self.repo.git.fetch('--all')
self.repo.git.reset('--hard', 'origin/master')
if synced_head != self.repo.head.ref.commit:
yield self.sync()
synced_head = self.repo.head.ref.commit
except:
logging.exception("Failed to pull repository.")
yield sleep(5)
@coroutine
def sync(self):
logging.info("Syncing '%s'.", REPO_DIRECTORY)
charts = yield Query(self.database, "Charts", manipulate=True).find()
for chart in charts:
path = chart["path"]
self.charts[path] = chart
discovered_charts = dict()
for subdir, _, files in os.walk(REPO_DIRECTORY):
for chart_file in files:
if chart_file == "Chart.yaml":
try:
discovered_charts[subdir] = yield self.import_chart(
subdir)
except Exception:
logging.exception("Failed to import chart at '%s'",
subdir)
for path, existing in self.charts.iteritems():
discovered = discovered_charts.get(path, None)
if discovered is None:
logging.debug("Deleting chart %(name)s", existing)
yield Query(self.database, 'Charts').remove(existing)
else:
discovered["_id"] = existing["_id"]
discovered["metadata"] = existing["metadata"]
if discovered["commit"] != existing["commit"]:
logging.debug("Updating existing chart %(name)s",
discovered)
yield Query(self.database, "Charts",
manipulate=True).update(discovered)
for path, discovered in discovered_charts.iteritems():
if discovered and "_id" not in discovered:
logging.debug("Inserting new chart %(name)s", discovered)
try:
yield Query(self.database, "Charts",
manipulate=True).insert(discovered)
except:
logging.error("Failed to insert chart %(name)s",
discovered)
self.charts = discovered_charts
@coroutine
def import_chart(self, directory):
chart_path = os.path.join(directory, "Chart.yaml")
with open(chart_path, "r") as stream:
chart = load(stream)
chart["path"] = directory
commit = self.repo.iter_commits(paths=chart_path).next()
chart["commit"] = binascii.hexlify(commit.binsha)
chart["committed_date"] = commit.committed_date
chart["resources"] = []
manifests = yield self.import_manifests(directory)
for _, manifest in manifests.iteritems():
if commit.committed_date < manifest["commit"].committed_date:
chart["commit"] = binascii.hexlify(
manifest["commit"].binsha)
chart["committed_date"] = manifest["commit"].committed_date
for resource in manifest["resources"]:
chart["resources"].append(resource)
raise Return(chart)
@coroutine
def import_manifests(self, directory):
manifests = dict()
manifests_path = os.path.join(directory, "manifests", "*.yaml")
for manifest in glob.glob(manifests_path):
with open(manifest, "r") as stream:
manifests[manifest] = dict(
resources=[resource for resource in load_all(stream)],
commit=self.repo.iter_commits(paths=manifest).next())
manifests_path = os.path.join(directory, "templates", "*.yaml")
for manifest in glob.glob(manifests_path):
manifest_filename = ntpath.basename(manifest)
rendered_manifest = check_output([
"tide", "view", "-f", "templates/" + manifest_filename,
directory
])
with io.TextIOWrapper(io.BytesIO(rendered_manifest)) as stream:
manifests[manifest] = dict(
resources=[resource for resource in load_all(stream)],
commit=self.repo.iter_commits(paths=manifest).next())
raise Return(manifests)
class GitSync(object):
def __init__(self, database):
logging.info("Initializing GitSync.")
self.database = database
self.charts = dict()
self.url = DEFAULT_GITREPO
try:
self.repo = Repo(REPO_DIRECTORY)
except InvalidGitRepositoryError:
logging.info("Cloning repository in %s", REPO_DIRECTORY)
self.repo = Repo.clone_from(self.url, REPO_DIRECTORY)
@coroutine
def update_repo(self, document):
if document['o']["charts"]["repo_url"] != self.url:
logging.info("Repo url updated to '%s'.", document['o']["charts"]["repo_url"])
self.url = document['o']["charts"]["repo_url"]
@coroutine
def sync_loop(self):
logging.info("Initializing sync loop.")
yield add_callback("Settings", self.update_repo)
synced_head = None
settings = yield Query(self.database, "Settings").find_one() or dict()
if settings["charts"]:
self.url = settings["charts"]["repo_url"]
while True:
try:
if self.url != self.repo.remotes.origin.url:
logging.info("Changing reference from '%s' to '%s'.", self.repo.remotes.origin.url, self.url)
self.repo.delete_remote('origin')
self.repo.create_remote('origin', url=self.url)
synced_head = None
self.repo.git.fetch('--all')
self.repo.git.reset('--hard', 'origin/master')
if synced_head != self.repo.head.ref.commit:
yield self.sync()
synced_head = self.repo.head.ref.commit
except:
logging.exception("Failed to pull repository.")
yield sleep(5)
@coroutine
def sync(self):
logging.info("Syncing '%s'.", REPO_DIRECTORY)
charts = yield Query(self.database, "Charts", manipulate=True).find()
for chart in charts:
path = chart["path"]
self.charts[path] = chart
discovered_charts = dict()
for subdir, _, files in os.walk(REPO_DIRECTORY):
for chart_file in files:
if chart_file == "Chart.yaml":
try:
discovered_charts[subdir] = yield self.import_chart(subdir)
except Exception:
logging.exception("Failed to import chart at '%s'", subdir)
for path, existing in self.charts.iteritems():
discovered = discovered_charts.get(path, None)
if discovered is None:
logging.debug("Deleting chart %(name)s", existing)
yield Query(self.database, 'Charts').remove(existing)
else:
discovered["_id"] = existing["_id"]
discovered["metadata"] = existing["metadata"]
if discovered["commit"] != existing["commit"]:
logging.debug("Updating existing chart %(name)s", discovered)
yield Query(self.database, "Charts", manipulate=True).update(discovered)
for path, discovered in discovered_charts.iteritems():
if discovered and "_id" not in discovered:
logging.debug("Inserting new chart %(name)s", discovered)
try:
yield Query(self.database, "Charts", manipulate=True).insert(discovered)
except:
logging.error("Failed to insert chart %(name)s", discovered)
self.charts = discovered_charts
@coroutine
def import_chart(self, directory):
chart_path = os.path.join(directory, "Chart.yaml")
with open(chart_path, "r") as stream:
chart = load(stream)
chart["path"] = directory
commit = self.repo.iter_commits(paths=chart_path).next()
chart["commit"] = binascii.hexlify(commit.binsha)
chart["committed_date"] = commit.committed_date
chart["resources"] = []
manifests = yield self.import_manifests(directory)
for _, manifest in manifests.iteritems():
if commit.committed_date < manifest["commit"].committed_date:
chart["commit"] = binascii.hexlify(manifest["commit"].binsha)
chart["committed_date"] = manifest["commit"].committed_date
for resource in manifest["resources"]:
chart["resources"].append(resource)
raise Return(chart)
@coroutine
def import_manifests(self, directory):
manifests = dict()
manifests_path = os.path.join(directory, "manifests", "*.yaml")
for manifest in glob.glob(manifests_path):
with open(manifest, "r") as stream:
manifests[manifest] = dict(
resources=[resource for resource in load_all(stream)],
commit=self.repo.iter_commits(paths=manifest).next()
)
manifests_path = os.path.join(directory, "templates", "*.yaml")
for manifest in glob.glob(manifests_path):
manifest_filename = ntpath.basename(manifest)
rendered_manifest = check_output(["tide", "view", "-f", "templates/" + manifest_filename, directory])
with io.TextIOWrapper(io.BytesIO(rendered_manifest)) as stream:
manifests[manifest] = dict(
resources=[resource for resource in load_all(stream)],
commit=self.repo.iter_commits(paths=manifest).next()
)
raise Return(manifests)
