def cli_copy(
image: str,
local_git_repo: str = './docker-mirror',
git_bin: str = 'git',
commit: bool = True,
push: bool = True,
debug: bool = False,
):
from git import Git
code = f'FROM {image}'
git_sub_path = image.replace(':', '/')
git_full_path = os.path.join(git_sub_path, "Dockerfile")
dest = os.path.join(local_git_repo, git_full_path)
if debug:
print(f"writting {code} into {dest}")
write_file(dest, code)
git = Git(
git_bin=git_bin,
cwd=local_git_repo,
)
git.add(git_full_path)
if commit:
git.commit(f"[Add] {image}", False)
if push:
git.push()
def commit_and_push(path: Path):
"""
Commits all files in the path directory and pushes to origin/master
:param path: The path to the directory
"""
repo = Git(path.absolute())
today = get_today()
repo.add(".")
repo.commit("-m", f"Reduction files for {today}")
repo.push("--set-upstream", "origin", "master")
def run(self):
git_dir = os.path.join(self.docs_dir, ".git")
if os.path.exists(git_dir):
import shutil
shutil.rmtree(git_dir)
from git import Git
g = Git(self.docs_dir)
g.init()
g.add(".")
g.commit("-m {}".format(self.commit_msg))
g.push("--force", self.repo, "master:gh-pages")
class TempClone:
"""Create a clone in a temp dir used to write and push file changes"""
def __init__(self, url: str):
import posixpath
self.path = join(gettempdir(), posixpath.basename(url))
if not isdir(self.path):
Repo.clone_from(url, self.path)
self.git = Git(self.path)
def write(self, path: str, content: str):
with open(join(self.path, path), 'w') as f:
f.write(content)
self.git.add(path)
self.git.commit(amend=True, no_edit=True)
self.git.push(force=True)
def setUp(self):
self.test_dir = tempfile.mkdtemp()
g = Git(self.test_dir)
g.init()
open('%s/testfile.txt' % (self.test_dir, ),
'w').write('This is a test')
g.add('testfile.txt')
g.commit('-m', 'First commit')
os.mkdir('%s/testdir' % (self.test_dir, ))
open('%s/testdir/file2.txt' % (self.test_dir, ),
'w').write('This is another test')
g.add('testdir/file2.txt')
g.commit('-m', 'Second commit')
self.vcs = VcsWrapper.vcs_for_path(self.test_dir)
def addSuperModuleCommit(self, id, hash, url, who, branch, project):
self.log.debug("branch: " + branch + ", project:" + project)
hasSuperModule = False
isSuperModuleBr = False
self.log.debug("Project names: " + str(self.config.projects))
projectNames = self.config.projects.keys()
for proj in projectNames:
self.log.debug("project: " + project + " proj: " + proj)
if project.lower() == proj:
hasSuperModule = True
break
for br in self.config.branches:
if branch == br:
isSuperModuleBr = True
break
self.log.debug("isSuperModuleBr: " + str(isSuperModuleBr) + " hasSuperModule: " + str(hasSuperModule))
if isSuperModuleBr and hasSuperModule:
self.log.debug("Git Profile Path: " + str(self.config.profile))
git = Git(self.config.profile)
self.checkoutTrackingBranch(git, branch)
git.pull()
git.submodule("update","--init")
gitSubmoduleProfile = {'git':self.config.superRepoPath + self.config.projects[project.lower()]}
gitSubmodule = Git(gitSubmoduleProfile)
self.log.debug("checking out hash: " + hash)
gitSubmodule.fetch()
if self.isOptOut(gitSubmodule, hash):
return
gitSubmodule.checkout(hash, True)
git.add(".")
commitMsg = "Auto checkin: " + self.getCommitMessage(gitSubmodule, hash) + "\nuser:"******"\nhash:" + hash + "\nproject: " + project
self.log.debug("commiting in super module: " + commitMsg)
git.commit(commitMsg)
self.log.debug("pushing super module to branch: " + branch)
git.push(branch)
else:
self.log.debug("No super module commit is required.")
class TempClone:
"""Create a clone in a temp dir used to write and push file changes"""
def __init__(self, url: str, branch: str=None):
import posixpath
self.path = join(gettempdir(), posixpath.basename(url))
if not isdir(self.path):
Repo.clone_from(url, self.path)
self.git = Git(self.path)
self.git.fetch()
if branch:
self.git.checkout(branch)
def write(self, path: str, content: str):
with open(join(self.path, path), 'w') as f:
f.write(content)
self.git.add(path)
self.git.commit(message="Automatic update of skill-metadata.json")
self.git.push()
def delete(self):
shutil.rmtree(self.path)
def created_quark(self, sender, new_quark):
options = sender.options
if self.git_is_installed():
g = Git(options['path'])
g.init()
default_ignore = os.path.join(os.path.dirname(__file__), '..', 'default_gitignore.txt')
ignore_path = os.path.join(options['path'], '.gitignore')
with open(ignore_path, 'w') as ignore_file, open(default_ignore, 'r') as default_file:
ignore_file.write(default_file.read())
user_ignores = settings.user_gitignore()
if user_ignores:
ignore_file.write('\n#SUPPLIED FROM ~/.qpmignore\n\n')
ignore_file.write(user_ignores)
g.add(new_quark.get_quarkfile_path(options['path']))
g.commit(m='Created quarkfile.')
g.add('.gitignore')
g.commit(m='Created gitignore.')
g.add('.')
if Repo(options['path']).is_dirty():
g.commit(m='Initial commit of existing files.')
g.tag(options['version'])
else:
self.msg(
'WARNING: Git is not installed. Try http://git-scm.com/ or run the following:\n'
+ '\truby -e "$(curl -fsSL https://raw.github.com/Homebrew/homebrew/go/install)"; brew install git'
)
if not os.path.isdir(dirpath):
os.mkdir(dirpath)
g = Git(dirpath)
# If is not Git repository, git init
if not git.repo.fun.is_git_dir(dirpath):
# bare repository
if args.__dict__.get('b'):
repo = g.init(dirpath, bare=True)
else:
# local repository
g.init()
# Make .gitignore
if not os.path.isfile(dirpath + '/.gitignore'):
f = open(dirpath + '/.gitignore', 'w')
f.write('')
f.close()
# git add .gitignore and first commit
if g.untracked_files or g.is_dirty():
# git add
g.add('.gitignore')
# git commit
g.commit(m='First commit')
except RuntimeError as e:
sys.stderr.write("ERROR: %s\n" % e)
class Mounts:
"""
The Mounts class is the class that does all the ETL from git to updating the file
** This is the magic sauce behind the api
"""
def __init__(self, app):
"""
Initilize mounts class by loading yaml into a variable
:input app - flask app
"""
self.app = app
self.git = Git(environ['GIT_DIRECTORY'])
self.git.pull()
self.nfs_file = join(environ['GIT_DIRECTORY'], 'data/common.yaml')
with open(self.nfs_file) as yaml_file:
self.nfs_info = yaml.safe_load(yaml_file.read())
if not self.nfs_info:
raise Exception('error loading mounts file')
def commit(self, hostname, option):
"""
Commits the codes and pushes back to git
:input hostname - name changed
:input option - option changed
"""
self.app.logger.info(f"successfully {option} for {hostname}")
self.update_current()
try:
self.git.add(self.nfs_file)
self.git.commit('-m', f"'updated - {option} for {hostname}'")
self.git.push()
except Exception as exc:
if 'nothing to commit' in str(exc):
raise Exception('No changes made')
else:
raise exc
def check_exists(self, host_type, name, mount) -> bool:
"""
Check if the mount exists
:input host_type - hosts/hostgroups
:input name - name of the host or hostgroup
:input mount - the mount option
:return boolean if exists
"""
try:
for temp_mount in self.nfs_info[f"nfs_mounts::{host_type}"][name]:
unmatched_item = set(temp_mount.items()) ^ set(mount.items())
if unmatched_item and dict(unmatched_item) and 'uuid' in dict(
unmatched_item).keys():
return True
except Exception as exc:
self.app.logger.warning(exc)
return False
def update_current(self):
"""
Writes to file
"""
with open(self.nfs_file, 'w') as yaml_file:
to_write = yaml.dump(self.nfs_info, default_flow_style=False)
yaml_file.write(to_write)
def add_nas_share(self,
local_path,
share_path,
options,
owner,
group,
host=None,
hostgroup=None):
"""
Add a new NAS Share to a host/hostgroup
"""
if not host and not hostgroup:
raise Exception('Missing host and hostgroup')
else:
host_type = 'hosts' if host else 'hostgroups'
name = host if host else hostgroup
if not self.nfs_info[f"nfs_mounts::{host_type}"]:
self.nfs_info[f"nfs_mounts::{host_type}"] = {}
if name.lower(
) not in self.nfs_info[f"nfs_mounts::{host_type}"].keys():
self.app.logger.info(f"{name}: has no mounts...appending")
self.nfs_info[f"nfs_mounts::{host_type}"].update({name: []})
self.app.logger.info(
self.nfs_info[f"nfs_mounts::{host_type}"].keys())
mount = {
'uuid': str(uuid.uuid4()),
'local_path': local_path,
'share_path': share_path,
'options': options,
'owner': owner,
'group': group
}
if not self.check_exists(host_type, name, mount):
self.app.logger.info(f"{name}: adding {mount}")
self.nfs_info[f"nfs_mounts::{host_type}"][name].append(mount)
self.commit(name, 'add')
return self.nfs_info[f"nfs_mounts::{host_type}"][name]
else:
raise ExistsException('mount point already exists')
def update_nas_share(self,
uuid_num,
replacement_dict,
host=None,
hostgroup=None):
"""
Modify an existing new NAS Share to a host/hostgroup
"""
if not host and not hostgroup:
raise Exception('Missing host and hostgroup')
else:
host_type = 'hosts' if host else 'hostgroups'
name = host if host else hostgroup
changed = False
for idx, val in enumerate(
self.nfs_info[f"nfs_mounts::{host_type}"][name]):
if uuid_num == val['uuid']:
self.nfs_info[f"nfs_mounts::{host_type}"][name][
idx].update(replacement_dict)
self.app.logger.info(f"{name}: updating {uuid_num}")
changed = True
if not changed:
raise IndexError('no index matching that uuid found')
self.commit(name, 'added')
return self.nfs_info[f"nfs_mounts::{host_type}"][name]
def delete_host_name(self, name, host_type):
"""
Remove a host/hostgroup
"""
del self.nfs_info[f"nfs_mounts::{host_type}"][name]
if host_type == 'hostgroups':
self.commit(name, 'deleted hostgroup')
else:
self.commit(name, 'deleted host')
def delete_host_mount(self, name, host_type, uuid_num):
"""
Remove an existing new NAS mount from a host/hostgroup
"""
self.nfs_info[f"nfs_mounts::{host_type}"][name] = [
x for x in self.nfs_info[f"nfs_mounts::{host_type}"][name]
if x['uuid'] != uuid_num
]
self.commit(name, 'deleted mount')
return self.nfs_info[f"nfs_mounts::{host_type}"][name]
