def init_repository(url=None):
"""Creates a new Gitless's repository in the cwd.
Args:
url: if given the local repository will be a clone of the remote repository
given by this url.
"""
cwd = os.getcwd()
try:
pygit2.discover_repository(cwd)
raise GlError('You are already in a Gitless repository')
except KeyError: # Expected
if not url:
repo = pygit2.init_repository(cwd)
# We also create an initial root commit
git.commit(allow_empty=True, m='Initialize repository')
return repo
try:
git.clone(url, cwd)
except ErrorReturnCode as e:
raise GlError(stderr(e))
# We get all remote branches as well and create local equivalents
repo = Repository()
remote = repo.remotes['origin']
for rb in (remote.lookup_branch(bn)
for bn in remote.listall_branches()):
if rb.branch_name == 'master':
continue
new_b = repo.create_branch(rb.branch_name, rb.head)
new_b.upstream = rb
return repo
def import_course(course, new_courseid, username, course_factory):
if not id_checker(new_courseid):
raise ImportCourseException("Course with invalid name: " + new_courseid)
course_fs = course_factory.get_course_fs(new_courseid)
if course_fs.exists("course.yaml") or course_fs.exists("course.json"):
raise ImportCourseException("Course with id " + new_courseid + " already exists.")
try:
git.clone(course.get_link(), course_fs.prefix)
except:
raise ImportCourseException(_("Couldn't clone course into your instance"))
try:
old_descriptor = course_factory.get_course_descriptor_content(new_courseid)
except:
old_descriptor ={}
try:
new_descriptor = {"description": old_descriptor.get("description", ""),
'admins': [username],
"accessible": False,
"tags": old_descriptor.get("tags", {})}
if "name" in old_descriptor:
new_descriptor["name"] = old_descriptor["name"] + " - " + new_courseid
else:
new_descriptor["name"] = new_courseid
if "toc" in old_descriptor:
new_descriptor["toc"] = old_descriptor["toc"]
course_factory.update_course_descriptor_content(new_courseid, new_descriptor)
except:
course_factory.delete_course(new_courseid)
raise ImportCourseException(_("An error occur while editing the course description"))
get_course_logger(new_courseid).info("Course %s cloned from the marketplace.", new_courseid)
def _create_git(self):
"""create a book from a git repo"""
if not os.path.exists(self.path):
LOG.info(("Creating book {0} from {1}, branch: {2}" +
"").format(self.path, self.git, self.branch))
git.clone(self.git, self.path)
else:
LOG.info("Book {0} already exists".format(self.path))
cwd = os.getcwd()
os.chdir(self.path)
if self.skiprepourlcheck:
remote_match_found = False
for remote in git("remote", "-v"):
remote_parts = remote.split()
if Url(remote_parts[1]) == Url(self.git):
remote_match_found = True
if remote_match_found:
LOG.debug('Found {0} in the list of remotes for {1}'.format(self.git, self.path))
else:
LOG.error('ERROR: {0} wasn\'t found in the list of remotes for {1}'.format(self.git, self.path))
if not self._check_branch():
LOG.info("Switching {0} to branch {1}".format(self.path,
self.branch))
git.fetch
git.checkout(self.branch)
os.chdir(cwd)
def clone_and_check_repo(user_name, repo_name, repo_clone_url, s3):
has_manifest = False
try:
os.makedirs('./tmp')
os.chdir('./tmp')
git.clone(repo_clone_url)
for fname in os.listdir(f'./{repo_name}'):
if fname == 'manifest.json' or fname == 'manifest.yaml':
has_manifest = True
manifest_suffix = ""
json_file_builder.get_has_manifest(repo_name=repo_name,
suffix=manifest_suffix,
is_valid=has_manifest)
upload_file(user_name, repo_name, manifest_suffix, s3)
os.chdir('../')
try:
shutil.rmtree('./tmp')
except OSError as e: ## if failed, report it back to the user ##
print("Error: %s - %s." % (e.filename, e.strerror))
except:
e = sys.exc_info()[0]
print("<p> Error: %s </p>" % e)
os.chdir('/webhooks')
shutil.rmtree('./tmp')
return str(has_manifest), 200
def _create_git(self):
"""create a book from a git repo"""
if not os.path.exists(self.path):
LOG.info(("Creating book {0} from {1}, branch: {2}" +
"").format(self.path, self.git, self.branch))
git.clone(self.git, self.path)
else:
LOG.info("Book {0} already exists".format(self.path))
cwd = os.getcwd()
os.chdir(self.path)
if self.skiprepourlcheck:
remote_match_found = False
for remote in git("remote", "-v"):
remote_parts = remote.split()
if Url(remote_parts[1]) == Url(self.git):
remote_match_found = True
if remote_match_found:
LOG.debug('Found {0} in the list of remotes for {1}'.format(self.git, self.path))
else:
LOG.error('ERROR: {0} wasn\'t found in the list of remotes for {1}'.format(self.git, self.path))
if not self._check_branch():
LOG.info("Switching {0} to branch {1}".format(self.path,
self.branch))
git.fetch
git.checkout(self.branch)
os.chdir(cwd)
def init():
if os.path.exists("repo"):
if not os.path.isdir("repo/.git"):
sys.stderr.write("repo/ exists, but is not a git repo")
sys.exit(1)
else:
git.clone(GIT_REMOTE, "repo")
def prepare_repo(repos_dir, repo_name, repo_url):
# Confirm that a git repo exists at the given local directory. If it does
# not exist, then attempt to clone it. If it already exists, then
# fetch new branches and prune old branches.
log.debug("Preparing repo " + repo_name)
repo_path = os.path.join(repos_dir, repo_name)
dot_git_path = os.path.join(repo_path, '.git')
clone_the_repo = False
if not os.path.isdir(repo_path):
log.warning(f"Directory {repo_path} not found")
clone_the_repo = True
elif not os.path.isdir(dot_git_path):
# If we find a directory in our mounted volume that is not a git
# repo, then something is seriously wrong. Delete the directory and
# start again with a clean repo.
log.warning(f"Deleting invalid git repo found at {repo_path}")
shutil.rmtree(repo_path)
clone_the_repo = True
else:
log.debug(f"Directory {repo_path} is apparently a git repo")
if clone_the_repo:
log.info(f"Cloning {repo_name} ...")
os.chdir(repos_dir)
git.clone(repo_url)
else:
# The git repo exists, so bring it up to date
log.info(f"Fetching changes to {repo_name}")
os.chdir(repo_path)
git.fetch('--all')
return repo_path
def init_repository(url=None):
"""Creates a new Gitless's repository in the cwd.
Args:
url: if given the local repository will be a clone of the remote repository
given by this url.
"""
cwd = os.getcwd()
try:
pygit2.discover_repository(cwd)
raise GlError('You are already in a Gitless repository')
except KeyError: # Expected
if not url:
repo = pygit2.init_repository(cwd)
# We also create an initial root commit
git.commit(allow_empty=True, m='Initialize repository')
return repo
try:
git.clone(url, cwd)
except ErrorReturnCode as e:
raise GlError(stderr(e))
# We get all remote branches as well and create local equivalents
repo = Repository()
remote = repo.remotes['origin']
for rb in (remote.lookup_branch(bn) for bn in remote.listall_branches()):
if rb.branch_name == 'master':
continue
new_b = repo.create_branch(rb.branch_name, rb.head)
new_b.upstream = rb
return repo
def clone(path, git_address, git_reference, host):
new_env = resetEnv(host)
logger.debug('Cloning {} with git_reference {} at {}'
.format(git_address, git_reference, path))
old_path = os.getcwd()
try:
os.makedirs(path)
os.chdir(path)
git_reference = git_reference.replace('tags/', '')
git.clone(
git_address,
'.',
'--recursive',
'--branch',
git_reference,
'--depth',
1,
_env=new_env
)
logger.debug('{} {} {} {} {} {} {}'.format(
git_address,
'.',
'--recursive',
'--branch',
git_reference,
'--depth',
1
))
except:
logger.error('Failed to clone project at {}'.format(path),
exc_info=True)
os.chdir(old_path)
def clone(path, git_address, git_reference, host):
new_env = resetEnv(host)
logger.debug('Cloning {} with git_reference {} at {}'.format(
git_address, git_reference, path))
old_path = os.getcwd()
try:
os.makedirs(path)
os.chdir(path)
git_reference = git_reference.replace('tags/', '')
git.clone(git_address,
'.',
'--recursive',
'--branch',
git_reference,
'--depth',
1,
_env=new_env)
logger.debug('{} {} {} {} {} {} {}'.format(git_address, '.',
'--recursive', '--branch',
git_reference, '--depth',
1))
except:
logger.error('Failed to clone project at {}'.format(path),
exc_info=True)
os.chdir(old_path)
def generate_csv(output_directory='./'):
# Set up root data dir
root_filename = ROOT + '/cfb-data/automated/wiki/' + ROOT + '/'
# Clone data repository for updates - continue if already exists
from sh import git
try:
shutil.rmtree(ROOT + '/cfb-data/')
except:
pass
git.clone(
'https://' + os.getenv('MACHINE_AUTH') +
'@github.com/coffenbacher/cfb-data.git', ROOT + '/cfb-data/')
# Create our root if required
if not os.path.exists(root_filename):
os.makedirs(root_filename)
# Set up logging
logging.basicConfig(
level='WARNING',
#format='%(asctime)s %(levelname)-8s %(message)s',
#datefmt='%a, %d %b %Y %H:%M:%S',
filename=root_filename + ROOT + '.log',
filemode='w')
# Extract all current names from Wiki
data = FN()
# Write everything to csv
with open(root_filename + ROOT + '.csv', 'w') as csvfile:
writer = csv.DictWriter(csvfile,
fieldnames=FIELDNAMES,
extrasaction='ignore')
for d in data:
asci = dict([(unidecode(k), unidecode(unicode(v)))
for k, v in d.items()])
writer.writerow(asci)
# [unidecode(unicode(d.get(field))) for field in FIELDNAMES]
# Write everything to json
with open(root_filename + ROOT + '.json', 'w') as jsonfile:
relevant = [{f: d.get(f, None) for f in FIELDNAMES} for d in data]
jsonfile.write(json.dumps(relevant))
with open(root_filename + ROOT + '_meta.json', 'w') as metafile:
d = {
'created': datetime.now().strftime('%x %X'),
'rows': len(data),
'headers': ','.join(FIELDNAMES),
}
metafile.write(json.dumps(d))
git = git.bake(**{
'git-dir': ROOT + '/cfb-data/.git/',
'work-tree': ROOT + '/cfb-data'
})
git.commit(m='Auto updating %s data' % ROOT, a=True)
git.push('origin', 'master')
def clone(self):
import os
from ambry.dbexceptions import ConflictError
if not self.remote_url:
raise RepositoryException("Can't clone without setting remote_url in constructor")
git.clone(self.remote_url, self.dir_)
def clone(self,url, dir_):
import os
from databundles.dbexceptions import ConflictError
if not os.path.exists(dir_):
git.clone(url,dir_)
else:
raise ConflictError("{} already exists".format(dir_))
def update(self):
if not os.path.isdir(CACHE_DIR):
os.makedirs(CACHE_DIR)
if os.path.isdir(self.dir):
self.git.pull()
else:
git.clone(addon['metadata']['source'], self.dir)
def get_models(repository):
print("Getting list of benchmarks from {}".format(repository))
tmpdir = tempfile.mkdtemp()
git.clone(repository, tmpdir)
with open(join(tmpdir, "benchmarks.json")) as fp:
models = json.load(fp)
shutil.rmtree(tmpdir)
return models
def update(self):
if not os.path.isdir(CACHE_DIR):
os.makedirs(CACHE_DIR)
if os.path.isdir(self.dir):
self.git.pull()
else:
git.clone(addon['metadata']['source'], self.dir)
def _git(self, dep, dep_dict, verbose):
"""Git clone from list."""
from sh import git # isort:skip
for repo in dep_dict["git_list"]:
if verbose:
print(f" cloning {dep} repo {repo}")
git.clone(repo)
def init(self, username, reponame, force, backend=None):
"""
Initialize a Git repo
Parameters
----------
username, reponame : Repo name is tuple (name, reponame)
force: force initialization of the repo even if exists
backend: backend that must be used for this (e.g. s3)
"""
key = self.key(username, reponame)
# In local filesystem-based server, add a repo
server_repodir = self.server_rootdir(username,
reponame,
create=False)
# Force cleanup if needed
if os.path.exists(server_repodir) and not force:
raise RepositoryExists()
if os.path.exists(server_repodir):
shutil.rmtree(server_repodir)
os.makedirs(server_repodir)
# Initialize the repo
with cd(server_repodir):
git.init(".", "--bare")
if backend is not None:
backend.init_repo(server_repodir)
# Now clone the filesystem-based repo
repodir = self.rootdir(username, reponame, create=False)
# Prepare it if needed
if os.path.exists(repodir) and not force:
raise Exception("Local repo already exists")
if os.path.exists(repodir):
shutil.rmtree(repodir)
os.makedirs(repodir)
# Now clone...
with cd(os.path.dirname(repodir)):
git.clone(server_repodir, '--no-hardlinks')
url = server_repodir
if backend is not None:
url = backend.url(username, reponame)
repo = Repo(username, reponame)
repo.manager = self
repo.remoteurl = url
repo.rootdir = self.rootdir(username, reponame)
self.add(repo)
return repo
def init(self, username, reponame, force, backend=None):
"""
Initialize a Git repo
Parameters
----------
username, reponame : Repo name is tuple (name, reponame)
force: force initialization of the repo even if exists
backend: backend that must be used for this (e.g. s3)
"""
key = self.key(username, reponame)
# In local filesystem-based server, add a repo
server_repodir = self.server_rootdir(username,
reponame,
create=False)
# Force cleanup if needed
if os.path.exists(server_repodir) and not force:
raise RepositoryExists()
if os.path.exists(server_repodir):
shutil.rmtree(server_repodir)
os.makedirs(server_repodir)
# Initialize the repo
with cd(server_repodir):
git.init(".", "--bare")
if backend is not None:
backend.init_repo(server_repodir)
# Now clone the filesystem-based repo
repodir = self.rootdir(username, reponame, create=False)
# Prepare it if needed
if os.path.exists(repodir) and not force:
raise Exception("Local repo already exists")
if os.path.exists(repodir):
shutil.rmtree(repodir)
os.makedirs(repodir)
# Now clone...
with cd(os.path.dirname(repodir)):
git.clone(server_repodir, '--no-hardlinks')
url = server_repodir
if backend is not None:
url = backend.url(username, reponame)
repo = Repo(username, reponame)
repo.manager = self
repo.remoteurl = url
repo.rootdir = self.rootdir(username, reponame)
self.add(repo)
return repo
def __init__(self, settings):
self.settings = settings
# Into the context
if not os.path.exists(self.settings.local_dir):
git.clone(self.settings.repo_url, self.settings.local_dir)
cd(settings.local_dir)
# Make sure checkout the right branch
git.checkout(settings.branch)
self.__current_commit = ""
def install_vundle(self):
vundle_dest = Path(path.expanduser("~/.vim/bundle/Vundle.vim"))
if not vundle_dest.exists():
git.clone("https://github.com/VundleVim/Vundle.vim.git", str(vundle_dest), _out=print)
return "Vundle has been downloaded successfully"
else:
git("-C", str(vundle_dest), "pull")
return "Vundle was already installed, it has been updated"
def sync(event, context):
event = Event(event)
sourcerepo = SourceRepo(event)
with tempfile.TemporaryDirectory() as tmpdir:
git.clone('--mirror', event.clone_url, _cwd=tmpdir)
git.push('--mirror', sourcerepo.url, _cwd=tmpdir)
return
def __init__(self, settings):
self.settings = settings
# Into the context
if not os.path.exists(self.settings.local_dir):
git.clone(self.settings.repo_url, self.settings.local_dir)
cd(settings.local_dir)
# Make sure checkout the right branch
git.checkout(settings.branch)
self.__current_commit = ""
def grab(data, name):
if 'type' not in data:
raise KeyError('type not defined in %s in data.yaml' % name)
if 'location' not in data:
raise KeyError('location not defined in %s in data.yaml' % name)
dest = '.components/%s' % name
print('Grabbing %s from %s' % (name, data['location']))
if data['type'] == 'git':
git.clone(data['location'], dest)
def __init__(self, repo):
if path.exists(self.repo_path):
os.chdir(self.repo_path)
git.pull('-f', '-u', 'origin', 'master')
else:
git.clone(repo, self.REPO_PATH)
os.chdir(self.repo_path)
git.submodule('update', '--init', self.ROLES)
self._metadata = dict()
def clone(self,url, dir_=None):
import os
from ambry.dbexceptions import ConflictError
dir_ = self.dir_ if not dir_ else dir_
if not os.path.exists(dir_):
git.clone(url,dir_)
else:
raise ConflictError("{} already exists".format(dir_))
def git_clone(git_repo_url, git_repo_dir):
try:
logger.info('Fetching from %s to %s', git_repo_url, git_repo_dir)
git.clone(git_repo_url,
git_repo_dir,
_out=sys.stdout,
_err=sys.stderr,
_fg=True)
except ErrorReturnCode as e:
raise CommandError(e)
def clone(self, origin, address):
'''
Clone a component repository
'''
from_path = self.path(origin)
to_path = self.path(address)
if os.path.exists(to_path):
raise Exception('Component repository already exists at %s' % to_path)
git.clone(from_path, to_path)
self.setup(to_path)
def clone(self, url, dir_=None):
import os
from ambry.dbexceptions import ConflictError
dir_ = self.dir_ if not dir_ else dir_
if not os.path.exists(dir_):
git.clone(url, dir_)
else:
raise ConflictError("{} already exists".format(dir_))
def run(self, sorted_repos):
self.log("cloning repositories...")
cd(self._basedir)
for repo in sorted_repos:
self.log("cloning {0}".format(repo))
rm("-rf", repo)
git.clone(self._repo_url(repo), repo)
self.log("done cloning repos.")
def install_addon(self, addon):
addonPath = os.path.join(self.get_addon_path(), addon.name)
if os.path.isdir(addonPath):
# Folder already exists
if not os.path.isdir(os.path.join(addonPath, '.git')):
raise Exception('WARNING',
'ALREADY_INSTALLED_UNLINKED',
'Addon %s is already installed, but it\'s state is unknown.\nPlease reinstall with:\n' % addon.name + note('ofx reinstall %s' % addon.name))
else:
os.chdir(addonPath)
currentSha = sh.git('rev-parse', 'HEAD').rstrip()
remoteSha = addon.latest_commit['sha']
if addon.version is not None:
try:
git.checkout(addon.version)
except Exception:
raise Exception('ERROR',
'UNKNOWN_VERSION',
'No version named %s found for %s' % (addon.version, addon.name))
print ok() + addon.name + ": %s checked out" % addon.version
else:
if currentSha == remoteSha:
raise Exception('OK',
'ALREADY_INSTALLED',
'Addon %s is already installed and up to date' % addon.name)
else:
raise Exception('WARNING',
'ALREADY_INSTALLED_OUTDATED',
'Addon %s is already installed but is not up to date. Consider running:\n' % addon.name + note("ofx update %s" % addon.name))
else:
print '>> Cloning %s' % addon.clone_url
os.chdir(self.get_addon_path())
git.clone(addon.clone_url)
os.chdir(addonPath)
sha = sh.git('rev-parse', 'HEAD').rstrip()
if not sha:
raise Exception('ERROR', 'CLONE_ERROR', "Something went wrong while cloning from "+addon.clone_url)
else:
print ok()+addon.name + ": Clone done"
if addon.version is not None:
try:
git.checkout(addon.version)
except Exception:
raise Exception('ERROR',
'UNKNOWN_VERSION',
'No version named %s found for %s' % (addon.version, addon.name))
print ok() + addon.name + ": %s checked out" % addon.version
self.install_dependencies(addon)
return True
def setup(job_sources):
job_sources_by_name = {}
for job_source in job_sources:
if os.path.isdir(job_source):
name = os.path.basename(os.path.realpath(job_source))
js = {"commitish": None, "directory": job_source, "url": None}
else:
url, commitish = job_source.split(";", 1)
match = re.search(
r"/([a-zA-Z0-9-_]+)(?:.git)?",
url
)
name = match.group(1)
js = {"commitish": commitish, "directory": name, "url": url}
job_sources_by_name[name] = js
sys.stderr.write("Job Sources: {} \n".format(job_sources_by_name))
paths_by_source = {}
for name, job_source in job_sources_by_name.items():
directory = job_source["directory"]
if job_source["url"]:
try:
git.clone(
"--branch",
job_source["commitish"],
job_source["url"],
directory
)
except ErrorReturnCode_128 as e:
sys.stderr.write(
"Clone Failure: {}, "
"retrying checkout.\n".format(e))
git.checkout(
job_source["commitish"],
_env={"GIT_DIR": directory + "/.git"}
)
try:
metadata_file = "{d}/component_metadata.yml".format(d=directory)
with open(metadata_file) as f:
metadata = yaml.load(f)
except IOError as e:
sys.stderr.write("Failed to load component metadata file: "
"{}, error: {}".format(metadata_file, e))
else:
paths = metadata.get("jenkins", {}).get("jjb_paths", [])
paths_by_source[name] = [
os.path.join(directory, path)
for path in paths
]
all_paths = chain(*paths_by_source.values())
click.echo(":".join(all_paths))
def clone(self, repo, path, branch='master'):
"""Equivalent to git clone """
if not os.path.exists('{0}'.format(path)):
global git
try:
git.clone('{0}'.format(repo), '{0}'.format(path),
'--branch={0}'.format(branch), '--depth=1')
except ErrorReturnCode as e:
Log.debug(self, "{0}".format(e))
Log.error(self, "Unable to git clone at {0} ".format(path))
else:
Log.debug(self, "WOGit: Path {0} already exist".format(path))
def updateRepo(url, name, local_storage):
repo_path = os.path.join(local_storage, name)
if os.path.exists(repo_path) and os.path.exists(os.path.join(repo_path, '.git')):
git.reset(hard= True, _cwd= repo_path)
git.pull(_cwd= repo_path)
else:
if os.path.exists(repo_path):
shutil.rmtree(repo_path)
git.clone(url, name, _cwd= local_storage)
git.config('core.fileMode', 'false', _cwd= repo_path)
git.config('core.sharedRepository', 'true', _cwd= repo_path)
def main():
options = json.load(sys.stdin)
github_remote_url = f"{get_github_url()}/{options['target_remote']}.git"
if len(list(pathlib.Path('.').iterdir())) > 0:
print("Found existing files in work directory", file=sys.stderr)
assert pathlib.Path('.git').exists(
), "if files are present in the work dir, it must be a git work tree"
remote = "origin"
if options["source_remote_name"] == remote:
remote = remote + "2"
add_or_set_git_remote(remote, github_remote_url)
print(f"Fetching from {github_remote_url}", file=sys.stderr)
git.fetch(remote)
print(
f"Checking out {options['target_branch']} from {remote}/{options['target_branch']}",
file=sys.stderr)
git.checkout("-B", options['target_branch'],
f"{remote}/{options['target_branch']}")
print(f"Cleaning work tree", file=sys.stderr)
git.reset("--hard", "HEAD")
git.clean("-fdx")
else:
print(f"Cloning {options['target_branch']} from {github_remote_url}",
file=sys.stderr)
git.clone("--branch", options['target_branch'], github_remote_url, ".")
if options['target_remote'] != options['source_remote']:
source_remote_name = options['source_remote_name']
add_or_set_git_remote(
source_remote_name,
f"{get_github_url()}/{options['source_remote']}.git")
print(
f"Fetching from {get_github_url()}/{options['source_remote']}.git",
file=sys.stderr)
git.fetch(source_remote_name)
set_git_author_info(f"GitHub Action {os.environ['GITHUB_ACTION']}",
"action@localhost")
try:
git("cherry-pick", options['source_commits'])
print(
f"Source commits ({options['source_commits']}) were successfully cherry-picked "
f"onto {options['target_remote']}:{options['target_branch']}",
file=sys.stderr)
except sh.ErrorReturnCode:
print(
f"Source commits ({options['source_commits']}) could not be cherry-picked "
f"onto {options['target_remote']}:{options['target_branch']}",
file=sys.stderr)
raise
def clone_repo(repo, target=None):
"""Clone repo to target and return the checkout directory path.
If target is not specified, clone_repo defaults to ./docsrc
"""
if target is None:
target = os.path.join(os.getcwd(), 'docsrc')
git.clone(repo, target)
return target
def set_remote(self, chdir=False):
# init
remote = WALIKI_DATA_DIR + '_remote'
git.clone(WALIKI_DATA_DIR, remote)
os.chdir(remote)
git.config("user.email", '*****@*****.**')
git.config("user.name", 'I am me, yo')
os.chdir(WALIKI_DATA_DIR)
git.remote('add', 'origin', remote)
if chdir:
os.chdir(remote)
return remote
def set_remote(self, chdir=False):
# init
remote = WALIKI_DATA_DIR + '_remote'
git.clone(WALIKI_DATA_DIR, remote)
os.chdir(remote)
git.config("user.email", '*****@*****.**')
git.config("user.name", 'I am me, yo')
os.chdir(WALIKI_DATA_DIR)
git.remote('add', 'origin', remote)
if chdir:
os.chdir(remote)
return remote
def fork(self, source, dest):
'''
Fork a component repository
Makes the forkee repo the "upstream" remote
'''
source_path = self.path(source)
dest_path = self.path(dest)
if os.path.exists(dest_path):
raise Exception('Component repository already exists at %s' % dest_path)
git.clone(source_path, dest_path)
self.git(dest_path, 'remote', 'rename', 'origin', 'upstream')
self.setup(dest_path)
def manage_checkout(self):
logger.info('manage checkout')
if not os.path.exists(self.checkouts_dir):
os.makedirs(self.checkouts_dir)
if not os.path.exists(self.checkout_dir):
logger.info('git clone %s %s' % (self.url, self.checkout_dir))
git.clone(self.url, self.checkout_dir)
else:
logger.info('git fetch -a')
git.fetch('-a', _cwd=self.checkout_dir)
logger.info('git pull --rebase')
git.pull('--rebase', _cwd=self.checkout_dir)
def repository(namespace, name, branch='master'):
'''Returns a repository'''
with TemporaryDirectory() as download_path:
old_directory = str(pwd()).strip()
try:
git.clone('https://github.com/{0}/{1}.git'.format(namespace, name), download_path)
cd(download_path)
git.fetch('origin', branch)
git.checkout(branch)
yield (download_path, git('rev-parse', 'HEAD'), redis.Dict(key="{0}.{1}".format(namespace, name)))
except ErrorReturnCode_128:
mkdir(download_path)
yield (None, None, None)
cd(old_directory)
def initial_deploy(repo_url):
os.chdir('%s' % REPOS_DIR)
git.clone(repo_url)
dest = destination_path(repo_url)
if not os.path.exists(dest):
os.makedirs(dest)
commit_sha = install_site(repo_url)
if commit_sha:
update_commit(repo_url, commit_sha)
installOK = True
else:
logger.error("Failed to install repo %s on initial deploy" %s)
installOK = False
return installOK
def clone_repo():
"""
Clone the Git repository 'github/gitignore' to the data directory.
This function clones the gitignore repository from GitHub and saves the
local copy in the workflow's data directory. It uses the module sh to invoke
the git executable. If the git executable cannot execute properly, an
exception is thrown.
"""
try:
os.chdir(workflow.datafile(""))
git.clone("https://github.com/github/gitignore.git")
except:
handle_exception()
return 0
def __git_clone(url, branch, path):
"""
Private method to clone a git repository.
This method clones a git repository specified by ``url`` and
``branch`` to a folder ``path``. The ``--depth=1`` option is passed
to the command to avoid cloning full history.
.. versionadded:: 0.1.0
"""
try:
git.clone(url, path, quiet=True, depth=1, branch=branch)
except BaseException:
print('There was a problem cloning {0}.'.format(url))
raise
def clone(self):
"""todo: Docstring for clone
:return:
:rtype:
"""
logger.debug("")
if not self.url:
estr = "Cannot install this repos without a URL. %s" % self.info()
logger.warning(estr)
raise ValueError(estr)
# check if the already exists, if so, don't install, update it?
url = urlparse(self.url)
url_path = url[2]
path_end = url_path.split('/')
path_end = path_end[len(path_end) - 1]
if url.scheme not in self.supported_schemes:
raise ValueError("Unsupported scheme '{}' for {}".format(url.scheme, self.url))
assert self.repo_dir, "Invalid repo directory."
# Clone it.
logger.debug("cloning %s into %s .", self.url, self.repo_dir)
self.pr_pass("\nInstalling %s ... " % self.url)
p = git.clone('--progress', self.url, self.repo_dir,
_out=self._sh_stdout('blue'), _err=self._sh_stderr('blue'))
p.wait()
def repository(namespace, name, branch='master'):
'''Returns a repository'''
with TemporaryDirectory() as download_path:
old_directory = str(pwd()).strip()
try:
git.clone('https://github.com/{0}/{1}.git'.format(namespace, name),
download_path)
cd(download_path)
git.fetch('origin', branch)
git.checkout(branch)
yield (download_path, git('rev-parse', 'HEAD'),
redis.Dict(key="{0}.{1}".format(namespace, name)))
except ErrorReturnCode_128:
mkdir(download_path)
yield (None, None, None)
cd(old_directory)
def main(input_file, pa_code):
with open (input_file, 'r') as f:
for line in f.readlines():
github_root, repo_name = line.partition(',')[::2]
github_root = github_root.strip()
repo_name = repo_name.strip()
github_url = "https://github.com/" + github_root + "/" + repo_name + ".git"
repo_path = "./" + pa_code + "/ucd-csci2312-" + pa_code + "-" + github_root
print "Cloning " + github_url + " to " + repo_path
try:
# output = git.clone(github_url, repo_path, _err=print_error, _out_bufsize=1)
output = git.clone(github_url, repo_path, _iter=True)
print "Output code: " + str(output.exit_code)
# output.wait() # ???
except ErrorReturnCode as e:
aggr = ""
print "RAN:"
for c in e.full_cmd:
aggr += c
print aggr
print "STDOUT:"
for c in e.stdout:
aggr += c
print aggr
print "STDERR:"
for c in e.stderr:
aggr += c
print aggr
def clone(self):
"""todo: Docstring for clone
:return:
:rtype:
"""
logger.debug("")
if not self.url:
estr = "Cannot install this repos without a URL. %s" % self.info()
logger.warning(estr)
raise ValueError(estr)
# check if the already exists, if so, don't install, update it?
url = urlparse(self.url)
url_path = url[2]
path_end = url_path.split('/')
path_end = path_end[len(path_end) - 1]
if url.scheme not in self.supported_schemes:
raise ValueError("Unsupported scheme '{}' for {}".format(
url.scheme, self.url))
assert self.repo_dir, "Invalid repo directory."
# Clone it.
logger.debug("cloning %s into %s .", self.url, self.repo_dir)
self.pr_pass("\nInstalling %s ... " % self.url)
p = git.clone('--progress',
self.url,
self.repo_dir,
_out=self._sh_stdout('blue'),
_err=self._sh_stderr('blue'))
p.wait()
def trigger(req):
e = create_event_from_request(req)
try:
e.validate(req.headers)
except Exception:
print(RuntimeError("Ignoring webhook event"))
return
bucket = os.environ['BUCKET']
project = os.environ['GCP_PROJECT']
config = {
'source': {
'bucket': bucket
},
'substitutions': {
'REPO_NAME': e.repo,
'BRANCH_NAME': e.branch,
'REVISION_ID': e.commit,
'COMMIT_SHA': e.commit,
'SHORT_SHA': e.commit[0:7]
}
}
with tempfile.TemporaryDirectory() as tmpdir:
#git.clone('--branch', e.base_branch, '--single-branch', e.base_clone_url, tmpdir)
#git.remote('add', 'head', e.head_clone_url, _cwd=tmpdir)
#git.fetch('head', f'+refs/heads/{e.head_branch}', _cwd=tmpdir)
#git.merge('-q', '--no-ff', '-m', 'trigger-merge', 'FETCH_HEAD', _cwd=tmpdir)
git.clone('--branch', e.branch, '--depth=1', '--single-branch',
e.clone_url, tmpdir)
cloudbuild_config_path = os.path.join(tmpdir, 'cloudbuild.yaml')
config.update(yaml.safe_load(open(cloudbuild_config_path)))
suffix = uuid.uuid1()
tarball = os.path.join(tmpdir, f'{e.commit}-{suffix}.tar.gz')
config['source']['object'] = tarball
git.archive('-o', tarball, 'HEAD', _cwd=tmpdir)
upload(tarball, bucket)
submit(project, config)
def process_repo(dirname, repo):
"""Process one git repository"""
if not Config.verbose:
print("Process " + dirname)
if not os.path.isdir(dirname):
print("Clone " + repo["url"])
git.clone("--depth", "2", repo['url'])
os.chdir(dirname)
git.pull()
mds = glob.glob("./_posts/*/*.md")
Result.totalCount += len(mds)
prefix = "https://" + dirname + repo['prefix']
for path in mds:
process_article(path, prefix)
if not Config.verbose:
print()
os.chdir("..")
def clone(repo):
global dest
dest = os.path.realpath(ARGS.outdir)
destrepo = os.path.join(dest, repo)
if not os.path.exists(destrepo):
os.chdir(dest)
# pp = git.clone("--progress", "[email protected]:%s/%s" %
# (ARGS.username, repo), _err=process_output,
# _out_bufsize=0)
git.clone("[email protected]:%s/%s" % (ARGS.username, repo),
_err=process_output,
_out_bufsize=0)
else:
# do git pull
os.chdir(destrepo)
pp = git.pull(_out=process_output, _out_bufsize=1)
pp.wait()
def setup_environment(exp):
"""
Do environment setup like downloading code and inputs.
"""
# Get the code, we only handle git repos.
code_urls = exp.code_urls.split(',')
for u in code_urls:
assert u[-4:] == '.git'
code_repos = [re.search('(\w+)\.git$', u).group(1) for u in code_urls]
code_hashes = exp.code_hashes.split(',')
assert len(code_urls) == len(code_hashes)
assert len(code_repos) == len(code_urls)
for c in code_urls:
git.clone(['--recursive', c])
orig_dir = os.getcwd()
for i, c in enumerate(code_hashes):
os.chdir(code_repos[i])
git.checkout(code_hashes[i])
git.submodule('update')
os.chdir(orig_dir)
data_urls = exp.data_urls.split(',')
data_hashes = exp.data_hashes.split(',')
for u in data_urls:
assert u[-7:] == '.tar.gz'
data_dirs = [re.search('(\w+)\.tar.gz$', u).group(1) for u in data_urls]
assert len(data_urls) == len(data_hashes)
assert len(data_dirs) == len(data_urls)
for u, d in zip(data_urls, data_dirs)
wget(d)
# Do build, then link in data.
if exp.codebase == 'MOM6':
mom6.build()
elif exp.codebase == 'MOM5':
mom5.build()
else:
assert False
return True
def get_releases(refresh=False):
cachedir = os.path.join(VARDIR, 'releases')
if not os.path.exists(cachedir):
os.makedirs(cachedir)
# make a devel checkout
dpath = os.path.join(cachedir, 'devel.git')
cmd = 'git clone %s %s' % (DEVEL_URL, dpath)
logger.info(cmd)
if not os.path.exists(dpath):
git.clone(DEVEL_URL, dpath)
if DEVEL_BRANCH:
(rc, so, se) = _run_command('cd %s; git branch | egrep --color=never ^\\* | head -n1' % dpath)
thisbranch = so.replace('*', '').strip()
if thisbranch != DEVEL_BRANCH:
logger.debug('%s != %s' % (thisbranch, DEVEL_BRANCH))
(rc, so, se) = _run_command('cd %s; git checkout %s' % (dpath, DEVEL_BRANCH))
assert rc == 0
def clone(self):
self.tmp_dir = tempfile.mkdtemp(suffix='facio')
try:
from sh import git
except ImportError:
raise Exception # TODO: Custom exception
try:
git = git.bake(_cwd=self.tmp_dir)
git.clone(self.repo, self.tmp_dir)
git.fetch('--all')
git.checkout('master') # TODO: Branch prompt to the user
except Exception:
raise Exception # TODO: Custom exception
rmtree(os.path.join(self.tmp_dir, '.git'))
with indent(4, quote=' >'):
puts(blue('Clone complete'))
def sync_git_to_bzr(
project_name,
github_user,
repositories_dir,
bzr_url):
"""
Using the provided {repositories_dir},
pull down the git project from github
([email protected]:{git_user}/{project_name}.git),
export all commits to a bzr repository,
and push to launchpad at lp:{project_name}.
If the launchpad repo already exists, this will fail the first time
because the history will be entirely different from the existing history.
Therefore, after running this the first time and getting an error,
`cd {project_name}-bzr/` and run `bzr push --overwrite`.
From then on, every subsequent time you run this, for the same
repository directory, it should work fine.
"""
git_dir = join(repositories_dir, project_name + '-git')
bzr_dir = join(repositories_dir, project_name + '-bzr')
git_url = '[email protected]:{0}/{1}.git'.format(github_user, project_name)
logger = ShLogger()
# Clone the git repo, or pull if it exists
if not isdir(git_dir):
logger.update_for_command(
"Cloning " + git_url,
git.clone(git_url, git_dir)
)
else:
logger.update_for_command(
"Pulling {0} changes".format(project_name),
git('-C', git_dir, 'pull')
)
# Always delete and recreate the bzr repo
logger.update_for_command(
"Removing bzr dir {0}".format(bzr_dir),
rm(bzr_dir, r=True, f=True)
)
try:
logger.update_for_command(
"Creating BZR repo",
bzr('init-repo', bzr_dir)
)
except ErrorReturnCode, sh_error:
logger.update('init-repo returned error code {0}. Message: {1}'.format(
str(sh_error.exit_code), sh_error.message
))
def clone(self):
""" Clone the git repository into a temporary directory. """
try:
from sh import git
except ImportError:
raise FacioException('Git must be installed to use git+ '
'template paths')
temp_diretory = self.get_temp_directory()
self.out('Git Cloning {0} to {1}'.format(self.path, temp_diretory))
try:
git = git.bake(_cwd=temp_diretory)
git.clone(self.path, temp_diretory)
git.fetch('--all')
git.checkout('master')
except:
raise FacioException('Failed to clone git repository '
'at {0}'.format(self.path))
return temp_diretory
def compile_slides(git_repo, base_path):
"""
"""
work_dir = tempfile.mkdtemp()
logging.debug("will work in {}".format(work_dir))
pd = Pandoc(work_dir)
git.clone(git_repo, '.', _cwd=work_dir)
in_file = find_markdown(work_dir)
#ensure that the directory exists
base_dir = os.path.dirname(base_path)
try:
os.makedirs(base_dir)
except OSError:
logging.warn("directory {} already exists".format(base_dir))
html_out = str(base_path) + ".html"
pdf_out = str(base_path) + ".pdf"
try:
logging.debug("compiling the html output")
logging.debug(html_out)
pd.compile_html(in_file, html_out)
except Exception as e:
logging.error("compilation fail for html")
logging.error(str(e))
try:
logging.debug("compiling the pdf output")
logging.debug(pdf_out)
pd.compile_pdf(in_file, pdf_out)
except Exception as e:
logging.error("compilation fail for pdf")
logging.error(str(e))
logging.debug("Cleaning working directory")
shutil.rmtree(work_dir)
def _get_code(self, nmpi_job, job_desc):
"""
Obtain the code and place it in the working directory.
If the experiment description is the URL of a Git repository, try to clone it.
If it is the URL of a zip or .tar.gz archive, download and unpack it.
Otherwise, the content of "code" is the code: write it to a file.
"""
url_candidate = urlparse(nmpi_job['code'])
logger.debug("Get code: %s %s", url_candidate.netloc, url_candidate.path)
if url_candidate.scheme and url_candidate.path.endswith((".tar.gz", ".zip", ".tgz")):
self._create_working_directory(job_desc.working_directory)
target = os.path.join(job_desc.working_directory, os.path.basename(url_candidate.path))
#urlretrieve(nmpi_job['code'], target) # not working via KIP https proxy
curl(nmpi_job['code'], '-o', target)
logger.info("Retrieved file from {} to local target {}".format(nmpi_job['code'], target))
if url_candidate.path.endswith((".tar.gz", ".tgz")):
tar("xzf", target, directory=job_desc.working_directory)
elif url_candidate.path.endswith(".zip"):
try:
# -o for auto-overwrite
unzip('-o', target, d=job_desc.working_directory)
except:
logger.error("Could not unzip file {}".format(target))
else:
try:
# Check the "code" field for a git url (clone it into the workdir) or a script (create a file into the workdir)
# URL: use git clone
git.clone('--recursive', nmpi_job['code'], job_desc.working_directory)
logger.info("Cloned repository {}".format(nmpi_job['code']))
except (sh.ErrorReturnCode_128, sh.ErrorReturnCode):
# SCRIPT: create file (in the current directory)
logger.info("The code field appears to be a script.")
self._create_working_directory(job_desc.working_directory)
with codecs.open(job_desc.arguments[0], 'w', encoding='utf8') as job_main_script:
job_main_script.write(nmpi_job['code'])
