def _resolve_git(url: str, destination: str) -> str:
"""
Clone a gh:// url to a local folder.
Args:
url: git url, the format is gh://organization/repo
destination: the destination directory for the repo, must be empty
Returns:
The destination directory.
Raises:
ValueError: The github url is not valid.
FileExistsError: The destination directory is not empty.
"""
url = url.lower()
if not url.startswith('gh://'):
raise ValueError(f'{url} is not a valid gh:// url.')
if len(os.listdir(destination)):
raise FileExistsError(
f'The working directory {destination} is not empty!')
parsed = urlparse(url)
org, repo = parsed.netloc, parsed.path[1:]
git_url = f'git://github.com/{org}/{repo}.git'
try:
Repo.clone_from(git_url, destination)
except Exception as e:
raise ValueError(f'The github url can not be cloned. Message: {e}')
logger.info(f'Cloned {url} to {destination}')
return destination
def rbenv_install(version: str = '2.4.2', force:bool = False):
logger.debug('verifying that dependencies are installed')
# Centos 7 Deps
# git bzip2 gcc make openssl-devel readline-devel zlib-devel docker
assert all([which(pkg) for pkg in ('git', 'bzip2', 'gcc', 'docker')])
# Get paths
rbenv = path.join(local_path, '.rbenv')
plugins = path.join(rbenv, 'plugins')
ruby_build = path.join(plugins, 'ruby-build')
rbenv_bin = path.join(rbenv, 'bin', 'rbenv')
gem = path.join(rbenv, 'shims', 'gem')
# Setup paths
if not path.exists(rbenv):
assert git
git.clone_from('https://github.com/rbenv/rbenv.git', rbenv)
if not path.exists(plugins):
mkdir(plugins)
if not path.exists:
assert git
git.clone_from('https://github.com/rbenv/ruby-build.git', ruby_build)
#
logger.debug('Installing ruby version {}'.format(version))
assert subprocess.Popen([rbenv_bin, 'install', version, '-f' if force else '-s']).wait() == 0
logger.debug('Setting ruby version {} as global'.format(version))
assert subprocess.Popen([rbenv_bin, 'global', version]).wait() == 0
logger.debug('installing bundle')
def CloneGitRepo(link,dir):
try:
from git.repo.base import Repo
except ImportError:
print("Please install GitPython package first!")
return None
Repo.clone_from(link, dir, progress=Progress())
def install_playbook(self, **kwargs):
"""
Validates that self._playbook is a valid path or url.
If it is a url git clone to /tmp. If it has a requires file
install dependencies.
"""
logger.debug('Installing playbook...')
if "url" in kwargs:
# Should probably extract the playbook name from the playbook
# URL
Repo.clone_from(
kwargs["url"],
self.path
)
elif "path" in kwargs:
if os.path.exists(kwargs["path"]):
path = None
if os.path.isfile(kwargs["path"]):
path = os.path.dirname(kwargs["path"])
else:
path = kwargs["path"]
os.symlink(os.path.realpath(path), self.path,)
else:
raise ValueError(
'The path value provided (%s) does not exists',
kwargs['path']
)
assert os.path.exists(self.path)
def test_get_pipeline_returns_none_if_non_existent(tmp_path: str) -> None:
"""Check get_pipeline returns None if it doesn't exist"""
Repo.init(tmp_path)
repo = Repository(str(tmp_path))
repo.set_active_stack("local_stack")
our_pipeline = repo.get_pipeline("not_a_pipeline")
assert our_pipeline is None
def clone_repo(start=0, end=100000):
"""
Clones the GitHub repositories if the repositories has not been cloned initially.
Parameters:
start(int): This is an integer that specifies which URL cloning should start from in the CSV file of the GitHub repositories. Defaults to zero.
end(int): This is an integer that specifies which URL cloning should end in the CSV file of the GitHub repositories. Defaults to 100000, which is the last URL.
Returns:
count(int): The number of repository cloned.
"""
repo_list = repo_url['URLs']
count = 0
for url in repo_list[start:end]:
url = str(url)
name = url.rsplit(
'/', 2) #get the repo name (last 2 part) of the repository url
last = name[-2] + '-' + name[-1]
try:
if not os.path.exists(last):
os.mkdir(last) #Make folder for a repo if it does not exist
repo = str(url) + '.git'
folder = r'repos'
Repo.clone_from(repo, last)
count += 1
print('cloned ', repo)
except:
continue
return count
def test_getting_active_stack_returns_local_stack(tmp_path: str, ) -> None:
"""Check getting the active stack"""
Repo.init(tmp_path)
repo = Repository(str(tmp_path))
repo.set_active_stack("local_stack")
assert repo.get_active_stack() == LocalService().get_stack("local_stack")
assert isinstance(repo.get_active_stack(), BaseStack)
def main(hub, archived, archived_only, forks, depth, prune, org):
"""
Clone an entire GitHub organization into the current directory.
Each repo becomes a subdirectory.
"""
if archived_only:
archived = True
dir_names = set()
for repo in hub.organization(org).repositories():
if repo.fork and not forks:
continue
if repo.archived and not archived:
continue
if not repo.archived and archived_only:
continue
dir_name = repo.name
dir_name = dir_name.lstrip("-") # avoid dirname/option confusion
dir_names.add(dir_name)
if os.path.exists(dir_name):
continue
print("Cloning {}".format(repo.full_name))
clone_args = {}
if depth:
clone_args['depth'] = depth
Repo.clone_from(repo.ssh_url, dir_name, **clone_args)
if prune:
for dir_name in os.listdir("."):
if os.path.isdir(dir_name):
if dir_name not in dir_names:
print("Pruning {}".format(dir_name))
shutil.rmtree(dir_name)
def clone():
try:
print(f'Using existing repo in {settings.REPOSITORY_DIR}')
Repo(settings.REPOSITORY_DIR)
except:
print('Repo does not exist')
Repo.clone_from(settings.GIT_REPOSITORY, settings.REPOSITORY_DIR)
def CheckFiles():
if os.path.isdir('scripts'):
print ("File Does exist")
checkall()
else:
print ("File not exist, Creating")
Repo.clone_from("https://github.com/ChokunPlayZ/ChatSpammer-Data.git", "scripts")
def clone(where, what):
from pathlib import Path
dest_fn = Path(GLOBS['kk_code_dir'], what)
if dest_fn.exists():
return (False, str(dest_fn))
"""ans = None
while ans not in "ynYN":
ans = input("The destination folder, {dest_fn}, exists already. Should we overwrite it?")
if ans in "nN":"""
if not dest_fn.exists():
print("Module doesn't exist. Attempting to load it from GitHub...")
if where[0] == '@':
where = where[1:]
url = "https://github.com/%s/%s" % (where, what)
print(f"Cloning '{where}/{what}' into '{dest_fn}' ...")
from git.repo.base import Repo
fld = Path(GLOBS['kk_code_dir'])
if not fld.exists():
fld.mkdir()
Repo.clone_from(url, dest_fn)
return (True, str(dest_fn))
def run(self):
"""
Removes, if exists, the current folder and clones the repository in that folder that you have passed by
- argument with space like delimiter (ie: !git_clone https://github.com/bilardi/sceptre-git-clone-hook my-folder)
- sceptre_user_data with properties GitRepository and RepositoryFolder
Raise (Exception): when the system does not find the arguments repository and folder in argument or sceptre_user_data
"""
if self.argument and self.DELIMITER in self.argument:
repository, folder = self.argument.split(self.DELIMITER, 1)
self.logger.debug(
"[{}] Parameters parsed from the argument".format(self.NAME))
elif self.stack.sceptre_user_data and 'GitRepository' in self.stack.sceptre_user_data and 'RepositoryFolder' in self.stack.sceptre_user_data:
repository = self.stack.sceptre_user_data.get('GitRepository', {})
folder = self.stack.sceptre_user_data.get('RepositoryFolder', {})
self.logger.debug(
"[{}] Parameters parsed from sceptre_user_data".format(
self.NAME))
else:
raise Exception(
"Parameters GitRepository and RepositoryFolder could not be parsed from sceptre_user_data or argument"
)
if os.path.exists(folder):
shutil.rmtree(folder)
self.logger.info(
"[{}] Removed the local repository folder {}".format(
self.NAME, folder))
Repo.clone_from(repository, folder)
self.logger.info(
"[{}] Downloaded the remote repository in the folder {}".format(
self.NAME, folder))
def parse(self, response, **kwargs):
# inspect_response(response, self)
links = response.xpath('/html/body/div/main/div/div/div/div/div[@class="org-repos repo-list"]/ul/li/div/div/h3[@class="wb-break-all"]/a[@data-hovercard-type="repository"]/@href').getall()
for link in links:
l = urljoin(self.start_urls[0], link)
dir = Path.joinpath(self.output_dir, l.split('/')[-1])
os.makedirs(dir, exist_ok=True)
Repo.clone_from(l, dir, branch='master')
def test_get_artifact_store_returns_artifact_store(tmp_path: str) -> None:
"""Test get_artifact_store returns artifact store."""
Repo.init(tmp_path)
repo = Repository(str(tmp_path))
local_service = repo.get_service()
artifact_store = local_service.get_artifact_store(LOCAL_ARTIFACT_STORE_NAME)
assert artifact_store is not None
assert isinstance(artifact_store, BaseArtifactStore)
def test_get_pipelines_returns_list(tmp_path: str) -> None:
"""Check get_pipelines returns a list"""
Repo.init(tmp_path)
repo = Repository(str(tmp_path))
repo.set_active_stack("local_stack")
our_pipelines = repo.get_pipelines()
assert our_pipelines is not None
assert isinstance(our_pipelines, list)
def test_get_metadata_store_returns_metadata_store(tmp_path: str) -> None:
"""Test get_metadata_store returns metadata store."""
Repo.init(tmp_path)
repo = Repository(str(tmp_path))
local_service = repo.get_service()
metadata_store = local_service.get_metadata_store(LOCAL_METADATA_STORE_NAME)
assert metadata_store is not None
assert isinstance(metadata_store, BaseMetadataStore)
def test_get_orchestrator_returns_orchestrator(tmp_path: str) -> None:
"""Test get_orchestrator returns orchestrator"""
Repo.init(tmp_path)
repo = Repository(str(tmp_path))
local_service = repo.get_service()
orchestrator = local_service.get_orchestrator(LOCAL_ORCHESTRATOR_NAME)
assert orchestrator is not None
assert isinstance(orchestrator, BaseOrchestrator)
def test_get_stack_raises_exception_when_key_does_not_exist(
tmp_path: str,
) -> None:
"""Test get_stack raises exception when key does not exist."""
Repo.init(tmp_path)
repo = Repository(str(tmp_path))
local_service = repo.get_service()
with pytest.raises(DoesNotExistException):
local_service.get_stack("made_up_stack")
def test_getting_the_active_service_returns_local_service(
tmp_path: str, ) -> None:
"""Check getting the active service"""
Repo.init(tmp_path)
repo = Repository(str(tmp_path))
assert repo.get_service() is not None
assert isinstance(repo.get_service(), BaseComponent)
assert isinstance(repo.get_service(), LocalService)
assert repo.get_service() == repo.service
def test_get_pipeline_returns_same_when_stack_specified(tmp_path: str) -> None:
"""Check get_pipeline returns the same if stack specified"""
Repo.init(tmp_path)
repo = Repository(str(tmp_path))
repo.set_active_stack("local_stack")
our_pipeline_default = repo.get_pipeline("pipeline_1")
our_pipeline_local = repo.get_pipeline("pipeline_1",
stack_key="local_stack")
assert our_pipeline_default == our_pipeline_local
def test_register_orchestrator_works_as_expected(tmp_path: str) -> None:
"""Test register_orchestrator method registers an orchestrator as expected."""
Repo.init(tmp_path)
repo = Repository(str(tmp_path))
local_service = repo.get_service()
orchestrator = LocalOrchestrator()
local_service.register_orchestrator("local_orchestrator_2", orchestrator)
assert local_service.get_orchestrator("local_orchestrator_2") is not None
local_service.delete_orchestrator("local_orchestrator_2")
def clone_git(self, github_repo, branch):
tmpRoot = self.root + '/' + github_repo + '/' + branch
git_url = self.github_base + github_repo + ".git"
if not os.path.isdir(tmpRoot):
os.makedirs(os.path.dirname(tmpRoot), exist_ok=True)
branch_opt = '--branch ' + branch
Repo.clone_from(git_url, tmpRoot, multi_options=[branch_opt])
def test_register_stack_raises_exception_when_key_already_exists(
tmp_path: str,
) -> None:
"""Test register_stack raises exception when key already exists."""
Repo.init(tmp_path)
repo = Repository(str(tmp_path))
local_service = repo.get_service()
local_stack1 = local_service.get_stack("local_stack")
with pytest.raises(AlreadyExistsException):
local_service.register_stack("local_stack", local_stack1)
def setup_bkrepo(repo_url, repo_path):
try:
repo = Repo(repo_path)
except NoSuchPathError:
logging.info('git clone: {}'.format(repo_url))
repo = Repo.clone_from(repo_url, repo_path)
else:
logging.info('git pull: {}'.format(repo_url))
repo.remote().pull()
return repo
def test_register_metadata_store_with_existing_key_fails(tmp_path: str) -> None:
"""Test register_metadata_store with existing key fails."""
Repo.init(tmp_path)
repo = Repository(str(tmp_path))
local_service = repo.get_service()
with pytest.raises(AlreadyExistsException):
local_service.register_metadata_store(
LOCAL_METADATA_STORE_NAME,
local_service.get_metadata_store(LOCAL_METADATA_STORE_NAME),
)
def test_delete_orchestrator_works(tmp_path: str) -> None:
"""Test delete_orchestrator works as expected."""
Repo.init(tmp_path)
repo = Repository(str(tmp_path))
local_service = repo.get_service()
orchestrator = LocalOrchestrator()
local_service.register_orchestrator("local_orchestrator_2", orchestrator)
local_service.delete_orchestrator("local_orchestrator_2")
with pytest.raises(DoesNotExistException):
local_service.get_orchestrator("local_orchestrator_2")
def test_register_orchestrator_with_existing_key_fails(tmp_path: str) -> None:
"""Test register_orchestrator with existing key fails."""
Repo.init(tmp_path)
repo = Repository(str(tmp_path))
local_service = repo.get_service()
with pytest.raises(AlreadyExistsException):
local_service.register_orchestrator(
LOCAL_ORCHESTRATOR_NAME,
local_service.get_orchestrator(LOCAL_ORCHESTRATOR_NAME),
)
def clone_repo(owner, repo, path_to_clone):
try:
if (not os.path.exists(path_to_clone)):
git_url = "https://github.com/" + owner + "/" + repo
Repo.clone_from(url=git_url, to_path=path_to_clone)
else:
print("Jumped repo because its folder already exists: " +
repo['id'])
except:
print("Error for: Owner -- {0} Repo -- {1}".format(owner, repo))
print(sys.exc_info()[0])
def test_local_service_can_access_orchestrators(tmp_path: str) -> None:
"""Local Service can access orchestrators."""
Repo.init(tmp_path)
repo = Repository(str(tmp_path))
local_service = repo.get_service()
assert local_service.orchestrators is not None
assert isinstance(local_service.orchestrators, dict)
assert isinstance(
local_service.orchestrators[LOCAL_ORCHESTRATOR_NAME],
BaseOrchestrator,
)
def include_git(self, github_repo, file_path, branch, local_path):
tmpRoot = self.root + '/' + github_repo + '/' + branch
git_url = self.github_base + github_repo + ".git"
if not os.path.isdir(tmpRoot):
os.makedirs(os.path.dirname(tmpRoot), exist_ok=True)
branch_opt = '--branch ' + branch
Repo.clone_from(git_url, tmpRoot, multi_options=[branch_opt,'--depth 1'])
self.include_file(tmpRoot + "/" + file_path, local_path)
def __init__(self, config: Config, path: str = None) -> None:
self.__config: Config = config
try:
if not path:
path = config.root_path
self.__repo = Repo(path)
assert not self.__repo.bare
self.origin = self.__repo.remote('origin')
except InvalidGitRepositoryError:
log_error("Path " + path + " is not a git repository. Please go to valid git repository!")
sys.exit()
def main(hub, forks, depth, org):
for repo in hub.organization(org).iter_repos():
if repo.fork and not forks:
continue
dir_name = repo.name
dir_name = dir_name.lstrip("-") # avoid dirname/option confusion
if os.path.exists(dir_name):
continue
print(repo.full_name)
clone_args = {}
if depth:
clone_args['depth'] = depth
Repo.clone_from(repo.ssh_url, dir_name, **clone_args)
def get_results(repos_dir, settings):
paths = []
collect_only_dirty = settings.get('dirty', False)
show_full_path = settings.get('full_path', False)
for path, dirs, files in os.walk(repos_dir):
if is_git_dir(path):
repo = Repo(path)
path = os.path.dirname(path)
if show_full_path:
path = PATH.format(heading=os.path.join(repos_dir, path))
else:
path = PATH.format(heading=os.path.relpath(path, repos_dir))
bare = repo.bare
dirty = repo.is_dirty()
if (collect_only_dirty and dirty) or (not collect_only_dirty):
paths.append({'path': path, 'bare': bare, 'dirty': dirty})
return paths
def load_config():
git_config = Repo().config_reader()
pivotal_items = git_config.items('pivotal')
token = None
filter = None
project_id = None
for item_name, item_value in pivotal_items:
if item_name == 'token':
token = item_value
if item_name == 'filter':
filter = item_value
if item_name == 'projectid':
project_id = item_value
return token, filter, project_id
def __init__(self , **kwargs):
url = kwargs.get("url")
local_path = kwargs.get("local_path")
self.work_dir = None
if url is None and local_path is None:
raise ValueError("Must supply one of url and local_path")
if url and local_path:
raise ValueError("Can only supply one of url and local_path")
if url:
self.work_dir = tempfile.mkdtemp( )
self.clone( url )
else:
self.work_dir = None
self.repo = Repo( local_path )
origin = self.repo.remote( name = "origin")
origin.fetch( )
def make_origin():
origin_root = tempfile.mkdtemp()
origin_base = "origin"
origin_url = os.path.join( origin_root , origin_base)
os.makedirs( origin_url )
repo = Repo.init( path = origin_url )
add_file( repo , "file.txt" , "Content")
add_file( repo , "tests/run_not_executable" , "content")
add_file( repo , "tests/subdir/file" , "Content")
add_executable( repo , "tests/run_OK" , "#!/bin/bash\nexit 0\n")
add_executable( repo , "tests/run_fail" , "#!/bin/bash\nexit 1\n")
repo.git.commit(m = "message")
repo.git.branch("version2")
repo.git.checkout("version2")
with open( os.path.join(origin_url , "file2.txt") , "w") as f:
f.write("File with content")
repo.git.add("file2.txt")
repo.git.commit(m = "message")
return repo
def git_repo(self, request, github_repo, branch=None, remote='origin', checkout_root=None):
"""
py.test fixture to clone a GitHub based repo onto the local disk.
Arguments:
github_repo (:class:`~github3.GitHub`): The repo to read from
branch (str): The branch to check out
Returns:
A :class:`~git.repo.base.Repo` object, with the master branch checked out
and up to date with the remote.
"""
if checkout_root is None:
checkout_root = request.config.option.checkout_root
if not os.path.exists(checkout_root):
os.makedirs(checkout_root)
repo_dir = os.path.join(
os.path.join(checkout_root, github_repo.owner.name),
github_repo.name
)
if github_repo.private:
repo_url = github_repo.ssh_url
else:
repo_url = github_repo.clone_url
if not os.path.exists(repo_dir):
repo = Repo.clone_from(repo_url, repo_dir)
else:
repo = Repo(repo_dir)
if github_repo not in SYNCED:
try:
remote_obj = repo.remote(remote)
except ValueError:
repo.create_remote(remote, repo_url)
remote_obj = repo.remote(remote)
if remote_obj.fetch != repo_url:
remote_obj.set_url(repo_url)
remote_obj.fetch()
SYNCED.add(github_repo)
if branch is None:
branch = github_repo.default_branch
head = repo.head
remote_branch = RemoteReference(repo, 'refs/remotes/{}/{}'.format(remote, branch))
local_branch = Head(repo, 'refs/heads/{}'.format(branch))
try:
if head.commit != remote_branch.commit:
local_branch.commit = remote_branch.commit
local_branch.checkout()
except ValueError:
pytest.xfail("Branch {} is empty".format(branch))
return repo
def run(self):
config_file = "development.ini"
if "BIOSHADOCK_CONFIG" in os.environ:
config_file = os.environ["BIOSHADOCK_CONFIG"]
config = ConfigParser.ConfigParser()
config.readfp(open(config_file))
logging.config.fileConfig(config_file)
while True:
logging.info("New build run")
if BioshadockDaemon.db_mongo is None:
mongo = MongoClient(config.get('app:main','mongo_url'))
BioshadockDaemon.db_mongo = mongo['mydockerhub']
if BioshadockDaemon.db_redis is None:
BioshadockDaemon.db_redis = redis.StrictRedis(host=config.get('app:main','redis_host'), port=int(config.get('app:main','redis_port')), db=0)
if BioshadockDaemon.cli is None:
if config.get('app:main', 'docker_connect'):
BioshadockDaemon.cli = Client(base_url=config.get('app:main',
'docker_connect'))
else:
BioshadockDaemon.cli = Client()
BioshadockDaemon.cli.login(username=config.get('app:main','push_auth_user'), password=config.get('app:main','push_auth_password'),
email=config.get('app:main','push_auth_email'),registry=config.get('app:main','service'))
build = BioshadockDaemon.db_redis.lpop('bioshadock:builds')
dockerfile = None
if build is not None:
dt = datetime.datetime.now()
timestamp = time.mktime(dt.timetuple())
build = loads(build)
logging.debug(str(build))
dockerfile = build['dockerfile']
gitrepo = build['git']
do_git = False
git_repo_dir = None
if gitrepo is not None and gitrepo and gitrepo != 'none':
# TODO clone repo in a dir, chdir to repo and optionally write
# dockerfile
git_repo_dir = tempfile.mkdtemp(suffix='.git')
git_info = gitrepo.split('#')
gitrepo = git_info[0]
selectedbranch = 'master'
subdir = None
if len(git_info) > 1:
branch_path = git_info[1].split(':')
if branch_path[0]:
selectedbranch = branch_path[0]
if len(branch_path) > 1 and branch_path[1]:
subdir = branch_path[1]
logging.info(str(gitrepo))
logging.info("Using branch "+selectedbranch)
logging.info("Directory: "+str(subdir))
try:
Repo.clone_from(gitrepo, git_repo_dir, branch=selectedbranch)
if subdir is not None:
git_repo_dir = os.path.join(git_repo_dir, subdir)
logging.debug(str(git_repo_dir))
os.chdir(git_repo_dir)
except Exception as e:
logging.error(str(e))
continue
#if dockerfile:
if not os.path.exists("Dockerfile"):
logging.debug("Overwrite Dockerfile")
f = open('Dockerfile', 'w')
f.write(dockerfile.encode('utf-8'))
f.close()
else:
logging.debug("Use git Dockerfile")
with open ("Dockerfile", "r") as gitDockerfile:
dockerfile=gitDockerfile.read().encode('utf-8')
f = BytesIO(dockerfile.encode('utf-8'))
build_tag = ''
if 'tag' in build and build['tag']:
build_tag = ':'+build['tag']
logging.info('Build: '+str(build['id']))
response = [line for line in BioshadockDaemon.cli.build(
fileobj=f, rm=True, tag=config.get('app:main', 'service')+"/"+build['id']+build_tag)]
build['response'] = response
if build['response']:
last = build['response'][len(build['response'])-1]
matches = re.search('Successfully built\s+(\w+)', last)
if matches is None:
build['status'] = False
else:
build['status'] = True
build['image_id'] = matches.group(1)
p= subprocess.Popen(["docker",
"push",
config.get('app:main', 'service')+"/"+build['id']])
else:
build['status'] = False
build['timestamp'] = timestamp
BioshadockDaemon.db_mongo['repository'].update({'id': build['id']},
{'$push': {'builds': build},
'$set':{'meta.Dockerfile': dockerfile}})
if do_git:
cur_dir = os.path.dirname(os.path.realpath(__file__))
os.chdir(cur_dir)
shutil.rmtree(git_repo_dir)
time.sleep(2)
def make_client( origin ):
name = os.path.basename( origin.working_tree_dir )
client_path = os.path.join( tempfile.mkdtemp() , name )
origin_url = "file://%s" % origin.working_tree_dir
return Repo.clone_from( origin_url , client_path )
def clone(self , url):
self.repo = Repo.clone_from( url , self.work_dir )
from git.repo.base import Repo
import sys, os
url = sys.argv[1]
path = "repos/{0}".format(url.split("/", 3)[-1])
if __name__ == '__main__':
if not os.path.isdir(path):
repo = Repo.clone_from(url, path)
def clone(uri, path):
return Repo.clone_from(uri, path, progress=ProgressPrinter())
backup_file_path = os.path.join(app_folder, repo_name + "-" + args.revert + ".zip")
get(sftp, backup_file_path, os.path.join(PATH, backup_file_name))
fh = open(os.path.join(PATH, backup_file_name), "rb")
z = zipfile.ZipFile(fh)
for name in z.namelist():
outpath = os.path.join(PATH, repo_name)
z.extract(name, outpath)
fh.close()
rm(sftp, os.path.join(app_folder, repo_name))
put_all(sftp, os.path.join(PATH, repo_name), app_folder)
exit(0)
try:
Repo.clone_from(git_repo, os.path.join(PATH, repo_name))
except Exception, e:
print "git error"
print e
sys.exit(2)
if not remote_file_exists(sftp, app_folder + repo_name):
# sftp.mkdir(app_folder+repo_name)
put_all(sftp, os.path.join(PATH, repo_name), app_folder)
shutil.rmtree(os.path.join(PATH, repo_name))
else:
os.mkdir(os.path.join(PATH, "tmp"))
get_all(sftp, app_folder + repo_name, os.path.join(PATH, "tmp"))
zip_name = repo_name + "-" + time.strftime("%d-%m-%Y-%I:%M:%S") + ".zip"
class GitModule(object):
def clone(self , url):
self.repo = Repo.clone_from( url , self.work_dir )
def __init__(self , **kwargs):
url = kwargs.get("url")
local_path = kwargs.get("local_path")
self.work_dir = None
if url is None and local_path is None:
raise ValueError("Must supply one of url and local_path")
if url and local_path:
raise ValueError("Can only supply one of url and local_path")
if url:
self.work_dir = tempfile.mkdtemp( )
self.clone( url )
else:
self.work_dir = None
self.repo = Repo( local_path )
origin = self.repo.remote( name = "origin")
origin.fetch( )
def __del__(self):
if self.work_dir and os.path.isdir( self.work_dir ):
shutil.rmtree( self.work_dir )
def getRoot(self):
return self.repo.working_tree_dir
def relPath(self , path):
if path.startswith( self.getRoot( ) ):
N = len(self.getRoot()) + 1
return path[N:]
else:
raise IOError("Not part of repo:%s ?!" % path)
def absPath(self , path):
repo_path = os.path.join( self.getRoot() , path)
if os.path.exists( repo_path ):
return repo_path
else:
raise IOError("No such entry in repo:%s" % path)
def checkout(self , ref):
self.repo.git.checkout( ref )
def getHeadSHA(self):
commit = self.repo.commit("HEAD")
return commit.hexsha
def runTests(self , cmd):
full_cmd = self.absPath( cmd )
if not os.path.isfile( full_cmd ):
raise IOError("Is not an executable file:%s" % cmd)
if not os.access( full_cmd , os.X_OK):
raise OSError("File not executable: %s" % cmd)
exit_code = subprocess.call( full_cmd )
if exit_code == 0:
return True
else:
return False
def install(self , target, files = [] , directories = []):
if os.path.exists( target ):
if not os.path.isdir( target ):
raise OSError("Target:%s already exists - and is not a directory" % target)
else:
os.makedirs( target )
for dir in directories:
repo_path = os.path.join( self.getRoot() , dir)
if not os.path.exists( repo_path ):
target_path = os.path.join( target , dir)
if not os.path.isdir( target_path ):
os.makedirs( target_path )
continue
if os.path.dirname( dir ):
target_path = os.path.join( target , os.path.dirname( dir ))
if not os.path.isdir( target_path ):
os.makedirs( target_path )
for dirpath , dirnames , filenames in os.walk( self.absPath(dir)):
target_path = os.path.join( target , self.relPath( dirpath ))
if not os.path.isdir( target_path ):
os.makedirs( target_path )
for file in filenames:
src_file = os.path.join( dirpath , file )
target_file = os.path.join( target_path , file )
shutil.copy( src_file , target_file )
for file in files:
target_file = os.path.join( target , file )
if os.path.dirname( file ):
target_path = os.path.join( target , os.path.dirname( file ))
if not os.path.isdir( target_path ):
os.makedirs( target_path )
shutil.copy( self.absPath( file ) , target_file )