def deploy(repo_name): for s in config: if s['repo_name'] == repo_name: git_repo = GitRepo(remote_url=s['repo_url'], local_folder=s['repo_name']) if s['deploy_type'] == 's3': server = S3Bucket(s['aws_key_id'], s['aws_key'], s['s3_bucket']) prev_hash = server.get_value(VERSION_FILE) if '' == prev_hash: files_to_upload = git_repo.all_files() files_to_delete = [] else: files_to_upload, files_to_delete = \ git_repo.changed_files(prev_hash) server.upload_files(files_to_upload, all_public=True) server.delete_files(files_to_delete) server.set_value(VERSION_FILE, git_repo.head_hash()) elif s['deploy_type'] == 'ssh': print 'did nothing' else: assert False, \ 'Wrong deploy type: %s. Only support S3 and SSH now' \ % s['deploy_type'] return
def deploy(repo_name): for s in config: if s['repo_name'] == repo_name: git_repo = GitRepo( remote_url=s['repo_url'], local_folder=s['repo_name']) if s['deploy_type'] == 's3': server = S3Bucket( s['aws_key_id'], s['aws_key'], s['s3_bucket']) prev_hash = server.get_value(VERSION_FILE) if '' == prev_hash: files_to_upload = git_repo.all_files() files_to_delete = [] else: files_to_upload, files_to_delete = \ git_repo.changed_files(prev_hash) server.upload_files(files_to_upload, all_public=True) server.delete_files(files_to_delete) server.set_value(VERSION_FILE, git_repo.head_hash()) elif s['deploy_type'] == 'ssh': print 'did nothing' else: assert False, \ 'Wrong deploy type: %s. Only support S3 and SSH now' \ % s['deploy_type'] return
def _clone_git(self, git_url): logger.debug('Cloning/pulling repository %s.', self.name) try: if os.path.exists(self.name): try: GitRepo(self.name).git.reset('--hard', 'origin') GitRepo(self.name).git.pull() logger.debug('Repository %s pulled.', self.name) except Exception: logger.error('Unable to pull invalid repository %s.', self.name) return generic_error(msg='Error updating Git Repo! ' + 'Please ensure the path ' + 'is a valid Git Repo.') else: os.makedirs(self.name) GitRepo.clone_from(git_url, self.name) logger.debug('Repository %s cloned.', self.name) except Exception: logger.error('Unable to access repository %s.', self.name) return generic_error(msg='Error cloning/pulling Git Repo! ' + 'Please ensure you have access.')
def get_git_repo(self, repo_url, local_repo): """ Get a git repository. Parameters ---------- repo_url: str The location of the git repository (an url if local is False, a local path otherwise) local_repo: bool If True, get the repository from a local directory instead of the web Returns ------- str The temporary path to which the repository has been copied GitRepo The repository object Raises ------ FileNotFoundError If repo_url is not an existing directory git.InvalidGitRepositoryError If the directory in repo_url is not a git repository git.GitCommandError If the url in repo_url is not a git repository, or access to the repository is denied """ project_path = tempfile.mkdtemp() if local_repo: project_path = os.path.join(tempfile.mkdtemp(), 'repo') try: shutil.copytree(repo_url, project_path) repo = GitRepo(project_path) except FileNotFoundError as e: shutil.rmtree(project_path) raise e except InvalidGitRepositoryError as e: shutil.rmtree(project_path) raise InvalidGitRepositoryError( f"\"{repo_url}\" is not a local git repository.") from e else: try: GitRepo.clone_from(repo_url, project_path) repo = GitRepo(project_path) except GitCommandError as e: shutil.rmtree(project_path) raise e return project_path, repo
def download(cls, manifest=None, localdir=None, version=None, override_existing=False, **kwargs): try: repo = GitRepo(localdir) except (InvalidGitRepositoryError, NoSuchPathError): repo = GitRepo.init(localdir) if manifest.git is not None: try: repo.create_remote('origin', manifest.git) except GitCommandError as e: pass try: if sys.stdout.isatty(): repo.remotes.origin.fetch(progress=GitProgressBar( label="%s@%s" % (str(manifest), version.version))) else: for fetch_info in repo.remotes.origin.fetch(): logger.debug( "Updated %s %s to %s" % (manifest.git, fetch_info.ref, fetch_info.commit)) # self.last_checked = datetime.now() except (GitCommandError, AttributeError) as e: logger.error("Could not fetch %s: %s" % (manifest.git, str(e))) if version.git_sha is not None: repo.git.checkout(version.git_sha)
def get_env(self, directory=None): """Determine environment name.""" if self.environment_override_name in self.env_vars: return self.env_vars[self.environment_override_name] if self.runway_config.get('ignore_git_branch', False): LOGGER.info('Skipping environment lookup from current git branch ' '("ignore_git_branch" is set to true in the runway ' 'config)') else: # These are not located with the top imports because they throw an # error if git isn't installed from git import Repo as GitRepo from git.exc import InvalidGitRepositoryError if directory is None: directory = self.module_root try: b_name = GitRepo( directory, search_parent_directories=True ).active_branch.name LOGGER.info('Deriving environment name from git branch %s...', b_name) return self.get_env_from_branch(b_name) except InvalidGitRepositoryError: pass LOGGER.info('Deriving environment name from directory %s...', self.env_root) return self.get_env_from_directory(os.path.basename(self.env_root))
def get_env(path, ignore_git_branch=False): """Determine environment name.""" if 'DEPLOY_ENVIRONMENT' in os.environ: return os.environ['DEPLOY_ENVIRONMENT'] if ignore_git_branch: LOGGER.info('Skipping environment lookup from current git branch ' '("ignore_git_branch" is set to true in the Runway ' 'config)') else: # These are not located with the top imports because they throw an # error if git isn't installed from git import Repo as GitRepo from git.exc import InvalidGitRepositoryError try: b_name = GitRepo(path, search_parent_directories=True).active_branch.name LOGGER.info('Deriving environment name from git branch %s...', b_name) return get_env_from_branch(b_name) except InvalidGitRepositoryError: pass except TypeError: LOGGER.warning('Unable to retrieve the current git branch name!') LOGGER.warning('Typically this occurs when operating in a ' 'detached-head state (e.g. what Jenkins uses when ' 'checking out a git branch). Set the ' 'DEPLOY_ENVIRONMENT environment variable to the ' 'name of the logical environment (e.g. "export ' 'DEPLOY_ENVIRONMENT=dev") to bypass this error.') sys.exit(1) LOGGER.info('Deriving environment name from directory %s...', path) return get_env_from_directory(os.path.basename(path))
def get_env(path, ignore_git_branch=False): """Determine environment name.""" if 'DEPLOY_ENVIRONMENT' in os.environ: return os.environ['DEPLOY_ENVIRONMENT'] if ignore_git_branch: LOGGER.info('Skipping environment lookup from current git branch ' '("ignore_git_branch" is set to true in the runway ' 'config)') else: # These are not located with the top imports because they throw an # error if git isn't installed from git import Repo as GitRepo from git.exc import InvalidGitRepositoryError try: b_name = GitRepo(path, search_parent_directories=True).active_branch.name LOGGER.info('Deriving environment name from git branch %s...', b_name) return get_env_from_branch(b_name) except InvalidGitRepositoryError: pass LOGGER.info('Deriving environment name from directory %s...', path) return get_env_from_directory(os.path.basename(path))
def checkout(ctx, self, version=None, retry=False): """Checkout a version of the repository.""" if ctx.dont_checkout: return if version is None: version = self.version if self._type == RepositoryType.ARCH: return elif self._type == RepositoryType.PLAT and self._source == UNIKRAFT_CORE: return elif version is not None: try: repo = GitRepo(self._localdir) except (NoSuchPathError, InvalidGitRepositoryError): logger.debug("Attempting to checkout %s before update!" % self) # Allow one retry if retry is False: self.update() self.checkout(version, True) return try: # If this throws an exception, it means we have never checked # out the repository before. commit_hash = str(repo.head.commit) # Determine if the repository has already been checked out at # this version if commit_hash.startswith(version) \ or version in self._known_versions.keys() and self._known_versions[version] == commit_hash: logger.debug("%s already at %s" % (self._name, version)) return except ValueError as e: pass logger.debug("Checking-out %s@%s..." % (self._name, version)) # First simply attempting what was specified try: repo.git.checkout(version) except GitCommandError as e1: # Don't try well-known branches with well-known RELEASE prefix: if version != BRANCH_MASTER and version != BRANCH_STAGING: try: repo.git.checkout('RELEASE-%s' % version) except GitCommandError as e2: if not ctx.ignore_checkout_errors: logger.error("Could not checkout %s@%s: %s" % (self._name, version, str(e2))) sys.exit(1) elif not ctx.ignore_checkout_errors: logger.error("Could not checkout %s@%s: %s" % (self._name, version, str(e1))) sys.exit(1)
def __init__(self, version: str): from git import Repo as GitRepo from svn.remote import RemoteClient as SvnRepo self.git_repo = None self.svn_repo = None if version is BLENDER_VERSION_MASTER: self.git_repo = GitRepo(Blender.GIT_BASE_URL) self.svn_repo = SvnRepo(os.path.join(Blender.SVN_BASE_URL, Blender.SVN_MASTER)) else: self.match = BLENDER_VERSION_REGEX.match(version) if self.match: matching_git_tags = [tag for tag in GitRepo(Blender.GIT_BASE_URL).tags if tag == self.match.group(0)] self.git_repo = matching_git_tags[0] if matching_git_tags else None if self.git_repo: pass else: raise Exception(f"Blender {version} does not exist in git") svn_version_tag = (f"blender-{self.match.group(1)}" f"{self.match.group(2) if not self.match.group(2).startswith("-rc")}-release") blender_svn_tags = os.path.join(Blender.SVN_BASE_URL, Blender.SVN_TAGS) matching_svn_tags = [os.path.join(blender_svn_tags, tag) for tag in SvnRepo(os.path.join(Blender.SVN_BASE_URL, Blender.SVN_TAGS)).list() if tag == svn_version_tag] self.svn_repo = SvnRepo(matching_svn_tags[0]) if matching_svn_tags else None
def __init__(self, full_name): self.full_name = full_name try: self.repo = GitRepo('repos/{}'.format(full_name)) except NoSuchPathError: self.repo = GitRepo.clone_from( '[email protected]:{}.git'.format(full_name), 'repos/{}'.format(full_name))
def meta_repo(): """Get a reference to the meta repo that contains the templates and lists of students Returns: git.Repo: metadata repo """ config = Config.load_config() return GitRepo(config.meta_path)
def clone_or_pull(self): try: if os.path.isdir(self.clone_dir): git_repo = GitRepo(self.clone_dir) git_repo.remotes.origin.pull("refs/heads/master:refs/heads/origin") else: git_repo = GitRepo.clone_from(self.git_url, self.clone_dir) except GitError: raise exceptions.GitException(self, "Cannot clone or pull Git repository") self.git_repo = git_repo
def git_initialise(): try: # attempt to open existing repository repository = GitRepo(GIT_REPO_PATH) print(f"Opened existing repository in '{GIT_REPO_PATH}'") except (InvalidGitRepositoryError, NoSuchPathError): # initialise bare repository os.makedirs(GIT_REPO_PATH, exist_ok=True) repository = GitRepo.init(GIT_REPO_PATH) print(f"Initialised new repository in '{GIT_REPO_PATH}'") return repository
def checkout(self, full_path: pathlib.Path): """Retrieve Blender code from Git Keyword Arguments: full_path {pathlib.Path} -- place to clone code to """ try: repo = GitRepo(str(full_path)) except: GitRepo.clone_from(self.BASE_URL, str(full_path)) repo = GitRepo(str(full_path)) repo.heads.master.checkout() repo.remotes.origin.pull() if self.tag is not None: repo.git.checkout(self.tag) repo.git.submodule('update', '--init', '--recursive') for submodule in repo.submodules: submodule_repo = submodule.module() submodule_repo.heads.master.checkout() submodule_repo.remotes.origin.pull() if self.tag is not None: try: submodule_repo.git.checkout(self.tag) except: pass
def __init__(self, repo_path: str): """ Initialize GitWrapper. Should be initialized by ZenML Repository. Args: repo_path: Raises: InvalidGitRepositoryError: If repository is not a git repository. NoSuchPathError: If the repo_path does not exist. """ # TODO [ENG-163]: Raise ZenML exceptions here instead. self.repo_path: str = repo_path self.git_root_path: str = os.path.join(repo_path, GIT_FOLDER_NAME) self.git_repo = GitRepo(self.repo_path)
def get_env(path, ignore_git_branch=False, prompt_if_unexpected=False): """Determine environment name. Args: path (str): Path to check for deploy environment name. ignore_git_branch (bool): Skip checking for git branch name. (*default:* ``False``) prompt_if_unexpected (bool): If the branch name is an unexpected format/value, the user will be prompted if they would like to enter a different deploy environment. (*default:* ``False``) Returns: str: Deploy environment. """ if 'DEPLOY_ENVIRONMENT' in os.environ: return os.environ['DEPLOY_ENVIRONMENT'] if ignore_git_branch: LOGGER.info('Skipping environment lookup from current git branch ' '("ignore_git_branch" is set to true in the Runway ' 'config)') else: # These are not located with the top imports because they throw an # error if git isn't installed from git import Repo as GitRepo # pylint: disable=import-outside-toplevel from git.exc import InvalidGitRepositoryError # pylint: disable=import-outside-toplevel try: b_name = GitRepo( path, search_parent_directories=True ).active_branch.name LOGGER.info('Deriving environment name from git branch %s...', b_name) return get_env_from_branch(b_name, prompt_if_unexpected) except InvalidGitRepositoryError: pass except TypeError: LOGGER.warning('Unable to retrieve the current git branch name!') LOGGER.warning('Typically this occurs when operating in a ' 'detached-head state (e.g. what Jenkins uses when ' 'checking out a git branch). Set the ' 'DEPLOY_ENVIRONMENT environment variable to the ' 'name of the logical environment (e.g. "export ' 'DEPLOY_ENVIRONMENT=dev") to bypass this error.') sys.exit(1) LOGGER.info('Deriving environment name from directory %s...', path) return get_env_from_directory(os.path.basename(path))
def get_git_repo(repo_path, init=False): if os.path.isdir(repo_path): try: return GitRepo(repo_path) except InvalidGitRepositoryError: if init: return GitRepo.init(repo_path) elif init: try: create_path(repo_path) return get_git_repo(repo_path, init=init) except FileNotFoundError: pass raise ValueError('Could not create a repo on this path {}'.format(repo_path))
def __attrs_post_init__(self): """Initialize computed attributes.""" #: Configure Renku path. path = Path(self.renku_home) if not path.is_absolute(): path = self.path / path path.relative_to(path) self.renku_path = path #: Create an instance of a Git repository for the given path. try: self.git = GitRepo(str(self.path)) except InvalidGitRepositoryError: self.git = None
def get_commit_hash(self): logger.debug('Getting commit hash for repository %s.', self.name) if not os.path.exists(self.name): logger.error('When getting commit hash, path of %s not found.', self.name) return no_code_dir_error() repo = GitRepo(self.name) sha = repo.head.object.hexsha logger.debug('Commit hash for repository %s obtained.', self.name) return {"sha": sha}
def clone_student_repo(student, dest=None): """Clones a student repo into a temporary directory Arguments: student (Student) Returns: git.Repo: repo at the cloned path """ logger.info('Cloning repo: %s', student.github) if dest is None: dest = tempfile.mkdtemp() repo_url = student.repo_url logger.debug('%s -> %s', repo_url, dest) GitRepo.clone_from(repo_url, dest) return GitRepo(dest)
def get(self, request, *args, **kwargs): if 'remove' in request.GET.keys(): self.get_object().delete() return redirect(self.success_url) elif 'generate' in request.GET.keys(): os.system( f"pycco repos/{self.get_object().relative_dir}/**/*.py -p") #pass elif 'update' in request.GET.keys(): repo = GitRepo( os.path.join(settings.REPOS_DIR, self.get_object().relative_dir)) o = repo.remotes.origin o.pull() return super().get(request, *args, **kwargs)
def __init__(self, repo_path: Text): """ Initialize GitWrapper. Should be initialize by ZenML Repository. Args: repo_path: Raises: InvalidGitRepositoryError: If repository is not a git repository. NoSuchPathError: If the repo_path does not exist. """ # TODO: [LOW] Raise ZenML exceptions here instead. self.repo_path: Text = repo_path self.git_root_path: Text = os.path.join(repo_path, GIT_FOLDER_NAME) self.git_repo: GitRepo = GitRepo(self.repo_path) # TODO: [LOW] Check whether this is necessary. assert not self.git_repo.bare
def clone_or_pull(self): if not self.remote_available(): raise exceptions.GitException( self, "Cannot clone or pull Git repository") try: if os.path.isdir(self.clone_dir): git_repo = GitRepo(self.clone_dir) git_repo.remotes.origin.fetch(tags=True, force=True) git_repo.git.reset("--hard", "origin/master") else: git_repo = GitRepo.clone_from(self.git_url, self.clone_dir) except GitError: raise exceptions.GitException( self, "Cannot clone or pull Git repository") self.git_repo = git_repo
def __init__(self, git_url, email, schema_type): super(Repo, self).__init__() parsed_git = giturlparse.parse(git_url) self.git_url = git_url self.owner = self.find_owner(parsed_git) self.name = parsed_git.name self.email = email if os.path.isdir(self.clone_dir): self.git_repo = GitRepo(self.clone_dir) else: self.git_repo = None self.current_tag = None self.cache_latest_valid_tag = None if schema_type not in self.SCHEMA_TYPES: raise exceptions.InvalidSchemaTypeException( self, "`%s` is not a supported schema type. Supported: %s" % (schema_type, ",".join(self.SCHEMA_TYPES)), ) self.schema_type = schema_type
def update(self): """Update this particular repository.""" repo = None try: repo = GitRepo(self._localdir) # Not a repository? No problem, let's clone it: except (InvalidGitRepositoryError, NoSuchPathError) as e: repo = GitRepo.init(self._localdir) repo.create_remote('origin', self._source) try: for fetch_info in repo.remotes.origin.fetch(): logger.debug("Updated %s %s to %s" % (self._source, fetch_info.ref, fetch_info.commit)) self._last_checked = datetime.now() except (GitCommandError, AttributeError) as e: logger.error("Could not fetch %s: %s" % (self._source, str(e)))
def is_type(cls, origin_url=None): if origin_url is None: return False try: if origin_url.startswith("file://"): origin_url = origin_url[7:] GitRepo(origin_url, search_parent_directories=True) return True except Exception: pass try: GitCmd().ls_remote(origin_url) return True except Exception: pass return False
def is_type(cls, source=None): if source is None: return False try: if source.startswith("file://"): source = source[7:] GitRepo(source, search_parent_directories=True) return True except Exception: pass try: GitCmd().ls_remote(source) return True except Exception: pass return False
def check_create_local_repo(config) -> GitRepo: """Create a local repo for saving state and push it to the user's personal ref. Checkout any existing version on the user's personal remote, or create a new commit""" path = Path(config.cwd, "_state") if not path.exists(): os.mkdir(path) try: repo = GitRepo(path) if "origin" not in [r.name for r in repo.remotes] and config._state_ref: repo.create_remote( 'origin', f"ssh://{config.GERRIT_HOST[8:]}/{config.GERRIT_STATE_PATH}") except exc.InvalidGitRepositoryError: repo = GitRepo.init(path) if config._state_ref: repo.create_remote( 'origin', f"ssh://{config.GERRIT_HOST[8:]}/{config.GERRIT_STATE_PATH}") fetch_and_checkout(config, repo) state_path = Path(repo.working_tree_dir, "state.bin") if not state_path.exists(): with open(state_path, 'wb') as state_file: pickle.dump({}, state_file) repo.index.add('state.bin') repo.index.commit("Empty state") if config._state_ref: repo.remotes.origin.push(['-f', f"HEAD:{config._state_ref}"]) if not config._state_ref: print( "\nWARN: Unable to create git remote for state!\n" "WARN: State will only be saved locally to _state/state.bin.\n" "INFO: Please configure an ssh user in ~/.ssh/config for your gerrit host\n" "INFO: as set by 'GERRIT_HOST' in config.yaml in order to save state in gerrit.\n" ) return repo
def check_repo(self, repo_url, git_token=None, local_repo=False): """ Check git token validity for the repository Parameters ---------- repo_url: str The location of a git repository (an url if local_repo is False, a local path otherwise) git_token: str, optional Git personal access token to authenticate to the git server local_repo: bool, optional If True, get the repository from a local directory instead of the web Returns ------- bool True if the git token is valid for the repository, False otherwise """ if local_repo: try: GitRepo(repo_url) except InvalidGitRepositoryError: return False, 'InvalidGitRepositoryError' except NoSuchPathError: return False, 'NoSuchPathError' else: g = git.cmd.Git() if git_token is not None and len(git_token) > 0: repo_url = repo_url.replace('https://', f'https://*****:*****@') try: g.ls_remote(repo_url) except GitCommandError: return False, 'GitCommandError' return True, None
def __init__(self, name, repopath, secret, sshkey='', action='default', branch='master', script=None): self.title = name + branch self.name = name self.repo = GitRepo(repopath) action = action.lower() if action in ['default', 'script']: self.action = action else: raise ValueError('action for {} is not supported'.format(name)) if action == 'script': if script is None: raise ValueError('Script not specified for {}'.format(name)) elif not app.config['TESTING'] and not isabs(script): raise ValueError( 'Script for {} is not a absolute path'.format(name)) self.script = script if sshkey != '' and isfile(sshkey): self.ssh_cmd = 'ssh -i {}'.format(sshkey) else: self.ssh_cmd = 'ssh' self.branch = branch self.repo.git.checkout('-B', self.branch) self.__update() self.secret = secret
def __init__(self, config): self.config = config self.repo = GitRepo.factory(self.config.general_options["git_repo"]) self.log_dir = self.config.general_options["git_repo"] + "/balog" if not os.path.isdir(self.log_dir): os.mkdir(self.log_dir)