Example #1
0
def _resolve_git(url: str, destination: str) -> str:
    """
    Clone a gh:// url to a local folder.

    Args:
        url: git url, the format is gh://organization/repo
        destination: the destination directory for the repo, must be empty

    Returns:
        The destination directory.

    Raises:
        ValueError: The github url is not valid.
        FileExistsError: The destination directory is not empty.
    """
    url = url.lower()
    if not url.startswith('gh://'):
        raise ValueError(f'{url} is not a valid gh:// url.')
    if len(os.listdir(destination)):
        raise FileExistsError(
            f'The working directory {destination} is not empty!')
    parsed = urlparse(url)
    org, repo = parsed.netloc, parsed.path[1:]
    git_url = f'git://github.com/{org}/{repo}.git'
    try:
        Repo.clone_from(git_url, destination)
    except Exception as e:
        raise ValueError(f'The github url can not be cloned. Message: {e}')
    logger.info(f'Cloned {url} to {destination}')
    return destination
Example #2
0
def rbenv_install(version: str = '2.4.2', force:bool = False):
    logger.debug('verifying that dependencies are installed')
    # Centos 7 Deps
    # git bzip2 gcc make openssl-devel readline-devel zlib-devel docker
    assert all([which(pkg) for pkg in ('git', 'bzip2', 'gcc', 'docker')])

    # Get paths
    rbenv = path.join(local_path, '.rbenv')
    plugins = path.join(rbenv, 'plugins')
    ruby_build = path.join(plugins, 'ruby-build')
    rbenv_bin = path.join(rbenv, 'bin', 'rbenv')
    gem = path.join(rbenv, 'shims', 'gem')

    # Setup paths
    if not path.exists(rbenv):
        assert git
        git.clone_from('https://github.com/rbenv/rbenv.git', rbenv)
    if not path.exists(plugins):
        mkdir(plugins)
    if not path.exists:
        assert git
        git.clone_from('https://github.com/rbenv/ruby-build.git', ruby_build)

    #
    logger.debug('Installing ruby version {}'.format(version))
    assert subprocess.Popen([rbenv_bin, 'install', version, '-f' if force else '-s']).wait() == 0
    logger.debug('Setting ruby version {} as global'.format(version))
    assert subprocess.Popen([rbenv_bin, 'global', version]).wait() == 0
    logger.debug('installing bundle')
Example #3
0
def CloneGitRepo(link,dir):
    try:
        from git.repo.base import Repo
    except ImportError:
        print("Please install GitPython package first!")
        return None
    Repo.clone_from(link, dir, progress=Progress())
Example #4
0
    def install_playbook(self, **kwargs):
        """
        Validates that self._playbook is a valid path or url.
        If it is a url git clone to /tmp. If it has a requires file
        install dependencies.
        """
        logger.debug('Installing playbook...')
        if "url" in kwargs:
            # Should probably extract the playbook name from the playbook
            # URL
            Repo.clone_from(
                kwargs["url"],
                self.path
            )
        elif "path" in kwargs:
            if os.path.exists(kwargs["path"]):
                path = None
                if os.path.isfile(kwargs["path"]):
                    path = os.path.dirname(kwargs["path"])
                else:
                    path = kwargs["path"]

                os.symlink(os.path.realpath(path), self.path,)
            else:
                raise ValueError(
                    'The path value provided (%s) does not exists',
                    kwargs['path']
                )

        assert os.path.exists(self.path)
Example #5
0
def test_get_pipeline_returns_none_if_non_existent(tmp_path: str) -> None:
    """Check get_pipeline returns None if it doesn't exist"""
    Repo.init(tmp_path)
    repo = Repository(str(tmp_path))
    repo.set_active_stack("local_stack")
    our_pipeline = repo.get_pipeline("not_a_pipeline")
    assert our_pipeline is None
def clone_repo(start=0, end=100000):
    """
    Clones the GitHub repositories if the repositories has not been cloned initially. 
    
    Parameters:
    start(int): This is an integer that specifies which URL cloning should start from in the CSV file  of     the GitHub repositories. Defaults to zero.
    end(int): This is an integer that specifies which URL cloning should end in the CSV file  of       the GitHub repositories. Defaults to 100000, which is the last URL.
    
    Returns:
    count(int): The number of repository cloned.
    
    """
    repo_list = repo_url['URLs']
    count = 0

    for url in repo_list[start:end]:
        url = str(url)
        name = url.rsplit(
            '/', 2)  #get the repo name (last 2 part) of the repository url
        last = name[-2] + '-' + name[-1]
        try:
            if not os.path.exists(last):
                os.mkdir(last)  #Make folder for a repo if it does not exist
                repo = str(url) + '.git'
                folder = r'repos'
                Repo.clone_from(repo, last)
                count += 1
                print('cloned ', repo)
        except:
            continue
    return count
Example #7
0
def test_getting_active_stack_returns_local_stack(tmp_path: str, ) -> None:
    """Check getting the active stack"""
    Repo.init(tmp_path)
    repo = Repository(str(tmp_path))
    repo.set_active_stack("local_stack")
    assert repo.get_active_stack() == LocalService().get_stack("local_stack")
    assert isinstance(repo.get_active_stack(), BaseStack)
Example #8
0
def main(hub, archived, archived_only, forks, depth, prune, org):
    """
    Clone an entire GitHub organization into the current directory.
    Each repo becomes a subdirectory.
    """
    if archived_only:
        archived = True
    dir_names = set()
    for repo in hub.organization(org).repositories():
        if repo.fork and not forks:
            continue
        if repo.archived and not archived:
            continue
        if not repo.archived and archived_only:
            continue
        dir_name = repo.name
        dir_name = dir_name.lstrip("-")     # avoid dirname/option confusion
        dir_names.add(dir_name)
        if os.path.exists(dir_name):
            continue

        print("Cloning {}".format(repo.full_name))
        clone_args = {}
        if depth:
            clone_args['depth'] = depth
        Repo.clone_from(repo.ssh_url, dir_name, **clone_args)

    if prune:
        for dir_name in os.listdir("."):
            if os.path.isdir(dir_name):
                if dir_name not in dir_names:
                    print("Pruning {}".format(dir_name))
                    shutil.rmtree(dir_name)
Example #9
0
 def clone():
     try:
         print(f'Using existing repo in {settings.REPOSITORY_DIR}')
         Repo(settings.REPOSITORY_DIR)
     except:
         print('Repo does not exist')
         Repo.clone_from(settings.GIT_REPOSITORY, settings.REPOSITORY_DIR)
Example #10
0
def CheckFiles():
    if os.path.isdir('scripts'):
        print ("File Does exist")
        checkall()
    else:
        print ("File not exist, Creating")
        Repo.clone_from("https://github.com/ChokunPlayZ/ChatSpammer-Data.git", "scripts")
Example #11
0
def clone(where, what):

    from pathlib import Path
    dest_fn = Path(GLOBS['kk_code_dir'], what)

    if dest_fn.exists():
        return (False, str(dest_fn))
        """ans = None
        while ans not in "ynYN":
            ans = input("The destination folder, {dest_fn}, exists already. Should we overwrite it?")
        if ans in "nN":"""

    if not dest_fn.exists():
        print("Module doesn't exist. Attempting to load it from GitHub...")

        if where[0] == '@':
            where = where[1:]

        url = "https://github.com/%s/%s" % (where, what)

        print(f"Cloning '{where}/{what}' into '{dest_fn}' ...")

        from git.repo.base import Repo

        fld = Path(GLOBS['kk_code_dir'])
        if not fld.exists():
            fld.mkdir()

        Repo.clone_from(url, dest_fn)
        return (True, str(dest_fn))
Example #12
0
    def run(self):
        """
        Removes, if exists, the current folder and clones the repository in that folder that you have passed by
        - argument with space like delimiter (ie: !git_clone https://github.com/bilardi/sceptre-git-clone-hook my-folder)
        - sceptre_user_data with properties GitRepository and RepositoryFolder
            Raise (Exception): when the system does not find the arguments repository and folder in argument or sceptre_user_data
        """
        if self.argument and self.DELIMITER in self.argument:
            repository, folder = self.argument.split(self.DELIMITER, 1)
            self.logger.debug(
                "[{}] Parameters parsed from the argument".format(self.NAME))
        elif self.stack.sceptre_user_data and 'GitRepository' in self.stack.sceptre_user_data and 'RepositoryFolder' in self.stack.sceptre_user_data:
            repository = self.stack.sceptre_user_data.get('GitRepository', {})
            folder = self.stack.sceptre_user_data.get('RepositoryFolder', {})
            self.logger.debug(
                "[{}] Parameters parsed from sceptre_user_data".format(
                    self.NAME))
        else:
            raise Exception(
                "Parameters GitRepository and RepositoryFolder could not be parsed from sceptre_user_data or argument"
            )

        if os.path.exists(folder):
            shutil.rmtree(folder)
            self.logger.info(
                "[{}] Removed the local repository folder {}".format(
                    self.NAME, folder))

        Repo.clone_from(repository, folder)
        self.logger.info(
            "[{}] Downloaded the remote repository in the folder {}".format(
                self.NAME, folder))
Example #13
0
 def parse(self, response, **kwargs):
     # inspect_response(response, self)
     links = response.xpath('/html/body/div/main/div/div/div/div/div[@class="org-repos repo-list"]/ul/li/div/div/h3[@class="wb-break-all"]/a[@data-hovercard-type="repository"]/@href').getall()
     for link in links:
         l = urljoin(self.start_urls[0], link)
         dir = Path.joinpath(self.output_dir, l.split('/')[-1])
         os.makedirs(dir, exist_ok=True)
         Repo.clone_from(l, dir, branch='master')
def test_get_artifact_store_returns_artifact_store(tmp_path: str) -> None:
    """Test get_artifact_store returns artifact store."""
    Repo.init(tmp_path)
    repo = Repository(str(tmp_path))
    local_service = repo.get_service()
    artifact_store = local_service.get_artifact_store(LOCAL_ARTIFACT_STORE_NAME)
    assert artifact_store is not None
    assert isinstance(artifact_store, BaseArtifactStore)
Example #15
0
def test_get_pipelines_returns_list(tmp_path: str) -> None:
    """Check get_pipelines returns a list"""
    Repo.init(tmp_path)
    repo = Repository(str(tmp_path))
    repo.set_active_stack("local_stack")
    our_pipelines = repo.get_pipelines()
    assert our_pipelines is not None
    assert isinstance(our_pipelines, list)
def test_get_metadata_store_returns_metadata_store(tmp_path: str) -> None:
    """Test get_metadata_store returns metadata store."""
    Repo.init(tmp_path)
    repo = Repository(str(tmp_path))
    local_service = repo.get_service()
    metadata_store = local_service.get_metadata_store(LOCAL_METADATA_STORE_NAME)
    assert metadata_store is not None
    assert isinstance(metadata_store, BaseMetadataStore)
def test_get_orchestrator_returns_orchestrator(tmp_path: str) -> None:
    """Test get_orchestrator returns orchestrator"""
    Repo.init(tmp_path)
    repo = Repository(str(tmp_path))
    local_service = repo.get_service()
    orchestrator = local_service.get_orchestrator(LOCAL_ORCHESTRATOR_NAME)
    assert orchestrator is not None
    assert isinstance(orchestrator, BaseOrchestrator)
def test_get_stack_raises_exception_when_key_does_not_exist(
    tmp_path: str,
) -> None:
    """Test get_stack raises exception when key does not exist."""
    Repo.init(tmp_path)
    repo = Repository(str(tmp_path))
    local_service = repo.get_service()
    with pytest.raises(DoesNotExistException):
        local_service.get_stack("made_up_stack")
Example #19
0
def test_getting_the_active_service_returns_local_service(
    tmp_path: str, ) -> None:
    """Check getting the active service"""
    Repo.init(tmp_path)
    repo = Repository(str(tmp_path))
    assert repo.get_service() is not None
    assert isinstance(repo.get_service(), BaseComponent)
    assert isinstance(repo.get_service(), LocalService)
    assert repo.get_service() == repo.service
Example #20
0
def test_get_pipeline_returns_same_when_stack_specified(tmp_path: str) -> None:
    """Check get_pipeline returns the same if stack specified"""
    Repo.init(tmp_path)
    repo = Repository(str(tmp_path))
    repo.set_active_stack("local_stack")
    our_pipeline_default = repo.get_pipeline("pipeline_1")
    our_pipeline_local = repo.get_pipeline("pipeline_1",
                                           stack_key="local_stack")
    assert our_pipeline_default == our_pipeline_local
def test_register_orchestrator_works_as_expected(tmp_path: str) -> None:
    """Test register_orchestrator method registers an orchestrator as expected."""
    Repo.init(tmp_path)
    repo = Repository(str(tmp_path))
    local_service = repo.get_service()
    orchestrator = LocalOrchestrator()
    local_service.register_orchestrator("local_orchestrator_2", orchestrator)
    assert local_service.get_orchestrator("local_orchestrator_2") is not None
    local_service.delete_orchestrator("local_orchestrator_2")
Example #22
0
    def clone_git(self, github_repo, branch):

        tmpRoot = self.root + '/' + github_repo + '/' + branch
        git_url = self.github_base + github_repo + ".git"

        if not os.path.isdir(tmpRoot):
            os.makedirs(os.path.dirname(tmpRoot), exist_ok=True)
            branch_opt = '--branch ' + branch
            Repo.clone_from(git_url, tmpRoot, multi_options=[branch_opt])
def test_register_stack_raises_exception_when_key_already_exists(
    tmp_path: str,
) -> None:
    """Test register_stack raises exception when key already exists."""
    Repo.init(tmp_path)
    repo = Repository(str(tmp_path))
    local_service = repo.get_service()
    local_stack1 = local_service.get_stack("local_stack")
    with pytest.raises(AlreadyExistsException):
        local_service.register_stack("local_stack", local_stack1)
Example #24
0
def setup_bkrepo(repo_url, repo_path):
    try:
        repo = Repo(repo_path)
    except NoSuchPathError:
        logging.info('git clone: {}'.format(repo_url))
        repo = Repo.clone_from(repo_url, repo_path)
    else:
        logging.info('git pull: {}'.format(repo_url))
        repo.remote().pull()
    return repo
def test_register_metadata_store_with_existing_key_fails(tmp_path: str) -> None:
    """Test register_metadata_store with existing key fails."""
    Repo.init(tmp_path)
    repo = Repository(str(tmp_path))
    local_service = repo.get_service()
    with pytest.raises(AlreadyExistsException):
        local_service.register_metadata_store(
            LOCAL_METADATA_STORE_NAME,
            local_service.get_metadata_store(LOCAL_METADATA_STORE_NAME),
        )
def test_delete_orchestrator_works(tmp_path: str) -> None:
    """Test delete_orchestrator works as expected."""
    Repo.init(tmp_path)
    repo = Repository(str(tmp_path))
    local_service = repo.get_service()
    orchestrator = LocalOrchestrator()
    local_service.register_orchestrator("local_orchestrator_2", orchestrator)
    local_service.delete_orchestrator("local_orchestrator_2")
    with pytest.raises(DoesNotExistException):
        local_service.get_orchestrator("local_orchestrator_2")
def test_register_orchestrator_with_existing_key_fails(tmp_path: str) -> None:
    """Test register_orchestrator with existing key fails."""
    Repo.init(tmp_path)
    repo = Repository(str(tmp_path))
    local_service = repo.get_service()
    with pytest.raises(AlreadyExistsException):
        local_service.register_orchestrator(
            LOCAL_ORCHESTRATOR_NAME,
            local_service.get_orchestrator(LOCAL_ORCHESTRATOR_NAME),
        )
def clone_repo(owner, repo, path_to_clone):
    try:
        if (not os.path.exists(path_to_clone)):
            git_url = "https://github.com/" + owner + "/" + repo
            Repo.clone_from(url=git_url, to_path=path_to_clone)
        else:
            print("Jumped repo because its folder already exists: " +
                  repo['id'])
    except:
        print("Error for: Owner -- {0} Repo -- {1}".format(owner, repo))
        print(sys.exc_info()[0])
def test_local_service_can_access_orchestrators(tmp_path: str) -> None:
    """Local Service can access orchestrators."""
    Repo.init(tmp_path)
    repo = Repository(str(tmp_path))
    local_service = repo.get_service()
    assert local_service.orchestrators is not None
    assert isinstance(local_service.orchestrators, dict)
    assert isinstance(
        local_service.orchestrators[LOCAL_ORCHESTRATOR_NAME],
        BaseOrchestrator,
    )
    def include_git(self, github_repo, file_path, branch, local_path):

        tmpRoot = self.root + '/' + github_repo + '/' + branch
        git_url = self.github_base + github_repo + ".git"

        if not os.path.isdir(tmpRoot):
            os.makedirs(os.path.dirname(tmpRoot), exist_ok=True)
            branch_opt = '--branch ' + branch
            Repo.clone_from(git_url, tmpRoot, multi_options=[branch_opt,'--depth 1'])

        self.include_file(tmpRoot + "/" + file_path, local_path)
Example #31
0
 def __init__(self, config: Config, path: str = None) -> None:
     self.__config: Config = config
     try:
         if not path:
             path = config.root_path
         self.__repo = Repo(path)
         assert not self.__repo.bare
         self.origin = self.__repo.remote('origin')
     except InvalidGitRepositoryError:
         log_error("Path " + path + " is not a git repository. Please go to valid git repository!")
         sys.exit()
Example #32
0
def main(hub, forks, depth, org):
    for repo in hub.organization(org).iter_repos():
        if repo.fork and not forks:
            continue
        dir_name = repo.name
        dir_name = dir_name.lstrip("-")     # avoid dirname/option confusion
        if os.path.exists(dir_name):
            continue

        print(repo.full_name)
        clone_args = {}
        if depth:
            clone_args['depth'] = depth
        Repo.clone_from(repo.ssh_url, dir_name, **clone_args)
Example #33
0
def get_results(repos_dir, settings):
    paths = []
    collect_only_dirty = settings.get('dirty', False)
    show_full_path = settings.get('full_path', False)
    for path, dirs, files in os.walk(repos_dir):
        if is_git_dir(path):
            repo = Repo(path)
            path = os.path.dirname(path)
            if show_full_path:
                path = PATH.format(heading=os.path.join(repos_dir, path))
            else:
                path = PATH.format(heading=os.path.relpath(path, repos_dir))
            bare = repo.bare
            dirty = repo.is_dirty()
            if (collect_only_dirty and dirty) or (not collect_only_dirty):
                paths.append({'path': path, 'bare': bare, 'dirty': dirty})
    return paths
def load_config():

    git_config = Repo().config_reader()
    pivotal_items = git_config.items('pivotal')

    token = None
    filter = None
    project_id = None

    for item_name, item_value in pivotal_items:
        if item_name == 'token':
            token = item_value

        if item_name == 'filter':
            filter = item_value

        if item_name == 'projectid':
            project_id = item_value

    return token, filter, project_id
Example #35
0
    def __init__(self , **kwargs):
        url = kwargs.get("url")
        local_path = kwargs.get("local_path")
        self.work_dir = None

        if url is None and local_path is None:
            raise ValueError("Must supply one of url and local_path")
            
        if url and local_path:
            raise ValueError("Can only supply one of url and local_path")
            
        if url:
            self.work_dir = tempfile.mkdtemp( )
            self.clone( url )
        else:
            self.work_dir = None
            self.repo = Repo( local_path )
            origin = self.repo.remote( name = "origin")
            origin.fetch( )
def make_origin():
    origin_root = tempfile.mkdtemp()
    origin_base = "origin"
    origin_url = os.path.join( origin_root , origin_base)
    os.makedirs( origin_url )
    repo = Repo.init( path = origin_url )

    add_file( repo , "file.txt" , "Content")
    add_file( repo , "tests/run_not_executable" , "content")
    add_file( repo , "tests/subdir/file" , "Content")
    add_executable( repo , "tests/run_OK" , "#!/bin/bash\nexit 0\n")
    add_executable( repo , "tests/run_fail" , "#!/bin/bash\nexit 1\n")

    repo.git.commit(m = "message")

    repo.git.branch("version2")
    repo.git.checkout("version2")
    with open( os.path.join(origin_url , "file2.txt") , "w") as f:
        f.write("File with content")
    repo.git.add("file2.txt")
    repo.git.commit(m = "message")

    return repo
Example #37
0
    def git_repo(self, request, github_repo, branch=None, remote='origin', checkout_root=None):
        """
        py.test fixture to clone a GitHub based repo onto the local disk.

        Arguments:
            github_repo (:class:`~github3.GitHub`): The repo to read from
            branch (str): The branch to check out

        Returns:
            A :class:`~git.repo.base.Repo` object, with the master branch checked out
            and up to date with the remote.
        """
        if checkout_root is None:
            checkout_root = request.config.option.checkout_root

        if not os.path.exists(checkout_root):
            os.makedirs(checkout_root)

        repo_dir = os.path.join(
            os.path.join(checkout_root, github_repo.owner.name),
            github_repo.name
        )

        if github_repo.private:
            repo_url = github_repo.ssh_url
        else:
            repo_url = github_repo.clone_url

        if not os.path.exists(repo_dir):
            repo = Repo.clone_from(repo_url, repo_dir)
        else:
            repo = Repo(repo_dir)

        if github_repo not in SYNCED:

            try:
                remote_obj = repo.remote(remote)
            except ValueError:
                repo.create_remote(remote, repo_url)
                remote_obj = repo.remote(remote)

            if remote_obj.fetch != repo_url:
                remote_obj.set_url(repo_url)

            remote_obj.fetch()
            SYNCED.add(github_repo)

        if branch is None:
            branch = github_repo.default_branch

        head = repo.head
        remote_branch = RemoteReference(repo, 'refs/remotes/{}/{}'.format(remote, branch))
        local_branch = Head(repo, 'refs/heads/{}'.format(branch))

        try:
            if head.commit != remote_branch.commit:
                local_branch.commit = remote_branch.commit
                local_branch.checkout()

        except ValueError:
            pytest.xfail("Branch {} is empty".format(branch))

        return repo
Example #38
0
  def run(self):
      config_file = "development.ini"
      if "BIOSHADOCK_CONFIG" in os.environ:
          config_file = os.environ["BIOSHADOCK_CONFIG"]
      config = ConfigParser.ConfigParser()
      config.readfp(open(config_file))
      logging.config.fileConfig(config_file)
      while True:
          logging.info("New build run")
          if BioshadockDaemon.db_mongo is None:
              mongo = MongoClient(config.get('app:main','mongo_url'))
              BioshadockDaemon.db_mongo = mongo['mydockerhub']
          if BioshadockDaemon.db_redis is None:
              BioshadockDaemon.db_redis = redis.StrictRedis(host=config.get('app:main','redis_host'), port=int(config.get('app:main','redis_port')), db=0)
          if BioshadockDaemon.cli is None:
              if config.get('app:main', 'docker_connect'):
                  BioshadockDaemon.cli = Client(base_url=config.get('app:main',
                                                'docker_connect'))
              else:
                  BioshadockDaemon.cli = Client()
              BioshadockDaemon.cli.login(username=config.get('app:main','push_auth_user'), password=config.get('app:main','push_auth_password'),
                                         email=config.get('app:main','push_auth_email'),registry=config.get('app:main','service'))

          build = BioshadockDaemon.db_redis.lpop('bioshadock:builds')
          dockerfile = None
          if build is not None:
              dt = datetime.datetime.now()
              timestamp = time.mktime(dt.timetuple()) 
              build = loads(build)
              logging.debug(str(build))
              dockerfile = build['dockerfile']
              gitrepo = build['git']
              do_git = False
              git_repo_dir = None
              if gitrepo is not None and gitrepo and gitrepo != 'none':
                  # TODO clone repo in a dir, chdir to repo and optionally write
                  # dockerfile
                  git_repo_dir = tempfile.mkdtemp(suffix='.git')
                  git_info = gitrepo.split('#')
                  gitrepo = git_info[0]
                  selectedbranch = 'master'
                  subdir = None
                  if len(git_info) > 1:
                      branch_path = git_info[1].split(':')
                      if branch_path[0]:
                          selectedbranch = branch_path[0]
                      if len(branch_path) > 1 and branch_path[1]:
                          subdir = branch_path[1]
                  logging.info(str(gitrepo))
                  logging.info("Using branch "+selectedbranch)
                  logging.info("Directory: "+str(subdir))
                  try:
                      Repo.clone_from(gitrepo, git_repo_dir, branch=selectedbranch)
                      if subdir is not None:
                          git_repo_dir = os.path.join(git_repo_dir, subdir)
                      logging.debug(str(git_repo_dir))
                      os.chdir(git_repo_dir)
                  except Exception as e:
                      logging.error(str(e))
                      continue
                  #if dockerfile:
                  if not os.path.exists("Dockerfile"):
                      logging.debug("Overwrite Dockerfile")
                      f = open('Dockerfile', 'w')
                      f.write(dockerfile.encode('utf-8'))
                      f.close()
                  else:
                      logging.debug("Use git Dockerfile")
                      with open ("Dockerfile", "r") as gitDockerfile:
                          dockerfile=gitDockerfile.read().encode('utf-8')

              f = BytesIO(dockerfile.encode('utf-8'))

              build_tag = ''
              if 'tag' in build and build['tag']:
                  build_tag = ':'+build['tag']
              logging.info('Build: '+str(build['id']))
              response = [line for line in BioshadockDaemon.cli.build(
                  fileobj=f, rm=True, tag=config.get('app:main', 'service')+"/"+build['id']+build_tag)]
              build['response'] = response
              if build['response']:
                  last = build['response'][len(build['response'])-1]
                  matches = re.search('Successfully built\s+(\w+)', last)
                  if matches is None:
                      build['status'] = False
                  else:
                      build['status'] = True
                      build['image_id'] = matches.group(1)
                      p= subprocess.Popen(["docker",
                                        "push",
                                        config.get('app:main', 'service')+"/"+build['id']])

              else:
                  build['status'] = False
              build['timestamp'] = timestamp
              BioshadockDaemon.db_mongo['repository'].update({'id': build['id']},
                                                       {'$push': {'builds': build},
                                                       '$set':{'meta.Dockerfile': dockerfile}})
              if do_git:
                  cur_dir = os.path.dirname(os.path.realpath(__file__))
                  os.chdir(cur_dir)
                  shutil.rmtree(git_repo_dir)
          time.sleep(2)
def make_client( origin ):
    name = os.path.basename( origin.working_tree_dir )
    client_path = os.path.join( tempfile.mkdtemp() , name )
    origin_url = "file://%s" % origin.working_tree_dir
    return Repo.clone_from( origin_url , client_path )
Example #40
0
 def clone(self , url):
     self.repo = Repo.clone_from( url , self.work_dir )
Example #41
0
from git.repo.base import Repo
import sys, os

url = sys.argv[1]
path = "repos/{0}".format(url.split("/", 3)[-1])

if __name__ == '__main__':
    if not os.path.isdir(path):
        repo = Repo.clone_from(url, path)
Example #42
0
def clone(uri, path):
    return Repo.clone_from(uri, path, progress=ProgressPrinter())
Example #43
0
            backup_file_path = os.path.join(app_folder, repo_name + "-" + args.revert + ".zip")
            get(sftp, backup_file_path, os.path.join(PATH, backup_file_name))

            fh = open(os.path.join(PATH, backup_file_name), "rb")
            z = zipfile.ZipFile(fh)
            for name in z.namelist():
                outpath = os.path.join(PATH, repo_name)
                z.extract(name, outpath)
            fh.close()
            rm(sftp, os.path.join(app_folder, repo_name))
            put_all(sftp, os.path.join(PATH, repo_name), app_folder)

            exit(0)

        try:
            Repo.clone_from(git_repo, os.path.join(PATH, repo_name))
        except Exception, e:
            print "git error"
            print e
            sys.exit(2)

        if not remote_file_exists(sftp, app_folder + repo_name):
            # sftp.mkdir(app_folder+repo_name)
            put_all(sftp, os.path.join(PATH, repo_name), app_folder)
            shutil.rmtree(os.path.join(PATH, repo_name))
        else:
            os.mkdir(os.path.join(PATH, "tmp"))
            get_all(sftp, app_folder + repo_name, os.path.join(PATH, "tmp"))

            zip_name = repo_name + "-" + time.strftime("%d-%m-%Y-%I:%M:%S") + ".zip"
Example #44
0
class GitModule(object):

    def clone(self , url):
        self.repo = Repo.clone_from( url , self.work_dir )

    
    def __init__(self , **kwargs):
        url = kwargs.get("url")
        local_path = kwargs.get("local_path")
        self.work_dir = None

        if url is None and local_path is None:
            raise ValueError("Must supply one of url and local_path")
            
        if url and local_path:
            raise ValueError("Can only supply one of url and local_path")
            
        if url:
            self.work_dir = tempfile.mkdtemp( )
            self.clone( url )
        else:
            self.work_dir = None
            self.repo = Repo( local_path )
            origin = self.repo.remote( name = "origin")
            origin.fetch( )
    

    def __del__(self):
        if self.work_dir and os.path.isdir( self.work_dir ):
            shutil.rmtree( self.work_dir )


    def getRoot(self):
        return self.repo.working_tree_dir

    def relPath(self , path):
        if path.startswith( self.getRoot( ) ):
            N = len(self.getRoot()) + 1
            return path[N:]
        else:
            raise IOError("Not part of repo:%s ?!" % path)


    def absPath(self , path):
        repo_path = os.path.join( self.getRoot() , path)
        if os.path.exists( repo_path ):
            return repo_path
        else:
            raise IOError("No such entry in repo:%s" % path)


    def checkout(self , ref):
        self.repo.git.checkout( ref )


    def getHeadSHA(self):
        commit = self.repo.commit("HEAD")
        return commit.hexsha

    def runTests(self , cmd):
        full_cmd = self.absPath( cmd )
        if not os.path.isfile( full_cmd ):
            raise IOError("Is not an executable file:%s" % cmd)

        if not os.access( full_cmd , os.X_OK):
            raise OSError("File not executable: %s" % cmd)
            
        exit_code = subprocess.call( full_cmd )
        if exit_code == 0:
            return True
        else:
            return False


    def install(self , target, files = [] , directories = []):
        if os.path.exists( target ):
            if not os.path.isdir( target ):
                raise OSError("Target:%s already exists - and is not a directory" % target)
        else:    
            os.makedirs( target )

        for dir in directories:
            repo_path = os.path.join( self.getRoot() , dir)
            if not os.path.exists( repo_path ):
                target_path = os.path.join( target , dir)
                if not os.path.isdir( target_path ):
                    os.makedirs( target_path )
                continue

            if os.path.dirname( dir ):
                target_path = os.path.join( target , os.path.dirname( dir ))
                if not os.path.isdir( target_path ):
                    os.makedirs( target_path )
            
            for dirpath , dirnames , filenames in os.walk( self.absPath(dir)):
                target_path = os.path.join( target , self.relPath( dirpath ))
                if not os.path.isdir( target_path ):
                    os.makedirs( target_path )

                for file in filenames:
                    src_file = os.path.join( dirpath , file )
                    target_file = os.path.join( target_path , file )
                    shutil.copy( src_file , target_file )



        for file in files:
            target_file = os.path.join( target , file )
            if os.path.dirname( file ):
                target_path = os.path.join( target , os.path.dirname( file ))
                if not os.path.isdir( target_path ):
                    os.makedirs( target_path )

            shutil.copy( self.absPath( file ) , target_file )