Ejemplo n.º 1
0
def clone_subrepository(subrepo_url, local_path):
    """clone a mercurial subrepository and handle all of the .hgsub stuff"""

    # clone the common subrepository
    hg_root = fabhelp.get_hg_root()
    fabhelp.progress("create the '%s' subrepository" % 
                     os.path.relpath(local_path, hg_root))
    local("hg clone %s %s"%(subrepo_url, local_path))

    # update .hgsub
    hgsub_filename = os.path.join(hg_root, ".hgsub")
    if os.path.exists(hgsub_filename):
        with open(hgsub_filename) as hgsub_file:
            hgsub = hgsub_file.read()
        # add a trailing newline character
        if hgsub!='' and not hgsub.endswith('\n'):
            hgsub += '\n'
    else:
        hgsub = ''
        
    with open(hgsub_filename,'w') as hgsub_file:
        hgsub_file.write(hgsub)
        hgsub_file.write("%s = %s\n"%(
            os.path.relpath(os.path.join(local("pwd",capture=True),local_path), hg_root),
            subrepo_url,
        ))
Ejemplo n.º 2
0
def update_remote(hg_revision='tip'):
    """update remote repository"""

    change_owner()
    group_writable_permissions()

    # get a list of all of the django projects in this 
    rel_django_project_dirs = []
    hg_root = fabhelp.get_hg_root()
    for root, dirs, files in os.walk(hg_root):
        d = os.path.join(hg_root, root)
        try:
            dirs.remove('.hg') # ignore mercurial repositories
        except ValueError:
            pass # .hg is not in dirs
        if fabhelp.is_django_project(d):
            rel_django_project_dirs.append(os.path.relpath(d, hg_root))
            dirs[:] = [] # no need to decend any further... save time!

    # get the remote directory from the apache file
    fabhelp.progress("updating remote repository")
    remote_repo_dir = get_remote_repo_dir()
    with cd(remote_repo_dir):
        with fabric_settings(warn_only = True):
            run("hg pull")

        # run database2fixture for each django repository in this repository
        for rel_django_project_dir in rel_django_project_dirs:
            with cd(rel_django_project_dir):
                if fabhelp.is_dbvcs_installed():
                    run("./manage.py database2fixture")

        run("hg update -r %s"%hg_revision)

    remote_project_dir = get_remote_project_dir()
    with cd(remote_project_dir):
        run("./manage.py setup_virtualenv") # add new packages when necessary
        group_writable_permissions()
Ejemplo n.º 3
0
def pull_remote_to_localhost(remotehost):
    """pull remotehost changes into local repository and update. this
    includes all subrepositories
    """

    assert remotehost in ("poisson", "noether",)
    
    # dump database and check in as necessary
    fabhelp.progress("dump remote database and auto check-in")
    remote_project_dir = get_remote_project_dir()
    with cd(remote_project_dir):
        if fabhelp.is_dbvcs_installed():
            run("./manage.py database2fixture")
        cmd="hg ci -m 'auto-checkin: adding data for pull_remote_to_localhost'"
        with fabric_settings(warn_only=True):
            result = run(cmd)
            if result.failed:
                # there were no changes to check in
                if result.return_code==1:
                    pass 
                # something else happened. this is a problem, so abort
                else:
                    abort("hg ci failed. see above output for details")

    # recursively get all of the subrepository directories. breadth
    # first search here
    fabhelp.progress("find all of the subrepository directories")
    from django.conf import settings as django_settings
    with lcd(django_settings.PROJECT_ROOT):
        local_repo_dir = fabhelp.get_hg_root()
    remote_repo_dir = get_remote_repo_dir()
    hgsub_directories=collections.deque([local_repo_dir])
    subrepo_directory_list = [local_repo_dir]
    while len(hgsub_directories):
        hgsub_directory = hgsub_directories.popleft()
        hgsub_filename = os.path.join(hgsub_directory, '.hgsub')
        if os.path.exists(hgsub_filename):
            hgsub = open(hgsub_filename)
            for line in hgsub:
                l = line.split('=')
                if len(l)==2:
                    directory = l[0].strip()
                    subrepo_directory = os.path.join(hgsub_directory,directory)
                    subrepo_directory_list.append(subrepo_directory)
                    hgsub_directories.append(subrepo_directory)
                elif line.strip(): # this is a non-empty line
                    raise TypeError("%s has unexpected format"%hgsub_filename)
            hgsub.close()

    # reorder subrepo reldir list to always get subdirectories first 
    subrepo_directory_list.reverse()

    # pull changes into base repository and all
    # subrepositories. iterate over all of the subrepo reldir's and
    # pull in the new information so that there aren't any problems
    # with remote heads, etc.
    fabhelp.progress("recursively pull subrepos and the base repo")
    for subrepo_directory in subrepo_directory_list:
        with lcd(subrepo_directory):
            subrepo_reldir = os.path.relpath(subrepo_directory, local_repo_dir)
            remote_dir = os.path.join(remote_repo_dir, subrepo_reldir)

            # if remote dir exists, pull from that
            # subrepository. remote_dir may not exist if there are
            # locally added subrepos that have not been put on
            # production yet.
            with fabric_settings(warn_only=True):
                exists = run("test -d %s"%remote_dir)
                if exists.succeeded:
                    local("hg pull ssh://%s/%s"%(remotehost, remote_dir))

    # # XXXX THIS IS CAUSING WIERD ERROR WITH PARAMIKO ?!?!
    # # locally update
    # fabhelp.progress("update local repository (or at least try)")
    # with lcd(local_repo_dir):
    #     with fabric_settings(warn_only=True):
    #         result = local("hg up", capture=True)
    #         if result.failed:
    #             pass # nothing to update
    msg="next: update your local repository to get all incoming change sets"
    fabhelp.progress(msg)
Ejemplo n.º 4
0
def add_dsa_app(app_name=None):
    """add an app from the DjangoApps collection of repositories"""
    
    # make sure project name is provided 
    if app_name is None:
        app_name = prompt("Please enter an app name: ")

    # remember some directories
    cur_dir, bin_dir, src_dir = fabhelp.get_cur_bin_src_directories()

    # make sure we are in the root of the django project
    if not fabhelp.is_django_project(cur_dir, in_root_dir=True):
        abort("This command must be run from root of django project")

    # abort if local changes to .hgignore or .hgsub
    hg_root = fabhelp.get_hg_root()
    hg_modified_files = [
        os.path.join(hg_root, ".hgsub"),
        os.path.join(cur_dir, "conf", "settings_modules.txt"),
    ]
    fabhelp.abort_if_local_changes(hg_modified_files)

    # clone the subrepository in the right place
    clone_subrepository(
        "ssh://poisson//srv/hg/DjangoApps/%s"%app_name, 
        os.path.join(cur_dir, "apps", app_name),
    )

    # create the settings file
    fabhelp.progress("creating the conf/%s.py settings for project"%app_name)
    from django.conf import settings as django_settings
    from django.template.loader import render_to_string
    try:
        django_settings.configure(
            DEBUG=True, 
            TEMPLATE_DEBUG=True,
            TEMPLATE_DIRS=(
                os.path.join(cur_dir, 'templates'),
            ),
        )
    except RuntimeError:
        pass # this is being run from the setup_dsa_app django command
             # --- hopefully
    f = open(os.path.join(cur_dir, "conf", "%s.py"%app_name), 'w')
    f.write(render_to_string(".settings_template.py", {
        "app_name": app_name,
        "is_dbvcs_installed": fabhelp.is_dbvcs_installed(),
    }))
    f.close()
    
    # add the settings to the settings_modules.txt file
    fabhelp.progress("add the settings file to conf/settings_modules.txt")
    f = open(os.path.join(cur_dir, "conf", "settings_modules.txt"), 'a')
    f.write("""
# automatically including %s
conf/%s.py
"""%(app_name, app_name))
    f.close()

    # add the media soft links
    fabhelp.progress("soft links for media")
    with lcd(".media"):
        local("ln -s ../apps/%s/media/%s %s"%(app_name, app_name, app_name))
    with lcd(".static"):
        local("ln -s ../apps/%s/static/%s %s"%(app_name, app_name, app_name))

    # synchronize the database and dump the database
    fabhelp.progress("syncdb and database2fixture (as necessary)!")
    local("./manage.py syncdb --noinput")
    if fabhelp.is_dbvcs_installed():
        local("./manage.py database2fixture --all")

    # add these fixtures to the repository. make sure to change into
    # the local directory first in case the dbvcs data directory is a
    # subrepository
    if fabhelp.is_dbvcs_installed():
        result = local("./manage.py listfixtures apps.%s" % app_name, 
                       capture=True)
        for filename in result.split():
            d, f = os.path.split(filename)
            with lcd(d):
                local("hg add %s"%f)

    # run hg add
    fabhelp.progress("add everything to the mercurial repository")
    local("hg add %s"%' '.join([
        os.path.join(cur_dir, "conf", "%s.py"%app_name),
        os.path.join(cur_dir, ".media", app_name),
        os.path.join(cur_dir, ".static", app_name),
    ]))
Ejemplo n.º 5
0
def render_apache_configuration(production_root_url, 
                                local_master_django_project_dir=''):
    """setup apache configuration files for BOTH staging and
    production. this does NOT manipulate apache configuration on
    remote servers. it is only intended torender the apache
    templates
    """
    msg = "setup apache configuration files for BOTH staging and production"
    fabhelp.progress(msg)

    # if the production root url has a suburl, the
    # local_master_django_project_dir must be specified!!! WARNING: do not
    # remove this condition --- it is used very strongly below to
    # assume when things need to be written to the staging apache conf
    if production_root_url.suburl and not local_master_django_project_dir:
        msg = "if production_root_url is a suburl django project,\n"
        msg += "you must specify the --parent_project option\n"
        raise TypeError(msg)

    # make sure that master_conf_filename exists if it is specified
    if local_master_django_project_dir:
        local_master_django_project_dir = \
            os.path.abspath(local_master_django_project_dir)
        if not fabhelp.is_django_project(local_master_django_project_dir):
            msg="--parent_project must point to a django project root.\n" 
            msg+="'%s' is not a django project root" % (
                local_master_django_project_dir,
            )
            abort(msg)

        # also need to make sure that local_master_django_project_dir
        # is actually hosting the same domain, otherwise could cause
        # problems!
        filename = os.path.join(local_master_django_project_dir,
                                "conf", "noether", "apache", "noether.conf")
        try:
            f = open(filename)
        except IOError:
            msg = "can not find parent project apache conf in %s\n"%filename
            msg += "have you run setup_staging in %s yet?" % (
                local_master_django_project_dir,
            )
            abort(msg)
        s = f.read()
        f.close()
        result = re.search(r"<VirtualHost (?P<domain>[\w\.]+):\d+>", s)
        if result is None:
            msg = "apache configuration file (%s)\n"%filename 
            msg += "does not appear to be the root VirtualHost"
            abort(msg)
        master_domain = result.groupdict()["domain"]
        if production_root_url.domain!=master_domain:
            msg = "domain of parent_project (%s)\n" % master_domain
            msg += "does not match specified domain (%s)"%(
                production_root_url.domain,
            )
            abort(msg)

    # get the project name --- remember, this command is always run
    # from within the django project
    project_dir = local('pwd', capture=True)
    project_name = os.path.basename(project_dir)

    # setup django
    from django.conf import settings as django_settings
    from django.template.loader import render_to_string
    try:
        django_settings.configure(
            DEBUG=True, 
            TEMPLATE_DEBUG=True,
            TEMPLATE_DIRS=(
                os.path.join(project_dir, 'common', 'templates'),
            ),
        )
    except RuntimeError:
        pass # this is being run from the setup_{staging,production}
             # django command --- hopefully

    # set things up for both staging and production
    for server in ("poisson", "noether",):

        # determing the template to use 
        suburl = production_root_url.suburl
        if server=="poisson" or local_master_django_project_dir:
            conf_template = os.path.join("apache", "suburl.conf")
            if server=="poisson":
                suburl = '/' + production_root_url.subdomain + suburl
        else:
            conf_template = os.path.join("apache", "root.conf")

        # write the configuration template for this server
        conf_filename = os.path.join(project_dir, "conf", 
                                     server, "apache", "%s.conf"%server)
        local("mkdir -p %s"%os.path.dirname(conf_filename))
        hg_root = fabhelp.get_hg_root()
        f = open(conf_filename, 'w')
        f.write(render_to_string(
                conf_template,
                {
                    "subdomain": production_root_url.subdomain,
                    "domain": production_root_url.domain,
                    "suburl": suburl,
                    "server": server,
                    "hg_path_to_django_project": os.path.join(
                        os.path.basename(hg_root),
                        os.path.relpath(project_dir, hg_root)),
                }))
        f.close()
        local("hg add %s"%os.path.relpath(conf_filename, project_dir))

        # include this new apache file as necessary in the appropriate
        # root url apache configuration
        if local_master_django_project_dir:
            remote_django_project_dir = get_remote_project_dir()
            root_apache_filename = os.path.join(
                local_master_django_project_dir, "conf", server, "apache",
                "%s.conf"%server)

            # XXXX TODO: this is probably not enough to simply place the
            # suburls directly before the ####XXXX line. we probably need
            # to order things in a smart way depending on what the suburls
            # are: the most restrictive first like /foo/bar before /foo.
            apache_file = open(root_apache_filename,'r')
            apache_conf = apache_file.read()
            apache_file.close()
            include_directive = "Include %s"%os.path.join(
                remote_django_project_dir, 
                "conf", server, "apache", 
                "%s.conf"%server
            ).replace('/', r'\/')
            if include_directive not in apache_conf:
                local(r"sed 's/####XXXX/%s\n####XXXX/' %s > kk"%(
                    include_directive,
                    root_apache_filename,
                ))
                local("mv kk %s"%root_apache_filename)

    with settings(warn_only=True):
        result = local("hg ci -m 'auto-added apache configuration setup'", 
                       capture=True)
        if result.failed:
            warn("apache configuration already added")
Ejemplo n.º 6
0
def setup_virtualenv(virtualenv_name, run_func=local):
    """setup the virtual environment"""

    # make sure that virtualenv is installed
    with settings(warn_only=True):
        if run_func == local:
            virtualenv_exe = run_func("which virtualenv", capture=True)
        else:
            virtualenv_exe = run_func("which virtualenv")
        if virtualenv_exe.failed:
            sudo("apt-get install python-virtualenv")
            if run_func == local:
                virtualenv_exe = run_func("which virtualenv", capture=True)
            else:
                virtualenv_exe = run_func("which virtualenv")

    # set up the virtual environment
    fabhelp.progress("set up the virtual environment")
    
    # change 10/31/2012- using environment variables to specify using
    # setuptools instead of the command option. hopefully this will
    # work across multiple versions of virtualenv
    run_func("export VIRTUALENV_DISTRIBUTE=false")
    run_func("export VIRTUALENV_USE_SETUPTOOLS=true")
    run_func("%s --no-site-packages %s"%(
        virtualenv_exe,
        virtualenv_name,
    ))
    virtualenv_requirements = [
        os.path.join("common", "conf", "virtualenv_requirements.txt"),
        os.path.join("conf", "virtualenv_requirements.txt"),
    ]
    pip_version = get_pip_version()
    PIP_REMOVED_ENVIRONMENT_OPTION = 1.1
    for virtualenv_requirement in virtualenv_requirements:
        if pip_version < PIP_REMOVED_ENVIRONMENT_OPTION:
            run_func("if [ -s %s ]; then pip install -E %s -r %s; fi"%(
                virtualenv_requirement,
                virtualenv_name,
                virtualenv_requirement,
            ))
        else:
            run_func("if [ -s %s ]; then virtualenv %s && %s/bin/pip install -r %s; fi"%(
                virtualenv_requirement,
                virtualenv_name,
                virtualenv_name,
                virtualenv_requirement,
            ))

    hg_root = fabhelp.get_hg_root()
    if run_func == local:
        project_dir = run_func("pwd", capture=True)
    else:
        project_dir = run_func("pwd")
    project_name = os.path.basename(project_dir)

    # add virtualenv to hgignore
    if run_func == local:
        fabhelp.progress("add virtualenv to .hgignore")
        ignore_patterns = (
            os.path.relpath(os.path.join(project_dir,virtualenv_name),hg_root),
        )
        fabhelp.add_to_hgignore(
            ignore_patterns,
            syntax="glob",
            comment="ignore virtualenv for project '%s'"%project_name,
        )