예제 #1
0
def _generate_images(cmd, dpi, width, target, source):
    command(cmd.format(cmd=_get_inkscape_cmd(),
                       dpi=dpi,
                       width=width,
                       target=target,
                       source=source))
    logger.debug('generated image file {0}'.format(target))
예제 #2
0
def json_output(conf):
    list_file = os.path.join(conf.paths.branch_output, 'json-file-list')
    public_list_file = os.path.join(conf.paths.public_site_output,
                                    'json', '.file_list')

    cmd = 'rsync --recursive --times --delete --exclude="*pickle" --exclude=".buildinfo" --exclude="*fjson" {src} {dst}'

    json_dst = os.path.join(conf.paths.public_site_output, 'json')

    if not os.path.exists(json_dst):
        logger.debug('created directories for {0}'.format(json_dst))
        os.makedirs(json_dst)

    builder = 'json'
    if 'edition' in conf.project and conf.project.edition != conf.project.name:
        try:
            builder += '-' + conf.project.edition
        except:
            pass

    command(cmd.format(src=os.path.join(conf.paths.branch_output, builder) + '/',
                       dst=json_dst))

    copy_if_needed(list_file, public_list_file)
    logger.info('deployed json files to local staging.')
예제 #3
0
def json_output(conf):
    list_file = os.path.join(conf.paths.branch_output, 'json-file-list')
    public_list_file = os.path.join(conf.paths.public_site_output, 'json',
                                    '.file_list')

    cmd = 'rsync --recursive --times --delete --exclude="*pickle" --exclude=".buildinfo" --exclude="*fjson" {src} {dst}'

    json_dst = os.path.join(conf.paths.public_site_output, 'json')

    if not os.path.exists(json_dst):
        logger.debug('created directories for {0}'.format(json_dst))
        os.makedirs(json_dst)

    builder = 'json'
    if 'edition' in conf.project and conf.project.edition != conf.project.name:
        try:
            builder += '-' + conf.project.edition
        except:
            pass

    command(
        cmd.format(src=os.path.join(conf.paths.branch_output, builder) + '/',
                   dst=json_dst))

    copy_if_needed(list_file, public_list_file)
    logger.info('deployed json files to local staging.')
예제 #4
0
def _generate_images(cmd, dpi, width, target, source):
    command(cmd.format(cmd=_get_inkscape_cmd(),
                       dpi=dpi,
                       width=width,
                       target=target,
                       source=source))
    print('[image]: generated image file {0}'.format(target))
예제 #5
0
def _generate_images(cmd, dpi, width, target, source):
    command(
        cmd.format(cmd=_get_inkscape_cmd(),
                   dpi=dpi,
                   width=width,
                   target=target,
                   source=source))
    print('[image]: generated image file {0}'.format(target))
예제 #6
0
def _generate_images(cmd, dpi, width, target, source):
    command(
        cmd.format(cmd=_get_inkscape_cmd(),
                   dpi=dpi,
                   width=width,
                   target=target,
                   source=source))
    logger.debug('generated image file {0}'.format(target))
예제 #7
0
def transfer_source(conf):
    target = os.path.join(conf.paths.projectroot, conf.paths.branch_output)

    if not os.path.exists(target):
        os.makedirs(target)
        print('[sphinx-prep]: created ' + target)
    elif not os.path.isdir(target):
        raise InvalidFile('[sphinx-prep]: {0} exists and is not a directory'.format(target))

    source_dir = os.path.join(conf.paths.projectroot, conf.paths.source)

    command('rsync --checksum --recursive --delete {0} {1}'.format(source_dir, target))
    print('[sphinx-prep]: updated source in {0}'.format(target))
예제 #8
0
def transfer_source(sconf, conf):
    target = os.path.join(conf.paths.projectroot, conf.paths.branch_source)

    if not os.path.exists(target):
        os.makedirs(target)
        logger.debug('created directory for sphinx build: {0}'.format(target))
    elif not os.path.isdir(target):
        msg = '"{0}" exists and is not a directory'.format(target)
        logger.error(msg)
        raise InvalidFile(msg)

    source_dir = os.path.join(conf.paths.projectroot, conf.paths.source)

    command('rsync --checksum --recursive --delete {0}/ {1}'.format(source_dir, target))
    logger.info('prepared source for sphinx build in {0}'.format(target))
예제 #9
0
def include_files(conf, files=None):
    if files is not None:
        return files
    else:
        source_dir = os.path.join(conf.paths.projectroot, conf.paths.source)
        grep = command(
            'grep -R ".. include:: /" {0} || exit 0'.format(source_dir),
            capture=True).out

        rx = re.compile(source_dir + r'(.*):.*\.\. include:: (.*)')

        s = [
            m.groups() for m in [rx.match(d) for d in grep.split('\n')]
            if m is not None
        ]

        def tuple_sort(k):
            return k[1]

        s.sort(key=tuple_sort)

        files = dict()

        for i in groupby(s, itemgetter(1)):
            files[i[0]] = set()
            for src in i[1]:
                if not src[0].endswith('~') and not src[0].endswith(
                        'overview.rst'):
                    files[i[0]].add(src[0])
            files[i[0]] = list(files[i[0]])
            files[i[0]].sort()

        files.update(generated_includes(conf))

        return files
예제 #10
0
def sphinx_build_worker(target, conf, sconf, do_post=True):
    sconf = sconf[target]

    dirpath = os.path.join(sconf.root, target)
    if not os.path.exists(dirpath):
        os.makedirs(dirpath)
        print('[{0}]: created {1}/{0}'.format(target, sconf.root))

    print('[sphinx] [{0}]: starting build at {1}'.format(target, timestamp()))

    tags = ' '.join(['-t ' + i for i in sconf.tags])
    cmd = 'sphinx-build -b {builder} {tags} -q -d {root}/doctrees-{builder} -c {config_path} {sphinx_args} {source} {root}/{builder}'

    sphinx_cmd = cmd.format(builder=sconf.target,
                            tags=tags,
                            root=sconf.root,
                            config_path=conf.paths.projectroot,
                            sphinx_args=get_sphinx_args(sconf.tags),
                            source=sconf.source
        )

    out = command(sphinx_cmd, capture=True)
    output_sphinx_stream('\n'.join([out.err, out.out]), target, conf)

    print('[sphinx] [{0}]: completed build at {1}'.format(target, timestamp()))

    finalize(target, sconf, conf)
예제 #11
0
def prereq(sconf, conf=None):
    conf = lazy_config(conf)

    jobs = itertools.chain()

    job_count = runner(jobs)
    print('[sphinx]: processed {0} build prerequisite jobs'.format(job_count))

    generate_source(sconf, conf)
    dep_count = refresh_dependencies(conf)
    print('[sphinx]: refreshed {0} dependencies'.format(dep_count))

    command(build_platform_notification('Sphinx',
            'Build in progress past critical phase.'))

    dump_file_hashes(conf.system.dependency_cache, conf)
예제 #12
0
def include_files(files=None, conf=None):
    if files is not None:
        return files
    else:
        conf = lazy_conf(conf)

        source_dir = os.path.join(conf.paths.projectroot, conf.paths.source)
        grep = command('grep -R ".. include:: /" {0} || exit 0'.format(source_dir), capture=True).out

        rx = re.compile(source_dir + r'(.*):.*\.\. include:: (.*)')

        s = [ m.groups()
              for m in [ rx.match(d)
                         for d in grep.split('\n') ]
              if m is not None
            ]

        def tuple_sort(k):
            return k[1]
        s.sort(key=tuple_sort)

        files = dict()

        for i in groupby(s, itemgetter(1) ):
            files[i[0]] = set()
            for src in i[1]:
                if not src[0].endswith('~'):
                    files[i[0]].add(src[0])
            files[i[0]] = list(files[i[0]])

        files.update(generated_includes(conf))

        return files
예제 #13
0
def finalize_dirhtml_build(builder, conf):
    pjoin = os.path.join

    error_pages(conf)

    single_html_dir = get_single_html_dir(conf)
    search_page = pjoin(conf.paths.branch_output, builder, 'index.html')

    if conf.project.name == 'mms' and mms_should_migrate(builder, conf) is False:
        return False

    if os.path.exists(search_page):
        copy_if_needed(source_file=search_page,
                       target_file=pjoin(single_html_dir, 'search.html'))

    dest = pjoin(conf.paths.projectroot, conf.paths.public_site_output)
    command('rsync -a {source}/ {destination}'.format(source=pjoin(conf.paths.projectroot,
                                                                 conf.paths.branch_output,
                                                                 builder),
                                                    destination=dest))

    print('[{0}]: migrated build to {1}'.format(builder, dest))

    if conf.git.branches.current in conf.git.branches.published:
        sitemap_exists = sitemap(config_path=None, conf=conf)

        if sitemap_exists is True:
            copy_if_needed(source_file=pjoin(conf.paths.projectroot,
                                             conf.paths.branch_output,
                                             'sitemap.xml.gz'),
                           target_file=pjoin(conf.paths.projectroot,
                                             conf.paths.public_site_output,
                                             'sitemap.xml.gz'))

    sconf = BuildConfiguration('sphinx.yaml', pjoin(conf.paths.projectroot,
                                                conf.paths.builddata))

    if 'dirhtml' in sconf and 'excluded_files' in sconf.dirhtml:
        fns = [ pjoin(conf.paths.projectroot,
                      conf.paths.public_site_output,
                      fn)
                for fn in sconf.dirhtml.excluded_files ]

        cleaner(fns)
        print('[dirhtml] [clean]: removed excluded files from output directory')
예제 #14
0
def json_output(conf=None):
    conf = lazy_conf(conf)

    list_file = os.path.join(conf.paths.branch_staging, 'json-file-list')
    public_list_file = os.path.join(conf.paths.public_site_output,
                                    'json', '.file_list')

    cmd = 'rsync --recursive --times --delete --exclude="*pickle" --exclude=".buildinfo" --exclude="*fjson" {src} {dst}'
    json_dst = os.path.join(conf.paths.public_site_output, 'json')

    if not os.path.exists(json_dst):
        os.makedirs(json_dst)

    command(cmd.format(src=os.path.join(conf.paths.branch_output, 'json') + '/',
                       dst=json_dst))

    copy_if_needed(list_file, public_list_file)
    print('[json]: deployed json files to local staging.')
예제 #15
0
def json_output(conf=None):
    conf = lazy_conf(conf)

    list_file = os.path.join(conf.paths.branch_staging, 'json-file-list')
    public_list_file = os.path.join(conf.paths.public_site_output, 'json',
                                    '.file_list')

    cmd = 'rsync --recursive --times --delete --exclude="*pickle" --exclude=".buildinfo" --exclude="*fjson" {src} {dst}'
    json_dst = os.path.join(conf.paths.public_site_output, 'json')

    if not os.path.exists(json_dst):
        os.makedirs(json_dst)

    command(
        cmd.format(src=os.path.join(conf.paths.branch_output, 'json') + '/',
                   dst=json_dst))

    copy_if_needed(list_file, public_list_file)
    print('[json]: deployed json files to local staging.')
예제 #16
0
def build_prerequisites(conf):
    jobs = itertools.chain(
        build_prereq_jobs(conf),
        manpage_jobs(conf),
        table_jobs(conf),
        api_jobs(conf),
        toc_jobs(conf),
        option_jobs(conf),
        steps_jobs(conf),
        release_jobs(conf),
        intersphinx_jobs(conf),
        image_jobs(conf),
    )

    try:
        res = runner(jobs, parallel="process")
        print("[sphinx-prep]: built {0} pieces of content".format(len(res)))
    except PoolResultsError:
        print(
            "[WARNING]: sphinx prerequisites encountered errors. "
            "See output above. Continuing as a temporary measure."
        )

    runner(external_jobs(conf), parallel="thread")

    buildinfo_hash(conf)

    if conf.project.name != "mms":
        # we copy source manually for mms in makefile.mms, avoiding this
        # operation to clarify the artifacts directory
        transfer_source(conf)

    print("[sphinx-prep]: resolving all intra-source dependencies now. (takes several seconds)")
    dep_count = refresh_dependencies(conf)
    print("[sphinx-prep]: bumped timestamps of {0} files".format(dep_count))

    command(build_platform_notification("Sphinx", "Build in progress pastb critical phase."), ignore=True)

    print("[sphinx-prep]: INFO - Build in progress past critical phase.")

    dump_file_hashes(conf.system.dependency_cache, conf)
    print("[sphinx-prep]: build environment prepared for sphinx.")
예제 #17
0
def finalize_dirhtml_build(builder, conf):
    pjoin = os.path.join

    single_html_dir = get_single_html_dir(conf)
    search_page = pjoin(conf.paths.branch_output, builder, 'index.html')

    if os.path.exists(search_page):
        copy_if_needed(source_file=search_page,
                       target_file=pjoin(single_html_dir, 'search.html'))

    dest = pjoin(conf.paths.projectroot, conf.paths.public_site_output)
    command('rsync -a {source}/ {destination}'.format(source=pjoin(conf.paths.projectroot,
                                                                 conf.paths.branch_output,
                                                                 builder),
                                                    destination=dest))

    logger.info('{0}: migrated build to {1}'.format(builder, dest))

    if 'published' in conf.git.branches and conf.git.branches.current in conf.git.branches.published:
        sitemap_exists = sitemap(config_path=None, conf=conf)

        if sitemap_exists is True:
            copy_if_needed(source_file=pjoin(conf.paths.projectroot,
                                             conf.paths.branch_output,
                                             'sitemap.xml.gz'),
                           target_file=pjoin(conf.paths.projectroot,
                                             conf.paths.public_site_output,
                                             'sitemap.xml.gz'))

    sconf = BuildConfiguration('sphinx.yaml', pjoin(conf.paths.projectroot,
                                                conf.paths.builddata))

    if 'dirhtml' in sconf and 'excluded_files' in sconf.dirhtml:
        fns = [ pjoin(conf.paths.projectroot,
                      conf.paths.public_site_output,
                      fn)
                for fn in sconf.dirhtml.excluded_files ]

        cleaner(fns)
        logging.info('removed excluded files from dirhtml output directory')
예제 #18
0
def build_job_prerequsites(sync, sconf, conf):
    runner(external_jobs(conf), parallel='thread')

    with update_source_lock:
        if conf.project.name != 'mms':
            if sync.satisfied('transfered_source') is False:
                transfer_source(sconf, conf)
                sync.transfered_source = True
            cond_toc = "build_toc"
        else:
            cond_name = 'transfered_' + sconf.edition
            cond_toc = 'build_toc_' + sconf.edition
            if sync.satisfied(cond_name) is False:
                cmd = 'make -C {0} {1}-source-dir={0}{2}{3} EDITION={1} generate-source-{1}'
                cmd = cmd.format(conf.paths.projectroot, sconf.edition, os.path.sep,
                                 conf.paths.branch_source)
                o = command(cmd, capture=True)
                if len(o.out.strip()) > 0:
                    print(o.out)

                sync[cond_name] = True

        if sync.satisfied(cond_toc) is False:
            # this has to go here so that MMS can generate different toc trees for
            # each edition.
            runner(toc_jobs(conf), parallel='process')
            sync[cond_toc] = True

    with update_deps_lock:
        if sync.satisfied('updated_deps') is False:
            print('[sphinx-prep]: resolving all intra-source dependencies now. (takes several seconds)')
            dep_count = refresh_dependencies(conf)
            print('[sphinx-prep]: bumped timestamps of {0} files'.format(dep_count))
            sync.updated_deps = True

            command(build_platform_notification('Sphinx', 'Build in progress pastb critical phase.'), ignore=True)
            print('[sphinx-prep]: INFO - Build in progress past critical phase ({0})'.format(conf.paths.branch_source))
            dump_file_hashes(conf)

    print('[sphinx-prep]: build environment prepared for sphinx.')
예제 #19
0
def finalize_dirhtml_build(builder, conf):
    pjoin = os.path.join

    single_html_dir = get_single_html_dir(conf)
    search_page = pjoin(conf.paths.branch_output, builder, 'index.html')

    if os.path.exists(search_page):
        copy_if_needed(source_file=search_page,
                       target_file=pjoin(single_html_dir, 'search.html'))

    dest = pjoin(conf.paths.projectroot, conf.paths.public_site_output)
    command('rsync -a {source}/ {destination}'.format(source=pjoin(
        conf.paths.projectroot, conf.paths.branch_output, builder),
                                                      destination=dest))

    logger.info('{0}: migrated build to {1}'.format(builder, dest))

    if 'published' in conf.git.branches and conf.git.branches.current in conf.git.branches.published:
        sitemap_exists = sitemap(config_path=None, conf=conf)

        if sitemap_exists is True:
            copy_if_needed(source_file=pjoin(conf.paths.projectroot,
                                             conf.paths.branch_output,
                                             'sitemap.xml.gz'),
                           target_file=pjoin(conf.paths.projectroot,
                                             conf.paths.public_site_output,
                                             'sitemap.xml.gz'))

    sconf = BuildConfiguration(
        'sphinx.yaml', pjoin(conf.paths.projectroot, conf.paths.builddata))

    if 'dirhtml' in sconf and 'excluded_files' in sconf.dirhtml:
        fns = [
            pjoin(conf.paths.projectroot, conf.paths.public_site_output, fn)
            for fn in sconf.dirhtml.excluded_files
        ]

        cleaner(fns)
        logging.info('removed excluded files from dirhtml output directory')
예제 #20
0
def generate_source(sconf, conf=None):
    conf = lazy_config(conf)

    target = sconf.build_source

    if not os.path.exists(target):
        os.makedirs(target)
        print('[sphinx-prep]: created ' + target)
    elif not os.path.isdir(sconf.build_source):
        abort('[sphinx-prep]: {0} exists and is not a directory'.format(target))

    r = command('rsync --checksum --recursive --delete {0}/ {1}'.format(sconf.source, target), capture=True)
    print('[sphinx]: updated source in {0}'.format(target))
예제 #21
0
def setup_config_remote(branch_name, conf):
    remotes = command('git remote', capture=True).out.split('\n')

    if 'config-upstream' not in remotes:
        if conf.git.remote.upstream.startswith('10gen'):
            git_url = '[email protected]:'
        else:
            git_url = 'git://github.com/'

        command('git remote add config-upstream {0}{1}.git'.format(git_url, conf.git.remote.upstream))

        command('git fetch config-upstream')

        if branch_name not in command('git branch', capture=True).out.split('\n'):
            command('git branch {0} config-upstream/{0}'.format(branch_name))
예제 #22
0
def build_prerequisites(conf):
    jobs = itertools.chain(build_prereq_jobs(conf),
                           manpage_jobs(conf),
                           table_jobs(conf),
                           api_jobs(conf),
                           toc_jobs(conf),
                           option_jobs(conf),
                           steps_jobs(conf),
                           release_jobs(conf),
                           intersphinx_jobs(conf),
                           image_jobs(conf))

    try:
        res = runner(jobs, parallel='process')
        print('[sphinx-prep]: built {0} pieces of content'.format(len(res)))
    except PoolResultsError:
        print('[WARNING]: sphinx prerequisites encountered errors. '
              'See output above. Continuing as a temporary measure.')

    runner(external_jobs(conf), parallel='thread')

    buildinfo_hash(conf)

    if conf.project.name != 'mms':
        # we copy source manually for mms in makefile.mms, avoiding this
        # operation to clarify the artifacts directory
        transfer_source(conf)

    print('[sphinx-prep]: resolving all intra-source dependencies now. (takes several seconds)')
    dep_count = refresh_dependencies(conf)
    print('[sphinx-prep]: bumped timestamps of {0} files'.format(dep_count))

    command(build_platform_notification('Sphinx', 'Build in progress pastb critical phase.'), ignore=True)

    print('[sphinx-prep]: INFO - Build in progress past critical phase.')

    dump_file_hashes(conf.system.dependency_cache, conf)
    print('[sphinx-prep]: build environment prepared for sphinx.')
예제 #23
0
def setup_config_remote(branch_name, conf):
    remotes = command('git remote', capture=True).out.split('\n')

    if 'config-upstream' not in remotes:
        if conf.git.remote.upstream.startswith('10gen'):
            git_url = '[email protected]:'
        else:
            git_url = 'git://github.com/'

        command('git remote add config-upstream {0}{1}.git'.format(
            git_url, conf.git.remote.upstream))

        command('git fetch config-upstream')

        if branch_name not in command('git branch',
                                      capture=True).out.split('\n'):
            command('git branch {0} config-upstream/{0}'.format(branch_name))
예제 #24
0
def render_versions(conf=None):
    if is_processed('versions', conf):
        return conf
    else:
        conf = lazy_conf(conf)

        if 'branches' not in conf.git:
            conf.git.branches = AttributeDict()

        version_config_file = os.path.join(conf.paths.builddata,
                                           'published_branches.yaml')

        try:
            vconf_data = get_file_from_branch(version_config_file, 'master')
        except CommandError:
            if get_branch() == 'master':
                return conf

            remotes = command('git remote', capture=True).out.split('\n')
            if 'origin' in remotes:
                return conf

            if 'config-upstream' not in remotes:
                command('git remote add config-upstream git://github.com/{0}.git'.format(conf.git.remote.upstream))

            command('git fetch config-upstream')

            if 'master' not in command('git branch', capture=True).out.split('\n'):
                command('git branch master config-upstream/master')

            vconf_data = get_file_from_branch(version_config_file, 'master')
        except CommandError:
            return conf

        vconf = AttributeDict(yaml.load(vconf_data))

        conf.version.update(vconf.version)

        conf.git.branches.update(vconf.git.branches)

        conf.system.processed.versions = True

        return conf
예제 #25
0
def render_versions(conf=None):
    if is_processed('versions', conf):
        return conf
    else:
        conf = lazy_conf(conf)

        if 'branches' not in conf.git:
            conf.git.branches = AttributeDict()

        version_config_file = os.path.join(conf.paths.builddata,
                                           'published_branches.yaml')

        try:
            vconf_data = get_file_from_branch(version_config_file, 'master')
        except CommandError:
            if get_branch() == 'master':
                return conf

            remotes = command('git remote', capture=True).out.split('\n')
            if 'origin' in remotes:
                return conf

            if 'config-upstream' not in remotes:
                command('git remote add config-upstream git://github.com/{0}.git'.format(conf.git.remote.upstream))

            command('git fetch config-upstream')

            if 'master' not in command('git branch', capture=True).out.split('\n'):
                command('git branch master config-upstream/master')

            vconf_data = get_file_from_branch(version_config_file, 'master')
        except CommandError:
            return conf

        vconf = AttributeDict(yaml.load(vconf_data))

        conf.version.update(vconf.version)

        conf.git.branches.update(vconf.git.branches)

        conf.system.processed.versions = True

        return conf
예제 #26
0
def build_job_prerequsites(sync, sconf, conf):
    runner(external_jobs(conf), parallel='thread')

    with update_source_lock:
        cond_toc = "build_toc"
        cond_name = 'transfered_source'
        cond_dep = 'updated_deps'
        cond_steps = 'build_step_files'

        if conf.project.name in ['mms', 'training']:
            cond_name += '_' + sconf.edition
            cond_toc += '_' + sconf.edition
            cond_dep += '_' + sconf.edition
            cond_steps += '_' + sconf.edition

        if sync.satisfied(cond_name) is False:
            transfer_source(sconf, conf)
            sync[cond_name] = True

        if 'excluded' in sconf:
            logger.info('removing excluded files')
            for fn in sconf.excluded:
                fqfn = os.path.join(conf.paths.projectroot,
                                    conf.paths.branch_source, fn[1:])
                if os.path.exists(fqfn):
                    if os.path.isdir(fqfn):
                        rmtree(fqfn)
                    else:
                        os.remove(fqfn)
                    logger.debug('removed {0}'.format(fqfn))

            logger.info('removed {0} files'.format(len(sconf.excluded)))

        with ProcessPool() as p:
            # these must run here so that MMS can generate different toc/steps/etc for
            # each edition.

            if sync.satisfied(cond_toc) is False:
                # even if this fails we don't want it to run more than once
                sync[cond_toc] = True
                tr = p.runner(toc_jobs(conf))
                logger.info('generated {0} toc files'.format(len(tr)))

            if sync.satisfied(cond_steps) is False:
                sync[cond_steps] = True
                sr = p.runner(steps_jobs(conf))
                logger.info('generated {0} step files'.format(len(sr)))

        if sync.satisfied(cond_dep) is False:
            logger.debug('using update deps lock.')

            logger.info(
                'resolving all intra-source dependencies now. for sphinx build. (takes several seconds)'
            )
            dep_count = refresh_dependencies(conf)
            logger.info('bumped {0} dependencies'.format(dep_count))
            sync[cond_dep] = True

            command(build_platform_notification(
                'Sphinx', 'Build in progress past critical phase.'),
                    ignore=True)
            logger.info(
                'sphinx build in progress past critical phase ({0})'.format(
                    conf.paths.branch_source))
            dump_file_hashes(conf)
        else:
            logger.debug('dependencies already updated, lock unneeded.')

        logger.debug('releasing dependency update lock.')

    logging.info('build environment prepared for sphinx build {0}.'.format(
        sconf.builder))
예제 #27
0
def get_commit(path=None):
    return command('git rev-parse --verify HEAD', capture=True).out
예제 #28
0
    def cmd(self, *args):
        args = ' '.join(args)

        return command(command='cd {0} ; git {1}'.format(self.path, args), capture=True)
예제 #29
0
def get_file_from_branch(path, branch='master'):
    cmd = 'git show {branch}:{path}'.format(branch=branch, path=path)

    return command(command=cmd, capture=True, ignore=False).out
예제 #30
0
def checkout_file(path, branch='master'):
    return command(command='git checkout {0} -- {1}'.format(branch, path),
                   capture=False, ignore=False)
예제 #31
0
def get_branch(path=None):
    return command('git symbolic-ref HEAD', capture=True).out.split('/')[2]
예제 #32
0
def get_commit(path=None):
    return command('git rev-parse --verify HEAD', capture=True).out
예제 #33
0
def dirhtml_migration(conf):
    cmd = 'rsync -a {source}/ {destination}'
    command(cmd.format(source=os.path.join(conf.paths.projectroot,
                                           conf.paths.output, 'dirhtml'),
                       destination=os.path.join(conf.paths.projectroot,
                                                conf.paths.public)))
예제 #34
0
def build_job_prerequsites(sync, sconf, conf):
    runner(external_jobs(conf), parallel='thread')

    with update_source_lock:
        cond_toc = "build_toc"
        cond_name = 'transfered_source'
        cond_dep = 'updated_deps'
        cond_steps = 'build_step_files'

        if conf.project.name in ['mms', 'training']:
            cond_name += '_' + sconf.edition
            cond_toc += '_' + sconf.edition
            cond_dep += '_' + sconf.edition
            cond_steps += '_' + sconf.edition

        if sync.satisfied(cond_name) is False:
            transfer_source(sconf, conf)
            sync[cond_name] = True

        if 'excluded' in sconf:
            logger.info('removing excluded files')
            for fn in sconf.excluded:
                fqfn = os.path.join(conf.paths.projectroot, conf.paths.branch_source, fn[1:])
                if os.path.exists(fqfn):
                    if os.path.isdir(fqfn):
                        rmtree(fqfn)
                    else:
                        os.remove(fqfn)
                    logger.debug('removed {0}'.format(fqfn))

            logger.info('removed {0} files'.format(len(sconf.excluded)))

        with ProcessPool() as p:
            # these must run here so that MMS can generate different toc/steps/etc for
            # each edition.

            if sync.satisfied(cond_toc) is False:
                # even if this fails we don't want it to run more than once
                sync[cond_toc] = True
                tr = p.runner(toc_jobs(conf))
                logger.info('generated {0} toc files'.format(len(tr)))

            if sync.satisfied(cond_steps) is False:
                sync[cond_steps] = True
                sr = p.runner(steps_jobs(conf))
                logger.info('generated {0} step files'.format(len(sr)))

        if sync.satisfied(cond_dep) is False:
            logger.debug('using update deps lock.')

            logger.info('resolving all intra-source dependencies now. for sphinx build. (takes several seconds)')
            dep_count = refresh_dependencies(conf)
            logger.info('bumped {0} dependencies'.format(dep_count))
            sync[cond_dep] = True

            command(build_platform_notification('Sphinx', 'Build in progress past critical phase.'), ignore=True)
            logger.info('sphinx build in progress past critical phase ({0})'.format(conf.paths.branch_source))
            dump_file_hashes(conf)
        else:
            logger.debug('dependencies already updated, lock unneeded.')

        logger.debug('releasing dependency update lock.')

    logging.info('build environment prepared for sphinx build {0}.'.format(sconf.builder))
예제 #35
0
def main():
    user = user_input()
    setup_logging(user)

    if user.repo == '[email protected]:mongodb/docs.git' and user.project != 'manual':
        msg = '[test]: project and repo are not correctly matched'
        logger.error(msg)
        exit(1)

    if not os.path.exists('build'):
        os.makedirs('build')
    elif not os.path.isdir('build'):
        logger.warning(
            'build exists but is not a directory. please investigate.')
        os.remove('build')

    root_path = os.path.abspath(os.getcwd())
    build_path = os.path.join('build', user.project)

    if os.path.exists(build_path):
        logger.info(
            'build directory exists. continuing with quasi-incremental build.')
    else:
        logger.info('cloning repository')
        command('git clone {0} {1}'.format(user.repo, build_path))
        logger.info('cloned repository')

    os.chdir(build_path)
    logger.debug('script working directory is now {0}'.format(os.getcwd()))

    command('git fetch --all')
    logger.debug('git fetched testing repository {0} in {1}'.format(
        user.project, build_path))

    try:
        command('git checkout {0}'.format(user.branch))
    except CommandError:
        command('git checkout -b {0} origin/{0}'.format(user.branch))
    except CommandError:
        logger.error('branch name {0} does not exist in remote'.format(
            user.branch))
        exit(1)

    command('git reset --hard HEAD~12')
    logger.debug('re-set testing head 12 commits')
    command('git pull')
    logger.debug('ran "git pull" on testing repo')

    bootstrapped_tools_path = os.path.join('build', 'docs-tools')

    if not os.path.exists(bootstrapped_tools_path):
        logger.debug("{0} does not exist".format(bootstrapped_tools_path))
        symlink(name=bootstrapped_tools_path, target=root_path)
        logger.debug('created tools symlink')
    elif os.path.islink(bootstrapped_tools_path):
        logger.debug(
            "{0} is a link. continuing.".format(bootstrapped_tools_path))
    elif os.path.isdir(bootstrapped_tools_path
                       ) and not os.path.islink(bootstrapped_tools_path):
        logger.warning('a tools directory currently exists, removing.')
        shutil.rmtree(bootstrapped_tools_path)
        symlink(name=bootstrapped_tools_path, target=root_path)
        logger.debug('created tools symlink')

    logger.info('bootstrapping.')
    command('python bootstrap.py safe')
    logger.info('moving on to build the publish target.')

    build_task = command('make publish', capture=True, ignore=True)
    logger.info('completed build task, moving to printing output')

    print_build_output(build_task)

    log_and_propogate_task_return(build_task)
예제 #36
0
def main():
    user = user_input()
    setup_logging(user)

    if user.repo == '[email protected]:mongodb/docs.git' and user.project != 'manual':
        msg = '[test]: project and repo are not correctly matched'
        logger.error(msg)
        exit(1)

    if not os.path.exists('build'):
        os.makedirs('build')
    elif not os.path.isdir('build'):
        logger.warning('build exists but is not a directory. please investigate.')
        os.remove('build')

    root_path = os.path.abspath(os.getcwd())
    build_path = os.path.join('build', user.project)

    command('git remote add origin [email protected]:mongodb/docs-tools.git', ignore=True)
    command('git branch master origin/master', ignore=True)

    if os.path.exists(build_path):
        logger.info('build directory exists. continuing with quasi-incremental build.')
    else:
        logger.info('cloning repository')
        command('git clone {0} {1}'.format(user.repo, build_path))
        logger.info('cloned repository')

    if user.toolchain == 'giza':
        os.chdir('giza')

        try:
            command('python setup.py install')
        except CommandError:
            command('python setup.py install')
        except CommandError:
            command('python setup.py install')
        except CommandError:
            logger.critical('giza installation failed after three attempts. declaring failure.')
            exit(1)

        os.chdir(root_path)
        logger.info('installed giza')

    os.chdir(build_path)
    logger.debug('script working directory is now {0}'.format(os.getcwd()))

    command('git fetch --all')
    logger.debug('git fetched testing repository {0} in {1}'.format(user.project, build_path))

    try:
        command('git checkout {0}'.format(user.branch))
    except CommandError:
        command('git checkout -b {0} origin/{0}'.format(user.branch))
    except CommandError:
        logger.error('branch name {0} does not exist in remote'.format(user.branch))
        exit(1)

    command('git reset --hard HEAD~3')
    logger.debug('re-set testing head 12 commits')
    command('git pull')
    logger.debug('ran "git pull" on testing repo')

    bootstrapped_tools_path = os.path.join('build', 'docs-tools')

    if user.toolchain == 'giza':
        if os.path.exists(bootstrapped_tools_path):
            try:
                os.remove(bootstrapped_tools_path)
            except:
                shutil.rmtree(bootstrapped_tools_path)
    else:
        if not os.path.exists(bootstrapped_tools_path):
            os.makedirs('build')
            logger.debug("{0} does not exist".format(bootstrapped_tools_path))
            symlink(name=bootstrapped_tools_path, target=root_path)
            logger.debug('created tools symlink')
        elif os.path.islink(bootstrapped_tools_path):
            logger.debug("{0} is a link. continuing.".format(bootstrapped_tools_path))
        elif os.path.isdir(bootstrapped_tools_path) and not os.path.islink(bootstrapped_tools_path):
            logger.warning('a tools directory currently exists, removing.')
            shutil.rmtree(bootstrapped_tools_path)
            symlink(name=bootstrapped_tools_path, target=root_path)
            logger.debug('created tools symlink')

    logger.info('bootstrapping.')
    command('python bootstrap.py safe')
    logger.info('moving on to build the publish target.')

    if user.toolchain == 'giza':
        build_task = command('make giza-publish', capture=True, ignore=True)
    elif user.toolchain == 'legacy':
        build_task = command('make publish', capture=True, ignore=True)

    logger.info('completed build task, moving to printing output')

    print_build_output(build_task)

    log_and_propogate_task_return(build_task)
예제 #37
0
    def cmd(self, *args):
        args = ' '.join(args)

        return command(command='cd {0} ; git {1}'.format(self.path, args),
                       capture=True)
예제 #38
0
def get_file_from_branch(path, branch='master'):
    cmd = 'git show {branch}:{path}'.format(branch=branch, path=path)

    return command(command=cmd, capture=True, ignore=False).out
예제 #39
0
def checkout_file(path, branch='master'):
    return command(command='git checkout {0} -- {1}'.format(branch, path),
                   capture=False,
                   ignore=False)
예제 #40
0
def get_branch(path=None):
    return command('git symbolic-ref HEAD', capture=True).out.split('/')[2]