Beispiel #1
0
def main(lang, src_info, version, lang_config, gendate,
         verbose=False, desconectado=False, procesar_articles=True):
    # don't affect the rest of the machine
    make_it_nicer()

    if procesar_articles:
        try:
            import SuffixTree  # NOQA
        except ImportError:
            logger.warning(NO_ST_MSG)

    # validate lang and versions, and fix config with selected data
    logger.info("Fixing config for lang=%r version=%r", lang, version)
    try:
        _lang_conf = config.imagtypes[lang]
    except KeyError:
        print "Not a valid language! try one of", config.imagtypes.keys()
        exit()
    try:
        config.imageconf = _lang_conf[version]
    except KeyError:
        print "Not a valid version! try one of", _lang_conf.keys()
        exit()
    config.langconf = lang_config

    logger.info("Starting!")
    preparaTemporal(procesar_articles)

    logger.info("Copying the assets and locale files")
    copy_assets(src_info, config.DIR_ASSETS)
    shutil.copytree('locale', path.join(config.DIR_CDBASE, "locale"))

    articulos = path.join(src_info, "articles")
    if procesar_articles:
        logger.info("Preprocessing")
        if not path.exists(articulos):
            logger.error("Couldn't find articles dir: %r", articulos)
            raise EnvironmentError("Directory not found, can't continue")
            sys.exit()
        cantnew, cantold = preprocesar.run(articulos)
        logger.info("Processed pages: %d new, %d from before",
                    cantnew, cantold)

        logger.info("Calculating which stay and which don't")
        preprocesar.pages_selector.calculate()

        logger.info("Generating the images log")
        taken, adesc = extract.run()
        logger.info("Extracted %d images, need to download %d", taken, adesc)
    else:
        logger.info("Avoid processing articles and generating images log")

    logger.info("Recalculating the reduction percentages.")
    calcular.run()

    if not desconectado:
        logger.info("Downloading the images from the internet")
        download.retrieve()

    logger.info("Reducing the downloaded images")
    reducir.run(verbose)

    logger.info("Putting the reduced images into blocks")
    # agrupamos las imagenes en bloques
    q_blocks, q_images = ImageManager.generar_bloques(verbose)
    logger.info("Got %d blocks with %d images", q_blocks, q_images)

    if not procesar_articles:
        logger.info("Not generating index and blocks (by user request)")
    elif preprocesar.pages_selector.same_info_through_runs:
        logger.info("Same articles than previous run "
                    "(not generating index and blocks)")
    else:
        logger.info("Generating the index")
        result = cdpindex.generar_de_html(articulos, verbose)
        logger.info("Got %d files", result)
        logger.info("Generating the articles blocks")
        q_blocks, q_files, q_redirs = ArticleManager.generar_bloques(lang,
                                                                     verbose)
        logger.info("Got %d blocks with %d files and %d redirects",
                    q_blocks, q_files, q_redirs)

    logger.info("Copying the sources")
    copy_sources()

    logger.info("Generating the links to blocks and indexes")
    # blocks
    dest = path.join(config.DIR_CDBASE, "cdpedia", "bloques")
    if os.path.exists(dest):
        os.remove(dest)
    os.symlink(path.abspath(config.DIR_BLOQUES), dest)
    # indexes
    dest = path.join(config.DIR_CDBASE, "cdpedia", "indice")
    if os.path.exists(dest):
        os.remove(dest)
    os.symlink(path.abspath(config.DIR_INDICE), dest)

    if config.imageconf["windows"]:
        logger.info("Copying Windows stuff")
        # generated by pyinstaller 2.0
        copy_dir("resources/autorun.win/cdroot", config.DIR_CDBASE)

    logger.info("Generating runtime config")
    genera_run_config()

    base_dest_name = "cdpedia-%s-%s-%s-%s" % (lang, config.VERSION, gendate, version)
    if config.imageconf["type"] == "iso":
        logger.info("Building the ISO: %r", base_dest_name)
        build_iso(base_dest_name)
    elif config.imageconf["type"] == "tarball":
        logger.info("Building the tarball: %r", base_dest_name)
        build_tarball(base_dest_name)
    else:
        raise ValueError("Unrecognized image type")

    logger.info("All done!")
Beispiel #2
0
def main(src_info, evitar_iso, verbose, desconectado,
         procesar_articles, include_windows, tarball):

    if procesar_articles:
        try:
            import SuffixTree
        except ImportError:
            print NO_ST_MSG

    articulos = path.join(src_info, "articles")

    mensaje("Comenzando!")
    preparaTemporal(procesar_articles)

    mensaje("Copiando los assets")
    copiarAssets(src_info, config.DIR_ASSETS)

    if procesar_articles:
        mensaje("Preprocesando")
        if not path.exists(articulos):
            print "\nERROR: No se encuentra el directorio %r" % articulos
            print "Este directorio es obligatorio para el procesamiento general"
            sys.exit()
        cantnew, cantold = preprocesar.run(articulos, verbose)
        print '  total %d páginas procesadas' % cantnew
        print '      y %d que ya estaban de antes' % cantold

        mensaje("Calculando los que quedan y los que no")
        preprocesar.calcula_top_htmls()

        mensaje("Generando el log de imágenes")
        taken, adesc = extraer.run(verbose)
        print '  total: %5d imágenes extraídas' % taken
        print '         %5d a descargar' % adesc
    else:
        mensaje("Evitamos procesar artículos y generar el log de imágenes")

    mensaje("Recalculando porcentajes de reducción")
    calcular.run(verbose)

    if not desconectado:
        mensaje("Descargando las imágenes de la red")
        download.traer(verbose)

    mensaje("Reduciendo las imágenes descargadas")
    notfound = reducir.run(verbose)

    mensaje("Emblocando las imágenes reducidas")
    # agrupamos las imagenes en bloques
    result = ImageManager.generar_bloques(verbose)
    print '  total: %d bloques con %d imags' % result

    if procesar_articles:
        mensaje("Generando el índice")
        result = cdpindex.generar_de_html(articulos, verbose)
        print '  total: %d archivos' % result

        mensaje("Generando los bloques de artículos")
        result = ArticleManager.generar_bloques(verbose)
        print '  total: %d bloques con %d archivos y %d redirects' % result
    else:
        mensaje("Evitamos generar el índice y los bloques")

    mensaje("Copiando las fuentes")
    copiarSources()

    mensaje("Copiando los indices")
    dest_src = path.join(config.DIR_CDBASE, "cdpedia", "indice")
    if os.path.exists(dest_src):
        shutil.rmtree(dest_src)
    shutil.copytree(config.DIR_INDICE, dest_src)

    if include_windows:
        mensaje("Copiando cosas para Windows")
        copy_dir("resources/autorun.win/cdroot", config.DIR_CDBASE)

    mensaje("Generamos la config para runtime")
    genera_run_config()

    if not evitar_iso:
        mensaje("Armamos el ISO")
        armarIso("cdpedia.iso")

    if tarball:
        mensaje("Armamos el tarball con %r" % (tarball,))
        build_tarball(tarball)

    mensaje("Todo terminado!")
Beispiel #3
0
def main(lang,
         src_info,
         version,
         lang_config,
         gendate,
         verbose=False,
         desconectado=False,
         procesar_articles=True):
    # don't affect the rest of the machine
    make_it_nicer()

    if procesar_articles:
        try:
            import SuffixTree  # NOQA
        except ImportError:
            logger.warning(
                "Import error on SuffixTree; compressed index generation will be REALLY slow. "
                "Please install it (download, python2 setup.py build, python2 setup.py install) "
                "from here:  http://taniquetil.com.ar/facundo/SuffixTree-0.7.1-8bit.tar.bz2"
            )

    # validate lang and versions, and fix config with selected data
    logger.info("Fixing config for lang=%r version=%r", lang, version)
    try:
        _lang_conf = config.imagtypes[lang]
    except KeyError:
        print("ERROR: %r is not a valid language! try one of %s" %
              (lang, config.imagtypes.keys()))
        exit()
    try:
        config.imageconf = _lang_conf[version]
    except KeyError:
        print("ERROR: %r is not a valid version! try one of %s" %
              (version, _lang_conf.keys()))
        exit()
    config.langconf = lang_config

    logger.info("Starting!")
    preparaTemporal(procesar_articles)

    logger.info("Copying the assets and locale files")
    copy_assets(src_info, config.DIR_ASSETS)
    shutil.copytree('locale', path.join(config.DIR_CDBASE, "locale"))

    articulos = path.join(src_info, "articles")
    if procesar_articles:
        logger.info("Preprocessing")
        if not path.exists(articulos):
            logger.error("Couldn't find articles dir: %r", articulos)
            raise EnvironmentError("Directory not found, can't continue")
            sys.exit()
        preprocesar.run(articulos)

        logger.info("Calculating which stay and which don't")
        preprocesar.pages_selector.calculate()

        logger.info("Generating the images log")
        taken, adesc = extract.run()
        logger.info("Extracted %d images, need to download %d", taken, adesc)
    else:
        logger.info("Avoid processing articles and generating images log")

    logger.info("Recalculating the reduction percentages.")
    calcular.run()

    if not desconectado:
        logger.info("Downloading the images from the internet")
        download.retrieve()

    logger.info("Reducing the downloaded images")
    reducir.run(verbose)

    logger.info("Putting the reduced images into blocks")
    # agrupamos las imagenes en bloques
    q_blocks, q_images = ImageManager.generar_bloques(verbose)
    logger.info("Got %d blocks with %d images", q_blocks, q_images)

    if not procesar_articles:
        logger.info("Not generating index and blocks (by user request)")
    elif preprocesar.pages_selector.same_info_through_runs:
        logger.info("Same articles than previous run "
                    "(not generating index and blocks)")
    else:
        logger.info("Generating the index")
        result = cdpindex.generar_de_html(articulos, verbose)
        logger.info("Got %d files", result)
        logger.info("Generating the articles blocks")
        q_blocks, q_files, q_redirs = ArticleManager.generar_bloques(
            lang, verbose)
        logger.info("Got %d blocks with %d files and %d redirects", q_blocks,
                    q_files, q_redirs)

    logger.info("Copying the sources")
    copy_sources()

    logger.info("Generating the links to blocks and indexes")
    # blocks
    dest = path.join(config.DIR_CDBASE, "cdpedia", "bloques")
    if os.path.exists(dest):
        os.remove(dest)
    os.symlink(path.abspath(config.DIR_BLOQUES), dest)
    # indexes
    dest = path.join(config.DIR_CDBASE, "cdpedia", "indice")
    if os.path.exists(dest):
        os.remove(dest)
    os.symlink(path.abspath(config.DIR_INDICE), dest)

    if config.imageconf["windows"]:
        logger.info("Copying Windows stuff")
        # generated by pyinstaller 2.0
        copy_dir("resources/autorun.win/cdroot", config.DIR_CDBASE)

    logger.info("Generating runtime config")
    genera_run_config()

    base_dest_name = "cdpedia-%s-%s-%s-%s" % (lang, config.VERSION, gendate,
                                              version)
    if config.imageconf["type"] == "iso":
        logger.info("Building the ISO: %r", base_dest_name)
        build_iso(base_dest_name)
    elif config.imageconf["type"] == "tarball":
        logger.info("Building the tarball: %r", base_dest_name)
        build_tarball(base_dest_name)
    else:
        raise ValueError("Unrecognized image type")

    logger.info("All done!")