def main(lang, src_info, version, lang_config, gendate,
verbose=False, desconectado=False, procesar_articles=True):
# don't affect the rest of the machine
make_it_nicer()
if procesar_articles:
try:
import SuffixTree # NOQA
except ImportError:
logger.warning(NO_ST_MSG)
# validate lang and versions, and fix config with selected data
logger.info("Fixing config for lang=%r version=%r", lang, version)
try:
_lang_conf = config.imagtypes[lang]
except KeyError:
print "Not a valid language! try one of", config.imagtypes.keys()
exit()
try:
config.imageconf = _lang_conf[version]
except KeyError:
print "Not a valid version! try one of", _lang_conf.keys()
exit()
config.langconf = lang_config
logger.info("Starting!")
preparaTemporal(procesar_articles)
logger.info("Copying the assets and locale files")
copy_assets(src_info, config.DIR_ASSETS)
shutil.copytree('locale', path.join(config.DIR_CDBASE, "locale"))
articulos = path.join(src_info, "articles")
if procesar_articles:
logger.info("Preprocessing")
if not path.exists(articulos):
logger.error("Couldn't find articles dir: %r", articulos)
raise EnvironmentError("Directory not found, can't continue")
sys.exit()
cantnew, cantold = preprocesar.run(articulos)
logger.info("Processed pages: %d new, %d from before",
cantnew, cantold)
logger.info("Calculating which stay and which don't")
preprocesar.pages_selector.calculate()
logger.info("Generating the images log")
taken, adesc = extract.run()
logger.info("Extracted %d images, need to download %d", taken, adesc)
else:
logger.info("Avoid processing articles and generating images log")
logger.info("Recalculating the reduction percentages.")
calcular.run()
if not desconectado:
logger.info("Downloading the images from the internet")
download.retrieve()
logger.info("Reducing the downloaded images")
reducir.run(verbose)
logger.info("Putting the reduced images into blocks")
# agrupamos las imagenes en bloques
q_blocks, q_images = ImageManager.generar_bloques(verbose)
logger.info("Got %d blocks with %d images", q_blocks, q_images)
if not procesar_articles:
logger.info("Not generating index and blocks (by user request)")
elif preprocesar.pages_selector.same_info_through_runs:
logger.info("Same articles than previous run "
"(not generating index and blocks)")
else:
logger.info("Generating the index")
result = cdpindex.generar_de_html(articulos, verbose)
logger.info("Got %d files", result)
logger.info("Generating the articles blocks")
q_blocks, q_files, q_redirs = ArticleManager.generar_bloques(lang,
verbose)
logger.info("Got %d blocks with %d files and %d redirects",
q_blocks, q_files, q_redirs)
logger.info("Copying the sources")
copy_sources()
logger.info("Generating the links to blocks and indexes")
# blocks
dest = path.join(config.DIR_CDBASE, "cdpedia", "bloques")
if os.path.exists(dest):
os.remove(dest)
os.symlink(path.abspath(config.DIR_BLOQUES), dest)
# indexes
dest = path.join(config.DIR_CDBASE, "cdpedia", "indice")
if os.path.exists(dest):
os.remove(dest)
os.symlink(path.abspath(config.DIR_INDICE), dest)
if config.imageconf["windows"]:
logger.info("Copying Windows stuff")
# generated by pyinstaller 2.0
copy_dir("resources/autorun.win/cdroot", config.DIR_CDBASE)
logger.info("Generating runtime config")
genera_run_config()
base_dest_name = "cdpedia-%s-%s-%s-%s" % (lang, config.VERSION, gendate, version)
if config.imageconf["type"] == "iso":
logger.info("Building the ISO: %r", base_dest_name)
build_iso(base_dest_name)
elif config.imageconf["type"] == "tarball":
logger.info("Building the tarball: %r", base_dest_name)
build_tarball(base_dest_name)
else:
raise ValueError("Unrecognized image type")
logger.info("All done!")
def main(src_info, evitar_iso, verbose, desconectado,
procesar_articles, include_windows, tarball):
if procesar_articles:
try:
import SuffixTree
except ImportError:
print NO_ST_MSG
articulos = path.join(src_info, "articles")
mensaje("Comenzando!")
preparaTemporal(procesar_articles)
mensaje("Copiando los assets")
copiarAssets(src_info, config.DIR_ASSETS)
if procesar_articles:
mensaje("Preprocesando")
if not path.exists(articulos):
print "\nERROR: No se encuentra el directorio %r" % articulos
print "Este directorio es obligatorio para el procesamiento general"
sys.exit()
cantnew, cantold = preprocesar.run(articulos, verbose)
print ' total %d páginas procesadas' % cantnew
print ' y %d que ya estaban de antes' % cantold
mensaje("Calculando los que quedan y los que no")
preprocesar.calcula_top_htmls()
mensaje("Generando el log de imágenes")
taken, adesc = extraer.run(verbose)
print ' total: %5d imágenes extraídas' % taken
print ' %5d a descargar' % adesc
else:
mensaje("Evitamos procesar artículos y generar el log de imágenes")
mensaje("Recalculando porcentajes de reducción")
calcular.run(verbose)
if not desconectado:
mensaje("Descargando las imágenes de la red")
download.traer(verbose)
mensaje("Reduciendo las imágenes descargadas")
notfound = reducir.run(verbose)
mensaje("Emblocando las imágenes reducidas")
# agrupamos las imagenes en bloques
result = ImageManager.generar_bloques(verbose)
print ' total: %d bloques con %d imags' % result
if procesar_articles:
mensaje("Generando el índice")
result = cdpindex.generar_de_html(articulos, verbose)
print ' total: %d archivos' % result
mensaje("Generando los bloques de artículos")
result = ArticleManager.generar_bloques(verbose)
print ' total: %d bloques con %d archivos y %d redirects' % result
else:
mensaje("Evitamos generar el índice y los bloques")
mensaje("Copiando las fuentes")
copiarSources()
mensaje("Copiando los indices")
dest_src = path.join(config.DIR_CDBASE, "cdpedia", "indice")
if os.path.exists(dest_src):
shutil.rmtree(dest_src)
shutil.copytree(config.DIR_INDICE, dest_src)
if include_windows:
mensaje("Copiando cosas para Windows")
copy_dir("resources/autorun.win/cdroot", config.DIR_CDBASE)
mensaje("Generamos la config para runtime")
genera_run_config()
if not evitar_iso:
mensaje("Armamos el ISO")
armarIso("cdpedia.iso")
if tarball:
mensaje("Armamos el tarball con %r" % (tarball,))
build_tarball(tarball)
mensaje("Todo terminado!")
def main(lang,
src_info,
version,
lang_config,
gendate,
verbose=False,
desconectado=False,
procesar_articles=True):
# don't affect the rest of the machine
make_it_nicer()
if procesar_articles:
try:
import SuffixTree # NOQA
except ImportError:
logger.warning(
"Import error on SuffixTree; compressed index generation will be REALLY slow. "
"Please install it (download, python2 setup.py build, python2 setup.py install) "
"from here: http://taniquetil.com.ar/facundo/SuffixTree-0.7.1-8bit.tar.bz2"
)
# validate lang and versions, and fix config with selected data
logger.info("Fixing config for lang=%r version=%r", lang, version)
try:
_lang_conf = config.imagtypes[lang]
except KeyError:
print("ERROR: %r is not a valid language! try one of %s" %
(lang, config.imagtypes.keys()))
exit()
try:
config.imageconf = _lang_conf[version]
except KeyError:
print("ERROR: %r is not a valid version! try one of %s" %
(version, _lang_conf.keys()))
exit()
config.langconf = lang_config
logger.info("Starting!")
preparaTemporal(procesar_articles)
logger.info("Copying the assets and locale files")
copy_assets(src_info, config.DIR_ASSETS)
shutil.copytree('locale', path.join(config.DIR_CDBASE, "locale"))
articulos = path.join(src_info, "articles")
if procesar_articles:
logger.info("Preprocessing")
if not path.exists(articulos):
logger.error("Couldn't find articles dir: %r", articulos)
raise EnvironmentError("Directory not found, can't continue")
sys.exit()
preprocesar.run(articulos)
logger.info("Calculating which stay and which don't")
preprocesar.pages_selector.calculate()
logger.info("Generating the images log")
taken, adesc = extract.run()
logger.info("Extracted %d images, need to download %d", taken, adesc)
else:
logger.info("Avoid processing articles and generating images log")
logger.info("Recalculating the reduction percentages.")
calcular.run()
if not desconectado:
logger.info("Downloading the images from the internet")
download.retrieve()
logger.info("Reducing the downloaded images")
reducir.run(verbose)
logger.info("Putting the reduced images into blocks")
# agrupamos las imagenes en bloques
q_blocks, q_images = ImageManager.generar_bloques(verbose)
logger.info("Got %d blocks with %d images", q_blocks, q_images)
if not procesar_articles:
logger.info("Not generating index and blocks (by user request)")
elif preprocesar.pages_selector.same_info_through_runs:
logger.info("Same articles than previous run "
"(not generating index and blocks)")
else:
logger.info("Generating the index")
result = cdpindex.generar_de_html(articulos, verbose)
logger.info("Got %d files", result)
logger.info("Generating the articles blocks")
q_blocks, q_files, q_redirs = ArticleManager.generar_bloques(
lang, verbose)
logger.info("Got %d blocks with %d files and %d redirects", q_blocks,
q_files, q_redirs)
logger.info("Copying the sources")
copy_sources()
logger.info("Generating the links to blocks and indexes")
# blocks
dest = path.join(config.DIR_CDBASE, "cdpedia", "bloques")
if os.path.exists(dest):
os.remove(dest)
os.symlink(path.abspath(config.DIR_BLOQUES), dest)
# indexes
dest = path.join(config.DIR_CDBASE, "cdpedia", "indice")
if os.path.exists(dest):
os.remove(dest)
os.symlink(path.abspath(config.DIR_INDICE), dest)
if config.imageconf["windows"]:
logger.info("Copying Windows stuff")
# generated by pyinstaller 2.0
copy_dir("resources/autorun.win/cdroot", config.DIR_CDBASE)
logger.info("Generating runtime config")
genera_run_config()
base_dest_name = "cdpedia-%s-%s-%s-%s" % (lang, config.VERSION, gendate,
version)
if config.imageconf["type"] == "iso":
logger.info("Building the ISO: %r", base_dest_name)
build_iso(base_dest_name)
elif config.imageconf["type"] == "tarball":
logger.info("Building the tarball: %r", base_dest_name)
build_tarball(base_dest_name)
else:
raise ValueError("Unrecognized image type")
logger.info("All done!")