Exemple #1
0
def calcImages(row, entityIdVar = "EntityId"):
    """Figure out if an Article has images and what the paths are

    Requires attributes:
    EntityId - the Product Id (same as Id_Articulo, PK on Articulos)
    Categoria_Seccion - from Articulos

    Up to four attributes are appended:
    SmallCover - the path to the small cover
    SmallCoverGeneric - a boolean indicating that the SmallCover is a generic
    LargeCover - if exists, the path to the large cover
    """

    # get some config values (be sure to make copies)
    config   = ecommerce.config.getConfig()
    basePath = config.get("paths.resources")
    small    = config.get("paths.cover.small", [ ])[:] # shallow copy
    smallDef = config.get("paths.cover.small-def")
    large    = config.get("paths.cover.large", [ ])[:] # shallow copy

    # figure out the variables we want to pass
    variables = {
        "EntityId"              : row.get(entityIdVar),
        "Categoria_Seccion"     : row.get("Categoria_Seccion")
    }

    # query the cache to find the images
    imagelist = tmklib.cache.findImages(variables["EntityId"])
    if imagelist is None:
        imagelist = [ ]

    # try to find the small image
    row["CoverSmall"] = None
    row["CoverSmallGeneric"] = True
    for i in range(len(small)):
        path = checkImage(imagelist, basePath, small[i], variables)
        if path is not None:
            row["CoverSmall"] = path
            row["CoverSmallGeneric"] = False
            break
    # still no image? => check for a default
    if row["CoverSmall"] is None and smallDef is not None:
        path = checkImageFile(basePath, smallDef, variables)
        if path is not None:
            row["CoverSmall"] = path
            row["CoverSmallGeneric"] = True

    # try to find the large image
    row["CoverLarge"] = None
    for i in range(len(large)):
        path = checkImage(imagelist, basePath, large[i], variables)
        if path is not None:
            row["CoverLarge"] = path
            break

    return row
Exemple #2
0
def author_texts(row):
    """Check if the author has texts (biography) and add the content"""

    # get a config object
    config = ecommerce.config.getConfig()

    # create an empty Text item in the record
    row["Texts"] = [ ]

    # if no contributor id => do nothing
    if "ContributorId" not in row:
        return row
    contributorId = row["ContributorId"].to_integral_value()

    # get the file name from the cache
    fname = tmklib.cache.findBiography(contributorId)
    if fname is None:
        return row

    # get the path to biographies
    biographyPath = config.get("paths.biography")
    fname = biographyPath + os.sep + fname

    # read the file
    text = None
    try:
        f = open(fname, "r")
        text = tmklib.support.decode(f.read(), 'iso-8859-1')
        f.close()

        # append only if there is text
        if text is None:
            return row

        # try to figure out the text format
        textFormat = "06"       # assume is plain text
        if re.search("<[a-zA-Z]( |>|\/)?", text) is not None:
            textFormat = "02"

        # append the entry
        row["Texts"].append( {
            "EntityType"        : "CONT",
            "EntityId"          : contributorId,
            "ContributorId"     : contributorId,
            "EntryCode"         : "13",
            "EntryCode_list"    : "ONIX.33",
            "EntryCode_desc"    : "Biographical note",
            "TextFormat"        : textFormat,
            "TextFormat_list"   : "ONIX.34",
            "TextFormat_desc"   : "HTML" if textFormat == "02" else "Default text format",
            "TextContent"       : text
        } )
    except:
        pass

    return row
Exemple #3
0
def getStorages(config, prefix):
    """Returns a dictionary of storages"""

    # sanity checks
    if config is None or prefix is None:
        raise ValueError("ecommerce.storage.getStorages called with None parameter(s)")

    # get the list of storages to create
    names = config.get(prefix, { }).keys()

    # prepare the result
    storages = { }
    for name in names:
        storages[name] = getStorage(config, prefix, name)

    return storages
Exemple #4
0
def solverInitialize(config = None):
    """Initialize the solver mechanism

    Accesses the configuration and figures out the global database
    """

    global _defaultDB
    global _code

    # instantiate the appropriate loader
    if config is None:
        config = ecommerce.config.getConfig()

    _defaultDB = config.get("db.dataset.database", ecommerce.db.getDefaultDB())

    # reset the imported library cache
    _code = { }
Exemple #5
0
def _init(config = None):
    """Initialize the module

    Steps executed:
    1: get the default database
    2: process each module in "db.modules"
    3: process each database in "db.databases" (set default if none)
    """

    # get the configuration
    if config is None:
        config = ecommerce.config.getConfig()

    # 1: get the default database
    defaultDB = config.get("db.default")

    # 2: process each module in "db.modules"
    modules = { }
    try:
        mDef = config.get("db.python", { })
    except:
        raise DBConfigurationException("Cannot find 'db.python' configuration entry")
    for m in mDef:
        modules[m] = {
            "params"  : mDef[m],
            "module"  : None,
            "connect" : None
        }

    # 3: process each database in "db.databases" (set default if none)
    databases = { }
    try:
        dblist = config.get("db.databases", { })
    except:
        raise DBConfigurationException("Cannot find 'db.databases' configuration entry")
    for db in dblist:

        # ignore errors
        try:
            # get db conf
            dbconf = config.getMulti("db", db)
            if dbconf is None:     # ignore if not present
                continue

            # get the module
            module = dbconf["python"]

            # prepare the params according to the module
            params = { }
            mDef = modules[module]["params"]
            for key in mDef:
                if key in dbconf:
                    if key == 'password':
                        params[key] = config.keychain.fetch(dbconf[key])
                    else:
                        params[key] = dbconf[key]

            # create the db entry
            databases[db] = {
                "def"      : dbconf,
                "module"   : module,
                "params"   : params
            }

            # if no default, use this one
            if defaultDB is None:
                defaultDB = db
        except:
            raise DBConfigurationException("Configuration error processing 'db.%s'" % db)

    # return the config
    return ( defaultDB, databases, modules )
Exemple #6
0
def getTexts(dataset, entityType, datasetName, idList):
    """Returns the texts fro products (PROD) entities
    
    Fetches texts from the database (Articulos_Textos) and
    completes with file system files
    """
    
    # prepare the result
    texts = { }

    # protect from exceptions
    try:

        # connect to the database and get a cursor
        conn = ecommerce.db.getConnection()
        cursor = conn.cursor()

        # build the query
        ids = ", ".join( [ str(id) for id in idList ] )
        query = """
SELECT          A.Id_Articulo, A.Tipo, A.Parte, A.Tipo_Texto, A.Texto, A.Idioma,
                Ref.RV_Meaning
        FROM        Articulos_Textos A
        LEFT JOIN   CG_Ref_Codes Ref
            ON      Ref.RV_Domain = 'ONIX:TextTypeCode' AND
                    A.Tipo = Ref.RV_Low_Value
        WHERE       Id_Articulo IN (""" + ids + """)
        ORDER BY    Id_Articulo, Tipo, Parte"""
        cursor.execute(query)

        # fetch the results
        row = cursor.fetchone()
        while row is not None:

            # get some values
            id_articulo = int(row[0])
            tipo        = row[1]
            parte       = int(row[2])
            tipo_texto  = row[3]
            texto       = tmklib.support.decode("" if row[4] is None else row[4], 'iso-8859-1')
            idioma      = row[5]
            tipo_desc   = tmklib.support.decode("" if row[6] is None else row[6], 'iso-8859-1')

            # convert tipo_texto
            tipos_texto     = {
                "00"        : ("06", "Default text format"),
                "01"        : ("02", "HTML"),
                "02"        : ("02", "HTML")
            }
            tipo_texto      = tipos_texto[tipo_texto if tipo_texto in tipos_texto else "00"][0]
            tipo_texto_desc = tipos_texto[tipo_texto if tipo_texto in tipos_texto else "00"][1]

            # create the key
            key = (id_articulo, tipo)
            if key not in texts:
                # create entry
                texts[key] = {
                    "EntityType"        : "PROD",
                    "EntityId"          : id_articulo,
                    "ProductId"         : id_articulo,
                    "EntryCode"         : tipo,
                    "EntryCode_list"    : "ONIX.153",
                    "EntryCode_desc"    : tipo_desc,
                    "TextFormat"        : tipo_texto,
                    "TextFormat_list"   : "ONIX.34",
                    "TextFormat_desc"   : tipo_texto_desc,
                    "TextContent"       : texto
                }
            else:
                # append texto to existing entry
                texts[key]["TextContent"] += texto

            # next entry
            row = cursor.fetchone()
    except:
        pass

    # get a config object
    config = ecommerce.config.getConfig()

    # append first-chapters to texts
    firstChapterPath = config.get("paths.firstChapter")
    if firstChapterPath is not None:
        texts = _loadFiles("firstChapter", firstChapterPath, idList, texts, {
            "EntityType"        : "PROD",
            "EntryCode"         : "24",
            "EntryCode_list"    : "ONIX.33",
            "EntryCode_desc"    : "First Chapter"
        } )

    # append interviews to texts
    interviewsPath = config.get("paths.interviews")
    if firstChapterPath is not None:
        texts = _loadFiles("interview", interviewsPath, idList, texts, {
            "EntityType"        : "PROD",
            "EntryCode"         : "40",
            "EntryCode_list"    : "ONIX.33",
            "EntryCode_desc"    : "Author interview"
        } )

    # create the real result
    result = { }

    # process the intermediate texts
    keys = sorted(texts.keys())
    for k in keys:

        # get the product id
        productId = Decimal(k[0])

        # if product not in result, create the entry
        if productId not in result:
            result[productId] = [ texts[k] ]
        else:
            result[productId].append(texts[k])

    return result
Exemple #7
0
def _loadConfig(config = None):
    """Try to get a viable configuration (use defaults wherever appropriate)"""

    # prepare the results (assume defaults)
    _config = {
        "dbName" :          None,
        "codeTable" :       "CodeTables",
        "tableId" :         "CodeTableId",
        "tableDomain" :     "TableDomain",
        "tableName" :       "TableName ",
        "flagGrouped" :     "FlagGrouped",
        "dataTableSchema" : "DataTableSchema",
        "dataTableName" :   "DataTableName ",
        "dataTableId" :     "CodeTableId",
        "dataTableCode" :   "DataTableCodeField",
        "dataTableDesc" :   "DataTableNameField"
    }

    # if we can find a config, try to get config
    if config is None:
        config = ecommerce.config.getConfig()
    if config is not None:
        # get the config values (using defaults)
        _config["dbName"]          = config.get("codetables.database",
                                                _config["dbName"])
        _config["codeTable"]       = config.get("codetables.codetable",
                                                _config["codeTable"])
        _config["tableId"]         = config.get("codetables.fields.tableId",
                                                _config["tableId"])
        _config["tableDomain"]     = config.get("codetables.fields.tableDomain",
                                                _config["tableDomain"])
        _config["tableName"]       = config.get("codetables.fields.tableName",
                                                _config["tableName"])
        _config["flagGrouped"]     = config.get("codetables.fields.flagGrouped",
                                                _config["flagGrouped"])
        _config["dataTableSchema"] = config.get("codetables.fields.dataTableSchema",
                                                _config["dataTableSchema"])
        _config["dataTableName"]   = config.get("codetables.fields.dataTableName",
                                                _config["dataTableName"])
        _config["dataTableId"]     = config.get("codetables.fields.dataTableId",
                                                _config["dataTableId"])
        _config["dataTableCode"]   = config.get("codetables.fields.dataTableCode",
                                                _config["dataTableCode"])
        _config["dataTableDesc"]   = config.get("codetables.fields.dataTableDesc",
                                                _config["dataTableDesc"])

    # return the configured elements
    return _config