def calcImages(row, entityIdVar = "EntityId"):
"""Figure out if an Article has images and what the paths are
Requires attributes:
EntityId - the Product Id (same as Id_Articulo, PK on Articulos)
Categoria_Seccion - from Articulos
Up to four attributes are appended:
SmallCover - the path to the small cover
SmallCoverGeneric - a boolean indicating that the SmallCover is a generic
LargeCover - if exists, the path to the large cover
"""
# get some config values (be sure to make copies)
config = ecommerce.config.getConfig()
basePath = config.get("paths.resources")
small = config.get("paths.cover.small", [ ])[:] # shallow copy
smallDef = config.get("paths.cover.small-def")
large = config.get("paths.cover.large", [ ])[:] # shallow copy
# figure out the variables we want to pass
variables = {
"EntityId" : row.get(entityIdVar),
"Categoria_Seccion" : row.get("Categoria_Seccion")
}
# query the cache to find the images
imagelist = tmklib.cache.findImages(variables["EntityId"])
if imagelist is None:
imagelist = [ ]
# try to find the small image
row["CoverSmall"] = None
row["CoverSmallGeneric"] = True
for i in range(len(small)):
path = checkImage(imagelist, basePath, small[i], variables)
if path is not None:
row["CoverSmall"] = path
row["CoverSmallGeneric"] = False
break
# still no image? => check for a default
if row["CoverSmall"] is None and smallDef is not None:
path = checkImageFile(basePath, smallDef, variables)
if path is not None:
row["CoverSmall"] = path
row["CoverSmallGeneric"] = True
# try to find the large image
row["CoverLarge"] = None
for i in range(len(large)):
path = checkImage(imagelist, basePath, large[i], variables)
if path is not None:
row["CoverLarge"] = path
break
return row
def author_texts(row):
"""Check if the author has texts (biography) and add the content"""
# get a config object
config = ecommerce.config.getConfig()
# create an empty Text item in the record
row["Texts"] = [ ]
# if no contributor id => do nothing
if "ContributorId" not in row:
return row
contributorId = row["ContributorId"].to_integral_value()
# get the file name from the cache
fname = tmklib.cache.findBiography(contributorId)
if fname is None:
return row
# get the path to biographies
biographyPath = config.get("paths.biography")
fname = biographyPath + os.sep + fname
# read the file
text = None
try:
f = open(fname, "r")
text = tmklib.support.decode(f.read(), 'iso-8859-1')
f.close()
# append only if there is text
if text is None:
return row
# try to figure out the text format
textFormat = "06" # assume is plain text
if re.search("<[a-zA-Z]( |>|\/)?", text) is not None:
textFormat = "02"
# append the entry
row["Texts"].append( {
"EntityType" : "CONT",
"EntityId" : contributorId,
"ContributorId" : contributorId,
"EntryCode" : "13",
"EntryCode_list" : "ONIX.33",
"EntryCode_desc" : "Biographical note",
"TextFormat" : textFormat,
"TextFormat_list" : "ONIX.34",
"TextFormat_desc" : "HTML" if textFormat == "02" else "Default text format",
"TextContent" : text
} )
except:
pass
return row
def getStorages(config, prefix):
"""Returns a dictionary of storages"""
# sanity checks
if config is None or prefix is None:
raise ValueError("ecommerce.storage.getStorages called with None parameter(s)")
# get the list of storages to create
names = config.get(prefix, { }).keys()
# prepare the result
storages = { }
for name in names:
storages[name] = getStorage(config, prefix, name)
return storages
def solverInitialize(config = None):
"""Initialize the solver mechanism
Accesses the configuration and figures out the global database
"""
global _defaultDB
global _code
# instantiate the appropriate loader
if config is None:
config = ecommerce.config.getConfig()
_defaultDB = config.get("db.dataset.database", ecommerce.db.getDefaultDB())
# reset the imported library cache
_code = { }
def _init(config = None):
"""Initialize the module
Steps executed:
1: get the default database
2: process each module in "db.modules"
3: process each database in "db.databases" (set default if none)
"""
# get the configuration
if config is None:
config = ecommerce.config.getConfig()
# 1: get the default database
defaultDB = config.get("db.default")
# 2: process each module in "db.modules"
modules = { }
try:
mDef = config.get("db.python", { })
except:
raise DBConfigurationException("Cannot find 'db.python' configuration entry")
for m in mDef:
modules[m] = {
"params" : mDef[m],
"module" : None,
"connect" : None
}
# 3: process each database in "db.databases" (set default if none)
databases = { }
try:
dblist = config.get("db.databases", { })
except:
raise DBConfigurationException("Cannot find 'db.databases' configuration entry")
for db in dblist:
# ignore errors
try:
# get db conf
dbconf = config.getMulti("db", db)
if dbconf is None: # ignore if not present
continue
# get the module
module = dbconf["python"]
# prepare the params according to the module
params = { }
mDef = modules[module]["params"]
for key in mDef:
if key in dbconf:
if key == 'password':
params[key] = config.keychain.fetch(dbconf[key])
else:
params[key] = dbconf[key]
# create the db entry
databases[db] = {
"def" : dbconf,
"module" : module,
"params" : params
}
# if no default, use this one
if defaultDB is None:
defaultDB = db
except:
raise DBConfigurationException("Configuration error processing 'db.%s'" % db)
# return the config
return ( defaultDB, databases, modules )
def getTexts(dataset, entityType, datasetName, idList):
"""Returns the texts fro products (PROD) entities
Fetches texts from the database (Articulos_Textos) and
completes with file system files
"""
# prepare the result
texts = { }
# protect from exceptions
try:
# connect to the database and get a cursor
conn = ecommerce.db.getConnection()
cursor = conn.cursor()
# build the query
ids = ", ".join( [ str(id) for id in idList ] )
query = """
SELECT A.Id_Articulo, A.Tipo, A.Parte, A.Tipo_Texto, A.Texto, A.Idioma,
Ref.RV_Meaning
FROM Articulos_Textos A
LEFT JOIN CG_Ref_Codes Ref
ON Ref.RV_Domain = 'ONIX:TextTypeCode' AND
A.Tipo = Ref.RV_Low_Value
WHERE Id_Articulo IN (""" + ids + """)
ORDER BY Id_Articulo, Tipo, Parte"""
cursor.execute(query)
# fetch the results
row = cursor.fetchone()
while row is not None:
# get some values
id_articulo = int(row[0])
tipo = row[1]
parte = int(row[2])
tipo_texto = row[3]
texto = tmklib.support.decode("" if row[4] is None else row[4], 'iso-8859-1')
idioma = row[5]
tipo_desc = tmklib.support.decode("" if row[6] is None else row[6], 'iso-8859-1')
# convert tipo_texto
tipos_texto = {
"00" : ("06", "Default text format"),
"01" : ("02", "HTML"),
"02" : ("02", "HTML")
}
tipo_texto = tipos_texto[tipo_texto if tipo_texto in tipos_texto else "00"][0]
tipo_texto_desc = tipos_texto[tipo_texto if tipo_texto in tipos_texto else "00"][1]
# create the key
key = (id_articulo, tipo)
if key not in texts:
# create entry
texts[key] = {
"EntityType" : "PROD",
"EntityId" : id_articulo,
"ProductId" : id_articulo,
"EntryCode" : tipo,
"EntryCode_list" : "ONIX.153",
"EntryCode_desc" : tipo_desc,
"TextFormat" : tipo_texto,
"TextFormat_list" : "ONIX.34",
"TextFormat_desc" : tipo_texto_desc,
"TextContent" : texto
}
else:
# append texto to existing entry
texts[key]["TextContent"] += texto
# next entry
row = cursor.fetchone()
except:
pass
# get a config object
config = ecommerce.config.getConfig()
# append first-chapters to texts
firstChapterPath = config.get("paths.firstChapter")
if firstChapterPath is not None:
texts = _loadFiles("firstChapter", firstChapterPath, idList, texts, {
"EntityType" : "PROD",
"EntryCode" : "24",
"EntryCode_list" : "ONIX.33",
"EntryCode_desc" : "First Chapter"
} )
# append interviews to texts
interviewsPath = config.get("paths.interviews")
if firstChapterPath is not None:
texts = _loadFiles("interview", interviewsPath, idList, texts, {
"EntityType" : "PROD",
"EntryCode" : "40",
"EntryCode_list" : "ONIX.33",
"EntryCode_desc" : "Author interview"
} )
# create the real result
result = { }
# process the intermediate texts
keys = sorted(texts.keys())
for k in keys:
# get the product id
productId = Decimal(k[0])
# if product not in result, create the entry
if productId not in result:
result[productId] = [ texts[k] ]
else:
result[productId].append(texts[k])
return result
def _loadConfig(config = None):
"""Try to get a viable configuration (use defaults wherever appropriate)"""
# prepare the results (assume defaults)
_config = {
"dbName" : None,
"codeTable" : "CodeTables",
"tableId" : "CodeTableId",
"tableDomain" : "TableDomain",
"tableName" : "TableName ",
"flagGrouped" : "FlagGrouped",
"dataTableSchema" : "DataTableSchema",
"dataTableName" : "DataTableName ",
"dataTableId" : "CodeTableId",
"dataTableCode" : "DataTableCodeField",
"dataTableDesc" : "DataTableNameField"
}
# if we can find a config, try to get config
if config is None:
config = ecommerce.config.getConfig()
if config is not None:
# get the config values (using defaults)
_config["dbName"] = config.get("codetables.database",
_config["dbName"])
_config["codeTable"] = config.get("codetables.codetable",
_config["codeTable"])
_config["tableId"] = config.get("codetables.fields.tableId",
_config["tableId"])
_config["tableDomain"] = config.get("codetables.fields.tableDomain",
_config["tableDomain"])
_config["tableName"] = config.get("codetables.fields.tableName",
_config["tableName"])
_config["flagGrouped"] = config.get("codetables.fields.flagGrouped",
_config["flagGrouped"])
_config["dataTableSchema"] = config.get("codetables.fields.dataTableSchema",
_config["dataTableSchema"])
_config["dataTableName"] = config.get("codetables.fields.dataTableName",
_config["dataTableName"])
_config["dataTableId"] = config.get("codetables.fields.dataTableId",
_config["dataTableId"])
_config["dataTableCode"] = config.get("codetables.fields.dataTableCode",
_config["dataTableCode"])
_config["dataTableDesc"] = config.get("codetables.fields.dataTableDesc",
_config["dataTableDesc"])
# return the configured elements
return _config