def load_module_loadone(options,modpath,db, preparse=False):
module=""
tables=[]
mtd_files={}
filetypes=["xml","ui","qry","kut","qs","mtd","ts", "pgsql"]
unicode_filetypes=["ui","ts"]
files=[]
pd = shelve.open("/tmp/pydba") # open -- file may get suffix added by low-level lib
for root, dirs, walk_files in os.walk(modpath):
for name in walk_files:
fname = os.path.join(root, name)
mtime = os.stat(fname)[ST_MTIME]
loadFile = True
if pd.has_key(fname):
if pd[fname]["mtime"]==mtime:
loadFile=False
if f_ext(name)=="mod":
module=name[:-4]
file_module=loadfile_inutf8(root, name)
module_parse=XMLParser(name)
module_parse.parseText(file_module)
d_module={
'name' : str(module_parse.root.module.name),
'alias' : str(module_parse.root.module.alias),
'area' : str(module_parse.root.module.area),
'areaname' : str(module_parse.root.module.areaname),
'version' : str(module_parse.root.module.version),
'icon' : str(module_parse.root.module.icon),
}
if loadFile:
d_module['icon_data']=loadfile_inutf8(root, d_module['icon'])
contents=""
contents_1=""
if f_ext(name) in filetypes:
contents_1=loadfile_inutf8(root,name)
contents=my_escape_string(contents_1)
if not loadFile:
try:
if pd[fname]["sha"] is None: loadFile = True # Some bug in database can cause sha is None.
except:
pass
if loadFile:
sha=SHA1(contents_1)
pd[fname]={"mtime":mtime, "sha":sha, 'root' : root,'name' : name}
options.sha_allowed_files |= set([pd[fname]["sha"]])
options.filenames_allowed_files |= set([pd[fname]["name"]])
if f_ext(name)=="qs" and loadFile==True and options.flscriptparser==True:
flscriptparser(root,name)
if f_ext(name)=="pgsql":
array_name=name.split(".")
# name.ext1.pgsql
# array_name = [ "nombre" , "view" , "pgsql" ]
fullfilename = os.path.join(root, name)
if options.verbose: print "Cargando PGSQL: ", fullfilename
ret = None
if len(array_name)==2:
ret = pydba_loadpgsql.loadpgsqlfile(fullfilename = fullfilename, options = options, database = db, pgname = array_name[0], pgtype = "sql1", pgtext = contents_1)
elif len(array_name)!=3:
print "ERROR: Al cargar un .pgsql Se esperaban 2 o 3 elementos al estilo nombre.view.pgsql o nombre.pgsql y se encontró %s " % name
continue
else:
ret = pydba_loadpgsql.loadpgsqlfile(fullfilename = fullfilename, options = options, database = db, pgname = array_name[0], pgtype = array_name[1], pgtext = contents_1)
if ret != True:
print "ERROR: Errores al cargar el fichero ", root, name
if f_ext(name)=="mtd":
table=name[:-4]
# print "### Table: " + table
tables+=[table]
mtd_files[table]=contents_1
if preparse:
xml=XMLParser(name)
xml.parseText(contents_1)
if xml.root==None:
#print "ERROR: Failed to parse xml %s" % (name)
xml=None
else:
import pydba_mtdparser
mtd=xml.root.tmd
mparser=pydba_mtdparser.MTDParser()
mparser.parse_mtd(mtd)
mtdquery = getattr(mtd, "query", None)
if not mtdquery:
pydba_mtdparser.Tables[table]=mparser
if contents and f_ext(name) in filetypes:
file={}
for key,val in pd[fname].iteritems():
file[key]=val
file["contents"]=contents
if (options.modules_loaded.has_key(name)):
print "ERROR: %s file was already loaded." % name
print "--> this file was found at %s" % root
print "--> previous file found at %s" % options.modules_loaded[name]['root']
print "* skipping file"
else:
options.modules_loaded[name]=file
files+=[file]
pd.close()
try:
os.chmod("/tmp/pydba", S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)
except:
pass
qry_areas=db.query("SELECT descripcion, bloqueo, idarea"
" FROM flareas WHERE idarea='%s'" % d_module['area'])
tareas=qry_areas.dictresult()
if len(tareas)==0:
print "Creando Area %s - %s " %(d_module['area'],d_module['areaname'])
db.query("INSERT INTO flareas (descripcion, bloqueo, idarea)"
"VALUES('%s','t','%s')" % (d_module['areaname'],d_module['area']))
habilitar_carga=False
qry_modulo=db.query("SELECT idmodulo, version, descripcion, bloqueo, idarea"
" FROM flmodules WHERE idmodulo='%s'" % module);
tmodulo=qry_modulo.dictresult()
cargado=False
for pmodulo in tmodulo:
cargado=True
# if pmodulo['bloqueo']=='t': # TRUE Es que NO está bloqueado. Está al revés.s
habilitar_carga=True
if cargado and options.updatemodules:
cargado = False
idmodulo = pg.escape_string(d_module['name'])
sql=("DELETE FROM flmodules WHERE idmodulo = '%s'" %
(idmodulo))
db.query(sql)
if not cargado:
print "Se procede a crear el módulo nuevo %s" % module
idmodulo = pg.escape_string(d_module['name'])
idarea = pg.escape_string(d_module['area'])
version = pg.escape_string(d_module['version'])
bloqueo = "t"
descripcion = pg.escape_string(d_module['alias'])
icono = pg.escape_string(d_module['icon_data'])
sql=("INSERT INTO flmodules (idmodulo, idarea, version, bloqueo, descripcion,icono) "
"VALUES('%s','%s','%s','%s','%s','%s')" %
(idmodulo, idarea, version, bloqueo, descripcion,icono))
db.query(sql)
habilitar_carga=True
if not habilitar_carga:
print "Error when trying to update the module '%s': non-loaded or locked module" % module
return 0
qry_modulos=db.query("SELECT nombre,sha FROM flfiles WHERE idmodulo='%s' " % (module));
tuplas_modulos=qry_modulos.dictresult()
dmodulos={}
for modulo in tuplas_modulos:
dmodulos[modulo['nombre']]=modulo;
loaded=[]
for file in files:
update=True
if (dmodulos.has_key(file['name'])):
dm=dmodulos[file['name']]
if (dm['sha']==file['sha']):
update=False
if options.loadbaselec:
update=(file['name']=='baselec.mtd')
if (update):
loaded+=[file['name']]
if (options.verbose):
print "* Loading file '%s' => '%s'..." % (file['name'],file['sha'])
sql="DELETE FROM flfiles WHERE nombre='%s';\n" % file['name']
db.query(sql)
file['module']=module
sql=("INSERT INTO flfiles (contenido, bloqueo, sha, idmodulo, nombre) "
"VALUES(E'%(contents)s', 't', '%(sha)s','%(module)s', '%(name)s')" % file)
try:
db.query(sql)
except Exception, e:
print e.__class__.__name__, e
print sql
raise
def repair_db(options,ddb=None,mode=0,odb=None):
if (options.verbose):
print "-> RepairDB"
if (not ddb):
if (not options.ddb):
print "RepairDB requiere una base de datos de destino y no proporcionó ninguna."
return 0
ddb=dbconnect(options)
if (not ddb):
return 0
if (not odb):
if (not options.odb):
print "RepairDB requiere una base de datos de origen y no proporcionó ninguna."
return 0
odb=odbconnect(options)
if (not odb):
return 0
where=""
if not options.full :
#if len(options.files_loaded)>0:
where+=" AND (nombre IN ('" + "','".join(options.files_loaded) + "' ) OR nombre LIKE '%.mtd')"
if len(options.modules)>0:
where+=" AND ( 0=1 "
for modname,module in options.modules.iteritems():
noloadtable=[]
loadtable=[]
for tablename, value in module.iteritems():
if value==False:
noloadtable.append(tablename + ".mtd")
else:
loadtable.append(tablename + ".mtd")
try:
default=module['_default_']
except:
default=False
if default==True:
where+=" OR ( idmodulo = '%s' AND nombre NOT IN ('" % modname + "','".join(noloadtable) + "'))"
else:
where+=" OR ( idmodulo = '%s' AND nombre IN ('" % modname + "','".join(loadtable) + "'))"
where+=" OR nombre LIKE '%.mtd' )"
if options.odb!=options.ddb:
where+=" AND nombre LIKE '%.mtd'"
if (options.verbose):
print "Inicializando reparación de la base de datos '%s'..." % options.ddb
print " * Calcular firmas SHA1 de files y metadata"
if options.debug:
print "* Revisando si existe una tabla de servidores réplica ..."
tabla = None
qry1 =ddb.query("SELECT relname FROM pg_class WHERE relname ='servidoresreplica'");
for row in qry1.dictresult():
tabla = row['relname']
if tabla:
if options.debug:
print "Encontrada una tabla %s." % tabla
qry1 =ddb.query("SELECT codservidor, activo, numero FROM %s" % tabla);
servsize = len(qry1.dictresult())
servnumber = -1
servname = None
for row in qry1.dictresult():
if row['activo'] == 't':
try:
assert(servnumber == -1)
except AssertionError:
print "PANIC: Hay mas de un servidor activo! El valor anterior era %s el nuevo %s. (activo vale %s)" % (
repr(servnumber),
repr(row['numero']),
repr(row['activo'])
)
raise
servnumber = row['numero']
servname = row['codservidor']
activar_servrep = True
if servsize < 1 or servsize > 10:
activar_servrep = False
print "WARN: No se activa la sincronización de secuencia para replicación master-master por error en la cantidad de servidores (%d)" % servsize
if servnumber < 0 or servnumber >= servsize:
activar_servrep = False
print "WARN: No se activa la sincronización de secuencia para replicación master-master por error el numero de este servidor (%d/%d)" % (servnumber,servsize)
if activar_servrep:
print "INFO: Activando sincro. de secuencias para %s: %d/%d" % (repr(servname), servnumber,servsize)
options.seqsync = (servnumber,servsize)
else:
if options.debug:
print "No se encontró tabla de servidores de replica"
if options.transactions:
odb.query("BEGIN;");
try:
lltables = "flfiles,flserial,flmetadata".split(",")
for ltable in lltables:
sql = "LOCK %s NOWAIT;" % ltable
if (options.verbose): print sql
odb.query(sql);
if (options.verbose): print "done."
except:
print "Error al bloquear la tabla %s , ¡algun otro usuario está conectado!" % ltable
odb.query("ROLLBACK;");
raise
tables_notrebuilt = []
pydba_loadpgsql.process_drop(options,ddb)
qry_omodulos=odb.query("SELECT sha " +
"FROM flfiles WHERE sha!='' AND nombre NOT LIKE '%%alteredtable%%.mtd' ORDER BY sha");
ofiles=[]
for row in qry_omodulos.dictresult():
ofiles.append(row['sha'])
qry_dmodulos=ddb.query("SELECT sha " +
"FROM flfiles WHERE sha!='' AND nombre NOT LIKE '%%alteredtable%%.mtd' ORDER BY sha");
dfiles=[]
for row in qry_dmodulos.dictresult():
if row['sha'] in ofiles:
ofiles.remove(row['sha'])
else:
dfiles.append(row['sha'])
# Eliminar los ficheros sobrantes.
qry_dmodulos=ddb.query("DELETE FROM flfiles WHERE sha IN ('" + "','".join(dfiles) + "')")
# Obtener los ficheros nuevos
qry_omodulos=odb.query("SELECT * FROM flfiles WHERE sha IN ('" + "','".join(ofiles) + "')")
# Insertarlos en la nueva DB.
for row in qry_omodulos.dictresult():
fields=row.keys()
values=[]
for field in fields:
campo=row[field]
if (campo is not None):#Si el valor es nulo
values.append("(E'" + pg.escape_string(str(campo)) + "')")
else:
values.append("NULL")
try:
qry_dmodulos=ddb.query("DELETE FROM flfiles WHERE nombre ='" + row['nombre'] + "'")
sql="INSERT INTO flfiles (" + ",".join(fields) + ") VALUES(" + ",".join(values) + ")"
ddb.query(sql)
except:
print sql
raise
sqlModulos = ("SELECT idmodulo, nombre, contenido, sha " +
"FROM flfiles WHERE sha!='' AND nombre NOT LIKE '%%alteredtable%%.mtd' "
+ where + " ORDER BY idmodulo, nombre")
# print sqlModulos
qry_modulos=ddb.query(sqlModulos);
modulos=qry_modulos.dictresult()
# print "%d resultados." % len(modulos)
sql=""
resha1="";
xmlfiles=("xml","ui","qry","kut","mtd","ts")
ficheros_actualizados=0
for modulo in modulos:
if options.loadbaselec:
if modulo['nombre'] != 'baselec.mtd':
continue
xml=None
if options.full and modulo.has_key('contenido'):
sha1=SHA1(modulo['contenido'])
else:
sha1=modulo['sha']
if (sha1==None):
print "ERROR: Carácteres no ISO en %s.%s (se omite SHA1)" % (modulo['idmodulo'],modulo['nombre'])
sha1=modulo['sha']
if f_ext(modulo['nombre']) in xmlfiles:
xml=XMLParser("%s.%s" % (modulo['idmodulo'],modulo['nombre']))
xml.parseText(modulo['contenido'])
if xml.root==None:
xml=None
resha1=SHA1(resha1+sha1)
if (modulo['sha']!=sha1):
ficheros_actualizados+=1
print "Updating " + modulo['nombre'] + " => " + sha1 + " ..."
sql+="UPDATE flfiles SET sha='%s' WHERE nombre='%s';\n" % (sha1,modulo['nombre'])
elif (options.debug):
print modulo['nombre'] + " is ok."
if (f_ext(modulo['nombre'])=="mtd"):
tabla=modulo['nombre'][:-4]
qry_modulos=ddb.query("SELECT xml FROM flmetadata WHERE tabla='%s'" % tabla);
tablas=qry_modulos.dictresult()
TablaCargada=False
sql_update_metadata = ""
for txml in tablas:
TablaCargada=True
if txml['xml']!=sha1:
sql_update_metadata="UPDATE flmetadata SET xml='%s' WHERE tabla='%s';\n" % (sha1,tabla)
if not TablaCargada:
print "Cargando tabla nueva %s ..." % tabla
sql_update_metadata=("INSERT INTO flmetadata (tabla,bloqueo,seq,xml)"
" VALUES('%s','f','0','%s');\n" % (tabla,sha1) )
if xml:
if sql_update_metadata or options.full or options.loadbaselec:
updatedTableStructure = load_mtd(options,odb,ddb,tabla,xml)
if updatedTableStructure or not TablaCargada:
if options.verbose:
print "Actualizando metadata para %s" % tabla
if sql_update_metadata:
ddb.query(sql_update_metadata)
elif sql_update_metadata:
if options.rebuildtables or options.forgottables:
print "Actualizando metadatos para tabla %s" % tabla
ddb.query(sql_update_metadata)
else:
tables_notrebuilt.append(modulo['nombre'])
if (len(sql)>1024):
ddb.query(sql)
sql=""
if (len(sql)>0):
ddb.query(sql)
sql=""
qry_d_pkeys=ddb.query("SELECT table_name, column_name,constraint_name FROM information_schema.constraint_column_usage WHERE constraint_name LIKE '%_pkey_%';")
for row in qry_d_pkeys.dictresult():
qry_d_pkeys2=ddb.query("""
SELECT table_name, column_name,constraint_name
FROM information_schema.constraint_column_usage
WHERE constraint_name = '%(table_name)s_pkey';
""" % row)
for row2 in qry_d_pkeys2.dictresult():
sql = """
ALTER TABLE %(table_name)s DROP CONSTRAINT %(constraint_name)s;
""" % row2
try:
ddb.query(sql)
print "Borrado pkey de la tabla %(table_name)s" % row2
except:
print "Error en query corrigiendo pkey:", row2
print traceback.format_exc()
print "SQL:"
print sql
sql = """
ALTER TABLE %(table_name)s DROP CONSTRAINT %(constraint_name)s;
ALTER TABLE %(table_name)s ADD PRIMARY KEY (%(column_name)s);
""" % row
try:
ddb.query(sql)
print "PK Regenerado: %(constraint_name)s" % row
try:
ddb.query("ALTER INDEX %(table_name)s_pkey SET (fillfactor = 80);" % row)
except:
pass
except:
print "Error en query corrigiendo pkey:", row
print traceback.format_exc()
print "SQL:"
print sql
# !---- Verificacion de modulos instalados
qry_dmodulos=ddb.query("SELECT DISTINCT idmodulo FROM flfiles");
modules_installed = set([])
modules_to_uninstall = set([])
for (idmodulo, ) in qry_dmodulos.getresult():
modules_installed.add(idmodulo)
if modules_installed:
# !--- Examinar los modulos ptes de desinstalacion
qry_dmodulos=ddb.query("SELECT idmodulo, descripcion FROM flmodules WHERE bloqueo = TRUE AND idmodulo NOT IN ('" + "', '".join(modules_installed) + "')");
for idmodulo, descripcion in qry_dmodulos.getresult():
modules_to_uninstall.add((idmodulo, descripcion))
if modules_to_uninstall:
if options.cleanfiles:
print "INFO: Los siguientes modulos ya no se usan y serán desinstalados:"
for idmodulo, nombre in modules_to_uninstall:
print " - %s : %s " % (idmodulo, nombre)
ddb.query("DELETE FROM flmodules WHERE idmodulo = '%s'" % idmodulo);
print ":: Finalizado."
else:
print "Existen %d modulos pendientes de desinstalar:" % (len(modules_to_uninstall)),
for idmodulo, nombre in modules_to_uninstall:
print idmodulo,
print
print "Use --cleanfiles para desinstalarlos"
# !---- Verificacion de areas instaladas
qry_dmodulos=ddb.query("SELECT DISTINCT idarea FROM flmodules");
areas_installed = set([])
areas_to_uninstall = set([])
for (idarea, ) in qry_dmodulos.getresult():
areas_installed.add(idarea)
if areas_installed:
# !--- Examinar las areas ptes de desinstalacion
qry_dmodulos=ddb.query("SELECT idarea, descripcion FROM flareas WHERE bloqueo = TRUE AND idarea NOT IN ('" + "', '".join(areas_installed) + "')");
for idarea, descripcion in qry_dmodulos.getresult():
areas_to_uninstall.add((idarea, descripcion))
if areas_to_uninstall:
if options.cleanfiles:
print "INFO: Las siguientes areas ya no se usan y serán desinstaladas:"
for idarea, nombre in areas_to_uninstall:
print " - %s : %s " % (idarea, nombre)
ddb.query("DELETE FROM flareas WHERE idarea = '%s'" % idarea);
print ":: Finalizado."
else:
print "Existen %d areas pendientes de desinstalar:" % (len(areas_to_uninstall)),
for idarea, nombre in areas_to_uninstall:
print idarea,
print
print "Use --cleanfiles para desinstalarlos"
qry_serial=ddb.query("SELECT sha FROM flserial");
serials=qry_serial.dictresult()
for serial in serials:
if (serial['sha']==resha1):
resha1=False
if resha1 and ficheros_actualizados>0:
if len(serials)>0 :
ddb.query("UPDATE flserial SET sha='%s';" % (resha1))
print "Updated flserial => %s." % (resha1)
else:
ddb.query("INSERT INTO flserial (serie,sha) VALUES(1,'%s')" % (resha1))
print "Created flserial => %s." % (resha1)
pydba_loadpgsql.process_create(options,ddb)
if options.transactions: odb.query("COMMIT;");
if tables_notrebuilt:
print "Tables pending rebuild:", ", ".join(tables_notrebuilt)
def repair_db(options,ddb=None,mode=0,odb=None):
if (options.verbose):
print "-> RepairDB"
if (not ddb):
if (not options.ddb):
print "RepairDB requiere una base de datos de destino y no proporcionó ninguna."
return 0
ddb=dbconnect(options)
if (not ddb):
return 0
if (not odb):
if (not options.odb):
print "RepairDB requiere una base de datos de origen y no proporcionó ninguna."
return 0
odb=odbconnect(options)
if (not odb):
return 0
where=""
if not options.full :
#if len(options.files_loaded)>0:
where+=" AND (nombre IN ('" + "','".join(options.files_loaded) + "' ) OR nombre LIKE '%.mtd')"
if len(options.modules)>0:
where+=" AND ( 0=1 "
for modname,module in options.modules.iteritems():
noloadtable=[]
loadtable=[]
for tablename, value in module.iteritems():
if value==False:
noloadtable.append(tablename + ".mtd")
else:
loadtable.append(tablename + ".mtd")
try:
default=module['_default_']
except:
default=False
if default==True:
where+=" OR ( idmodulo = '%s' AND nombre NOT IN ('" % modname + "','".join(noloadtable) + "'))"
else:
where+=" OR ( idmodulo = '%s' AND nombre IN ('" % modname + "','".join(loadtable) + "'))"
where+=" OR nombre LIKE '%.mtd' )"
if options.odb!=options.ddb:
where+=" AND nombre LIKE '%.mtd'"
if (options.verbose):
print "Inicializando reparación de la base de datos '%s'..." % options.ddb
print " * Calcular firmas SHA1 de files y metadata"
if options.debug:
print "* Revisando si existe una tabla de servidores réplica ..."
tabla = None
qry1 =ddb.query("SELECT relname FROM pg_class WHERE relname ='servidoresreplica'");
for row in qry1.dictresult():
tabla = row['relname']
if tabla:
if options.debug:
print "Encontrada una tabla %s." % tabla
qry1 =ddb.query("SELECT codservidor, activo, numero FROM %s" % tabla);
servsize = len(qry1.dictresult())
servnumber = -1
servname = None
for row in qry1.dictresult():
if row['activo'] == 't':
try:
assert(servnumber == -1)
except AssertionError:
print "PANIC: Hay mas de un servidor activo! El valor anterior era %s el nuevo %s. (activo vale %s)" % (
repr(servnumber),
repr(row['numero']),
repr(row['activo'])
)
raise
servnumber = row['numero']
servname = row['codservidor']
activar_servrep = True
if servsize < 1 or servsize > 10:
activar_servrep = False
print "WARN: No se activa la sincronización de secuencia para replicación master-master por error en la cantidad de servidores (%d)" % servsize
if servnumber < 0 or servnumber >= servsize:
activar_servrep = False
print "WARN: No se activa la sincronización de secuencia para replicación master-master por error el numero de este servidor (%d/%d)" % (servnumber,servsize)
if activar_servrep:
print "INFO: Activando sincro. de secuencias para %s: %d/%d" % (repr(servname), servnumber,servsize)
options.seqsync = (servnumber,servsize)
else:
if options.debug:
print "No se encontró tabla de servidores de replica"
if options.transactions:
odb.query("BEGIN;");
try:
lltables = "flfiles,flserial,flmetadata".split(",")
for ltable in lltables:
sql = "LOCK %s NOWAIT;" % ltable
if (options.verbose): print sql
odb.query(sql);
if (options.verbose): print "done."
except:
print "Error al bloquear la tabla %s , ¡algun otro usuario está conectado!" % ltable
odb.query("ROLLBACK;");
raise
tables_notrebuilt = []
pydba_loadpgsql.process_drop(options,ddb)
qry_omodulos=odb.query("SELECT sha " +
"FROM flfiles WHERE sha!='' AND nombre NOT LIKE '%%alteredtable%%.mtd' ORDER BY sha");
ofiles=[]
for row in qry_omodulos.dictresult():
ofiles.append(row['sha'])
qry_dmodulos=ddb.query("SELECT sha " +
"FROM flfiles WHERE sha!='' AND nombre NOT LIKE '%%alteredtable%%.mtd' ORDER BY sha");
dfiles=[]
for row in qry_dmodulos.dictresult():
if row['sha'] in ofiles:
ofiles.remove(row['sha'])
else:
dfiles.append(row['sha'])
# Eliminar los ficheros sobrantes.
qry_dmodulos=ddb.query("DELETE FROM flfiles WHERE sha IN ('" + "','".join(dfiles) + "')")
# Obtener los ficheros nuevos
qry_omodulos=odb.query("SELECT * FROM flfiles WHERE sha IN ('" + "','".join(ofiles) + "')")
# Insertarlos en la nueva DB.
for row in qry_omodulos.dictresult():
fields=row.keys()
values=[]
for field in fields:
campo=row[field]
if (campo is not None):#Si el valor es nulo
values.append("(E'" + pg.escape_string(str(campo)) + "')")
else:
values.append("NULL")
try:
qry_dmodulos=ddb.query("DELETE FROM flfiles WHERE nombre ='" + row['nombre'] + "'")
sql="INSERT INTO flfiles (" + ",".join(fields) + ") VALUES(" + ",".join(values) + ")"
ddb.query(sql)
except:
print sql
raise
sqlModulos = ("SELECT idmodulo, nombre, contenido, sha " +
"FROM flfiles WHERE sha!='' AND nombre NOT LIKE '%%alteredtable%%.mtd' "
+ where + " ORDER BY idmodulo, nombre")
# print sqlModulos
qry_modulos=ddb.query(sqlModulos);
modulos=qry_modulos.dictresult()
# print "%d resultados." % len(modulos)
sql=""
resha1="";
xmlfiles=("xml","ui","qry","kut","mtd","ts")
ficheros_actualizados=0
for modulo in modulos:
if options.loadbaselec:
if modulo['nombre'] != 'baselec.mtd':
continue
xml=None
if options.full and modulo.has_key('contenido'):
sha1=SHA1(modulo['contenido'])
else:
sha1=modulo['sha']
if (sha1==None):
print "ERROR: Carácteres no ISO en %s.%s (se omite SHA1)" % (modulo['idmodulo'],modulo['nombre'])
sha1=modulo['sha']
if f_ext(modulo['nombre']) in xmlfiles:
xml=XMLParser("%s.%s" % (modulo['idmodulo'],modulo['nombre']))
xml.parseText(modulo['contenido'])
if xml.root==None:
xml=None
resha1=SHA1(resha1+sha1)
if (modulo['sha']!=sha1):
ficheros_actualizados+=1
print "Updating " + modulo['nombre'] + " => " + sha1 + " ..."
sql+="UPDATE flfiles SET sha='%s' WHERE nombre='%s';\n" % (sha1,modulo['nombre'])
elif (options.debug):
print modulo['nombre'] + " is ok."
if (f_ext(modulo['nombre'])=="mtd"):
tabla=modulo['nombre'][:-4]
qry_modulos=ddb.query("SELECT xml FROM flmetadata WHERE tabla='%s'" % tabla);
tablas=qry_modulos.dictresult()
TablaCargada=False
sql_update_metadata = ""
for txml in tablas:
TablaCargada=True
if txml['xml']!=sha1:
sql_update_metadata="UPDATE flmetadata SET xml='%s' WHERE tabla='%s';\n" % (sha1,tabla)
if not TablaCargada:
print "Cargando tabla nueva %s ..." % tabla
sql_update_metadata=("INSERT INTO flmetadata (tabla,bloqueo,seq,xml)"
" VALUES('%s','f','0','%s');\n" % (tabla,sha1) )
if xml:
if sql_update_metadata or options.full or options.loadbaselec:
updatedTableStructure = load_mtd(options,odb,ddb,tabla,xml)
if updatedTableStructure or not TablaCargada:
if options.verbose:
print "Actualizando metadata para %s" % tabla
if sql_update_metadata:
ddb.query(sql_update_metadata)
elif sql_update_metadata:
if options.rebuildtables or options.forgottables:
print "Actualizando metadatos para tabla %s" % tabla
ddb.query(sql_update_metadata)
else:
tables_notrebuilt.append(modulo['nombre'])
if (len(sql)>1024):
ddb.query(sql)
sql=""
if (len(sql)>0):
ddb.query(sql)
sql=""
qry_d_pkeys=ddb.query("SELECT table_name, column_name,constraint_name FROM information_schema.constraint_column_usage WHERE constraint_name LIKE '%_pkey_%';")
for row in qry_d_pkeys.dictresult():
qry_d_pkeys2=ddb.query("""
SELECT table_name, column_name,constraint_name
FROM information_schema.constraint_column_usage
WHERE constraint_name = '%(table_name)s_pkey';
""" % row)
for row2 in qry_d_pkeys2.dictresult():
sql = """
ALTER TABLE %(table_name)s DROP CONSTRAINT %(constraint_name)s;
""" % row2
try:
ddb.query(sql)
print "Borrado pkey de la tabla %(table_name)s" % row2
except:
print "Error en query corrigiendo pkey:", row2
print traceback.format_exc()
print "SQL:"
print sql
sql = """
ALTER TABLE %(table_name)s DROP CONSTRAINT %(constraint_name)s;
ALTER TABLE %(table_name)s ADD PRIMARY KEY (%(column_name)s);
""" % row
try:
ddb.query(sql)
print "PK Regenerado: %(constraint_name)s" % row
try:
ddb.query("ALTER INDEX %(table_name)s_pkey SET (fillfactor = 80);" % row)
except:
pass
except:
print "Error en query corrigiendo pkey:", row
print traceback.format_exc()
print "SQL:"
print sql
# !---- Verificacion de modulos instalados
qry_dmodulos=ddb.query("SELECT DISTINCT idmodulo FROM flfiles");
modules_installed = set([])
modules_to_uninstall = set([])
for (idmodulo, ) in qry_dmodulos.getresult():
modules_installed.add(idmodulo)
if modules_installed:
# !--- Examinar los modulos ptes de desinstalacion
qry_dmodulos=ddb.query("SELECT idmodulo, descripcion FROM flmodules WHERE bloqueo = TRUE AND idmodulo NOT IN ('" + "', '".join(modules_installed) + "')");
for idmodulo, descripcion in qry_dmodulos.getresult():
modules_to_uninstall.add((idmodulo, descripcion))
if modules_to_uninstall:
if options.cleanfiles:
print "INFO: Los siguientes modulos ya no se usan y serán desinstalados:"
for idmodulo, nombre in modules_to_uninstall:
print " - %s : %s " % (idmodulo, nombre)
ddb.query("DELETE FROM flmodules WHERE idmodulo = '%s'" % idmodulo);
print ":: Finalizado."
else:
print "Existen %d modulos pendientes de desinstalar:" % (len(modules_to_uninstall)),
for idmodulo, nombre in modules_to_uninstall:
print idmodulo,
print
print "Use --cleanfiles para desinstalarlos"
# !---- Verificacion de areas instaladas
qry_dmodulos=ddb.query("SELECT DISTINCT idarea FROM flmodules");
areas_installed = set([])
areas_to_uninstall = set([])
for (idarea, ) in qry_dmodulos.getresult():
areas_installed.add(idarea)
if areas_installed:
# !--- Examinar las areas ptes de desinstalacion
qry_dmodulos=ddb.query("SELECT idarea, descripcion FROM flareas WHERE bloqueo = TRUE AND idarea NOT IN ('" + "', '".join(areas_installed) + "')");
for idarea, descripcion in qry_dmodulos.getresult():
areas_to_uninstall.add((idarea, descripcion))
if areas_to_uninstall:
if options.cleanfiles:
print "INFO: Las siguientes areas ya no se usan y serán desinstaladas:"
for idarea, nombre in areas_to_uninstall:
print " - %s : %s " % (idarea, nombre)
ddb.query("DELETE FROM flareas WHERE idarea = '%s'" % idarea);
print ":: Finalizado."
else:
print "Existen %d areas pendientes de desinstalar:" % (len(areas_to_uninstall)),
for idarea, nombre in areas_to_uninstall:
print idarea,
print
print "Use --cleanfiles para desinstalarlos"
qry_serial=ddb.query("SELECT sha FROM flserial");
serials=qry_serial.dictresult()
for serial in serials:
if (serial['sha']==resha1):
resha1=False
if resha1 and ficheros_actualizados>0:
if len(serials)>0 :
ddb.query("UPDATE flserial SET sha='%s';" % (resha1))
print "Updated flserial => %s." % (resha1)
else:
ddb.query("INSERT INTO flserial (serie,sha) VALUES(1,'%s')" % (resha1))
print "Created flserial => %s." % (resha1)
pydba_loadpgsql.process_create(options,ddb)
if options.transactions: odb.query("COMMIT;");
if tables_notrebuilt:
print "Tables pending rebuild:", ", ".join(tables_notrebuilt)
def load_module_loadone(options, modpath, db, preparse=False):
module = ""
tables = []
mtd_files = {}
filetypes = [
"xml", "ui", "qry", "kut", "qs", "mtd", "ts", "pgsql", "py", "qs.py"
]
allowed_extensions = [
"ar", "kut", "mod", "mtd", "qry", "qs", "ts", "ui", "xml", "xpm",
"qs.py", "py"
]
omitted_exts = []
unicode_filetypes = ["ui", "ts"]
files = []
pd = shelve.open(
"/tmp/pydba") # open -- file may get suffix added by low-level lib
for root, dirs, walk_files in os.walk(modpath):
for name in walk_files:
if not name.endswith(".ts") and f_ext(
name) not in allowed_extensions:
if f_ext(name) not in omitted_exts:
omitted_exts.append(f_ext(name))
#print "Extension %r omitida: %r" % (f_ext(name) , name)
continue
fname = os.path.join(root, name)
mtime = os.stat(fname)[ST_MTIME]
loadFile = True
if pd.has_key(fname):
if pd[fname]["mtime"] == mtime:
loadFile = False
if f_ext(name) == "mod":
module = name[:-4]
file_module = loadfile_inutf8(root, name)
module_parse = XMLParser(name)
module_parse.parseText(file_module)
d_module = {
'name': str(module_parse.root.module.name),
'alias': str(module_parse.root.module.alias),
'area': str(module_parse.root.module.area),
'areaname': str(module_parse.root.module.areaname),
'version': str(module_parse.root.module.version),
'icon': str(module_parse.root.module.icon),
}
if loadFile:
d_module['icon_data'] = loadfile_inutf8(
root, d_module['icon'])
contents = ""
contents_1 = ""
if name.endswith(".ts") or f_ext(name) in filetypes:
contents_1 = loadfile_inutf8(root, name)
contents = my_escape_string(contents_1)
if not loadFile:
try:
if pd[fname]["sha"] is None:
loadFile = True # Some bug in database can cause sha is None.
except:
pass
if loadFile:
sha = SHA1(contents_1)
pd[fname] = {
"mtime": mtime,
"sha": sha,
'root': root,
'name': name
}
options.sha_allowed_files |= set([pd[fname]["sha"]])
options.filenames_allowed_files |= set([pd[fname]["name"]])
if f_ext(
name
) == "qs" and loadFile == True and options.flscriptparser == True:
flscriptparser(root, name)
if f_ext(name) == "pgsql":
array_name = name.split(".")
# name.ext1.pgsql
# array_name = [ "nombre" , "view" , "pgsql" ]
fullfilename = os.path.join(root, name)
if options.verbose: print "Cargando PGSQL: ", fullfilename
ret = None
if len(array_name) == 2:
ret = pydba_loadpgsql.loadpgsqlfile(
fullfilename=fullfilename,
options=options,
database=db,
pgname=array_name[0],
pgtype="sql1",
pgtext=contents_1)
elif len(array_name) != 3:
print "ERROR: Al cargar un .pgsql Se esperaban 2 o 3 elementos al estilo nombre.view.pgsql o nombre.pgsql y se encontró %s " % name
continue
else:
ret = pydba_loadpgsql.loadpgsqlfile(
fullfilename=fullfilename,
options=options,
database=db,
pgname=array_name[0],
pgtype=array_name[1],
pgtext=contents_1)
if ret != True:
print "ERROR: Errores al cargar el fichero ", root, name
if f_ext(name) == "mtd":
table = name[:-4]
# print "### Table: " + table
tables += [table]
mtd_files[table] = contents_1
if preparse:
xml = XMLParser(name)
xml.parseText(contents_1)
if xml.root == None:
#print "ERROR: Failed to parse xml %s" % (name)
xml = None
else:
import pydba_mtdparser
mtd = xml.root.tmd
mparser = pydba_mtdparser.MTDParser()
mparser.parse_mtd(mtd)
mtdquery = getattr(mtd, "query", None)
if not mtdquery:
pydba_mtdparser.Tables[table] = mparser
if contents and f_ext(name) in filetypes:
file = {}
for key, val in pd[fname].iteritems():
file[key] = val
file["contents"] = contents
if (options.modules_loaded.has_key(name)):
print "ERROR: %s file was already loaded." % name
print "--> this file was found at %s" % root
print "--> previous file found at %s" % options.modules_loaded[
name]['root']
print "* skipping file"
else:
options.modules_loaded[name] = file
files += [file]
pd.close()
try:
os.chmod("/tmp/pydba",
S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)
except:
pass
qry_areas = db.query("SELECT descripcion, bloqueo, idarea"
" FROM flareas WHERE idarea='%s'" % d_module['area'])
tareas = qry_areas.dictresult()
if len(tareas) == 0:
print "Creando Area %s - %s " % (d_module['area'],
d_module['areaname'])
db.query("INSERT INTO flareas (descripcion, bloqueo, idarea)"
"VALUES('%s','t','%s')" %
(d_module['areaname'], d_module['area']))
habilitar_carga = False
qry_modulo = db.query(
"SELECT idmodulo, version, descripcion, bloqueo, idarea"
" FROM flmodules WHERE idmodulo='%s'" % module)
tmodulo = qry_modulo.dictresult()
cargado = False
for pmodulo in tmodulo:
cargado = True
# if pmodulo['bloqueo']=='t': # TRUE Es que NO está bloqueado. Está al revés.s
habilitar_carga = True
if cargado and options.updatemodules:
cargado = False
idmodulo = pg.escape_string(d_module['name'])
sql = ("DELETE FROM flmodules WHERE idmodulo = '%s'" % (idmodulo))
db.query(sql)
if not cargado:
print "Se procede a crear el módulo nuevo %s" % module
idmodulo = pg.escape_string(d_module['name'])
idarea = pg.escape_string(d_module['area'])
version = pg.escape_string(d_module['version'])
bloqueo = "t"
descripcion = pg.escape_string(d_module['alias'])
icono = pg.escape_string(d_module['icon_data'])
sql = (
"INSERT INTO flmodules (idmodulo, idarea, version, bloqueo, descripcion,icono) "
"VALUES('%s','%s','%s','%s','%s','%s')" %
(idmodulo, idarea, version, bloqueo, descripcion, icono))
db.query(sql)
habilitar_carga = True
if not habilitar_carga:
print "Error when trying to update the module '%s': non-loaded or locked module" % module
return 0
qry_modulos = db.query(
"SELECT nombre,sha FROM flfiles WHERE idmodulo='%s' " % (module))
tuplas_modulos = qry_modulos.dictresult()
dmodulos = {}
for modulo in tuplas_modulos:
dmodulos[modulo['nombre']] = modulo
loaded = []
for file in files:
update = True
if (dmodulos.has_key(file['name'])):
dm = dmodulos[file['name']]
if (dm['sha'] == file['sha']):
update = False
if options.loadbaselec:
update = (file['name'] == 'baselec.mtd')
if (update):
loaded += [file['name']]
if (options.verbose):
print "* Loading file '%s' => '%s'..." % (file['name'],
file['sha'])
sql = "DELETE FROM flfiles WHERE nombre='%s';\n" % file['name']
db.query(sql)
file['module'] = module
sql = (
"INSERT INTO flfiles (contenido, bloqueo, sha, idmodulo, nombre) "
"VALUES(E'%(contents)s', 't', '%(sha)s','%(module)s', '%(name)s')"
% file)
try:
db.query(sql)
except Exception, e:
print e.__class__.__name__, e
print sql
raise