Beispiel #1
0
def load_module_loadone(options,modpath,db, preparse=False):
    module=""
    tables=[]
    mtd_files={}
    filetypes=["xml","ui","qry","kut","qs","mtd","ts", "pgsql"]
    unicode_filetypes=["ui","ts"]
    
    files=[]
    pd = shelve.open("/tmp/pydba") # open -- file may get suffix added by low-level lib
                          
    
    for root, dirs, walk_files in os.walk(modpath):
            for name in walk_files:
                fname = os.path.join(root, name)
                mtime = os.stat(fname)[ST_MTIME]
                loadFile = True
                if pd.has_key(fname):
                    if pd[fname]["mtime"]==mtime:
                        loadFile=False
                
                if f_ext(name)=="mod":
                    module=name[:-4]
                    file_module=loadfile_inutf8(root, name)
                    module_parse=XMLParser(name)
                    module_parse.parseText(file_module)
                    d_module={
                        'name' :        str(module_parse.root.module.name),
                        'alias' :     str(module_parse.root.module.alias),
                        'area' :        str(module_parse.root.module.area),
                        'areaname' :        str(module_parse.root.module.areaname),
                        'version' :        str(module_parse.root.module.version),
                        'icon' :        str(module_parse.root.module.icon),
                        }
                    if loadFile:
                        d_module['icon_data']=loadfile_inutf8(root, d_module['icon'])
                
                contents=""
                contents_1=""
                if f_ext(name) in filetypes:
                    contents_1=loadfile_inutf8(root,name)
                    contents=my_escape_string(contents_1)
                    if not loadFile:
                        try:
                            if pd[fname]["sha"] is None: loadFile = True # Some bug in database can cause sha is None.
                        except:
                            pass
                            
                    if loadFile:                    
                        sha=SHA1(contents_1)
                        pd[fname]={"mtime":mtime, "sha":sha, 'root' : root,'name' : name}
                    options.sha_allowed_files |= set([pd[fname]["sha"]])    
                    options.filenames_allowed_files |= set([pd[fname]["name"]])    
                    
                if f_ext(name)=="qs" and loadFile==True and options.flscriptparser==True:
                    flscriptparser(root,name)              
                    
                
                if f_ext(name)=="pgsql":
                    array_name=name.split(".")
                    # name.ext1.pgsql
                    # array_name = [ "nombre" , "view" , "pgsql" ]
                    fullfilename = os.path.join(root, name)
                    if options.verbose: print "Cargando PGSQL: ", fullfilename
                    ret = None
                    if len(array_name)==2:
                        ret = pydba_loadpgsql.loadpgsqlfile(fullfilename = fullfilename, options = options, database = db, pgname = array_name[0], pgtype = "sql1", pgtext = contents_1)
                    elif len(array_name)!=3:
                        print "ERROR: Al cargar un .pgsql Se esperaban 2 o 3 elementos al estilo nombre.view.pgsql o nombre.pgsql y se encontró %s " % name
                        continue
                    else:
                        ret = pydba_loadpgsql.loadpgsqlfile(fullfilename = fullfilename, options = options, database = db, pgname = array_name[0], pgtype = array_name[1], pgtext = contents_1)
                    
                    if ret != True:
                        print "ERROR: Errores al cargar el fichero ", root, name
                        
                        
                        
                        
                    
                
                    
                
                if f_ext(name)=="mtd":
                    table=name[:-4]
                    # print "### Table: " + table
                    tables+=[table]
                    mtd_files[table]=contents_1
                    if preparse:
                        xml=XMLParser(name)
                        xml.parseText(contents_1)
                        if xml.root==None:
                            #print "ERROR: Failed to parse xml %s" %  (name)
                            xml=None
                        else:
                            import pydba_mtdparser
                            
                            mtd=xml.root.tmd
                            
                            mparser=pydba_mtdparser.MTDParser()
                            mparser.parse_mtd(mtd)
                            mtdquery = getattr(mtd, "query", None)
                            if not mtdquery:
                                pydba_mtdparser.Tables[table]=mparser
                            
                                

                        
                    
                
                if contents and f_ext(name) in filetypes:
                    file={}
                    for key,val in pd[fname].iteritems():
                        file[key]=val
                    file["contents"]=contents
                    if (options.modules_loaded.has_key(name)):
                        print "ERROR: %s file was already loaded." % name
                        print "--> this file was found at %s" % root
                        print "--> previous file found at %s" % options.modules_loaded[name]['root']
                        print "* skipping file"
                    else:
                        options.modules_loaded[name]=file
                        files+=[file]
    
    pd.close()
    try:
        os.chmod("/tmp/pydba", S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)
    except:
        pass

    qry_areas=db.query("SELECT descripcion, bloqueo, idarea"
                                                " FROM flareas WHERE idarea='%s'" % d_module['area'])
    tareas=qry_areas.dictresult()                        
    if len(tareas)==0:
        print "Creando Area %s - %s " %(d_module['area'],d_module['areaname'])
        db.query("INSERT INTO flareas (descripcion, bloqueo, idarea)"
                "VALUES('%s','t','%s')" % (d_module['areaname'],d_module['area']))
                    
                                                    
    
    habilitar_carga=False
    qry_modulo=db.query("SELECT idmodulo, version, descripcion, bloqueo, idarea"
                            " FROM flmodules WHERE idmodulo='%s'" % module);
    tmodulo=qry_modulo.dictresult() 
    cargado=False
    for pmodulo in tmodulo:
        cargado=True
        # if pmodulo['bloqueo']=='t':    # TRUE Es que NO está bloqueado. Está al revés.s
        habilitar_carga=True
    if cargado and options.updatemodules:
        cargado = False
        idmodulo        = pg.escape_string(d_module['name']) 
        sql=("DELETE FROM flmodules WHERE idmodulo = '%s'"  % 
                            (idmodulo))
        db.query(sql)     
        
    if not cargado:
        print "Se procede a crear el módulo nuevo %s" % module
        
        idmodulo        = pg.escape_string(d_module['name']) 
        idarea            = pg.escape_string(d_module['area'])
        version         = pg.escape_string(d_module['version'])
        bloqueo         = "t"
        descripcion = pg.escape_string(d_module['alias'])
        icono             = pg.escape_string(d_module['icon_data'])
        
        sql=("INSERT INTO flmodules (idmodulo, idarea, version, bloqueo, descripcion,icono) "    
                    "VALUES('%s','%s','%s','%s','%s','%s')" % 
                            (idmodulo, idarea, version, bloqueo, descripcion,icono))
        db.query(sql)     
        habilitar_carga=True     

    if not habilitar_carga:
        print "Error when trying to update the module '%s': non-loaded or locked module" % module
        return 0
    
    qry_modulos=db.query("SELECT nombre,sha FROM flfiles WHERE idmodulo='%s' " % (module));
    tuplas_modulos=qry_modulos.dictresult() 
    dmodulos={}
    for modulo in tuplas_modulos:
        dmodulos[modulo['nombre']]=modulo;
        
    loaded=[]
    
    for file in files:
        
        update=True        
        if (dmodulos.has_key(file['name'])):
            dm=dmodulos[file['name']]
            if (dm['sha']==file['sha']):
                update=False
        
        if options.loadbaselec:
            update=(file['name']=='baselec.mtd')
              
        if (update):
            loaded+=[file['name']]
            if (options.verbose):
                print "* Loading file '%s' => '%s'..." % (file['name'],file['sha'])
            sql="DELETE FROM flfiles WHERE nombre='%s';\n" % file['name']
            db.query(sql)
        
            file['module']=module
            
            sql=("INSERT INTO flfiles (contenido, bloqueo, sha, idmodulo, nombre) "    
                    "VALUES(E'%(contents)s', 't', '%(sha)s','%(module)s', '%(name)s')" % file)
            try:
                db.query(sql)
            except Exception, e:
                print e.__class__.__name__, e
                print sql
                
                raise
Beispiel #2
0
def repair_db(options,ddb=None,mode=0,odb=None):
    if (options.verbose):
        print "-> RepairDB"
    if (not ddb):
        if (not options.ddb):
            print "RepairDB requiere una base de datos de destino y no proporcionó ninguna."
            return 0
        ddb=dbconnect(options)
        if (not ddb): 
            return 0
    
    if (not odb):
        if (not options.odb):
            print "RepairDB requiere una base de datos de origen y no proporcionó ninguna."
            return 0
        odb=odbconnect(options)
        if (not odb): 
            return 0
        
    where=""
    if not options.full :
        #if len(options.files_loaded)>0:
        where+=" AND (nombre IN ('" + "','".join(options.files_loaded) + "' ) OR nombre LIKE '%.mtd')"
          
        if len(options.modules)>0:
            where+=" AND ( 0=1 "
            for modname,module in options.modules.iteritems():
                noloadtable=[]
                loadtable=[]
                for tablename, value in module.iteritems():
                    if value==False:
                        noloadtable.append(tablename + ".mtd")
                    else:
                        loadtable.append(tablename + ".mtd")
                
                try:
                    default=module['_default_']
                except:
                    default=False
                
                if default==True:
                    where+=" OR ( idmodulo = '%s' AND nombre NOT IN ('" % modname + "','".join(noloadtable) + "'))" 
                else:
                    where+=" OR ( idmodulo = '%s' AND nombre IN ('" % modname + "','".join(loadtable) + "'))" 
            where+=" OR nombre LIKE '%.mtd' )"
                       
        if options.odb!=options.ddb:
            where+=" AND nombre LIKE '%.mtd'"
        
    if (options.verbose):
        print "Inicializando reparación de la base de datos '%s'..." % options.ddb
        print " * Calcular firmas SHA1 de files y metadata"
    
    if options.debug:
        print "* Revisando si existe una tabla de servidores réplica ..."
    tabla = None
    qry1 =ddb.query("SELECT relname FROM pg_class WHERE relname ='servidoresreplica'");
    for row in qry1.dictresult():
        tabla = row['relname']
    
    if tabla:
        if options.debug:
            print "Encontrada una tabla %s." % tabla
        qry1 =ddb.query("SELECT codservidor, activo, numero  FROM %s" % tabla);
        servsize = len(qry1.dictresult())
        servnumber = -1
        servname = None
        for row in qry1.dictresult():
            if row['activo'] == 't':
                try:
                    assert(servnumber == -1)
                except AssertionError:
                    print "PANIC: Hay mas de un servidor activo! El valor anterior era %s el nuevo %s. (activo vale %s)" % (
                        repr(servnumber),
                        repr(row['numero']),
                        repr(row['activo'])
                        )
                    raise
                servnumber = row['numero']
                servname = row['codservidor']
        activar_servrep = True
        if servsize < 1 or servsize > 10:
            activar_servrep = False
            print "WARN: No se activa la sincronización de secuencia para replicación master-master por error en la cantidad de servidores (%d)" % servsize
            
        if servnumber < 0 or servnumber >= servsize:
            activar_servrep = False
            print "WARN: No se activa la sincronización de secuencia para replicación master-master por error el numero de este servidor (%d/%d)" % (servnumber,servsize)
        
        if activar_servrep:
            print "INFO: Activando sincro. de secuencias para %s: %d/%d" % (repr(servname), servnumber,servsize)
            options.seqsync = (servnumber,servsize)
        
    else:
        if options.debug:
            print "No se encontró tabla de servidores de replica"
            
        
    
    
    if options.transactions: 
        odb.query("BEGIN;");
        try:
            lltables = "flfiles,flserial,flmetadata".split(",")
            for ltable in lltables:
                sql = "LOCK %s NOWAIT;" % ltable
                if (options.verbose): print sql
                odb.query(sql);
                if (options.verbose): print "done."
        except:
            print "Error al bloquear la tabla %s , ¡algun otro usuario está conectado!" % ltable
            odb.query("ROLLBACK;");
            raise
            
    tables_notrebuilt = []
    pydba_loadpgsql.process_drop(options,ddb)
    

    qry_omodulos=odb.query("SELECT sha " +
            "FROM flfiles WHERE sha!='' AND nombre NOT LIKE '%%alteredtable%%.mtd' ORDER BY sha");
    ofiles=[]
    for row in qry_omodulos.dictresult():
        ofiles.append(row['sha'])
    
    qry_dmodulos=ddb.query("SELECT sha " +
            "FROM flfiles WHERE sha!='' AND nombre NOT LIKE '%%alteredtable%%.mtd' ORDER BY sha");
    dfiles=[]
    for row in qry_dmodulos.dictresult():
        if row['sha'] in ofiles:
            ofiles.remove(row['sha'])
        else:
            dfiles.append(row['sha'])
    
    # Eliminar los ficheros sobrantes.
    qry_dmodulos=ddb.query("DELETE FROM flfiles WHERE sha IN ('" + "','".join(dfiles) + "')")
    
    # Obtener los ficheros nuevos
    qry_omodulos=odb.query("SELECT * FROM flfiles WHERE sha IN ('" + "','".join(ofiles) + "')")
    
    # Insertarlos en la nueva DB.
    for row in qry_omodulos.dictresult():
        fields=row.keys()
        values=[]
        for field in fields:
            campo=row[field]
            if (campo is not None):#Si el valor es nulo
                values.append("(E'" + pg.escape_string(str(campo)) + "')")
            else:
                values.append("NULL")
        try:
            qry_dmodulos=ddb.query("DELETE FROM flfiles WHERE nombre ='" + row['nombre'] + "'")
            
            sql="INSERT INTO flfiles (" + ",".join(fields) + ") VALUES(" + ",".join(values) + ")"
            ddb.query(sql)
        except:
            print sql
            raise
        
        
    
    sqlModulos = ("SELECT idmodulo, nombre, contenido, sha " +
                    "FROM flfiles WHERE sha!='' AND nombre NOT LIKE '%%alteredtable%%.mtd' "
                    + where + " ORDER BY idmodulo, nombre")
    # print sqlModulos
    qry_modulos=ddb.query(sqlModulos);
                                
    modulos=qry_modulos.dictresult() 
    # print "%d resultados." % len(modulos)
    sql=""
    resha1="";
    xmlfiles=("xml","ui","qry","kut","mtd","ts")
    ficheros_actualizados=0
    for modulo in modulos:
        if options.loadbaselec:
            if modulo['nombre'] != 'baselec.mtd': 
                continue
        xml=None
        if options.full and modulo.has_key('contenido'):
            sha1=SHA1(modulo['contenido'])
        else:            
            sha1=modulo['sha']
        
        if (sha1==None):
            print "ERROR: Carácteres no ISO en %s.%s (se omite SHA1)" % (modulo['idmodulo'],modulo['nombre'])
            sha1=modulo['sha']
        
        if f_ext(modulo['nombre']) in xmlfiles:
            xml=XMLParser("%s.%s" % (modulo['idmodulo'],modulo['nombre']))
            xml.parseText(modulo['contenido'])
            if xml.root==None:
                xml=None
            
        
        resha1=SHA1(resha1+sha1)
        if (modulo['sha']!=sha1):
            ficheros_actualizados+=1
            print "Updating " + modulo['nombre'] + " => " + sha1 + " ..."
            sql+="UPDATE flfiles SET sha='%s' WHERE nombre='%s';\n" %    (sha1,modulo['nombre'])
        elif (options.debug):
            print modulo['nombre'] + " is ok."
        
        if (f_ext(modulo['nombre'])=="mtd"):
            tabla=modulo['nombre'][:-4]
            
            qry_modulos=ddb.query("SELECT xml FROM flmetadata WHERE tabla='%s'" % tabla);
            tablas=qry_modulos.dictresult() 
            TablaCargada=False
            sql_update_metadata = ""
            for txml in tablas:
                TablaCargada=True
                if txml['xml']!=sha1:
                    sql_update_metadata="UPDATE flmetadata SET xml='%s' WHERE tabla='%s';\n" % (sha1,tabla)
            if not TablaCargada:
                    print "Cargando tabla nueva %s ..." % tabla
                    sql_update_metadata=("INSERT INTO flmetadata (tabla,bloqueo,seq,xml)"
                        " VALUES('%s','f','0','%s');\n" % (tabla,sha1) )
            if xml:
                if sql_update_metadata or options.full or options.loadbaselec:    
                    updatedTableStructure = load_mtd(options,odb,ddb,tabla,xml)
                    if updatedTableStructure or not TablaCargada:
                        if options.verbose:
                            print "Actualizando metadata para %s" % tabla
                        if sql_update_metadata:
                            ddb.query(sql_update_metadata)
                    elif sql_update_metadata:
                        if options.rebuildtables or options.forgottables:
                            print "Actualizando metadatos para tabla %s" % tabla
                            ddb.query(sql_update_metadata)
                        else:
                            tables_notrebuilt.append(modulo['nombre'])
                
        if (len(sql)>1024):
            ddb.query(sql)
            sql=""
    
    if (len(sql)>0):    
        ddb.query(sql)
        sql=""
    
    qry_d_pkeys=ddb.query("SELECT table_name, column_name,constraint_name FROM information_schema.constraint_column_usage WHERE constraint_name LIKE '%_pkey_%';")
    for row in qry_d_pkeys.dictresult():
        qry_d_pkeys2=ddb.query("""
        SELECT table_name, column_name,constraint_name 
        FROM information_schema.constraint_column_usage 
        WHERE constraint_name = '%(table_name)s_pkey';
        """ % row)
        for row2 in qry_d_pkeys2.dictresult():
            sql = """
            ALTER TABLE %(table_name)s DROP CONSTRAINT %(constraint_name)s;
            """ % row2
            try: 
                ddb.query(sql)
                print "Borrado pkey de la tabla %(table_name)s" % row2
            except:
                print "Error en query corrigiendo pkey:", row2
                print traceback.format_exc()
                print "SQL:"
                print sql
        
        sql = """
        ALTER TABLE %(table_name)s DROP CONSTRAINT %(constraint_name)s;
        ALTER TABLE %(table_name)s ADD PRIMARY KEY (%(column_name)s);
        """ % row
        try: 
            ddb.query(sql)
            print "PK Regenerado: %(constraint_name)s" % row
            try:
                ddb.query("ALTER INDEX %(table_name)s_pkey SET (fillfactor = 80);" % row)
            except:
                pass
        except:
            print "Error en query corrigiendo pkey:", row
            print traceback.format_exc()
            print "SQL:"
            print sql

    # !---- Verificacion de modulos instalados
    qry_dmodulos=ddb.query("SELECT DISTINCT idmodulo FROM flfiles");
    modules_installed = set([])
    modules_to_uninstall = set([])
    for (idmodulo, ) in qry_dmodulos.getresult():
        modules_installed.add(idmodulo)
    
    if modules_installed:
        # !--- Examinar los modulos ptes de desinstalacion
        qry_dmodulos=ddb.query("SELECT idmodulo, descripcion FROM flmodules WHERE bloqueo = TRUE AND idmodulo NOT IN ('" + "', '".join(modules_installed) + "')");
        for idmodulo, descripcion in qry_dmodulos.getresult():
            modules_to_uninstall.add((idmodulo, descripcion))
    
    if modules_to_uninstall:
        if options.cleanfiles:
            print "INFO: Los siguientes modulos ya no se usan y serán desinstalados:"
            for idmodulo, nombre in modules_to_uninstall:
                print " - %s : %s " % (idmodulo, nombre)
                ddb.query("DELETE FROM flmodules WHERE idmodulo = '%s'" % idmodulo);
            print ":: Finalizado."
        else:
            print "Existen %d modulos pendientes de desinstalar:" % (len(modules_to_uninstall)),
            for idmodulo, nombre in modules_to_uninstall:
                print idmodulo, 
            print
            print "Use --cleanfiles para desinstalarlos"
    # !---- Verificacion de areas instaladas
    qry_dmodulos=ddb.query("SELECT DISTINCT idarea FROM flmodules");
    areas_installed = set([])
    areas_to_uninstall = set([])
    for (idarea, ) in qry_dmodulos.getresult():
        areas_installed.add(idarea)
    
    if areas_installed:
        # !--- Examinar las areas ptes de desinstalacion
        qry_dmodulos=ddb.query("SELECT idarea, descripcion FROM flareas WHERE bloqueo = TRUE AND idarea NOT IN ('" + "', '".join(areas_installed) + "')");
        for idarea, descripcion in qry_dmodulos.getresult():
            areas_to_uninstall.add((idarea, descripcion))
    
    if areas_to_uninstall:
        if options.cleanfiles:
            print "INFO: Las siguientes areas ya no se usan y serán desinstaladas:"
            for idarea, nombre in areas_to_uninstall:
                print " - %s : %s " % (idarea, nombre)
                ddb.query("DELETE FROM flareas WHERE idarea = '%s'" % idarea);
            print ":: Finalizado."
        else:
            print "Existen %d areas pendientes de desinstalar:" % (len(areas_to_uninstall)),
            for idarea, nombre in areas_to_uninstall:
                print idarea, 
            print
            print "Use --cleanfiles para desinstalarlos"
            
    qry_serial=ddb.query("SELECT sha FROM flserial");
    serials=qry_serial.dictresult() 
    for serial in serials:
        if (serial['sha']==resha1):
            resha1=False
                
    if resha1 and ficheros_actualizados>0:
        if len(serials)>0 :
            ddb.query("UPDATE flserial SET sha='%s';" % (resha1))
            print "Updated flserial => %s." % (resha1)     
        else:        
            ddb.query("INSERT INTO flserial (serie,sha) VALUES(1,'%s')" % (resha1))
            print "Created flserial => %s." % (resha1)     
    
    pydba_loadpgsql.process_create(options,ddb)
    if options.transactions: odb.query("COMMIT;");
    if tables_notrebuilt:
        print "Tables pending rebuild:", ", ".join(tables_notrebuilt)
Beispiel #3
0
def repair_db(options,ddb=None,mode=0,odb=None):
    if (options.verbose):
        print "-> RepairDB"
    if (not ddb):
        if (not options.ddb):
            print "RepairDB requiere una base de datos de destino y no proporcionó ninguna."
            return 0
        ddb=dbconnect(options)
        if (not ddb): 
            return 0
    
    if (not odb):
        if (not options.odb):
            print "RepairDB requiere una base de datos de origen y no proporcionó ninguna."
            return 0
        odb=odbconnect(options)
        if (not odb): 
            return 0
        
    where=""
    if not options.full :
        #if len(options.files_loaded)>0:
        where+=" AND (nombre IN ('" + "','".join(options.files_loaded) + "' ) OR nombre LIKE '%.mtd')"
          
        if len(options.modules)>0:
            where+=" AND ( 0=1 "
            for modname,module in options.modules.iteritems():
                noloadtable=[]
                loadtable=[]
                for tablename, value in module.iteritems():
                    if value==False:
                        noloadtable.append(tablename + ".mtd")
                    else:
                        loadtable.append(tablename + ".mtd")
                
                try:
                    default=module['_default_']
                except:
                    default=False
                
                if default==True:
                    where+=" OR ( idmodulo = '%s' AND nombre NOT IN ('" % modname + "','".join(noloadtable) + "'))" 
                else:
                    where+=" OR ( idmodulo = '%s' AND nombre IN ('" % modname + "','".join(loadtable) + "'))" 
            where+=" OR nombre LIKE '%.mtd' )"
                       
        if options.odb!=options.ddb:
            where+=" AND nombre LIKE '%.mtd'"
        
    if (options.verbose):
        print "Inicializando reparación de la base de datos '%s'..." % options.ddb
        print " * Calcular firmas SHA1 de files y metadata"
    
    if options.debug:
        print "* Revisando si existe una tabla de servidores réplica ..."
    tabla = None
    qry1 =ddb.query("SELECT relname FROM pg_class WHERE relname ='servidoresreplica'");
    for row in qry1.dictresult():
        tabla = row['relname']
    
    if tabla:
        if options.debug:
            print "Encontrada una tabla %s." % tabla
        qry1 =ddb.query("SELECT codservidor, activo, numero  FROM %s" % tabla);
        servsize = len(qry1.dictresult())
        servnumber = -1
        servname = None
        for row in qry1.dictresult():
            if row['activo'] == 't':
                try:
                    assert(servnumber == -1)
                except AssertionError:
                    print "PANIC: Hay mas de un servidor activo! El valor anterior era %s el nuevo %s. (activo vale %s)" % (
                        repr(servnumber),
                        repr(row['numero']),
                        repr(row['activo'])
                        )
                    raise
                servnumber = row['numero']
                servname = row['codservidor']
        activar_servrep = True
        if servsize < 1 or servsize > 10:
            activar_servrep = False
            print "WARN: No se activa la sincronización de secuencia para replicación master-master por error en la cantidad de servidores (%d)" % servsize
            
        if servnumber < 0 or servnumber >= servsize:
            activar_servrep = False
            print "WARN: No se activa la sincronización de secuencia para replicación master-master por error el numero de este servidor (%d/%d)" % (servnumber,servsize)
        
        if activar_servrep:
            print "INFO: Activando sincro. de secuencias para %s: %d/%d" % (repr(servname), servnumber,servsize)
            options.seqsync = (servnumber,servsize)
        
    else:
        if options.debug:
            print "No se encontró tabla de servidores de replica"
            
        
    
    
    if options.transactions: 
        odb.query("BEGIN;");
        try:
            lltables = "flfiles,flserial,flmetadata".split(",")
            for ltable in lltables:
                sql = "LOCK %s NOWAIT;" % ltable
                if (options.verbose): print sql
                odb.query(sql);
                if (options.verbose): print "done."
        except:
            print "Error al bloquear la tabla %s , ¡algun otro usuario está conectado!" % ltable
            odb.query("ROLLBACK;");
            raise
            
    tables_notrebuilt = []
    pydba_loadpgsql.process_drop(options,ddb)
    

    qry_omodulos=odb.query("SELECT sha " +
            "FROM flfiles WHERE sha!='' AND nombre NOT LIKE '%%alteredtable%%.mtd' ORDER BY sha");
    ofiles=[]
    for row in qry_omodulos.dictresult():
        ofiles.append(row['sha'])
    
    qry_dmodulos=ddb.query("SELECT sha " +
            "FROM flfiles WHERE sha!='' AND nombre NOT LIKE '%%alteredtable%%.mtd' ORDER BY sha");
    dfiles=[]
    for row in qry_dmodulos.dictresult():
        if row['sha'] in ofiles:
            ofiles.remove(row['sha'])
        else:
            dfiles.append(row['sha'])
    
    # Eliminar los ficheros sobrantes.
    qry_dmodulos=ddb.query("DELETE FROM flfiles WHERE sha IN ('" + "','".join(dfiles) + "')")
    
    # Obtener los ficheros nuevos
    qry_omodulos=odb.query("SELECT * FROM flfiles WHERE sha IN ('" + "','".join(ofiles) + "')")
    
    # Insertarlos en la nueva DB.
    for row in qry_omodulos.dictresult():
        fields=row.keys()
        values=[]
        for field in fields:
            campo=row[field]
            if (campo is not None):#Si el valor es nulo
                values.append("(E'" + pg.escape_string(str(campo)) + "')")
            else:
                values.append("NULL")
        try:
            qry_dmodulos=ddb.query("DELETE FROM flfiles WHERE nombre ='" + row['nombre'] + "'")
            
            sql="INSERT INTO flfiles (" + ",".join(fields) + ") VALUES(" + ",".join(values) + ")"
            ddb.query(sql)
        except:
            print sql
            raise
        
        
    
    sqlModulos = ("SELECT idmodulo, nombre, contenido, sha " +
                    "FROM flfiles WHERE sha!='' AND nombre NOT LIKE '%%alteredtable%%.mtd' "
                    + where + " ORDER BY idmodulo, nombre")
    # print sqlModulos
    qry_modulos=ddb.query(sqlModulos);
                                
    modulos=qry_modulos.dictresult() 
    # print "%d resultados." % len(modulos)
    sql=""
    resha1="";
    xmlfiles=("xml","ui","qry","kut","mtd","ts")
    ficheros_actualizados=0
    for modulo in modulos:
        if options.loadbaselec:
            if modulo['nombre'] != 'baselec.mtd': 
                continue
        xml=None
        if options.full and modulo.has_key('contenido'):
            sha1=SHA1(modulo['contenido'])
        else:            
            sha1=modulo['sha']
        
        if (sha1==None):
            print "ERROR: Carácteres no ISO en %s.%s (se omite SHA1)" % (modulo['idmodulo'],modulo['nombre'])
            sha1=modulo['sha']
        
        if f_ext(modulo['nombre']) in xmlfiles:
            xml=XMLParser("%s.%s" % (modulo['idmodulo'],modulo['nombre']))
            xml.parseText(modulo['contenido'])
            if xml.root==None:
                xml=None
            
        
        resha1=SHA1(resha1+sha1)
        if (modulo['sha']!=sha1):
            ficheros_actualizados+=1
            print "Updating " + modulo['nombre'] + " => " + sha1 + " ..."
            sql+="UPDATE flfiles SET sha='%s' WHERE nombre='%s';\n" %    (sha1,modulo['nombre'])
        elif (options.debug):
            print modulo['nombre'] + " is ok."
        
        if (f_ext(modulo['nombre'])=="mtd"):
            tabla=modulo['nombre'][:-4]
            
            qry_modulos=ddb.query("SELECT xml FROM flmetadata WHERE tabla='%s'" % tabla);
            tablas=qry_modulos.dictresult() 
            TablaCargada=False
            sql_update_metadata = ""
            for txml in tablas:
                TablaCargada=True
                if txml['xml']!=sha1:
                    sql_update_metadata="UPDATE flmetadata SET xml='%s' WHERE tabla='%s';\n" % (sha1,tabla)
            if not TablaCargada:
                    print "Cargando tabla nueva %s ..." % tabla
                    sql_update_metadata=("INSERT INTO flmetadata (tabla,bloqueo,seq,xml)"
                        " VALUES('%s','f','0','%s');\n" % (tabla,sha1) )
            if xml:
                if sql_update_metadata or options.full or options.loadbaselec:    
                    updatedTableStructure = load_mtd(options,odb,ddb,tabla,xml)
                    if updatedTableStructure or not TablaCargada:
                        if options.verbose:
                            print "Actualizando metadata para %s" % tabla
                        if sql_update_metadata:
                            ddb.query(sql_update_metadata)
                    elif sql_update_metadata:
                        if options.rebuildtables or options.forgottables:
                            print "Actualizando metadatos para tabla %s" % tabla
                            ddb.query(sql_update_metadata)
                        else:
                            tables_notrebuilt.append(modulo['nombre'])
                
        if (len(sql)>1024):
            ddb.query(sql)
            sql=""
    
    if (len(sql)>0):    
        ddb.query(sql)
        sql=""
    
    qry_d_pkeys=ddb.query("SELECT table_name, column_name,constraint_name FROM information_schema.constraint_column_usage WHERE constraint_name LIKE '%_pkey_%';")
    for row in qry_d_pkeys.dictresult():
        qry_d_pkeys2=ddb.query("""
        SELECT table_name, column_name,constraint_name 
        FROM information_schema.constraint_column_usage 
        WHERE constraint_name = '%(table_name)s_pkey';
        """ % row)
        for row2 in qry_d_pkeys2.dictresult():
            sql = """
            ALTER TABLE %(table_name)s DROP CONSTRAINT %(constraint_name)s;
            """ % row2
            try: 
                ddb.query(sql)
                print "Borrado pkey de la tabla %(table_name)s" % row2
            except:
                print "Error en query corrigiendo pkey:", row2
                print traceback.format_exc()
                print "SQL:"
                print sql
        
        sql = """
        ALTER TABLE %(table_name)s DROP CONSTRAINT %(constraint_name)s;
        ALTER TABLE %(table_name)s ADD PRIMARY KEY (%(column_name)s);
        """ % row
        try: 
            ddb.query(sql)
            print "PK Regenerado: %(constraint_name)s" % row
            try:
                ddb.query("ALTER INDEX %(table_name)s_pkey SET (fillfactor = 80);" % row)
            except:
                pass
        except:
            print "Error en query corrigiendo pkey:", row
            print traceback.format_exc()
            print "SQL:"
            print sql

    # !---- Verificacion de modulos instalados
    qry_dmodulos=ddb.query("SELECT DISTINCT idmodulo FROM flfiles");
    modules_installed = set([])
    modules_to_uninstall = set([])
    for (idmodulo, ) in qry_dmodulos.getresult():
        modules_installed.add(idmodulo)
    
    if modules_installed:
        # !--- Examinar los modulos ptes de desinstalacion
        qry_dmodulos=ddb.query("SELECT idmodulo, descripcion FROM flmodules WHERE bloqueo = TRUE AND idmodulo NOT IN ('" + "', '".join(modules_installed) + "')");
        for idmodulo, descripcion in qry_dmodulos.getresult():
            modules_to_uninstall.add((idmodulo, descripcion))
    
    if modules_to_uninstall:
        if options.cleanfiles:
            print "INFO: Los siguientes modulos ya no se usan y serán desinstalados:"
            for idmodulo, nombre in modules_to_uninstall:
                print " - %s : %s " % (idmodulo, nombre)
                ddb.query("DELETE FROM flmodules WHERE idmodulo = '%s'" % idmodulo);
            print ":: Finalizado."
        else:
            print "Existen %d modulos pendientes de desinstalar:" % (len(modules_to_uninstall)),
            for idmodulo, nombre in modules_to_uninstall:
                print idmodulo, 
            print
            print "Use --cleanfiles para desinstalarlos"
    # !---- Verificacion de areas instaladas
    qry_dmodulos=ddb.query("SELECT DISTINCT idarea FROM flmodules");
    areas_installed = set([])
    areas_to_uninstall = set([])
    for (idarea, ) in qry_dmodulos.getresult():
        areas_installed.add(idarea)
    
    if areas_installed:
        # !--- Examinar las areas ptes de desinstalacion
        qry_dmodulos=ddb.query("SELECT idarea, descripcion FROM flareas WHERE bloqueo = TRUE AND idarea NOT IN ('" + "', '".join(areas_installed) + "')");
        for idarea, descripcion in qry_dmodulos.getresult():
            areas_to_uninstall.add((idarea, descripcion))
    
    if areas_to_uninstall:
        if options.cleanfiles:
            print "INFO: Las siguientes areas ya no se usan y serán desinstaladas:"
            for idarea, nombre in areas_to_uninstall:
                print " - %s : %s " % (idarea, nombre)
                ddb.query("DELETE FROM flareas WHERE idarea = '%s'" % idarea);
            print ":: Finalizado."
        else:
            print "Existen %d areas pendientes de desinstalar:" % (len(areas_to_uninstall)),
            for idarea, nombre in areas_to_uninstall:
                print idarea, 
            print
            print "Use --cleanfiles para desinstalarlos"
            
    qry_serial=ddb.query("SELECT sha FROM flserial");
    serials=qry_serial.dictresult() 
    for serial in serials:
        if (serial['sha']==resha1):
            resha1=False
                
    if resha1 and ficheros_actualizados>0:
        if len(serials)>0 :
            ddb.query("UPDATE flserial SET sha='%s';" % (resha1))
            print "Updated flserial => %s." % (resha1)     
        else:        
            ddb.query("INSERT INTO flserial (serie,sha) VALUES(1,'%s')" % (resha1))
            print "Created flserial => %s." % (resha1)     
    
    pydba_loadpgsql.process_create(options,ddb)
    if options.transactions: odb.query("COMMIT;");
    if tables_notrebuilt:
        print "Tables pending rebuild:", ", ".join(tables_notrebuilt)
Beispiel #4
0
def load_module_loadone(options, modpath, db, preparse=False):
    module = ""
    tables = []
    mtd_files = {}
    filetypes = [
        "xml", "ui", "qry", "kut", "qs", "mtd", "ts", "pgsql", "py", "qs.py"
    ]
    allowed_extensions = [
        "ar", "kut", "mod", "mtd", "qry", "qs", "ts", "ui", "xml", "xpm",
        "qs.py", "py"
    ]
    omitted_exts = []
    unicode_filetypes = ["ui", "ts"]

    files = []
    pd = shelve.open(
        "/tmp/pydba")  # open -- file may get suffix added by low-level lib

    for root, dirs, walk_files in os.walk(modpath):
        for name in walk_files:
            if not name.endswith(".ts") and f_ext(
                    name) not in allowed_extensions:
                if f_ext(name) not in omitted_exts:
                    omitted_exts.append(f_ext(name))
                    #print "Extension %r omitida: %r" % (f_ext(name) , name)

                continue
            fname = os.path.join(root, name)
            mtime = os.stat(fname)[ST_MTIME]
            loadFile = True
            if pd.has_key(fname):
                if pd[fname]["mtime"] == mtime:
                    loadFile = False

            if f_ext(name) == "mod":
                module = name[:-4]
                file_module = loadfile_inutf8(root, name)
                module_parse = XMLParser(name)
                module_parse.parseText(file_module)
                d_module = {
                    'name': str(module_parse.root.module.name),
                    'alias': str(module_parse.root.module.alias),
                    'area': str(module_parse.root.module.area),
                    'areaname': str(module_parse.root.module.areaname),
                    'version': str(module_parse.root.module.version),
                    'icon': str(module_parse.root.module.icon),
                }
                if loadFile:
                    d_module['icon_data'] = loadfile_inutf8(
                        root, d_module['icon'])

            contents = ""
            contents_1 = ""
            if name.endswith(".ts") or f_ext(name) in filetypes:
                contents_1 = loadfile_inutf8(root, name)
                contents = my_escape_string(contents_1)
                if not loadFile:
                    try:
                        if pd[fname]["sha"] is None:
                            loadFile = True  # Some bug in database can cause sha is None.
                    except:
                        pass

                if loadFile:
                    sha = SHA1(contents_1)
                    pd[fname] = {
                        "mtime": mtime,
                        "sha": sha,
                        'root': root,
                        'name': name
                    }
                options.sha_allowed_files |= set([pd[fname]["sha"]])
                options.filenames_allowed_files |= set([pd[fname]["name"]])

            if f_ext(
                    name
            ) == "qs" and loadFile == True and options.flscriptparser == True:
                flscriptparser(root, name)

            if f_ext(name) == "pgsql":
                array_name = name.split(".")
                # name.ext1.pgsql
                # array_name = [ "nombre" , "view" , "pgsql" ]
                fullfilename = os.path.join(root, name)
                if options.verbose: print "Cargando PGSQL: ", fullfilename
                ret = None
                if len(array_name) == 2:
                    ret = pydba_loadpgsql.loadpgsqlfile(
                        fullfilename=fullfilename,
                        options=options,
                        database=db,
                        pgname=array_name[0],
                        pgtype="sql1",
                        pgtext=contents_1)
                elif len(array_name) != 3:
                    print "ERROR: Al cargar un .pgsql Se esperaban 2 o 3 elementos al estilo nombre.view.pgsql o nombre.pgsql y se encontró %s " % name
                    continue
                else:
                    ret = pydba_loadpgsql.loadpgsqlfile(
                        fullfilename=fullfilename,
                        options=options,
                        database=db,
                        pgname=array_name[0],
                        pgtype=array_name[1],
                        pgtext=contents_1)

                if ret != True:
                    print "ERROR: Errores al cargar el fichero ", root, name

            if f_ext(name) == "mtd":
                table = name[:-4]
                # print "### Table: " + table
                tables += [table]
                mtd_files[table] = contents_1
                if preparse:
                    xml = XMLParser(name)
                    xml.parseText(contents_1)
                    if xml.root == None:
                        #print "ERROR: Failed to parse xml %s" %  (name)
                        xml = None
                    else:
                        import pydba_mtdparser

                        mtd = xml.root.tmd

                        mparser = pydba_mtdparser.MTDParser()
                        mparser.parse_mtd(mtd)
                        mtdquery = getattr(mtd, "query", None)
                        if not mtdquery:
                            pydba_mtdparser.Tables[table] = mparser

            if contents and f_ext(name) in filetypes:
                file = {}
                for key, val in pd[fname].iteritems():
                    file[key] = val
                file["contents"] = contents
                if (options.modules_loaded.has_key(name)):
                    print "ERROR: %s file was already loaded." % name
                    print "--> this file was found at %s" % root
                    print "--> previous file found at %s" % options.modules_loaded[
                        name]['root']
                    print "* skipping file"
                else:
                    options.modules_loaded[name] = file
                    files += [file]

    pd.close()
    try:
        os.chmod("/tmp/pydba",
                 S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)
    except:
        pass

    qry_areas = db.query("SELECT descripcion, bloqueo, idarea"
                         " FROM flareas WHERE idarea='%s'" % d_module['area'])
    tareas = qry_areas.dictresult()
    if len(tareas) == 0:
        print "Creando Area %s - %s " % (d_module['area'],
                                         d_module['areaname'])
        db.query("INSERT INTO flareas (descripcion, bloqueo, idarea)"
                 "VALUES('%s','t','%s')" %
                 (d_module['areaname'], d_module['area']))

    habilitar_carga = False
    qry_modulo = db.query(
        "SELECT idmodulo, version, descripcion, bloqueo, idarea"
        " FROM flmodules WHERE idmodulo='%s'" % module)
    tmodulo = qry_modulo.dictresult()
    cargado = False
    for pmodulo in tmodulo:
        cargado = True
        # if pmodulo['bloqueo']=='t':    # TRUE Es que NO está bloqueado. Está al revés.s
        habilitar_carga = True
    if cargado and options.updatemodules:
        cargado = False
        idmodulo = pg.escape_string(d_module['name'])
        sql = ("DELETE FROM flmodules WHERE idmodulo = '%s'" % (idmodulo))
        db.query(sql)

    if not cargado:
        print "Se procede a crear el módulo nuevo %s" % module

        idmodulo = pg.escape_string(d_module['name'])
        idarea = pg.escape_string(d_module['area'])
        version = pg.escape_string(d_module['version'])
        bloqueo = "t"
        descripcion = pg.escape_string(d_module['alias'])
        icono = pg.escape_string(d_module['icon_data'])

        sql = (
            "INSERT INTO flmodules (idmodulo, idarea, version, bloqueo, descripcion,icono) "
            "VALUES('%s','%s','%s','%s','%s','%s')" %
            (idmodulo, idarea, version, bloqueo, descripcion, icono))
        db.query(sql)
        habilitar_carga = True

    if not habilitar_carga:
        print "Error when trying to update the module '%s': non-loaded or locked module" % module
        return 0

    qry_modulos = db.query(
        "SELECT nombre,sha FROM flfiles WHERE idmodulo='%s' " % (module))
    tuplas_modulos = qry_modulos.dictresult()
    dmodulos = {}
    for modulo in tuplas_modulos:
        dmodulos[modulo['nombre']] = modulo

    loaded = []

    for file in files:

        update = True
        if (dmodulos.has_key(file['name'])):
            dm = dmodulos[file['name']]
            if (dm['sha'] == file['sha']):
                update = False

        if options.loadbaselec:
            update = (file['name'] == 'baselec.mtd')

        if (update):
            loaded += [file['name']]
            if (options.verbose):
                print "* Loading file '%s' => '%s'..." % (file['name'],
                                                          file['sha'])
            sql = "DELETE FROM flfiles WHERE nombre='%s';\n" % file['name']
            db.query(sql)

            file['module'] = module

            sql = (
                "INSERT INTO flfiles (contenido, bloqueo, sha, idmodulo, nombre) "
                "VALUES(E'%(contents)s', 't', '%(sha)s','%(module)s', '%(name)s')"
                % file)
            try:
                db.query(sql)
            except Exception, e:
                print e.__class__.__name__, e
                print sql

                raise