Пример #1
0
    def initDB(self):
        '''connect db and create cursor obj'''
        if not isinstance(self._config, SimpleDBConfig):
            return CBasicResult(-1, "config is not instance of SimpleDBConfig buf %s" % type(self._config))

        checkrst = self._config.checkConfig()
        if checkrst._resultcode != 0:
            return checkrst

        try:
            self._connect = MySQLdb.connect(host=self._config._host, port=self._config._port,
                                            user=self._config._user, passwd=self._config._passwd,
                                            connect_timeout=self._config._connect_timeout)

            self._connect.autocommit(True)

            if self._config._dbname != '':
                self._connect.select_db(self._config._dbname)

            # use dictCursor way to get result
            self._cursor = self._connect.cursor(MySQLdb.cursors.DictCursor)

            self._useable = True
            return CBasicResult()

        except MySQLdb.Error, e:
            self._useable = False
            return CBasicResult(-1, "SimpleDB init failed", -1, e)
Пример #2
0
    def SendFileMail(sender,
                     receivers,
                     cc,
                     subject,
                     mailfile,
                     accessoryfile='',
                     htmloption=False,
                     passwd=None,
                     log=None):
        cmd = "sendmail2 '%s' '%s' '%s' '%s' %s " % (sender, receivers, cc,
                                                     subject, mailfile)

        if len(accessoryfile) > 0:
            cmd = cmd + " "
            cmd = cmd + accessoryfile

        if htmloption:
            cmd = cmd + " --html"

        if None != passwd:
            passwdinfo = "--pass=%s" % passwd
            cmd = cmd + passwdinfo

        if None != log:
            log.info(cmd)
        else:
            print cmd

        ret = os.system(cmd)
        if 0 != ret:
            return CBasicResult(-1, "SendMail %s failed" % subject, -1, cmd)

        return CBasicResult()
Пример #3
0
    def getLogInstance(name):
        try:
            if GlobalIns.__log.has_key(
                    name) and GlobalIns.__log[name]['IsInited']:
                return CBasicResult(), GlobalIns.__log[name]['obj']

            rst, cfg = GlobalIns.getCfgInstance()
            if 0 != rst._resultcode or None == cfg:
                errinfo = "getLogInstance: getCfgInstance failed"
                return CBasicResult(-1, errinfo, -1, rst), None

            if not GlobalIns.__log.has_key(name):
                GlobalIns.__log[name] = copy.deepcopy(INSTANCE_TEMP)

            loglevel = cfg.get(name, 'level')
            logpath = "%s.%s.log" % (cfg.get(
                name, 'logname'), datetime.datetime.today().strftime("%Y%m%d"))
            logconfig = simplelog.CLogConfig(level=loglevel, path=logpath)
            GlobalIns.__log[name]['obj'] = simplelog.initLog(logconfig, name)
            if GlobalIns.__log[name]['obj'] == None:
                errinfo = "getLogInstance create log with [%s] failed" % name
                return CBasicResult(-1, errinfo, -1, errinfo), None
            else:
                GlobalIns.__log[name]['IsInited'] = True

            return CBasicResult(), GlobalIns.__log[name]['obj']

        except ConfigParser.Error, e:
            return CBasicResult(-1, "getLogInstance failed", -1, e), None
Пример #4
0
def CommitDBRecord(sdb,
                   dbname,
                   tbname,
                   records_dict,
                   fields_map={},
                   exclude_fields_set=set([]),
                   log=None):
    '''Commit dict records to db, the dict key name must be the same to the db table's field name'''
    fields_array = []
    keys_array = []
    rst = GetFields(sdb, dbname, tbname, fields_array, keys_array)
    if 0 != rst._resultcode:
        return rst

    fields_set = set(fields_array)
    for record in records_dict.values():
        fields_key_array = []
        fields_value_array = []
        for field_key, field_value in record.items():
            # the fields in exclude_fields_set not need to insert to db
            if field_key in exclude_fields_set:
                continue

            # need map record field to table field
            if field_key in fields_map.keys():
                field_key = fields_map[field_key]

            # dict value's key must be in table fields
            if not field_key in fields_set:
                errinfo = "dict value field:%s not in %s.%s table fields" % (
                    field_key, dbname, tbname)
                if not None == log:
                    log.error("CommitDBRecord failed:%s" % errinfo)
                    return CBasicResult(-1, errinfo)

            fields_key_array.append(field_key)
            fields_value_array.append(field_value)

        if len(fields_key_array) == 0:
            continue

        key_sql = MakeSqlKeyFields(fields_key_array)
        value_sql = MakeSqlValueFields(fields_value_array)
        sql = '''insert into %s.%s(%s) values(%s)''' % (dbname, tbname,
                                                        key_sql, value_sql)

        if not None == log:
            log.info("CommitDBRecord commitsql:%s" % sql)
            rst = sdb.execute(sql)
            if 0 != rst._resultcode:
                if not None == log:
                    log.error(
                        "CommitDBRecord db execute failed,sql:%s, err:%s" %
                        (sql, rst))
                return rst

    return CBasicResult()
Пример #5
0
    def setConfigPath(config_fullpath):
        try:
            if os.path.exists(config_fullpath):
                GlobalIns.__config_fullpath = config_fullpath
                return CBasicResult()
            else:
                errinfo = "%s not exists" % GlobalIns.__config_fullpath
                return CBasicResult(-1, errinfo, -1, errinfo)

        except Exception, e:
            return CBasicResult(-1, "setConfigPath failed", -1, e)
Пример #6
0
    def getLogExInstance(name):
        try:
            if GlobalIns.__logex.has_key(
                    name) and GlobalIns.__logex[name]['IsInited']:
                return CBasicResult(), GlobalIns.__logex[name]['obj']

            rst, cfg = GlobalIns.getCfgInstance()
            if 0 != rst._resultcode or None == cfg:
                return CBasicResult(-1, "getCfgInstance failed", -1, rst), None

            if not GlobalIns.__logex.has_key(name):
                GlobalIns.__logex[name] = copy.deepcopy(INSTANCE_TEMP)

            logconfig = {}
            logconfig['loglevel'] = cfg.get(name, 'level')
            logconfig['logpath'] = cfg.get(name, 'logname')
            logconfig['logtype'] = cfg.get(name, 'logtype')

            if cfg.has_option(name, 'formatter'):
                logconfig['formatter'] = cfg.get(name, 'formatter')
            else:
                logconfig['formatter'] = 'simple'

            handlerconfig_array = []

            if logconfig['logtype'].lower() == 'size':
                rst, handlerconfig = GlobalIns._initSizeRotatedHandlerConfig(
                    cfg, name, logconfig)
                if 0 != rst._resultcode:
                    return rst, None

                print handlerconfig
                handlerconfig_array.append(handlerconfig)
            elif logconfig['logtype'].lower() == 'naturetime':
                rst, handlerconfig = GlobalIns._initNatureTimeRotateLog(
                    cfg, name, logconfig)
                if 0 != rst._resultcode:
                    return rst, None

                print handlerconfig
                handlerconfig_array.append(handlerconfig)
            else:
                return CBasicResult(-1, "logtype %s invalid"%logconfig['logtype'], \
                                    -1, "logtype %s invalid"%logconfig['logtype']), None

            GlobalIns.__logex[name]['obj'] = simplelogex.initLog(logmodule = name, \
                                                                 loglevel = logconfig['loglevel'], \
                                                                 handlerConfigs = handlerconfig_array)
            GlobalIns.__logex[name]['IsInited'] = True
            return CBasicResult(), GlobalIns.__logex[name]['obj']

        except Exception, e:
            return CBasicResult(-1, "getLogExInstance failed", -1, e), None
Пример #7
0
    def getLogExInstance(name):
        try:
            if GlobalIns.__logex.has_key(
                    name) and GlobalIns.__logex[name]['IsInited']:
                return CBasicResult(), GlobalIns.__logex[name]['obj']

            cfg = GlobalIns.getCfgInstance()
            if None == cfg:
                return CBasicResult(-1, "getCfgInstance failed", -1,
                                    "getCfgInstance failed")

        except Exception, e:
            printerr(e)
            return CBasicResult(-1, "getLogExInstance failed", -1, e), None
Пример #8
0
 def execute(self, sql, again=False):
     try:
         sdbrst = SimpleDBRst()
         sdbrst._affect_rows = self._cursor.execute(sql)
         sdbrst._rst = self._cursor.fetchall()
         return sdbrst
     except MySQLdb.Error, e:
         if not again and ((not self._useable) or 2013 == e.args[0] or 2006 == e.args[0]):
             rst = self.reInitDB()
             if 0 != rst._resultcode:
                 # need to know the affect sql, so shadow the inner error
                 return CBasicResult(-1, "SimpleDB execute failed [%s]" % sql, rst._errorcode, rst._errorinfo)
             else:
                 return self.execute(sql, True)
         else:
             return CBasicResult(-1, "SimpleDB execute failed [%s]" % sql, -1, e)
Пример #9
0
    def _initSizeRotatedHandlerConfig(cfg, sectionname, logconfig):
        if cfg.has_option(sectionname, 'rotatesize'):
            logconfig['rotatesize'] = cfg.getint(sectionname,
                                                 'rotatesize') * 1024 * 1024
        else:
            logconfig['rotatesize'] = 4 * 1024 * 1024 * 1024

        if cfg.has_option(sectionname, 'openmode'):
            logconfig['openmode'] = cfg.get(sectionname, 'openmode')
        else:
            logconfig['openmode'] = 'a'

        if cfg.has_option(sectionname, 'encoding'):
            #logconfig['encoding'] = cfg.get(sectionname, 'encoding')
            logconfig['encoding'] = None
        else:
            logconfig['encoding'] = None

        if cfg.has_option(sectionname, 'backupcount'):
            logconfig['backupcount'] = cfg.getint(sectionname, 'backupcount')
        else:
            logconfig['backupcount'] = 10000

        handlerconfig = simplelogex.CRotatingFileHandlerConfig(level = logconfig['loglevel'], formatter = logconfig['formatter'],\
filename = "%s.log.%s"%(logconfig['logpath'], datetime.datetime.today().strftime("%Y-%m-%d")), \
maxBytes = logconfig['rotatesize'], mode = logconfig['openmode'], \
                        encoding = logconfig['encoding'], backupCount = logconfig['backupcount'])

        return CBasicResult(), handlerconfig
Пример #10
0
def GetDBRecord(sdb, dbname, tbname, records_dict, keys_array=[], exclude_fields_set=set([]), \
                where_condition="", keysep="-", exclude_keyfield_set=set([]), log=None):
    '''GetDBRecord is suitable for complex db field extract'''
    fields_array = []
    keys_array = []
    rst = GetFields(sdb, dbname, tbname, fields_array, keys_array,
                    exclude_fields_set, exclude_keyfield_set)
    if 0 != rst._resultcode:
        return rst

    fields_str = MakeSqlKeyFields(fields_array)

    sql = "select %s from %s.%s %s;" % (fields_str, dbname, tbname,
                                        where_condition)

    if not log == None:
        log.info("GetDBRecord sql:%s" % sql)

    rst = sdb.execute(sql)
    if 0 != rst._resultcode:
        return rst

    for l in rst._rst:
        key = MakeKey(l, keys_array, keysep)
        records_dict[key] = l

    return CBasicResult()
Пример #11
0
 def __del__(self):
     try:
         if hasattr(self, '_cursor'):
             self._cursor.close()
         if hasattr(self, '_connect'):
             self._connect.close()
     except MySQLdb.Error, e:
         print CBasicResult(-1, "SimpleDB __del__ failed %s" % e)
Пример #12
0
 def addImportedModule(modulename):
     filename = "%s.py" % modulename.replace(".", "/")
     if not os.path.exists(filename):
         return CBasicResult(-1, "%s not exist" % filename, -1,
                             "%s not exist" % filename), None
     try:
         statinfo = os.stat(filename)
         m = imp.load_source(modulename, filename)
         minfo = copy.deepcopy(TMP_MODULE_INFO)
         minfo['name'] = modulename
         minfo['module'] = m
         minfo['valid'] = True
         minfo['fstat'] = statinfo
         SingleDImport._dynimport_dict[minfo['name']] = minfo
         return CBasicResult(), m
     except Exception, e:
         return CBasicResult(-1, "addImportedModule failed", -1, e), None
Пример #13
0
 def getModule(name):
     if SingleDImport.isImported(
             name) and not SingleDImport.isModified(name):
         # print "use inited"
         return CBasicResult(
         ), SingleDImport._dynimport_dict[name]['module']
     else:
         # print "use new"
         return SingleDImport.addImportedModule(name)
Пример #14
0
    def getCfgInstance(needreparse=False):
        try:
            if GlobalIns.__cfg['IsInited'] and not needreparse:
                return CBasicResult(), GlobalIns.__cfg['obj']

            if not os.path.exists(GlobalIns.__config_fullpath):
                errinfo = "%s not exists" % GlobalIns.__config_fullpath
                return CBasicResult(-1, errinfo, -1, errinfo), None

            GlobalIns.__cfg['obj'] = ConfigParser.ConfigParser()
            l = GlobalIns.__cfg['obj'].read(GlobalIns.__config_fullpath)
            if len(l) == 0:
                return CBasicResult(-1, 'read config [%s] failed'%GlobalIns.__config_fullpath,\
                                    -1, 'read config [%s] failed'%GlobalIns.__config_fullpath), None

            return CBasicResult(), GlobalIns.__cfg['obj']

        except ConfigParser.Error, e:
            return CBasicResult(-1, "getCfgInstance failed", -1, e), None
Пример #15
0
    def getSDBInstance(name):
        try:
            if GlobalIns.__sdb.has_key(
                    name) and GlobalIns.__sdb[name]['IsInited']:
                return GlobalIns.__sdb[name]['obj']

            rst, cfg = GlobalIns.getCfgInstance()
            if 0 != rst._resultcode or None == cfg:
                return CBasicResult(-1, "getSDBInstance failed", -1,
                                    "getCfgInstance None"), None

            if not GlobalIns.__sdb.has_key(name):
                GlobalIns.__sdb[name] = copy.deepcopy(INSTANCE_TEMP)
            db_host = cfg.get(name, 'ip')
            db_port = int(cfg.get(name, 'port'))
            db_user = cfg.get(name, 'user')
            db_passwd = cfg.get(name, 'passwd')
            db_timeout = 5
            if cfg.has_option(name, 'timeout'):
                db_timeout = int(cfg.get(name, 'timeout'))
            db_dbname = ''
            if cfg.has_option(name, 'dbname'):
                db_dbname = cfg.get(name, 'dbname')

            db_tbname = ''
            if cfg.has_option(name, 'tablename'):
                db_tbname = cfg.get(name, 'tablename')

            sdbconfig = simpledb.SimpleDBConfig(db_host, db_port, db_user,
                                                db_passwd, db_timeout,
                                                db_dbname, db_tbname)
            GlobalIns.__sdb[name]['obj'] = simpledb.SimpleDB(sdbconfig)

            rst = GlobalIns.__sdb[name]['obj'].initDB()
            if 0 != rst._resultcode:
                return CBasicResult(-1, "getSDBInstance failed", -1, rst), None

            GlobalIns.__sdb[name]['IsInited'] = True
            return CBasicResult(), GlobalIns.__sdb[name]['obj']
        except ConfigParser.Error, e:
            return CBasicResult(-1, "getSDBInstance failed", -1, e), None
Пример #16
0
    def reInitDB(self):
        '''some time the connection is flash crash, so we need to rebuild connection,
        this function first close old connect then build a new one'''
        try:
            if self._useable:
                self._cursor.close()
                self._connect.close()

            return self.initDB()

        except MySQLdb.Error, e:
            return CBasicResult(-1, "SimpleDB reInitDB failed", -1, e)
Пример #17
0
    def Rsync_Pull(ip,
                   port,
                   username,
                   src_path,
                   src_name,
                   dst_path,
                   dst_name,
                   force=False,
                   log=None):
        srcfull = "%s/%s" % (src_path, src_name)
        dstfull = "%s/%s" % (dst_path, dst_name)
        if os.path.exists(dstfull) and not force:
            return CBasicResult(), dstfull

        if os.path.exists(dstfull):
            os.remove(dstfull)

        cmd = "rsync --port=%s %s@%s::%s %s" % (port, username, ip, srcfull,
                                                dst_path)

        if None != log:
            log.info(cmd)

        print cmd

        ret = os.system(cmd)
        if 0 != ret:
            return CBasicResult(-1, "%s failed" % cmd, -1,
                                "%s failed" % cmd), dstfull

        old_name = "%s/%s" % (dst_path, src_name)
        if not os.path.exists(old_name):
            return CBasicResult(-1, "%s failed" % cmd, -1,
                                "%s failed" % cmd), dstfull

        if src_name != dst_name:
            os.rename(old_name, dstfull)

        return CBasicResult(), dstfull
Пример #18
0
def GetFields(sdb,
              dbname,
              tbname,
              fields_array,
              keys_array,
              exclude_fields_set=set([]),
              exclude_keyfield_set=set([])):
    sql = "desc %s.%s" % (dbname, tbname)
    rst = sdb.execute(sql)
    if 0 != rst._resultcode:
        return rst

    for l in rst._rst:
        if l['Key'] == 'PRI':
            if l['Field'] in exclude_fields_set:
                return CBasicResult(
                    -1,
                    "GetFields key field:%s shold not be in exclude_fields_set"
                    % l['Field'])
            if l['Field'] in exclude_keyfield_set:  # filter key
                if not l['Field'] in exclude_fields_set:
                    fields_array.append(l['Field'])
                continue
            keys_array.append(l['Field'])

        # exclude field should not in fields_array
        if l['Field'] in exclude_fields_set:
            continue
        fields_array.append(l['Field'])

    if len(fields_array) == 0:
        return CBasicResult(-1, "GetFields fields_array null")
    if len(keys_array) == 0:
        return CBasicResult(-1, "GetFields keys_array null")

    return CBasicResult()
Пример #19
0
    def SendCfgFileMail(cfg_sec_name, mailfile, accessoryfile='', htmloption=False, \
                        passwd=None, log=None, refresh=False, subject_suffix=''):
        # use cfg_sec_name(config section name) as key, before use this method you need to use globalins_ex to init
        if not CSendMail._mailinfo.has_key(cfg_sec_name):
            rst, cfg = globalins_ex.GlobalIns.getCfgInstance(refresh)
            if 0 != rst._resultcode:
                return rst

            try:
                mail_cfg = copy.deepcopy(MAIL_CFG_TEMPLATE)
                mail_cfg['sender'] = cfg.get(cfg_sec_name, 'sender')
                mail_cfg['receivers'] = cfg.get(cfg_sec_name, 'receivers')
                mail_cfg['cc'] = cfg.get(cfg_sec_name, 'cc')
                subject = cfg.get(cfg_sec_name, 'subject') + subject_suffix
                mail_cfg['subject'] = subject
                CSendMail._mailinfo[cfg_sec_name] = mail_cfg

            except ConfigParser.NoSectionError, e:
                return CBasicResult(-1, e, -1, e)
Пример #20
0
def test():
    rst = GlobalIns.setConfigPath("globalins_demo_config.ini")
    if 0 != rst._resultcode:
        return rst
    
    rst, logex = GlobalIns.getLogExInstance('logex_demo2')
    if 0 != rst._resultcode:
        return rst

    #test log rotate by size
    #for i in range(100):
    #	logex.debug("你好")

    #test log rotate by nature time
    while True:
        logex.info('你好')
        sleep(1)
        
        return CBasicResult()
Пример #21
0
def GetSqlRecords(sdb,
                  sql,
                  keys_mutiarray,
                  records_dict,
                  keysep="-",
                  log=None):
    '''get sql records and add record to dict, this can have multi level dicts(each level have the same struct), 
    such as {key1:{sub_key1:value1, sub_key2:value2}, key2:value2}
    keys_mutiarray : [[field1, field2], [field2,]]'''

    rst = sdb.execute(sql)
    if 0 != rst._resultcode:
        return rst
    for line in rst._rst:
        keys_array = []
        for fields in keys_mutiarray:
            keys_array.append(MakeKey(line, fields, keysep))

        SetMultiDict(keys_array, line, records_dict)
    return CBasicResult()
Пример #22
0
    def CfgRsync_Pull(sec_name, src_name, dst_name, force=False, log=None):
        # you need first init GlobalIns
        rst, cfg = GlobalIns.getCfgInstance()

        if 0 != rst._resultcode:
            return rst, None

        if not CRsync._rsync_cfg.has_key(sec_name):
            rsync_cfg = {}
            try:
                rsync_cfg['ip'] = cfg.get(sec_name, 'ip')
                rsync_cfg['port'] = cfg.get(sec_name, 'port')
                rsync_cfg['user'] = cfg.get(sec_name, 'user')
                rsync_cfg['src_path'] = cfg.get(sec_name, 'src_path')
                rsync_cfg['dst_path'] = cfg.get(sec_name, 'dst_path')

                CRsync._rsync_cfg[sec_name] = rsync_cfg

            except ConfigParser.NoSectionError, e:
                return CBasicResult(-1, e, -1, e), None
Пример #23
0
    def _initNatureTimeRotateLog(cfg, sectionname, logconfig):
        #handlerconfig = []

        when_map = {'day': 'D', 'hour': 'H', 'minute': 'M', 'second': 'S'}

        if cfg.has_option(sectionname, 'when'):
            when = cfg.get(sectionname, 'when')
            if when not in when_map.keys():
                logconfig['when'] = when_map['day']
            else:
                logconfig['when'] = when_map[when]
        else:
            logconfig['when'] = when_map['day']

        if cfg.has_option(sectionname, 'interval'):
            logconfig['interval'] = cfg.getint(sectionname, 'interval')
        else:
            logconfig['interval'] = 1

        if cfg.has_option(sectionname, 'openmode'):
            logconfig['openmode'] = cfg.get(sectionname, 'openmode')
        else:
            logconfig['openmode'] = 'a'

        if cfg.has_option(sectionname, 'encoding'):
            #logconfig['encoding'] = cfg.get(sectionname, 'encoding')
            logconfig['encoding'] = None
        else:
            logconfig['encoding'] = None

        if cfg.has_option(sectionname, 'backupcount'):
            logconfig['backupcount'] = cfg.getint(sectionname, 'backupcount')
        else:
            logconfig['backupcount'] = 10000

        handlerconfig = simplelogex.CNatureTimedRotatingFileHandlerConfig(level = logconfig['loglevel'], \
formatter = logconfig['formatter'], filename = logconfig['logpath'], when = logconfig['when'], interval = logconfig['interval'],\
mode = logconfig['openmode'], backupCount = logconfig['backupcount'], encoding=logconfig['encoding'])

        return CBasicResult(), handlerconfig
Пример #24
0
            if not os.path.exists(GlobalIns.__config_fullpath):
                errinfo = "%s not exists" % GlobalIns.__config_fullpath
                return CBasicResult(-1, errinfo, -1, errinfo), None

            GlobalIns.__cfg['obj'] = ConfigParser.ConfigParser()
            l = GlobalIns.__cfg['obj'].read(GlobalIns.__config_fullpath)
            if len(l) == 0:
                return CBasicResult(-1, 'read config [%s] failed'%GlobalIns.__config_fullpath,\
                                    -1, 'read config [%s] failed'%GlobalIns.__config_fullpath), None

            return CBasicResult(), GlobalIns.__cfg['obj']

        except ConfigParser.Error, e:
            return CBasicResult(-1, "getCfgInstance failed", -1, e), None
        except Exception, e:
            return CBasicResult(-1, "getCfgInstance failed", -1, e), None

    @staticmethod
    def getLogInstance(name):
        try:
            if GlobalIns.__log.has_key(
                    name) and GlobalIns.__log[name]['IsInited']:
                return CBasicResult(), GlobalIns.__log[name]['obj']

            rst, cfg = GlobalIns.getCfgInstance()
            if 0 != rst._resultcode or None == cfg:
                errinfo = "getLogInstance: getCfgInstance failed"
                return CBasicResult(-1, errinfo, -1, rst), None

            if not GlobalIns.__log.has_key(name):
                GlobalIns.__log[name] = copy.deepcopy(INSTANCE_TEMP)
Пример #25
0
    def checkConfig(self):
        if self._host == '' or self._port <= 0 or self._user == '' or self._connect_timeout <= 0:
            return CBasicResult(-1, "config not valid [%s]" % self.__str__())

        return CBasicResult()