def initDB(self):
'''connect db and create cursor obj'''
if not isinstance(self._config, SimpleDBConfig):
return CBasicResult(-1, "config is not instance of SimpleDBConfig buf %s" % type(self._config))
checkrst = self._config.checkConfig()
if checkrst._resultcode != 0:
return checkrst
try:
self._connect = MySQLdb.connect(host=self._config._host, port=self._config._port,
user=self._config._user, passwd=self._config._passwd,
connect_timeout=self._config._connect_timeout)
self._connect.autocommit(True)
if self._config._dbname != '':
self._connect.select_db(self._config._dbname)
# use dictCursor way to get result
self._cursor = self._connect.cursor(MySQLdb.cursors.DictCursor)
self._useable = True
return CBasicResult()
except MySQLdb.Error, e:
self._useable = False
return CBasicResult(-1, "SimpleDB init failed", -1, e)
def SendFileMail(sender,
receivers,
cc,
subject,
mailfile,
accessoryfile='',
htmloption=False,
passwd=None,
log=None):
cmd = "sendmail2 '%s' '%s' '%s' '%s' %s " % (sender, receivers, cc,
subject, mailfile)
if len(accessoryfile) > 0:
cmd = cmd + " "
cmd = cmd + accessoryfile
if htmloption:
cmd = cmd + " --html"
if None != passwd:
passwdinfo = "--pass=%s" % passwd
cmd = cmd + passwdinfo
if None != log:
log.info(cmd)
else:
print cmd
ret = os.system(cmd)
if 0 != ret:
return CBasicResult(-1, "SendMail %s failed" % subject, -1, cmd)
return CBasicResult()
def getLogInstance(name):
try:
if GlobalIns.__log.has_key(
name) and GlobalIns.__log[name]['IsInited']:
return CBasicResult(), GlobalIns.__log[name]['obj']
rst, cfg = GlobalIns.getCfgInstance()
if 0 != rst._resultcode or None == cfg:
errinfo = "getLogInstance: getCfgInstance failed"
return CBasicResult(-1, errinfo, -1, rst), None
if not GlobalIns.__log.has_key(name):
GlobalIns.__log[name] = copy.deepcopy(INSTANCE_TEMP)
loglevel = cfg.get(name, 'level')
logpath = "%s.%s.log" % (cfg.get(
name, 'logname'), datetime.datetime.today().strftime("%Y%m%d"))
logconfig = simplelog.CLogConfig(level=loglevel, path=logpath)
GlobalIns.__log[name]['obj'] = simplelog.initLog(logconfig, name)
if GlobalIns.__log[name]['obj'] == None:
errinfo = "getLogInstance create log with [%s] failed" % name
return CBasicResult(-1, errinfo, -1, errinfo), None
else:
GlobalIns.__log[name]['IsInited'] = True
return CBasicResult(), GlobalIns.__log[name]['obj']
except ConfigParser.Error, e:
return CBasicResult(-1, "getLogInstance failed", -1, e), None
def CommitDBRecord(sdb,
dbname,
tbname,
records_dict,
fields_map={},
exclude_fields_set=set([]),
log=None):
'''Commit dict records to db, the dict key name must be the same to the db table's field name'''
fields_array = []
keys_array = []
rst = GetFields(sdb, dbname, tbname, fields_array, keys_array)
if 0 != rst._resultcode:
return rst
fields_set = set(fields_array)
for record in records_dict.values():
fields_key_array = []
fields_value_array = []
for field_key, field_value in record.items():
# the fields in exclude_fields_set not need to insert to db
if field_key in exclude_fields_set:
continue
# need map record field to table field
if field_key in fields_map.keys():
field_key = fields_map[field_key]
# dict value's key must be in table fields
if not field_key in fields_set:
errinfo = "dict value field:%s not in %s.%s table fields" % (
field_key, dbname, tbname)
if not None == log:
log.error("CommitDBRecord failed:%s" % errinfo)
return CBasicResult(-1, errinfo)
fields_key_array.append(field_key)
fields_value_array.append(field_value)
if len(fields_key_array) == 0:
continue
key_sql = MakeSqlKeyFields(fields_key_array)
value_sql = MakeSqlValueFields(fields_value_array)
sql = '''insert into %s.%s(%s) values(%s)''' % (dbname, tbname,
key_sql, value_sql)
if not None == log:
log.info("CommitDBRecord commitsql:%s" % sql)
rst = sdb.execute(sql)
if 0 != rst._resultcode:
if not None == log:
log.error(
"CommitDBRecord db execute failed,sql:%s, err:%s" %
(sql, rst))
return rst
return CBasicResult()
def setConfigPath(config_fullpath):
try:
if os.path.exists(config_fullpath):
GlobalIns.__config_fullpath = config_fullpath
return CBasicResult()
else:
errinfo = "%s not exists" % GlobalIns.__config_fullpath
return CBasicResult(-1, errinfo, -1, errinfo)
except Exception, e:
return CBasicResult(-1, "setConfigPath failed", -1, e)
def getLogExInstance(name):
try:
if GlobalIns.__logex.has_key(
name) and GlobalIns.__logex[name]['IsInited']:
return CBasicResult(), GlobalIns.__logex[name]['obj']
rst, cfg = GlobalIns.getCfgInstance()
if 0 != rst._resultcode or None == cfg:
return CBasicResult(-1, "getCfgInstance failed", -1, rst), None
if not GlobalIns.__logex.has_key(name):
GlobalIns.__logex[name] = copy.deepcopy(INSTANCE_TEMP)
logconfig = {}
logconfig['loglevel'] = cfg.get(name, 'level')
logconfig['logpath'] = cfg.get(name, 'logname')
logconfig['logtype'] = cfg.get(name, 'logtype')
if cfg.has_option(name, 'formatter'):
logconfig['formatter'] = cfg.get(name, 'formatter')
else:
logconfig['formatter'] = 'simple'
handlerconfig_array = []
if logconfig['logtype'].lower() == 'size':
rst, handlerconfig = GlobalIns._initSizeRotatedHandlerConfig(
cfg, name, logconfig)
if 0 != rst._resultcode:
return rst, None
print handlerconfig
handlerconfig_array.append(handlerconfig)
elif logconfig['logtype'].lower() == 'naturetime':
rst, handlerconfig = GlobalIns._initNatureTimeRotateLog(
cfg, name, logconfig)
if 0 != rst._resultcode:
return rst, None
print handlerconfig
handlerconfig_array.append(handlerconfig)
else:
return CBasicResult(-1, "logtype %s invalid"%logconfig['logtype'], \
-1, "logtype %s invalid"%logconfig['logtype']), None
GlobalIns.__logex[name]['obj'] = simplelogex.initLog(logmodule = name, \
loglevel = logconfig['loglevel'], \
handlerConfigs = handlerconfig_array)
GlobalIns.__logex[name]['IsInited'] = True
return CBasicResult(), GlobalIns.__logex[name]['obj']
except Exception, e:
return CBasicResult(-1, "getLogExInstance failed", -1, e), None
def getLogExInstance(name):
try:
if GlobalIns.__logex.has_key(
name) and GlobalIns.__logex[name]['IsInited']:
return CBasicResult(), GlobalIns.__logex[name]['obj']
cfg = GlobalIns.getCfgInstance()
if None == cfg:
return CBasicResult(-1, "getCfgInstance failed", -1,
"getCfgInstance failed")
except Exception, e:
printerr(e)
return CBasicResult(-1, "getLogExInstance failed", -1, e), None
def execute(self, sql, again=False):
try:
sdbrst = SimpleDBRst()
sdbrst._affect_rows = self._cursor.execute(sql)
sdbrst._rst = self._cursor.fetchall()
return sdbrst
except MySQLdb.Error, e:
if not again and ((not self._useable) or 2013 == e.args[0] or 2006 == e.args[0]):
rst = self.reInitDB()
if 0 != rst._resultcode:
# need to know the affect sql, so shadow the inner error
return CBasicResult(-1, "SimpleDB execute failed [%s]" % sql, rst._errorcode, rst._errorinfo)
else:
return self.execute(sql, True)
else:
return CBasicResult(-1, "SimpleDB execute failed [%s]" % sql, -1, e)
def _initSizeRotatedHandlerConfig(cfg, sectionname, logconfig):
if cfg.has_option(sectionname, 'rotatesize'):
logconfig['rotatesize'] = cfg.getint(sectionname,
'rotatesize') * 1024 * 1024
else:
logconfig['rotatesize'] = 4 * 1024 * 1024 * 1024
if cfg.has_option(sectionname, 'openmode'):
logconfig['openmode'] = cfg.get(sectionname, 'openmode')
else:
logconfig['openmode'] = 'a'
if cfg.has_option(sectionname, 'encoding'):
#logconfig['encoding'] = cfg.get(sectionname, 'encoding')
logconfig['encoding'] = None
else:
logconfig['encoding'] = None
if cfg.has_option(sectionname, 'backupcount'):
logconfig['backupcount'] = cfg.getint(sectionname, 'backupcount')
else:
logconfig['backupcount'] = 10000
handlerconfig = simplelogex.CRotatingFileHandlerConfig(level = logconfig['loglevel'], formatter = logconfig['formatter'],\
filename = "%s.log.%s"%(logconfig['logpath'], datetime.datetime.today().strftime("%Y-%m-%d")), \
maxBytes = logconfig['rotatesize'], mode = logconfig['openmode'], \
encoding = logconfig['encoding'], backupCount = logconfig['backupcount'])
return CBasicResult(), handlerconfig
def GetDBRecord(sdb, dbname, tbname, records_dict, keys_array=[], exclude_fields_set=set([]), \
where_condition="", keysep="-", exclude_keyfield_set=set([]), log=None):
'''GetDBRecord is suitable for complex db field extract'''
fields_array = []
keys_array = []
rst = GetFields(sdb, dbname, tbname, fields_array, keys_array,
exclude_fields_set, exclude_keyfield_set)
if 0 != rst._resultcode:
return rst
fields_str = MakeSqlKeyFields(fields_array)
sql = "select %s from %s.%s %s;" % (fields_str, dbname, tbname,
where_condition)
if not log == None:
log.info("GetDBRecord sql:%s" % sql)
rst = sdb.execute(sql)
if 0 != rst._resultcode:
return rst
for l in rst._rst:
key = MakeKey(l, keys_array, keysep)
records_dict[key] = l
return CBasicResult()
def __del__(self):
try:
if hasattr(self, '_cursor'):
self._cursor.close()
if hasattr(self, '_connect'):
self._connect.close()
except MySQLdb.Error, e:
print CBasicResult(-1, "SimpleDB __del__ failed %s" % e)
def addImportedModule(modulename):
filename = "%s.py" % modulename.replace(".", "/")
if not os.path.exists(filename):
return CBasicResult(-1, "%s not exist" % filename, -1,
"%s not exist" % filename), None
try:
statinfo = os.stat(filename)
m = imp.load_source(modulename, filename)
minfo = copy.deepcopy(TMP_MODULE_INFO)
minfo['name'] = modulename
minfo['module'] = m
minfo['valid'] = True
minfo['fstat'] = statinfo
SingleDImport._dynimport_dict[minfo['name']] = minfo
return CBasicResult(), m
except Exception, e:
return CBasicResult(-1, "addImportedModule failed", -1, e), None
def getModule(name):
if SingleDImport.isImported(
name) and not SingleDImport.isModified(name):
# print "use inited"
return CBasicResult(
), SingleDImport._dynimport_dict[name]['module']
else:
# print "use new"
return SingleDImport.addImportedModule(name)
def getCfgInstance(needreparse=False):
try:
if GlobalIns.__cfg['IsInited'] and not needreparse:
return CBasicResult(), GlobalIns.__cfg['obj']
if not os.path.exists(GlobalIns.__config_fullpath):
errinfo = "%s not exists" % GlobalIns.__config_fullpath
return CBasicResult(-1, errinfo, -1, errinfo), None
GlobalIns.__cfg['obj'] = ConfigParser.ConfigParser()
l = GlobalIns.__cfg['obj'].read(GlobalIns.__config_fullpath)
if len(l) == 0:
return CBasicResult(-1, 'read config [%s] failed'%GlobalIns.__config_fullpath,\
-1, 'read config [%s] failed'%GlobalIns.__config_fullpath), None
return CBasicResult(), GlobalIns.__cfg['obj']
except ConfigParser.Error, e:
return CBasicResult(-1, "getCfgInstance failed", -1, e), None
def getSDBInstance(name):
try:
if GlobalIns.__sdb.has_key(
name) and GlobalIns.__sdb[name]['IsInited']:
return GlobalIns.__sdb[name]['obj']
rst, cfg = GlobalIns.getCfgInstance()
if 0 != rst._resultcode or None == cfg:
return CBasicResult(-1, "getSDBInstance failed", -1,
"getCfgInstance None"), None
if not GlobalIns.__sdb.has_key(name):
GlobalIns.__sdb[name] = copy.deepcopy(INSTANCE_TEMP)
db_host = cfg.get(name, 'ip')
db_port = int(cfg.get(name, 'port'))
db_user = cfg.get(name, 'user')
db_passwd = cfg.get(name, 'passwd')
db_timeout = 5
if cfg.has_option(name, 'timeout'):
db_timeout = int(cfg.get(name, 'timeout'))
db_dbname = ''
if cfg.has_option(name, 'dbname'):
db_dbname = cfg.get(name, 'dbname')
db_tbname = ''
if cfg.has_option(name, 'tablename'):
db_tbname = cfg.get(name, 'tablename')
sdbconfig = simpledb.SimpleDBConfig(db_host, db_port, db_user,
db_passwd, db_timeout,
db_dbname, db_tbname)
GlobalIns.__sdb[name]['obj'] = simpledb.SimpleDB(sdbconfig)
rst = GlobalIns.__sdb[name]['obj'].initDB()
if 0 != rst._resultcode:
return CBasicResult(-1, "getSDBInstance failed", -1, rst), None
GlobalIns.__sdb[name]['IsInited'] = True
return CBasicResult(), GlobalIns.__sdb[name]['obj']
except ConfigParser.Error, e:
return CBasicResult(-1, "getSDBInstance failed", -1, e), None
def reInitDB(self):
'''some time the connection is flash crash, so we need to rebuild connection,
this function first close old connect then build a new one'''
try:
if self._useable:
self._cursor.close()
self._connect.close()
return self.initDB()
except MySQLdb.Error, e:
return CBasicResult(-1, "SimpleDB reInitDB failed", -1, e)
def Rsync_Pull(ip,
port,
username,
src_path,
src_name,
dst_path,
dst_name,
force=False,
log=None):
srcfull = "%s/%s" % (src_path, src_name)
dstfull = "%s/%s" % (dst_path, dst_name)
if os.path.exists(dstfull) and not force:
return CBasicResult(), dstfull
if os.path.exists(dstfull):
os.remove(dstfull)
cmd = "rsync --port=%s %s@%s::%s %s" % (port, username, ip, srcfull,
dst_path)
if None != log:
log.info(cmd)
print cmd
ret = os.system(cmd)
if 0 != ret:
return CBasicResult(-1, "%s failed" % cmd, -1,
"%s failed" % cmd), dstfull
old_name = "%s/%s" % (dst_path, src_name)
if not os.path.exists(old_name):
return CBasicResult(-1, "%s failed" % cmd, -1,
"%s failed" % cmd), dstfull
if src_name != dst_name:
os.rename(old_name, dstfull)
return CBasicResult(), dstfull
def GetFields(sdb,
dbname,
tbname,
fields_array,
keys_array,
exclude_fields_set=set([]),
exclude_keyfield_set=set([])):
sql = "desc %s.%s" % (dbname, tbname)
rst = sdb.execute(sql)
if 0 != rst._resultcode:
return rst
for l in rst._rst:
if l['Key'] == 'PRI':
if l['Field'] in exclude_fields_set:
return CBasicResult(
-1,
"GetFields key field:%s shold not be in exclude_fields_set"
% l['Field'])
if l['Field'] in exclude_keyfield_set: # filter key
if not l['Field'] in exclude_fields_set:
fields_array.append(l['Field'])
continue
keys_array.append(l['Field'])
# exclude field should not in fields_array
if l['Field'] in exclude_fields_set:
continue
fields_array.append(l['Field'])
if len(fields_array) == 0:
return CBasicResult(-1, "GetFields fields_array null")
if len(keys_array) == 0:
return CBasicResult(-1, "GetFields keys_array null")
return CBasicResult()
def SendCfgFileMail(cfg_sec_name, mailfile, accessoryfile='', htmloption=False, \
passwd=None, log=None, refresh=False, subject_suffix=''):
# use cfg_sec_name(config section name) as key, before use this method you need to use globalins_ex to init
if not CSendMail._mailinfo.has_key(cfg_sec_name):
rst, cfg = globalins_ex.GlobalIns.getCfgInstance(refresh)
if 0 != rst._resultcode:
return rst
try:
mail_cfg = copy.deepcopy(MAIL_CFG_TEMPLATE)
mail_cfg['sender'] = cfg.get(cfg_sec_name, 'sender')
mail_cfg['receivers'] = cfg.get(cfg_sec_name, 'receivers')
mail_cfg['cc'] = cfg.get(cfg_sec_name, 'cc')
subject = cfg.get(cfg_sec_name, 'subject') + subject_suffix
mail_cfg['subject'] = subject
CSendMail._mailinfo[cfg_sec_name] = mail_cfg
except ConfigParser.NoSectionError, e:
return CBasicResult(-1, e, -1, e)
def test():
rst = GlobalIns.setConfigPath("globalins_demo_config.ini")
if 0 != rst._resultcode:
return rst
rst, logex = GlobalIns.getLogExInstance('logex_demo2')
if 0 != rst._resultcode:
return rst
#test log rotate by size
#for i in range(100):
# logex.debug("你好")
#test log rotate by nature time
while True:
logex.info('你好')
sleep(1)
return CBasicResult()
def GetSqlRecords(sdb,
sql,
keys_mutiarray,
records_dict,
keysep="-",
log=None):
'''get sql records and add record to dict, this can have multi level dicts(each level have the same struct),
such as {key1:{sub_key1:value1, sub_key2:value2}, key2:value2}
keys_mutiarray : [[field1, field2], [field2,]]'''
rst = sdb.execute(sql)
if 0 != rst._resultcode:
return rst
for line in rst._rst:
keys_array = []
for fields in keys_mutiarray:
keys_array.append(MakeKey(line, fields, keysep))
SetMultiDict(keys_array, line, records_dict)
return CBasicResult()
def CfgRsync_Pull(sec_name, src_name, dst_name, force=False, log=None):
# you need first init GlobalIns
rst, cfg = GlobalIns.getCfgInstance()
if 0 != rst._resultcode:
return rst, None
if not CRsync._rsync_cfg.has_key(sec_name):
rsync_cfg = {}
try:
rsync_cfg['ip'] = cfg.get(sec_name, 'ip')
rsync_cfg['port'] = cfg.get(sec_name, 'port')
rsync_cfg['user'] = cfg.get(sec_name, 'user')
rsync_cfg['src_path'] = cfg.get(sec_name, 'src_path')
rsync_cfg['dst_path'] = cfg.get(sec_name, 'dst_path')
CRsync._rsync_cfg[sec_name] = rsync_cfg
except ConfigParser.NoSectionError, e:
return CBasicResult(-1, e, -1, e), None
def _initNatureTimeRotateLog(cfg, sectionname, logconfig):
#handlerconfig = []
when_map = {'day': 'D', 'hour': 'H', 'minute': 'M', 'second': 'S'}
if cfg.has_option(sectionname, 'when'):
when = cfg.get(sectionname, 'when')
if when not in when_map.keys():
logconfig['when'] = when_map['day']
else:
logconfig['when'] = when_map[when]
else:
logconfig['when'] = when_map['day']
if cfg.has_option(sectionname, 'interval'):
logconfig['interval'] = cfg.getint(sectionname, 'interval')
else:
logconfig['interval'] = 1
if cfg.has_option(sectionname, 'openmode'):
logconfig['openmode'] = cfg.get(sectionname, 'openmode')
else:
logconfig['openmode'] = 'a'
if cfg.has_option(sectionname, 'encoding'):
#logconfig['encoding'] = cfg.get(sectionname, 'encoding')
logconfig['encoding'] = None
else:
logconfig['encoding'] = None
if cfg.has_option(sectionname, 'backupcount'):
logconfig['backupcount'] = cfg.getint(sectionname, 'backupcount')
else:
logconfig['backupcount'] = 10000
handlerconfig = simplelogex.CNatureTimedRotatingFileHandlerConfig(level = logconfig['loglevel'], \
formatter = logconfig['formatter'], filename = logconfig['logpath'], when = logconfig['when'], interval = logconfig['interval'],\
mode = logconfig['openmode'], backupCount = logconfig['backupcount'], encoding=logconfig['encoding'])
return CBasicResult(), handlerconfig
if not os.path.exists(GlobalIns.__config_fullpath):
errinfo = "%s not exists" % GlobalIns.__config_fullpath
return CBasicResult(-1, errinfo, -1, errinfo), None
GlobalIns.__cfg['obj'] = ConfigParser.ConfigParser()
l = GlobalIns.__cfg['obj'].read(GlobalIns.__config_fullpath)
if len(l) == 0:
return CBasicResult(-1, 'read config [%s] failed'%GlobalIns.__config_fullpath,\
-1, 'read config [%s] failed'%GlobalIns.__config_fullpath), None
return CBasicResult(), GlobalIns.__cfg['obj']
except ConfigParser.Error, e:
return CBasicResult(-1, "getCfgInstance failed", -1, e), None
except Exception, e:
return CBasicResult(-1, "getCfgInstance failed", -1, e), None
@staticmethod
def getLogInstance(name):
try:
if GlobalIns.__log.has_key(
name) and GlobalIns.__log[name]['IsInited']:
return CBasicResult(), GlobalIns.__log[name]['obj']
rst, cfg = GlobalIns.getCfgInstance()
if 0 != rst._resultcode or None == cfg:
errinfo = "getLogInstance: getCfgInstance failed"
return CBasicResult(-1, errinfo, -1, rst), None
if not GlobalIns.__log.has_key(name):
GlobalIns.__log[name] = copy.deepcopy(INSTANCE_TEMP)
def checkConfig(self):
if self._host == '' or self._port <= 0 or self._user == '' or self._connect_timeout <= 0:
return CBasicResult(-1, "config not valid [%s]" % self.__str__())
return CBasicResult()