def host_mongod_state(client_ins, shardinfo):
hostname = client_ins.client_hostname
commands = [{
"command": "ps",
"parameter": "-auxf | grep mongod| grep -v grep",
"resultsplit":
"USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND",
"resultprocess": ["PID", "COMMAND"]
}, {
"command": "netstat",
"parameter": " -lptu|grep mongod",
"resultsplit":
"Proto Recv-Q Send-Q Local_Address Foreign_Address State PID/Program_name",
"resultprocess": ["Local_Address", "PID/Program_name"]
}]
command_result = host_command(client_ins, commands)
host_mongodnodes = [_ for _ in shardinfo['mongodnodes'] if hostname in _]
host_mongodport = [_.split(":")[1] for _ in host_mongodnodes]
host_mongodcommand = {}
for port_ in host_mongodport:
for netstat_ in command_result['netstat']:
if port_ in netstat_['Local_Address']:
portpid = netstat_['PID/Program_name'].split("/")[0]
print(f"found the process {portpid}")
for ps_ in command_result['ps']:
if portpid == ps_["PID"]:
host_mongodcommand[port_] = ps_['COMMAND']
print(ps_['COMMAND'])
if len(host_mongodport) != len(host_mongodcommand):
raise AppException(
"mongo", 801,
f"{hostname} only have {host_mongodcommand.keys()} running. it shoudl have {host_mongodport}"
)
print(host_mongodcommand)
def database(self, dbname=None, status=False, new=False):
"""Get mongo database object
Args:
dbname string database name
if not name passed, will return admin database
Return:
db_ins object database object
Exepption:
201 database is not found
"""
dblist = self.conn.list_database_names()
if dbname is None:
db_ins = self.conn.get_database("admin")
else:
if dbname in dblist:
db_ins = self.conn.get_database(dbname)
else:
if new:
db_ins = getattr(self.conn, dbname)
else:
print(dblist)
raise AppException("mongo", 201,
f"database name is {dbname}")
if status:
dbstats = self.database_stats(db_ins)
return db_ins
def host_command(client_ins, commands):
validresults = {}
for commandset in commands:
command = f"{commandset['command']} {commandset['parameter']}"
returnresult = client_ins.execute_command(command)
resultmark = commandset['resultsplit'].split()
commandresult = []
if len(returnresult) == 3:
rr = [_ for _ in returnresult[1].split("\n") if _ is not ""]
for rrl in rr:
rrlist = rrl.split()
if len(rrlist) != len(resultmark):
lastnum = len(resultmark) - 1
lastvalue = " ".join(rrlist[lastnum:])
rrlisttmp = rrlist[:lastnum] + [lastvalue]
rrlist = rrlisttmp
rrdict = dict(zip(resultmark, rrlist))
validresult = {}
for key_ in commandset['resultprocess']:
validresult[key_] = rrdict[key_]
commandresult.append(validresult)
else:
raise AppException("bash", 101, returnresult)
validresults[commandset['command']] = commandresult
return validresults
def wrapper(*args):
log = get_log()
class_obj = args[0]
input_kwargs = args[1]
required = args[2]
optional = args[3]
newfields = args[4]
log.debug(f"start to set {class_obj} required properties")
for field in required:
if field in input_kwargs:
setattr(class_obj, field, input_kwargs[field])
log.debug(
f"set attribute {field} with value {input_kwargs[field]}")
else:
raise AppException(
"base", 101,
f"{field} is not in the {class_obj} initial dict")
log.debug(f"start to set {class_obj} initial optional properties")
for field in optional:
if field in input_kwargs:
setattr(class_obj, field, input_kwargs[field])
log.debug(
f"set attribute {field} with value {input_kwargs[field]}")
else:
log.debug(f"attribute {field}is not set")
log.debug(f"start to set {class_obj} properties")
for field in newfields:
setattr(class_obj, field, None)
log.debug(f"set attribute {field}")
return func(*args)
def _smtp_send(self, msg, toaddrs, fromaddr):
""" send email with smtp"""
smtp_ins = smtplib.SMTP(self.server)
retcode = smtp_ins.sendmail(fromaddr, toaddrs, msg)
if retcode != {}:
raise AppException(self.appname, 101, retcode)
return retcode
def email_folder2email(self, email_, folder, extlist=None, group=5):
""" send files in a folder to email"""
filegroup = file_splitgroup(folder_getfiles(folder, extlist), group)
filesizes = []
emaillist = []
for filename in filegroup:
timestamp = str(tc())
if not email_['attachment']:
email_['body'], filesize = webfile2mimebody(
filename, WEBFILENAME)
email_[
'subject'] = f"Email: Folder {folder} convert to HTML at {timestamp}"
filenames = None
else:
email_['body'], filesize = file2attach(filename)
email_[
'subject'] = f"Email: Folder {folder} convert to attachment at {timestamp}"
filenames = filename
filesizes += filesize
if len(filesize) != len(filename):
raise AppException(self.appname, 102, "file size is not match")
emaillist.append((email_['subject'], filesize))
self.email_mimeemail(email_, filenames=filenames)
reportstring = folder_report(filegroup, filesizes, emaillist, group,
email_['attachment'])
if email_['attachment']:
email_[
'subject'] = f"Email: Folder {folder} convert to attachment Report"
else:
email_[
'subject'] = f"Email: Folder {folder} convert to HTML Report"
email_['body'] = [(reportstring, "plain")]
self.email_mimeemail(email_)
def connection_check(self):
"""check mongo client connection """
try:
serverinfo = self.conn.server_info()
except errors:
raise AppException("mongo", 101, f"{errors}")
serverinfo['mongos'] = self.conn.is_mongos
serverinfo['primary'] = self.conn.is_primary
self.serverinfo = serverinfo
def shard_collection(self,
dbname=None,
collectionname=None,
enableshard=True):
""" collection shard information bassed on database and collection"""
if dbname is None:
dbs = self.conn.list_database_name()
collectionname = None
else:
dbs = [dbname]
self.log.debug(f"Will check the following dbs {dbs}")
shardinfos = []
for db in dbs:
self.log.debug(f"start to check {db} shard info")
shardinfo = {}
db_ = self.database(dbname=db, status=True)
self.log.debug(f"get database instance {db_}")
if collectionname is None:
collectionlist = db_.list_collection_names()
else:
collectionlist = [collectionname]
self.log.debug(
f"will check the following collections {collectionlist}")
for collname in collectionlist:
self.log.debug(f"get collection {collname} stats")
coll_info = db_.command("collstats", collname, 1)
if coll_info['sharded']:
self.log.debug(f"collection {collname} is sharded")
shardinfo[collname] = {}
shardinfo[collname]["shards"] = coll_info['shards'].keys()
self.log.debug(
f"there are {len(coll_info['shards'])} shards")
shardinfo[collname]['count'] = coll_info['count']
self.log.debug(
f"there are total {coll_info['count']} docuemnts")
shardinfo[collname]['nchunks'] = coll_info['nchunks']
self.log.debug(
f"there are total {coll_info['nchunks']} chuncks")
for shardins in coll_info['shards']:
shardinfo[collname][shardins] = coll_info['shards'][
shardins]['count']
self.log.debug(f"colleciton info is {shardinfo}")
else:
self.log.debug(f"{collname} is not sharded")
if enableshard:
db_admin = self.database()
db_admin.command({"enableSharding": db})
else:
raise AppException("mongo", 201,
f"{db} is not sharded")
shardinfos.append(shardinfo)
self.log.debug(f"shard collection info:\n{shardinfos}")
return shardinfos
def arg_assign(self, kwargs, required_fields, optional_fields):
""" Assign class attribute"""
all_keys = kwargs.keys()
for field in required_fields:
if field in all_keys:
setattr(self, field, kwargs[field])
else:
raise AppException(self.appname, 201,
f"input doens't have {field}")
for field in optional_fields:
if field in all_keys:
setattr(self, field, kwargs[field])
else:
setattr(self, field, None)
def collection(self, db_ins, collname, new=False):
""" Get mongo database colleciotn ojbects
Args:
db_ins object database instance
collname string collection name madatory
Return:
coll_ins object collection objects
Exception:
301 collection is not found
"""
collectionlist = db_ins.list_collection_names()
if collname in collectionlist:
# coll_ins = getattr(db_ins, collname)
coll_ins = db_ins.get_collection(collname)
else:
if new:
coll_ins = getattr(db_ins, collname)
else:
raise AppException("mongo", 301,
f"collection name is {collname}")
return coll_ins