def host_mongod_state(client_ins, shardinfo): hostname = client_ins.client_hostname commands = [{ "command": "ps", "parameter": "-auxf | grep mongod| grep -v grep", "resultsplit": "USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND", "resultprocess": ["PID", "COMMAND"] }, { "command": "netstat", "parameter": " -lptu|grep mongod", "resultsplit": "Proto Recv-Q Send-Q Local_Address Foreign_Address State PID/Program_name", "resultprocess": ["Local_Address", "PID/Program_name"] }] command_result = host_command(client_ins, commands) host_mongodnodes = [_ for _ in shardinfo['mongodnodes'] if hostname in _] host_mongodport = [_.split(":")[1] for _ in host_mongodnodes] host_mongodcommand = {} for port_ in host_mongodport: for netstat_ in command_result['netstat']: if port_ in netstat_['Local_Address']: portpid = netstat_['PID/Program_name'].split("/")[0] print(f"found the process {portpid}") for ps_ in command_result['ps']: if portpid == ps_["PID"]: host_mongodcommand[port_] = ps_['COMMAND'] print(ps_['COMMAND']) if len(host_mongodport) != len(host_mongodcommand): raise AppException( "mongo", 801, f"{hostname} only have {host_mongodcommand.keys()} running. it shoudl have {host_mongodport}" ) print(host_mongodcommand)
def database(self, dbname=None, status=False, new=False): """Get mongo database object Args: dbname string database name if not name passed, will return admin database Return: db_ins object database object Exepption: 201 database is not found """ dblist = self.conn.list_database_names() if dbname is None: db_ins = self.conn.get_database("admin") else: if dbname in dblist: db_ins = self.conn.get_database(dbname) else: if new: db_ins = getattr(self.conn, dbname) else: print(dblist) raise AppException("mongo", 201, f"database name is {dbname}") if status: dbstats = self.database_stats(db_ins) return db_ins
def host_command(client_ins, commands): validresults = {} for commandset in commands: command = f"{commandset['command']} {commandset['parameter']}" returnresult = client_ins.execute_command(command) resultmark = commandset['resultsplit'].split() commandresult = [] if len(returnresult) == 3: rr = [_ for _ in returnresult[1].split("\n") if _ is not ""] for rrl in rr: rrlist = rrl.split() if len(rrlist) != len(resultmark): lastnum = len(resultmark) - 1 lastvalue = " ".join(rrlist[lastnum:]) rrlisttmp = rrlist[:lastnum] + [lastvalue] rrlist = rrlisttmp rrdict = dict(zip(resultmark, rrlist)) validresult = {} for key_ in commandset['resultprocess']: validresult[key_] = rrdict[key_] commandresult.append(validresult) else: raise AppException("bash", 101, returnresult) validresults[commandset['command']] = commandresult return validresults
def wrapper(*args): log = get_log() class_obj = args[0] input_kwargs = args[1] required = args[2] optional = args[3] newfields = args[4] log.debug(f"start to set {class_obj} required properties") for field in required: if field in input_kwargs: setattr(class_obj, field, input_kwargs[field]) log.debug( f"set attribute {field} with value {input_kwargs[field]}") else: raise AppException( "base", 101, f"{field} is not in the {class_obj} initial dict") log.debug(f"start to set {class_obj} initial optional properties") for field in optional: if field in input_kwargs: setattr(class_obj, field, input_kwargs[field]) log.debug( f"set attribute {field} with value {input_kwargs[field]}") else: log.debug(f"attribute {field}is not set") log.debug(f"start to set {class_obj} properties") for field in newfields: setattr(class_obj, field, None) log.debug(f"set attribute {field}") return func(*args)
def _smtp_send(self, msg, toaddrs, fromaddr): """ send email with smtp""" smtp_ins = smtplib.SMTP(self.server) retcode = smtp_ins.sendmail(fromaddr, toaddrs, msg) if retcode != {}: raise AppException(self.appname, 101, retcode) return retcode
def email_folder2email(self, email_, folder, extlist=None, group=5): """ send files in a folder to email""" filegroup = file_splitgroup(folder_getfiles(folder, extlist), group) filesizes = [] emaillist = [] for filename in filegroup: timestamp = str(tc()) if not email_['attachment']: email_['body'], filesize = webfile2mimebody( filename, WEBFILENAME) email_[ 'subject'] = f"Email: Folder {folder} convert to HTML at {timestamp}" filenames = None else: email_['body'], filesize = file2attach(filename) email_[ 'subject'] = f"Email: Folder {folder} convert to attachment at {timestamp}" filenames = filename filesizes += filesize if len(filesize) != len(filename): raise AppException(self.appname, 102, "file size is not match") emaillist.append((email_['subject'], filesize)) self.email_mimeemail(email_, filenames=filenames) reportstring = folder_report(filegroup, filesizes, emaillist, group, email_['attachment']) if email_['attachment']: email_[ 'subject'] = f"Email: Folder {folder} convert to attachment Report" else: email_[ 'subject'] = f"Email: Folder {folder} convert to HTML Report" email_['body'] = [(reportstring, "plain")] self.email_mimeemail(email_)
def connection_check(self): """check mongo client connection """ try: serverinfo = self.conn.server_info() except errors: raise AppException("mongo", 101, f"{errors}") serverinfo['mongos'] = self.conn.is_mongos serverinfo['primary'] = self.conn.is_primary self.serverinfo = serverinfo
def shard_collection(self, dbname=None, collectionname=None, enableshard=True): """ collection shard information bassed on database and collection""" if dbname is None: dbs = self.conn.list_database_name() collectionname = None else: dbs = [dbname] self.log.debug(f"Will check the following dbs {dbs}") shardinfos = [] for db in dbs: self.log.debug(f"start to check {db} shard info") shardinfo = {} db_ = self.database(dbname=db, status=True) self.log.debug(f"get database instance {db_}") if collectionname is None: collectionlist = db_.list_collection_names() else: collectionlist = [collectionname] self.log.debug( f"will check the following collections {collectionlist}") for collname in collectionlist: self.log.debug(f"get collection {collname} stats") coll_info = db_.command("collstats", collname, 1) if coll_info['sharded']: self.log.debug(f"collection {collname} is sharded") shardinfo[collname] = {} shardinfo[collname]["shards"] = coll_info['shards'].keys() self.log.debug( f"there are {len(coll_info['shards'])} shards") shardinfo[collname]['count'] = coll_info['count'] self.log.debug( f"there are total {coll_info['count']} docuemnts") shardinfo[collname]['nchunks'] = coll_info['nchunks'] self.log.debug( f"there are total {coll_info['nchunks']} chuncks") for shardins in coll_info['shards']: shardinfo[collname][shardins] = coll_info['shards'][ shardins]['count'] self.log.debug(f"colleciton info is {shardinfo}") else: self.log.debug(f"{collname} is not sharded") if enableshard: db_admin = self.database() db_admin.command({"enableSharding": db}) else: raise AppException("mongo", 201, f"{db} is not sharded") shardinfos.append(shardinfo) self.log.debug(f"shard collection info:\n{shardinfos}") return shardinfos
def arg_assign(self, kwargs, required_fields, optional_fields): """ Assign class attribute""" all_keys = kwargs.keys() for field in required_fields: if field in all_keys: setattr(self, field, kwargs[field]) else: raise AppException(self.appname, 201, f"input doens't have {field}") for field in optional_fields: if field in all_keys: setattr(self, field, kwargs[field]) else: setattr(self, field, None)
def collection(self, db_ins, collname, new=False): """ Get mongo database colleciotn ojbects Args: db_ins object database instance collname string collection name madatory Return: coll_ins object collection objects Exception: 301 collection is not found """ collectionlist = db_ins.list_collection_names() if collname in collectionlist: # coll_ins = getattr(db_ins, collname) coll_ins = db_ins.get_collection(collname) else: if new: coll_ins = getattr(db_ins, collname) else: raise AppException("mongo", 301, f"collection name is {collname}") return coll_ins