def start_client(step):
try:
QSIZE = 1024
CRATIO = 120 # Common ratio for send interval progression
db_manager = util.DBManager(world.config['connections']['mysql'])
db = db_manager.get_db()
where1 = and_(db.messages.type == 'out')
where2 = and_(db.messages.message_version == 2)
where3 = and_(func.unix_timestamp(db.messages.dtlasthandleattempt) +\
db.messages.handle_attempts * CRATIO < func.unix_timestamp(
func.now()))
msgs = db.messages.filter(db.messages.status==0,\
where1, where2, where3).order_by(
desc(db.messages.id)).all()[0:QSIZE]
global right_msgs
right_msgs = msgs
cnf = ETC_DIR + '/config.yml'
subps.Popen([
'python', '-m', 'scalrpy.messaging.messaging_client', '--start',
'-c', cnf
])
time.sleep(2)
ps = subps.Popen(['ps -ef'], shell=True, stdout=subps.PIPE)
output = ps.stdout.read()
ps.stdout.close()
ps.wait()
assert 'messaging_client' in output
except:
assert False
finally:
db.session.close()
db.session.remove()
def run(self):
self.db_manager = util.DBManager(self.config['connections']['mysql'])
db = self.db_manager.get_db()
while True:
# XXX next try-except block force load tables
try:
db.messages
db.servers
db.server_properties
break
except db_exc.OperationalError:
LOG.error(util.exc_info())
time.sleep(5)
LOG.debug('Reconnecting...')
try:
global pool
persistent_gm_workers = [_GMWorker(self) for i in range(2)]
for wrk in persistent_gm_workers:
wrk.register_task('message.send', send)
pool.add(gevent.spawn(wrk.work))
gevent.spawn(heartbeat, 10, self)
pool.join()
except:
LOG.critical(util.exc_info())
sys.exit(1)
def create_server_properties_table(config):
try:
db_manager = util.DBManager(config)
db = db_manager.get_db()
db.session.connection().execute('commit')
try:
db.session.connection().execute("CREATE TABLE `server_properties` ("+\
"`id` int(11) NOT NULL AUTO_INCREMENT,"+\
"`server_id` varchar(36) DEFAULT NULL,"+\
"`name` varchar(255) DEFAULT NULL,"+\
"`value` text,"+\
"PRIMARY KEY (`id`),"+\
"UNIQUE KEY `serverid_name` (`server_id`,`name`),"+\
"KEY `serverid` (`server_id`),"+\
"KEY `name_value` (`name`(20),`value`(20)),"+\
"CONSTRAINT `server_properties_ibfk_1` FOREIGN KEY "+\
"(`server_id`) REFERENCES `servers` (`server_id`)"+\
"ON DELETE CASCADE ON UPDATE NO ACTION) "+\
"ENGINE=InnoDB AUTO_INCREMENT=533922744 "+\
"DEFAULT CHARSET=latin1")
db.session.connection().execute('commit')
db.session.connection().close()
ret = True
except sql_exc.InternalError as e:
print sys.exc_info()
if e.orig[0] == 1050:
ret = True
else:
print sys.exc_info()
ret = False
except:
print sys.exc_info()
ret = False
return ret
def fill_tables(step, count, st, tp):
try:
world.msgs_id = {}
world.srvs_id = []
db_manager = util.DBManager(world.config['connections']['mysql'])
db = db_manager.get_db()
for i in range(int(count)):
while True:
msg_id = ''.join(random.choice(string.ascii_uppercase +\
string.digits) for x in range(75))
if db.messages.filter(
db.messages.messageid == msg_id).first() is None:
break
continue
while True:
srv_id = ''.join(random.choice(string.ascii_uppercase +\
string.digits) for x in range(36))
if db.servers.filter(
db.servers.server_id == srv_id).first() is None:
break
continue
db.messages.insert(messageid=msg_id,
status=int(st),
handle_attempts=0,
message_name='ExecScript__',
dtlasthandleattempt=func.now(),
message='some text here',
server_id=srv_id,
type='%s' % tp,
message_version=2)
world.msgs_id.setdefault(msg_id, {}).setdefault('status', st)
db.servers.insert(server_id=srv_id,
env_id=1,
status='Running',
remote_ip='1.0.0.1')
world.srvs_id.append(srv_id)
db.commit()
for srv_id in world.srvs_id:
db.server_properties.insert(server_id=srv_id,
name='scalarizr.key',
value='hoho')
db.server_properties.insert(server_id=srv_id,
name='scalarizr.ctrl_port',
value='8013')
db.commit()
db.session.close()
assert True
except:
assert False
def check_message_deleted(step):
db_manager = util.DBManager(world.config['connections']['mysql'])
db = db_manager.get_db()
try:
_id = [_.messageid for _ in right_msgs]
msgs = db.messages.filter(db.messages.messageid.in_(_id)).all()
assert len(msgs) == 0
except:
assert False
finally:
try:
db.session.close()
db.session.remove()
except:
pass
def create_servers_table(config):
try:
db_manager = util.DBManager(config)
db = db_manager.get_db()
db.session.connection().execute('commit')
try:
db.session.connection().execute("CREATE TABLE `servers` ("+\
"`id` int(11) NOT NULL AUTO_INCREMENT,"+\
"`server_id` varchar(36) DEFAULT NULL,"+\
"`farm_id` int(11) DEFAULT NULL,"+\
"`farm_roleid` int(11) DEFAULT NULL,"+\
"`client_id` int(11) DEFAULT NULL,"+\
"`env_id` int(11) NOT NULL,"+\
"`role_id` int(11) DEFAULT NULL,"+\
"`platform` varchar(10) DEFAULT NULL,"+\
"`status` varchar(25) DEFAULT NULL,"+\
"`remote_ip` varchar(15) DEFAULT NULL,"+\
"`local_ip` varchar(15) DEFAULT NULL,"+\
"`dtadded` datetime DEFAULT NULL,"+\
"`index` int(11) DEFAULT NULL,"+\
"`dtshutdownscheduled` datetime DEFAULT NULL,"+\
"`dtrebootstart` datetime DEFAULT NULL,"+\
"`replace_server_id` varchar(36) DEFAULT NULL,"+\
"`dtlastsync` datetime DEFAULT NULL,"+\
"PRIMARY KEY (id),"+\
"KEY serverid (server_id),"+\
"KEY farm_roleid (farm_roleid),"+\
"KEY farmid_status (farm_id,status),"+\
"KEY local_ip (local_ip),"+\
"KEY env_id (env_id),"+\
"KEY role_id (role_id),"+\
"KEY client_id (client_id) )"+\
"ENGINE=InnoDB AUTO_INCREMENT=817009 "+\
"DEFAULT CHARSET=latin1")
db.session.connection().execute('commit')
db.session.connection().close()
ret = True
except sql_exc.InternalError as e:
print sys.exc_info()
if e.orig[0] == 1050:
ret = True
else:
print sys.exc_info()
ret = False
except:
print sys.exc_info()
ret = False
return ret
def check_message_status(step, st):
db_manager = util.DBManager(world.config['connections']['mysql'])
db = db_manager.get_db()
try:
_id = [_.messageid for _ in right_msgs]
msgs = db.messages.filter(db.messages.messageid.in_(_id)).all()
for msg in msgs:
assert msg.status == 1
except:
assert False
finally:
try:
db.session.close()
db.session.remove()
except:
pass
def create_clients_table(config):
try:
db_manager = util.DBManager(config)
db = db_manager.get_db()
db.session.connection().execute('commit')
try:
db.session.connection().execute("CREATE TABLE `clients` ("+\
"`id` int(11) NOT NULL AUTO_INCREMENT,"+\
"`name` varchar(255) DEFAULT NULL,"+\
"`status` varchar(50) DEFAULT NULL,"+\
"`isbilled` tinyint(1) DEFAULT '0',"+\
"`dtdue` datetime DEFAULT NULL,"+\
"`isactive` tinyint(1) DEFAULT '0',"+\
"`fullname` varchar(60) DEFAULT NULL,"+\
"`org` varchar(60) DEFAULT NULL,"+\
"`country` varchar(60) DEFAULT NULL,"+\
"`state` varchar(60) DEFAULT NULL,"+\
"`city` varchar(60) DEFAULT NULL,"+\
"`zipcode` varchar(60) DEFAULT NULL,"+\
"`address1` varchar(60) DEFAULT NULL,"+\
"`address2` varchar(60) DEFAULT NULL,"+\
"`phone` varchar(60) DEFAULT NULL,"+\
"`fax` varchar(60) DEFAULT NULL,"+\
"`dtadded` datetime DEFAULT NULL,"+\
"`iswelcomemailsent` tinyint(1) DEFAULT '0',"+\
"`login_attempts` int(5) DEFAULT '0',"+\
"`dtlastloginattempt` datetime DEFAULT NULL,"+\
"`comments` text,"+\
"`priority` int(4) NOT NULL DEFAULT '0',"+\
"PRIMARY KEY (`id`)) "+\
"ENGINE=InnoDB AUTO_INCREMENT=9587 DEFAULT CHARSET=latin1")
db.session.connection().execute('commit')
db.session.connection().close()
ret = True
except sql_exc.InternalError as e:
print sys.exc_info()
if e.orig[0] == 1050:
ret = True
else:
print sys.exc_info()
ret = False
except:
print sys.exc_info()
ret = False
return ret
def create_messages_table(config):
try:
db_manager = util.DBManager(config)
db = db_manager.get_db()
db.session.connection().execute('commit')
try:
db.session.connection().execute("CREATE TABLE `messages` ("+\
"`id` int(11) NOT NULL AUTO_INCREMENT,"+\
"`messageid` varchar(75) DEFAULT NULL,"+\
"`instance_id` varchar(15) DEFAULT NULL,"+\
"`status` tinyint(1) DEFAULT '0',"+\
"`handle_attempts` int(2) DEFAULT '1',"+\
"`dtlasthandleattempt` datetime DEFAULT NULL,"+\
"`lock_time` datetime DEFAULT NULL,"+\
"`message` longtext,"+\
"`server_id` varchar(36) DEFAULT NULL,"+\
"`type` enum('in','out') DEFAULT NULL,"+\
"`isszr` tinyint(1) DEFAULT '0',"+\
"`message_name` varchar(30) DEFAULT NULL,"+\
"`message_version` int(2) DEFAULT NULL,"+\
"`in_processing` tinyint(1) DEFAULT '0',"+\
"PRIMARY KEY (id),"+\
"UNIQUE KEY server_message (messageid(36),server_id),"+\
"KEY server_id (server_id),"+\
"KEY serverid_isszr (server_id,isszr),"+\
"KEY messageid (messageid),"+\
"KEY status (status,type),"+\
"KEY message_name (message_name),"+\
"KEY dt (dtlasthandleattempt)) "+\
"ENGINE=MyISAM AUTO_INCREMENT=42920410 "+\
"DEFAULT CHARSET=latin1")
db.session.connection().execute('commit')
db.session.connection().close()
ret = True
except sql_exc.InternalError as e:
print sys.exc_info()
if e.orig[0] == 1050:
ret = True
else:
print sys.exc_info()
ret = False
except:
print sys.exc_info()
ret = False
return ret
def create_farms_table(config):
try:
db_manager = util.DBManager(config)
db = db_manager.get_db()
db.session.connection().execute('commit')
try:
db.session.connection().execute("CREATE TABLE `farms` ("+\
"`id` int(11) NOT NULL AUTO_INCREMENT,"+\
"`clientid` int(11) DEFAULT NULL,"+\
"`env_id` int(11) NOT NULL,"+\
"`name` varchar(255) DEFAULT NULL,"+\
"`iscompleted` tinyint(1) DEFAULT '0',"+\
"`hash` varchar(25) DEFAULT NULL,"+\
"`dtadded` datetime DEFAULT NULL,"+\
"`status` tinyint(1) DEFAULT '1',"+\
"`dtlaunched` datetime DEFAULT NULL,"+\
"`term_on_sync_fail` tinyint(1) DEFAULT '1',"+\
"`region` varchar(255) DEFAULT 'us-east-1',"+\
"`farm_roles_launch_order` tinyint(1) DEFAULT '0',"+\
"`comments` text,"+\
"`created_by_id` int(11) DEFAULT NULL,"+\
"`created_by_email` varchar(250) DEFAULT NULL,"+\
"PRIMARY KEY (`id`),"+\
"KEY `clientid` (`clientid`),"+\
"KEY `env_id` (`env_id`),"+\
"CONSTRAINT `farms_ibfk_1` FOREIGN KEY (`clientid`) "+\
"REFERENCES `clients` (`id`) ON DELETE CASCADE ON UPDATE NO ACTION) "+\
"ENGINE=InnoDB AUTO_INCREMENT=12552 DEFAULT CHARSET=latin1")
db.session.connection().execute('commit')
db.session.connection().close()
ret = True
except sql_exc.InternalError as e:
print sys.exc_info()
if e.orig[0] == 1050:
ret = True
else:
print sys.exc_info()
ret = False
except:
print sys.exc_info()
ret = False
return ret
def drop_db(config):
try:
db_manager = util.DBManager(config)
db = db_manager.get_db()
db.session.connection().execute('commit')
db.session.connection().execute('drop database %s' %config['name'])
db.session.connection().execute('commit')
db.session.connection().close()
ret = True
except sql_exc.InternalError as e:
if e.orig[0] == 1049:
ret = True
else:
print sys.exc_info()
ret = False
except:
print sys.exc_info()
ret = False
return ret
def create_db(config):
try:
db_manager = util.DBManager(config)
db = db_manager.get_db()
db.session.connection()
db.session.close()
ret = True
except:
try:
db_engine = sqlalchemy.create_engine('%s://%s:%s@%s'
%(config['driver'], config['user'],
config['pass'], config['host']))
conn = db_engine.connect()
conn.execute('commit')
conn.execute('create database %s' %config['name'])
conn.execute('commit')
conn.close()
ret = True
except:
print sys.exc_info()
ret = False
return ret
def create_farm_roles_table(config):
try:
db_manager = util.DBManager(config)
db = db_manager.get_db()
db.session.connection().execute('commit')
try:
db.session.connection().execute("CREATE TABLE `farm_roles` ("+\
"`id` int(11) NOT NULL AUTO_INCREMENT,"+\
"`farmid` int(11) DEFAULT NULL,"+\
"`dtlastsync` datetime DEFAULT NULL,"+\
"`reboot_timeout` int(10) DEFAULT '300',"+\
"`launch_timeout` int(10) DEFAULT '300',"+\
"`status_timeout` int(10) DEFAULT '20',"+\
"`launch_index` int(5) DEFAULT '0',"+\
"`role_id` int(11) DEFAULT NULL,"+\
"`new_role_id` int(11) DEFAULT NULL,"+\
"`platform` varchar(20) DEFAULT NULL,"+\
"`cloud_location` varchar(50) DEFAULT NULL,"+\
"PRIMARY KEY (`id`),"+\
"KEY `role_id` (`role_id`),"+\
"KEY `farmid` (`farmid`),"+\
"KEY `platform` (`platform`),"+\
"CONSTRAINT `farm_roles_ibfk_1` FOREIGN KEY (`farmid`) "+\
"REFERENCES `farms` (`id`) ON DELETE CASCADE ON UPDATE NO ACTION )"+\
"ENGINE=InnoDB AUTO_INCREMENT=43156 DEFAULT CHARSET=latin1")
db.session.connection().execute('commit')
db.session.connection().close()
ret = True
except sql_exc.InternalError as e:
print sys.exc_info()
if e.orig[0] == 1050:
ret = True
else:
print sys.exc_info()
ret = False
except:
print sys.exc_info()
ret = False
return ret
def run(self):
db_manager = util.DBManager(self.config['connections']['mysql'])
db = db_manager.get_db()
self._set_new_gm_server(block=True)
timestep = 5
while True:
session = db.session
try:
gm_adm_client = gearman.GearmanAdminClient([self.gm_host])
gm_adm_client.ping_server()
# fix gearman v2.0.2 memory leak bug
self.gm_client = gearman.GearmanClient([self.gm_host])
self._update_submitted_jobs()
if len(self.submitted_jobs) > 5000:
LOG.warning('Too much of a submitted jobs. Skip iteration')
time.sleep(timestep)
continue
where1 = and_(db.messages.type == 'out',
db.messages.status == 0,
db.messages.message_version == 2)
where2 = and_(
func.unix_timestamp(db.messages.dtlasthandleattempt) +\
db.messages.handle_attempts *\
CRATIO < func.unix_timestamp(func.now()))
if self.submitted_jobs:
where3 = and_(
not_(
db.messages.messageid.in_(
self.submitted_jobs.keys())))
msgs = session.query(
db.messages.messageid,
db.messages.handle_attempts).filter(
where1, where2, where3).order_by(
asc(db.messages.id)).all()[0:QSIZE]
else:
msgs = session.query(
db.messages.messageid,
db.messages.handle_attempts).filter(
where1, where2).order_by(asc(
db.messages.id)).all()[0:QSIZE]
for msg in msgs:
# simple unique version
req = self.gm_client.submit_job('message.send',
msg.messageid,
unique=msg.messageid[0:64],
wait_until_complete=False)
# sha256 unique version
'''
req = self.gm_client.submit_job('message.send', msg.messageid,
unique=hashlib.sha256(msg.messageid).hexdigest(),
wait_until_complete=False)
'''
self.gm_client.wait_until_jobs_accepted([req])
self.submitted_jobs.update({
msg.messageid:
(req,
int(time.time() + CRATIO * (msg.handle_attempts + 1)))
})
LOG.info('Sumbit message: msg_id:%s' % msg.messageid)
except db_exc.OperationalError:
LOG.error(util.exc_info())
time.sleep(5)
except gearman.errors.ServerUnavailable:
LOG.error(util.exc_info())
self._set_new_gm_server(block=True)
except:
LOG.error(util.exc_info())
raise
finally:
session.close()
session.remove()
time.sleep(timestep)
def scheduler(connection, mode, metric):
"""This function gets farms and puts it into farm queue then run farm workers"""
'''
:type connection: dictionary
:param connection: Database connection info. Example:
{
'user':'******',
'pass':'******',
'host':'localhost',
'name':'scalr',
'driver':'mysql+pymysql',
'pool_recycle':120,
'pool_size':4}
:type mode: string
:param mode: Mode [snmp | scalarizr_api]
:type task: list of strings
:param task: Tasks list [cpu | mem | net | la | io]
'''
global farm_queue
db_manager = util.DBManager(connection)
db = db_manager.get_db()
session = db.session
clients = [
cli.id for cli in session.query(db.clients.id).filter_by(
status='Active').all()
]
if not clients:
LOG.info('Nothing to do. Exit')
return
where_farms = and_(db.farms.status == 1, db.farms.clientid.in_(clients))
farms = dict((farm.id, farm.hash) for farm in session.query(
db.farms.id, db.farms.hash).filter(where_farms).all())
if not farms:
LOG.info('Nothing to do. Exit')
return
servers = session.query(db.servers.server_id, db.servers.farm_id,
db.servers.farm_roleid, db.servers.index,
db.servers.remote_ip).filter(
and_(db.servers.farm_id.in_(farms.keys()),
db.servers.status == 'Running',
db.servers.remote_ip != 'None')).all()
servers_id = [srv.server_id for srv in servers]
if not servers_id:
LOG.info('Nothing to do. Exit')
return
if mode == 'snmp':
where_port = and_(db.server_properties.server_id.in_(servers_id),
db.server_properties.name == 'scalarizr.snmp_port')
elif mode == 'scalarizr_api':
where_port = and_(db.server_properties.server_id.in_(servers_id),
db.server_properties.name == 'scalarizr.api_port')
where_key = and_(db.server_properties.server_id.in_(servers_id),
db.server_properties.name == 'scalarizr.key')
keys = dict((prop.server_id, prop.value) for prop in session.query(
db.server_properties.server_id, db.server_properties.value).filter(
where_key).all())
ports = dict((prop.server_id, prop.value) for prop in session.query(
db.server_properties.server_id, db.server_properties.value).filter(
where_port).all())
session.close()
session.remove()
farms_to_process = {}
for srv in servers:
farm_id = srv.farm_id
farm_role_id = srv.farm_roleid
index = srv.index
host = srv.remote_ip
if mode == 'snmp':
community = farms[farm_id]
try:
port = ports[srv.server_id]
except:
port = 161
connection_info = [host, port, community]
elif mode == 'scalarizr_api':
try:
key = keys[srv.server_id]
except:
LOG.warning('Scalarizr key not found')
continue
try:
port = ports[srv.server_id]
except:
port = 8010
connection_info = [host, port, key]
farms_to_process.setdefault(farm_id, {}).setdefault(farm_role_id, []).append(\
[index, connection_info])
farm_workers = [
multiprocessing.Process(target=FarmWorker(), args=(
mode,
metric,
)) for n in range(F_WRK_NUM)
]
try:
for p in farm_workers:
p.start()
for farm in farms_to_process.items():
farm_queue.put(['farm', farm])
farm_queue.put(['done'])
LOG.debug('Put farm task done')
for p in farm_workers:
p.join()
except KeyboardInterrupt:
LOG.debug('Kill all farm workers')
for p in farm_workers:
LOG.debug('Terminate...')
p.terminate()
raise