class SaltByLocalApi(object):
'''
Saltapi class 通过salt本地接口调用,需和salt-master服务在同一台机器
'''
def __init__(self, main_config):
self.opts = master_config(main_config)
self.wheel = Wheel(self.opts)
self.client = LocalClient()
self.connected_minions_list = self.wheel.call_func('minions.connected')
self.key_dict = self.wheel.call_func('key.list_all')
self.total = len(self.key_dict['minions']) + len(
self.key_dict['minions_pre']) + len(
self.key_dict['minions_denied']) + len(
self.key_dict['minions_rejected'])
def get_minions_key_status(self):
reject = len(self.key_dict['minions_rejected'])
unaccept = len(self.key_dict['minions_pre'])
accept = len(self.key_dict['minions'])
return [accept, reject, unaccept]
def get_minions_status(self):
return [
self.total,
len(self.connected_minions_list),
self.total - len(self.connected_minions_list)
]
def get_host_info(self):
minions = self.connected_minions_list
ret = self.client.cmd(
minions,
'grains.item', [
'mem_total', 'osfullname', 'host', 'osrelease', 'num_cpus',
'ipv4', 'group', 'area', 'usage'
],
expr_form='list')
host_info_dict = {}
for k, v in ret.iteritems():
v['ipv4'].remove('127.0.0.1')
ips = '/'.join(v['ipv4']) if len(v['ipv4']) > 1 else v['ipv4'][0]
values = [
v['host'], ips, v['osfullname'] + v['osrelease'],
str(v['num_cpus']) + ' cores',
str(v['mem_total']) + ' MB', v['group'], v['area'], v['usage']
]
host_info_dict[k] = values
return host_info_dict
def get_master_config(self):
return self.opts
def get_grains(self):
if self.connected_minions_list is None or len(
self.connected_minions_list) < 1:
return None
return self.client.cmd(self.connected_minions_list[0], 'grains.items',
[])
class SaltByLocalApi(object):
'''
Saltapi class 通过salt本地接口调用,需和salt-master服务在同一台机器
'''
def __init__(self,main_config):
self.opts=master_config(main_config)
self.wheel=Wheel(self.opts)
self.client=LocalClient()
self.connected_minions_list=self.wheel.call_func('minions.connected')
self.key_dict=self.wheel.call_func('key.list_all')
self.total=len(self.key_dict['minions'])+len(self.key_dict['minions_pre'])+len(self.key_dict['minions_denied'])+len(self.key_dict['minions_rejected'])
def get_minions_key_status(self):
reject=len(self.key_dict['minions_rejected'])
unaccept=len(self.key_dict['minions_pre'])
accept=len(self.key_dict['minions'])
return [accept,reject,unaccept]
def get_minions_status(self):
online=len(self.get_host_info())
return [self.total,online,self.total-online]
def get_host_info(self):
minions=self.connected_minions_list
ret=self.client.cmd(minions,'grains.item',['mem_total',
'osfullname',
'host',
'osrelease',
'num_cpus',
'ipv4',
'group',
'area',
'usage'],expr_form='list')
host_info_dict={}
for k,v in ret.iteritems():
v['ipv4'].remove('127.0.0.1')
ips='/'.join(v['ipv4']) if len(v['ipv4'])>1 else v['ipv4'][0]
values=[v['host'],
ips,
v['osfullname']+v['osrelease'],
str(v['num_cpus'])+' cores',
str(v['mem_total'])+' MB',
v['group'],
v['area'],
v['usage']]
host_info_dict[k]=values
return host_info_dict
def get_master_config(self):
return self.opts
def get_grains(self):
if self.connected_minions_list is None or len(self.connected_minions_list)<1:
return None
return self.client.cmd(self.connected_minions_list[0],'grains.items',[])
def syncAsset():
salt = LocalClient()
grains = salt.cmd('*', 'grains.items')
obj = Asset.objects.all()
host_list = []
for i in obj:
host_list.append(i.hostname)
try:
for host in grains.keys():
ip = grains[host]['ipv4'][-1]
hostname_id = grains[host]['id']
cpu = grains[host]['cpu_model']
memory = grains[host]['mem_total']
if grains[host].has_key('virtual'):
asset_type = grains[host]['virtual']
else:
asset_type = 'physical'
if grains[host].has_key('osfinger'):
os = grains[host]['osfinger']
else:
os = grains[host]['osfullname']
if host not in host_list:
try:
Asset.objects.create(ip=ip,
hostname=hostname_id,
system_type=os,
cpu=cpu,
memory=memory,
asset_type=asset_type)
except Exception, e:
print e
except Exception, e:
print e
def export_table(handler,
panel,
server_name,
dash_user,
export_type='pdf',
table_file='/tmp/table',
args=[],
provider=None,
kwargs={},
filter_field=''):
table_func = 'va_utils.get_%s' % export_type
table_file = table_file + '.' + export_type
if not args:
args = list(args)
cl = LocalClient()
panel = yield get_panel_for_user(handler=handler,
panel=panel,
server_name=server_name,
dash_user=dash_user,
args=args,
provider=provider,
kwargs=kwargs)
print('Getting ', export_type, ' with filter : ', filter_field)
result = cl.cmd('G@role:va-master',
fun=table_func,
tgt_type='compound',
kwarg={
'panel': panel,
'table_file': table_file,
'filter_field': filter_field
})
yield handler.serve_file(table_file)
def get_panel_pdf(handler,
panel,
server_name,
dash_user,
pdf_file='/tmp/table.pdf',
args=[],
provider=None,
kwargs={},
filter_field=''):
if not args:
args = list(args)
cl = LocalClient()
panel = yield get_panel_for_user(handler=handler,
panel=panel,
server_name=server_name,
dash_user=dash_user,
args=args,
provider=provider,
kwargs=kwargs)
result = cl.cmd('va-master',
'va_utils.get_pdf',
kwarg={
'panel': panel,
'pdf_file': pdf_file,
'filter_field': filter_field
})
if not result['va-master']:
yield handler.serve_file(pdf_file)
raise tornado.gen.Return({'data_type': 'file'})
raise Exception('PDF returned a value - probably because of an error. ')
def get_all_monitoring_data(datastore_handler):
"""
description: Returns all icinga data from connected monitoring minions.
"""
cl = LocalClient()
result = cl.cmd('G@role:monitoring',
fun='va_monitoring.icinga2',
tgt_type='compound')
monitoring_errors = []
for minion in result:
if type(result[minion]) == str:
print(
'Error getting monitoring data for %s, salt returned %s, but will go on as usual. '
% (minion, result[minion]))
monitoring_errors.append(minion)
continue
for host in result[minion]:
if 'va_master' in host['host_name']:
panel = {'icon': 'fa-circle'}
else:
panel = yield datastore_handler.find_panel_for_server(
host['host_name'])
host['icon'] = panel['icon']
if monitoring_errors:
monitoring_errors = 'There was an error with the monitoring server(s): ' + ', '.join(
monitoring_errors)
result = {
'success': True,
'data': result,
'message': monitoring_errors
}
raise tornado.gen.Return(result)
def get_all_roles():
cl = LocalClient()
result = cl.cmd('*', 'grains.get', arg=['role'])
return {
x: result[x]
for x in result if result[x] not in error_msgs and result[x]
}
def flushMemcached(request):
data = request.POST.getlist('mcName')
project = 'memcache flush'
username = request.user
ip = request.META['REMOTE_ADDR']
saltCmd = LocalClient()
result = []
for name in data:
for info in memcache.objects.filter(memcacheName=name):
try:
cmd = saltCmd.cmd(
info.saltMinion.saltname, 'cmd.run',
['echo "flush_all" | nc %s %s' % (info.ip, info.port)])
result.append(cmd)
if cmd[info.saltMinion.saltname] == 'OK':
msg = 'Success'
else:
msg = 'error'
host = info.ip + ":" + info.port
dingding_robo(host, project, msg, username,
request.POST['phone_number'])
print result
except Exception, e:
print e
def restart_infra_impl(infra_name, target_server):
# check if infra is supported
if infra_name not in supported_infra:
raise NoSuchInfrastructureException('', 1)
logger.info('restart infra:%s on %s' % (infra_name, target_server))
sls_arg = supported_infra[infra_name]
# send saltstack cmd to restart infra
client = LocalClient()
client.cmd(target_server, 'state.apply', [sls_arg])
# check if the infrastructure is up
if not check_module_is_up(target_server, infra_name):
raise InfraIsNotUpException(infra_name)
logger.info('restart infra:%s on %s success' % (infra_name, target_server))
def run(self, quiet=False):
'''
do all the work
note that 'quiet' applies only to remotely
run, and the same is true for returning the contents.
maybe we want to fix that
'''
while True:
client = LocalClient()
module_args = [self.path, self.batchno,
self.batchsize, quiet]
result = client.cmd([self.host],
"retentionaudit.examine_dir",
module_args, expr_form='list',
timeout=self.timeout)
if self.host in result:
lines = result[self.host].split("\n")
maxlen = 0
for line in lines:
if (line.startswith("WARNING:") or
line.startswith("INFO:")):
continue
else:
try:
entry = json.loads(
line, object_hook=JsonHelper.decode_dict)
if len(entry['path']) > maxlen:
maxlen = len(entry['path'])
except:
continue
if not quiet:
for line in lines:
if (line.startswith("WARNING:") or
line.startswith("INFO:")):
print line
else:
try:
entry = json.loads(
line,
object_hook=JsonHelper.decode_dict)
EntryInfo.display_from_dict(
entry, True, maxlen)
except:
print line
return result[self.host]
else:
print "Failed to retrieve dir content for", self.path, "on", self.host
continuing = ("Try again? Y/N [N]: ")
if continuing == "":
continuing = "N"
if continuing.upper() != "Y":
return None
def get_all_salt_functions(datastore_handler):
""" Gets all salt functions for all minions. """
cl = LocalClient()
states = yield datastore_handler.get_states_and_apps()
functions = cl.cmd('*', 'sys.doc')
result = {[i for i in x if x[states[x]['module']] in i] for x in functions}
raise tornado.gen.Return(result)
def salt_runner(*args, **kwargs):
print("In salt_runner")
client = LocalClient()
t = client.cmd(target, instruction, arg=cmd_arg, timeout=120)
if (instruction in ("state.sls", "state.sls_id")):
ans = formatter_state(t)
else:
ans = formatter_raw(t)
return ans
def salt_runner(*args, **kwargs):
print "In salt_runner"
client = LocalClient()
t = client.cmd(target, instruction, arg=cmd_arg, timeout=120)
if (instruction == "state.sls"):
ans = formatter_state(t)
else:
ans = formatter_raw(t)
return ans
def run(self, quiet=False):
'''
do all the work
note that 'quiet' applies only to remotely
run, and the same is true for returning the contents.
maybe we want to fix that
'''
while True:
client = LocalClient()
module_args = [self.path, self.batchno, self.batchsize, quiet]
result = client.cmd([self.host],
"retentionaudit.examine_dir",
module_args,
expr_form='list',
timeout=self.timeout)
if self.host in result:
lines = result[self.host].split("\n")
maxlen = 0
for line in lines:
if (line.startswith("WARNING:")
or line.startswith("INFO:")):
continue
else:
try:
entry = json.loads(
line, object_hook=JsonHelper.decode_dict)
if len(entry['path']) > maxlen:
maxlen = len(entry['path'])
except:
continue
if not quiet:
for line in lines:
if (line.startswith("WARNING:")
or line.startswith("INFO:")):
print line
else:
try:
entry = json.loads(
line, object_hook=JsonHelper.decode_dict)
EntryInfo.display_from_dict(
entry, True, maxlen)
except:
print line
return result[self.host]
else:
print "Failed to retrieve dir content for", self.path, "on", self.host
continuing = ("Try again? Y/N [N]: ")
if continuing == "":
continuing = "N"
if continuing.upper() != "Y":
return None
def system(hostname):
try:
local = LocalClient()
sys = local.cmd(hostname,'grains.item',['kernel'])
if sys[hostname]['kernel'] == 'Linux':
return 'linux'
elif sys[hostname]['kernel'] == 'Windows':
return 'windows'
except Exception,e:
print e
def install_zfs(cls, data):
client = LocalClient()
subprocess.call(["mkdir",
"-p",
"/srv/salt"])
subprocess.call(["mv",
"%(app_root)s/application/utils/install_zfs.sh" % {'app_root': CONFIGS['APP_ROOT']},
"/srv/salt/"])
response = client.cmd(data['Storage-Node'],
"cmd.script",
["salt://install_zfs.sh"])
return response
def run_cmd(command, retcodes=[0], **kwargs):
from salt.client import LocalClient
global salt_client
if not salt_client:
salt_client = LocalClient()
res = salt_client.cmd('*', 'cmd.run', [command], full_return=True)
if retcodes:
for _id, _res in res.items():
assert _res['retcode'] in retcodes
return res
def restart_module_impl(module_name, target_server, settings):
module_settings = settings[module_name]
# container info
container_info = ContainerInfoParser(module_settings)
container_name = container_info.parse_container_name()
# remove old container
remove_container(target_server, container_name)
logger.info('restart module:%s on %s' % (module_name, target_server))
# send docker cmd
docker_cmd = compose_docker_cmd(module_settings)
client = LocalClient()
client.cmd(target_server, 'cmd.run', [docker_cmd])
# check if the module is up
if not check_module_is_up(target_server, container_name):
raise ChatModuleIsNotUpException(module_name)
logger.info('restart module:%s on %s success' %
(module_name, target_server))
def do_macro(registry, user, instruction, match):
macro = match.group('target')
if (macro not in deployments.keys()):
return {'success': False, 'answer': "I don't currently know about %s" % macro}
tgt = deployments[macro]
print "Starting macro %s"%macro
client = LocalClient()
t = client.cmd(tgt['target'], tgt['instruction'], arg=tgt['arg'], timeout=120)
if (tgt['instruction'] == "state.sls"):
ans = formatter_state(t)
else:
ans = formatter_raw(t)
return {'success': True, 'answer': ans}
def handle_app(datastore_handler, server, role, app_type):
''' Helping function for the manage_server_type function. If a server is added as an app, it adds a panel, sets the type, calls add_minion_to_server, which installs salt and runs highstate, and then inserts the server to the datastore. '''
server_name = server['server_name']
print ('Handling app!')
if not role:
raise Exception('Tried to convert ' + str(server_name) + " to app, but the role argument is empty. ")
if app_type == 'salt':
cl = LocalClient()
ping = cl.cmd(server_name, 'test.ping').get(server_name)
print ('Salt ping is : ', ping)
if not ping:
minion_kwargs = {'username' : server['username']}
if server.get('password'):
minion_kwargs['password'] = server['password']
else:
minion_kwargs['key_filename'] = datastore_handler.config.ssh_key_path + datastore_handler.config.ssh_key_name + '.pem'
print ('No ping, adding minion to server ', server_name, server['ip_address'], role, minion_kwargs)
yield add_minion_to_server(datastore_handler, server_name, server['ip_address'], role, **minion_kwargs)
print ('Server is : ', server)
server['type'] = 'app'
server['available_actions'] = server.get('available_actions', {}) # TODO get panel actions and add here
cl.cmd(server_name, 'grains.append', arg = ['role', role])
highstate = cl.cmd(server_name, 'state.highstate')
print ('Highstate is : ', highstate)
yield panels.new_panel(datastore_handler, server_name = server_name, role = role)
raise tornado.gen.Return(server)
def call_master_cmd(fun, arg=[], kwarg={}):
''' Calls the salt function on the va-master. Used to work with salt-call but a recent salt version made it incompatible with tornado, so we resort to using the `role` grain to find the va-master and call the function that way. '''
cl = LocalClient()
result = cl.cmd('G@role:va-master',
fun=fun,
tgt_type='compound',
arg=arg,
kwarg=kwarg)
result = [result[i] for i in result if result[i]]
if not result:
raise Exception('Tried to run ' + str(fun) +
' on va-master, but there was no response. arg was ' +
str(arg) + ' and kwarg was ' + str(kwarg))
return result[0]
def get_salt_functions():
cl = LocalClient()
salt_functions = cl.cmd('G@role:va-master',
fun='va_utils.get_documented_module_functions',
tgt_type='compound')
salt_functions = salt_functions.items()[0][1]
print('Salt : ', salt_functions)
salt_functions = {
method: [[function[0], yaml.load(function[1])]
for function in salt_functions[method]
if function_is_documented(function[1], func_name=function[0])]
for method in salt_functions
}
return salt_functions
def run(self):
'''
do all the work
'''
client = LocalClient()
module_args = [self.path, self.num_lines]
result = client.cmd([self.host],
"retentionaudit.examine_file",
module_args,
expr_form='list',
timeout=self.timeout)
if self.host in result:
if not self.quiet:
print result[self.host]
return result[self.host]
def run(self):
'''
do all the work
'''
client = LocalClient()
module_args = [self.path,
self.num_lines]
result = client.cmd([self.host],
"retentionaudit.examine_file",
module_args, expr_form='list',
timeout=self.timeout)
if self.host in result:
if not self.quiet:
print result[self.host]
return result[self.host]
def run(config, argv):
args = docopt(__doc__, argv=argv)
profile_name = args['--profile-name']
state = args['--state']
data = {'salt': {}}
master = config.master_config_data(files=['/etc/salt/master'])
instance_name = instances.instance_name_for_state(state, config)
instance_id = instance_name.rsplit('-')[-1]
minion_id = '{0}{1}'.format(state, instance_id)
pem, pub = salt.gen_keys()
client = LocalClient()
grains = client.cmd('master', 'grains.item', ['fqdn'])
try:
master_fqdn = grains['master']['fqdn']
except KeyError:
return
minion_data = {
'master': master_fqdn,
'id': minion_id
}
profile = config.profile_for_key(profile_name)
minion_data.update(profile.get('minion', {}))
data['salt']['minion_pub'] = pub
data['salt']['minion_pem'] = pem
data['salt']['minion'] = Filesystem.encode(
config.minion_config_data(minion_data))
salt.accept_key(master['pki_dir'], pub, minion_id)
instances.create_instance(
config=config,
profile_name=profile_name,
state=state,
data=data,
instance_name=instance_name)
def do_run(cmd, channel, body):
# {'cid': 'blahblah', 'target': '*', 'instruction': 'test.ping', cmd_arg: ['']}
try:
cmd_arg = body.get('cmd_arg', ())
instruction = body.get('instruction', 'test.ping')
target = body.get('target', '*')
client = LocalClient()
t = client.cmd(target, instruction, arg=cmd_arg, timeout=240)
if (instruction == "state.sls"):
ans = formatter_state(t)
else:
ans = formatter_raw(t)
result = {'cid': body.get('cid', None), 'result': ans}
zmq_sock.send_multipart('MSG', 'input/salt/run', json.dumps(result))
except:
traceback.print_exc(None)
result = {'cid': body.get('cid', None), 'result': 'An exception occurred'}
zmq_sock.send_multipart('MSG', 'input/salt/run', json.dumps(result))
def getServices(request):
action = request.POST.get('action')
data = request.POST.getlist('id')
username = request.user
saltCmd = LocalClient()
result = []
ip = request.META['REMOTE_ADDR']
for v in data:
goName,host = v.split(',')
getMes = saltCmd.cmd('%s'%host,'cmd.run',['supervisorctl %s %s'% (action,goName)])
result.append(getMes)
info = action + ' ' + goName
dingding_robo(host,info,getMes,username,request.POST['phone_number'])
logs(username,ip,'%s services' % action,result)
return render(request,'getdata.html',{'result':result})
def run(self):
try:
from salt.client import LocalClient
client = LocalClient(c_path=get_config().getstring('salt', 'master_config_path',
'/etc/salt/master'))
try:
log.debug('Running action against "%s": %s args: %s',
self.hostname, self.action, self.args)
data = client.cmd(self.hostname, self.action, arg=self.args)
except SystemExit as e:
log.error('Failed action %s on host: %s (%s)', self.action, self.hostname, e)
self.q.put(cPickle.dumps({}))
else:
pdata = cPickle.dumps(data)
self.q.put(pdata)
except Exception as e:
log.err(system='salt-local')
self.q.put(cPickle.dumps({'_error': e}))
def run(self):
try:
from salt.client import LocalClient
client = LocalClient(c_path=get_config().getstring(
'salt', 'master_config_path', '/etc/salt/master'))
try:
log.debug('Running action against "%s": %s args: %s',
self.hostname, self.action, self.args)
data = client.cmd(self.hostname, self.action, arg=self.args)
except SystemExit as e:
log.error('Failed action %s on host: %s (%s)', self.action,
self.hostname, e)
self.q.put(cPickle.dumps({}))
else:
pdata = cPickle.dumps(data)
self.q.put(pdata)
except Exception as e:
log.err(system='salt-local')
self.q.put(cPickle.dumps({'_error': e}))
def do_macro(registry, user, instruction, match):
macro = match.group('target')
if (macro not in deployments.keys()):
return {
'success': False,
'answer': "I don't currently know about %s" % macro
}
tgt = deployments[macro]
print "Starting macro %s" % macro
client = LocalClient()
t = client.cmd(tgt['target'],
tgt['instruction'],
arg=tgt['arg'],
timeout=120)
if (tgt['instruction'] == "state.sls"):
ans = formatter_state(t)
else:
ans = formatter_raw(t)
return {'success': True, 'answer': ans}
def do_run(cmd, channel, body):
# {'cid': 'blahblah', 'target': '*', 'instruction': 'test.ping', cmd_arg: ['']}
try:
cmd_arg = body.get('cmd_arg', ())
instruction = body.get('instruction', 'test.ping')
target = body.get('target', '*')
client = LocalClient()
t = client.cmd(target, instruction, arg=cmd_arg, timeout=240)
if (instruction == "state.sls"):
ans = formatter_state(t)
else:
ans = formatter_raw(t)
result = {'cid': body.get('cid', None), 'result': ans}
zmq_sock.send_multipart('MSG', 'input/salt/run', json.dumps(result))
except:
traceback.print_exc(None)
result = {
'cid': body.get('cid', None),
'result': 'An exception occurred'
}
zmq_sock.send_multipart('MSG', 'input/salt/run', json.dumps(result))
def server_list():
try:
linux_k_list=[]
local = LocalClient()
for item in hostname('/etc/salt/pki/master/minions'):
sys = local.cmd(item, 'grains.item', ['kernel'])
if sys[item]['kernel'] == 'Linux':
for k,v in sys.items():
linux_k = unicode.encode(k)
#os_list_path = os.path.abspath(os.path.dirname(os.getcwd()) + os.path.sep + ".." + os.path.sep + "data")
os_list_path = os.path.abspath(os.getcwd() + os.path.sep + "data")
os_list_file_linux = os.path.join(os_list_path,"linux_list.txt")
f = open(os_list_file_linux,'a')
f.write("|")
f.write(linux_k)
f.close()
elif sys[item]['kernel'] == 'Windows':
pass
else:
print 'other os'
except Exception,e:
print e
class goPublish:
def __init__(self, env):
self.env = env
self.saltCmd = LocalClient()
self.svnInfo = models.svn.objects.all()
def getNowTime(self):
return time.strftime("%Y-%m-%d_%H:%M:%S", time.localtime(time.time()))
def test_deployGo(self,
name,
services,
username,
ip,
tower_url,
phone_number,
svn_revision='head',
rb=True):
self.name = name # Go project name
self.services = services # Go service name
self.username = username
self.ip = ip
self.tower_url = tower_url
self.phone_number = phone_number
self.svn_revision = svn_revision
hostInfo = {}
result = []
rev_head = None
# assert svn info
res_svn = {
'vagrant-ubuntu-trusty-64':
"Updating '/srv/d2d':\nRestored '/srv/d2d/d2d'\nAt revision 18.\n------------------------------------------------------------------------\nr18 | chenye | 2017-03-08 03:56:55 +0000 (Wed, 08 Mar 2017) | 1 line\n\nupdate d2d\n------------------------------------------------------------------------"
}
# get head revision from svn
if self.svn_revision == 'head':
try:
for _, v in res_svn.items():
m = re.search('---+\nr(?P<rev>\d+)\s+\|\s+', v)
if m and rev_head is None:
rev_head = m.group('rev')
break
except Exception as e:
print str(e)
result.append({'get head-revision FAILED': str(e)})
result.append(res_svn)
print '-------------------svn:', self.svn_revision
print '----rev_head: ', rev_head
print '----svn_rev: ', svn_revision
if self.svn_revision == 'head':
# ROLLBACK to last successful revision if failed
if rb:
rev_last = get_rev_latest(services)
print '----rev_last:', rev_last
if rev_last:
result += self.deployGo(name,
services,
username,
ip,
tower_url,
phone_number,
svn_revision=rev_last)
else:
try:
# rev_head = get_rev_head(name)
update_rev_latest(services, rev_head)
except Exception as e:
print str(e)
result.append({'save head revision FAILED': str(e)})
return result
def deployGo(self,
name,
services,
username,
ip,
tower_url,
phone_number,
svn_revision='head'):
self.name = name # Go project name
self.services = services # Go service name
self.username = username
self.ip = ip
self.tower_url = tower_url
self.phone_number = phone_number
self.svn_revision = svn_revision
hostInfo = {}
result = []
rev_head = None
minionHost = commands.getstatusoutput(
'salt-key -l accepted')[1].split()[2:]
groupname = gogroup.objects.all()
for name in groupname:
if self.name == name.name:
for obj in goservices.objects.filter(env=self.env).filter(
group_id=name.id):
for saltname in minion.objects.filter(
id=obj.saltminion_id):
saltHost = saltname.saltname
if saltHost not in minionHost:
notMinion = 'No minions matched the %s host.' % saltHost
result.append(notMinion)
if obj.name == self.services:
golist = [self.services]
hostInfo[saltHost] = golist
for host, goname in hostInfo.items():
for p in self.svnInfo:
if p.project.name == self.name:
deploy_pillar = "pillar=\"{'project':'" + self.name + "'}\""
os.system("salt '%s' state.sls logs.gologs %s" %
(host, deploy_pillar))
currentTime = self.getNowTime()
self.saltCmd.cmd('%s' % host, 'cmd.run', [
'mv %s %s/%s_%s' %
(p.executefile, p.movepath, self.name, currentTime)
])
svn = self.saltCmd.cmd('%s' % host, 'cmd.run', [
'svn update -r%s --username=%s --password=%s --non-interactive %s && svn log -l 1 --username=%s --password=%s --non-interactive %s'
% (self.svn_revision, p.username, p.password,
p.localpath, p.username, p.password, p.localpath)
])
# get head revision from svn
if self.svn_revision == 'head':
try:
for _, v in svn.items():
m = re.search('---+\nr(?P<rev>\d+)\s+\|\s+', v)
if m and rev_head is None:
rev_head = m.group('rev')
break
except Exception as e:
print str(e)
result.append({'get head-revision FAILED': str(e)})
result.append(svn)
allServices = " ".join(goname)
restart = self.saltCmd.cmd(
'%s' % host, 'cmd.run',
['supervisorctl restart %s' % allServices])
result.append(restart)
info = self.name + "(" + tower_url + ")"
if self.svn_revision == 'head':
action = 'deploy ' + info
dingding_robo(host,
info,
restart,
self.username,
self.phone_number,
types=1)
else:
action = 'revert ' + info
dingding_robo(host,
info,
restart,
self.username,
self.phone_number,
types=3)
print '-------------------svn:', self.svn_revision
logs(self.username, self.ip, action, result)
publish_logs(self.username, self.ip, self.tower_url, result)
if self.svn_revision == 'head':
# ROLLBACK to last successful revision if failed
if not get_service_status(services):
rev_last = get_rev_latest(services)
if rev_last:
result += self.deployGo(self.name,
self.services,
self.username,
self.ip,
self.tower_url,
self.phone_number,
svn_revision=rev_last)
else:
try:
# rev_head = get_rev_head(name)
update_rev_latest(services, rev_head)
except Exception as e:
print str(e)
result.append({'save head revision FAILED': str(e)})
return result
def go_revert(self, project, revertFile, host, username, ip):
self.project = project
self.revertFile = revertFile
self.host = host
self.username = username
self.ip = ip
result = []
for p in self.svnInfo:
if p.project.name == self.project:
currentTime = self.getNowTime()
rename = p.revertpath + '_revert_' + currentTime
runCmd = "'mv " + p.executefile + " " + rename + "'"
os.system("salt %s state.sls logs.revert" % self.host)
self.saltCmd.cmd('%s' % self.host, 'cmd.run', ['%s' % runCmd])
revertResult = commands.getstatusoutput(
"salt '%s' cmd.run 'cp %s/%s %s'" %
(self.host, p.movepath, self.revertFile, p.executefile))
if revertResult[0] == 0:
for obj in goservices.objects.filter(env=self.env):
if obj.group.name == self.project and self.host == obj.saltminion.saltname:
restart = self.saltCmd.cmd(
'%s' % self.host, 'cmd.run',
['supervisorctl restart %s' % obj.name])
result.append(restart)
mes = 'revert to %s version is successful.' % revertFile
#mes = {self.host:mes}
#result.append(mes)
else:
mes = 'revert to %s version is failed.' % revertFile
mes = {self.host: mes}
info = 'revert ' + self.project
dingding_robo(self.host, info, mes, username)
result.append(mes)
action = 'revert ' + self.project
logs(self.username, self.ip, action, result)
return result
def goConf(self, project, usernmae, ip, phone_number):
self.project = project
self.username = usernmae
self.ip = ip
self.phone_number = phone_number
result = []
conf = goconf.objects.all()
for p in conf:
try:
if str(p.env) == self.env and p.project.name == self.project:
#print p.username,p.password,p.localpath,p.hostname
confCmd = "svn update --username=%s --password=%s --non-interactive %s" % (
p.username, p.password, p.localpath)
confResult = self.saltCmd.cmd('%s' % p.hostname, 'cmd.run',
['%s' % confCmd])
result.append(confResult)
info = self.project + ' conf'
dingding_robo(p.hostname, info, confResult, self.username,
self.phone_number)
except Exception, e:
print e
action = 'conf ' + self.project
logs(self.username, self.ip, action, result)
return result
def move_files():
'''Migrate file in Migration collection'''
from salt.client import LocalClient
cv = Condition()
app = mongo.app
tmp_dst_path = '/var/www/STO-%d/temp'
dst_path = '/var/www/STO-%d/FshareFS/mega/%s/%s'
client = LocalClient()
aria = None
migratings = {}
files_moving = {}
resum = {}
def res_grains_to_dict(salt_res):
ret = {}
for i in salt_res:
osts = salt_res[i]['ret']['osts']
if len(osts) > 1:
for ost in osts:
ret[ost] = [i, salt_res[i]['ret']['default_addr']]
continue
ret[osts[0]]=[i, salt_res[i]['ret']['default_addr']]
return ret
def check_file_done(dst_id, file_id, file_path):
sto_id = None
new_file_name = None
folder_path = None
src_move = None
dst_move = None
minion_id = None
with app.test_request_context():
res = cache_ost_grains_item(dst_id, args=['default_addr', 'osts'])
if not res or len(res) == 0 or type(res[res.keys()[0]]['ret'])=='str':
return
minion_id = res_grains_to_dict(res)
minion_id = minion_id[dst_id][0]
file_name = file_path.rsplit('/',1)[1]
with app.test_request_context():
try_connect_mysql()
sysFile = SystemFile.query.filter_by(id = file_id).first()
# Record in DB is deleted before moving file
if sysFile == None:
client.cmd(minion_id, 'storage.delete_files', [dst_id] + [file_path], timeout = 60*10, expr_form='glob')
mongo.Migration.collection.update({'_id': file_id}, {'$set': {'stat': "REMOVE_BEFORE"}})
return
folder_path = sysFile.folder_path
sub_folder = folder_path.split('/')
if len(sub_folder) == 1 and len(folder_path) > 6:
folder_path = folder_path[:6] + '/' + folder_path[6:]
src_move = '%s/%s' % (tmp_dst_path % dst_id, file_name)
if file_name.startswith('migrate_'):
file_name = file_name[8:]
dst_move = dst_path % (dst_id, folder_path, file_name)
while True:
jid = client.cmd_async(minion_id, 'storage.move_file', [src_move, dst_move, 400, dst_id], expr_form='glob')
ret = wait_return_salt_cmd(jid, timeout=60*10)
if len(ret) == 0:
continue
new_file_name = ret[ret.keys()[0]]['ret']
with app.test_request_context():
if ' ' in new_file_name: # filename not have space
mongo.Migration.collection.update({'_id': file_id}, {'$set': {'stat': "MOVE_ERR"}})
return
try_connect_mysql()
sysFile = SystemFile.query.filter_by(id = file_id).first()
ost = dst_id
sto_id = sysFile.storage_id
file_path = dst_path % (sysFile.storage_id, sysFile.folder_path, sysFile.name)
if sysFile == None:# Record in DB is deleted after moving file
new_file_path = dst_path % (ost, folder_path, new_file_name)
client.cmd(minion_id, 'storage.delete_files', [ost] + [new_file_path],
timeout=60*10, expr_form='glob')
mongo.Migration.collection.update({'_id': sid}, {'$set': {'stat': "REMOVE_AFTER"}})
return
with app.test_request_context():
# Update DB
count = 3
try_connect_mysql() # Update record in DB
while count > 0:
try:
sess = sql.session()
sess.query(SystemFile).filter(SystemFile.id == file_id).update({"storage_id": long(dst_id),
"name": new_file_name, "folder_path": folder_path })
sess.commit()
break
except OperationalError as e:
if '1205' in e[0]:
time.sleep(1)
count -= 1
if count == 0:
return
else:
sess.rollback()
raise e
mongo.FileRemoveSchedule.collection.insert({
'file_path' : file_path,
'ost' : sto_id,
'schedule_time' : datetime.datetime.utcnow() + datetime.timedelta(hours=24),
})
mongo.Migration.collection.remove({'_id': file_id})
break
def get_result(migratings):
while True:
wait = 60
for dst_id in migratings.keys():
ret = {}
with app.test_request_context():
res = cache_ost_grains_item(dst_id, args=['default_addr', 'osts'])
if not res or len(res) == 0 or type(res[res.keys()[0]]['ret']) == 'str':
continue
ret = res_grains_to_dict(res)
jid = client.cmd_async(ret[dst_id][0], 'aria2.tellStatus', [migratings[dst_id][0]], expr_form='glob')
aria = wait_return_salt_cmd(jid, timeout=20)
if len(aria) == 0 or not isinstance(aria[aria.keys()[0]]['ret'], dict):
with app.test_request_context():
current_app.logger.error('salt %s aria2.tellStatus(%s) not return (jid: %s)',
migratings[dst_id][2], migratings[dst_id][0], jid)
continue
stat = aria[aria.keys()[0]]['ret']
if stat['status'] == 'complete':
check_file_done(dst_id, migratings[dst_id][1], stat['files'][0]['path'])
# Delete cac system file id da download xong/error ra de add system file id khac vao download tiep
del migratings[dst_id]
# remove result
client.cmd_async(aria.keys()[0], 'aria2.removeDownloadResult', [stat['gid']])
cv.acquire()
cv.notify()
cv.release()
elif stat['status'] == 'error':
# Update stat -> ERROR & gid in mongo
with app.test_request_context():
mongo.Migration.collection.update({'_id': migratings[dst_id][1]} , {'$set': {'stat': "ARIA_ERROR_" + stat['errorCode']}})
current_app.logger.error('gid %s error %s: fid %d', stat['gid'], stat['errorCode'], migratings[dst_id][1])
# Delete cac system file id da download xong/error ra de add system file id khac vao download tiep
del migratings[dst_id]
cv.acquire()
cv.notify()
cv.release()
else:
# Maximum time waiting is 60s
if (long(stat['downloadSpeed']) == 0):
wait = 60
else:
wait = (long(stat['files'][0]['length']) - long(stat['files'][0]['completedLength'])) / long(stat['downloadSpeed'])
wait = min(wait, 60)
if wait < 10:
wait = 10
time.sleep(wait)
if not write_pid('move_files', 30*60):
return
th = Thread(target = get_result, args = (migratings,))
th.daemon = True
th.start()
count = 3
while count > 0:
try:
ret = mongo.Migration.find({'stat': "MIGRATING"})
for m in ret:
migratings[m['dst_id']] = [m['gid'], m['_id']]
break
except OperationFailure as e:
patt = re.compile(r"cursor id '(\d+)' not valid at server")
if patt.match(e.message):
time.sleep(1)
count -= 1
if count == 0:
return
else:
raise e
while True:
if not th.isAlive():
return
count = 3
while count > 0:
try:
list_queue = mongo.Migration.find({'stat': "QUEUE"})
for m in list_queue:
if m['dst_id'] in migratings:
continue
try_connect_mysql()
q = sql.session.query(SystemFile.name, SystemFile._checksum, SystemFile.storage_id, SystemFile.folder_path, UserFile.name). \
join(UserFile, SystemFile.id==UserFile.pid).filter(SystemFile.id == m['_id']).first()
# Record in DB is deleted before move file from source to dest's temp dir
if q == None or q[2] != m['src_id']:
mongo.Migration.collection.remove({'_id': m['_id']})
continue
src_id = cache_ost_grains_item(m['src_id'], args=['default_addr', 'osts'])
dst_id = cache_ost_grains_item(m['dst_id'], args=['default_addr', 'osts'])
if (not src_id or len(src_id) == 0 or type(src_id[src_id.keys()[0]]['ret']) == 'str') or \
(not dst_id or len(dst_id) == 0 or type(dst_id[dst_id.keys()[0]]['ret']) == 'str'):
continue
src_id = res_grains_to_dict(src_id)
dst_id = res_grains_to_dict(dst_id)
url = 'http://%s:89/private/STO-%d/FshareFS/mega/%s/%s' % (src_id[m['src_id']][1] , m['src_id'], q[3], q[0])
checksum = hexlify(q[1])
file_name = q[4]
file_name = filename_shortcut(file_name, lenght = 70, lower = True)
if file_name.startswith('_'):
file_name = checksum + file_name
else:
file_name = checksum + '_'+ file_name
file_name = 'migrate_' + file_name
ret = client.cmd(dst_id[m['dst_id']][0], 'aria2.addUri',
[url, 'dir=%s' % tmp_dst_path % m['dst_id'], 'out=%s' % file_name],
timeout = 60, expr_form='glob')
if len(ret) == 0 or len(ret[ret.keys()[0]]) > 16 or len(ret[ret.keys()[0]]) == 0:
continue
gid = ret[ret.keys()[0]]
mongo.Migration.collection.update({'_id': m['_id']}, {'$set': {'stat': "MIGRATING", 'gid': gid}})
migratings[m['dst_id']] = [gid, m['_id']]
break
except OperationFailure as e:
patt = re.compile(r"cursor id '(\d+)' not valid at server")
if patt.match(e.message):
time.sleep(1)
count -= 1
if count == 0:
return
else:
raise e
update_pid_expire('move_files', 30*60)
cv.acquire()
cv.wait(60*2)
cv.release()
class SaltLocalClient(SaltClientBase):
c_path: str = attr.ib(
default='/etc/salt/master',
metadata={inputs.METADATA_ARGPARSER: {
'help': "config path"
}})
# targets: str = RunArgs.targets
targets: str = attr.ib(default=ALL_MINIONS,
metadata={
inputs.METADATA_ARGPARSER: {
'help': "command's host targets"
}
})
shared: bool = False
_client: LocalClient = attr.ib(init=False, default=None)
def __attrs_post_init__(self):
"""Do post init."""
if self.shared:
self._client = salt_local_client()
else:
self._client = LocalClient(c_path=str(self.c_path))
@property
def _cmd_args_t(self) -> Type[SaltArgsBase]:
return SaltLocalClientArgs
@property
def _salt_client_res_t(self) -> Type[SaltClientResultBase]:
return SaltLocalClientResult
def _build_cmd_args(self,
fun: str,
fun_args: Union[Tuple, None] = None,
fun_kwargs: Union[Dict, None] = None,
secure=False,
**kwargs) -> SaltArgsBase:
cmd_args = super()._build_cmd_args(fun, fun_args, fun_kwargs, secure,
**kwargs)
_set_auth(cmd_args.kw)
cmd_args.kw['full_return'] = True
return cmd_args
def _run(self, cmd_args: SaltArgsBase):
return self._client.cmd(*cmd_args.args, **cmd_args.kwargs)
def run(self,
fun: str,
fun_args: Union[Tuple, None] = None,
fun_kwargs: Union[Dict, None] = None,
secure=False,
**kwargs):
targets = kwargs.pop('targets', self.targets)
if targets:
kwargs['targets'] = targets
return super().run(fun, fun_args, fun_kwargs, secure, **kwargs)
class goPublish:
def __init__(self, env):
self.env = env
self.saltCmd = LocalClient()
self.svnInfo = models.svn.objects.all()
def getNowTime(self):
return time.strftime("%Y-%m-%d_%H:%M:%S", time.localtime(time.time()))
def deployGo(self,
name,
services,
username,
ip,
tower_url,
phone_number,
svn_revision='head',
gotemplate_svn_revision='head'):
self.name = name # Go project name
self.services = services # Go service name
self.username = username
self.ip = ip
self.tower_url = tower_url
self.phone_number = phone_number
self.svn_revision = svn_revision
self.gotemplate_svn_revision = gotemplate_svn_revision
hostInfo = {}
result = []
rev_head = None
#minionHost = commands.getstatusoutput('salt-key -l accepted')[1].split()[2:]
services_info = goservices.objects.filter(env=self.env).filter(
name=self.services)
for s in services_info:
go_template = GOTemplate.objects.filter(project=s.group).filter(
hostname=s.saltminion).first()
# update conf file
if go_template:
svn_gotemplate = self.saltCmd.cmd(
'%s' % s.saltminion.saltname, 'cmd.run', [
'svn update -r%s --username=%s --password=%s --non-interactive %s && svn log -l 1 --username=%s --password=%s --non-interactive %s'
% (self.gotemplate_svn_revision, go_template.username,
go_template.password, go_template.localpath,
go_template.username, go_template.password,
go_template.localpath)
])
else:
svn_gotemplate = {
'Warning':
'#####################Not gotemplate file######################'
}
result.append(svn_gotemplate)
for p in self.svnInfo:
if p.project.name == self.name:
deploy_pillar = "pillar=\"{'project':'" + self.name + "'}\""
os.system("salt '%s' state.sls logs.gologs %s" %
(s.saltminion.saltname, deploy_pillar))
currentTime = self.getNowTime()
# move go binary
self.saltCmd.cmd('%s' % s.saltminion.saltname, 'cmd.run', [
'mv %s %s/%s_%s' %
(p.executefile, p.movepath, self.name, currentTime)
])
# deploy go project
svn = self.saltCmd.cmd(
'%s' % s.saltminion.saltname, 'cmd.run', [
'svn update -r%s --username=%s --password=%s --non-interactive %s && svn log -l 1 --username=%s --password=%s --non-interactive %s'
%
(self.svn_revision, p.username, p.password,
p.localpath, p.username, p.password, p.localpath)
])
result.append(svn)
restart = self.saltCmd.cmd(
'%s' % s.saltminion.saltname, 'cmd.run',
['supervisorctl restart %s' % self.services])
result.append(restart)
info = self.name + "(" + tower_url + ")"
if self.svn_revision == 'head':
action = 'deploy ' + info
dingding_robo(s.saltminion.saltname,
info,
restart,
self.username,
self.phone_number,
types=1)
else:
action = 'revert ' + info
dingding_robo(s.saltminion.saltname,
info,
restart,
self.username,
self.phone_number,
types=3)
print '-------------------svn:', self.svn_revision
logs(self.username, self.ip, action, result)
publish_logs(self.username, self.ip, self.tower_url, result)
if self.svn_revision == 'head':
# ROLLBACK to last successful revision if failed
if not get_service_status(services):
goservice_rev_last = get_rev_latest(self.name)
print '#####goservice_rev_last.gotemplate_last_rev', goservice_rev_last
if goservice_rev_last:
revert_info = {
'Warning':
'#####################Roll back to the previous version######################'
}
result.append(revert_info)
result += self.deployGo(
self.name,
self.services,
self.username,
self.ip,
self.tower_url,
self.phone_number,
svn_revision=goservice_rev_last[0],
gotemplate_svn_revision=goservice_rev_last[1])
else:
try:
# get head revision from svn
rev_head = get_svn_revision(svn)
gotemplate_rev_head = get_svn_revision(svn_gotemplate)
update_rev_latest(self.name, rev_head, gotemplate_rev_head)
except Exception as e:
print str(e)
result.append({'save head revision FAILED': str(e)})
return result
def go_revert(self, project, revertFile, host, username, ip):
self.project = project
self.revertFile = revertFile
self.host = host
self.username = username
self.ip = ip
result = []
for p in self.svnInfo:
if p.project.name == self.project:
currentTime = self.getNowTime()
rename = p.revertpath + '_revert_' + currentTime
runCmd = "'mv " + p.executefile + " " + rename + "'"
os.system("salt %s state.sls logs.revert" % self.host)
self.saltCmd.cmd('%s' % self.host, 'cmd.run', ['%s' % runCmd])
revertResult = commands.getstatusoutput(
"salt '%s' cmd.run 'cp %s/%s %s'" %
(self.host, p.movepath, self.revertFile, p.executefile))
if revertResult[0] == 0:
for obj in goservices.objects.filter(env=self.env):
if obj.group.name == self.project and self.host == obj.saltminion.saltname:
restart = self.saltCmd.cmd(
'%s' % self.host, 'cmd.run',
['supervisorctl restart %s' % obj.name])
result.append(restart)
mes = 'revert to %s version is successful.' % revertFile
#mes = {self.host:mes}
#result.append(mes)
else:
mes = 'revert to %s version is failed.' % revertFile
mes = {self.host: mes}
info = 'revert ' + self.project
dingding_robo(self.host, info, mes, username)
result.append(mes)
action = 'revert ' + self.project
logs(self.username, self.ip, action, result)
return result
def goConf(self, project, usernmae, ip, phone_number):
self.project = project
self.username = usernmae
self.ip = ip
self.phone_number = phone_number
result = []
conf = goconf.objects.all()
for p in conf:
try:
if str(p.env) == self.env and p.project.name == self.project:
#print p.username,p.password,p.localpath,p.hostname
confCmd = "svn update --username=%s --password=%s --non-interactive %s" % (
p.username, p.password, p.localpath)
confResult = self.saltCmd.cmd('%s' % p.hostname, 'cmd.run',
['%s' % confCmd])
result.append(confResult)
info = self.project + ' conf'
dingding_robo(p.hostname, info, confResult, self.username,
self.phone_number)
except Exception, e:
print e
action = 'conf ' + self.project
logs(self.username, self.ip, action, result)
return result
def get_minion_role(minion_name = '*'):
cl = LocalClient()
role = cl.cmd(minion_name, 'grains.get', arg = ['role'])
if minion_name != '*':
role = role[minion_name]
return role
def test1():
client=LocalClient()
result=client.cmd('self','oss:centos',['ls /root'],'grain')
print result
def test1():
client = LocalClient()
result = client.cmd('self', 'oss:centos', ['ls /root'], 'grain')
print result
def test_cmd():
local = LocalClient()
ret = local.cmd('*', 'test.ping')
assert ret == {'master': True}
