def make_docker_image(self): if self.order_obj.project.deploy_type != DOCKER: return None logger.debug('开始生成docker镜像') self.f.write('> 开始生成docker镜像\n') self.f.flush() minion = settings.DEPLOY.get('M_MINION') rets = saltapi.state_sls( minion, **{ 'pillar': { 'project': self.order_obj.project.name, 'env': self.order_obj.env.code, 'tag': self.order_obj.version }, 'mods': 'make_docker_image', 'saltenv': 'deploy' }) self.f.write(json.dumps(rets, indent=4)) logger.debug('执行生成docker镜像完成') self.f.write('\n> 执行生成docker镜像完成\n') self.f.flush() rets = rets.get('return', [])[0].get(minion) for k, v in rets.items(): if not v.get('result'): raise Exception('生成docker镜像失败')
def sgdRMSPropNestorov(w0, x, y, f, grad, learning_rate=0.01, batch_size=100, max_epochs=1000, alpha=0.9, delta=1e-6, ro=0.9, eps=1e-6, shuffle=False, rng=None): tm = Timer() n = x.shape[0] n_batches = get_num_batches(n, batch_size) w = np.copy(w0) v = np.zeros(len(w0), dtype=w0.dtype) # velocity r = np.zeros(len(w0), dtype=w0.dtype) # gradient accumulation variable epoch_losses = np.zeros(max_epochs, dtype=float) epoch = 0 w_best = np.copy(w0) loss_best = np.inf if n <= batch_size: # no need to shuffle since all instances will be used up in one batch shuffle = False if shuffle: shuffled_idxs = np.arange(n) if rng is None: np.random.shuffle(shuffled_idxs) else: rng.shuffle(shuffled_idxs) else: shuffled_idxs = None prev_loss = np.inf while epoch < max_epochs: losses = np.zeros(n_batches, dtype=float) for i in range(n_batches): xi, yi = get_sgd_batch(x, y, i, batch_size, shuffled_idxs=shuffled_idxs) tw = w + alpha * v g = grad(tw, xi, yi) r[:] = ro * r + (1 - ro) * np.multiply(g, g) dw_scale = (learning_rate / (np.sqrt(delta + r))) v = alpha * v - np.multiply(dw_scale, g) w[:] = w + v losses[i] = f(w, xi, yi) loss = np.mean(losses) if np.isnan(loss): logger.debug("loss is nan") logger.debug("|w|=%f" % w.dot(w)) raise ArithmeticError("loss is nan in sgd") epoch_losses[epoch] = loss if loss < loss_best: # pocket algorithm np.copyto(w_best, w) loss_best = loss epoch += 1 if (loss < eps or np.abs(loss - prev_loss) < eps or avg_loss_check(epoch_losses, epoch, n=20, eps=eps)): break prev_loss = loss debug_log_sgd_losses("sgdRMSPropNestorov", epoch_losses, epoch, n=20, timer=tm) # logger.debug("epochs: %d" % epoch) # logger.debug("net losses:") # logger.debug("epoch losses:\n%s" % str(epoch_losses[0:epoch])) # logger.debug("best loss: %f" % loss_best) return w_best
def append(self, test_name, **parameters): logger.info("Append name \"%s\" " % (test_name)) if parameters == {}: obj = tests[test_name]() else: obj = tests[test_name](**parameters) if obj is not None: logger.debug("Appended name \"%s\" " % (test_name)) self.__data[test_name + str(self.__id)] = obj self.__id = self.__id + 1
def closeConnect(host, port, s): global messageForMPQueue logger.debug("closing MP: " + host + " : " + str(port)) if s in outputs: outputs.remove(s) d = getOrBuildMPClient(host, port) if messageForMPQueue.has_key(d): q = messageForMPQueue[d] messageForMPQueue.pop(d) del deviceConnectionToIPMap[s] inputs.remove(s) s.close()
def __init__(self, **params): logger.debug("Init axis %s", params['ip']) vc.__init__(self, **params) auth = '%s:%s' % (self.user, self.password) auth = auth.encode('ascii') userAndPass = b64encode(auth).decode("ascii") self.__headers_auth = {'Authorization': 'Basic %s' % userAndPass} logger.debug("Connection string http://%s:%s@%s" % (self.user, self.password, self.ip)) try: self.__conn = httplib.HTTPConnection("%s" % self.ip) except httplib.HTTPException as e: logger.error("Execution failed: %s", e)
def run_script(self, tgt, path): data = { 'client': 'local', 'fun': 'cmd.script', 'tgt': tgt, 'arg': 'salt://' + path } req = self.post(data=data) logger.debug(req.json()) if req.status_code != 200: response = {'code': req.status_code, 'detail': req.text} else: response = req.json().get('return', req.json()) return response
def debug_log_sgd_losses(sgd_type, losses, epoch, n=20, timer=None): if False: # disable logging -- should be used in PRODUCTION return elif True: # minimal info logger.debug("[%s] epochs: %d; avg last %d losses:%f%s" % (sgd_type, epoch, n, np.mean(losses[(epoch-min(n, epoch)):(epoch)]), "" if timer is None else "; time: %f" % timer.elapsed())) else: # maximum info logger.debug("[%s] epochs: %d; avg last %d losses:%f\n%s\n%s" % (sgd_type, epoch, n, np.mean(losses[(epoch-min(n, epoch)):(epoch)]), str(list(losses[0:min(n, epoch)])), str(list(losses[(epoch-min(n, epoch)):(epoch)]))))
def download_package(self): logger.debug('开始下载代码') self.f.write('> 开始下载代码\n') self.f.flush() build_number = self.deploy_cache['build_number'] self.set_step_cache(self.cache_name, self.deploy_cache) jenkins_api.download_package(self.order_obj.project.package_url, self.order_obj.project.jenkins_job, build_number) logger.debug('代码下载完成') self.f.write('> 代码下载完成\n') self.f.flush()
def state(self): try: logger.debug("%s: Check service state", self.service_name) FNULL = open(os.devnull, 'w') retcode = call("pgrep %s" % (self.service_name), stdout=FNULL, stderr=subprocess.STDOUT, shell=True) if retcode == 0: logger.info("%s: Service runnig" % (self.service_name)) return Service_State.run else: logger.info("%s: Service stopped" % (self.service_name)) return Service_State.stop except OSError as e: logger.error("Execution failed: %s", e)
def get_grains_items(self, key_id): prefix = '/minions/{}'.format(key_id) try: req = self.get(prefix=prefix) logger.debug(req.json()) data = req.json() response = data['return'][0][key_id] if req.status_code != 200: response = {'code': req.status_code, 'detail': '请求异常'} elif not response: response = {'code': -1, 'detail': 'minion 返回false'} else: response['code'] = req.status_code except Exception as e: logger.error(e) response = {'code': -1, 'detail': 'salt api返回数据异常'} return response
def check_iif_state(self, id_iff): logger.debug("Check iif %s from %s", id_iff, self.ip) res = True snmp_res = netsnmp.snmpget('.1.3.6.1.4.1.8691.7.19.1.9.1.1.3.' + id_iff, Version=2, Community=self.snmp_rw_pass, DestHost=self.ip) if snmp_res[0] == 0: res = False # self.__tn = telnetlib.Telnet(self.ip) # try: # self.__tn.read_until("login as: ".encode(), 5) # except EOFError as e: # logger.error("%s: Execution failed: %s" % (self.ip, e)) # return # # try: # self.__tn.write(self.user.encode() + "\n".encode()) # except EOFError as e: # logger.error("%s: Execution failed: %s" % (self.ip, e)) # return # # if self.password: # try: # self.__tn.read_until("password: "******"%s: Execution failed: %s" % (self.ip, e)) # return # # try: # self.__tn.write(self.password.encode() + "\n\n".encode()) # except EOFError as e: # logger.error("%s: Execution failed: %s" % (self.ip, e)) # # request = "show interfaces ethernet %s\n" % (id_iff) # self.__tn.write(request.encode()) # self.__tn.write("exit\n".encode()) # str = self.__tn.read_all().decode() # if ("Enable" in str): # res = True # elif ("Disable" in str): # res = False # return res
def download_package(self, package_url, name, build_number): URI = settings.JENKINS.get('URI') download_url = '{}/job/{}/{}/artifact/{}'.format( URI, name, build_number, package_url) logger.debug(download_url) local_filename = download_url.split('/')[-1] code_path = os.path.join(settings.DEPLOY.get('CODE_PATH'), 'packages') local_full_filename = os.path.join(code_path, local_filename) # with requests.get(url, stream=True, # auth=HTTPBasicAuth("zhoujinliang", "117c911a35acf51e428e29f3ccb363f53f")) as r: with requests.get(download_url, stream=True) as r: r.raise_for_status() with open(local_full_filename, 'wb') as f: for chunk in r.iter_content(chunk_size=8192): if chunk: # filter out keep-alive new chunks f.write(chunk) return local_full_filename
def sgd(w0, x, y, f, grad, learning_rate=0.01, batch_size=100, max_epochs=1000, eps=1e-6, shuffle=False, rng=None): tm = Timer() n = x.shape[0] n_batches = get_num_batches(n, batch_size) w = np.copy(w0) epoch_losses = np.zeros(max_epochs, dtype=float) epoch = 0 w_best = np.copy(w0) loss_best = np.inf if n <= batch_size: shuffle = False # no need to shuffle since all instances will be used up in one batch if shuffle: shuffled_idxs = np.arange(n) if rng is None: np.random.shuffle(shuffled_idxs) else: rng.shuffle(shuffled_idxs) else: shuffled_idxs = None while epoch < max_epochs: losses = np.zeros(n_batches, dtype=float) for i in range(n_batches): xi, yi = get_sgd_batch(x, y, i, batch_size, shuffled_idxs=shuffled_idxs) if xi.shape[0] == 0: raise ValueError("Batch size of 0") g = grad(w, xi, yi) w -= learning_rate * g losses[i] = f(w, xi, yi) if False: g_norm = g.dot(g) if np.isnan(g_norm) or np.isinf(g_norm): logger.debug("|grad|=%f, i=%d/%d, epoch:%d" % (g.dot(g), i+1, n_batches, epoch)) logger.debug("|w0|=%f" % w0.dot(w0)) raise ArithmeticError("grad is nan/inf in sgd") loss = np.mean(losses) if np.isnan(loss): logger.debug("loss is nan") logger.debug("|w|=%f" % w.dot(w)) raise ArithmeticError("loss is nan in sgd") epoch_losses[epoch] = loss if loss < loss_best: # pocket algorithm np.copyto(w_best, w) loss_best = loss epoch += 1 if loss < eps: break debug_log_sgd_losses("sgd", epoch_losses, epoch, n=20, timer=tm) # logger.debug("epochs: %d" % epoch) # logger.debug("net losses:") # logger.debug("epoch losses:\n%s" % str(epoch_losses[0:epoch])) # logger.debug("best loss: %f" % loss_best) return w_best
def reject_key(self, key_id): data = { 'client': 'wheel', 'fun': 'key.reject', 'match': key_id, 'include_accepted': True, 'include_denied': True } try: req = self.post(data=data) logger.debug(req.json()) if req.status_code != 200: response = {'code': req.status_code, 'detail': '请求异常'} else: data = req.json() status = data['return'][0]['data']['success'] # salt-api 返回执行成功的成员,任意saltID都会返回status,最终结果还得看numbers numbers = data['return'][0]['data']['return'] if numbers and status: response = { 'code': 200, 'status': status, 'detail': '驳回成功' } else: if not status: response = { 'code': 200, 'status': status, 'detail': '驳回失败' } else: response = { 'code': 200, 'status': False, 'detail': '未知的minion' } except Exception as e: logger.error(e) response = {'code': -1, 'detail': 'salt api返回数据异常'} return response
def post(self, headers=None, data=None, json=None, prefix='/'): salt_token = cache.get('salt-token') logger.debug('{}:{}'.format(self.__token, self.__token_expire)) logger.debug(salt_token) if not headers and data: headers = { 'X-Auth-Token': self.__token, 'Accept': 'application/json', } elif not headers and json: headers = { 'X-Auth-Token': self.__token, 'Accept': 'application/json', 'Content-type': 'application/json' } url = '{}{}'.format(self.__url, prefix) # token过期,重新获取 if time.time() >= self.__token_expire: self.get_token() headers['X-Auth-Token'] = self.__token ret = requests.post(url, headers=headers, data=data, json=json, verify=False, timeout=self.__timeout) logger.debug(JSON.dumps(ret.json(), indent=4)) return ret
def get_token(self, prefix='/login'): ''' 登录获取token ''' data = { "username": self.__username, "password": self.__password, "eauth": "pam" } headers = {'Accept': 'application/json'} url = '{}{}'.format(self.__url, prefix) try: req = requests.post(url, headers=headers, data=data, verify=False, timeout=self.__timeout) logger.debug(JSON.dumps(req.json(), indent=4)) if req.status_code != 200: return {'code': req.status_code, 'detail': '请求异常'} req = req.json() self.__token = req['return'][0]['token'] self.__token_expire = req['return'][0]['expire'] start = req['return'][0]['start'] cache.set('salt-token', { 'token': self.__token, 'start': start, 'expire': self.__token_expire }, timeout=86400) except Exception as e: logger.critical(e) raise e
def sendMessageToMPClient(msg, device): global deviceConnectionToIPMap, outputs, messageForMPQueue for conn in deviceConnectionToIPMap.keys(): addr = deviceConnectionToIPMap[conn] d = getOrBuildMPClient(addr[0], addr[1]) if d.code: try: # 只发给对应用的MP if device.code != d.code: continue q = messageForMPQueue[d] #如果满了,则弹出一个 if q.full(): q.get_nowait() q.put_nowait(msg) if not conn in outputs: outputs.append(conn) logger.debug("MP outputs conn: " + str(len(outputs))) except: traceback.print_exc() logger.error("add MP msg QUEUE ERROR :" + binascii.b2a_hex(msg)) pass
def delete_key(self, key_id): data = {'client': 'wheel', 'fun': 'key.delete', 'match': key_id} try: req = self.post(data=data) logger.debug(req.json()) if req.status_code != 200: response = {'code': req.status_code, 'detail': '请求异常'} else: data = req.json() status = data['return'][0]['data']['success'] response = {'code': 200, 'status': status, 'detail': '删除key成功'} # delete key 无论key是否存在都返回空 if not status: response['detail'] = '删除key失败' except Exception as e: logger.error(e) response = {'code': -1, 'detail': '删除key失败'} return response
def deploy_state_sls(self): logger.debug('开始执行salt SLS') self.f.write('> 开始执行salt SLS\n') self.f.flush() salt_id_list = [s.saltID for s in self.order_obj.get_deploy_servers] rets = saltapi.state_sls( salt_id_list, **{ 'pillar': { 'project': self.order_obj.project.name, 'order_id': str(self.order_obj.id), 'env': self.order_obj.env.code, 'devops_env': settings.ENV, 'tag': self.order_obj.version, 'private_vars': self.order_obj.get_private_vars }, 'mods': self.order_obj.project.name, 'saltenv': 'deploy' }) self.f.write(json.dumps(rets, indent=4)) logger.debug('salt SLS 执行完成') self.f.write('\n> salt SLS 执行完成\n') self.f.flush() brief = self.__deal_with_salt_ret(rets) for salt_id in salt_id_list: if brief.get(salt_id, 'default') == 'default': brief[salt_id] = False failed_salt = [k for k, v in brief.items() if v == False] if failed_salt: raise Exception('{}执行salt sls失败'.format(','.join(failed_salt)))
def __new__(cls, **params): logger.debug("Create vc %s" % (params['name'])) inst = device.__new__(cls, **params) return inst
def sgdAdam(w0, x, y, f, grad, learning_rate=0.01, batch_size=100, max_epochs=1000, delta=1e-8, ro1=0.9, ro2=0.999, eps=1e-6, shuffle=False, rng=None): tm = Timer() n = x.shape[0] n_batches = get_num_batches(n, batch_size) w = np.copy(w0) s = np.zeros(len(w0), dtype=w0.dtype) # first moment variable s_hat = np.zeros(len(w0), dtype=w0.dtype) # first moment corrected for bias r = np.zeros(len(w0), dtype=w0.dtype) # second moment variable r_hat = np.zeros(len(w0), dtype=w0.dtype) # second moment corrected for bias t = 0 # time step epoch_losses = np.zeros(max_epochs, dtype=float) epoch = 0 w_best = np.copy(w0) loss_best = np.inf if n <= batch_size: # no need to shuffle since all instances will be used up in one batch shuffle = False if shuffle: shuffled_idxs = np.arange(n) if rng is None: np.random.shuffle(shuffled_idxs) else: rng.shuffle(shuffled_idxs) else: shuffled_idxs = None prev_loss = np.inf while epoch < max_epochs: losses = np.zeros(n_batches, dtype=float) for i in range(n_batches): xi, yi = get_sgd_batch(x, y, i, batch_size, shuffled_idxs=shuffled_idxs) g = grad(w, xi, yi) t += 1 s[:] = ro1 * s + (1 - ro1) * g r[:] = ro2 * r + (1 - ro2) * np.multiply(g, g) # correct bias in first moment s_hat[:] = (1./(1 - ro1 ** t)) * s # correct bias in second moment r_hat[:] = (1./(1 - ro2 ** t)) * r dw_scale = (learning_rate / (np.sqrt(delta + r_hat))) dw = np.multiply(dw_scale, s_hat) w[:] = w - dw losses[i] = f(w, xi, yi) loss = np.mean(losses) if np.isnan(loss): logger.debug("loss is nan") logger.debug("|w|=%f" % w.dot(w)) raise ArithmeticError("loss is nan in sgd") epoch_losses[epoch] = loss if loss < loss_best: # pocket algorithm np.copyto(w_best, w) loss_best = loss epoch += 1 if (loss < eps or np.abs(loss - prev_loss) < eps or avg_loss_check(epoch_losses, epoch, n=20, eps=eps)): break prev_loss = loss debug_log_sgd_losses("sgdAdam", epoch_losses, epoch, n=20, timer=tm) # logger.debug("epochs: %d" % epoch) # logger.debug("net losses:") # logger.debug("epoch losses:\n%s" % str(epoch_losses[0:epoch])) # logger.debug("best loss: %f" % loss_best) return w_best
def missionPlannerServer(): global inputs, outputs, MISSION_PLANNER_S, deviceConnectionToIPMap, messageForMPQueue, logger logger.info("Mission Planner Server start at port: " + str(MP_PORT)) # TCP 入口主程序,负责接收网络连接,然后分发到子线程处理 while True: # global MP_PORT try: readables, writables, exceptional = select.select( inputs, outputs, [], .5) except: if not (readables or writables or exceptional): break else: # 循环处理 可读 列表 for s in readables: if s is server: # 接收网络请求 connection, addr = s.accept() connection.setblocking(0) logger.info("MP connection from: " + addr[0] + ":" + str(addr[1])) inputs.append(connection) # 接受客户端连接请求 deviceConnectionToIPMap[connection] = addr d = getOrBuildMPClient(addr[0], addr[1], connection) messageForMPQueue[d] = Queue.Queue(QUEUE_SIZE) # 如果是客户端上传数据进来 else: host, port = deviceConnectionToIPMap[s] if deviceConnectionToIPMap.has_key(s): try: data = s.recv(BUF_SIZE) except socket.error, e: if 10035 == e.errno: continue # 如果对方强制关闭 else: closeConnect(host, port, s) except: continue else: # 如果读数据成功,则处理由客户端上传数据 if data: hexData = binascii.b2a_hex(data) logger.debug("data from MP " + host + ":" + str(port) + " " + hexData) # 获取连接设备 d = getOrBuildMPClient(addr[0], addr[1], s) d.counter += 1 # 第一次连接,第一个上行数据包为两个字节时 # 第一个字节,为指令,00则认为是关注哪个FC (此版忽略) # 第二个字节,为数据,当第一个字节为00时,则代码关注哪个设备 if 5 > d.counter and len(data) == 2: d.code = hexData[2:4] else: devicesServer.sendMessageToDevice( d.code, data) # 处理客户端上传的数据 # if s not in outputs: # outputs.append(s) # 如果为客户端关闭请求 else: closeConnect(host, port, s) pass
def __init__(self): service_control.__init__(self, 'aksusbd') logger.debug("Init HASP service")
def __new__(cls, **parameters): logger.debug("New complex thread test") inst = complex.__new__(cls, **parameters) return inst
def deviceServer(): global messageForDeviceQueue, outputs, logger logger.info("Device Server start at port: " + str(PORT)) # TCP 入口主程序,负责接收网络连接,然后分发到子线程处理 while True: try: #global PORT readables, writables, exceptional = select.select( inputs, outputs, [], .5) except: if not (readables or writables or exceptional): break else: # 循环处理 可读 列表 for s in readables: if s is server: # 接收网络请求 connection, addr = s.accept() connection.setblocking(0) logger.info("Device connection from: " + addr[0] + ":" + str(addr[1])) inputs.append(connection) deviceConnToHostMap[connection] = addr messageForDeviceQueue[(addr[0], addr[1])] = Queue.Queue(QUEUE_SIZE) getOrBuildDevice(addr[0], addr[1], connection) # 如果是客户端其它数据进来 else: host, port = deviceConnToHostMap[s] try: data = s.recv(BUF_SIZE) except socket.error, e: if 10035 == e.errno: continue # 如果对方强制关闭 else: logger.info("closing FC:" + host) closeConnection(host, port, s) except: continue else: # 0000000000000000000000000000000000000000000000000000000000000000 host, port = deviceConnToHostMap[s] # 如果读数据成功,则处理由客户端上传数据 if data: d = getOrBuildDevice(host, port, s) d.counter += 1 hexData = binascii.b2a_hex(data) # 如果上传数据大于,则取第四个字节做为设备编号 if "-1" == d.code and len( data) > 4 and "fe" == hexData[0:2]: d.code = hexData[6:8] logger.debug("get the FC code:" + d.code + " msg:" + hexData) # 处理设备上传的数据 # service.logServerService.receiveData(data, d) # 信息透传到 MissionPlanner missionPlannerServer.sendMessageToMPClient(data, d) logger.debug("data from FC " + host + " " + hexData) # if s not in outputs: # outputs.append(s) # 如果为客户端关闭请求 else: closeConnection(host, port, s)
def __init__(self, **parameters): self.__name = parameters['name'] self.__run_handler = self.handler self.__init_state = {} logger.debug("Init test \"%s\"", self.__name)
def save_init_state(self): logger.debug("Save init state test \"%s\"", self.__name) self._runned = True
def return_init_state(self): if self._runned == False: return logger.debug("Return init state test \"%s\"", self.__name) self._runned = False
def __init__(self, service_name): self.__service_name = service_name self.__state = Service_State.unknown logger.debug("Init service object")
else: closeConnection(host, port, s) # 循环处理 可写 处理列表 for s in writables: host, port = deviceConnToHostMap[s] try: # 如果有消息放在此连接的消息队列中,则进行下发 msg = messageForDeviceQueue[(host, port)].get_nowait() except Exception, e: if s in outputs: outputs.remove(s) pass else: # 111111111111111111111111111111111111111111111111111111111111111111111111 logger.debug("sending to FC " + host + " " + binascii.b2a_hex(msg)) try: s.sendall(msg) except: if s in outputs: outputs.remove(s) logger.error("sending to FC FAILED " + host + ":" + str(port) + " : " + binascii.b2a_hex(msg)) pass for s in exceptional: logger.error(" exception condition on " + s.getpeername()) if s in outputs: outputs.remove(s) inputs.remove(s) s.close()