def client_send(self):
#hello = 10
hello = int(self.response)
#print "respone.....",type(self.response)
print "hello.....",type(hello)
#print hello
data = str(transnit_data(self.filename).data_collect())
data_do = transnit_data(self.filename).data_collect()
try:
print data
print "host is :",self.host
print "port is :",self.port
logging.info("Miontor agent started Successfully!")
while True:
self.do_working(data_do)
try:
host = self.host
port = self.port
s = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
s.connect((host,int(port)))
logging.info(("Miontor agent has been successfully connected to the server!!"))
s.sendall(data)
logging.debug(data)
buf = s.recv(8092)
if not len(data): break
s.close()
except:
logging.error('socket.error: [Errno 111] Connection refused')
continue
finally:
time.sleep(hello)
except:
logging.error('The socket connect to the server failed!!!')
def get_html_text(self, set_url=''):
url = ''
if set_url != '':
url = set_url
else:
url = self.url
try:
#self.response = urllib.request.urlopen(self.request, timeout=5)
#return self.response.read().decode(self.encoding)
req = requests.get(url, params=self.params, headers=headers)
if req.encoding == 'ISO-8859-1':
encodings = requests.utils.get_encodings_from_content(req.text)
if encodings:
encoding = encodings[0]
else:
encoding = req.apparent_encoding
else:
encoding = req.encoding
if req.encoding is None:
encoding = self.encoding
#print('use encoding: %s' % encoding)
#return req.content.decode(encoding, 'ignore').encode('utf-8', 'ignore')
#print(req.content.decode(encoding, 'ignore'))
return req.content.decode(encoding, 'ignore')
except (Exception, ConnectionError) as e:
log.error(e)
log.error('get rul %s fail, code : %d' %
(self.url, self.get_response_code()))
return ''
def url(self):
if not self.__uri:
log.error("url不能为空")
assert False, "url不能为空"
if not self.__uri.startswith("/"):
self.__uri = "/" + self.__uri
return self.__base_url + self.__uri
def update_stock_data_to_file(self, file_list=''):
#codes = self.stock_list
if file_list != '':
filename = file_list
else:
filename = 'all_list_filter.csv'
test_file = data_dir + 'sh.000001.csv'
if os.path.exists(test_file):
print('get test file')
dates = pd.read_csv(test_file, encoding='gbk')['date']
dates = list(dates)
last_date = dates[-1]
year = int(last_date.split('-')[0])
month = int(last_date.split('-')[1])
day = int(last_date.split('-')[2])
date = datetime.datetime(year, month,
day) + datetime.timedelta(days=1)
last_date = date.strftime('%Y-%m-%d')
else:
last_date = '2019-01-01'
now_date = datetime.datetime.now().strftime('%Y-%m-%d')
codes = list(pd.read_csv(filename, encoding='gbk')['code'])
print(last_date)
print(now_date)
if last_date == now_date:
print('data is update to new')
return None
df_list = sb.download_data(codes, last_date, now_date)
if os.path.exists(data_dir) is False:
print('create dir:' + data_dir)
os.mkdir(data_dir)
for df in df_list:
try:
if df.empty == True:
print('skip...')
continue
#base = dst_dir + '/data_csv/bs_data/'
base = data_dir
name = base + ('%s' % df['code'][0]) + '.csv'
if os.path.exists(name):
#print('append')
df.to_csv(name,
encoding="gbk",
mode='a',
index=False,
header=False)
else:
#print('create')
df.to_csv(name, encoding="gbk", index=False)
except (Exception, KeyError, IndexError) as e:
log.error(e)
log.error('write data fail, code:' + df['code'][1])
continue
def get_config(self, section, option):
if section not in self.__get_sections():
log.error("section：{} 在config.ini文件中不存在".format(section))
return None
if option not in self.__get_options(section):
log.error("option：{}在config.ini文件的section：{}中不存在".format(option, section))
return None
return config.get(section, option)
def closeConnection(conn):
try:
conn.close()
except:
print('Failed to close the connection.')
logging.error('Failed to close the connection.')
return 1
print('Connection is closed...')
logging.info('Connection is closed...')
def __init__(self, json_file):
if len(self.json_data_files) == 0:
self.get_all_json_file()
json_file = json_file.replace("\\", "/")
json_file = self.get_file_key(json_file)
# log.debug(json_file)
if json_file not in self.json_data_files:
log.error("json文件：{} 不存在".format(json_file))
assert False, "json文件：{} 不存在".format(json_file)
self.data = self.read_file(self.json_data_files[json_file])
def run(self):
while True:
try:
url, flag = self.queue.get(timeout=60)
log.info('downloading %s' % url)
self.download(url, flag)
time.sleep(1)
except Exception as e:
log.error(e)
break
def closeConnection(conn):
try:
conn.close()
except:
print('Failed to close the connection.')
logging.error('Failed to close the connection.')
return 1
print('Connection is closed...')
logging.info('Connection is closed...')
def getDomInfoByID(conn):
print('----get domain info by ID -----')
logging.debug('----get domain info by ID -----')
try:
#myDom = conn.lookupByID(id)
myDom = conn.listDomainsID()
return myDom
except:
print('Failed to find the domain with ID %s' % id)
logging.error('Failed to find the domain with ID %s' % id)
return 1
def createConnection():
conn = libvirt.open('qemu:///system')
if conn == None:
print('Failed to open connection to qemu:///system',file=sys.stderr)
logging.error('Failed to open connection to qemu:///system',file=sys.stderr)
exit(1)
else:
print('--Connection is created successfully--')
logging.info('--Connection is created successfully--')
return conn
def getDomInfoByName(conn):
print('----get domain info by name -----')
logging.debug('----get domain info by name -----')
try:
#myDom = conn.lookupByName(name)
myDom = conn.listDefinedDomains()
return myDom
except:
print('Failed to find the domain with name %s' % name)
logging.error('Failed to find the domain with name %s' % name)
return 1
def getDomInfoByID(conn):
print('----get domain info by ID -----')
logging.debug('----get domain info by ID -----')
try:
#myDom = conn.lookupByID(id)
myDom = conn.listDomainsID()
return myDom
except:
print('Failed to find the domain with ID %s' % id)
logging.error('Failed to find the domain with ID %s' % id)
return 1
def getDomInfoByName(conn):
print('----get domain info by name -----')
logging.debug('----get domain info by name -----')
try:
#myDom = conn.lookupByName(name)
myDom = conn.listDefinedDomains()
return myDom
except:
print('Failed to find the domain with name %s' % name)
logging.error('Failed to find the domain with name %s' % name)
return 1
def createConnection():
conn = libvirt.open('qemu:///system')
if conn == None:
print('Failed to open connection to qemu:///system', file=sys.stderr)
logging.error('Failed to open connection to qemu:///system',
file=sys.stderr)
exit(1)
else:
print('--Connection is created successfully--')
logging.info('--Connection is created successfully--')
return conn
def startDomaction(conn, name):
print('----Start domain info by Name -----')
logging.debug('----Start domain info by Name -----')
dom = conn.lookupByName(name)
try:
dom.create()
print('Dome %s boot sucessfully' % dom.name())
logging.info('Dome %s boot sucessfully' % dom.name())
return 0
except:
print('Dome %s boot failed' % dom.name())
logging.error('Dome %s boot failed' % dom.name())
return 1
def resetDom(conn,ID):
print('---Reset Domin status-----')
logging.info('---Reset Domin status-----')
dom = conn.lookupByID(ID)
try:
dom.reset()
print('Dom %s State %s' %(dom.name(),dom.info()[0]))
logging.info('Dom %s State %s' %(dom.name(),dom.info()[0]))
return 0
except:
print('Dom %s reset failed...' % dom.name())
logging.error('Dom %s reset failed...' % dom.name())
return 1
def startDomaction(conn,name):
print('----Start domain info by Name -----')
logging.debug('----Start domain info by Name -----')
dom = conn.lookupByName(name)
try:
dom.create()
print('Dome %s boot sucessfully' %dom.name())
logging.info('Dome %s boot sucessfully'%dom.name())
return 0
except:
print('Dome %s boot failed' %dom.name())
logging.error('Dome %s boot failed' %dom.name())
return 1
def shutdownDomaction(conn,ID):
print('----Shutdown domain info by ID -----')
logging.debug('----Shutdown domain info by ID -----')
dom = conn.lookupByID(ID)
try:
dom.destroy()
print('Dom %s State %s' %(dom.name(),dom.info()[0]))
logging.info('Dom %s State %s' %(dom.name(),dom.info()[0]))
return 0
except:
print('Dom %s shutdown failed...' % dom.name())
logging.error('Dom %s shutdown failed...' % dom.name())
return 1
def shutdownDomaction(conn, ID):
print('----Shutdown domain info by ID -----')
logging.debug('----Shutdown domain info by ID -----')
dom = conn.lookupByID(ID)
try:
dom.destroy()
print('Dom %s State %s' % (dom.name(), dom.info()[0]))
logging.info('Dom %s State %s' % (dom.name(), dom.info()[0]))
return 0
except:
print('Dom %s shutdown failed...' % dom.name())
logging.error('Dom %s shutdown failed...' % dom.name())
return 1
def resetDom(conn, ID):
print('---Reset Domin status-----')
logging.info('---Reset Domin status-----')
dom = conn.lookupByID(ID)
try:
dom.reset()
print('Dom %s State %s' % (dom.name(), dom.info()[0]))
logging.info('Dom %s State %s' % (dom.name(), dom.info()[0]))
return 0
except:
print('Dom %s reset failed...' % dom.name())
logging.error('Dom %s reset failed...' % dom.name())
return 1
def open_organisation(organisation, password=None):
conf = configuration.configuration()
try:
with open(os.path.join(conf.root_dir, organisation), 'rb') as openFile:
data = pickle.load(openFile)
if password:
orgdata = openssl.decrypt(conf, data, password)
else:
orgdata = data
return pickle.loads(orgdata)
except:
out = 'Cannot open organisation %s'%(os.path.join(conf.root_dir,organisation))
log.error(out)
raise OrganisationError(out)
def get_db_config(self, app):
"""
获取应用数据库连接配置信息
:param app: config.ini文件中应用名
:return:
"""
app = "{}_{}_db".format(self.env, app)
if app not in self.__get_sections():
log.error("db配置：{} 在config.ini文件中不存在".format(app))
return None
db = self.__get_items(app)
if "port" in db and isinstance(db["port"], str):
db["port"] = int(db["port"])
return db
def download(self, url, flag):
resp = requests.get(url, headers={
'User-Agent': ua.get_ua(),
'Cookie': auth.auth()}, proxies=proxy.get_proxies()[0]
)
if resp.status_code == 200:
log.info('GET %s 200 OK' % url)
html = to_html(resp.content)
if flag:
self.parse(html)
else:
self.parse_detail(html)
self.filter_urls.append(url)
else:
log.error('Error %s %s ' % (url, resp.status_code))
def files(self):
if not self.__files:
return None
if isinstance(self.__files, dict):
root_path = ParseConfig().get_root_path()
for k, v in self.__files.items():
file_path = os.path.join(root_path, v)
if os.path.isfile(file_path):
self.__files[k] = open(file_path, 'rb')
else:
log.error("文件：{}不存在".format(file_path))
assert False, "文件：{}不存在".format(file_path)
else:
log.error('files字段需要字典格式，示例：{"file":"test_case\\data\\name.xls"}')
assert False, 'files字段需要字典格式，示例：{"file":"test_case\\data\\name.xls"}'
return self.__files
def save(self, password=None):
try:
orgdata = pickle.dumps(self)
if password:
data = openssl.encrypt(self.conf, orgdata, password)
else:
data = orgdata
except:
out = 'Cannot save Organisation in %s'%(os.path.join(self.conf.root_dir,self.name))
log.error(out)
raise OrganisationError(out)
try:
with open(os.path.join(self.conf.root_dir, self.name), 'wb') \
as dump_file:
pickle.dump(data, dump_file)
except IOError:
out = 'Cannot open %s to save Organisation'%(os.path.join(self.conf.root_dir,self.name))
log.error(out)
raise OrganisationError(out)
except Exception:
out = 'Cannot save Organisation in %s'%(os.path.join(self.conf.root_dir,self.name))
dump_file.close()
log.error(out)
raise OrganisationError(out)
def checkDom(conn,name,ip):
print('---check Domin status-----')
logging.info('---check Domin status-----')
def check_ip(ip):
cmd = 'fping %s' %ip
ping_result = shell.shell_cmd(cmd)[0]
if 'alive' in ping_result:
return 'active'
else:
return 'inactive'
dom = conn.lookupByName(name)
try:
if (dom.state() == [1,1]) and (check_ip(ip) == 'active'):
return 'fluent'
elif (dom.state() == [1,1]) or (check_ip(ip) == 'inactive'):
return 'active'
else:
return 'inactive'
except:
print('Dom %s check failed...' %dom.name())
logging.error('Dom %s check failed...' %dom.name())
def checkDom(conn, name, ip):
print('---check Domin status-----')
logging.info('---check Domin status-----')
def check_ip(ip):
cmd = 'fping %s' % ip
ping_result = shell.shell_cmd(cmd)[0]
if 'alive' in ping_result:
return 'active'
else:
return 'inactive'
dom = conn.lookupByName(name)
try:
if (dom.state() == [1, 1]) and (check_ip(ip) == 'active'):
return 'fluent'
elif (dom.state() == [1, 1]) or (check_ip(ip) == 'inactive'):
return 'active'
else:
return 'inactive'
except:
print('Dom %s check failed...' % dom.name())
logging.error('Dom %s check failed...' % dom.name())
def get_response_code(self):
if self.response is not None:
return self.response.getcode()
else:
log.error('not get response')
return 404
parser.add_argument(
'--output-directory',
action='store',
help='Specify the directory where files will be written')
options = parser.parse_args()
# This dict stores the component and the previous version to track in case of rollback
previous_component_version_dict = {}
proposed_component_version_dict = {}
if options.output_directory is not None:
dump_files_here = options.output_directory
else:
dump_files_here = "./"
if options.blue_print is not None:
if "_env" not in options.blue_print[0]:
log.error(
"Unexpected input for blueprint. Expected blueprint with _env")
parser.print_help()
sys.exit(1)
list_of_components = options.component_name + options.blue_print
else:
list_of_components = options.component_name
backup_location = "%s.backup" % options.config_file
shutil.copyfile(options.config_file, backup_location)
log.info("physical file backed up to: %s" % backup_location)
env_file_section_to_find_env_name = 'environment'
variable_name_of_environment_name = 'ENV_NAME'
config = ConfigParser.RawConfigParser()
config.optionxform = str # make keys case sensitive
config.read(options.config_file)
def server_receive(self):
ip = self.host
port = self.port
s = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1)
s.bind((ip,int(port)))
s.listen(9)
check_file = '/root/script/monitoring_kvm/tools/check_client_conn'
try:
while True:
####################################
# Process the connection
try:
clientconn,clientaddr=s.accept()
except (KeyboardInterrupt, SystemExit):
raise
logging.error("you have CTRL+C,Now quit")
except:
traceback.print_exc()
logging.error(traceback.print_exc())
try:
print "Got connection from %s",clientconn.getpeername()
logging.info("Got connection from %s", clientconn.getpeername())
with open(check_file,'w') as f:
f.writelines('1')
while True:
remote_data=clientconn.recv(8094)
if not len(remote_data):
clientconn.send('welcome',remote_data)
logging.info(("The client data has been received and connection will be disconected!"))
break
clientconn.sendall(remote_data)
recv_data = eval(remote_data)
remote_data += remote_data
print "Data is :",remote_data
logging.debug(remote_data)#####
self.do_working(recv_data)
except (KeyboardInterrupt, SystemExit):
raise
logging.error("you have CTRL+C,Now quit")
except:
traceback.print_exc()
logging.error(traceback.print_exc())
# Close the connection
try:
clientconn.close()
except KeyboardInterrupt:
raise
logging.error("you have CTRL+C,Now quit")
except:
traceback.print_exc()
logging.error(traceback.print_exc())
except (KeyboardInterrupt, SystemExit):
print "you have CTRL+C,Now quit"
raise
logging.error(traceback.print_exc())
except:
traceback.print_exc()
logging.error(traceback.print_exc())
finally:
s.close()
bs.logout()
#print(queue)
#print(data_df_list)
return data_df_list
if __name__ == '__main__':
# 获取指定日期全部股票的日K线数据
#df = all_stock_list()
#df.to_csv('./all_list.csv', encoding="gbk", index=False)
#exit()
#codes = list(pd.read_csv('all_list_filter.csv', encoding='gbk')['code'])
codes = list(pd.read_csv('test.csv', encoding='gbk')['code'])
df_list = download_data(codes, '2020-07-18', '2020-07-21')
for df in df_list:
try:
base = './bs_data/'
name = base + ('%s' % df['code'][1]) + '.csv'
if os.path.exists(name):
df.to_csv(name,
encoding="gbk",
mode='a',
index=False,
header=False)
else:
df.to_csv(name, encoding="gbk", index=False)
except (Exception, KeyError, IndexError) as e:
log.error(e)
log.error('write data fail, code:' + df['code'][1])
continue
os.environ['ENV_FILE'] = options.config_file
user = common.get_stack_user()
gateway_ip = common.get_gateway_ip()
env_options = common.get_env_options()
gateway_session = ssh.SSHSession(gateway_ip, user)
admin01_session = gateway_session.get_remote_session('admin01')
error_encountered = []
if options.diff_file:
diff_file_data = open(options.diff_file).read()
for component in json.loads(diff_file_data):
pod_status_dict = {}
deployment_version_dict = {}
container_status_dict = {}
error_encountered.append(inspect_load_state(component, admin01_session, pod_status_dict,
deployment_version_dict, container_status_dict))
if options.component_name:
pod_status_dict = {}
deployment_version_dict = {}
container_status_dict = {}
error_encountered.append(inspect_load_state(options.component_name, admin01_session, pod_status_dict,
deployment_version_dict, container_status_dict))
for exits in error_encountered:
if exits == True:
number_of_errors = error_encountered.count(True)
log.error("\n%s%s components had errors during this validation check\n" % (textColours.FAIL,
number_of_errors))
sys.exit(1)
else:
print("Validate has completed")
parser.add_argument('--artifactory', action='store_true', help='Attempt to retrieve build information from '
'Artifactory (major version, blueprint version etc)')
parser.add_argument('--major-version', action='append', help='The major version of a blueprint')
parser.add_argument('--output-directory', action='store', help='Specify the directory where files will be written')
options = parser.parse_args()
# This dict stores the component and the previous version to track in case of rollback
previous_component_version_dict = {}
proposed_component_version_dict = {}
if options.output_directory is not None:
dump_files_here = options.output_directory
else:
dump_files_here = "./"
if options.blue_print is not None:
if "_env" not in options.blue_print[0]:
log.error("Unexpected input for blueprint. Expected blueprint with _env")
parser.print_help()
sys.exit(1)
list_of_components = options.component_name + options.blue_print
else:
list_of_components = options.component_name
backup_location = "%s.backup" % options.config_file
shutil.copyfile(options.config_file, backup_location)
log.info("physical file backed up to: %s" % backup_location)
env_file_section_to_find_env_name = 'environment'
variable_name_of_environment_name = 'ENV_NAME'
config = ConfigParser.RawConfigParser()
config.optionxform = str # make keys case sensitive
config.read(options.config_file)
