def getOilFundData(self):
try:
html = requests.get(url=self.url).content.decode('utf-8')
except:
wxPusher = Helper.WxPusher()
wxPusher.sendMessage(title='发生未知错误！',
text='访问haoETF网站获取数据失败，请检查代码接口！')
raise
soup = BeautifulSoup(html, features='lxml')
#获取表头
thead = soup.body.table.thead.tr
thead_list = []
for th in thead.contents:
if th.string != '\n':
thead_list.append(th.string)
#通过判断数据类型去除不需要的隐藏字段
for th in thead_list:
#print(type(th))
if isinstance(th, bs4.element.Comment) == True:
thead_list.remove(th)
#获取表内容
tbody = soup.body.table.tbody
tr_list = []
for tr in tbody.find_all('tr'):
td_list = []
for td in tr.find_all('td'):
td_list.append(td.string)
tr_list.append(td_list)
#创建表
table = pd.DataFrame(tr_list, columns=thead_list)
#设定变量字段名
discount_rt = thead_list[2] #溢价率
volume = thead_list[6] #成交额
limit = thead_list[-1] #申购限额
#去除百分比%字符
table[discount_rt] = table[discount_rt].str.replace('%', '')
#将字符串转化为数字格式
table[discount_rt] = pd.to_numeric(table[discount_rt], errors='ignore')
table[volume] = pd.to_numeric(table[volume], errors='ignore')
#选取溢价率超过4%且成交额>500万且不开放申购的基金
table = table[(table[discount_rt] >= 4)
& (table[limit].str.contains('暂停') == False) &
(table[volume] > 500)].sort_values(discount_rt,
ascending=False)
#截取需要用到的字段
seleted = table.loc[:, ['代码', '名称', discount_rt, '现价', 'T-1估值']]
return seleted
def getDate(self,serverUrl):
try :
jsl = requests.get(url=serverUrl)
except :
wxPusher = Helper.WxPusher()
wxPusher.sendMessage(title = '发生未知错误！' , text = '访问集思录网站获取数据失败，请检查代码接口！')
raise
data = jsl.json()
#将json数据转换成列表
list = []
for row in data['rows']:
list.append(row['cell'])
#创建DataFrame
stocks = pd.DataFrame(data=list)
return stocks
import pandas as pd
import datetime
import pytz
from common import DataApi, Helper
#定义时区=东八区
tz = pytz.timezone('Asia/Shanghai')
#获取当前时间
now = datetime.datetime.now(tz)
#格式化日期时间
today = now.strftime('%Y%m%d')
#测试
#today = '20200306'
#仅在工作日执行后续操作
tradeDay = Helper.TradeDay()
if tradeDay.isHoliday(today) == False:
print('今天不是交易日哟~')
else:
#获取集思录数据
jslData = DataApi.JslData()
qdiiData = jslData.getQdiiData()
stockLofData = jslData.getStockLofData()
indexLofData = jslData.getIndexLofData()
#将三张表合并在一起
selected = pd.concat([qdiiData, stockLofData, indexLofData],
axis=0,
join='outer')
#获取haoETF原油数据
import config
from common import Helper
import requests
import json
import base64
from requests.utils import quote
env = Helper.get_env()
def diagnostics():
endpoint = config.hostnames[env] + 'diagnostics'
headers = {'content-Type': 'application/json'}
return requests.get(endpoint, headers=headers)
def edit_get(edit_by_option, payload):
endpoint = config.hostnames[env] + 'editGet/'
headers = {'content-Type': 'application/json'}
if edit_by_option == 'by_url':
url_encoded_url = quote(payload['url'], safe='')
endpoint = endpoint + 'editByUrl?' + payload['operation'] + '&url=' + url_encoded_url
elif edit_by_option == 'by_url_encoded_operation':
url_encoded_url = quote(payload['url'], safe='')
base64_encoded_operation = base64.b64encode(json.dumps(payload['operation']))
endpoint = endpoint + 'editByUrlEncodedOperation?' +\
'operation=' + base64_encoded_operation + '&url=' + url_encoded_url
elif edit_by_option == 'by_data':
url_encoded_image = quote(payload['image'])
endpoint = endpoint + 'editByData?' + payload['operation'] + '&image=' + url_encoded_image
def get_collection_manifest(cls, api_root, **query_parameters):
objects_query = None
manifest_query = None
version_range = None
added_after_range = None
size = int(query_parameters.get('limit'))
max_page_size = PAGE_SIZE
added_after = query_parameters.get('added_after')
sort_by = {'date_added': {'order': 'asc'}}
types = query_parameters.get('types')
ids = query_parameters.get('ids')
versions = query_parameters.get('versions')
spec_versions = query_parameters.get('spec_versions')
base_page = 0
next_id = 0
objects_query = None
manifest_query = None
version_range = None
added_after_range = None
log_debug(
f"Request to Get The objects Manifest of Collection: {query_parameters.get('collection_id')} "
f"in the Feed Root: {api_root}")
if query_parameters is None:
query_parameters = {}
try:
# Create a Query to filter Objects by collection id, types and spec_versions
objects_query = f"collection : {query_parameters.get('collection_id')}"
if types:
types = types.replace(",", " OR ")
objects_query = objects_query + f" AND type : ('{types}')"
if spec_versions:
spec_versions = spec_versions.replace(",", " OR ")
objects_query = objects_query + f" AND spec_version : ('{spec_versions}')"
objects_query_string = QueryString(query=objects_query,
default_operator="and")
# Create a Query to filter Manifest by collection id, object id's, versions and added after dates
manifest_query = f"collection : {query_parameters.get('collection_id')}"
if ids:
ids = ids.replace(",", " OR ")
manifest_query = manifest_query + f" AND id : ('{ids}')"
if added_after:
added_after_range = Range(
**{'date_added': {
'gt': f'{added_after}'
}})
manifests_query_string = QueryString(query=manifest_query,
default_operator="and")
# Get the intersect of both Objects and Manifest Queries
intersected_results = cls.es_client.manifest_intersect(
intersect_by='id',
objects_index=f'{api_root}-objects',
objects_query_string=objects_query_string,
manifests_index=f'{api_root}-manifest',
manifests_query_string=manifests_query_string,
added_after_range=added_after_range)
# Version and Paginate The Results
if intersected_results:
manifest_ids = ",".join(intersected_results).replace(
',', ' OR ')
query_string = QueryString(query=f"id:('{manifest_ids}')",
default_operator="AND")
pre_versioning_results = cls.es_client.scan(
index=f'{api_root}-manifest', query_string=query_string)
pre_pagination_results = Helper.fetch_objects_by_versions(
stix_objects=pre_versioning_results, versions=versions)
if -1 < size < max_page_size:
results = cls.es_client.search(
index=f'{api_root}-manifest',
query_string=query_string,
search_from=base_page,
size=size,
sort_by=sort_by)
else:
results = cls.es_client.search(
index=f'{api_root}-manifest',
query_string=query_string,
search_from=base_page,
size=max_page_size,
sort_by=sort_by)
results = {'objects': pre_pagination_results}
else:
results = {"objects": []}
return results
except Exception as e:
log_error(e)
if query_parameters.get('next'):
return EXCEPTIONS.get('NextNotFoundException', {})
else:
return EXCEPTIONS.get('CollectionNotFoundException', {})
def main():
helper = Helper()
config_helper = ConfigHelper()
config_helper.create_config()
# shutil.copy('mail.config', 'src/mail.config')
# REPLACE TOKENS IN FILES
tokens = {
"SUPER_SECRET_KEY": config_helper.get_encoded_secret(),
"HOME_DIRECTORY": parentdir
}
helper.prepare_file(snoopy_filename, tokens)
tokens = {"HOME_DIRECTORY": '{}/src'.format(parentdir)}
helper.prepare_file(snoopy_spec_filename, tokens)
# INSTALL PYTHON DEPENDENCIES
# COMPILE RELOADER -- needs to happen before we install snoopy
print("compiling reloader...")
subprocess.check_output(['gcc', 'src/darwin/create_launchd.c'])
shutil.copy('a.out', 'src/a.out')
# COMPILE PYTHON
helper.compile(snoopy_spec_filename)
# CLEANUP
helper.replace_original(snoopy_filename)
helper.replace_original(snoopy_spec_filename)
# INSTART
local_config_file = '{}/{}'.format(config_dir, plist_filename)
runtime_config_file = '{}/{}'.format(launchd_path, plist_filename)
print("starting daemon...")
copy_plist(local_config_file, runtime_config_file, parentdir)
load_plist(runtime_config_file)
print("finished!")
def main():
helper = Helper()
# install python dependencies
# pip install --upgrade google-api-python-client
tokens = {"HOME_DIRECTORY": '{}/src'.format(parentdir)}
helper.prepare_file(filename, tokens)
tokens = {"HOME_DIRECTORY": '{}/src'.format(parentdir)}
helper.prepare_file(spec_filename, tokens)
helper.compile(spec_filename)
subprocess.check_output(['rm', '-rf', 'collector/'])
subprocess.check_output(['mkdir', 'collector/'])
subprocess.check_output(['cp', 'installers/collector.sh', 'collector/'])
subprocess.check_output(
['cp', 'config/com.reedcwilson.collector.plist', 'collector/'])
subprocess.check_output(['cp', '-r', 'dist', 'build', 'collector/'])
os.chdir('collector/')
subprocess.check_output(['zip', '-r', 'collector.zip', '.'])
subprocess.check_output(['cp', 'collector.zip', '..'])
os.chdir('..')
subprocess.check_output([
'rm', '-rf', 'collector/', 'collector.sh',
'com.reedcwilson.collector.plist'
])
# CLEANUP
helper.replace_original(filename)
helper.replace_original(spec_filename)
print("finished!")
def main():
helper = Helper()
config_helper = ConfigHelper()
config_helper.create_config()
# REPLACE TOKENS IN FILES
tokens = {
"SUPER_SECRET_KEY": config_helper.get_encoded_secret(),
"OPERATING_USER": getpass.getuser(),
"HOME_DIRECTORY": parentdir
}
helper.prepare_file(snoopy_filename, tokens)
tokens = {"HOME_DIRECTORY": parentdir}
helper.prepare_file(sidecar_filename, tokens)
tokens = {"HOME_DIRECTORY": '{}/src'.format(parentdir)}
helper.prepare_file(snoopy_spec_filename, tokens)
tokens = {"HOME_DIRECTORY": '{}/src'.format(parentdir)}
helper.prepare_file(sidecar_spec_filename, tokens)
# INSTALL PYTHON DEPENDENCIES
# COMPILE
helper.compile(snoopy_spec_filename)
helper.compile(sidecar_spec_filename)
# CLEANUP
helper.replace_original(snoopy_filename)
helper.replace_original(sidecar_filename)
helper.replace_original(snoopy_spec_filename)
helper.replace_original(sidecar_spec_filename)
# INSTART
print("starting daemon...")
subprocess.check_output([start, snoopy_systemd_file, snoopy_systemd_name])
subprocess.check_output(
[start, sidecar_systemd_file, sidecar_systemd_name])
print("finished!")
import json
from flask import Flask, request, Response
from common import Helper
app = Flask(__name__)
hlp = Helper()
@app.route("/v1/get-html-content", methods=["GET"])
def get_html_content():
try:
if request.args:
if len(request.args) == 1:
fileName = request.args.get("fileName")
filePath = hlp.get_file_path(fileName)
if filePath:
allText = hlp.get_all_content(filePath)
return Response(allText["content"],
mimetype=allText["type"])
else:
return "File name not valid. Please enter valid file format file<1-4>.txt"
else:
fileName = request.args.get("fileName")
startLine = int(request.args.get("start"))
endLine = int(request.args.get("end"))
filePath = hlp.get_file_path(fileName)
if filePath:
specText = hlp.get_all_content(filePath, startLine,
endLine)
return Response(specText["content"],
mimetype=specText["type"])
def main():
cleanup(snoopy_service)
cleanup(sidecar_service)
home_directory = rootdir.replace('\\', '\\\\')
helper = Helper()
config_helper = ConfigHelper()
config_helper.create_config()
# REPLACE TOKENS
tokens = {
"SUPER_SECRET_KEY": config_helper.get_encoded_secret(),
"HOME_DIRECTORY": home_directory
}
helper.prepare_file(snoopy_filename, tokens)
tokens = {
"SUPER_SECRET_KEY": config_helper.get_encoded_secret(),
"HOME_DIRECTORY": home_directory
}
helper.prepare_file(sidecar_filename, tokens)
tokens = {"HOME_DIRECTORY": r'{}\src'.format(home_directory)}
helper.prepare_file(snoopy_spec_filename, tokens)
tokens = {"HOME_DIRECTORY": r'{}\src'.format(home_directory)}
helper.prepare_file(sidecar_spec_filename, tokens)
# INSTALL PYTHON DEPENDENCIES
# COMPILE PROGRAMS
helper.compile(snoopy_spec_filename)
helper.compile(sidecar_spec_filename)
# CLEANUP
helper.replace_original(snoopy_filename)
helper.replace_original(sidecar_filename)
helper.replace_original(snoopy_spec_filename)
helper.replace_original(sidecar_spec_filename)
# INSTART
install(snoopy_service, snoopy)
install(sidecar_service, sidecar)
time.sleep(2)
start(snoopy_service)
start(sidecar_service)
print("finished!")