def feature_about():
# 获取特征列表
feature_dict = NewsUtil.get_feature()
# 获取新闻中出现特征后最近的5个词及其属性
logger.info("In Prepare Raw News...")
raw_news_data = CommonUtil.read_excel(RAW_NEWS_DEMO_PATH)
raw_news_table = raw_news_data.sheet_by_index(0)
raw_news_rows = raw_news_table.nrows
segmentor = Segmentor() # 初始化实例
segmentor.load_with_lexicon(cws_model_path,
CFETSFX_LEXICON_PATH) # 加载模型，第二个参数是您的外部词典文件路径
feature_about_list = list()
for rowN in range(0, raw_news_rows):
news_content = raw_news_table.cell_value(rowN, 2)
sentences = SentenceSplitter.split(news_content)
for sentence in sentences:
print(sentence)
# 分词
words = segmentor.segment(sentence)
print(list(words))
for word_index in range(0, len(words)):
word = words[word_index]
for feature_word in feature_dict.values():
if feature_word in word:
about_list = list()
count = 0
while word_index < len(words) and count < 6:
about_list.append(words[word_index])
count += 1
word_index += 1
feature_about_list.append(about_list)
print(about_list)
break
segmentor.release()
CommonUtil.write_csv(FEATURE_ABOUT_PATH, feature_about_list)
def get_feature_value():
feature_vector_list = list()
feature_name_list = list()
for feature_name in FEATURE_NAME_LIST:
feature_name_list.append(feature_name)
feature_list = CommonUtil.read_csv(MarketDataCrawler.MARKET_DATA_PATH +
'/' + feature_name + '.csv')
feature_dict = dict()
for feature_i in range(1, len(feature_list)):
date = CommonUtil.get_datetime_from_string_(
feature_list[feature_i][0]).date()
# 开盘价
feature_value = float(feature_list[feature_i][1])
feature_dict[date] = feature_value
for value_key in valueDict.keys():
if value_key in feature_dict.keys():
feature_value = feature_dict[value_key]
else:
feature_value = 'N/A'
if value_key in featureValue.keys():
feature_items = featureValue[value_key]
else:
feature_items = list()
feature_items.append(feature_value)
featureValue[value_key] = feature_items
feature_name_list.append(VALUE_NAME)
feature_name_list.insert(0, 'DATE')
feature_vector_list.append(feature_name_list)
for key in featureValue.keys():
feature_items = featureValue[key]
feature_items.append(valueDict[key])
feature_items.insert(0, key)
feature_vector_list.append(feature_items)
CommonUtil.write_csv('../files/marketdata/FEATURE_VECTOR.csv',
feature_vector_list)
def news_sentiment():
logger.info("In News Sentiment...")
count = 1
for mapped_news in newsMappedList:
feature_vector_item = list()
news_index = mapped_news[0]
news_time = mapped_news[1]
feature_vector_item.append(news_index)
feature_vector_item.append(news_time)
feature_vector = list()
keyword_sentiment_dict = dict()
# 下标从0开始，减1
news_mapped = newsList[news_index - 1]
for mapped_news_index in range(2, len(mapped_news)):
keyword = mapped_news[mapped_news_index]
sentiment_result = BaiduNLPProcessor.sentiment_classify(
news_mapped[2])
keyword_sentiment_dict[keyword] = sentiment_result
keys = featureDict.keys()
for key in keys:
if featureDict[key] in keyword_sentiment_dict.keys():
feature_vector.append(keyword_sentiment_dict[featureDict[key]])
else:
feature_vector.append(0)
feature_vector_item.append(feature_vector)
newsItemList.append(feature_vector_item)
feature_vector.insert(0, news_time)
newsFeatureList.append(feature_vector)
logger.info(count)
count += 1
CommonUtil.write_csv(NEWS_ITEM_PATH, newsItemList)
CommonUtil.write_csv(NEWS_FEATURE_PATH, newsFeatureList)
logger.info("News Sentiment Done!")
def link_vec(NEWS_VEC, PRICE_VEC, p):
ALL_VEC = []
# print(NEWS_VEC[0][1])
# print(PRICE_VEC[0][0])
for each_price_vec in PRICE_VEC:
flag = 0
for each_news_vec in NEWS_VEC:
# print("each_news_vec[0] , each_price_vec[0]", each_news_vec[0] , each_price_vec[0])
if each_news_vec[0] == each_price_vec[0]:
print("matched!")
vec_of_all = each_news_vec[::]
vec_of_all.append(each_price_vec[1])
vec_of_all.append(each_price_vec[2])
ALL_VEC.append(vec_of_all)
flag = 1
break
if flag == 0:
print("not matched! Created!")
list_temp = []
list_temp.append(each_price_vec[0])
list_temp += [0] * (len(each_news_vec) - 1)
list_temp.append(each_price_vec[1])
list_temp.append(each_price_vec[2])
ALL_VEC.append(list_temp)
CommonUtil.write_csv(p, ALL_VEC)
def get_value_list():
value_list = CommonUtil.read_csv(MarketDataCrawler.MARKET_DATA_PATH + '/' +
VALUE_NAME + '.csv')
for value_i in range(1, len(value_list)):
date = CommonUtil.get_datetime_from_string_(
value_list[value_i][0]).date()
# 收盘价
value = float(value_list[value_i][2])
valueDict[date] = value
def get_news_item(start_date, end_date):
# 从最新到最远获取新闻
reverse_year = int(end_date[0:4])
reverse_month = int(end_date[4:6])
reverse_day = int(end_date[6:8])
reverse_pattern = (reverse_year, reverse_month, reverse_day, 23, 59, 59,
99, 99, 99)
reverse_cursor = CommonUtil.convert_date_to_long(reverse_pattern)
logger.info("reverseCursor is %s" % reverse_cursor)
finished_year = int(start_date[0:4])
finished_month = int(start_date[4:6])
finished_day = int(start_date[6:8])
finished_pattern = (finished_year, finished_month, finished_day, 0, 0, 0,
0, 0, 0)
finished_cursor = CommonUtil.convert_date_to_long(finished_pattern)
logger.info("finishedCursor is %s" % finished_cursor)
# 需要爬数据的网址
url_pattern = 'https://api-prod.wallstreetcn.com/apiv1/content/lives?' \
'channel=weex-channel,gold-channel,gold-forex-channel,' \
'forex-channel,goldc-channel,oil-channel&client=pc'
news_limit = 100
cursor = reverse_cursor
file_content = ''
# 页面计数器
page_num = 0
# 循环开始
while int(cursor) > int(finished_cursor):
page_num += 1
url = url_pattern + "&cursor=" + str(cursor) + "&" + str(news_limit)
logger.info(url)
page = requests.Session().get(url)
page.encoding = 'utf-8'
if page.status_code == 200:
data_all = json.loads(page.text)
res_data = data_all['data']
data_items = res_data['items']
cursor = res_data['next_cursor']
for item_i in range(len(data_items)):
display_time = data_items[item_i]['display_time']
context_text = data_items[item_i]['content_text']
context = context_text.strip().replace('\n', '')
context = context.replace('\r', '')
time = CommonUtil.convert_long_to_date(display_time)
file_content = file_content + time + "," + context + "\n"
# print(item_i+1, ": " , time, ", ", context_text)
CommonUtil.save_to_file(
'../files/wallstreetcn_%s_%s.csv' % (start_date, end_date),
file_content)
file_content = ''
# 无下一页数据时退出循环
if cursor == '':
break
logger.info("Finished With %s Pages Crawled." % page_num)
def news_segment():
logger.info("In Segment News...")
count = 1
for news_item in newsList:
word_list = BaiduNLPProcessor.lexer(news_item[2])
word_list.insert(0, news_item[0])
word_list.insert(1, CommonUtil.get_string_from_datetime(news_item[1]))
newsSegmentationList.append(word_list)
logger.info(count)
count += 1
CommonUtil.write_csv(SEGMENTED_NEWS_PATH, newsSegmentationList)
logger.info("Segment News...Done!")
def adjust_feature_vector():
feature_vector_list = CommonUtil.read_csv(
'../files/marketdata/FEATURE_VECTOR.csv')
pre_item = feature_vector_list[0]
current_item = pre_item
for vector_i in range(1, len(feature_vector_list)):
current_item = feature_vector_list[vector_i]
for i in range(1, len(current_item)):
if current_item[i] == 'N/A':
current_item[i] = pre_item[i]
feature_vector_list[vector_i] = current_item
pre_item = current_item
CommonUtil.write_csv('../files/marketdata/ADJUSTED_FEATURE_VECTOR.csv',
feature_vector_list)
def check_connection(self, url):
"""
Check connection status to the server currently selected by user and
show a status box indicating current operation.
:param url: The link of the server chose by user.This string could be
a domain name or the IP address of a server.
.. seealso:: :attr:`link` in
:meth:`~util.common.CommonUtil.check_connection`.
:type url: str
:return: A flag indicating connection status is good or not.
.. seealso:: :meth:`~util.common.CommonUtil.check_connection`. in
:class:`~util.common.CommonUtil` class.
:rtype: int
"""
print("Checking Server Status...")
conn = CommonUtil.check_connection(url)
if conn:
self.statusinfo[0][1] = "OK"
self.statusinfo[0][2] = "GREEN"
else:
self.statusinfo[0][1] = "Error"
self.statusinfo[0][2] = "RED"
return conn
def _delete_local_unused_branch(branch_name=None):
"""
批量删除本地无用分支
:param branch_name: 需要删除的branch，支持正则表达式
"""
if not branch_name:
print("Error: 请指定要删除的branchName，支持正则，例如：python3 git.py del '7.22|earning|feature'")
return
dirs_arr = CommonUtil.get_dirs(Constants.YY_ROOT_DIR, Constants.EXCLUDE_DIR)
print(dirs_arr)
print()
err_output = open(os.devnull, 'w') # 隐藏错误输出
for d in dirs_arr:
result = os.path.join(Constants.YY_ROOT_DIR, d)
os.chdir(result)
try:
result = subprocess.Popen("git branch | grep -E -i '%s' | xargs git branch -D" % branch_name,
shell=True, stdout=subprocess.PIPE, stderr=err_output)
print("%s: " % d)
while result.poll() is None:
line = result.stdout.readline().strip().decode("utf-8")
if line:
print(line, end='')
print()
print()
except Exception as e:
print("del exception = %s" % e, end='')
print()
def __init__(self):
"""
Initialize a new TUI session.
* Load server list from a configuration file under working directory.
* Try to load the hosts data file under working directory if it
exists.
.. note:: IF hosts data file does not exists correctly in current
working directory, a warning message box would popup. And
operations to change the hosts file on current system could be
done only until a new data file has been downloaded.
.. seealso:: :meth:`~tui.curses_d.CursesDaemon.session_daemon` method
in :class:`~tui.curses_d.CursesDaemon`.
.. seealso:: :meth:`~gui.hostsutil.HostsUtil.init_main` in
:class:`~gui.hostsutil.HostsUtil` class.
"""
super(HostsUtil, self).__init__()
# Set mirrors
self.settings[0][2] = CommonUtil.set_network("network.conf")
# Read data file and set function list
try:
self.set_platform()
RetrieveData.unpack()
RetrieveData.connect_db()
self.set_info()
self.set_func_list()
except IOError:
self.messagebox("No data file found! Press F6 to get data file "
"first.", 1)
except BadZipfile:
self.messagebox("Incorrect Data file! Press F6 to get a new data "
"file first.", 1)
def prepare_raw_news():
logger.info("In Prepare Raw News...")
raw_news_data = CommonUtil.read_excel(RAW_NEWS_PATH)
raw_news_table = raw_news_data.sheet_by_index(0)
raw_news_rows = raw_news_table.nrows
for rowN in range(0, raw_news_rows):
news_item = list()
news_index = int(raw_news_table.cell_value(rowN, 0))
news_time = CommonUtil.get_datetime_from_cell(
raw_news_table.cell_value(rowN, 1))
news_content = raw_news_table.cell_value(rowN, 2)
news_item.append(news_index)
news_item.append(news_time)
news_item.append(news_content)
newsList.append(news_item)
logger.info("Prepare Raw News...Done!")
def check_connection(self, url):
"""
Check connection status to the server currently selected by user and
show a status box indicating current operation.
:param url: The link of the server chose by user.This string could be
a domain name or the IP address of a server.
.. seealso:: :attr:`link` in
:meth:`~util.common.CommonUtil.check_connection`.
:type url: str
:return: A flag indicating connection status is good or not.
.. seealso:: :meth:`~util.common.CommonUtil.check_connection`. in
:class:`~util.common.CommonUtil` class.
:rtype: int
"""
self.messagebox("Checking Server Status...")
conn = CommonUtil.check_connection(url)
if conn:
self.statusinfo[0][1] = "OK"
self.statusinfo[0][2] = "GREEN"
else:
self.statusinfo[0][1] = "Error"
self.statusinfo[0][2] = "RED"
self.status()
return conn
def set_info(self):
"""
Set the information of the current local data file.
"""
info = RetrieveData.get_info()
build = info["Buildtime"]
self.hostsinfo["Version"] = info["Version"]
self.hostsinfo["Release"] = CommonUtil.timestamp_to_date(build)
def get_custom_conf_path(cls):
if not CommonUtil.check_platform()[0] == "Windows":
path = os.path.expanduser('~/.custom.hosts')
if not os.path.isfile(path):
path = os.path.expanduser('~/custom.hosts')
if os.path.isfile(path):
return path
return None
def reduce_feature_vector():
logger.info("In Reduce Feature Vector...")
prepare_feature()
origin_feature_num = len(featureDict.keys())
global featureVectorList
reduced_feature_vector_list = list()
feature_list = list()
feature_count_threshold = 2
file_path = FEATURE_VECTOR_PATH + '_' + str(
PRICE_SAMPLE_MINUTE) + CSV_FILE_SUFFIX
featureVectorList = CommonUtil.read_csv(file_path)
feature_count_dict = dict()
feature_count_list = [0] * origin_feature_num
is_title = True
for feature_vector in featureVectorList:
if is_title:
is_title = False
else:
for feature_value_index in range(0, origin_feature_num):
if feature_vector[feature_value_index] != '0.0':
feature_count_list[feature_value_index] += 1
feature_index = 0
for key in featureDict.keys():
feature_count = feature_count_list[feature_index]
feature_count_dict[key] = feature_count
if feature_count >= feature_count_threshold:
feature_list.append(feature_index)
feature_index += 1
logger.info(str('Reduce Feature Vector to: ' + str(len(feature_list))))
feature_list.append(origin_feature_num)
# 拼装计数超过阈值的特征向量
for feature_vector in featureVectorList:
reduced_feature_vector = list()
for feature_value_index in range(0, origin_feature_num + 1):
if feature_value_index in feature_list:
try:
reduced_feature_vector.append(
feature_vector[feature_value_index])
except IndexError:
logger.error(feature_vector)
logger.error(feature_value_index)
reduced_feature_vector_list.append(reduced_feature_vector)
file_path = REDUCED_FEATURE_VECTOR_PATH + '_' + str(
PRICE_SAMPLE_MINUTE) + CSV_FILE_SUFFIX
CommonUtil.write_csv(file_path, reduced_feature_vector_list)
logger.info("Reduce Feature Vector Done!")
def run(self):
"""
Start operations to check the network connection with a specified
server.
"""
self.trigger.emit(-1)
status = CommonUtil.check_connection(self.link)
self.trigger.emit(status)
def get_custom_conf_path(cls):
if not CommonUtil.check_platform()[0] == "Windows":
path = os.path.expanduser('~/.custom.hosts')
if not os.path.isfile(path):
path = os.path.expanduser('~/custom.hosts')
if os.path.isfile(path):
return path
return None
def process_bar(self, done, block, total, mode=1):
"""
Draw `Process Bar` at the bottom which is used to indicate progress of
downloading operation.
.. note:: This method is a callback function responses to
:meth:`urllib.urlretrieve` method while fetching hosts data
file.
:param done: Block count of packaged retrieved.
:type done: int
:param block: Block size of the data pack retrieved.
:type block: int
:param total: Total size of the hosts data file.
:type total: int
:param mode: A flag indicating the status of `Process Bar`.
The default value of `mode` is `1`.
==== ====================================
mode `Process Bar` status
==== ====================================
1 Downloading operation is processing.
0 Not downloading.
==== ====================================
:type mode: int
"""
screen = self._stdscr.subwin(2, 80, 20, 0)
screen.bkgd(' ', curses.color_pair(4))
normal = curses.A_NORMAL
line_width = 76
prog_len = line_width - 20
# Progress Bar
if mode:
done *= block
prog = 1.0 * prog_len * done / total
progress = ''.join(['=' * int(prog), '-' * int(2 * prog % 2)])
progress = progress.ljust(prog_len)
total = CommonUtil.convert_size(total).ljust(7)
done = CommonUtil.convert_size(done).rjust(7)
else:
progress = ' ' * prog_len
done = total = 'N/A'.center(7)
# Show Progress
prog_bar = "[%s] %s | %s" % (progress, done, total)
screen.addstr(1, 2, prog_bar, normal)
screen.refresh()
def set_info(self):
"""
Set the information of the current local data file.
"""
info = RetrieveData.get_info()
build = info["Buildtime"]
self.hostsinfo["Version"] = info["Version"]
self.hostsinfo["Release"] = CommonUtil.timestamp_to_date(build)
def set_info(cls):
"""
Set the information of the current local data file.
Modified from tui.hostsutil.HostsUtil.set_info()
"""
info = RetrieveData.get_info()
build = info["Buildtime"]
cls.hostsinfo["Version"] = info["Version"]
cls.hostsinfo["Release"] = CommonUtil.timestamp_to_date(build)
def set_progress(self, done, block, total):
"""
Send message to the main dialog to set the progress bar.
:param done: Block count of packaged retrieved.
:type done: int
:param block: Block size of the data pack retrieved.
:type block: int
:param total: Total size of the hosts data file.
:type total: int
"""
done = done * block
if total <= 0:
total = self.filesize
prog = 100 * done / total
done = CommonUtil.convert_size(done)
total = CommonUtil.convert_size(total)
text = unicode(_translate(
"Util", "Downloading: %s / %s", None)) % (done, total)
self.prog_trigger.emit(prog, text)
def set_progress(self, done, block, total):
"""
Send message to the main dialog to set the progress bar.
:param done: Block count of packaged retrieved.
:type done: int
:param block: Block size of the data pack retrieved.
:type block: int
:param total: Total size of the hosts data file.
:type total: int
"""
done = done * block
if total <= 0:
total = self.filesize
prog = 100 * done / total
done = CommonUtil.convert_size(done)
total = CommonUtil.convert_size(total)
text = unicode(_translate("Util", "Downloading: %s / %s",
None)) % (done, total)
self.prog_trigger.emit(prog, text)
def set_info(self):
"""
Set the information of the current local data file.
"""
info = RetrieveData.get_info()
ver = info["Version"]
self._cur_ver = ver
self.set_label_text(self.ui.labelVersionData, ver)
build = info["Buildtime"]
build = CommonUtil.timestamp_to_date(build)
self.set_label_text(self.ui.labelReleaseData, unicode(build))
def check_connection(cls):
"""
Check connection status to the servers.
Modified from tui.curses_d.CursesDaemon.check_connection()
"""
cls.show_status("Checking Server Status ...")
cls.settings[0][2] = CommonUtil.set_network(cls.netfile)
srv_id = 0
for srv_id,server in enumerate(cls.settings[0][2]):
cls.show_status("Trying to connect to "+server["tag"]+"...")
url = server["test_url"]
conn = CommonUtil.check_connection(url)
if conn :
cls.show_status("Connected!")
cls.settings[0][1] = srv_id
return
cls.show_status("Failed.")
cls.show_status("Can not connect to any server!")
cls.show_status("Script exits now...")
sys.exit(1)
def decay_influence(dt_news_time, dt_current_time):
delta_seconds = CommonUtil.get_interval_seconds(dt_current_time,
dt_news_time)
if delta_seconds > NEWS_INFLUENCE_DACAY_THRESHOLD * 60:
return 0
# 0-60秒
if delta_seconds <= NEWS_INFLUENCE_MOST * 60:
influence_score = 1 / (NEWS_INFLUENCE_MOST * 60) * delta_seconds
else:
influence_score = NEWS_INFLUENCE_DACAY_THRESHOLD/(NEWS_INFLUENCE_DACAY_THRESHOLD - NEWS_INFLUENCE_MOST) \
- 1/(NEWS_INFLUENCE_DACAY_THRESHOLD - NEWS_INFLUENCE_MOST) * delta_seconds / 60
return round(influence_score * FEATURE_VECTOR_SCALE,
CURRENCY_PAIR_PRECISION)
def set_platform(cls):
"""
Set the information about current operating system.
Modified from tui.hostsutil.HostsUtil.set_platform()
"""
system, hostname, path, encode, flag = CommonUtil.check_platform()
cls.platform = system
cls.hostname = hostname
cls.hosts_path = path
if encode == "win_ansi":
cls.sys_eol = "\r\n"
else:
cls.sys_eol = "\n"
def set_platform(self):
"""
Set the information of current operating system platform.
"""
system, hostname, path, encode, flag = CommonUtil.check_platform()
self.platform = system
self.hostname = hostname
self.hosts_path = path
self.plat_flag = flag
if encode == "win_ansi":
self.sys_eol = "\r\n"
else:
self.sys_eol = "\n"
def set_platform(self):
"""
Set the information of current operating system platform.
"""
system, hostname, path, encode, flag = CommonUtil.check_platform()
self.platform = system
self.hostname = hostname
self.hosts_path = path
self.plat_flag = flag
if encode == "win_ansi":
self.sys_eol = "\r\n"
else:
self.sys_eol = "\n"
def prepare_feature():
logger.info("In Prepare Feature...")
# 获取sheet
feature_data = CommonUtil.read_excel(FEATURE_PATH)
feature_table = feature_data.sheet_by_index(0)
# 获取总行数
feature_rows = feature_table.nrows
# 获取总列数
# feature_cols = feature_table.ncols
for rowNum in range(1, feature_rows):
key = feature_table.cell_value(rowNum, 0)
value = feature_table.cell_value(rowNum, 1)
featureDict[key] = value
logger.info("Prepare Feature...Done!")
def _checkout_branch(branch_name=None):
"""
批量检出指定分支，支持正则表达式
:param branch_name 分支名，不可为空
"""
if not branch_name:
print("请输入要切换的分支名称，支持正则")
return
dirs_arr = CommonUtil.get_dirs(Constants.YY_ROOT_DIR, Constants.EXCLUDE_DIR)
# dirs_arr = ['pluginlivebasemedia', 'ycloud', 'entmobile', 'entlive', 'livebasebiz']
for d in dirs_arr:
result = os.path.join(Constants.YY_ROOT_DIR, d)
os.chdir(result)
try:
print("%s: " % d)
# 先检查本地分支
try:
local_branch = subprocess.check_output('git branch | grep %s' % branch_name, shell=True).decode()
local_branch_arr = list(filter(None, local_branch.split("\n"))) # type: list
if len(local_branch_arr) == 1:
if '*' not in local_branch_arr[0]:
subprocess.call('git checkout %s' % local_branch_arr[0], shell=True) # 检出匹配到的本地分支
else:
print("当前分支 %s 已经是目标分支，无需切换" % local_branch_arr[0])
continue
elif len(local_branch_arr) > 1:
print("匹配到多个本地分支，请细化正则表达式后重试")
continue
except:
# 再检查远程分支
subprocess.check_output(['git', 'fetch'])
try:
remote_branch = subprocess.check_output('git branch -r | grep %s' % branch_name,
shell=True).decode()
remote_branch_arr = list(filter(None, remote_branch.split("\n"))) # type: list
if len(remote_branch_arr) == 1:
first_remote_branch = remote_branch_arr[0]
format_remote_branch = first_remote_branch.split("/")[-1]
subprocess.call('git checkout -b %s %s' % (format_remote_branch, first_remote_branch),
shell=True) # 检出匹配到的远程分支
continue
elif len(remote_branch_arr) > 1:
print("匹配到多个远程分支，请细化正则表达式后重试")
continue
except:
print("未匹配到任何分支，请检查正则表达式是否正确")
except Exception as e:
print(e, end='')
def check_writable(self):
"""
Check if current session has write privileges to the hosts file.
.. note:: IF current session does not has the write privileges to the
hosts file of current system, a warning message box would popup.
.. note:: ALL operation would change the `hosts` file on current
system could only be done while current session has write
privileges to the file.
"""
self._writable = CommonUtil.check_privileges()[1]
if not self._writable:
print("Please check if you have writing\n"
"privileges to the hosts file!")
def set_platform(self):
"""
Set the information about current operating system.
"""
system, hostname, path, encode, flag = CommonUtil.check_platform()
color = "GREEN" if flag else "RED"
self.platform = system
self.statusinfo[1][1] = system
self.hostname = hostname
self.hosts_path = path
self.statusinfo[1][2] = color
if encode == "win_ansi":
self.sys_eol = "\r\n"
else:
self.sys_eol = "\n"
def set_platform(self):
"""
Set the information about current operating system.
"""
system, hostname, path, encode, flag = CommonUtil.check_platform()
color = "GREEN" if flag else "RED"
self.platform = system
self.statusinfo[1][1] = system
self.hostname = hostname
self.hosts_path = path
self.statusinfo[1][2] = color
if encode == "win_ansi":
self.sys_eol = "\r\n"
else:
self.sys_eol = "\n"
def get_feature():
# 特征字典[AAAA:黄金]
feature_dict = dict()
# 获取sheet
feature_data = CommonUtil.read_excel(FEATURE_PATH)
feature_table = feature_data.sheet_by_index(0)
# 获取总行数
feature_rows = feature_table.nrows
# 获取总列数
# feature_cols = feature_table.ncols
for rowNum in range(1, feature_rows):
key = feature_table.cell_value(rowNum, 0)
value = feature_table.cell_value(rowNum, 1)
feature_dict[key] = value
return feature_dict
def get_market_data(i_type, i_count, s_data_type):
url_pattern = 'https://forexdata.wallstreetcn.com/kline?prod_code=' + s_data_type + \
'&candle_period=' + str(i_type) + \
'&fields=time_stamp,open_px,close_px,high_px,low_px,ma5,ma10,ma20,ma60,upper,mid,lower,diff,dea,' \
'macd,k,d,j,rsi6,rsi12,rsi24&data_count=' + str(i_count) # 需要爬数据的网址
logger.info(url_pattern)
page = requests.Session().get(url_pattern)
page.encoding = 'utf-8'
file_content = ''
if page.status_code == 200:
data_all = json.loads(page.text)
res_data = data_all['data']
candle_data = res_data['candle']
# 处理标题
data_fields = candle_data['fields']
for item_i in range(len(data_fields)):
file_content += data_fields[item_i] + ','
file_content += '\n'
# 处理数据
data_list = candle_data[s_data_type]
for item_i in range(len(data_list)):
data_items = data_list[item_i]
data_item = ''
for item_j in range(len(data_items)):
# 日期格式转换
if item_j == 0:
data_item += CommonUtil.convert_long_to_date(
data_items[item_j]) + ','
else:
data_item += str(data_items[item_j]) + ','
file_content += data_item + '\n'
logger.info("Finished With %s Items Crawled." % (len(data_list)))
CommonUtil.save_to_file(MARKET_DATA_PATH + '/%s.csv' % s_data_type,
file_content)
else:
logger.warning("Response Code is %s, Please Check!" % page.status_code)
def check_writable(self):
"""
Check if current session is ran with root privileges.
.. note:: IF current session does not has the write privileges to the
hosts file of current system, a warning message box would popup.
.. note:: ALL operation would change the `hosts` file on current
system could only be done while current session has write
privileges to the file.
"""
writable = CommonUtil.check_privileges()[1]
self._writable = writable
if not writable:
self.warning_permission()
def check_writable(self):
"""
Check if current session is ran with root privileges.
.. note:: IF current session does not has the write privileges to the
hosts file of current system, a warning message box would popup.
.. note:: ALL operation would change the `hosts` file on current
system could only be done while current session has write
privileges to the file.
"""
writable = CommonUtil.check_privileges()[1]
self._writable = writable
if not writable:
self.warning_permission()
def _list_branch():
dirs_arr = CommonUtil.get_dirs(Constants.YY_ROOT_DIR, Constants.EXCLUDE_DIR)
print(dirs_arr)
print()
for d in dirs_arr:
path = os.path.join(Constants.YY_ROOT_DIR, d)
print(path)
os.chdir(path)
try:
result = subprocess.check_output('git branch', shell=True).decode()
if result:
print("%s: " % d)
print(result, end='')
print()
except Exception as e:
print(e, end='')
def weixin_login(self):
url = "https://mp.weixin.qq.com/cgi-bin/bizlogin?action=startlogin"
params = {
'username': self.username,
'pwd': CommonUtil.md5(self.password),
'imgcode': '',
'f': 'json'
}
response = self.session.post(url,
data=params,
headers=self.headers,
verify=False)
if response.status_code == 200:
target = response.content.decode('utf-8')
print(target)
self.get_weixin_login_qrcode()
def check_writable(self):
"""
Check if current session has write privileges to the hosts file.
.. note:: IF current session does not has the write privileges to the
hosts file of current system, a warning message box would popup.
.. note:: ALL operation would change the `hosts` file on current
system could only be done while current session has write
privileges to the file.
"""
self._writable = CommonUtil.check_privileges()[1]
if not self._writable:
self.messagebox(
"Please check if you have writing\n"
"privileges to the hosts file!", 1)
exit()
def launch_tui(cls):
"""
Start a Text-based User Interface (TUI) session of
`Hosts Setup Utility`.
.. note:: This is a `classmethod`.
"""
# Set code page for Windows
system = CommonUtil.check_platform()[0]
cp = "437"
if system == "Windows":
chcp = os.popen("chcp")
cp = chcp.read().split()[-1]
chcp.close()
os.popen("chcp 437")
main = tui.HostsUtil()
main.start()
# Restore the default code page for Windows
if system == "Windows":
os.popen("chcp " + cp)
def init_main(self):
"""
Set up the elements on the main dialog. Check the environment of
current operating system and current session.
* Load server list from a configuration file under working directory.
* Try to load the hosts data file under working directory if it
exists.
.. note:: IF hosts data file does not exists correctly in current
working directory, a warning message box would popup. And
operations to change the hosts file on current system could be
done only until a new data file has been downloaded.
.. seealso:: Method :meth:`~tui.hostsutil.HostsUtil.__init__` in
:class:`~tui.hostsutil.HostsUtil` class.
"""
self.ui.SelectMirror.clear()
self.set_version()
# Set mirrors
self.mirrors = CommonUtil.set_network("network.conf")
self.set_mirrors()
# Read data file and set function list
try:
RetrieveData.unpack()
RetrieveData.connect_db()
self.set_func_list(1)
self.refresh_func_list()
self.set_info()
except IOError:
self.warning_no_datafile()
except BadZipfile:
self.warning_incorrect_datafile()
# Check if current session have root privileges
self.check_writable()
self.init_flag += 1
for name_format in excludes:
ex_files.extend(glob.glob(file_path(name_format)))
for name_format in includes:
files = glob.glob(file_path(name_format))
for src_file in files:
if src_file not in ex_files:
tar_path = os.path.join(prefix + VERSION,
src_file[rel_len:])
tar.add(src_file, tar_path)
print "compressing: %s" % src_file
tar.close()
exit(1)
from util import CommonUtil
system = CommonUtil.check_platform()[0]
if system == "Windows":
# Build binary executables for Windows
import struct
import zipfile
from distutils.core import setup
import py2exe
# Set working directories
WORK_DIR = SCRIPT_DIR + "work/"
DIR_NAME = "HostsTool"
DIST_DIR = WORK_DIR + DIR_NAME + '/'
WIN_OPTIONS = {
"includes": ["sip"],
"excludes": ["_scproxy", "_sysconfigdata"],
"dll_excludes": ["MSVCP90.dll"],
def messagebox(msg, mode=0):
"""
Draw a `Message Box` with :attr:`msg` in a specified type defined by
:attr:`mode`.
.. note:: This is a `static` method.
:param msg: The information to be displayed in message box.
:type msg: str
:param mode: A flag indicating the type of message box to be
displayed. The default value of `mode` is `0`.
==== ===========================================================
mode Message Box Type
==== ===========================================================
0 A simple message box showing a message without any buttons.
1 A message box with an `OK` button for user to confirm.
2 A message box with `OK` and `Cancel` buttons for user to
choose.
==== ===========================================================
:type mode: int
:return: A flag indicating the choice made by user.
====== =====================================================
Return Condition
====== =====================================================
0 #. No button is pressed while :attr:`mode` is `0`.
#. `OK` button pressed while :attr:`mode` is `1`.
#. `Cancel` button pressed while :attr:`mode` is `2`.
1 `OK` button pressed while :attr:`mode` is `2`.
====== =====================================================
:rtype: int
"""
pos_x = 24 if mode == 0 else 20
pos_y = 10
width = 30 if mode == 0 else 40
height = 2
messages = CommonUtil.cut_message(msg, width - 4)
height += len(messages)
if mode:
height += 2
# Draw Shadow
shadow = curses.newwin(height, width, pos_y + 1, pos_x + 1)
shadow.bkgd(' ', curses.color_pair(8))
shadow.refresh()
# Draw Subwindow
screen = curses.newwin(height, width, pos_y, pos_x)
screen.box()
screen.bkgd(' ', curses.color_pair(2))
screen.keypad(1)
# Set local variable
normal = curses.A_NORMAL
select = curses.A_REVERSE
# Insert messages
for i in range(len(messages)):
screen.addstr(1 + i, 2, messages[i].center(width - 4), normal)
if mode == 0:
screen.refresh()
else:
# Draw subwindow frame
line_height = 1 + len(messages)
screen.hline(line_height, 1, curses.ACS_HLINE, width - 2)
screen.addch(line_height, 0, curses.ACS_SSSB)
screen.addch(line_height, width - 1, curses.ACS_SBSS)
tab = 0
key_in = None
while key_in != 27:
if mode == 1:
choices = ["OK"]
elif mode == 2:
choices = ["OK", "Cancel"]
else:
return 0
for i, item in enumerate(choices):
item_str = ''.join(['[', item, ']'])
tab_pos_x = 6 + 20 * i if mode == 2 else 18
screen.addstr(line_height + 1, tab_pos_x, item_str,
select if i == tab else normal)
screen.refresh()
key_in = screen.getch()
if mode == 2:
# OK or Cancel
if key_in in [9, curses.KEY_LEFT, curses.KEY_RIGHT]:
tab = [1, 0][tab]
if key_in in [ord('a'), ord('c')]:
key_in -= (ord('a') - ord('A'))
if key_in in [ord('C'), ord('O')]:
return [ord('C'), ord('O')].index(key_in)
if key_in in [10, 32]:
return not tab
return 0