def return_title_urls(self, if_test_programm='false'):
"""调用其它方法,返回所有的title网址
Parameters
----------
if_test_programm : str, optional
注意,这里只可填写 true 和 false
减少循环,方便对程序进行测试 (the default is False, which 不循环)
Returns
-------
list
返回所有的title网址
"""
url = GetConfig.get_config()['root_url']
log = LogginInfoOnlyStream()
temp_res = []
while url:
log.info("getting title url in: " + url)
soup = self.get_soup(url)
self.sleep_program.sleep(0)
temp_res.extend(self.get_title_urls(soup))
url = self.get_next_page(soup)
if if_test_programm == 'true':
break
return temp_res
@Author: xuruizeng
@Data: 2018/7/18
@File: token_candle_test.py
@Description: 检查K线数据
"""
import os
import random
import unittest
from common.log import MyLog
from common.mongodb import BcfMongo
from common.request_data import ConfigRequest
from get_config import GetConfig
config = GetConfig()
base_url = config.get_url('Base_Url')
test_url = config.get_url('token_candle')
log = MyLog.get_log()
logger = log.get_logger()
logger.info('start run {}'.format(__file__))
class TokenCandle(unittest.TestCase):
"""
测试K线接口
"""
def setUp(self):
logger.info('test "{}" start'.format(test_url.format('Bitcoin', '1d')))
import datetime
import queue
import os
import requests
from pic_file_handle import PicFileHandle
from random_sleep_time import RandomSleepTime
from get_title_urls import GetTitleUrls
from get_pic_info_in_title_pages import GetPicInfoInTitlePages
from logging_info import LogginInfo
from logging_info import LogginInfoOnlyStream
from get_config import GetConfig
NOW_DATE = datetime.datetime.strftime(datetime.datetime.now(), '%Y%m%d')
NOW_TIME = datetime.datetime.strftime(datetime.datetime.now(), '%Y%m%d_%H%M%S')
IF_TEST_PROGRAMM = GetConfig.get_config()['if_test_programm']
assert IF_TEST_PROGRAMM in ('true', 'false'), '请在config.ini中填写 true 或 false'
class DownloadPicByThreading(threading.Thread):
"""多线程下载
Parameters
----------
threading : thread
python多线程的包
"""
def __init__(self, que_, log_path, f_write_urls):
threading.Thread.__init__(self)
@Author: xuruizeng
@Data: 2018/7/18
@File: symbol_ranking_volume_test.py
@Description: 检查交易对按volume排行数据
"""
import os
import random
import unittest
from common.log import MyLog
from common.mongodb import BcfMongo
from common.request_data import ConfigRequest
from get_config import GetConfig
config = GetConfig()
base_url = config.get_url('Base_Url')
test_url = config.get_url('symbol_rank_volume')
log = MyLog.get_log()
logger = log.get_logger()
logger.info('start run {}'.format(__file__))
class SymbolRankingVolume(unittest.TestCase):
"""
测试交易对按volume接口
"""
def setUp(self):
logger.info('test "{}" start'.format(test_url))
# -*- coding: UTF-8 -*-
"""
@Version: 1.0
@Project:
@Author: Zhang Kaiming
@Data: 2018/7/20
@Description: 检查新闻接口
"""
import time, datetime
import unittest
from common.log import MyLog
from get_config import GetConfig
from common.request_data import ConfigRequest
config = GetConfig()
news_api = config.get_url("news")
log = MyLog.get_log()
logger = log.get_logger()
class GetNews(unittest.TestCase):
def setUp(self):
pass
def tearDown(self):
pass
def test_get(self):
"""
"""
@Version: 1.0
@Project: Monitor
@Author: long
@Data: 2018/7/27
@File: monitor_test.py
@Description: 交易对资金流向接口
"""
from unittest import TestCase
import requests
from get_config import GetConfig
import time
cfg = GetConfig()
url_base = cfg.get_value("Urls", "Base_Url")
api = cfg.get_value("Urls", "monitor")
class Monitor(TestCase):
"""脱壳关注"""
id1 = ""
def setUp(self):
pass
def tearDown(self):
pass
def check(self, mid="", limit=20):
global id1
url = url_base + api.format(mid, str(limit))
"""
@Version: 1.0
@Project: Home Page
@Author: Zhang Kaiming
@Data: 2018/7/20
@Description: 检查获取所有token的接口
"""
import random
import os
import unittest
from common.log import MyLog
from get_config import GetConfig
from common.request_data import ConfigRequest
config = GetConfig()
all_token_api = config.get_url("Get_Token_All")
log = MyLog.get_log()
logger = log.get_logger()
class GetAllToken(unittest.TestCase):
"获取token列表"
def setUp(self):
pass
def tearDown(self):
pass
@Author: xuruizeng
@Data: 2018/7/18
@File: symbol_list_test.py
@Description: 检查交易对列表数据
"""
import os
import random
import unittest
from common.log import MyLog
from common.mongodb import BcfMongo
from common.request_data import ConfigRequest
from get_config import GetConfig
config = GetConfig()
base_url = config.get_url('Base_Url')
test_url = config.get_url('symbol_list')
log = MyLog.get_log()
logger = log.get_logger()
logger.info('start run {}'.format(__file__))
class SymbolList(unittest.TestCase):
"""
测试交易对列表接口
"""
def setUp(self):
logger.info('test "{}" start'.format(test_url))
@Author: xuruizeng
@Data: 2018/7/18
@File: f10_links_test.py
@Description: 检查f10基本信息links部分
"""
import os
import random
import unittest
from common.log import MyLog
from common.mongodb import BcfMongo
from common.request_data import ConfigRequest
from get_config import GetConfig
config = GetConfig()
base_url = config.get_url('Base_Url')
test_url = config.get_url('f10_links')
log = MyLog.get_log()
logger = log.get_logger()
logger.info('start run {}'.format(__file__))
class F10Links(unittest.TestCase):
"""
测试f10基本信息接口
"""
def setUp(self):
logger.info('test "{}" start'.format(test_url.format('Bitcoin')))
# coding=gbk
"""
@Version: 1.0
@Project: Token_Grade_One
@Author: long
@Data: 2018/7/19
@File: Token_Grade_One.py
@Description: 获取推荐币体检详情
"""
from unittest import TestCase
import requests
from common.logger import logfile
from get_config import GetConfig
cfg = GetConfig()
url_base = cfg.get_value("Urls", "Base_Url")
api = cfg.get_value("Urls", "Token_Grade")
# mylog = logfile("Token_Grade_One.log", type=0)
class TokenGradeThree(TestCase):
"""获取3个优质币"""
def setUp(self):
pass
def tearDown(self):
pass
def test_get(self):
"""执行获取3个优质币接口"""
from email.mime import text, multipart, image
import smtplib
from get_config import GetConfig
import os, re
BASEDIR = os.path.dirname(os.path.dirname(__file__))
#
# email_info=configparser.ConfigParser()
# email_info.read(os.path.join(BASEDIR,'conf','data'),encoding='utf-8')
c = GetConfig()
class MYEMAIL:
'''
params
:smtp_addr
'''
def __init__(self):
self._smtp_addr = c.get_value('email', 'smtp_addr')
self._from_addr = c.get_value('email', 'from_addr')
self._emai_pwd = c.get_value('email', 'emai_pwd')
self._to_addr = c.get_value('email', 'to_addr').split(',')
self._s = smtplib.SMTP(self._smtp_addr)
self._s.login(self._from_addr, self._emai_pwd)
def email_text(self, text_info, subject=None, sendername=None):
#定义邮件内容msg
self.text_obj = text.MIMEText(_text=text_info)
self._email_conf(subject, sendername)
self._send_eamil()
def email_html(self, url_info, subject=None, sendername=None):
"""
@Version: 1.0
@Project: Symbol_Flow
@Author: long
@Data: 2018/7/19
@File: Symbol_Flow.py
@Description: 交易对资金流向接口
"""
from unittest import TestCase
import requests
from common.logger import logfile
from get_config import GetConfig
import time
cfg = GetConfig()
url_base = cfg.get_value("Urls", "Base_Url")
api = cfg.get_value("Urls", "Symbol_Flow")
# mylog = logfile("Symbol_Flow.log", type=0)
class GetSymbolFlow(TestCase):
"""交易对资金流向接口"""
def setUp(self):
pass
def tearDown(self):
pass
def check(self, market, symbol, cycle, ts):
url = url_base + api.format(market, symbol, cycle, ts)
try:
rep = requests.get(url, timeout=10).json()
#coding=gbk
"""
@Version: 1.0
@Project: Version
@Author: long
@Data: 2018/7/19
@File: Version.py
@Description: 获取版本信息
"""
from unittest import TestCase
import requests
from common.logger import logfile
from get_config import GetConfig
cfg = GetConfig()
url_base = cfg.get_value("Urls", "Base_Url")
api = cfg.get_value("Urls", "Version")
# mylog=logfile("Version.log",type=0)
class LapVersion(TestCase):
"""获取版本信息"""
def setUp(self):
pass
def tearDown(self):
pass
def test_get(self):
"""获取版本信息"""
url = url_base + api
@Data: 2018/7/18
@File: f10_intro_test.py
@Description: 检查f10基本信息intro部分
"""
import os
import random
import unittest
from common.log import MyLog
from common.mongodb import BcfMongo
from common.request_data import ConfigRequest
from get_config import GetConfig
config = GetConfig()
base_url = config.get_url('Base_Url')
test_url = config.get_url('f10_intro')
log = MyLog.get_log()
logger = log.get_logger()
logger.info('start run {}'.format(__file__))
class F10Intro(unittest.TestCase):
"""
测试f10基本信息接口
"""
def setUp(self):
logger.info('test "{}" start'.format(test_url.format('Bitcoin')))
"""
@Version: 1.0
@Project: telegram
@Author: long
@Data: 2018/7/20
@File: telegram_test.py
@Description: 获取telegram列表
"""
from unittest import TestCase
import requests
from common.logger import logfile
from get_config import GetConfig
import time
cfg = GetConfig()
url_base = cfg.get_value("Urls", "Base_Url")
api = cfg.get_value("Urls", "Telegram")
# mylog=logfile("Telegram.log",type=0)
class LapTelegram(TestCase):
"""获取telegram列表"""
def setUp(self):
pass
def tearDown(self):
pass
def check(self, token, lau, ts, size):
'执行telegram列表接口获取最新的N条（默认50）'
# coding=gbk
"""
@Version: 1.0
@Project: Refreshtoken
@Author: long
@Data: 2018/7/19
@File: Refreshtoken.py
@Description: 获取特定币体检详情
"""
from unittest import TestCase
import requests
from common.logger import logfile
from get_config import GetConfig
cfg = GetConfig()
url_base = cfg.get_value("Urls", "Base_Url")
login_api = cfg.get_value("Urls", "Login")
api = cfg.get_value("Urls", "refreshtoken")
# mylog = logfile("Refreshtoken.log", type=0)
mobile = cfg.get_value("Login", "Mobile")
valid = cfg.get_value("Login", "Valid")
class Refreshtoken(TestCase):
"""刷新用户的token和过期时间"""
def setUp(self):
'执行预置条件'
pass
def tearDown(self):
"""
@Version: 1.0
@Project: Message
@Author: long
@Data: 2018/7/19
@File: messasge_test.py
@Description: 获取新闻列表
"""
from unittest import TestCase
import requests
from common.logger import logfile
from get_config import GetConfig
import time
cfg = GetConfig()
url_base = cfg.get_value("Urls", "Base_Url")
api = cfg.get_value("Urls", "Message")
# mylog=logfile("Message.log",type=0)
class LapMessage(TestCase):
"""获取新闻列表"""
def setUp(self):
pass
def tearDown(self):
pass
def check(self, token, lau, ts, size):
'执行新闻列表接口获取最新的N条（默认50）'
class PicFileHandle():
"""处理此程序中关于路径的类
Returns
-------
None
处理此程序中关于路径的类
"""
platform = sys.platform
if platform == 'win32':
root_folder_flag = 'windows_root_folder_path'
elif platform == 'linux':
root_folder_flag = 'linux_root_folder_path'
else:
raise EnvironmentError('未知系统')
__root_folder_path = GetConfig.get_config()[root_folder_flag]
def __init__(self):
pass
@staticmethod
def replace_invalid_char(path):
r"""剔除windows路径或文件上的非法字符
规则: r'？|//*|\*|\?|"|<|>|\||\u3000'
如果路径为文件或者文件夹,则直接剔除
否则对路径进行截取前50个字符
Parameters
----------
path : str
文件或文件夹路径
Returns
-------
str
剔除后的路径
"""
re_compile = re.compile(r'？|//*|\*|\?|"|<|>|\||\u3000')
# print('path in replace_invalid_char is ' + path)
# return re.sub(re_compile, '_', path)
# 假定路径为文件夹或者文件
if os.path.isdir(path) or os.path.isfile(path):
return re.sub(re_compile, '_', path)
else:
return re.sub(re_compile, '_', path)[:50]
@staticmethod
def create_folder(path):
"""根据path创建文件夹
Parameters
----------
path : str
文件夹路径
"""
if not os.path.exists(PicFileHandle().replace_invalid_char(path)):
os.makedirs(path)
@staticmethod
def remove_file(path):
"""删除文件,防止文件重复
Parameters
----------
path : str
文件路径
"""
if os.path.isfile(PicFileHandle().replace_invalid_char(path)):
os.remove(path)
@staticmethod
def path_join(path, *paths):
"""同os.path.join,合并N个路径
20181118_193554
这个方法有很大的问题
比如,在如果传入的path本身就是一个路径,当调用replace_invalid_char()的时候
会把原有路径中的正反斜杠给替换掉
所以,正确的写法是直接调用os.path.join(),在需要的地方调用replace_invalid_char()
Parameters
----------
path : str
路径
Returns
-------
str
合并后的路径
"""
warnings.warn('此方法作废,因为造成正确的地址中的正反斜杠变为下划线')
fisrt_path = PicFileHandle.replace_invalid_char(path)
other_paths = []
for tmp_path in paths:
other_paths.append(PicFileHandle.replace_invalid_char(tmp_path))
return os.path.join(fisrt_path, *other_paths)
@staticmethod
def isfile(path):
"""判断path路径是否为文件
Parameters
----------
path : str
文件路径
Returns
-------
bool
true -> 是
flase -> 不是
"""
return os.path.isfile(path)
@staticmethod
def get_root_folder_path():
"""返回config中配置的根目录
Returns
-------
str
根目录路径
"""
# return GetConfig.get_config()['root_folder_path']
return PicFileHandle.__root_folder_path
@staticmethod
def get_pic_folder_path(pic_title):
"""返回存储pic的文件夹路径
Parameters
----------
pic_title : str
使用pic_title作为文件夹名称
Returns
-------
str
存储pic的文件夹路径
"""
# root_folder_path = PicFileHandle.get_root_folder_path()
pic_title = PicFileHandle.replace_invalid_char(pic_title)
pic_folder_path = os.path.join(PicFileHandle.__root_folder_path,
NOW_DATE, pic_title)
return pic_folder_path
@staticmethod
def get_pic_file_path_bak(url, prefix_name, pic_folder_path):
"""返回存储pic的路径,此方法作废
Parameters
----------
url : str
pic网址,用于获取文件类型
prefix_name : str
文件名称,一般为图片下方的说明
pic_folder_path : str
存储pic的文件夹路径
Returns
-------
str
返回存储pic的路径
Modify
-------
由于有些图片没有后缀,导致这个方法的suffix_name失效
而且有些gif图片格式后缀为jpg,导致下载到本地的时候变为静态图片
准确的方法是在获取图片内容时,使用r.headers['Content-Type']来判断图片类型
所以需要修改相应的方法来匹配r.headers['Content-Type']
此方法作废
"""
warnings.warn('此方法作废,因为可能无法正确获取图片的后缀')
suffix_name = '.' + url.rsplit('.', 1)[1]
pic_name = os.path.join(pic_folder_path, prefix_name + suffix_name)
return pic_name
@staticmethod
def get_logger_file_path():
"""返回log的路径
Returns
-------
str
log的路径
"""
# root_folder_path = PicFileHandle.get_root_folder_path()
pic_logger_path = os.path.join(PicFileHandle.__root_folder_path,
NOW_DATE, 'LOG',
'pic_log_%s.log' % NOW_TIME)
PicFileHandle.create_folder(os.path.split(pic_logger_path)[0])
return pic_logger_path
@staticmethod
def get_downloaded_urls_path():
"""返回downloaded_urls.txt的路径
Returns
-------
str
downloaded_urls.txt的路径
"""
return os.path.join(PicFileHandle.__root_folder_path,
'downloaded_urls.txt')
@staticmethod
def get_downloaded_urls():
"""获取已下载的pic地址
Returns
-------
set
已下载的pic地址
"""
st = set()
mid_path = PicFileHandle.get_downloaded_urls_path()
PicFileHandle.clear_downloaded_urls()
with open(mid_path) as f:
for i in f.readlines():
st.add(''.join(i.split()))
return st
@staticmethod
def clear_downloaded_urls():
"""重置(清空)downloaded_urls.txt
"""
mid_path = PicFileHandle.get_downloaded_urls_path()
if not PicFileHandle.isfile(mid_path):
with open(mid_path, 'w'):
pass
@staticmethod
def write_pic_explain(title_url, pic_explain, pic_folder_path):
"""将pic_explain的内容写入到pic根目录
Parameters
----------
title_url : str
正在爬取的pic网址,方便之后查找
pic_explain : str
title说明
pic_folder_path : str
pic的根目录
"""
mid_path = os.path.join(pic_folder_path, 'pic_explain.txt')
with open(mid_path, 'w', encoding='utf-8') as f:
f.write(title_url + '\n')
f.write(pic_explain)
"""
@Version: 1.0
@Project: Get_Token_All
@Author: long
@Data: 2018/7/19
@File: Get_Token_All.py
@Description: 获取token列表接口
"""
from unittest import TestCase
import requests
from common.logger import logfile
from get_config import GetConfig
cfg = GetConfig()
url_base = cfg.get_value("Urls", "Base_Url")
api = cfg.get_value("Urls", "Get_Token_All")
# mylog = logfile("Get_Token_All.log", type=0)
class GetAllToken(TestCase):
"""获取token列表"""
def setUp(self):
pass
def tearDown(self):
pass
def test_get(self):
"""执行获取token列表接口"""
url = url_base + api
try:
# -*- coding: UTF-8 -*-
"""
@Version: 1.0
@Project: My Points
@Author: Zhang Kaiming
@Data: 2018/7/23
@Description: 检查手机账户邀请二维码的生成
"""
import time, datetime
import unittest
from common.log import MyLog
from get_config import GetConfig
from common.request_data import ConfigRequest
config = GetConfig()
log = MyLog.get_log()
logger = log.get_logger()
class ExamineCode(unittest.TestCase):
def setUp(self):
pass
def tearDown(self):
pass
def test_examine(self):
"""
检查返回正确examine结果
@Author: xuruizeng
@Data: 2018/7/18
@File: message_news_test.py
@Description: 检查新闻数据
"""
import os
import random
import unittest
from common.log import MyLog
from common.mongodb import BcfMongo
from common.request_data import ConfigRequest
from get_config import GetConfig
config = GetConfig()
base_url = config.get_url('Base_Url')
test_url = config.get_url('message_news')
log = MyLog.get_log()
logger = log.get_logger()
logger.info('start run {}'.format(__file__))
class MessageNews(unittest.TestCase):
"""
测试新闻接口
"""
def setUp(self):
logger.info('test "{}" start'.format(test_url.format('Bitcoin')))
@Author: Kevin Chang
@Data: 2018/7/12
@File: login_logout_test.py
@Description: 检查是否可以登录退出
"""
import random
import os
import time
import unittest
from common.log import MyLog
from get_config import GetConfig
from common.request_data import ConfigRequest
config = GetConfig()
base_url = config.get_url("Base_Url")
log = MyLog.get_log()
logger = log.get_logger()
class LoginLogout(unittest.TestCase):
def setUp(self):
self.token = ""
print("Test start")
def tearDown(self):
print("Test end")
def test_login(self):
"""
"""
@Version: 1.0
@Project: twitter
@Author: long
@Data: 2018/7/20
@File: twitter_test.py
@Description: 获取twitter列表
"""
from unittest import TestCase
import requests
from common.logger import logfile
from get_config import GetConfig
import time
cfg = GetConfig()
url_base = cfg.get_value("Urls", "Base_Url")
api = cfg.get_value("Urls", "Twitter")
# mylog=logfile("twitter.log",type=0)
class LapTwitter(TestCase):
"获取twitter列表"
def setUp(self):
pass
def tearDown(self):
pass
def check(self, token, lau, ts, size):