# -*- coding: utf-8 -*-
import re
from pathlib import Path
from datetime import datetime, timedelta
import pandas as pd
from src.data import conn
from src.data import DATE_PATTERN, INSTRUMENT_TYPE, RAW_HQ_DIR, BACKUP_DIR
from src.data import get_future_hq
from log import LogHandler
log = LogHandler('future.log')
# 期货历史交易日历，返回铜期货指数交易日期，2000/1/4开始
def get_future_calender(start=None, end=None, country='cn'):
"""
:param start:
:param end:
:param country: 默认是中国市场('cn')，目前仅支持中国市场
:return: pd.Index
"""
# if start_date > datetime.today():
# return pd.DatetimeIndex(freq='1D')
df = get_future_hq('cuL9', start=start, end=end)
if df is None:
df = get_future_hq('cuL9', start=start)
return df.index
# -*- coding: utf-8 -*-
import re
from datetime import datetime
import pandas as pd
from pymongo import ASCENDING, DESCENDING
from src.api import conn
from src.util import connect_mongo
from src.api.cons import FREQ
from src.setting import DATA_ANALYST, ANALYST_PWD
from log import LogHandler
log = LogHandler('api.log')
def get_trading_dates(start_date, end_date, country='cn'):
"""
:param start_date: datetime.datetime, 开始日期
:param end_date: datetime.datetime, 结束日期
:param country: 默认是中国市场('cn')，目前仅支持中国市场
:return: datetime.datetime list - 交易日期列表
"""
pass
def get_previous_trading_date(date, n=1, country='cn'):
"""
:param date: datetime.datetime 指定日期
# -*- coding: utf-8 -*-
import pandas as pd
from pymongo import MongoClient
from src.setting import MONGODB_URI, MONGODB_PORT, DATA_COLLECTOR, COLLECTOR_PWD, DATA_ANALYST, ANALYST_PWD
from log import LogHandler
log = LogHandler('data.log')
# client = MongoClient("mongodb+srv://unistar:<password>@cluster0-y9smy.mongodb.net/test?retryWrites=true")
# api = client.test
def connect_mongo(db,
username=DATA_COLLECTOR,
password=COLLECTOR_PWD,
host=MONGODB_URI,
port=MONGODB_PORT):
""" A util for making a connection to mongo """
if username and password:
mongo_uri = 'mongodb://{}:{}@{}:{}/{}'.format(username, password, host,
port, db)
conn = MongoClient(mongo_uri)
else:
conn = MongoClient(host, port)
return conn[db]
def read_mongo(database,
# parse_blocks.py - usage:
# tp = TextgridBlocksParser();
# tp.read(src_file);
# intervals = tp.parse_blocks();
# process intervals
# class to parse intevals in textgrid, generates a list containing all inforamtion
# author: xiao yang <*****@*****.**>
# date: 2016.Mar.09
import re
import chardet
from itertools import cycle
from log import LogHandler
logger = LogHandler('textgrid_parser.log', stdout=True)
class UpperLayerException(Exception):
pass
class CycleIterator(object):
""" a wrapper for the itertools.cycle """
def __init__(self, iterable):
super(CycleIterator, self).__init__()
self.iterable = iterable
self.iterator = cycle(iterable)
self.value = None
def head(self):
# -*- coding: utf-8 -*-
import requests
from lxml import etree
from log import LogHandler
log = LogHandler('util.crawler.log')
HEADERS = {
'Connection': 'keep-alive',
'Cache-Control': 'max-age=0',
'Upgrade-Insecure-Requests': '1',
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko)',
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'zh-CN,zh;q=0.8',
}
def get_html_text(url, headers=HEADERS, encoding=None):
try:
response = requests.get(url, headers=headers, timeout=30)
response.raise_for_status()
if encoding is None:
response.encoding = response.apparent_encoding
else:
response.encoding = encoding
return response.text
except:
log.info('{} is {}'.format(url, response.status_code))
# -*- coding: utf-8 -*-
import datetime
import json
import time
import numpy as np
import pandas as pd
from log import LogHandler
from src.util import get_html_tree, get_html_text, convert_percent
from src.data.setting import RAW_DATA_DIR # 不能使用相对路径 单文件运行时 import 错误
from src.data.jsl.setting import *
log = LogHandler('jsl.log')
def is_not_trading():
begin_time = datetime.time(9, 30)
end_noon = datetime.time(11, 30)
begin_noon = datetime.time(13)
end_time = datetime.time(15)
current_weekday = datetime.datetime.weekday(datetime.datetime.now())
current_time = datetime.datetime.time(datetime.datetime.now())
return current_weekday in [0, 6] \
or not (current_time in [begin_time, end_noon] or current_time in [begin_noon, end_time])
def get_convertible_list():
file_path = RAW_DATA_DIR / "convertible_list.csv"
file_name = str(file_path)
# -*- coding: utf-8 -*-
import datetime as dt
from log import LogHandler
from src.data.tdx.hq import get_future_day_hq, get_future_min_hq
from src.data.tdx.basic import get_future_basic
log = LogHandler('tdx.log')
# TODO 周期转换的处理， 日内的数据由于有夜盘，处理起来较复杂
# tdx处理是夜盘数据算作下一个交易日，resample函数处理起来会按照时间排序
# conversion = {'open': 'first', 'high': 'max', 'low': 'min',
# 'close': 'last', 'openInt' : 'last', 'volume': 'sum',
# 'comment': 'last'}
# m30 = df.resample('30min', closed='right', label='right).apply(conversion).dropna()
def get_future_hq(code, start=dt.datetime(1970, 1, 1), end=None, freq='d'):
"""
根据contractid找到对应的market，调用对应的行情函数
:param code: IL8 主力合约 IL9 期货指数
:param start: 开始时间
:param end: 结束时间
:param freq: 周期'1m'，'5m'
:return:
"""
future_basic_info = get_future_basic()
from openpyxl.styles import Alignment
from openpyxl.utils.dataframe import dataframe_to_rows
from openpyxl.worksheet.table import Table
from pymongo import DESCENDING
from log import LogHandler
from src.analysis import conn
from src.analysis.setting import REPORT_DIR, TABLE_STYLE, COLOR_RULE, PERCENT_FORMAT, COMMA0_FORMAT, DATE_FORMAT, \
PERCENT0_FORMAT
from src.analysis.utils import histogram
from src.api import get_peak_start_date, get_price, get_roll_yield
from src.data.future.setting import CODE2NAME_MAP
# from src.util import count_percentile
log = LogHandler('analysis.log')
def get_instrument_symbols(by='amount', threshold=1e9, instrument='future'):
"""
根据流动性选择交易品种，30天内的平均值作为选择标准，由于节假日原因，取平均的数据个数可能不一样。
:param instrument:
:param by:过滤的因子
:param threshold:过滤的阈值
:return:
"""
# conn = connect_mongo(db='quote', username=DATA_ANALYST, password=ANALYST_PWD)
pipeline = [
{
'$match': {
# -*- coding: utf-8 -*-
# audio.py - usage:
# class to provide interface to process common operation
# author: xiao yang <*****@*****.**>
# date: 2016.Mar.11
import os
import sys
import wave
import math
from struct import pack, unpack
from log import LogHandler
logger = LogHandler('audio_process.log', stdout=True)
class Wave(object):
PACKTYPE = {
1: 'c',
2: 'h',
4: 'l',
8: 'q',
}
PACKENDIAN = {
'little': '<',
'big': '>',
}
def __init__(self, filename, header, content):
from log import LogHandler
from datetime import datetime, timedelta
import pandas as pd
import numpy as np
import scipy.signal as signal
# TODO 改用动态接口
# get_history_hq_api(code, start=None, end=None, freq='d')
from src.features import conn
from src.data.tdx import get_future_hq
from src.util import connect_mongo
from src.api.cons import FREQ
get_history_hq_api = get_future_hq
log = LogHandler('features.log')
class TsBlock:
def __init__(self, code):
self.code = code
self.__peaks = dict.fromkeys(FREQ, pd.DataFrame())
self.__segments = dict.fromkeys(FREQ, pd.DataFrame())
self.__blocks = dict.fromkeys(FREQ, pd.DataFrame())
# 从数据库读取数据
def get_peaks(self, start=None, end=None, freq='d'):
"""
only append the data at the end, ascending sort
:param start: datetime
:param end: datetime