from spells.rest import Spell as BaseSpell
from utils import log_util
logger = log_util.get_logger('tohsaka.fund')
class Spell(BaseSpell):
REQUIRED_OPTIONS = ['code']
def __init__(self, config):
code = config['code']
print(config)
config[
'endpoint'] = f'https://fund.10jqka.com.cn/data/client/myfund/{code}'
BaseSpell.__init__(self, config)
@classmethod
def name(self):
return 'Fund'
@classmethod
def intro(self):
return 'Get Fund data'
def go(self):
for item in BaseSpell.go(self):
data = item['data'][0]
net = float(data['net'])
yield {
'name': data['name'],
import os
import hashlib
import time
from datetime import datetime
from feedgen.feed import FeedGenerator
import pytz
from dateutil import parser
from utils import file_util
from utils import log_util
from outputters.base_outputter import BaseOutputter
logger = log_util.get_logger('tohsaka.rss')
SECONDS_OF_DAY = 60 * 60 * 24
class Outputter(BaseOutputter):
@property
def REQUIRED_FIELDS(self):
return ['description', 'pubDate', 'title']
def __init__(self, config):
BaseOutputter.__init__(self, config)
self.file = config.get('filename', 'output') + '.xml'
self.title = config.get('title', 'Sample RSS')
self.description = config.get('description', 'Sample')
self.cache = config.get('cache', False)
self.base_link = config.get('host')
self.temp_dir = file_util.get_temp_dir()
from utils.log_util import get_logger
logger = get_logger('tohsaka.qualifier')
class Qualifier:
def __init__(self, config):
self.config = config.get('qualify', None)
def _qualify(self, item):
if not self.config:
return True
fields = self.config.get('field', [])
value = item
for f in fields:
value = value.get(f, {})
if type(value) == dict:
return False
condition = self.config.get('condition', {}).get('type', None)
target_value = int(self.config.get('condition', {}).get('value', 0))
if condition == '>':
if value > target_value:
return True
else:
logger.debug(
f'{item.get("title")} doesnt pass the qualify {value}/{target_value}'
)
return False
import os
from requests_html import HTMLSession
from spells.base_spell import BaseSpell
from utils import log_util
logger = log_util.get_logger('tohsaka.fourm')
class Spell(BaseSpell):
REQUIRED_OPTIONS = ['entry']
def __init__(self, config):
BaseSpell.__init__(self, config)
self.headers = config.get('headers', {})
@classmethod
def name(cls):
return 'Forum'
@classmethod
def intro(cls):
return 'Get forum-like data'
def get_date(self, element):
return element.text
def get_description(self, element):
return element.html
import importlib.util
from glob import glob
from os.path import join as pathjoin
import os
from utils import log_util
from utils.file_util import load_json
from qualifiers.qualifier import Qualifier
from spells import TohsakaException
logger = log_util.get_logger('tohsaka')
class Tohsaka:
item_per_log = 10
MYSTIC_PATH = pathjoin(os.path.dirname(os.path.realpath(__file__)),
'mystic')
SPELL_PATH = pathjoin(os.path.dirname(os.path.realpath(__file__)),
'spells')
OUTPUTTER_PATH = pathjoin(os.path.dirname(os.path.realpath(__file__)),
'outputters')
@classmethod
def get_mystic_codes(cls):
mystic = []
for mystic_file in glob(pathjoin(cls.MYSTIC_PATH, '*.json')):
mystic_json = load_json(mystic_file)
import requests
from spells.base_spell import BaseSpell
from utils import log_util
logger = log_util.get_logger('tohsaka.rest')
class Spell(BaseSpell):
REQUIRED_OPTIONS = ['endpoint']
def __init__(self, config):
BaseSpell.__init__(self, config)
@classmethod
def name(self):
return 'Rest'
@classmethod
def intro(self):
return 'Get REST data'
def get_headers(self):
return {
'Accept': 'application/json',
'Content-Type': 'application/json'
}
def go(self):
endpoint = self.config.get('endpoint')
r = requests.get(endpoint, headers=self.get_headers())
import os
from requests_html import HTMLSession
from spells.forum import Spell as BaseSpell
from utils import log_util
logger = log_util.get_logger('tohsaka.ranking')
class Spell(BaseSpell):
def __init__(self, config):
BaseSpell.__init__(self, config)
@classmethod
def name(cls):
return 'Ranking'
@classmethod
def intro(cls):
return 'Get ranking-like data'
def process_item(self, item):
try:
title = item.find(self.config.get('titleSelector'), first=True).text
pub_date = self.get_date(item.find(self.config.get('dateSelector'), first=True))
description = self.get_description(item.find(self.config.get('contentSelector'), first=True))
addition = self.get_addition(item)
except Exception as err:
logger.warning(f'failed to process url {item}')
logger.warning(str(err))
import json
import tempfile
import os
from utils import log_util
from pathlib import Path
logger = log_util.get_logger('tohsaka.file_util')
def load_json(filepath):
result = None
try:
with open(filepath) as json_file:
result = json.load(json_file)
except:
raise Exception(f'Failed to load file {filepath}.')
return result
def get_temp_dir():
path = os.path.join(tempfile.gettempdir(), 'tohsaka')
if not os.path.isdir(path):
os.mkdir(path)
logger.debug(f'Using temp folder {path}')
return path
def touch(path):