def __init__(self):
self.proxies = ProxyHandler()
self.tasker = Tasker()
self.saver = Saver()
self.log_handler = LogHandler()
self.tieba_task = tieba_task
self.loop_num = LOOP_NUM
def __init__(self, database=None):
if not database:
database = self.DEFAULT_DATABASE_URI
self.database_uri = database
engine = create_engine(self.database_uri)
Base.metadata.create_all(engine)
Base.metadata.bind = engine
session = sessionmaker(bind=engine)
self.db = session()
self.tasker = Tasker(self.db)
self._run_path = sys.argv[0]
# Scan through $PATH, and determine if this could be run without the full path.
run_directory = os.path.dirname(sys.argv[0])
if run_directory[-1] != os.path.sep:
run_directory = run_directory + os.path.sep
len_run_directory = len(run_directory)
for a_path in os.environ['PATH'].split(os.pathsep):
if a_path[-1] != os.path.sep:
a_path = a_path + os.path.sep
if a_path == run_directory:
self._run_path = self._run_path[len_run_directory:]
break
# Attempt to load up all intervals from the intervals directory.
self.all_cadences = {}
intervals_dir = os.path.realpath(
os.path.join(os.path.dirname(__file__), 'intervals'))
for f in os.listdir(intervals_dir):
# Only take py files
if not f.endswith('.py') or f in ('__init__.py',
'base_interval.py',
'interval_factory.py'):
continue
interval_name = f.replace('.py', '')
try:
interval = IntervalFactory.get(interval_name)
except UnsupportedIntervalException:
pass
self.all_cadences[interval.approximate_period()] = (interval_name,
interval)
class Scheduler(object):
def __init__(self):
self.proxies = ProxyHandler()
self.tasker = Tasker()
self.saver = Saver()
self.log_handler = LogHandler()
self.tieba_task = tieba_task
self.loop_num = LOOP_NUM
def end_task(self, loop):
# 保存数据到mongodb
self.saver.run()
if self.tasker.check_retries():
self.retry_task(loop)
else:
loop.close()
self.log_handler.logger().info('完成循环')
print('Finish!!')
def retry_task(self, loop):
#已经分片好的每次重试的url列表
per_step_urls_list = self.tasker.get_perloop_retry()
# 获取代理池中的代理
proxy_list = self.proxies.get_proxies()
for per_step_urls in per_step_urls_list:
# 待处理
tasks = tieba_task('retry', per_step_urls, proxy_list)
# 启动
loop.run_until_complete(asyncio.gather(*tasks))
self.end_task(loop)
# @run_time
def start_task(self):
loop = asyncio.get_event_loop()
# 获取代理池中的代理
proxy_list = self.proxies.get_proxies()
# 每次所有任务
one_task = self.tasker.get_task()
# self.loop_num: 每个任务循环几次, 避免Semaphore量太大
for index in range(self.loop_num):
# 每次循环的urls
per_step_urls = self.tasker.get_perloop(one_task)
# 待处理
tasks = tieba_task(index, per_step_urls, proxy_list)
# 启动
loop.run_until_complete(asyncio.gather(*tasks))
self.end_task(loop)
def run():
# 协程池
loop = asyncio.get_event_loop()
tasker = Tasker()
each_task = tasker.get_task()
# LOOP_NUM: 每个任务循环几次, 避免Semaphore量太大
for i in range(LOOP_NUM):
# 每次循环的urls
per_step_urls = tasker.get_urls(each_task)
# 待处理
tasks = tasker.create_task(i, per_step_urls)
# 启动
loop.run_until_complete(asyncio.gather(*tasks))
# 保存数据到mongodb
saver = Saver()
saver.run()
loop.close()
def __init__(self, *args, **kwargs):
if len(args) > 0:
f = args[0]
if isinstance(f, collections.Callable):
# the decorator is called without args,
# hence 1st arg is our function to decorate
self.related_tasker = Tasker.get_tasker_for_modulename(f.__module__)
self.related_tasker.taskfunc_list.append((f.__name__, ))
self._decorate(f)
else:
self.depends = args
else:
self.depends = ()
def call_task(task_file, *tasks):
from tasker import Tasker
subtasker = Tasker(task_file)
subtasker.run(tasks)
def __init__(self):
print("Server started on %s:%s." % (LOCAL, PORT))
self.loop = None
self.lobby = Tasker()
class TaskerCli(object):
DEFAULT_DATABASE_URI = 'sqlite:///{}'.format(
os.path.join(os.path.expanduser('~'), '.tasker.sqlite'))
def __init__(self, database=None):
if not database:
database = self.DEFAULT_DATABASE_URI
self.database_uri = database
engine = create_engine(self.database_uri)
Base.metadata.create_all(engine)
Base.metadata.bind = engine
session = sessionmaker(bind=engine)
self.db = session()
self.tasker = Tasker(self.db)
self._run_path = sys.argv[0]
# Scan through $PATH, and determine if this could be run without the full path.
run_directory = os.path.dirname(sys.argv[0])
if run_directory[-1] != os.path.sep:
run_directory = run_directory + os.path.sep
len_run_directory = len(run_directory)
for a_path in os.environ['PATH'].split(os.pathsep):
if a_path[-1] != os.path.sep:
a_path = a_path + os.path.sep
if a_path == run_directory:
self._run_path = self._run_path[len_run_directory:]
break
# Attempt to load up all intervals from the intervals directory.
self.all_cadences = {}
intervals_dir = os.path.realpath(
os.path.join(os.path.dirname(__file__), 'intervals'))
for f in os.listdir(intervals_dir):
# Only take py files
if not f.endswith('.py') or f in ('__init__.py',
'base_interval.py',
'interval_factory.py'):
continue
interval_name = f.replace('.py', '')
try:
interval = IntervalFactory.get(interval_name)
except UnsupportedIntervalException:
pass
self.all_cadences[interval.approximate_period()] = (interval_name,
interval)
def create_task(self):
name = self._get_task_name()
cadence = self._get_cadence()
while True:
start = self._get_first_date()
try:
self.tasker.assert_start_date_valid(cadence, start)
break
except InvalidStartDateException as e:
print >> sys.stderr, e.message
self.tasker.create_task(name, cadence, start)
def print_tasks(self):
self.tasker.schedule_tasks()
self._print_remaining_tasks()
def complete_task(self, ti_id):
self.tasker.complete_task_instance(ti_id)
def _print_remaining_tasks(self):
task_instances = self.tasker.get_incomplete_task_instances()
if len(task_instances):
# Get the highest TI id, so that the indent can be exactly 4 spaces in from the longest ID.
max_id = max(ti[0] for ti in task_instances)
rjust = 4 + len(str(max_id))
print 'Things to do:'
for row in task_instances:
ti_id, name, date, done = row
print '{}. ({}) {}'.format(str(ti_id).rjust(rjust), date, name)
database_str = ' '
if self.database_uri != self.DEFAULT_DATABASE_URI:
database_str = ' --database "{}" '.format(self.database_uri)
print 'To complete any task, use:\n {}{}{} N'.format(
self._run_path, database_str, TaskerCliOptions.COMPLETE)
def _get_task_name(self):
while True:
name = raw_input('Enter task name: ').strip()
if not name:
continue
try:
self.tasker.assert_name_unique(name)
return name
except DuplicateNameException as e:
print >> sys.stderr, e.message
def _get_cadence(self):
all_cadences_keys = sorted(self.all_cadences.keys())
cadence_lst = [
' {}. {}'.format(i + 1, self.all_cadences[o][0].title())
for i, o in enumerate(all_cadences_keys)
]
while True:
print 'Available cadences:'
print '\n'.join(cadence_lst)
cadence = raw_input('Select cadence: ').strip()
if not cadence:
continue
try:
cadence = self.all_cadences[all_cadences_keys[int(cadence) -
1]][0]
except ValueError:
pass
cadence = cadence.lower()
try:
self.tasker.assert_cadence_valid(cadence)
return cadence
except InvalidCadenceException as e:
print >> sys.stderr, e.message
def _get_first_date(self):
while True:
start = raw_input(
'When does this start (YYYY-MM-DD; default today): ').strip()
if start == '':
return date.today()
try:
return date(*[int(i) for i in start.split('-')])
except (TypeError, ValueError) as e:
print >> sys.stderr, 'Not a valid (YYYY-MM-DD) ({})'.format(
e.message)
def __call__(self, f):
t = tuple(self.depends) + (f.__name__, )
self.related_tasker = Tasker.get_tasker_for_modulename(f.__module__)
self.related_tasker.taskfunc_list.append(t)
return self._decorate(f)
# instantiate the JournaldLogHandler to hook into systemd
journald_handler = JournaldLogHandler()
# set a formatter to include the level name
journald_handler.setFormatter(
logging.Formatter('[%(levelname)s] %(asctime)s - %(message)s'))
# add the journald handler to the current logger
logger.addHandler(journald_handler)
# optionally set the logging level
logger.setLevel(logging.INFO)
logging.info("Starting new session...")
# Queue
mainQueue = Tasker(
) # for tasks that change users data (register, kill, password, etc..)
secondQueue = Tasker() # for reading-only tasks (whois, etc..)
# Discord bot
load_dotenv()
TOKEN = os.getenv('DISCORD_TOKEN')
bot = commands.Bot(command_prefix='$')
# Other
config = getConfig()
db = getDb()
serverManager = MortalManager.from_save(config, db)
# --------------- Bot commands ---------------
