def __init__(self): self.proxies = ProxyHandler() self.tasker = Tasker() self.saver = Saver() self.log_handler = LogHandler() self.tieba_task = tieba_task self.loop_num = LOOP_NUM
def __init__(self, database=None): if not database: database = self.DEFAULT_DATABASE_URI self.database_uri = database engine = create_engine(self.database_uri) Base.metadata.create_all(engine) Base.metadata.bind = engine session = sessionmaker(bind=engine) self.db = session() self.tasker = Tasker(self.db) self._run_path = sys.argv[0] # Scan through $PATH, and determine if this could be run without the full path. run_directory = os.path.dirname(sys.argv[0]) if run_directory[-1] != os.path.sep: run_directory = run_directory + os.path.sep len_run_directory = len(run_directory) for a_path in os.environ['PATH'].split(os.pathsep): if a_path[-1] != os.path.sep: a_path = a_path + os.path.sep if a_path == run_directory: self._run_path = self._run_path[len_run_directory:] break # Attempt to load up all intervals from the intervals directory. self.all_cadences = {} intervals_dir = os.path.realpath( os.path.join(os.path.dirname(__file__), 'intervals')) for f in os.listdir(intervals_dir): # Only take py files if not f.endswith('.py') or f in ('__init__.py', 'base_interval.py', 'interval_factory.py'): continue interval_name = f.replace('.py', '') try: interval = IntervalFactory.get(interval_name) except UnsupportedIntervalException: pass self.all_cadences[interval.approximate_period()] = (interval_name, interval)
class Scheduler(object): def __init__(self): self.proxies = ProxyHandler() self.tasker = Tasker() self.saver = Saver() self.log_handler = LogHandler() self.tieba_task = tieba_task self.loop_num = LOOP_NUM def end_task(self, loop): # 保存数据到mongodb self.saver.run() if self.tasker.check_retries(): self.retry_task(loop) else: loop.close() self.log_handler.logger().info('完成循环') print('Finish!!') def retry_task(self, loop): #已经分片好的每次重试的url列表 per_step_urls_list = self.tasker.get_perloop_retry() # 获取代理池中的代理 proxy_list = self.proxies.get_proxies() for per_step_urls in per_step_urls_list: # 待处理 tasks = tieba_task('retry', per_step_urls, proxy_list) # 启动 loop.run_until_complete(asyncio.gather(*tasks)) self.end_task(loop) # @run_time def start_task(self): loop = asyncio.get_event_loop() # 获取代理池中的代理 proxy_list = self.proxies.get_proxies() # 每次所有任务 one_task = self.tasker.get_task() # self.loop_num: 每个任务循环几次, 避免Semaphore量太大 for index in range(self.loop_num): # 每次循环的urls per_step_urls = self.tasker.get_perloop(one_task) # 待处理 tasks = tieba_task(index, per_step_urls, proxy_list) # 启动 loop.run_until_complete(asyncio.gather(*tasks)) self.end_task(loop)
def run(): # 协程池 loop = asyncio.get_event_loop() tasker = Tasker() each_task = tasker.get_task() # LOOP_NUM: 每个任务循环几次, 避免Semaphore量太大 for i in range(LOOP_NUM): # 每次循环的urls per_step_urls = tasker.get_urls(each_task) # 待处理 tasks = tasker.create_task(i, per_step_urls) # 启动 loop.run_until_complete(asyncio.gather(*tasks)) # 保存数据到mongodb saver = Saver() saver.run() loop.close()
def __init__(self, *args, **kwargs): if len(args) > 0: f = args[0] if isinstance(f, collections.Callable): # the decorator is called without args, # hence 1st arg is our function to decorate self.related_tasker = Tasker.get_tasker_for_modulename(f.__module__) self.related_tasker.taskfunc_list.append((f.__name__, )) self._decorate(f) else: self.depends = args else: self.depends = ()
def call_task(task_file, *tasks): from tasker import Tasker subtasker = Tasker(task_file) subtasker.run(tasks)
def __init__(self): print("Server started on %s:%s." % (LOCAL, PORT)) self.loop = None self.lobby = Tasker()
class TaskerCli(object): DEFAULT_DATABASE_URI = 'sqlite:///{}'.format( os.path.join(os.path.expanduser('~'), '.tasker.sqlite')) def __init__(self, database=None): if not database: database = self.DEFAULT_DATABASE_URI self.database_uri = database engine = create_engine(self.database_uri) Base.metadata.create_all(engine) Base.metadata.bind = engine session = sessionmaker(bind=engine) self.db = session() self.tasker = Tasker(self.db) self._run_path = sys.argv[0] # Scan through $PATH, and determine if this could be run without the full path. run_directory = os.path.dirname(sys.argv[0]) if run_directory[-1] != os.path.sep: run_directory = run_directory + os.path.sep len_run_directory = len(run_directory) for a_path in os.environ['PATH'].split(os.pathsep): if a_path[-1] != os.path.sep: a_path = a_path + os.path.sep if a_path == run_directory: self._run_path = self._run_path[len_run_directory:] break # Attempt to load up all intervals from the intervals directory. self.all_cadences = {} intervals_dir = os.path.realpath( os.path.join(os.path.dirname(__file__), 'intervals')) for f in os.listdir(intervals_dir): # Only take py files if not f.endswith('.py') or f in ('__init__.py', 'base_interval.py', 'interval_factory.py'): continue interval_name = f.replace('.py', '') try: interval = IntervalFactory.get(interval_name) except UnsupportedIntervalException: pass self.all_cadences[interval.approximate_period()] = (interval_name, interval) def create_task(self): name = self._get_task_name() cadence = self._get_cadence() while True: start = self._get_first_date() try: self.tasker.assert_start_date_valid(cadence, start) break except InvalidStartDateException as e: print >> sys.stderr, e.message self.tasker.create_task(name, cadence, start) def print_tasks(self): self.tasker.schedule_tasks() self._print_remaining_tasks() def complete_task(self, ti_id): self.tasker.complete_task_instance(ti_id) def _print_remaining_tasks(self): task_instances = self.tasker.get_incomplete_task_instances() if len(task_instances): # Get the highest TI id, so that the indent can be exactly 4 spaces in from the longest ID. max_id = max(ti[0] for ti in task_instances) rjust = 4 + len(str(max_id)) print 'Things to do:' for row in task_instances: ti_id, name, date, done = row print '{}. ({}) {}'.format(str(ti_id).rjust(rjust), date, name) database_str = ' ' if self.database_uri != self.DEFAULT_DATABASE_URI: database_str = ' --database "{}" '.format(self.database_uri) print 'To complete any task, use:\n {}{}{} N'.format( self._run_path, database_str, TaskerCliOptions.COMPLETE) def _get_task_name(self): while True: name = raw_input('Enter task name: ').strip() if not name: continue try: self.tasker.assert_name_unique(name) return name except DuplicateNameException as e: print >> sys.stderr, e.message def _get_cadence(self): all_cadences_keys = sorted(self.all_cadences.keys()) cadence_lst = [ ' {}. {}'.format(i + 1, self.all_cadences[o][0].title()) for i, o in enumerate(all_cadences_keys) ] while True: print 'Available cadences:' print '\n'.join(cadence_lst) cadence = raw_input('Select cadence: ').strip() if not cadence: continue try: cadence = self.all_cadences[all_cadences_keys[int(cadence) - 1]][0] except ValueError: pass cadence = cadence.lower() try: self.tasker.assert_cadence_valid(cadence) return cadence except InvalidCadenceException as e: print >> sys.stderr, e.message def _get_first_date(self): while True: start = raw_input( 'When does this start (YYYY-MM-DD; default today): ').strip() if start == '': return date.today() try: return date(*[int(i) for i in start.split('-')]) except (TypeError, ValueError) as e: print >> sys.stderr, 'Not a valid (YYYY-MM-DD) ({})'.format( e.message)
def __call__(self, f): t = tuple(self.depends) + (f.__name__, ) self.related_tasker = Tasker.get_tasker_for_modulename(f.__module__) self.related_tasker.taskfunc_list.append(t) return self._decorate(f)
# instantiate the JournaldLogHandler to hook into systemd journald_handler = JournaldLogHandler() # set a formatter to include the level name journald_handler.setFormatter( logging.Formatter('[%(levelname)s] %(asctime)s - %(message)s')) # add the journald handler to the current logger logger.addHandler(journald_handler) # optionally set the logging level logger.setLevel(logging.INFO) logging.info("Starting new session...") # Queue mainQueue = Tasker( ) # for tasks that change users data (register, kill, password, etc..) secondQueue = Tasker() # for reading-only tasks (whois, etc..) # Discord bot load_dotenv() TOKEN = os.getenv('DISCORD_TOKEN') bot = commands.Bot(command_prefix='$') # Other config = getConfig() db = getDb() serverManager = MortalManager.from_save(config, db) # --------------- Bot commands ---------------