class TimeSchedule: def __init__(self, lock, host='127.0.0.1', port='6800'): config = Config() self.db = glv.get_value(key='sqlite_db') self.user_name = config.get('auth_username', '') self.user_password = config.get('auth_password', '') self.start_time = time.strftime("%Y %m %d %H %M %S", time.localtime()) self.server_port = 'http://{}:{}/'.format(host, port) self.schedule_post_url = 'http://{}:{}/schedule.json'.format(host, port) self.listproject_url = 'http://{}:{}/listprojects.json'.format(host, port) self.spider_task_dic = dict() self.projects = None self.db_lock = lock self.ts_lock = threading.Lock() self._keys_set = { "year", "month", "day", "week", "hour", "minute", "second", "y", "m", "d", "w", "H", "M", "S", } self._keys_dic = { "y": "year", "m": "month", "d": "day", "w": "week", "H": "hour", "M": "minute", "S": "second", } self._keys_set_lis = [[y for y in x] for x in self._keys_set] self.CPU_THRESHOLD = 93 self.MEMORY_THRESHOLD = 96 self.schedule_logger = Logger(namespace='- Scheduler -') def run(self): time.sleep(3) self.projects = self.list_projects() self.schedule_logger.info('scheduler is running') count = 1 while True: schedule_sta = self.task_scheduler() if not schedule_sta and count == 1: self.schedule_logger.info('No Scheduled Spider in Database') count += 1 elif not schedule_sta and count != 1: count += 1 else: count = 1 time.sleep(1) def task_scheduler(self): self.ts_lock.acquire(blocking=True) self.db_lock.acquire() db_result = self.db.get(model_name='SpiderScheduleModel', key_list=['hash_str', 'project', 'spider', 'schedule', 'args', 'runtime', 'status']) self.db_lock.release() self.ts_lock.release() schedule_list_raw = [ {'hash_str': x.hash_str, 'project': x.project, 'spider': x.spider, 'schedule': x.schedule, 'args': x.args, 'runtime': x.runtime, 'status': x.status} for x in db_result if int(x.status) != 0 ] if db_result else [] schedule_sta = False if schedule_list_raw: for each_schedule in schedule_list_raw: project = each_schedule.get('project') runtime = int(each_schedule.get('runtime')) if project in self.projects and runtime > 0: schedule = each_schedule.get('schedule') if any([x in schedule for x in self._keys_set]): try: schedule = json.loads(schedule) except: schedule = eval(schedule) try: if isinstance(schedule, dict): for key in schedule.keys(): if key not in self._keys_set: mean_key = self._check_key(key) raise ValueError( 'found "{}" in your schedule dict, maybe you mean "{}"'.format(key, mean_key)) if key in self._keys_dic: val = schedule.pop(key) schedule[self._keys_dic[key]] = val next_time_sep = self.cal_time_sep(**schedule) else: next_time_sep = self.cal_time_sep(schedule_str=schedule, is_str=True) next_time_sep = int(next_time_sep) + 1 if next_time_sep > 1: each_schedule['schedule'] = next_time_sep item = '{}-{}'.format(each_schedule['project'], each_schedule['spider']) self.ts_lock.acquire(blocking=True) if self.spider_task_dic.get(item) != 'waiting': self.spider_task_dic[item] = 'waiting' t = threading.Thread(target=self.poster, args=(each_schedule,)) try: t.start() except Exception as THError: self.schedule_logger.warn('start new job error [ {} ]: {}'.format(item, THError)) self.ts_lock.release() except ValueError as V: self.schedule_logger.error('spider runtime schedule error, please check the database: {}'.format(V)) schedule_sta = True return schedule_sta def poster(self, dic): hash_str = dic.get('hash_str') status = int(dic.pop('status')) project = dic.get('project') spider = dic.get('spider') job_str = " %s-%s " % (project, spider) args = dic.get('args') try: args = json.loads(args) except: args = eval(args) wait_time = dic.get('schedule') item = '{}-{}'.format(project, spider) if project and spider: data = {'project': project, 'spider': spider, 'un': self.user_name, 'pwd': self.user_password} if args: args = self._spider_args_method(args, hash_str) data.update(args) self.schedule_logger.info('job {} is waiting, countdown {}s'.format(item, wait_time)) time.sleep(wait_time - 1) another_wait_time = 0 spider_runtime_avg = self.spiders_runtime(project=project, spider=spider) if status == 1: while not self.is_system_ok(): self.schedule_logger.warn('system is fully functioning, wait another 2 seconds to post schedule') time.sleep(2) another_wait_time += 3 if another_wait_time >= (wait_time - spider_runtime_avg): self.schedule_logger.warning('wait too long, cancel the job %s' % job_str) return None res = json.loads(requests.post(url=self.schedule_post_url, data=data).content) elif status == 2: res = json.loads(requests.post(url=self.schedule_post_url, data=data).content) elif status == 3: res = json.loads(requests.post(url=self.schedule_post_url, data=data).content) else: res = json.loads(requests.post(url=self.schedule_post_url, data=data).content) spider_status = res.get('status') if spider_status != 'ok': spider_status = 'error' else: self.schedule_logger.error('job project: {}, spider: {} post fail!'.format(project, spider)) spider_status = 'error' self.ts_lock.acquire(blocking=True) if spider_status == 'ok': self._run_countdown(project=project, spider=spider) self.spider_task_dic[item] = spider_status self.ts_lock.release() def _spider_args_method(self, args, hash_str): args_raw = args.copy() if args: method = args.pop('method', 'normal') if method == 'auto_increment': next_args = {k: str(int(v)+1) if isinstance(v, int) or (isinstance(v, str) and v.isdigit()) else v for k, v in args.items()} elif isinstance(method, dict): ex_md = method.get('expression') fc_md = method.get('function') if ex_md: next_args = eval(ex_md) if fc_md: exec(fc_md) else: next_args = args next_args.update({'method': method}) self.db.update('SpiderScheduleModel', update_dic={'args': next_args}, filter_dic={"hash_str": hash_str}) return args return args_raw def spiders_runtime(self, project, spider): self.db_lock.acquire() res = self.db.get(model_name='SpiderMonitor', key_list=['runtime'], filter_dic={'project': project, 'spider': spider}) self.db_lock.release() spider_list = [int(x.runtime) for x in res if x.runtime.isdigit()] if res else [0] return sum(spider_list) / len(spider_list) def list_projects(self): res = requests.get(url=self.listproject_url) projects = {} if res: projects_list = json.loads(res.content).get('projects') if projects_list: projects = set(projects_list) return projects def cal_time_sep(self, year='*', month='*', day='*', week='*', hour='*', minute='*', second='*', schedule_str=None, is_str=False ): """ "%Y-%m-%d %H:%M:%S %w" """ if is_str: s = [int(x.strip()) for x in schedule_str.split(',')] time_sep = (datetime.datetime(s[0], s[1], s[2], s[3], s[4], s[5]) - datetime.datetime.now()).total_seconds() return time_sep y = int(time.strftime("%Y", time.localtime())) if year != '*' and '*' in year: y = int(year.split('/')[-1]) + y elif year.isdigit(): y = int(year) if week == '*': m = int(time.strftime("%m", time.localtime())) if month != '*' and '*' in month: m_raw = int(month.split('/')[-1]) if m_raw >= 12: raise ValueError('month value is too large, please set the year instead') m = m_raw + m if m > 12: y += m // 12 m = m % 12 elif month.isdigit(): m = int(month) days_in_this_month = self.how_many_days_in_this_month(y, m) d = int(time.strftime("%d", time.localtime())) if day != '*' and '*' in day: d_raw = int(day.split('/')[-1]) if d_raw > days_in_this_month: raise ValueError('day value is too large, please set the month or the year instead') d = d_raw + d if d > days_in_this_month: d = d - days_in_this_month m += 1 if m > 12: y += 1 m = m - 12 elif day.isdigit(): d = int(day) days_in_this_month = self.how_many_days_in_this_month(y, m) H = int(time.strftime("%H", time.localtime())) if hour != '*' and '*' in hour: H_raw = int(hour.split('/')[-1]) if H_raw > 24: raise ValueError('hour value is too large, please set the day instead') H = H_raw + H if H >= 24: H = H - 24 d += 1 if d > days_in_this_month: d = d - days_in_this_month m += 1 if m > 12: y += 1 m = m - 12 elif hour.isdigit(): H = int(hour) days_in_this_month = self.how_many_days_in_this_month(y, m) M = int(time.strftime("%M", time.localtime())) if minute != '*' and '*' in minute: M_raw = int(minute.split('/')[-1]) if M_raw > 60: raise ValueError('minute value is too large, please set the hour instead') M = M_raw + M if M >= 60: M = M - 60 H += 1 if H >= 24: H = H - 24 d += 1 if d > days_in_this_month: d = d - days_in_this_month m += 1 if m > 12: y += 1 m = m - 12 elif minute.isdigit(): M = int(minute) days_in_this_month = self.how_many_days_in_this_month(y, m) S = int(time.strftime("%S", time.localtime())) if second != '*' and '*' in second: S_raw = int(second.split('/')[-1]) if S_raw > 60: raise ValueError('second value is too large, please set the minute instead') S = S_raw + S if S >= 60: S = S - 60 M += 1 if M >= 60: M = M - 60 H += 1 if H >= 24: H = H - 24 d += 1 if d > days_in_this_month: d = d - days_in_this_month m += 1 if m > 12: y += 1 m = m - 12 elif second.isdigit(): S = int(second) time_sep = eval( "(datetime.datetime({},{},{}, {},{},{}) - datetime.datetime.now()).total_seconds()".format(y, m, d, H, M, S)) else: week_in_this_year = int(time.strftime("%U", time.localtime())) w = int(time.strftime("%w", time.localtime())) if '*' in week: w_raw = int(week.split('/')[-1]) if w_raw >= 7: raise ValueError('week value is too large, please set the day or the month instead') if w_raw < w: week_in_this_year += 1 w = w_raw if week_in_this_year > 53: y += 1 week_in_this_year = week_in_this_year - 53 elif week.isdigit(): w = int(week) if int(week) < w: week_in_this_year += 1 H = int(time.strftime("%H", time.localtime())) if hour != '*' and '*' in hour: H_raw = int(hour.split('/')[-1]) if H_raw >= 24: raise ValueError('hour value is too large, please set the day instead') H = H_raw + H if H >= 24: H = H - 24 w += 1 if w >= 7: w = w - 7 week_in_this_year += 1 if week_in_this_year > 53: y += 1 week_in_this_year = week_in_this_year - 53 elif hour.isdigit(): H = int(hour) M = int(time.strftime("%M", time.localtime())) if minute != '*' and '*' in minute: M_raw = int(minute.split('/')[-1]) if M_raw >= 60: raise ValueError('minute value is too large, please set the hour instead') M = M_raw + M if M >= 60: M = M - 60 H += 1 if H >= 24: H = H - 24 w += 1 if w > 7: w = w - 7 week_in_this_year += 1 if week_in_this_year > 53: y += 1 week_in_this_year = week_in_this_year - 53 elif minute.isdigit(): M = int(minute) S = int(time.strftime("%S", time.localtime())) if second != '*' and '*' in second: S_raw = int(second.split('/')[-1]) if S_raw >= 60: raise ValueError('second value is too large, please set the minute instead') S = S_raw + S if S >= 60: S = S - 60 M += 1 if M >= 60: M = M - 60 H += 1 if H >= 24: H = H - 24 w += 1 if w > 7: w = w - 7 week_in_this_year += 1 if week_in_this_year > 53: y += 1 week_in_this_year = week_in_this_year - 53 elif second.isdigit(): S = int(second) if S >= 60: S = S - 60 M += 1 if M >= 60: M = M - 60 H += 1 if H >= 24: H = H - 24 w += 1 if w > 7: w = w - 7 week_in_this_year += 1 if week_in_this_year > 53: y += 1 week_in_this_year = week_in_this_year - 53 m, d = self.get_month_and_days_by_week(year=y, week_in_this_year=week_in_this_year, week=w) time_sep = eval( "(datetime.datetime({},{},{}, {},{},{}) - datetime.datetime.now()).total_seconds()".format(y, m, d, H, M, S)) return time_sep def get_month_and_days_by_week(self, year, week_in_this_year, week): days = week_in_this_year * 7 + week if (year % 4 == 0 and year % 100 != 0) or (year % 400 == 0): Fe = 29 else: Fe = 28 month_lis = [31, Fe, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] month_count = 1 days_count = 0 for month_days in month_lis: days = days - month_days if days > 0: month_count += 1 elif days == 0: days_count = 0 month_count += 1 break else: days_count = days + month_days break return [month_count, days_count] def how_many_days_in_this_month(self, y, m): if m in (1, 3, 5, 7, 8, 10, 12): days = 31 elif m in (4, 6, 9, 11): days = 30 else: if (y % 4 == 0 and y % 100 != 0) or (y % 400 == 0): days = 29 else: days = 28 return days def is_system_ok(self): is_pass = True cpu_list = psutil.cpu_percent(interval=1, percpu=True) memory_percent = psutil.virtual_memory().percent if cpu_list and memory_percent: is_cpu_ok = True if min(cpu_list) > self.CPU_THRESHOLD: is_cpu_ok = False is_memo_ok = True if memory_percent > self.MEMORY_THRESHOLD: is_memo_ok = False if not is_cpu_ok or not is_memo_ok: is_pass = False return is_pass def _check_key(self, key): key_lis = [x for x in key] count_dic = dict() for ksl in self._keys_set_lis: o_key = ''.join(ksl) score = 0 for k in key_lis: if k in ksl: score += 1 count_dic[o_key] = score best_math = sorted(count_dic, key=count_dic.__getitem__, reverse=True)[0] return best_math def _run_countdown(self, project, spider): db_schedule = self.db.get(model_name='SpiderScheduleModel', key_list=['id', 'runtime'], filter_dic={'project': project, 'spider': spider}) run_time_in_db = [x.runtime for x in db_schedule][0] if db_schedule else 0 the_id = [x.id for x in db_schedule][0] if db_schedule else None if run_time_in_db > 0 and the_id is not None: rt = int(run_time_in_db) - 1 self.db.update(model_name='SpiderScheduleModel', update_dic={"runtime": rt}, filter_dic={"id": the_id})
class FSProtocol(basic.LineReceiver): class State(Enum): IDLE = 0 POLLING = 1 POLLED = 2 RAW = 3 def __init__(self): self.log = Logger() self.setIdleState() def setIdleState(self): self.my_state = self.State.IDLE self.wait_for_list = None self.file_transfer_peer = None self.setLineMode() def connectionMade(self): self.log.info("New connection established: {peer!r}, {pid!s}", peer=self.transport.getPeer(), pid=id(self)) self.factory.protocols.append(self) self.transport.write(b'Hello form server\r\n') def send(self, command, argument): if command != "": command += " " self.transport.write(command.upper().encode() + argument.encode() + b'\r\n') # self.log.info("Sending to {peer!r}: {command!s} {argument!s}", # command=command, argument=argument) def doReject(self, line): self.log.error("We had to say no to {peer!r}: {line}", peer=self.transport.getPeer(), line=line) self.transport.write(b'REJECT ' + line.encode() + b'\r\n') # def doRequest(self, regex): # self.log.debug("server asks for files: {regex!s}", regex=regex) # self.transport.write(b'REQUEST ' + regex.encode() + b'\r\n') def requestFiles(self, regex): ''' Called by factory, if one of users requested a search ''' if self.my_state != self.State.IDLE or self.wait_for_list is not None: return defer.fail(RuntimeError("Connection is busy")) # Go to POLLED state, so no one will interrupt us self.my_state = self.State.POLLED self.send("REQUEST", regex) def debuglogc(data): self.log.debug("wait_for_list:requestFiles:callback: {data!r}", data=data) return data def debugloge(err): self.log.debug("wait_for_list:requestFiles:errback: {err!r}", err=err) return err self.wait_for_list = defer.Deferred().addCallbacks( debuglogc, debugloge) # We create Deferred, that will be fired as the response comes from user return self.wait_for_list def trySetStateIdle(self): # Some operation has finished # Check if there is file request # if len(self.outer_req) > 0: # # Set state # self.state = self.State.POLLED # # Ask peer # regex,d = self.outer_req[0] # self.transport.write("TODO some message" + regex + b'\r\n') pass def handleFind(self, regex): ''' Hanles the FIND request from user ''' self.my_state = self.State.POLLING # The factory will ask all the others about our regexp # The result here is the string d = self.factory.getFiles(self, regex) def response(ans): self.log.info("FIND command returns next {filelist!s}", filelist=ans) self.send('RESPONSE', ans) def error(e): self.doReject(e.getErrorMessage()) def idle(ignore): self.my_state = self.State.IDLE d.addCallbacks(response, error) d.addBoth(idle) def lineReceived(self, line): self.log.info("lineReceived: {peer!r} got message: {line!s}", peer=self.transport.getPeer(), line=line) line = line.decode() data = cleanInput(line) if len(data) == 0 or data == '': self.log.info("Oh, never mind, it's gone") self.doReject("Seems like a bunch of tabs") return command = data[0].upper() argument = data[1] if len(data) > 1 else None # Analize the command if command == 'FIND': # Find files from other users if self.my_state != self.State.IDLE: self.doReject("Another operation is in run") return if argument is None: self.doReject("Regex expected") return self.handleFind(argument) elif command == 'FILES': # Polled user returns list of files here if self.my_state != self.State.POLLED: self.doReject( "Got list of files, but there is no request for them") return # self.log.debug("Got list of files from {peer!r}", peer=self.transport.getPeer()) if argument is None: self.wait_for_list.errback( RuntimeError("This peer has no such files")) else: # Return self id and list of files self.wait_for_list.callback(':'.join([str(id(self)), argument])) # Cleanup self.wait_for_list = None self.my_state = self.State.IDLE elif command == 'GET': # User asks for file ''' This is a routine of connection, whitch needs file ''' # Check if we ready if self.my_state != self.State.IDLE: self.doReject("Can not give you file at the moment") return # Get arguments if argument is None: self.doReject("Expected filename for GET operation") return try: pid, filename = argument.split(':', 1) except (IndexError): self.doReject("not valid filename") return # Find the peer peer = self.factory.findConnById(pid) # Check if we are ready if peer is None: self.doReject("Sorry, the peer seems afk") return if peer.my_state != peer.State.IDLE: self.doReject("Sorry, the peer seems busy") return # Ready to begin, don't interrupt us please peer.my_state = peer.State.RAW self.my_state = self.State.RAW peer.file_transfer_peer = self self.file_transfer_peer = peer peer.send("OBTAIN", filename) self.send("RAW", filename) self.setRawMode() peer.setRawMode() elif command == 'ALICE': # User remembers how this project started self.send('', "nice girl") else: self.doReject("not a protocol command: %s" % command) def rawDataReceived(self, data): ''' Should be called, if user sends data to peer in response ''' if self.file_transfer_peer is None: self.log.warning("RECIEVING RAW DATA WITHOUT PEER TO RESEND") return peer = self.file_transfer_peer self.log.info("RAW: resending data: %d KB" % len(data)) peer.transport.write(data) if not data.endswith(b'\r\n'): # Transmission not done yet return self.log.info("RAW: Transmission complete") peer.setIdleState() self.setIdleState() def connectionLost(self, reason): self.log.info("Connection closed: {peer!r}, {reason!r}", peer=self.transport.getPeer(), reason=reason) self.factory.protocols.remove(self)
class TimeSchedule: def __init__(self, lock, host='127.0.0.1', port='6800'): config = Config() self.db = glv.get_value(key='sqlite_db') self.user_name = config.get('auth_username', '') self.user_password = config.get('auth_password', '') self.start_time = time.strftime("%Y %m %d %H %M %S", time.localtime()) self.server_port = 'http://{}:{}/'.format(host, port) self.schedule_post_url = 'http://{}:{}/schedule.json'.format( host, port) self.listproject_url = 'http://{}:{}/listprojects.json'.format( host, port) self.spider_task_dic = dict() self.projects = None self.db_lock = lock self.ts_lock = threading.Lock() self.CPU_THRESHOLD = 93 self.MEMORY_THRESHOLD = 96 self.schedule_logger = Logger(namespace='- Scheduler -') def run(self): time.sleep(3) self.projects = self.list_projects() self.schedule_logger.info('scheduler is running') count = 1 while True: schedule_sta = self.task_scheduler() if not schedule_sta and count == 1: self.schedule_logger.info('No Scheduled Spider in Database') count += 1 elif not schedule_sta and count != 1: count += 1 else: count = 1 time.sleep(1) def task_scheduler(self): self.ts_lock.acquire(blocking=True) self.db_lock.acquire() db_result = self.db.get( model_name='SpiderScheduleModel', key_list=['project', 'spider', 'schedule', 'args', 'status']) self.db_lock.release() self.ts_lock.release() schedule_list_raw = [{ 'project': x.project, 'spider': x.spider, 'schedule': x.schedule, 'args': x.args, 'status': x.status } for x in db_result if int(x.status) != 0] if db_result else [] schedule_sta = False if schedule_list_raw: for each_schedule in schedule_list_raw: project = each_schedule.get('project') if project in self.projects: schedule = each_schedule.get('schedule').replace('\\', '') try: schedule = json.loads(schedule) except: schedule = eval(schedule) try: next_time_sep = self.cal_time_sep(**schedule) next_time_sep = int(next_time_sep) + 1 if next_time_sep > 1: each_schedule['schedule'] = next_time_sep item = '{}-{}'.format(each_schedule['project'], each_schedule['spider']) self.ts_lock.acquire(blocking=True) if self.spider_task_dic.get(item) != 'waiting': self.spider_task_dic[item] = 'waiting' t = threading.Thread(target=self.poster, args=(each_schedule, )) try: t.start() except Exception as THError: self.schedule_logger.warn( 'start new job error [ {} ]: {}'. format(item, THError)) self.ts_lock.release() except ValueError: self.schedule_logger.error( 'spider runtime schedule error, please check the database' ) schedule_sta = True return schedule_sta def poster(self, dic): status = int(dic.pop('status')) project = dic.get('project') spider = dic.get('spider') job_str = " %s-%s " % (project, spider) args = dic.get('args').replace('\\', '') if args: args = eval(args) wait_time = dic.get('schedule') item = '{}-{}'.format(project, spider) if project and spider: data = { 'project': project, 'spider': spider, 'un': self.user_name, 'pwd': self.user_password } if args: data.update(args) self.schedule_logger.info( 'job {} is waiting, countdown {}s'.format(item, wait_time)) time.sleep(wait_time - 1) another_wait_time = 0 spider_runtime_avg = self.spiders_runtime(project=project, spider=spider) if status == 1: while not self.is_system_ok(): self.schedule_logger.warn( 'system is fully functioning, wait another 2 seconds to post schedule' ) time.sleep(2) another_wait_time += 3 if another_wait_time >= (wait_time - spider_runtime_avg): self.schedule_logger.warning( 'wait too long, cancel the job %s' % job_str) return None res = json.loads( requests.post(url=self.schedule_post_url, data=data).content) elif status == 2: res = json.loads( requests.post(url=self.schedule_post_url, data=data).content) elif status == 3: res = json.loads( requests.post(url=self.schedule_post_url, data=data).content) else: res = json.loads( requests.post(url=self.schedule_post_url, data=data).content) spider_status = res.get('status') if spider_status != 'ok': spider_status = 'error' else: self.schedule_logger.error( 'job project: {}, spider: {} post fail!'.format( project, spider)) spider_status = 'error' self.ts_lock.acquire(blocking=True) self.spider_task_dic[item] = spider_status self.ts_lock.release() def spiders_runtime(self, project, spider): self.db_lock.acquire() res = self.db.get(model_name='SpiderMonitor', key_list=['runtime'], filter_dic={ 'project': project, 'spider': spider }) self.db_lock.release() spider_list = [int(x.runtime) for x in res if x.runtime.isdigit()] if res else [0] return sum(spider_list) / len(spider_list) def list_projects(self): res = requests.get(url=self.listproject_url) projects = {} if res: projects_list = json.loads(res.content).get('projects') if projects_list: projects = set(projects_list) return projects def cal_time_sep( self, year='*', month='*', day='*', week='*', hour='*', minute='*', second='*', ): """ "%Y-%m-%d %H:%M:%S %w" """ y = int(time.strftime("%Y", time.localtime())) if year != '*' and '*' in year: y = int(year.split('/')[-1]) + y elif year.isdigit(): y = int(year) if week == '*': m = int(time.strftime("%m", time.localtime())) if month != '*' and '*' in month: m_raw = int(month.split('/')[-1]) if m_raw >= 12: raise ValueError( 'month value is too large, please set the year instead' ) m = m_raw + m if m > 12: y += m // 12 m = m % 12 elif month.isdigit(): m = int(month) days_in_this_month = self.how_many_days_in_this_month(y, m) d = int(time.strftime("%d", time.localtime())) if day != '*' and '*' in day: d_raw = int(day.split('/')[-1]) if d_raw > days_in_this_month: raise ValueError( 'day value is too large, please set the month or the year instead' ) d = d_raw + d if d > days_in_this_month: d = d - days_in_this_month m += 1 if m > 12: y += 1 m = m - 12 elif day.isdigit(): d = int(day) days_in_this_month = self.how_many_days_in_this_month(y, m) H = int(time.strftime("%H", time.localtime())) if hour != '*' and '*' in hour: H_raw = int(hour.split('/')[-1]) if H_raw > 24: raise ValueError( 'hour value is too large, please set the day instead') H = H_raw + H if H >= 24: H = H - 24 d += 1 if d > days_in_this_month: d = d - days_in_this_month m += 1 if m > 12: y += 1 m = m - 12 elif hour.isdigit(): H = int(hour) days_in_this_month = self.how_many_days_in_this_month(y, m) M = int(time.strftime("%M", time.localtime())) if minute != '*' and '*' in minute: M_raw = int(minute.split('/')[-1]) if M_raw > 60: raise ValueError( 'minute value is too large, please set the hour instead' ) M = M_raw + M if M >= 60: M = M - 60 H += 1 if H >= 24: H = H - 24 d += 1 if d > days_in_this_month: d = d - days_in_this_month m += 1 if m > 12: y += 1 m = m - 12 elif minute.isdigit(): M = int(minute) days_in_this_month = self.how_many_days_in_this_month(y, m) S = int(time.strftime("%S", time.localtime())) if second != '*' and '*' in second: S_raw = int(second.split('/')[-1]) if S_raw > 60: raise ValueError( 'second value is too large, please set the minute instead' ) S = S_raw + S if S >= 60: S = S - 60 M += 1 if M >= 60: M = M - 60 H += 1 if H >= 24: H = H - 24 d += 1 if d > days_in_this_month: d = d - days_in_this_month m += 1 if m > 12: y += 1 m = m - 12 elif second.isdigit(): S = int(second) time_sep = eval( "(datetime.datetime({},{},{}, {},{},{}) - datetime.datetime.now()).total_seconds()" .format(y, m, d, H, M, S)) else: week_in_this_year = int(time.strftime("%U", time.localtime())) w = int(time.strftime("%w", time.localtime())) if '*' in week: w_raw = int(week.split('/')[-1]) if w_raw >= 7: raise ValueError( 'week value is too large, please set the day or the month instead' ) if w_raw < w: week_in_this_year += 1 w = w_raw if week_in_this_year > 53: y += 1 week_in_this_year = week_in_this_year - 53 elif week.isdigit(): w = int(week) if int(week) < w: week_in_this_year += 1 H = int(time.strftime("%H", time.localtime())) if hour != '*' and '*' in hour: H_raw = int(hour.split('/')[-1]) if H_raw >= 24: raise ValueError( 'hour value is too large, please set the day instead') H = H_raw + H if H >= 24: H = H - 24 w += 1 if w >= 7: w = w - 7 week_in_this_year += 1 if week_in_this_year > 53: y += 1 week_in_this_year = week_in_this_year - 53 elif hour.isdigit(): H = int(hour) M = int(time.strftime("%M", time.localtime())) if minute != '*' and '*' in minute: M_raw = int(minute.split('/')[-1]) if M_raw >= 60: raise ValueError( 'minute value is too large, please set the hour instead' ) M = M_raw + M if M >= 60: M = M - 60 H += 1 if H >= 24: H = H - 24 w += 1 if w > 7: w = w - 7 week_in_this_year += 1 if week_in_this_year > 53: y += 1 week_in_this_year = week_in_this_year - 53 elif minute.isdigit(): M = int(minute) S = int(time.strftime("%S", time.localtime())) if second != '*' and '*' in second: S_raw = int(second.split('/')[-1]) if S_raw >= 60: raise ValueError( 'second value is too large, please set the minute instead' ) S = S_raw + S if S >= 60: S = S - 60 M += 1 if M >= 60: M = M - 60 H += 1 if H >= 24: H = H - 24 w += 1 if w > 7: w = w - 7 week_in_this_year += 1 if week_in_this_year > 53: y += 1 week_in_this_year = week_in_this_year - 53 elif second.isdigit(): S = int(second) if S >= 60: S = S - 60 M += 1 if M >= 60: M = M - 60 H += 1 if H >= 24: H = H - 24 w += 1 if w > 7: w = w - 7 week_in_this_year += 1 if week_in_this_year > 53: y += 1 week_in_this_year = week_in_this_year - 53 m, d = self.get_month_and_days_by_week( year=y, week_in_this_year=week_in_this_year, week=w) time_sep = eval( "(datetime.datetime({},{},{}, {},{},{}) - datetime.datetime.now()).total_seconds()" .format(y, m, d, H, M, S)) return time_sep def get_month_and_days_by_week(self, year, week_in_this_year, week): days = week_in_this_year * 7 + week if (year % 4 == 0 and year % 100 != 0) or (year % 400 == 0): Fe = 29 else: Fe = 28 month_lis = [31, Fe, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] month_count = 1 days_count = 0 for month_days in month_lis: days = days - month_days if days > 0: month_count += 1 elif days == 0: days_count = 0 month_count += 1 break else: days_count = days + month_days break return [month_count, days_count] def how_many_days_in_this_month(self, y, m): if m in (1, 3, 5, 7, 8, 10, 12): days = 31 elif m in (4, 6, 9, 11): days = 30 else: if (y % 4 == 0 and y % 100 != 0) or (y % 400 == 0): days = 29 else: days = 28 return days def is_system_ok(self): is_pass = True cpu_list = psutil.cpu_percent(interval=1, percpu=True) memory_percent = psutil.virtual_memory().percent if cpu_list and memory_percent: is_cpu_ok = True if min(cpu_list) > self.CPU_THRESHOLD: is_cpu_ok = False is_memo_ok = True if memory_percent > self.MEMORY_THRESHOLD: is_memo_ok = False if not is_cpu_ok or not is_memo_ok: is_pass = False return is_pass