class TimeSchedule:
def __init__(self, lock, host='127.0.0.1', port='6800'):
config = Config()
self.db = glv.get_value(key='sqlite_db')
self.user_name = config.get('auth_username', '')
self.user_password = config.get('auth_password', '')
self.start_time = time.strftime("%Y %m %d %H %M %S", time.localtime())
self.server_port = 'http://{}:{}/'.format(host, port)
self.schedule_post_url = 'http://{}:{}/schedule.json'.format(host, port)
self.listproject_url = 'http://{}:{}/listprojects.json'.format(host, port)
self.spider_task_dic = dict()
self.projects = None
self.db_lock = lock
self.ts_lock = threading.Lock()
self._keys_set = {
"year",
"month",
"day",
"week",
"hour",
"minute",
"second",
"y",
"m",
"d",
"w",
"H",
"M",
"S",
}
self._keys_dic = {
"y": "year",
"m": "month",
"d": "day",
"w": "week",
"H": "hour",
"M": "minute",
"S": "second",
}
self._keys_set_lis = [[y for y in x] for x in self._keys_set]
self.CPU_THRESHOLD = 93
self.MEMORY_THRESHOLD = 96
self.schedule_logger = Logger(namespace='- Scheduler -')
def run(self):
time.sleep(3)
self.projects = self.list_projects()
self.schedule_logger.info('scheduler is running')
count = 1
while True:
schedule_sta = self.task_scheduler()
if not schedule_sta and count == 1:
self.schedule_logger.info('No Scheduled Spider in Database')
count += 1
elif not schedule_sta and count != 1:
count += 1
else:
count = 1
time.sleep(1)
def task_scheduler(self):
self.ts_lock.acquire(blocking=True)
self.db_lock.acquire()
db_result = self.db.get(model_name='SpiderScheduleModel',
key_list=['hash_str', 'project', 'spider', 'schedule', 'args', 'runtime', 'status'])
self.db_lock.release()
self.ts_lock.release()
schedule_list_raw = [
{'hash_str': x.hash_str, 'project': x.project, 'spider': x.spider, 'schedule': x.schedule, 'args': x.args, 'runtime': x.runtime,
'status': x.status}
for x in db_result if int(x.status) != 0
] if db_result else []
schedule_sta = False
if schedule_list_raw:
for each_schedule in schedule_list_raw:
project = each_schedule.get('project')
runtime = int(each_schedule.get('runtime'))
if project in self.projects and runtime > 0:
schedule = each_schedule.get('schedule')
if any([x in schedule for x in self._keys_set]):
try:
schedule = json.loads(schedule)
except:
schedule = eval(schedule)
try:
if isinstance(schedule, dict):
for key in schedule.keys():
if key not in self._keys_set:
mean_key = self._check_key(key)
raise ValueError(
'found "{}" in your schedule dict, maybe you mean "{}"'.format(key, mean_key))
if key in self._keys_dic:
val = schedule.pop(key)
schedule[self._keys_dic[key]] = val
next_time_sep = self.cal_time_sep(**schedule)
else:
next_time_sep = self.cal_time_sep(schedule_str=schedule, is_str=True)
next_time_sep = int(next_time_sep) + 1
if next_time_sep > 1:
each_schedule['schedule'] = next_time_sep
item = '{}-{}'.format(each_schedule['project'], each_schedule['spider'])
self.ts_lock.acquire(blocking=True)
if self.spider_task_dic.get(item) != 'waiting':
self.spider_task_dic[item] = 'waiting'
t = threading.Thread(target=self.poster, args=(each_schedule,))
try:
t.start()
except Exception as THError:
self.schedule_logger.warn('start new job error [ {} ]: {}'.format(item, THError))
self.ts_lock.release()
except ValueError as V:
self.schedule_logger.error('spider runtime schedule error, please check the database: {}'.format(V))
schedule_sta = True
return schedule_sta
def poster(self, dic):
hash_str = dic.get('hash_str')
status = int(dic.pop('status'))
project = dic.get('project')
spider = dic.get('spider')
job_str = " %s-%s " % (project, spider)
args = dic.get('args')
try:
args = json.loads(args)
except:
args = eval(args)
wait_time = dic.get('schedule')
item = '{}-{}'.format(project, spider)
if project and spider:
data = {'project': project, 'spider': spider, 'un': self.user_name, 'pwd': self.user_password}
if args:
args = self._spider_args_method(args, hash_str)
data.update(args)
self.schedule_logger.info('job {} is waiting, countdown {}s'.format(item, wait_time))
time.sleep(wait_time - 1)
another_wait_time = 0
spider_runtime_avg = self.spiders_runtime(project=project, spider=spider)
if status == 1:
while not self.is_system_ok():
self.schedule_logger.warn('system is fully functioning, wait another 2 seconds to post schedule')
time.sleep(2)
another_wait_time += 3
if another_wait_time >= (wait_time - spider_runtime_avg):
self.schedule_logger.warning('wait too long, cancel the job %s' % job_str)
return None
res = json.loads(requests.post(url=self.schedule_post_url, data=data).content)
elif status == 2:
res = json.loads(requests.post(url=self.schedule_post_url, data=data).content)
elif status == 3:
res = json.loads(requests.post(url=self.schedule_post_url, data=data).content)
else:
res = json.loads(requests.post(url=self.schedule_post_url, data=data).content)
spider_status = res.get('status')
if spider_status != 'ok':
spider_status = 'error'
else:
self.schedule_logger.error('job project: {}, spider: {} post fail!'.format(project, spider))
spider_status = 'error'
self.ts_lock.acquire(blocking=True)
if spider_status == 'ok':
self._run_countdown(project=project, spider=spider)
self.spider_task_dic[item] = spider_status
self.ts_lock.release()
def _spider_args_method(self, args, hash_str):
args_raw = args.copy()
if args:
method = args.pop('method', 'normal')
if method == 'auto_increment':
next_args = {k: str(int(v)+1) if isinstance(v, int) or (isinstance(v, str) and v.isdigit()) else v for k, v in args.items()}
elif isinstance(method, dict):
ex_md = method.get('expression')
fc_md = method.get('function')
if ex_md:
next_args = eval(ex_md)
if fc_md:
exec(fc_md)
else:
next_args = args
next_args.update({'method': method})
self.db.update('SpiderScheduleModel', update_dic={'args': next_args}, filter_dic={"hash_str": hash_str})
return args
return args_raw
def spiders_runtime(self, project, spider):
self.db_lock.acquire()
res = self.db.get(model_name='SpiderMonitor', key_list=['runtime'],
filter_dic={'project': project, 'spider': spider})
self.db_lock.release()
spider_list = [int(x.runtime) for x in res if x.runtime.isdigit()] if res else [0]
return sum(spider_list) / len(spider_list)
def list_projects(self):
res = requests.get(url=self.listproject_url)
projects = {}
if res:
projects_list = json.loads(res.content).get('projects')
if projects_list:
projects = set(projects_list)
return projects
def cal_time_sep(self,
year='*',
month='*',
day='*',
week='*',
hour='*',
minute='*',
second='*',
schedule_str=None,
is_str=False
):
"""
"%Y-%m-%d %H:%M:%S %w"
"""
if is_str:
s = [int(x.strip()) for x in schedule_str.split(',')]
time_sep = (datetime.datetime(s[0], s[1], s[2], s[3], s[4], s[5]) - datetime.datetime.now()).total_seconds()
return time_sep
y = int(time.strftime("%Y", time.localtime()))
if year != '*' and '*' in year:
y = int(year.split('/')[-1]) + y
elif year.isdigit():
y = int(year)
if week == '*':
m = int(time.strftime("%m", time.localtime()))
if month != '*' and '*' in month:
m_raw = int(month.split('/')[-1])
if m_raw >= 12:
raise ValueError('month value is too large, please set the year instead')
m = m_raw + m
if m > 12:
y += m // 12
m = m % 12
elif month.isdigit():
m = int(month)
days_in_this_month = self.how_many_days_in_this_month(y, m)
d = int(time.strftime("%d", time.localtime()))
if day != '*' and '*' in day:
d_raw = int(day.split('/')[-1])
if d_raw > days_in_this_month:
raise ValueError('day value is too large, please set the month or the year instead')
d = d_raw + d
if d > days_in_this_month:
d = d - days_in_this_month
m += 1
if m > 12:
y += 1
m = m - 12
elif day.isdigit():
d = int(day)
days_in_this_month = self.how_many_days_in_this_month(y, m)
H = int(time.strftime("%H", time.localtime()))
if hour != '*' and '*' in hour:
H_raw = int(hour.split('/')[-1])
if H_raw > 24:
raise ValueError('hour value is too large, please set the day instead')
H = H_raw + H
if H >= 24:
H = H - 24
d += 1
if d > days_in_this_month:
d = d - days_in_this_month
m += 1
if m > 12:
y += 1
m = m - 12
elif hour.isdigit():
H = int(hour)
days_in_this_month = self.how_many_days_in_this_month(y, m)
M = int(time.strftime("%M", time.localtime()))
if minute != '*' and '*' in minute:
M_raw = int(minute.split('/')[-1])
if M_raw > 60:
raise ValueError('minute value is too large, please set the hour instead')
M = M_raw + M
if M >= 60:
M = M - 60
H += 1
if H >= 24:
H = H - 24
d += 1
if d > days_in_this_month:
d = d - days_in_this_month
m += 1
if m > 12:
y += 1
m = m - 12
elif minute.isdigit():
M = int(minute)
days_in_this_month = self.how_many_days_in_this_month(y, m)
S = int(time.strftime("%S", time.localtime()))
if second != '*' and '*' in second:
S_raw = int(second.split('/')[-1])
if S_raw > 60:
raise ValueError('second value is too large, please set the minute instead')
S = S_raw + S
if S >= 60:
S = S - 60
M += 1
if M >= 60:
M = M - 60
H += 1
if H >= 24:
H = H - 24
d += 1
if d > days_in_this_month:
d = d - days_in_this_month
m += 1
if m > 12:
y += 1
m = m - 12
elif second.isdigit():
S = int(second)
time_sep = eval(
"(datetime.datetime({},{},{}, {},{},{}) - datetime.datetime.now()).total_seconds()".format(y, m, d, H,
M, S))
else:
week_in_this_year = int(time.strftime("%U", time.localtime()))
w = int(time.strftime("%w", time.localtime()))
if '*' in week:
w_raw = int(week.split('/')[-1])
if w_raw >= 7:
raise ValueError('week value is too large, please set the day or the month instead')
if w_raw < w:
week_in_this_year += 1
w = w_raw
if week_in_this_year > 53:
y += 1
week_in_this_year = week_in_this_year - 53
elif week.isdigit():
w = int(week)
if int(week) < w:
week_in_this_year += 1
H = int(time.strftime("%H", time.localtime()))
if hour != '*' and '*' in hour:
H_raw = int(hour.split('/')[-1])
if H_raw >= 24:
raise ValueError('hour value is too large, please set the day instead')
H = H_raw + H
if H >= 24:
H = H - 24
w += 1
if w >= 7:
w = w - 7
week_in_this_year += 1
if week_in_this_year > 53:
y += 1
week_in_this_year = week_in_this_year - 53
elif hour.isdigit():
H = int(hour)
M = int(time.strftime("%M", time.localtime()))
if minute != '*' and '*' in minute:
M_raw = int(minute.split('/')[-1])
if M_raw >= 60:
raise ValueError('minute value is too large, please set the hour instead')
M = M_raw + M
if M >= 60:
M = M - 60
H += 1
if H >= 24:
H = H - 24
w += 1
if w > 7:
w = w - 7
week_in_this_year += 1
if week_in_this_year > 53:
y += 1
week_in_this_year = week_in_this_year - 53
elif minute.isdigit():
M = int(minute)
S = int(time.strftime("%S", time.localtime()))
if second != '*' and '*' in second:
S_raw = int(second.split('/')[-1])
if S_raw >= 60:
raise ValueError('second value is too large, please set the minute instead')
S = S_raw + S
if S >= 60:
S = S - 60
M += 1
if M >= 60:
M = M - 60
H += 1
if H >= 24:
H = H - 24
w += 1
if w > 7:
w = w - 7
week_in_this_year += 1
if week_in_this_year > 53:
y += 1
week_in_this_year = week_in_this_year - 53
elif second.isdigit():
S = int(second)
if S >= 60:
S = S - 60
M += 1
if M >= 60:
M = M - 60
H += 1
if H >= 24:
H = H - 24
w += 1
if w > 7:
w = w - 7
week_in_this_year += 1
if week_in_this_year > 53:
y += 1
week_in_this_year = week_in_this_year - 53
m, d = self.get_month_and_days_by_week(year=y, week_in_this_year=week_in_this_year, week=w)
time_sep = eval(
"(datetime.datetime({},{},{}, {},{},{}) - datetime.datetime.now()).total_seconds()".format(y, m, d, H,
M, S))
return time_sep
def get_month_and_days_by_week(self, year, week_in_this_year, week):
days = week_in_this_year * 7 + week
if (year % 4 == 0 and year % 100 != 0) or (year % 400 == 0):
Fe = 29
else:
Fe = 28
month_lis = [31, Fe, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
month_count = 1
days_count = 0
for month_days in month_lis:
days = days - month_days
if days > 0:
month_count += 1
elif days == 0:
days_count = 0
month_count += 1
break
else:
days_count = days + month_days
break
return [month_count, days_count]
def how_many_days_in_this_month(self, y, m):
if m in (1, 3, 5, 7, 8, 10, 12):
days = 31
elif m in (4, 6, 9, 11):
days = 30
else:
if (y % 4 == 0 and y % 100 != 0) or (y % 400 == 0):
days = 29
else:
days = 28
return days
def is_system_ok(self):
is_pass = True
cpu_list = psutil.cpu_percent(interval=1, percpu=True)
memory_percent = psutil.virtual_memory().percent
if cpu_list and memory_percent:
is_cpu_ok = True
if min(cpu_list) > self.CPU_THRESHOLD:
is_cpu_ok = False
is_memo_ok = True
if memory_percent > self.MEMORY_THRESHOLD:
is_memo_ok = False
if not is_cpu_ok or not is_memo_ok:
is_pass = False
return is_pass
def _check_key(self, key):
key_lis = [x for x in key]
count_dic = dict()
for ksl in self._keys_set_lis:
o_key = ''.join(ksl)
score = 0
for k in key_lis:
if k in ksl:
score += 1
count_dic[o_key] = score
best_math = sorted(count_dic, key=count_dic.__getitem__, reverse=True)[0]
return best_math
def _run_countdown(self, project, spider):
db_schedule = self.db.get(model_name='SpiderScheduleModel', key_list=['id', 'runtime'],
filter_dic={'project': project, 'spider': spider})
run_time_in_db = [x.runtime for x in db_schedule][0] if db_schedule else 0
the_id = [x.id for x in db_schedule][0] if db_schedule else None
if run_time_in_db > 0 and the_id is not None:
rt = int(run_time_in_db) - 1
self.db.update(model_name='SpiderScheduleModel', update_dic={"runtime": rt}, filter_dic={"id": the_id})
class FSProtocol(basic.LineReceiver):
class State(Enum):
IDLE = 0
POLLING = 1
POLLED = 2
RAW = 3
def __init__(self):
self.log = Logger()
self.setIdleState()
def setIdleState(self):
self.my_state = self.State.IDLE
self.wait_for_list = None
self.file_transfer_peer = None
self.setLineMode()
def connectionMade(self):
self.log.info("New connection established: {peer!r}, {pid!s}",
peer=self.transport.getPeer(),
pid=id(self))
self.factory.protocols.append(self)
self.transport.write(b'Hello form server\r\n')
def send(self, command, argument):
if command != "":
command += " "
self.transport.write(command.upper().encode() + argument.encode() +
b'\r\n')
# self.log.info("Sending to {peer!r}: {command!s} {argument!s}",
# command=command, argument=argument)
def doReject(self, line):
self.log.error("We had to say no to {peer!r}: {line}",
peer=self.transport.getPeer(),
line=line)
self.transport.write(b'REJECT ' + line.encode() + b'\r\n')
# def doRequest(self, regex):
# self.log.debug("server asks for files: {regex!s}", regex=regex)
# self.transport.write(b'REQUEST ' + regex.encode() + b'\r\n')
def requestFiles(self, regex):
'''
Called by factory, if one of users requested a search
'''
if self.my_state != self.State.IDLE or self.wait_for_list is not None:
return defer.fail(RuntimeError("Connection is busy"))
# Go to POLLED state, so no one will interrupt us
self.my_state = self.State.POLLED
self.send("REQUEST", regex)
def debuglogc(data):
self.log.debug("wait_for_list:requestFiles:callback: {data!r}",
data=data)
return data
def debugloge(err):
self.log.debug("wait_for_list:requestFiles:errback: {err!r}",
err=err)
return err
self.wait_for_list = defer.Deferred().addCallbacks(
debuglogc, debugloge)
# We create Deferred, that will be fired as the response comes from user
return self.wait_for_list
def trySetStateIdle(self):
# Some operation has finished
# Check if there is file request
# if len(self.outer_req) > 0:
# # Set state
# self.state = self.State.POLLED
# # Ask peer
# regex,d = self.outer_req[0]
# self.transport.write("TODO some message" + regex + b'\r\n')
pass
def handleFind(self, regex):
'''
Hanles the FIND request from user
'''
self.my_state = self.State.POLLING
# The factory will ask all the others about our regexp
# The result here is the string
d = self.factory.getFiles(self, regex)
def response(ans):
self.log.info("FIND command returns next {filelist!s}",
filelist=ans)
self.send('RESPONSE', ans)
def error(e):
self.doReject(e.getErrorMessage())
def idle(ignore):
self.my_state = self.State.IDLE
d.addCallbacks(response, error)
d.addBoth(idle)
def lineReceived(self, line):
self.log.info("lineReceived: {peer!r} got message: {line!s}",
peer=self.transport.getPeer(),
line=line)
line = line.decode()
data = cleanInput(line)
if len(data) == 0 or data == '':
self.log.info("Oh, never mind, it's gone")
self.doReject("Seems like a bunch of tabs")
return
command = data[0].upper()
argument = data[1] if len(data) > 1 else None
# Analize the command
if command == 'FIND': # Find files from other users
if self.my_state != self.State.IDLE:
self.doReject("Another operation is in run")
return
if argument is None:
self.doReject("Regex expected")
return
self.handleFind(argument)
elif command == 'FILES': # Polled user returns list of files here
if self.my_state != self.State.POLLED:
self.doReject(
"Got list of files, but there is no request for them")
return
# self.log.debug("Got list of files from {peer!r}", peer=self.transport.getPeer())
if argument is None:
self.wait_for_list.errback(
RuntimeError("This peer has no such files"))
else:
# Return self id and list of files
self.wait_for_list.callback(':'.join([str(id(self)),
argument]))
# Cleanup
self.wait_for_list = None
self.my_state = self.State.IDLE
elif command == 'GET': # User asks for file
'''
This is a routine of connection, whitch needs file
'''
# Check if we ready
if self.my_state != self.State.IDLE:
self.doReject("Can not give you file at the moment")
return
# Get arguments
if argument is None:
self.doReject("Expected filename for GET operation")
return
try:
pid, filename = argument.split(':', 1)
except (IndexError):
self.doReject("not valid filename")
return
# Find the peer
peer = self.factory.findConnById(pid)
# Check if we are ready
if peer is None:
self.doReject("Sorry, the peer seems afk")
return
if peer.my_state != peer.State.IDLE:
self.doReject("Sorry, the peer seems busy")
return
# Ready to begin, don't interrupt us please
peer.my_state = peer.State.RAW
self.my_state = self.State.RAW
peer.file_transfer_peer = self
self.file_transfer_peer = peer
peer.send("OBTAIN", filename)
self.send("RAW", filename)
self.setRawMode()
peer.setRawMode()
elif command == 'ALICE': # User remembers how this project started
self.send('', "nice girl")
else:
self.doReject("not a protocol command: %s" % command)
def rawDataReceived(self, data):
'''
Should be called, if user sends data to peer in response
'''
if self.file_transfer_peer is None:
self.log.warning("RECIEVING RAW DATA WITHOUT PEER TO RESEND")
return
peer = self.file_transfer_peer
self.log.info("RAW: resending data: %d KB" % len(data))
peer.transport.write(data)
if not data.endswith(b'\r\n'):
# Transmission not done yet
return
self.log.info("RAW: Transmission complete")
peer.setIdleState()
self.setIdleState()
def connectionLost(self, reason):
self.log.info("Connection closed: {peer!r}, {reason!r}",
peer=self.transport.getPeer(),
reason=reason)
self.factory.protocols.remove(self)
class TimeSchedule:
def __init__(self, lock, host='127.0.0.1', port='6800'):
config = Config()
self.db = glv.get_value(key='sqlite_db')
self.user_name = config.get('auth_username', '')
self.user_password = config.get('auth_password', '')
self.start_time = time.strftime("%Y %m %d %H %M %S", time.localtime())
self.server_port = 'http://{}:{}/'.format(host, port)
self.schedule_post_url = 'http://{}:{}/schedule.json'.format(
host, port)
self.listproject_url = 'http://{}:{}/listprojects.json'.format(
host, port)
self.spider_task_dic = dict()
self.projects = None
self.db_lock = lock
self.ts_lock = threading.Lock()
self.CPU_THRESHOLD = 93
self.MEMORY_THRESHOLD = 96
self.schedule_logger = Logger(namespace='- Scheduler -')
def run(self):
time.sleep(3)
self.projects = self.list_projects()
self.schedule_logger.info('scheduler is running')
count = 1
while True:
schedule_sta = self.task_scheduler()
if not schedule_sta and count == 1:
self.schedule_logger.info('No Scheduled Spider in Database')
count += 1
elif not schedule_sta and count != 1:
count += 1
else:
count = 1
time.sleep(1)
def task_scheduler(self):
self.ts_lock.acquire(blocking=True)
self.db_lock.acquire()
db_result = self.db.get(
model_name='SpiderScheduleModel',
key_list=['project', 'spider', 'schedule', 'args', 'status'])
self.db_lock.release()
self.ts_lock.release()
schedule_list_raw = [{
'project': x.project,
'spider': x.spider,
'schedule': x.schedule,
'args': x.args,
'status': x.status
} for x in db_result if int(x.status) != 0] if db_result else []
schedule_sta = False
if schedule_list_raw:
for each_schedule in schedule_list_raw:
project = each_schedule.get('project')
if project in self.projects:
schedule = each_schedule.get('schedule').replace('\\', '')
try:
schedule = json.loads(schedule)
except:
schedule = eval(schedule)
try:
next_time_sep = self.cal_time_sep(**schedule)
next_time_sep = int(next_time_sep) + 1
if next_time_sep > 1:
each_schedule['schedule'] = next_time_sep
item = '{}-{}'.format(each_schedule['project'],
each_schedule['spider'])
self.ts_lock.acquire(blocking=True)
if self.spider_task_dic.get(item) != 'waiting':
self.spider_task_dic[item] = 'waiting'
t = threading.Thread(target=self.poster,
args=(each_schedule, ))
try:
t.start()
except Exception as THError:
self.schedule_logger.warn(
'start new job error [ {} ]: {}'.
format(item, THError))
self.ts_lock.release()
except ValueError:
self.schedule_logger.error(
'spider runtime schedule error, please check the database'
)
schedule_sta = True
return schedule_sta
def poster(self, dic):
status = int(dic.pop('status'))
project = dic.get('project')
spider = dic.get('spider')
job_str = " %s-%s " % (project, spider)
args = dic.get('args').replace('\\', '')
if args:
args = eval(args)
wait_time = dic.get('schedule')
item = '{}-{}'.format(project, spider)
if project and spider:
data = {
'project': project,
'spider': spider,
'un': self.user_name,
'pwd': self.user_password
}
if args:
data.update(args)
self.schedule_logger.info(
'job {} is waiting, countdown {}s'.format(item, wait_time))
time.sleep(wait_time - 1)
another_wait_time = 0
spider_runtime_avg = self.spiders_runtime(project=project,
spider=spider)
if status == 1:
while not self.is_system_ok():
self.schedule_logger.warn(
'system is fully functioning, wait another 2 seconds to post schedule'
)
time.sleep(2)
another_wait_time += 3
if another_wait_time >= (wait_time - spider_runtime_avg):
self.schedule_logger.warning(
'wait too long, cancel the job %s' % job_str)
return None
res = json.loads(
requests.post(url=self.schedule_post_url,
data=data).content)
elif status == 2:
res = json.loads(
requests.post(url=self.schedule_post_url,
data=data).content)
elif status == 3:
res = json.loads(
requests.post(url=self.schedule_post_url,
data=data).content)
else:
res = json.loads(
requests.post(url=self.schedule_post_url,
data=data).content)
spider_status = res.get('status')
if spider_status != 'ok':
spider_status = 'error'
else:
self.schedule_logger.error(
'job project: {}, spider: {} post fail!'.format(
project, spider))
spider_status = 'error'
self.ts_lock.acquire(blocking=True)
self.spider_task_dic[item] = spider_status
self.ts_lock.release()
def spiders_runtime(self, project, spider):
self.db_lock.acquire()
res = self.db.get(model_name='SpiderMonitor',
key_list=['runtime'],
filter_dic={
'project': project,
'spider': spider
})
self.db_lock.release()
spider_list = [int(x.runtime) for x in res
if x.runtime.isdigit()] if res else [0]
return sum(spider_list) / len(spider_list)
def list_projects(self):
res = requests.get(url=self.listproject_url)
projects = {}
if res:
projects_list = json.loads(res.content).get('projects')
if projects_list:
projects = set(projects_list)
return projects
def cal_time_sep(
self,
year='*',
month='*',
day='*',
week='*',
hour='*',
minute='*',
second='*',
):
"""
"%Y-%m-%d %H:%M:%S %w"
"""
y = int(time.strftime("%Y", time.localtime()))
if year != '*' and '*' in year:
y = int(year.split('/')[-1]) + y
elif year.isdigit():
y = int(year)
if week == '*':
m = int(time.strftime("%m", time.localtime()))
if month != '*' and '*' in month:
m_raw = int(month.split('/')[-1])
if m_raw >= 12:
raise ValueError(
'month value is too large, please set the year instead'
)
m = m_raw + m
if m > 12:
y += m // 12
m = m % 12
elif month.isdigit():
m = int(month)
days_in_this_month = self.how_many_days_in_this_month(y, m)
d = int(time.strftime("%d", time.localtime()))
if day != '*' and '*' in day:
d_raw = int(day.split('/')[-1])
if d_raw > days_in_this_month:
raise ValueError(
'day value is too large, please set the month or the year instead'
)
d = d_raw + d
if d > days_in_this_month:
d = d - days_in_this_month
m += 1
if m > 12:
y += 1
m = m - 12
elif day.isdigit():
d = int(day)
days_in_this_month = self.how_many_days_in_this_month(y, m)
H = int(time.strftime("%H", time.localtime()))
if hour != '*' and '*' in hour:
H_raw = int(hour.split('/')[-1])
if H_raw > 24:
raise ValueError(
'hour value is too large, please set the day instead')
H = H_raw + H
if H >= 24:
H = H - 24
d += 1
if d > days_in_this_month:
d = d - days_in_this_month
m += 1
if m > 12:
y += 1
m = m - 12
elif hour.isdigit():
H = int(hour)
days_in_this_month = self.how_many_days_in_this_month(y, m)
M = int(time.strftime("%M", time.localtime()))
if minute != '*' and '*' in minute:
M_raw = int(minute.split('/')[-1])
if M_raw > 60:
raise ValueError(
'minute value is too large, please set the hour instead'
)
M = M_raw + M
if M >= 60:
M = M - 60
H += 1
if H >= 24:
H = H - 24
d += 1
if d > days_in_this_month:
d = d - days_in_this_month
m += 1
if m > 12:
y += 1
m = m - 12
elif minute.isdigit():
M = int(minute)
days_in_this_month = self.how_many_days_in_this_month(y, m)
S = int(time.strftime("%S", time.localtime()))
if second != '*' and '*' in second:
S_raw = int(second.split('/')[-1])
if S_raw > 60:
raise ValueError(
'second value is too large, please set the minute instead'
)
S = S_raw + S
if S >= 60:
S = S - 60
M += 1
if M >= 60:
M = M - 60
H += 1
if H >= 24:
H = H - 24
d += 1
if d > days_in_this_month:
d = d - days_in_this_month
m += 1
if m > 12:
y += 1
m = m - 12
elif second.isdigit():
S = int(second)
time_sep = eval(
"(datetime.datetime({},{},{}, {},{},{}) - datetime.datetime.now()).total_seconds()"
.format(y, m, d, H, M, S))
else:
week_in_this_year = int(time.strftime("%U", time.localtime()))
w = int(time.strftime("%w", time.localtime()))
if '*' in week:
w_raw = int(week.split('/')[-1])
if w_raw >= 7:
raise ValueError(
'week value is too large, please set the day or the month instead'
)
if w_raw < w:
week_in_this_year += 1
w = w_raw
if week_in_this_year > 53:
y += 1
week_in_this_year = week_in_this_year - 53
elif week.isdigit():
w = int(week)
if int(week) < w:
week_in_this_year += 1
H = int(time.strftime("%H", time.localtime()))
if hour != '*' and '*' in hour:
H_raw = int(hour.split('/')[-1])
if H_raw >= 24:
raise ValueError(
'hour value is too large, please set the day instead')
H = H_raw + H
if H >= 24:
H = H - 24
w += 1
if w >= 7:
w = w - 7
week_in_this_year += 1
if week_in_this_year > 53:
y += 1
week_in_this_year = week_in_this_year - 53
elif hour.isdigit():
H = int(hour)
M = int(time.strftime("%M", time.localtime()))
if minute != '*' and '*' in minute:
M_raw = int(minute.split('/')[-1])
if M_raw >= 60:
raise ValueError(
'minute value is too large, please set the hour instead'
)
M = M_raw + M
if M >= 60:
M = M - 60
H += 1
if H >= 24:
H = H - 24
w += 1
if w > 7:
w = w - 7
week_in_this_year += 1
if week_in_this_year > 53:
y += 1
week_in_this_year = week_in_this_year - 53
elif minute.isdigit():
M = int(minute)
S = int(time.strftime("%S", time.localtime()))
if second != '*' and '*' in second:
S_raw = int(second.split('/')[-1])
if S_raw >= 60:
raise ValueError(
'second value is too large, please set the minute instead'
)
S = S_raw + S
if S >= 60:
S = S - 60
M += 1
if M >= 60:
M = M - 60
H += 1
if H >= 24:
H = H - 24
w += 1
if w > 7:
w = w - 7
week_in_this_year += 1
if week_in_this_year > 53:
y += 1
week_in_this_year = week_in_this_year - 53
elif second.isdigit():
S = int(second)
if S >= 60:
S = S - 60
M += 1
if M >= 60:
M = M - 60
H += 1
if H >= 24:
H = H - 24
w += 1
if w > 7:
w = w - 7
week_in_this_year += 1
if week_in_this_year > 53:
y += 1
week_in_this_year = week_in_this_year - 53
m, d = self.get_month_and_days_by_week(
year=y, week_in_this_year=week_in_this_year, week=w)
time_sep = eval(
"(datetime.datetime({},{},{}, {},{},{}) - datetime.datetime.now()).total_seconds()"
.format(y, m, d, H, M, S))
return time_sep
def get_month_and_days_by_week(self, year, week_in_this_year, week):
days = week_in_this_year * 7 + week
if (year % 4 == 0 and year % 100 != 0) or (year % 400 == 0):
Fe = 29
else:
Fe = 28
month_lis = [31, Fe, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
month_count = 1
days_count = 0
for month_days in month_lis:
days = days - month_days
if days > 0:
month_count += 1
elif days == 0:
days_count = 0
month_count += 1
break
else:
days_count = days + month_days
break
return [month_count, days_count]
def how_many_days_in_this_month(self, y, m):
if m in (1, 3, 5, 7, 8, 10, 12):
days = 31
elif m in (4, 6, 9, 11):
days = 30
else:
if (y % 4 == 0 and y % 100 != 0) or (y % 400 == 0):
days = 29
else:
days = 28
return days
def is_system_ok(self):
is_pass = True
cpu_list = psutil.cpu_percent(interval=1, percpu=True)
memory_percent = psutil.virtual_memory().percent
if cpu_list and memory_percent:
is_cpu_ok = True
if min(cpu_list) > self.CPU_THRESHOLD:
is_cpu_ok = False
is_memo_ok = True
if memory_percent > self.MEMORY_THRESHOLD:
is_memo_ok = False
if not is_cpu_ok or not is_memo_ok:
is_pass = False
return is_pass
