def destroy(self, request, *args, **kwargs):
instance = self.get_object()
self.perform_destroy(instance)
ret = {}
ret['code'] = 0 #有没权限的时候有两层code
#####################################
# 日志删除时间和具体操作
print("hihhi")
write_to_log(self.request.user, instance, '0', request)
#####################################
return Response(ret)
def get_phantomjs_driver():
conf = dict(service_args=['--ssl-protocol=any'])
if os.environ.get('OPENSHIFT_DATA_DIR'):
capabilities = DesiredCapabilities.PHANTOMJS
capabilities['acceptSslCerts'] = True
driver = webdriver.Remote(command_executor='http://'+os.environ.get(
'OPENSHIFT_PYTHON_IP')+':15005', desired_capabilities=capabilities,
keep_alive=True)
write_to_log('Connected to remote web driver')
return driver
driver = webdriver.PhantomJS(**conf)
return driver
def perform_update(self, serializer):
instance = self.get_object()
print(instance.return_amount)
serializer.save()
data = serializer.data
print(instance.return_amount)
#####################################
# 日志修改时间和具体操作
write_to_log(self.request.user, instance, '1')
#####################################
serializer._data = {}
serializer._data['code'] = 0
serializer._data['data'] = data
def destroy(self, request, *args, **kwargs):
instance = self.get_object()
self.perform_destroy(instance)
ret = {}
ret['code'] = 0
#####################################
# 项目删除时间和具体操作
#print(instance)
print(instance.state)
if instance.state == '1':
write_to_log(self.request.user, instance, '0', request)
# oobj = repr(instance)
# OperatorLog.objects.create(otime=datetime.datetime.now(), oman=self.request.user, oobj=oobj, otype='0')
#####################################
return Response(ret)
def is_altered(self, request, pk=None, *args, **kwargs):
'''修改
字段仅仅限于is_delete,进行中的项目要通知管理员'''
aimpro = Project.objects.get(id=pk)
instance = self.get_object()
print(self.action)
#####################################
# 项目修改时间和具体操作
if instance.state == '1':
write_to_log(self.request.user, instance, '1', request)
#####################################
#if aimpro.state in ['１',]:
# 写操作日志，对应visit字段为False表示未读.而每个管理员登录后，都会更新当前的未读的操作日志有多少条，但这些操作日志是共享的
self.partial_update(request, *args, **kwargs)
res = {}
res['code'] = 0
return Response(res)
def _handle_tamper_file(self, filepath):
relative_pos = filepath[self.__website_path_len:].lstrip('/')
backup_filepath = os.path.join(self.backup_path, relative_pos)
if not os.path.exists(backup_filepath):
content = " 警告：发现可疑文件%s！" % filepath
# move to isolation area.
filename = os.path.basename(filepath)
if not os.path.exists(self.isolation_path):
os.mkdir(self.isolation_path)
if self.auto_cure:
try:
os.replace(filepath,
os.path.join(self.isolation_path, filename))
content += '已隔离到%s' % self.isolation_path
except:
print('Isolate Error: %s is in use!' % filepath)
else:
if not os.path.exists(filepath):
content = ' 警告：文件%s被删除！' % filepath
else:
if filecmp.cmp(filepath, backup_filepath):
return
content = " 警告：文件%s的内容发生变动！" % filepath
if self.auto_cure:
try:
shutil.copy2(backup_filepath, filepath)
content += "文件已恢复。"
except Exception as err_msg:
#Provisionally skip the file in use.
content += '文件恢复失败：' + str(err_msg)
if self.check_if_send(filepath):
try:
subject = "页面篡改问题"
send_alarm_mail(self.mail_receivers, subject, content)
except Exception as err_msg:
content += '发送告警邮件失败：' + err_msg
write_to_log(content)
def driver_start(self):
write_to_log('Preparing driver to parsing')
self.driver = self.get_phantomjs_driver()
time.sleep(5)
write_to_log('Driver trying to open url %s' % self.url)
self.driver.get(self.url)
write_to_log('Url %s opened successfully' % self.url)
self.driver.set_window_size(1920, 1080)
assert self.url in self.driver.current_url, "Can't open url: %s" % self.url
def parse_url_channels(self):
# run_phantomjs()
time.sleep(20)
self.driver_start()
write_to_log('Start channels parsing')
func_tm = time.time()
page_height = 0
elements = {}
scroll_height_script = """ return window.innerHeight + window.scrollY """
while page_height != self.driver.execute_script(scroll_height_script):
page_height = self.driver.execute_script(scroll_height_script)
self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(5)
channels = self.driver.find_elements_by_css_selector(self.get_channel_css_selector())
write_to_log('Found %s channels' % len(channels))
for channel in channels:
time.sleep(1)
name = channel.find_element_by_css_selector(
'span.tv-channel-title__text').text.encode('utf-8')
href = channel.get_attribute('href').encode('utf-8')
icon = channel.find_elements_by_css_selector(
'div.tv-channel-title__icon > span[class$="image_type_channel"] > span')
if icon:
icon = self.get_background_image(icon[0]).encode('utf-8')
if (href is not None) and (href not in elements.keys()):
elements[href] = {'name': name, 'icon': icon}
save_records = SaveRecordsToDb()
new_elements_count = save_records.save_channels_to_db(elements)
func_tm = int(time.time()-func_tm)
text_for_log = 'Channels parsed successfully.{elements_count} new channels.' \
'Execution time: {func_tm}'.\
format(elements_count=new_elements_count,
func_tm=hours_minutes_seconds_from_seconds(func_tm))
send_email(subject='Parser notification',
text=text_for_log)
write_to_log(text_for_log)
SaveRecordsToDb.insert_log_info(execution_time=func_tm, new_items=new_elements_count)
self.driver.close()
def run_scheduler():
try:
write_to_log('Run scheduler')
# if type_of_parser == 'channels':
# write_to_log('Preparing to parse channels')
# driver.parse_url_channels()
# elif type_of_parser == 'programs':
# write_to_log('Preparing to parse programs')
# driver.parse_tv_programs()
schedule.every().day.at("15:29").do(driver.parse_url_channels)
schedule.every().day.at("15:59").do(driver.parse_tv_programs)
# schedule.every().monday.at('"06:00"').do(driver.parse_tv_programs)
# schedule.every().sunday.at("'12:00'").do(driver.parse_url_channels)
while True:
schedule.run_pending()
time.sleep(2)
# else:
# subprocess.Popen("echo Unknown argument for parser."
# "Type --help for more information".split())
except Exception as e:
write_to_log(error=e)
send_email(subject='Parse error. Exception', exception=e)
write_to_log('Kill scheduler after error')
kill_process(get_pid_by_name('run_scheduler.py'))
def import_projectdata_excel(self, request):
admin_user = request.user
ret = {}
ret['data'] = {}
print("ghgggg")
file = request.FILES.get('file')
# aftername＝time.time()
filename = "./files/" + "1"
with open(filename, 'wb+') as destination:
for chunk in file.chunks():
destination.write(chunk)
data = xlrd.open_workbook(filename)
table = data.sheets()[0]
nrows = table.nrows
ncols = table.ncols
print("aaa")
if ncols != 8:
print("bbb")
ret['detail'] = "文件格式与模板不符，请下载最新模板填写！"
return JsonResponse(ret)
rtable = {}
mobile_list = []
dup = {}
try:
for i in range(1, nrows):
temp = []
for j in range(ncols):
cell = table.cell(i, j)
value = cell.value
print(value)
if j == 0:
id = int(value)
print(id)
project = Project.objects.get(id=id)
temp.append(id)
elif j == 2:
value = str(value).strip()
if value == u"首投":
temp.append(False)
elif value == u"复投":
temp.append(True)
else:
raise Exception(u"必须为首投或复投。")
elif j == 3:
if (cell.ctype != 3):
raise Exception("投资日期列格式错误，请修改后重新提交。")
else:
time2 = xlrd.xldate.xldate_as_datetime(value, 0)
temp.append(time2)
elif j == 4:
try:
mobile = str(int(value)).strip()
except Exception as e:
mobile = str(value).strip()
if len(mobile) == 11:
temp.append(mobile)
else:
raise Exception("手机号必须是11位，请修改后重新提交。")
elif j == 5 or j == 7:
try:
temp.append(Decimal(value))
except:
raise Exception("投资金额必须为数字")
elif j == 6:
try:
temp.append(int(value))
except Exception as e:
raise Exception("投资标期必须为数字，请修改后重新提交。")
else:
temp.append(value)
tid = temp[0]
if not temp[2]:
print(dup)
if tid in dup:
if temp[4] in dup[tid]:
continue
else:
dup[tid].append(temp[4])
else:
dup[tid] = [
temp[4],
]
if tid in rtable:
rtable[tid].append(temp)
else:
rtable[tid] = [
temp,
]
except Exception as e:
logger.info(e)
# traceback.print_exc()
ret['detail'] = e.__str__()
return JsonResponse(ret)
####开始去重
investdata_list = []
duplicate_mobile_list = []
try:
with cache.lock('investdata_first', timeout=2):
# db_key = DBlock.objects.select_for_update().get(index='investdata')
for id, values in rtable.items():
print(id)
temp = ProjectInvestDataModel.objects.filter(
project_id=id).values('invest_mobile')
db_mobile_list = map(lambda x: x['invest_mobile'], temp)
for item in values:
pid = item[0]
time = item[3]
mob = item[4]
is_futou = item[2]
amount = item[5]
term = item[6]
settle = item[7]
source = ''
remark = ''
if not is_futou and mob in db_mobile_list:
duplicate_mobile_list.append(mob)
else:
obj = ProjectInvestDataModel(project_id=pid,
invest_mobile=mob,
settle_amount=settle,
invest_amount=amount,
invest_term=term,
invest_time=time,
state='0',
remark=remark,
source=source,
is_futou=is_futou,
apply_man=admin_user)
investdata_list.append(obj)
ProjectInvestDataModel.objects.bulk_create(investdata_list)
except Exception as e:
logger.info(e)
# traceback.print_exc()
ret['detail'] = e.__str__()
return JsonResponse(ret)
succ_num = len(investdata_list)
duplic_num2 = len(duplicate_mobile_list)
duplic_num1 = nrows - 1 - succ_num - duplic_num2
duplic_mobile_list_str = u'，'.join(duplicate_mobile_list)
ret['data'].update(num=succ_num,
dup1=duplic_num1,
dup2=duplic_num2,
anum=nrows - 1,
dupstr=duplic_mobile_list_str)
ret['code'] = 0
#####################################
#日志记录导入人的id和导入文件名
write_to_log(self.request.user, filename, '2', request)
#####################################
return JsonResponse(ret)
def import_audit_projectdata_excel(self, request):
admin_user = request.user
ret = {'code': -1}
ret['data'] = {}
# print(dir(request))
file = request.FILES.get('file')
aftername = time.time()
filename = "./files/audit_" + str(int(aftername * 1000))
with open(filename, 'wb+') as destination:
for chunk in file.chunks():
destination.write(chunk)
data = xlrd.open_workbook(filename)
table = data.sheets()[0]
print(table)
nrows = table.nrows
ncols = table.ncols
if ncols != 13:
ret['detail'] = "文件格式与模板不符，请下载最新模板填写！"
return JsonResponse(ret)
rtable = []
try:
for i in range(1, nrows):
row = table.row_values(i)
temp = {}
id = int(row[0])
project_id = int(row[1])
term = int(row[7])
mobile = row[5]
consume = Decimal(row[8])
remark = row[12]
date = row[4]
date = xlrd.xldate.xldate_as_datetime(date, 0)
try:
mobile = str(int(mobile)).strip()
except Exception as e:
mobile = str(mobile).strip()
if len(mobile) != 11:
raise Exception("手机号必须是11位，请修改后重新提交。")
if row[9] == "是":
result = True
temp['state'] = '1'
elif row[9] == "否":
result = False
temp['state'] = '0'
else:
raise Exception("审核结果必须为是或否。")
if row[10]:
return_amount = Decimal(row[10])
if return_amount > consume:
raise Exception("返现金额不能大于结算金额，请检查表格")
elif result:
raise Exception("审核结果为是时，返现金额不能为空或零。")
else:
return_amount = 0
if row[11] == "网站":
source = 'site'
elif row[11] == "渠道":
source = 'channel'
else:
raise Exception("必须为网站或渠道。")
temp['id'] = id
temp['project_id'] = project_id
temp['source'] = source
temp['return_amount'] = return_amount
temp['consume'] = consume
temp['remark'] = remark
temp['mobile'] = mobile
temp['date'] = date
temp['term'] = term
rtable.append(temp)
except Exception as e:
logger.info(e)
print(e)
# traceback.print_exc()
ret['detail'] = e.__str__()
return JsonResponse(ret)
####开始去重
admin_user = request.user
suc_num = 0
try:
for row in rtable:
id = row['id']
return_amount = row['return_amount']
source = row['source']
remark = row['remark']
mobile = row['mobile']
consume = row['consume']
project_id = row['project_id']
state = row['state']
date = row['date']
term = row['term']
print(id)
event = ProjectInvestDataModel.objects.get(id=id)
# if event.state != '1':
# continue
event.state = state
event.return_amount = return_amount
event.audit_time = datetime.datetime.now()
event.source = source
event.remark = remark
event.settle_amount = consume
event.project_id = project_id
event.invest_mobile = mobile
event.invest_time = date
event.invest_term = term
event.save()
suc_num += 1
ret['code'] = 0
except Exception as e:
exstr = traceback.format_exc()
logger.info(exstr)
ret['code'] = 1
ret['detail'] = e.__str__()
ret['data']['num'] = suc_num
#####################################
# 日志记录导入人的id和导入文件名
write_to_log(self.request.user, filename, '2', request)
#####################################
return JsonResponse(ret)
def parse_tv_programs(self):
# run_phantomjs()
time.sleep(20)
self.driver_start()
write_to_log('Start programs parsing')
func_tm = time.time()
ids_and_links = GetRecordsFromDb().get_channels_id_and_link()
date_today = get_date_and_time_with_timezone()
count_programs = 0
for id_and_link in ids_and_links:
channel = Channel(channel_id=id_and_link['id'])
channel.update()
if id_and_link.get('link'):
self.driver.get(id_and_link.get('link'))
time.sleep(4)
if '404' not in self.driver.title:
if not channel.description or not channel.web_site:
channel_description = self.driver.find_elements_by_css_selector(
"tr.b-row div.b-tv-channel-content__text")
channel_description = channel_description[0].text.encode('utf-8')\
if channel_description else "This channel does not have description"
channel_web_site = self.driver.find_elements_by_css_selector(
"div.b-tv-channel-content__channel-info > "
"div.b-tv-channel-content__links > a")
channel_web_site = channel_web_site[0].get_attribute(
'href').encode('utf-8') \
if channel_web_site else "This channel does not have web site"
if len(channel_description) > Channel.description['length']:
channel_description = channel_description[:Channel.description[
'length']]
if len(channel_web_site) > Channel.web_site['length']:
channel_web_site = channel_web_site[:Channel.web_site['length']]
channel.description, channel.web_site = \
channel_description, channel_web_site
channel.update()
dates_of_week = list()
for date in self.driver.find_elements_by_css_selector(
'div.tv-filter-days__viewport > div.tv-filter-days__items > '
'div.tv-filter-days__item'):
date_of_week = re.findall(r'(\d{4}-\d{2}-\d{2})T',
date.get_attribute('data-bem'))[0]
if datetime.datetime.strptime(date_today, '%Y-%m-%d') <= \
datetime.datetime.strptime(date_of_week, '%Y-%m-%d'):
dates_of_week.append(date_of_week)
dates_of_week = dates_of_week[:7] if len(dates_of_week) > 7 else dates_of_week
for day in dates_of_week:
self.driver.get("%(channel_link)s?date=%(date)s" %
{'channel_link': id_and_link['link'], 'date': day})
time.sleep(1)
channels_tags = self.driver.find_elements_by_css_selector(
'div.b-tv-channel-schedule__items > '
'div.b-tv-channel-schedule__item > a')
tv_channels = []
for channel in channels_tags:
program_name = channel.find_element_by_class_name(
'tv-event__title-inner').text
show_time = channel.find_element_by_class_name(
'tv-event__time-text').text + ':00'
show_date = datetime.datetime.strptime(day, '%Y-%m-%d')
genre = json.loads(channel.get_attribute(
'data-bem'))['tv-event']['genre']
tv_channels.append(TvProgram(name=program_name, genre=genre,
show_date=show_date, show_time=show_time))
count_programs += 1
SaveRecordsToDb.save_programs(id_and_link['id'], tv_channels)
else:
write_to_log('Error. Page {page} not found'.format(
page=self.driver.current_url))
send_email(subject='Page not found',
text='Page {page} not found'.format(page=self.driver.current_url))
else:
write_to_log('Wrong channel link %s. Channel id %s' %
(id_and_link.get('link'), id_and_link.get('id')))
func_tm = time.time() - func_tm
text_for_log = 'Tv programs parsed successfully.' \
'Execution time: %s' % hours_minutes_seconds_from_seconds(func_tm)
send_email(subject='Parser notification',
text=text_for_log)
write_to_log(text_for_log)
SaveRecordsToDb.insert_log_info(parser_name='tv_programs', new_items=count_programs,
execution_time=func_tm)
self.driver.close()
