def destroy(self, request, *args, **kwargs): instance = self.get_object() self.perform_destroy(instance) ret = {} ret['code'] = 0 #有没权限的时候有两层code ##################################### # 日志删除时间和具体操作 print("hihhi") write_to_log(self.request.user, instance, '0', request) ##################################### return Response(ret)
def get_phantomjs_driver(): conf = dict(service_args=['--ssl-protocol=any']) if os.environ.get('OPENSHIFT_DATA_DIR'): capabilities = DesiredCapabilities.PHANTOMJS capabilities['acceptSslCerts'] = True driver = webdriver.Remote(command_executor='http://'+os.environ.get( 'OPENSHIFT_PYTHON_IP')+':15005', desired_capabilities=capabilities, keep_alive=True) write_to_log('Connected to remote web driver') return driver driver = webdriver.PhantomJS(**conf) return driver
def perform_update(self, serializer): instance = self.get_object() print(instance.return_amount) serializer.save() data = serializer.data print(instance.return_amount) ##################################### # 日志修改时间和具体操作 write_to_log(self.request.user, instance, '1') ##################################### serializer._data = {} serializer._data['code'] = 0 serializer._data['data'] = data
def destroy(self, request, *args, **kwargs): instance = self.get_object() self.perform_destroy(instance) ret = {} ret['code'] = 0 ##################################### # 项目删除时间和具体操作 #print(instance) print(instance.state) if instance.state == '1': write_to_log(self.request.user, instance, '0', request) # oobj = repr(instance) # OperatorLog.objects.create(otime=datetime.datetime.now(), oman=self.request.user, oobj=oobj, otype='0') ##################################### return Response(ret)
def is_altered(self, request, pk=None, *args, **kwargs): '''修改 字段仅仅限于is_delete,进行中的项目要通知管理员''' aimpro = Project.objects.get(id=pk) instance = self.get_object() print(self.action) ##################################### # 项目修改时间和具体操作 if instance.state == '1': write_to_log(self.request.user, instance, '1', request) ##################################### #if aimpro.state in ['1',]: # 写操作日志,对应visit字段为False表示未读.而每个管理员登录后,都会更新当前的未读的操作日志有多少条,但这些操作日志是共享的 self.partial_update(request, *args, **kwargs) res = {} res['code'] = 0 return Response(res)
def _handle_tamper_file(self, filepath): relative_pos = filepath[self.__website_path_len:].lstrip('/') backup_filepath = os.path.join(self.backup_path, relative_pos) if not os.path.exists(backup_filepath): content = " 警告:发现可疑文件%s!" % filepath # move to isolation area. filename = os.path.basename(filepath) if not os.path.exists(self.isolation_path): os.mkdir(self.isolation_path) if self.auto_cure: try: os.replace(filepath, os.path.join(self.isolation_path, filename)) content += '已隔离到%s' % self.isolation_path except: print('Isolate Error: %s is in use!' % filepath) else: if not os.path.exists(filepath): content = ' 警告:文件%s被删除!' % filepath else: if filecmp.cmp(filepath, backup_filepath): return content = " 警告:文件%s的内容发生变动!" % filepath if self.auto_cure: try: shutil.copy2(backup_filepath, filepath) content += "文件已恢复。" except Exception as err_msg: #Provisionally skip the file in use. content += '文件恢复失败:' + str(err_msg) if self.check_if_send(filepath): try: subject = "页面篡改问题" send_alarm_mail(self.mail_receivers, subject, content) except Exception as err_msg: content += '发送告警邮件失败:' + err_msg write_to_log(content)
def driver_start(self): write_to_log('Preparing driver to parsing') self.driver = self.get_phantomjs_driver() time.sleep(5) write_to_log('Driver trying to open url %s' % self.url) self.driver.get(self.url) write_to_log('Url %s opened successfully' % self.url) self.driver.set_window_size(1920, 1080) assert self.url in self.driver.current_url, "Can't open url: %s" % self.url
def parse_url_channels(self): # run_phantomjs() time.sleep(20) self.driver_start() write_to_log('Start channels parsing') func_tm = time.time() page_height = 0 elements = {} scroll_height_script = """ return window.innerHeight + window.scrollY """ while page_height != self.driver.execute_script(scroll_height_script): page_height = self.driver.execute_script(scroll_height_script) self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight);") time.sleep(5) channels = self.driver.find_elements_by_css_selector(self.get_channel_css_selector()) write_to_log('Found %s channels' % len(channels)) for channel in channels: time.sleep(1) name = channel.find_element_by_css_selector( 'span.tv-channel-title__text').text.encode('utf-8') href = channel.get_attribute('href').encode('utf-8') icon = channel.find_elements_by_css_selector( 'div.tv-channel-title__icon > span[class$="image_type_channel"] > span') if icon: icon = self.get_background_image(icon[0]).encode('utf-8') if (href is not None) and (href not in elements.keys()): elements[href] = {'name': name, 'icon': icon} save_records = SaveRecordsToDb() new_elements_count = save_records.save_channels_to_db(elements) func_tm = int(time.time()-func_tm) text_for_log = 'Channels parsed successfully.{elements_count} new channels.' \ 'Execution time: {func_tm}'.\ format(elements_count=new_elements_count, func_tm=hours_minutes_seconds_from_seconds(func_tm)) send_email(subject='Parser notification', text=text_for_log) write_to_log(text_for_log) SaveRecordsToDb.insert_log_info(execution_time=func_tm, new_items=new_elements_count) self.driver.close()
def run_scheduler(): try: write_to_log('Run scheduler') # if type_of_parser == 'channels': # write_to_log('Preparing to parse channels') # driver.parse_url_channels() # elif type_of_parser == 'programs': # write_to_log('Preparing to parse programs') # driver.parse_tv_programs() schedule.every().day.at("15:29").do(driver.parse_url_channels) schedule.every().day.at("15:59").do(driver.parse_tv_programs) # schedule.every().monday.at('"06:00"').do(driver.parse_tv_programs) # schedule.every().sunday.at("'12:00'").do(driver.parse_url_channels) while True: schedule.run_pending() time.sleep(2) # else: # subprocess.Popen("echo Unknown argument for parser." # "Type --help for more information".split()) except Exception as e: write_to_log(error=e) send_email(subject='Parse error. Exception', exception=e) write_to_log('Kill scheduler after error') kill_process(get_pid_by_name('run_scheduler.py'))
def import_projectdata_excel(self, request): admin_user = request.user ret = {} ret['data'] = {} print("ghgggg") file = request.FILES.get('file') # aftername=time.time() filename = "./files/" + "1" with open(filename, 'wb+') as destination: for chunk in file.chunks(): destination.write(chunk) data = xlrd.open_workbook(filename) table = data.sheets()[0] nrows = table.nrows ncols = table.ncols print("aaa") if ncols != 8: print("bbb") ret['detail'] = "文件格式与模板不符,请下载最新模板填写!" return JsonResponse(ret) rtable = {} mobile_list = [] dup = {} try: for i in range(1, nrows): temp = [] for j in range(ncols): cell = table.cell(i, j) value = cell.value print(value) if j == 0: id = int(value) print(id) project = Project.objects.get(id=id) temp.append(id) elif j == 2: value = str(value).strip() if value == u"首投": temp.append(False) elif value == u"复投": temp.append(True) else: raise Exception(u"必须为首投或复投。") elif j == 3: if (cell.ctype != 3): raise Exception("投资日期列格式错误,请修改后重新提交。") else: time2 = xlrd.xldate.xldate_as_datetime(value, 0) temp.append(time2) elif j == 4: try: mobile = str(int(value)).strip() except Exception as e: mobile = str(value).strip() if len(mobile) == 11: temp.append(mobile) else: raise Exception("手机号必须是11位,请修改后重新提交。") elif j == 5 or j == 7: try: temp.append(Decimal(value)) except: raise Exception("投资金额必须为数字") elif j == 6: try: temp.append(int(value)) except Exception as e: raise Exception("投资标期必须为数字,请修改后重新提交。") else: temp.append(value) tid = temp[0] if not temp[2]: print(dup) if tid in dup: if temp[4] in dup[tid]: continue else: dup[tid].append(temp[4]) else: dup[tid] = [ temp[4], ] if tid in rtable: rtable[tid].append(temp) else: rtable[tid] = [ temp, ] except Exception as e: logger.info(e) # traceback.print_exc() ret['detail'] = e.__str__() return JsonResponse(ret) ####开始去重 investdata_list = [] duplicate_mobile_list = [] try: with cache.lock('investdata_first', timeout=2): # db_key = DBlock.objects.select_for_update().get(index='investdata') for id, values in rtable.items(): print(id) temp = ProjectInvestDataModel.objects.filter( project_id=id).values('invest_mobile') db_mobile_list = map(lambda x: x['invest_mobile'], temp) for item in values: pid = item[0] time = item[3] mob = item[4] is_futou = item[2] amount = item[5] term = item[6] settle = item[7] source = '' remark = '' if not is_futou and mob in db_mobile_list: duplicate_mobile_list.append(mob) else: obj = ProjectInvestDataModel(project_id=pid, invest_mobile=mob, settle_amount=settle, invest_amount=amount, invest_term=term, invest_time=time, state='0', remark=remark, source=source, is_futou=is_futou, apply_man=admin_user) investdata_list.append(obj) ProjectInvestDataModel.objects.bulk_create(investdata_list) except Exception as e: logger.info(e) # traceback.print_exc() ret['detail'] = e.__str__() return JsonResponse(ret) succ_num = len(investdata_list) duplic_num2 = len(duplicate_mobile_list) duplic_num1 = nrows - 1 - succ_num - duplic_num2 duplic_mobile_list_str = u','.join(duplicate_mobile_list) ret['data'].update(num=succ_num, dup1=duplic_num1, dup2=duplic_num2, anum=nrows - 1, dupstr=duplic_mobile_list_str) ret['code'] = 0 ##################################### #日志记录导入人的id和导入文件名 write_to_log(self.request.user, filename, '2', request) ##################################### return JsonResponse(ret)
def import_audit_projectdata_excel(self, request): admin_user = request.user ret = {'code': -1} ret['data'] = {} # print(dir(request)) file = request.FILES.get('file') aftername = time.time() filename = "./files/audit_" + str(int(aftername * 1000)) with open(filename, 'wb+') as destination: for chunk in file.chunks(): destination.write(chunk) data = xlrd.open_workbook(filename) table = data.sheets()[0] print(table) nrows = table.nrows ncols = table.ncols if ncols != 13: ret['detail'] = "文件格式与模板不符,请下载最新模板填写!" return JsonResponse(ret) rtable = [] try: for i in range(1, nrows): row = table.row_values(i) temp = {} id = int(row[0]) project_id = int(row[1]) term = int(row[7]) mobile = row[5] consume = Decimal(row[8]) remark = row[12] date = row[4] date = xlrd.xldate.xldate_as_datetime(date, 0) try: mobile = str(int(mobile)).strip() except Exception as e: mobile = str(mobile).strip() if len(mobile) != 11: raise Exception("手机号必须是11位,请修改后重新提交。") if row[9] == "是": result = True temp['state'] = '1' elif row[9] == "否": result = False temp['state'] = '0' else: raise Exception("审核结果必须为是或否。") if row[10]: return_amount = Decimal(row[10]) if return_amount > consume: raise Exception("返现金额不能大于结算金额,请检查表格") elif result: raise Exception("审核结果为是时,返现金额不能为空或零。") else: return_amount = 0 if row[11] == "网站": source = 'site' elif row[11] == "渠道": source = 'channel' else: raise Exception("必须为网站或渠道。") temp['id'] = id temp['project_id'] = project_id temp['source'] = source temp['return_amount'] = return_amount temp['consume'] = consume temp['remark'] = remark temp['mobile'] = mobile temp['date'] = date temp['term'] = term rtable.append(temp) except Exception as e: logger.info(e) print(e) # traceback.print_exc() ret['detail'] = e.__str__() return JsonResponse(ret) ####开始去重 admin_user = request.user suc_num = 0 try: for row in rtable: id = row['id'] return_amount = row['return_amount'] source = row['source'] remark = row['remark'] mobile = row['mobile'] consume = row['consume'] project_id = row['project_id'] state = row['state'] date = row['date'] term = row['term'] print(id) event = ProjectInvestDataModel.objects.get(id=id) # if event.state != '1': # continue event.state = state event.return_amount = return_amount event.audit_time = datetime.datetime.now() event.source = source event.remark = remark event.settle_amount = consume event.project_id = project_id event.invest_mobile = mobile event.invest_time = date event.invest_term = term event.save() suc_num += 1 ret['code'] = 0 except Exception as e: exstr = traceback.format_exc() logger.info(exstr) ret['code'] = 1 ret['detail'] = e.__str__() ret['data']['num'] = suc_num ##################################### # 日志记录导入人的id和导入文件名 write_to_log(self.request.user, filename, '2', request) ##################################### return JsonResponse(ret)
def parse_tv_programs(self): # run_phantomjs() time.sleep(20) self.driver_start() write_to_log('Start programs parsing') func_tm = time.time() ids_and_links = GetRecordsFromDb().get_channels_id_and_link() date_today = get_date_and_time_with_timezone() count_programs = 0 for id_and_link in ids_and_links: channel = Channel(channel_id=id_and_link['id']) channel.update() if id_and_link.get('link'): self.driver.get(id_and_link.get('link')) time.sleep(4) if '404' not in self.driver.title: if not channel.description or not channel.web_site: channel_description = self.driver.find_elements_by_css_selector( "tr.b-row div.b-tv-channel-content__text") channel_description = channel_description[0].text.encode('utf-8')\ if channel_description else "This channel does not have description" channel_web_site = self.driver.find_elements_by_css_selector( "div.b-tv-channel-content__channel-info > " "div.b-tv-channel-content__links > a") channel_web_site = channel_web_site[0].get_attribute( 'href').encode('utf-8') \ if channel_web_site else "This channel does not have web site" if len(channel_description) > Channel.description['length']: channel_description = channel_description[:Channel.description[ 'length']] if len(channel_web_site) > Channel.web_site['length']: channel_web_site = channel_web_site[:Channel.web_site['length']] channel.description, channel.web_site = \ channel_description, channel_web_site channel.update() dates_of_week = list() for date in self.driver.find_elements_by_css_selector( 'div.tv-filter-days__viewport > div.tv-filter-days__items > ' 'div.tv-filter-days__item'): date_of_week = re.findall(r'(\d{4}-\d{2}-\d{2})T', date.get_attribute('data-bem'))[0] if datetime.datetime.strptime(date_today, '%Y-%m-%d') <= \ datetime.datetime.strptime(date_of_week, '%Y-%m-%d'): dates_of_week.append(date_of_week) dates_of_week = dates_of_week[:7] if len(dates_of_week) > 7 else dates_of_week for day in dates_of_week: self.driver.get("%(channel_link)s?date=%(date)s" % {'channel_link': id_and_link['link'], 'date': day}) time.sleep(1) channels_tags = self.driver.find_elements_by_css_selector( 'div.b-tv-channel-schedule__items > ' 'div.b-tv-channel-schedule__item > a') tv_channels = [] for channel in channels_tags: program_name = channel.find_element_by_class_name( 'tv-event__title-inner').text show_time = channel.find_element_by_class_name( 'tv-event__time-text').text + ':00' show_date = datetime.datetime.strptime(day, '%Y-%m-%d') genre = json.loads(channel.get_attribute( 'data-bem'))['tv-event']['genre'] tv_channels.append(TvProgram(name=program_name, genre=genre, show_date=show_date, show_time=show_time)) count_programs += 1 SaveRecordsToDb.save_programs(id_and_link['id'], tv_channels) else: write_to_log('Error. Page {page} not found'.format( page=self.driver.current_url)) send_email(subject='Page not found', text='Page {page} not found'.format(page=self.driver.current_url)) else: write_to_log('Wrong channel link %s. Channel id %s' % (id_and_link.get('link'), id_and_link.get('id'))) func_tm = time.time() - func_tm text_for_log = 'Tv programs parsed successfully.' \ 'Execution time: %s' % hours_minutes_seconds_from_seconds(func_tm) send_email(subject='Parser notification', text=text_for_log) write_to_log(text_for_log) SaveRecordsToDb.insert_log_info(parser_name='tv_programs', new_items=count_programs, execution_time=func_tm) self.driver.close()