def process_response(self, request, response): # try to not break the response just because of logging information try: # record the connection to backend API if request.path.startswith('/api') and hasattr(request, 'user'): if request.user.is_authenticated(): self._log(request, response) request.user.last_connected = datetime.datetime.now() request.user.save() # not arbitrary request or response could satisfy the following requirements if getattr(request.resolver_match.func, 'cls', None) in self.RECORD_VIEW_CLASS and \ request.method == 'GET': Usage.objects.create( user=request.user, resource=request.path, response_http_status=response.status_code, response_business_status= get_response_business_status_code(response), method=request.method, link_from=request.GET.get('from'), params=getattr(request, '_usage', {}).get('params')) else: self._log_unauthenticated(request, response) # don't break normal work except Exception as e: gated_debug_logger.error( "RecordUsageMiddleware failed to log: {}".format(str(e))) return response
def append_jira_comment(self, request, pk=None): issue = self.get_object() conclusion = request.data.get('conclusion') comment = request.data.get('comment') kst = request.data.get('from', "") form_id = request.data.get('formId') status_code = status.HTTP_400_BAD_REQUEST issue_status = issue.status.name if not (conclusion or comment): result = {'msg': "字段 'conclusion' 和 'comment' 至少要有一个"} else: try: # need to sync to JIRA comments = change_log = [] if issue.jira_link: if not jira_issue_is_avaliable(issue.jira_link): # jira上issue已删除,去除对应JIRA ID.并在接下去按照没有JIRA ID # 的情况尝试处理 issue.jira_link = None issue.save() else: issue_status, change_log = zc_set_jira_status( issue.jira_link, flag=conclusion, comment=comment) issue.status = IssueStatus.objects.get( name=issue_status) # save new status first, even update comments failed issue.save() change_log = self._update_issue_change_log( issue, change_log) # if no jira_link,but issue from weixin, need to set the status of issue if not issue.jira_link and is_issue_from_weixin(issue): if conclusion == "验证通过": issue_status = "关闭" if conclusion == "验证不通过": issue_status = "处理中" issue.status = IssueStatus.objects.get(name=issue_status) issue.save() if comment: new_comment = format_comment(self.request.user, comment) comments = update_issue_comment(issue, new_comment, kst) if form_id and not issue.set_ext_field_value( 'formId', form_id): gated_debug_logger.error( 'set form id failed for issue {}'.format(issue.id)) result = { 'comments': comments, 'status': issue_status, 'changeLog': change_log } status_code = status.HTTP_200_OK except JiraError as e: result = {'msg': "Jira 错误: {}".format(str(e))} except (ValueError, JSONDecodeError) as e: result = {'msg': format(str(e))} return Response(result, status=status_code)
def update_version_download(): ''' 版本下载量实现如下: 1、在页面的为策略每步的下载量的总和 2、直接获取apphub的版本的下载量,不进行条件过滤 3、版本下载量数据更新周期暂定: 5分钟,在celery定时任务中执行 4、步骤结束后,停止更新此步骤指定的版本下载量 5、如果步骤有更新了版本,则前一版本停止更新下载量,更新当前步骤的下载量 6、如果任务结束,不再更新下载量 ''' task_list = GrayTask.objects.filter( current_status__name=GrayStatus.TESTING_STATUS).all() for task in task_list: version = task.grayappversion_set.all().last() if not version: gated_debug_logger.debug("Task: {}, not gray versions".format( task.id)) continue elif version.current_step > task.inner_strategy_steps: gated_debug_logger.debug( "Task {}, step {} not need update download info; ".format( task.id, version.current_step)) continue data = get_app_versions(download_urls=[ version.ios_download_url, version.android_download_url ]) if not data.get("data"): gated_debug_logger.error(", ".join([ "Task {}, step {} update version fail; ".format( task.id, version.current_step), "ios: {} ,android: {}".format(version.ios_version, version.android_version), "data : {}".format(data) ])) continue for s in data["data"]: if version.android_download_url == s["url_download"]: version.android_download_number = s["amount_of_download"] if version.ios_download_url == s["url_download"]: version.ios_download_number = s["amount_of_download"] version.save() gated_debug_logger.debug(", ".join([ "Task {}, step {} update download successful; ".format( task.id, version.current_step), "ios {},{}".format(version.ios_version, version.ios_download_number), "android {},{}".format(version.android_version, version.android_download_number), ])) return True
def _weixin_code_to_openid(js_code): result = False response = requests.get('{0}?appid={1}&secret={2}&js_code={3}&grant_type=authorization_code'.format( WEIXIN_API_URL, WEIXIN_APP_ID, WEIXIN_APP_SECRET, js_code, )) if response.status_code == status.HTTP_200_OK: if not response.json().get('openid'): gated_debug_logger.error('weixin get openid failed: ' + str(response.content)) else: result = response.json().get('openid') return result
def update_status(self, status): if self.current_status.id < status.id: self.current_status = status self.save() gated_debug_logger.debug( "Update task:{},status:{}, successfully".format( self.id, status.name)) else: gated_debug_logger.error("Update task:{},status:{}, Fail".format( self.id, status.name)) raise ValidationError({ "msg": "Task can not be set to this status: {}".format(status.name) })
def validate(self, data): validated_data = super().validate(data) task_name = validated_data.get("task_name") if not (self.instance and task_name == self.instance.task_name ) and GrayTask.objects.filter(task_name=task_name).exists(): msg = "The name is not allowed duplicate." gated_debug_logger.error( "Create Or Put Task Validate Data Fail, Error msg: {}; data: {}" .format(msg, validated_data)) raise serializers.ValidationError({ "msg": "Create Or Put Task Validate Data Fail, {}".format(msg) }) return validated_data
def update(self, instance, validated_data): jira_id = instance.jira_link if not jira_id: user = self.current_user() generate_change_log(instance, validated_data, user) ext_fields_info = validated_data.pop('extFields', None) prev_status = instance.status with transaction.atomic(): super().update(instance, validated_data) self._create_or_update_extended_fields(instance, ext_fields_info, True) # 微信问题不管是否转jira,只要通过众测平台有update,kst_unread_flag置True if is_issue_from_weixin(instance) and not instance.kst_unread_flag: instance.kst_unread_flag = True instance.save() # if issue has jira_id then update to jira if jira_id: params = gen_params(instance) result = requests.put(JIRA_API_URL + jira_id + '/', params) if result.json().get("status") == status.HTTP_200_OK: comments = result.json().get("data", {}).get("comments", "") change_log = result.json().get("data", {}).get("changeLog", "") # 微信问题和众测问题非首次更新时,other字段不为空,需要更新原有other if instance.other: other_json = json.loads(instance.other) other_json.update({ "comments": comments, "changeLog": change_log }) # 由于众测问题首次更新时,other字段为空,直接赋值 else: other_json = { "comments": comments, "changeLog": change_log } instance.other = json.dumps(other_json) instance.save() res = "Update jira %s successfully!" % jira_id gated_debug_logger.debug(msg=res) else: res = "Update jira %s failed! error msg: %s" % (jira_id, result.text) gated_debug_logger.error(msg=res) # 根据最终数据决定是否发送微信提醒 may_send_wechat_notification(prev_status, instance, self.current_user()) return instance
def set_is_display(self, is_display): try: with transaction.atomic(): GrayTask.objects.filter( app=self.app, is_display=True).update(is_display=False) self.is_display = True if is_display == 'true' else False self.save() gated_debug_logger.debug( "Set App {}, Task: {} DisPlay Successful".format( self.app.id, self.id)) except IntegrityError: gated_debug_logger.error( "Set App {}, Task: {} DisPlay Fail".format( self.app.id, self.id)) return False return self
def update_version(data): """ 1、步骤总数不能大于策略和, 2、每次只能更新当前步骤或者+1步, 3、没有版本时步骤必须为1,即第一个步骤 4、任务状态不能为 结束 :param data: :return: """ task = data['gray_task'] if task.current_status.name == GrayStatus.FINISH_STATUS: raise ValidationError({"msg": "Task was test completed!"}) if task.total_strategy_steps < data['current_step']: raise ValidationError( {"msg": "current_step is large then max step"}) versions = GrayAppVersion.objects.filter( gray_task=data['gray_task']).order_by('-current_step').all() if versions and data['current_step'] not in ( versions[0].current_step, min(versions[0].current_step + 1, task.total_strategy_steps)): raise ValidationError( {"msg": "current_step mast step by step or not large max"}) if not versions and data['current_step'] != 1: raise ValidationError({"msg": "current_step must be 1"}) snap = SnapshotInnerStrategy if data[ 'current_step'] <= task.inner_strategy_steps else SnapshotOuterStrategy strategy = snap.objects.filter(gray_task=task, index=data['current_step']).first() data['current_step_name'] = strategy.name try: with transaction.atomic(): if data['current_step'] == 1: task.current_status = GrayStatus.objects.filter( name=GrayStatus.TESTING_STATUS).first() task.save() gray_version = GrayAppVersion(**data) gray_version.save() except IntegrityError: gated_debug_logger.error("Start Test Fail {}".format(data)) return False return True
def send_push_channels(sender, instance, **kwargs): gated_debug_logger.debug("Receive Signals, Task:{}, step: {}".format( instance.gray_task_id, instance.current_step)) if not kwargs.get('created'): gated_debug_logger.debug( "Task:{}, step:{} is Update, Not need to send sms and mail".format( instance.gray_task_id, instance.current_step)) return None try: celery_id = push_channels_task(instance.gray_task_id, instance.current_step) except Exception as e: gated_debug_logger.error("Celery run fail. {}".format(e)) return False gated_debug_logger.debug( "Start pushing celery task successful, id: {}".format(celery_id)) return True
def process_view(self, request, view_func, view_args, view_kwargs): if 'login' in request.path: return if request.method == 'GET' and request.GET: request._usage = {'params': request.GET.dict()} elif request.method == 'POST' and request.POST: request._usage = {'params': request.POST.dict()} else: try: if request.body: charset = request.encoding if request.encoding else settings.DEFAULT_CHARSET request._usage = { 'params': request.body.decode(encoding=charset) } except (ValueError, UnicodeError, RawPostDataException): # do nothing, don't break normal work pass except Exception as e: # unexpected exception, log it gated_debug_logger.error(str(e))
def strategy_push_content(task_id, step): gated_debug_logger.debug( "Start strategy push by Signals, Task {}, steps : {}".format( task_id, step)) instance = GrayAppVersion.objects.filter(gray_task_id=task_id, current_step=step) count = instance.count() if not instance: gated_debug_logger.error("Task {}, steps : {}".format(task_id, step)) return None # 判断内部策略还是外部策略 instance = instance.last() if instance.current_step <= instance.gray_task.inner_strategy_steps: gated_debug_logger.debug("Inner Strategys Push Start") gated_debug_logger.debug("Task: {}, step: {}; start Inner PUSH".format( instance.gray_task.id, instance.current_step)) inner = SnapshotInnerStrategy.objects.filter( gray_task=instance.gray_task, index=instance.current_step).first() if count > 1: gated_debug_logger.debug("Update version not need to send push") return None from apps.common.tasks import send_sms_task, send_email_task members = get_user_model().objects.filter( usergroup__in=inner.user_groups.all()).distinct().all() content = inner.push_content channels = [c.name for c in inner.push_channels.all()] phones = [m.phone for m in members] gated_debug_logger.debug("Send sms phone list : {}".format(phones)) mails = [m.email for m in members] gated_debug_logger.debug("Send email list : {}".format(mails)) if 'sms' in channels and TURN_ON_STRATEGY_SMS_AND_EMAIL: gated_debug_logger.debug("Task:{} Send SMS".format( inner.gray_task_id)) celery_id = send_sms_task.delay(phones, content) gated_debug_logger.debug("Task:{} Send SMS celery id {}".format( inner.gray_task_id, celery_id)) if 'mail' in channels and TURN_ON_STRATEGY_SMS_AND_EMAIL: gated_debug_logger.debug("Task:{} Send Mail".format(inner)) # 发送的邮件标题为 task的名称 celery_id = send_email_task.delay(mails, inner.gray_task.task_name, content) gated_debug_logger.debug("Task:{} Send Mail celery id {}".format( inner.gray_task_id, celery_id)) gated_debug_logger.debug("Inner Strategys Push Complated") else: gated_debug_logger.debug("Task: {}, step: {}; start Outer PUSH".format( instance.gray_task.id, instance.current_step)) outer_version = instance.android_version out_snap = SnapshotOuterStrategy.objects.filter( gray_task=instance.gray_task, index=instance.current_step).first() # 外部策略版本发推送 res = BpConfigOnline(out_snap.id, outer_version) if res["status"] != 200: gated_debug_logger.error("Outer Strategy Version Update Fail") gated_debug_logger.error(res) gated_debug_logger.debug("Strategy Push Completed") gated_debug_logger.debug("End strategy push by Signals")
def create(self, validated_data): inner_list = validated_data.get("inner_strategy", []) outer_list = validated_data.get("outer_strategy", []) validated_data['inner_strategy_steps'] = len(inner_list) validated_data['total_strategy_steps'] = len(inner_list) + len( outer_list) try: with transaction.atomic(): instance = GrayTask(**validated_data) # 新建任务时候默认状态 instance.current_status = GrayStatus.get_original_status() instance.creator = self.current_user() instance.save() index = 1 for k in inner_list: if not isinstance(k.get("id"), int): raise serializers.ValidationError( {"msg": "strategy id must be int"}) q = get_object_or_404(InnerStrategy, id=k.get("id")) i = copy_model_to_model(q, SnapshotInnerStrategy, pop_change_dict={ 'app_id': 'app', 'creator_id': 'creator' }, change_dict={ 'id': None, 'gray_task': instance, 'index': index }, data_change_dict={ 'user_groups': {}, 'push_channels': {} }) i.save() i.user_groups = q.user_groups.all() i.push_channels = q.push_channels.all() if k.get("pushContent"): # 更新推送内容 i.push_content = k.get("pushContent") i.save() index += 1 for k in outer_list: q = get_object_or_404(OuterStrategy, id=k) i = copy_model_to_model(q, SnapshotOuterStrategy, pop_change_dict={ 'app_id': 'app', 'creator_id': 'creator' }, change_dict={ 'id': None, 'gray_task': instance, 'index': index }) i.save() index += 1 except Exception as e: gated_debug_logger.error( "Create Task Fail, Error msg: {}; data: {}".format( e, validated_data)) raise serializers.ValidationError( {"msg": "Create Task Fail, {}".format(e)}) return instance
def to_dict(self, detail=False): try: res = { "id": self.id, "name": self.task_name, "isDisplay": self.is_display, "isJoinKesutong": self.is_join_kesutong, "startDate": self.start_date, "endDate": self.end_date, "innerStrategy": self.inner_strategy, "outerStrategy": self.outer_strategy, "imageId": self.image.image_id if self.image else "", "creator": self.creator.username, "currentStatus": self.current_status.description, "createdDate": self.created_time.date(), "versionDesc": self.version_desc, "awardRule": self.award_rule, "contact": self.contact, "app": { "appId": self.app.id, "name": self.app.name, "desc": self.app.desc, } } grap_version = self.grayappversion_set.all() # android,ios下载次数需要每步求和 from apps.issue.models import Issue res["qualityData"] = { "androidDownload": sum([d.android_download_number for d in grap_version]), "iosDownload": sum([d.ios_download_number for d in grap_version]), "feedback": Issue.objects.filter(task=self).count() } res["appVersion"] = [{ "step": a.current_step, "stepName": a.current_step_name, "androidVersion": a.android_version, "androidURL": a.android_download_url, "androidCreateDate": a.android_release_date, "iosVersion": a.ios_version, "iosURL": a.ios_download_url, "iosCreateDate": a.ios_release_date, "updatedTime": a.updated_time.strftime("%Y-%m-%d %H:%M:%S") } for a in grap_version] res['steps'] = SnapshotInnerStrategy.to_list( self) + SnapshotOuterStrategy.to_list(self) # 当前步骤是最后的值, 如果没有版本信息,则步骤为0 res["current_step"] = res["appVersion"][-1]["step"] if res[ "appVersion"] else 0 res["current_step_name"] = \ res["steps"][res["current_step"] - 1]["name"] if res[ "steps"] else 0 except Exception as e: gated_debug_logger.error( "Task:{}, instance to dict fail, {}".format(self.id, e)) raise ValidationError({ "msg": "Task: {}, instance to dict fail, {}".format(self.id, e) }) return res
def startTest(self, request, pk=None): ''' PATCH: api/v1/tasks/{id}/startTest/ { "currentStep":1, "appVersion":{ "android":{ "urlDownload":"http://apphub.ffan.com/api/appdownload/ffan/0/0/develop/4.20.0.0.1848//None/None/Feifan_o2o_4_20_0_0_DEV_1848_2017_07_25_release.apk", "createDate":"2017-07-25", "versionId":"4.20.0.0.1848" }, "ios":{ "urlDownload":"http://apphub.ffan.com/api/appdownload/ffan/0/0/develop/4.20.0.0.1848//None/None/Feifan_o2o_4_20_0_0_DEV_1848_2017_07_25_release.apk", "createDate":"2017-07-25", "versionId":"4.20.0.0.1848" } } } :param request: :param args: :param kwargs: :return: ''' app_dict = {} param_dict = request.data app_version = param_dict.get("appVersion", {}) android_version = app_version.get("android") ios_version = app_version.get("ios") if android_version: try: app_dict['android_version'] = android_version.get("versionId") if android_version.get("createDate"): app_dict['android_release_date'] = datetime.strptime( android_version.get("createDate"), '%Y-%m-%d') except Exception as e: gated_debug_logger.error("{}".format(e)) raise ValidationError({"msg": "{}".format(e)}) app_dict['android_download_url'] = android_version.get( "urlDownload") if ios_version: try: app_dict['ios_version'] = ios_version.get("versionId") if ios_version.get("createDate"): app_dict['ios_release_date'] = datetime.strptime( ios_version.get("createDate"), '%Y-%m-%d') except Exception as e: gated_debug_logger.error("{}".format(e)) raise ValidationError({"msg": "{}".format(e)}) app_dict['ios_download_url'] = ios_version.get("urlDownload") app_dict['current_step'] = param_dict.get("currentStep") task = self.get_object() # 外灰步骤android必须提供--外灰的内容暂时不用 # if app_dict['current_step'] > task.inner_strategy_steps and (not app_dict['android_version']): # raise ValidationError({'msg': "Outer Strategy must have android version"}) app_dict['gray_task'] = task app_dict['operator'] = request.user GrayAppVersion.update_version(app_dict) return Response(task.to_dict())
def wrapper(*args, **kwargs): try: return func(*args, **kwargs) except Exception as e: gated_debug_logger.error(str(e)) raise e
def get(self, request): """ :param request: token, appId, taskId, linkFrom, startTime, endTime :return: { "status": 200, "msg": "成功", "data": { "appId": 5, //参数:app id,全表唯一 "taskId": 0, //参数:task id,全表唯一 "linkFrom": null, //参数:参与途径 "startTime": null, //参数:查询开始时间 "endTime": null, //参数:查询结束时间 "taskDetailEntry": { "userCount": 2, //进入任务详情页人数--按照人员去重 "eventCount": 70 //进入任务详情页人次 }, "appDownload": { "userCount": 1, //下载人数--按照人员去重 "eventCount": 19 //下载人次 }, "feedbackSubmitSuccess": { "userCount": 0, //反馈问题提交成功人数--按照人员去重 "eventCount": 0 //反馈问题提交成功人次 } } } """ try: app_id, task_id, link_from, start_time, end_time = self._param_handler( request) if (start_time and not end_time) or (not start_time and end_time): gated_debug_logger.error( "startTime and endTime must be provided at the same time") raise Exception( 'startTime and endTime must be provided at the same time') if app_id and task_id: gated_debug_logger.error( "appId and taskId can not be provided at the same time") raise Exception( 'appId and taskId can not be provided at the same time') except ValueError as e: return Response(data={'results': str(e)}, status=status.HTTP_400_BAD_REQUEST) res = { 'appId': app_id, 'taskId': task_id, 'linkFrom': link_from, 'startTime': start_time, 'endTime': end_time } for event in [ 'taskDetailEntry', 'appDownload', 'feedbackSubmitSuccess' ]: # 获取要统计的事件的事件id event_type = EventType.objects.get(name=event) # 没有时间要求,直接按事件类型查询事件id_list if not start_time and not end_time: id_list = EventTracking.objects.values('id').filter( type=event_type) # 有时间要求,按照时间过滤id_list if start_time and end_time: id_list = EventTracking.objects.values('id').filter( type=event_type).filter(created_time__range=(start_time, end_time)) # 如果appId和taskId都没有传,上面所得id_ist不需要按照appId或者taskId过滤 # 按照app维度查询,按照appId过滤id_list if app_id != 0: id_list = Property.objects.values('event_id').filter( event__in=id_list).filter(key='appId', value=app_id) # 按照task维度查询,按照taskId过滤id_list if task_id != 0: id_list = Property.objects.values('event_id').filter( event__in=id_list).filter(key='taskId', value=task_id) # 如果没有要求来源,直接计算人数和人次 if not link_from: user_count = EventTracking.objects.filter( id__in=id_list).distinct().values("user_id").count() event_count = len(id_list) # 如果要求来源,再在property表中过滤一次from,之后,再计算人数和人次 else: id_list = Property.objects.values('event_id').filter( event__in=id_list).filter(key='from', value=link_from) user_count = EventTracking.objects.filter( id__in=id_list).distinct().values("user_id").count() event_count = len(id_list) res[event] = {'userCount': user_count, 'eventCount': event_count} return Response(data=res)