def process_response(self, request, response):
# try to not break the response just because of logging information
try:
# record the connection to backend API
if request.path.startswith('/api') and hasattr(request, 'user'):
if request.user.is_authenticated():
self._log(request, response)
request.user.last_connected = datetime.datetime.now()
request.user.save()
# not arbitrary request or response could satisfy the following requirements
if getattr(request.resolver_match.func, 'cls', None) in self.RECORD_VIEW_CLASS and \
request.method == 'GET':
Usage.objects.create(
user=request.user,
resource=request.path,
response_http_status=response.status_code,
response_business_status=
get_response_business_status_code(response),
method=request.method,
link_from=request.GET.get('from'),
params=getattr(request, '_usage',
{}).get('params'))
else:
self._log_unauthenticated(request, response)
# don't break normal work
except Exception as e:
gated_debug_logger.error(
"RecordUsageMiddleware failed to log: {}".format(str(e)))
return response
def append_jira_comment(self, request, pk=None):
issue = self.get_object()
conclusion = request.data.get('conclusion')
comment = request.data.get('comment')
kst = request.data.get('from', "")
form_id = request.data.get('formId')
status_code = status.HTTP_400_BAD_REQUEST
issue_status = issue.status.name
if not (conclusion or comment):
result = {'msg': "字段 'conclusion' 和 'comment' 至少要有一个"}
else:
try:
# need to sync to JIRA
comments = change_log = []
if issue.jira_link:
if not jira_issue_is_avaliable(issue.jira_link):
# jira上issue已删除，去除对应JIRA ID.并在接下去按照没有JIRA ID
# 的情况尝试处理
issue.jira_link = None
issue.save()
else:
issue_status, change_log = zc_set_jira_status(
issue.jira_link, flag=conclusion, comment=comment)
issue.status = IssueStatus.objects.get(
name=issue_status)
# save new status first, even update comments failed
issue.save()
change_log = self._update_issue_change_log(
issue, change_log)
# if no jira_link，but issue from weixin, need to set the status of issue
if not issue.jira_link and is_issue_from_weixin(issue):
if conclusion == "验证通过":
issue_status = "关闭"
if conclusion == "验证不通过":
issue_status = "处理中"
issue.status = IssueStatus.objects.get(name=issue_status)
issue.save()
if comment:
new_comment = format_comment(self.request.user, comment)
comments = update_issue_comment(issue, new_comment, kst)
if form_id and not issue.set_ext_field_value(
'formId', form_id):
gated_debug_logger.error(
'set form id failed for issue {}'.format(issue.id))
result = {
'comments': comments,
'status': issue_status,
'changeLog': change_log
}
status_code = status.HTTP_200_OK
except JiraError as e:
result = {'msg': "Jira 错误： {}".format(str(e))}
except (ValueError, JSONDecodeError) as e:
result = {'msg': format(str(e))}
return Response(result, status=status_code)
def update_version_download():
'''
版本下载量实现如下：
1、在页面的为策略每步的下载量的总和
2、直接获取apphub的版本的下载量，不进行条件过滤
3、版本下载量数据更新周期暂定： 5分钟，在celery定时任务中执行
4、步骤结束后，停止更新此步骤指定的版本下载量
5、如果步骤有更新了版本，则前一版本停止更新下载量，更新当前步骤的下载量
6、如果任务结束，不再更新下载量
'''
task_list = GrayTask.objects.filter(
current_status__name=GrayStatus.TESTING_STATUS).all()
for task in task_list:
version = task.grayappversion_set.all().last()
if not version:
gated_debug_logger.debug("Task: {}, not gray versions".format(
task.id))
continue
elif version.current_step > task.inner_strategy_steps:
gated_debug_logger.debug(
"Task {}, step {} not need update download info; ".format(
task.id, version.current_step))
continue
data = get_app_versions(download_urls=[
version.ios_download_url, version.android_download_url
])
if not data.get("data"):
gated_debug_logger.error(", ".join([
"Task {}, step {} update version fail; ".format(
task.id, version.current_step),
"ios: {} ,android: {}".format(version.ios_version,
version.android_version),
"data : {}".format(data)
]))
continue
for s in data["data"]:
if version.android_download_url == s["url_download"]:
version.android_download_number = s["amount_of_download"]
if version.ios_download_url == s["url_download"]:
version.ios_download_number = s["amount_of_download"]
version.save()
gated_debug_logger.debug(", ".join([
"Task {}, step {} update download successful; ".format(
task.id, version.current_step),
"ios {},{}".format(version.ios_version,
version.ios_download_number),
"android {},{}".format(version.android_version,
version.android_download_number),
]))
return True
def _weixin_code_to_openid(js_code):
result = False
response = requests.get('{0}?appid={1}&secret={2}&js_code={3}&grant_type=authorization_code'.format(
WEIXIN_API_URL, WEIXIN_APP_ID, WEIXIN_APP_SECRET, js_code,
))
if response.status_code == status.HTTP_200_OK:
if not response.json().get('openid'):
gated_debug_logger.error('weixin get openid failed: ' + str(response.content))
else:
result = response.json().get('openid')
return result
def update_status(self, status):
if self.current_status.id < status.id:
self.current_status = status
self.save()
gated_debug_logger.debug(
"Update task:{},status:{}, successfully".format(
self.id, status.name))
else:
gated_debug_logger.error("Update task:{},status:{}, Fail".format(
self.id, status.name))
raise ValidationError({
"msg":
"Task can not be set to this status: {}".format(status.name)
})
def validate(self, data):
validated_data = super().validate(data)
task_name = validated_data.get("task_name")
if not (self.instance and task_name == self.instance.task_name
) and GrayTask.objects.filter(task_name=task_name).exists():
msg = "The name is not allowed duplicate."
gated_debug_logger.error(
"Create Or Put Task Validate Data Fail, Error msg: {}; data: {}"
.format(msg, validated_data))
raise serializers.ValidationError({
"msg":
"Create Or Put Task Validate Data Fail, {}".format(msg)
})
return validated_data
def update(self, instance, validated_data):
jira_id = instance.jira_link
if not jira_id:
user = self.current_user()
generate_change_log(instance, validated_data, user)
ext_fields_info = validated_data.pop('extFields', None)
prev_status = instance.status
with transaction.atomic():
super().update(instance, validated_data)
self._create_or_update_extended_fields(instance, ext_fields_info,
True)
# 微信问题不管是否转jira，只要通过众测平台有update，kst_unread_flag置True
if is_issue_from_weixin(instance) and not instance.kst_unread_flag:
instance.kst_unread_flag = True
instance.save()
# if issue has jira_id then update to jira
if jira_id:
params = gen_params(instance)
result = requests.put(JIRA_API_URL + jira_id + '/', params)
if result.json().get("status") == status.HTTP_200_OK:
comments = result.json().get("data", {}).get("comments", "")
change_log = result.json().get("data", {}).get("changeLog", "")
# 微信问题和众测问题非首次更新时，other字段不为空，需要更新原有other
if instance.other:
other_json = json.loads(instance.other)
other_json.update({
"comments": comments,
"changeLog": change_log
})
# 由于众测问题首次更新时，other字段为空，直接赋值
else:
other_json = {
"comments": comments,
"changeLog": change_log
}
instance.other = json.dumps(other_json)
instance.save()
res = "Update jira %s successfully!" % jira_id
gated_debug_logger.debug(msg=res)
else:
res = "Update jira %s failed! error msg: %s" % (jira_id,
result.text)
gated_debug_logger.error(msg=res)
# 根据最终数据决定是否发送微信提醒
may_send_wechat_notification(prev_status, instance,
self.current_user())
return instance
def set_is_display(self, is_display):
try:
with transaction.atomic():
GrayTask.objects.filter(
app=self.app, is_display=True).update(is_display=False)
self.is_display = True if is_display == 'true' else False
self.save()
gated_debug_logger.debug(
"Set App {}, Task: {} DisPlay Successful".format(
self.app.id, self.id))
except IntegrityError:
gated_debug_logger.error(
"Set App {}, Task: {} DisPlay Fail".format(
self.app.id, self.id))
return False
return self
def update_version(data):
"""
1、步骤总数不能大于策略和，
2、每次只能更新当前步骤或者+1步，
3、没有版本时步骤必须为1,即第一个步骤
4、任务状态不能为 结束
:param data:
:return:
"""
task = data['gray_task']
if task.current_status.name == GrayStatus.FINISH_STATUS:
raise ValidationError({"msg": "Task was test completed!"})
if task.total_strategy_steps < data['current_step']:
raise ValidationError(
{"msg": "current_step is large then max step"})
versions = GrayAppVersion.objects.filter(
gray_task=data['gray_task']).order_by('-current_step').all()
if versions and data['current_step'] not in (
versions[0].current_step,
min(versions[0].current_step + 1, task.total_strategy_steps)):
raise ValidationError(
{"msg": "current_step mast step by step or not large max"})
if not versions and data['current_step'] != 1:
raise ValidationError({"msg": "current_step must be 1"})
snap = SnapshotInnerStrategy if data[
'current_step'] <= task.inner_strategy_steps else SnapshotOuterStrategy
strategy = snap.objects.filter(gray_task=task,
index=data['current_step']).first()
data['current_step_name'] = strategy.name
try:
with transaction.atomic():
if data['current_step'] == 1:
task.current_status = GrayStatus.objects.filter(
name=GrayStatus.TESTING_STATUS).first()
task.save()
gray_version = GrayAppVersion(**data)
gray_version.save()
except IntegrityError:
gated_debug_logger.error("Start Test Fail {}".format(data))
return False
return True
def send_push_channels(sender, instance, **kwargs):
gated_debug_logger.debug("Receive Signals, Task:{}, step: {}".format(
instance.gray_task_id, instance.current_step))
if not kwargs.get('created'):
gated_debug_logger.debug(
"Task:{}, step:{} is Update, Not need to send sms and mail".format(
instance.gray_task_id, instance.current_step))
return None
try:
celery_id = push_channels_task(instance.gray_task_id,
instance.current_step)
except Exception as e:
gated_debug_logger.error("Celery run fail. {}".format(e))
return False
gated_debug_logger.debug(
"Start pushing celery task successful, id: {}".format(celery_id))
return True
def process_view(self, request, view_func, view_args, view_kwargs):
if 'login' in request.path:
return
if request.method == 'GET' and request.GET:
request._usage = {'params': request.GET.dict()}
elif request.method == 'POST' and request.POST:
request._usage = {'params': request.POST.dict()}
else:
try:
if request.body:
charset = request.encoding if request.encoding else settings.DEFAULT_CHARSET
request._usage = {
'params': request.body.decode(encoding=charset)
}
except (ValueError, UnicodeError, RawPostDataException):
# do nothing, don't break normal work
pass
except Exception as e:
# unexpected exception, log it
gated_debug_logger.error(str(e))
def strategy_push_content(task_id, step):
gated_debug_logger.debug(
"Start strategy push by Signals, Task {}, steps : {}".format(
task_id, step))
instance = GrayAppVersion.objects.filter(gray_task_id=task_id,
current_step=step)
count = instance.count()
if not instance:
gated_debug_logger.error("Task {}, steps : {}".format(task_id, step))
return None
# 判断内部策略还是外部策略
instance = instance.last()
if instance.current_step <= instance.gray_task.inner_strategy_steps:
gated_debug_logger.debug("Inner Strategys Push Start")
gated_debug_logger.debug("Task: {}, step: {}; start Inner PUSH".format(
instance.gray_task.id, instance.current_step))
inner = SnapshotInnerStrategy.objects.filter(
gray_task=instance.gray_task, index=instance.current_step).first()
if count > 1:
gated_debug_logger.debug("Update version not need to send push")
return None
from apps.common.tasks import send_sms_task, send_email_task
members = get_user_model().objects.filter(
usergroup__in=inner.user_groups.all()).distinct().all()
content = inner.push_content
channels = [c.name for c in inner.push_channels.all()]
phones = [m.phone for m in members]
gated_debug_logger.debug("Send sms phone list : {}".format(phones))
mails = [m.email for m in members]
gated_debug_logger.debug("Send email list : {}".format(mails))
if 'sms' in channels and TURN_ON_STRATEGY_SMS_AND_EMAIL:
gated_debug_logger.debug("Task:{} Send SMS".format(
inner.gray_task_id))
celery_id = send_sms_task.delay(phones, content)
gated_debug_logger.debug("Task:{} Send SMS celery id {}".format(
inner.gray_task_id, celery_id))
if 'mail' in channels and TURN_ON_STRATEGY_SMS_AND_EMAIL:
gated_debug_logger.debug("Task:{} Send Mail".format(inner))
# 发送的邮件标题为 task的名称
celery_id = send_email_task.delay(mails, inner.gray_task.task_name,
content)
gated_debug_logger.debug("Task:{} Send Mail celery id {}".format(
inner.gray_task_id, celery_id))
gated_debug_logger.debug("Inner Strategys Push Complated")
else:
gated_debug_logger.debug("Task: {}, step: {}; start Outer PUSH".format(
instance.gray_task.id, instance.current_step))
outer_version = instance.android_version
out_snap = SnapshotOuterStrategy.objects.filter(
gray_task=instance.gray_task, index=instance.current_step).first()
# 外部策略版本发推送
res = BpConfigOnline(out_snap.id, outer_version)
if res["status"] != 200:
gated_debug_logger.error("Outer Strategy Version Update Fail")
gated_debug_logger.error(res)
gated_debug_logger.debug("Strategy Push Completed")
gated_debug_logger.debug("End strategy push by Signals")
def create(self, validated_data):
inner_list = validated_data.get("inner_strategy", [])
outer_list = validated_data.get("outer_strategy", [])
validated_data['inner_strategy_steps'] = len(inner_list)
validated_data['total_strategy_steps'] = len(inner_list) + len(
outer_list)
try:
with transaction.atomic():
instance = GrayTask(**validated_data)
# 新建任务时候默认状态
instance.current_status = GrayStatus.get_original_status()
instance.creator = self.current_user()
instance.save()
index = 1
for k in inner_list:
if not isinstance(k.get("id"), int):
raise serializers.ValidationError(
{"msg": "strategy id must be int"})
q = get_object_or_404(InnerStrategy, id=k.get("id"))
i = copy_model_to_model(q,
SnapshotInnerStrategy,
pop_change_dict={
'app_id': 'app',
'creator_id': 'creator'
},
change_dict={
'id': None,
'gray_task': instance,
'index': index
},
data_change_dict={
'user_groups': {},
'push_channels': {}
})
i.save()
i.user_groups = q.user_groups.all()
i.push_channels = q.push_channels.all()
if k.get("pushContent"): # 更新推送内容
i.push_content = k.get("pushContent")
i.save()
index += 1
for k in outer_list:
q = get_object_or_404(OuterStrategy, id=k)
i = copy_model_to_model(q,
SnapshotOuterStrategy,
pop_change_dict={
'app_id': 'app',
'creator_id': 'creator'
},
change_dict={
'id': None,
'gray_task': instance,
'index': index
})
i.save()
index += 1
except Exception as e:
gated_debug_logger.error(
"Create Task Fail, Error msg: {}; data: {}".format(
e, validated_data))
raise serializers.ValidationError(
{"msg": "Create Task Fail, {}".format(e)})
return instance
def to_dict(self, detail=False):
try:
res = {
"id": self.id,
"name": self.task_name,
"isDisplay": self.is_display,
"isJoinKesutong": self.is_join_kesutong,
"startDate": self.start_date,
"endDate": self.end_date,
"innerStrategy": self.inner_strategy,
"outerStrategy": self.outer_strategy,
"imageId": self.image.image_id if self.image else "",
"creator": self.creator.username,
"currentStatus": self.current_status.description,
"createdDate": self.created_time.date(),
"versionDesc": self.version_desc,
"awardRule": self.award_rule,
"contact": self.contact,
"app": {
"appId": self.app.id,
"name": self.app.name,
"desc": self.app.desc,
}
}
grap_version = self.grayappversion_set.all()
# android，ios下载次数需要每步求和
from apps.issue.models import Issue
res["qualityData"] = {
"androidDownload":
sum([d.android_download_number for d in grap_version]),
"iosDownload":
sum([d.ios_download_number for d in grap_version]),
"feedback":
Issue.objects.filter(task=self).count()
}
res["appVersion"] = [{
"step":
a.current_step,
"stepName":
a.current_step_name,
"androidVersion":
a.android_version,
"androidURL":
a.android_download_url,
"androidCreateDate":
a.android_release_date,
"iosVersion":
a.ios_version,
"iosURL":
a.ios_download_url,
"iosCreateDate":
a.ios_release_date,
"updatedTime":
a.updated_time.strftime("%Y-%m-%d %H:%M:%S")
} for a in grap_version]
res['steps'] = SnapshotInnerStrategy.to_list(
self) + SnapshotOuterStrategy.to_list(self)
# 当前步骤是最后的值, 如果没有版本信息，则步骤为0
res["current_step"] = res["appVersion"][-1]["step"] if res[
"appVersion"] else 0
res["current_step_name"] = \
res["steps"][res["current_step"] - 1]["name"] if res[
"steps"] else 0
except Exception as e:
gated_debug_logger.error(
"Task:{}, instance to dict fail, {}".format(self.id, e))
raise ValidationError({
"msg":
"Task: {}, instance to dict fail, {}".format(self.id, e)
})
return res
def startTest(self, request, pk=None):
'''
PATCH: api/v1/tasks/{id}/startTest/
{
"currentStep":1,
"appVersion":{
"android":{
"urlDownload":"http://apphub.ffan.com/api/appdownload/ffan/0/0/develop/4.20.0.0.1848//None/None/Feifan_o2o_4_20_0_0_DEV_1848_2017_07_25_release.apk",
"createDate":"2017-07-25",
"versionId":"4.20.0.0.1848"
},
"ios":{
"urlDownload":"http://apphub.ffan.com/api/appdownload/ffan/0/0/develop/4.20.0.0.1848//None/None/Feifan_o2o_4_20_0_0_DEV_1848_2017_07_25_release.apk",
"createDate":"2017-07-25",
"versionId":"4.20.0.0.1848"
}
}
}
:param request:
:param args:
:param kwargs:
:return:
'''
app_dict = {}
param_dict = request.data
app_version = param_dict.get("appVersion", {})
android_version = app_version.get("android")
ios_version = app_version.get("ios")
if android_version:
try:
app_dict['android_version'] = android_version.get("versionId")
if android_version.get("createDate"):
app_dict['android_release_date'] = datetime.strptime(
android_version.get("createDate"), '%Y-%m-%d')
except Exception as e:
gated_debug_logger.error("{}".format(e))
raise ValidationError({"msg": "{}".format(e)})
app_dict['android_download_url'] = android_version.get(
"urlDownload")
if ios_version:
try:
app_dict['ios_version'] = ios_version.get("versionId")
if ios_version.get("createDate"):
app_dict['ios_release_date'] = datetime.strptime(
ios_version.get("createDate"), '%Y-%m-%d')
except Exception as e:
gated_debug_logger.error("{}".format(e))
raise ValidationError({"msg": "{}".format(e)})
app_dict['ios_download_url'] = ios_version.get("urlDownload")
app_dict['current_step'] = param_dict.get("currentStep")
task = self.get_object()
# 外灰步骤android必须提供--外灰的内容暂时不用
# if app_dict['current_step'] > task.inner_strategy_steps and (not app_dict['android_version']):
# raise ValidationError({'msg': "Outer Strategy must have android version"})
app_dict['gray_task'] = task
app_dict['operator'] = request.user
GrayAppVersion.update_version(app_dict)
return Response(task.to_dict())
def wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except Exception as e:
gated_debug_logger.error(str(e))
raise e
def get(self, request):
"""
:param request: token, appId, taskId, linkFrom, startTime, endTime
:return:
{
"status": 200,
"msg": "成功",
"data": {
"appId": 5, //参数：app id，全表唯一
"taskId": 0, //参数：task id，全表唯一
"linkFrom": null, //参数：参与途径
"startTime": null, //参数：查询开始时间
"endTime": null, //参数：查询结束时间
"taskDetailEntry": {
"userCount": 2, //进入任务详情页人数--按照人员去重
"eventCount": 70 //进入任务详情页人次
}，
"appDownload": {
"userCount": 1, //下载人数--按照人员去重
"eventCount": 19 //下载人次
},
"feedbackSubmitSuccess": {
"userCount": 0, //反馈问题提交成功人数--按照人员去重
"eventCount": 0 //反馈问题提交成功人次
}
}
}
"""
try:
app_id, task_id, link_from, start_time, end_time = self._param_handler(
request)
if (start_time and not end_time) or (not start_time and end_time):
gated_debug_logger.error(
"startTime and endTime must be provided at the same time")
raise Exception(
'startTime and endTime must be provided at the same time')
if app_id and task_id:
gated_debug_logger.error(
"appId and taskId can not be provided at the same time")
raise Exception(
'appId and taskId can not be provided at the same time')
except ValueError as e:
return Response(data={'results': str(e)},
status=status.HTTP_400_BAD_REQUEST)
res = {
'appId': app_id,
'taskId': task_id,
'linkFrom': link_from,
'startTime': start_time,
'endTime': end_time
}
for event in [
'taskDetailEntry', 'appDownload', 'feedbackSubmitSuccess'
]:
# 获取要统计的事件的事件id
event_type = EventType.objects.get(name=event)
# 没有时间要求，直接按事件类型查询事件id_list
if not start_time and not end_time:
id_list = EventTracking.objects.values('id').filter(
type=event_type)
# 有时间要求，按照时间过滤id_list
if start_time and end_time:
id_list = EventTracking.objects.values('id').filter(
type=event_type).filter(created_time__range=(start_time,
end_time))
# 如果appId和taskId都没有传，上面所得id_ist不需要按照appId或者taskId过滤
# 按照app维度查询，按照appId过滤id_list
if app_id != 0:
id_list = Property.objects.values('event_id').filter(
event__in=id_list).filter(key='appId', value=app_id)
# 按照task维度查询，按照taskId过滤id_list
if task_id != 0:
id_list = Property.objects.values('event_id').filter(
event__in=id_list).filter(key='taskId', value=task_id)
# 如果没有要求来源，直接计算人数和人次
if not link_from:
user_count = EventTracking.objects.filter(
id__in=id_list).distinct().values("user_id").count()
event_count = len(id_list)
# 如果要求来源，再在property表中过滤一次from，之后，再计算人数和人次
else:
id_list = Property.objects.values('event_id').filter(
event__in=id_list).filter(key='from', value=link_from)
user_count = EventTracking.objects.filter(
id__in=id_list).distinct().values("user_id").count()
event_count = len(id_list)
res[event] = {'userCount': user_count, 'eventCount': event_count}
return Response(data=res)