def startspiderbyid(request):
# 设置模块查找目录
sys.path.append(SpiderBaseDir)
# 保存get请求的值
spider_id = request.GET['spider_id']
spider_argv = request.GET['run_argvs']
# 取出目标爬虫中的模块名称
result = Spider.objects.filter(id=spider_id)
spider_obj = result[0]
func_name = spider_obj.spider_runfunction
# 如果模块名称没有定义就不会执行新的任务，否则使用新的进程运行爬虫程序
if func_name != 'no info':
# 使用新进程开始任务
mymodule = __import__(func_name)
if spider_argv != '':
spider_argv = tuple(spider_argv.split(' '))
p = Process(target=mymodule.main, args=spider_argv)
markcontent = '爬虫启动成功, 参数为：' + '<b>' + request.GET[
'run_argvs'] + '</b>'
else:
p = Process(target=mymodule.main)
markcontent = '爬虫启动成功, 无自定义参数：'
p.start()
# 创建通知对象
me_obj = Message()
mytime = str(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()))
me_obj.action_time = mytime
me_obj.content = markcontent
me_obj.user = '******'
me_obj.spider = spider_obj
me_obj.save()
# 改变数据库中爬虫运行状态
spider_obj.spider_runing = True
spider_obj.save()
print(
'主进程结束标志1：此视图结束，爬虫程序自己运行，但是运行状态会放在redis和log中，使用视图调用并传递到ajax和mysql')
return JsonResponse({'data': '启动爬虫成功，请到实时监控界面查看！', 'doing': 1})
else:
# 改变数据库中爬虫运行状态
spider_obj.spider_runing = False
spider_obj.save()
# 创建通知对象
me_obj = Message()
mytime = str(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()))
me_obj.action_time = mytime
me_obj.content = '启动爬虫失败：没有足够的权限'
me_obj.user = '******'
me_obj.spider = spider_obj
me_obj.save()
print(
'主进程结束标志2：此视图结束，爬虫程序自己运行，但是运行状态会放在redis和log中，使用视图调用并传递到ajax和mysql')
return JsonResponse({'data': '权限不足，无法操作！', 'doing': 0})
def detail_message(request, conv_id):
"""
view untuk menampilkan detail pesan
beserta percakapan
"""
conv = get_object_or_404(Conversation, id=conv_id, users=request.user, membership__deleted=False)
list_messages = Message.objects.filter(conversation=conv)
msg = list_messages[0]
if request.POST:
form = SendConversationForm(request.POST)
if form.is_valid():
msg = Message()
msg.conversation = conv
msg.user = request.user
msg.content = form.cleaned_data['content']
msg.save()
to2 = list(conv.users.all())
to2.remove(request.user)
notification.send(to2, 'new_message', {'user': request.user, 'message': conv})
messages.info(request, _('Message sent'))
return HttpResponseRedirect(conv.get_absolute_url())
else:
form = SendConversationForm()
return render_to_response('message/detail_message.html',
{"conv": conv,
'form': form,
'msg': msg,
'list_messages': list_messages},
context_instance=RequestContext(request))
def save_message(user_id, message):
type_name = message.get('type_name')
args = message.get('args')
file = message.get('file', '')
conversation_id = message.get('conversation_id')
mt = MessageType.select().where(MessageType.name == type_name).first()
u = User.select().where(User.id == user_id).first()
cps = ConversationParty.select().where(ConversationParty.conversation == conversation_id)
number_of_conversationees = cps.count()
if not mt or not u or not cps or not number_of_conversationees:
raise InvalidMessageDataException('Couldn\'t save message: invalid message data')
m = Message()
with database.transaction():
m.conversation = conversation_id
m.message_type = mt
m.sender_id = user_id
m.ts = datetime.now()
m.file = file
m.content, m.display_content = __save_content(args, mt, user_id, cps)
m.save()
update_conversation(conversation_id=conversation_id,
last_message=m)
mark_message_as_read(user_id=user_id,
message=m,
conversation_id=conversation_id)
message_object = get_message_json(message=m)
message_object['recipient_ids'] = [cp.user.id for cp in cps]
return json.dumps(message_object)
def sendMessage(from_id, to_id, content):
message = Message()
message.from_user_id = from_id
message.to_user_id = to_id
message.content = content
message.date_publish = datetime.datetime.now()
message.save()
def save_message(name, content):
message = Message()
message.name = name
message.content = content
db.session.add(message)
db.session.commit()
return 1
def post(self):
# Set up the message instance
message = Message()
message.author = users.get_current_user()
if self.request.get('content') != '':
message.content = self.request.get('content')
message.put()
self.redirect('/')
def getspiderlogbyid(request):
# 根据ID获取爬虫任务名称
spider_id = request.GET['spider_id']
result = Spider.objects.filter(id=spider_id)
spider_obj = result[0]
func_name = spider_obj.spider_runfunction
# 先判断爬虫的对应模块是否是默认值no info，否则就返回'done'给前端，前端就会停止循环发送请求
tags = ''
if func_name == 'no info':
return JsonResponse({'data': 'done', 'content': tags})
else:
# 拼接log文件名称，并且试图读取其中的信息，如果没有这个文件就会返回空值
logfilename = SpiderBaseDir + func_name + '/' + func_name + '.log'
try:
with open(logfilename, 'r') as f:
content = f.read()
print '读取log文件成功'
except Exception as e:
print '读取log文件失败', e, logfilename
content = ''
# 生成html标签内容
tags = ''
for each in content.split('\n'):
li_tag = '<li>' + each + '</li>'
tags += li_tag
# 根据名称，在redis查找任务信息中的runing，如果是False前端就会结束循环
redisclient = redis.Redis(host='127.0.0.1', port=6379, db=1)
info = redisclient.get(func_name + ':runing')
# 根据redis的状态判断是否要作为最后一次发送数据
if info == b'False':
# 改变数据库中爬虫运行状态
try:
spider_obj.spider_runing = False
spider_obj.save()
except Exception as e:
print '改变mysql中爬虫状态失败', e
print 'log信息传输结束'
# 创建通知对象
me_obj = Message()
mytime = str(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()))
me_obj.action_time = mytime
me_obj.content = '爬虫运行结束并退出'
me_obj.user = '******'
me_obj.spider = spider_obj
me_obj.save()
return JsonResponse({'data': 'done', 'content': tags})
else:
return JsonResponse({'data': 'continue', 'content': tags})
def post(self):
message = Message()
user = users.get_current_user()
if user:
message.author = user
#Added the content to the message:
message.content = cgi.escape(self.request.get('content'))
#Assigned the id to the message:
message.postID = Message.max_id() + 1
message.put()
def storeMessage(self, messageProtocolEntity):
if messageProtocolEntity.isOutgoing():
conversation = self.getConversation(messageProtocolEntity.getTo())
else:
conversation = self.getConversation(messageProtocolEntity.getFrom())
if messageProtocolEntity.isOutgoing():
messageGenId = messageProtocolEntity.getId()
try:
while True:
Message.get(id_gen = messageGenId)
messageIdDis = messageGenId.split('-')
if len(messageGenIdDis) == 2:
messageIdCount = int(messageIdDis[1]) + 1
else:
messageIdCount = 1
messageGenId = "%s-%s" % (messageIdDis[0], messageIdCount)
except peewee.DoesNotExist:
pass
messageProtocolEntity._id = messageGenId
message = Message(
id_gen = messageProtocolEntity.getId(),
conversation = conversation,
t_sent = datetime.datetime.fromtimestamp(messageProtocolEntity.getTimestamp())
)
if messageProtocolEntity.getType() == MessageProtocolEntity.MESSAGE_TYPE_MEDIA:
media = self.getMedia(messageProtocolEntity, message)
media.save()
message.media = media
else:
message.content = messageProtocolEntity.getBody()
if type(message.content) is bytearray:
message.content = message.content.decode('latin-1')
message.save()
return message
def dataopinfo(request):
file_name = request.GET['FN']
dir_name = request.GET['DIR']
spider_obj = Spider.objects.filter(spider_runfunction=dir_name)[0]
me_obj = Message()
mytime = str(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()))
me_obj.action_time = mytime
me_obj.content = '下载文件：%s' % file_name
me_obj.user = '******'
me_obj.spider = spider_obj
me_obj.save()
return JsonResponse({'data': '1'})
def post(self, request, **kwargs):
file = request.FILES.get('file')
content = request.POST.get('message')
sender = request.user
message = Message()
message.file=file
# print message.file
message.content=content
message.receiver = kwargs['receiver']
message.sender=User.objects.get(username=sender)
message.sender_full_name = message.sender.userprofile.get_full_name()
print message.sender_full_name
message.save()
# print message
#
# print message.file
# print (request.user)
messages = Message.objects.filter(Q(sender=request.user, receiver=kwargs['receiver']) | Q(sender=User.objects.get(username=kwargs['receiver']), receiver=request.user))
messages5 = Message.objects.filter(Q(sender=request.user) | Q(receiver=request.user))[::-1]
if len(messages5) <=5:
message5 = messages5
else :
message5 = messages5[:5]
# message5=message5[::-1]
# print profile
profile = User.objects.filter(username=kwargs['receiver'])[0]
auth_form = AuthenticateForm()
return render(request, 'message.html', {'messages':messages, 'profile':profile ,'auth_form':auth_form, 'message5':message5})
def create_message(request, username=None):
"""
view untuk menampilkan pembuatan pesan
"""
to = []
if username:
to.append(username)
if request.POST:
to = request.POST.getlist('to')
form = CreateMessageForm(request.POST)
if form.is_valid():
title = form.cleaned_data['title']
message = form.cleaned_data['content']
location = form.cleaned_data['location']
conv = Conversation()
conv.title = title
conv.origin = request.user
conv.location = location
conv.save()
Membership.objects.create(user=request.user,
conversation=conv)
to2 = []
for person in to:
to_person = User.objects.get(username=person)
to2.append(to_person)
Membership.objects.create(user=to_person,
conversation=conv)
msg = Message()
msg.conversation = conv
msg.content = message
msg.user = request.user
msg.save()
notification.send(to2, 'new_message',
{'user': request.user, 'message': conv})
messages.info(request, _('Message sent'))
return HttpResponseRedirect(reverse('message_list'))
else:
form = CreateMessageForm()
return render_to_response('message/create_message.html',
{'form': form,
'to': to},
context_instance=(RequestContext(request)))
def thread(request, **args):
t = loader.get_template("discussion.view")
thread = None
if "id" in args:
thread = Thread.objects.get(id=int(args["id"]))
message_form = None
if thread and thread.openForEvent():
if request.method == "POST":
# Post dictionary needs to be mutable in order to remove the value from the messagebox
post_vars = {}
for key in request.POST:
post_vars[key] = request.POST[key]
message_form = MessageForm(post_vars)
if message_form.is_valid():
message = Message()
message.content = message_form.cleaned_data["content"]
message_form.data["content"] = ""
message.time = datetime.now()
message.user = request.user
message.thread = thread
message.save()
else:
message_form = MessageForm()
c = {
"thread": thread,
"thread_message": thread.getStatusMessage(),
"thread_status": thread.getStatus(),
"messages": None,
"message_form": message_form,
"is_authenticated": request.user.is_authenticated(),
}
if thread:
messages = Message.objects.filter(thread__id=thread.id)
c["messages"] = messages
c.update(csrf(request))
return HttpResponse(t.render(Context(c)))
def thread(request, **args):
t = loader.get_template('discussion.view')
thread = None
if 'id' in args:
thread = Thread.objects.get(id=int(args['id']))
message_form = None
if thread and thread.openForEvent():
if request.method == 'POST':
# Post dictionary needs to be mutable in order to remove the value from the messagebox
post_vars = {}
for key in request.POST:
post_vars[key] = request.POST[key]
message_form = MessageForm(post_vars)
if message_form.is_valid():
message = Message()
message.content = message_form.cleaned_data['content']
message_form.data['content'] = ''
message.time = datetime.now()
message.user = request.user
message.thread = thread
message.save()
else:
message_form = MessageForm()
c = {
'thread': thread,
'thread_message': thread.getStatusMessage(),
'thread_status': thread.getStatus(),
'messages': None,
'message_form': message_form,
'is_authenticated': request.user.is_authenticated()
}
if thread:
messages = Message.objects.filter(thread__id=thread.id)
c['messages'] = messages
c.update(csrf(request))
return HttpResponse(t.render(Context(c)))
def stopspiderbyid(request):
spider_id = request.GET['spider_id']
result = Spider.objects.filter(id=spider_id)
spider_obj = result[0]
func_name = spider_obj.spider_runfunction
redisclient = redis.Redis(host='127.0.0.1', port=6379, db=1)
redisclient.getset(func_name + ':runing', 'False')
spider_obj.spider_runing = False
spider_obj.save()
# 创建通知对象
me_obj = Message()
mytime = str(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()))
me_obj.action_time = mytime
me_obj.content = '手动停止爬虫'
me_obj.user = '******'
me_obj.spider = spider_obj
me_obj.save()
print('修改redis和mysql状态，爬虫退出')
return JsonResponse({'data': '停止采集成功！'})
def post(self):
entry = Message()
entry.name = self.request.get("name")
entry.content = self.request.get("content")
entry.put()
self.redirect("/")
