def __init__(self, **arg):
self.conn = MongoClient('localhost', 27017)
self.db = self.conn[setting.DATABASES]
tb = self.db[setting.TASKS_LIST] #总任务列表
tb.ensure_index('guid', unique=False) # 为guid创建唯一索引
db = self.conn[setting.TMP_DB]
self.fs = GridFS(db, 'body') # 存储任务body的大文档，无空间限制
self.result = {'success': True, 'error': "error reason", 'content': ''}
self.obj_db = db_oprate.collection_db() # 数据库操作对象
self.save_upload_data = save_clientdata_inter.save_Data() #保存上传数据对象
def __init__(self,concurrency=100):#并发个数
self.mongo_obj = db_oprate.collection_db()#mongo数据库对象
self.headersParameters = { # 发送HTTP请求时的HEAD信息，用于伪装为浏览器
'Connection': 'Keep-Alive',
'Accept': 'text/html, application/xhtml+xml, */*',
'Accept-Language': 'en-US,en;q=0.8,zh-Hans-CN;q=0.5,zh-Hans;q=0.3',
'Accept-Encoding': 'gzip, deflate',
'User-Agent': ua.random
}
self.proxies ={}#requestsd的代理
self.timeout = 10
self.phant_path = '/home/topinfo/phantomjs-2(crawl).1.1-linux-x86_64/bin/phantomjs'#无头浏览器的路径
self.pgevent = pool.Pool(concurrency)
self.db_obj = DB_Connect.task_opt()
#from django.test import TestCase
# Create your tests here.
from datetime import datetime
import db_oprate
import os, sys
mongo_obj = db_oprate.collection_db() #数据库操作对象
task_tb = mongo_obj.choice_main_table('task_main')
task_obj = mongo_obj.find_data(task_tb, {})
task_type = mongo_obj.obj_distinct(task_obj, 'topic') # 得到具体的任务类型
for item in task_type:
task_data = mongo_obj.find_data(task_tb, {'topic': item})
task_sum = task_data.count()
inter_list = mongo_obj.obj_distinct(task_data, 'interval')
timeout_list = mongo_obj.obj_distinct(task_data, 'timeout')
print({
'topic': item,
'interval': inter_list,
'timeount': timeout_list,
'count': task_sum
})
import db_oprate
db_obj = db_oprate.collection_db() #操作数据库对象
data_tb = db_obj.choice_data_table() # 切换到存储数据表
data = db_obj.find_modify(data_tb, {'upload_flag': 0},
{'$set': {
'upload_flag': 0
}})
obj_id = data['body'] # 得到存储数据的id
body = db_obj.gridfs_get_data(obj_id) # 从文档中读出body字段{'result':'',data:''}
body = eval(body) # 还原body
data['result'] = body['result']
data['data'] = body['data']
print(data)
from bson.objectid import ObjectId
import db_oprate
obj = db_oprate.collection_db()
a = obj.find_data(obj.tb, {}, 1)
for i in a:
_id = i["_id"]
_id = str(_id)
print(type(_id))
break
_id = ObjectId(_id)
a = obj.find_data(obj.tb, {"_id": _id}, 1)
for i in a:
print(i)
def __init__(self,**arg):
self.obj_db = db_oprate.collection_db()#数据库操作对象
self.save_upload_data =save_clientdata_inter.save_Data()#保存上传数据对象
self.result = {'success': True, 'error': "error reason", 'content': ''}
def __init__(self):
self.obj_db = db_oprate.collection_db() #数据库操作对象
def __init__(self, **arg):
self.obj_db = db_oprate.collection_db() #数据库操作对象
self.result = {'success': True, 'error': "error reason", 'content': ''}