def __init__(self, **arg): self.conn = MongoClient('localhost', 27017) self.db = self.conn[setting.DATABASES] tb = self.db[setting.TASKS_LIST] #总任务列表 tb.ensure_index('guid', unique=False) # 为guid创建唯一索引 db = self.conn[setting.TMP_DB] self.fs = GridFS(db, 'body') # 存储任务body的大文档,无空间限制 self.result = {'success': True, 'error': "error reason", 'content': ''} self.obj_db = db_oprate.collection_db() # 数据库操作对象 self.save_upload_data = save_clientdata_inter.save_Data() #保存上传数据对象
def __init__(self,concurrency=100):#并发个数 self.mongo_obj = db_oprate.collection_db()#mongo数据库对象 self.headersParameters = { # 发送HTTP请求时的HEAD信息,用于伪装为浏览器 'Connection': 'Keep-Alive', 'Accept': 'text/html, application/xhtml+xml, */*', 'Accept-Language': 'en-US,en;q=0.8,zh-Hans-CN;q=0.5,zh-Hans;q=0.3', 'Accept-Encoding': 'gzip, deflate', 'User-Agent': ua.random } self.proxies ={}#requestsd的代理 self.timeout = 10 self.phant_path = '/home/topinfo/phantomjs-2(crawl).1.1-linux-x86_64/bin/phantomjs'#无头浏览器的路径 self.pgevent = pool.Pool(concurrency) self.db_obj = DB_Connect.task_opt()
#from django.test import TestCase # Create your tests here. from datetime import datetime import db_oprate import os, sys mongo_obj = db_oprate.collection_db() #数据库操作对象 task_tb = mongo_obj.choice_main_table('task_main') task_obj = mongo_obj.find_data(task_tb, {}) task_type = mongo_obj.obj_distinct(task_obj, 'topic') # 得到具体的任务类型 for item in task_type: task_data = mongo_obj.find_data(task_tb, {'topic': item}) task_sum = task_data.count() inter_list = mongo_obj.obj_distinct(task_data, 'interval') timeout_list = mongo_obj.obj_distinct(task_data, 'timeout') print({ 'topic': item, 'interval': inter_list, 'timeount': timeout_list, 'count': task_sum })
import db_oprate db_obj = db_oprate.collection_db() #操作数据库对象 data_tb = db_obj.choice_data_table() # 切换到存储数据表 data = db_obj.find_modify(data_tb, {'upload_flag': 0}, {'$set': { 'upload_flag': 0 }}) obj_id = data['body'] # 得到存储数据的id body = db_obj.gridfs_get_data(obj_id) # 从文档中读出body字段{'result':'',data:''} body = eval(body) # 还原body data['result'] = body['result'] data['data'] = body['data'] print(data)
from bson.objectid import ObjectId import db_oprate obj = db_oprate.collection_db() a = obj.find_data(obj.tb, {}, 1) for i in a: _id = i["_id"] _id = str(_id) print(type(_id)) break _id = ObjectId(_id) a = obj.find_data(obj.tb, {"_id": _id}, 1) for i in a: print(i)
def __init__(self,**arg): self.obj_db = db_oprate.collection_db()#数据库操作对象 self.save_upload_data =save_clientdata_inter.save_Data()#保存上传数据对象 self.result = {'success': True, 'error': "error reason", 'content': ''}
def __init__(self): self.obj_db = db_oprate.collection_db() #数据库操作对象
def __init__(self, **arg): self.obj_db = db_oprate.collection_db() #数据库操作对象 self.result = {'success': True, 'error': "error reason", 'content': ''}