def save_fans_info(task): task_id_collection = Collection(db, "fans_info") task_id_collection.update({'share_id': task['share_id']}, {'$set': task}, True)
def get_collection(db: Database, name: str) -> Collection: collection = Collection(db, name) return collection
def handle_get_task(): task_id_collection = Collection(db, 'task_id') return task_id_collection.find_one_and_delete({})
def save_data(item): """ 原始插入数据, 已被save_task()代替 """ data_collections = Collection(db, 'douyin_data') data_collections.insert(item)
def process_item(self, item, spider): db_collections = Collection(self.db_data, 'mafengwo_article') db_collections.update({'from_url': item['from_url']}, item, True) return item
def save_task(task): task_collections = Collection(db, 'task_id') task_collections.update({'share_id': task['share_id']}, task, True)
def insert_item(self, item): db_collection = Collection(self.db_data, 'dou_guo_mei_shi_item') db_collection.insert(item)
def insert_item(self, item): #表名 db_collection = Collection(self.db_data, 'douyin_chanmama_info_data') db_collection.insert(item)
def get_collection(self): return Collection(self.get_db(),self.collection)
# -*- coding:utf-8 -*- import pymongo import os from pymongo.collection import Collection # client = pymongo.MongoClient(host='192.168.209.128', port=27017) client = pymongo.MongoClient(host='127.0.0.1', port=27017) db = client['douyin'] collection = Collection(db, 'taskid') collection_user_info = Collection(db, 'userinfo') def init_task(): shareid = os.path.dirname( os.path.realpath(__file__)) + "\\utils\\shareid.txt" with open(shareid) as f: for shareid in f.readlines(): task_id = {} task_id['share_id'] = shareid.replace('\n', '') collection.insert_one(task_id) # print(task_id) def save_task(task): try: collection.update({'share_id': task['share_id']}, task, True) except: pass # collection.update_one({'share_id': task['share_id']}, {'$set': {'share_id': task['share_id']}})
def create_collection( self, name: str, codec_options: Optional[CodecOptions] = None, read_preference: Optional[_ServerMode] = None, write_concern: Optional["WriteConcern"] = None, read_concern: Optional["ReadConcern"] = None, session: Optional["ClientSession"] = None, **kwargs: Any, ) -> Collection[_DocumentType]: """Create a new :class:`~pymongo.collection.Collection` in this database. Normally collection creation is automatic. This method should only be used to specify options on creation. :class:`~pymongo.errors.CollectionInvalid` will be raised if the collection already exists. :Parameters: - `name`: the name of the collection to create - `codec_options` (optional): An instance of :class:`~bson.codec_options.CodecOptions`. If ``None`` (the default) the :attr:`codec_options` of this :class:`Database` is used. - `read_preference` (optional): The read preference to use. If ``None`` (the default) the :attr:`read_preference` of this :class:`Database` is used. - `write_concern` (optional): An instance of :class:`~pymongo.write_concern.WriteConcern`. If ``None`` (the default) the :attr:`write_concern` of this :class:`Database` is used. - `read_concern` (optional): An instance of :class:`~pymongo.read_concern.ReadConcern`. If ``None`` (the default) the :attr:`read_concern` of this :class:`Database` is used. - `collation` (optional): An instance of :class:`~pymongo.collation.Collation`. - `session` (optional): a :class:`~pymongo.client_session.ClientSession`. - `**kwargs` (optional): additional keyword arguments will be passed as options for the `create collection command`_ All optional `create collection command`_ parameters should be passed as keyword arguments to this method. Valid options include, but are not limited to: - ``size`` (int): desired initial size for the collection (in bytes). For capped collections this size is the max size of the collection. - ``capped`` (bool): if True, this is a capped collection - ``max`` (int): maximum number of objects if capped (optional) - ``timeseries`` (dict): a document specifying configuration options for timeseries collections - ``expireAfterSeconds`` (int): the number of seconds after which a document in a timeseries collection expires - ``validator`` (dict): a document specifying validation rules or expressions for the collection - ``validationLevel`` (str): how strictly to apply the validation rules to existing documents during an update. The default level is "strict" - ``validationAction`` (str): whether to "error" on invalid documents (the default) or just "warn" about the violations but allow invalid documents to be inserted - ``indexOptionDefaults`` (dict): a document specifying a default configuration for indexes when creating a collection - ``viewOn`` (str): the name of the source collection or view from which to create the view - ``pipeline`` (list): a list of aggregation pipeline stages - ``comment`` (str): a user-provided comment to attach to this command. This option is only supported on MongoDB >= 4.4. .. versionchanged:: 3.11 This method is now supported inside multi-document transactions with MongoDB 4.4+. .. versionchanged:: 3.6 Added ``session`` parameter. .. versionchanged:: 3.4 Added the collation option. .. versionchanged:: 3.0 Added the codec_options, read_preference, and write_concern options. .. _create collection command: https://docs.mongodb.com/manual/reference/command/create """ with self.__client._tmp_session(session) as s: # Skip this check in a transaction where listCollections is not # supported. if (not s or not s.in_transaction ) and name in self.list_collection_names(filter={"name": name}, session=s): raise CollectionInvalid("collection %s already exists" % name) return Collection( self, name, True, codec_options, read_preference, write_concern, read_concern, session=s, **kwargs, )
def get_infsoync_collections(self, collection='form_answer', create=False): database = get_infosync_connection() return Collection(database['db'], collection, create)
def get_collections(self, collection='form_answer', create=False): database = self.get_user_connection() return Collection(database['db'], collection, create)
def create_collection(self, collection, user_connection): if config['CREATE'] and collection in user_connection['db'].collection_names(): oldCollection = user_connection['db'][collection] oldCollection.drop() newCollection = Collection(user_connection['db'], collection, create=config['CREATE']) return newCollection
for d in gd: for n in d: try: if n == "SECCODE": pass else: d[n] = float(d[n].replace(",", "")) except: pass # print(d) ol.append(d) try: # 建立连接 client = pymongo.MongoClient('localhost', 27017) # 建立数据库 db = client["XinlangFinance"] # 从原有的txt文件导入share_id: # 表的对象化 mgtable = Collection(db, 'FinanceReport_data2') mgtable.insert_many(ol) except: print("写入出错!!") pass # print(gd)
def delete_flag(self,collection_name): db_collections = Collection(self.db_data,collection_name) return db_collections.find_one_and_delete({})
def test_get_coll(self): db = Database(self.client, "pymongo_test") self.assertEqual(db.test, db["test"]) self.assertEqual(db.test, Collection(db, "test")) self.assertNotEqual(db.test, Collection(db, "mike")) self.assertEqual(db.test.mike, db["test.mike"])
def delete_all_flag(self): db_collections = Collection(self.db_data,'didi_flag') db_collections.remove({})
def handle_insert_douyin(douyin_info): task_id_collections = Collection(db, 'douyin_info') task_id_collections.insert(douyin_info)
def insert_data(self,collection_name,item): db_collections = Collection(self.db_data,collection_name) db_collections.update({'shop_id':item['shop_id']},item,True)
def save_task(task): """ 更新,不存在则插入 """ task_collections = Collection(db, 'douyin_task') task_collections.update({'share_id': task['share_id']}, task, True)
def find_data(self,collection_name,item): db_collections = Collection(self.db_data,collection_name) shop_id_result = db_collections.find_one({'shop_id':item}) return shop_id_result
def __init__(self, db, collection_Name): self._collection = Collection(db, name=collection_Name) self._logger = logging.getLogger(f'StockCollection_{collection_Name}')
def save_flag(self,collection_name,item): db_collections = Collection(self.db_data,collection_name) db_collections.update({'flag':item['flag']},item,True)
#{'username':'******','pwd':'gengxiaochan'}, {'username':'******','pwd':'ppjnelv2'}, #{'username':'******','pwd':'xkexbta8'}, #{'username':'******','pwd':'liuchenggang1988'}, ] agent = 'Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0' headers = { 'User-Agent': agent } session = requests.session() mongo_client = MongoClient(host = '172.16.40.12', port = 27017) mongo_oreo = mongo_client.get_database('oreo') mongo_cookies_col = Collection(mongo_oreo,'cookies') #mongo_oreo = mongo_client.get_database(settings.MONGO_DB_NAME) def getCookies(accounts,mongo_cookies_col): cookies=[] loginURL="" for e in accounts: account=e['username'] password=e['pwd'] cookie=login(account,password) cookie['userID'] = account cookies.append(cookie) mongo_cookies_col.update({'userID':cookie['userID']} , cookie , True) print cookie return cookies
from pymongo.connection import Connection from pymongo.database import Database from pymongo.collection import Collection import FreebaseApi import AlchemyApiCategory import nerd import DbPediaApi # constants and globals MONGO_CON = '107.22.242.25:27017' mongo_con = Connection(MONGO_CON) MONGO_DB = 'queries' mongo_db = Database(mongo_con, MONGO_DB) MONGO_COL = 'hott_rends' mongo_col = Collection(mongo_db, MONGO_COL) # for specific data LOG_FILE = '/home/penser/tmp/specifics.txt' # for simple data STAT_PAPER = '/home/penser/tmp/stats.txt' # number of queries LIMIT = 420 # threshold to cut off unusual types THRESHOLD = 2 class HotTrendsAnalysisAl: ''' docs ''' def __init__(self): self.freebaseApi = FreebaseApi.FreebaseApi()
def insert_item(self, item): #表名 db_collection = Collection(self.db_data, 'wx_info_data') db_collection.insert(item)
def create_collection(self, name, codec_options=None, read_preference=None, write_concern=None, read_concern=None, **kwargs): """Create a new :class:`~pymongo.collection.Collection` in this database. Normally collection creation is automatic. This method should only be used to specify options on creation. :class:`~pymongo.errors.CollectionInvalid` will be raised if the collection already exists. Options should be passed as keyword arguments to this method. Supported options vary with MongoDB release. Some examples include: - "size": desired initial size for the collection (in bytes). For capped collections this size is the max size of the collection. - "capped": if True, this is a capped collection - "max": maximum number of objects if capped (optional) See the MongoDB documentation for a full list of supported options by server version. :Parameters: - `name`: the name of the collection to create - `codec_options` (optional): An instance of :class:`~bson.codec_options.CodecOptions`. If ``None`` (the default) the :attr:`codec_options` of this :class:`Database` is used. - `read_preference` (optional): The read preference to use. If ``None`` (the default) the :attr:`read_preference` of this :class:`Database` is used. - `write_concern` (optional): An instance of :class:`~pymongo.write_concern.WriteConcern`. If ``None`` (the default) the :attr:`write_concern` of this :class:`Database` is used. - `read_concern` (optional): An instance of :class:`~pymongo.read_concern.ReadConcern`. If ``None`` (the default) the :attr:`read_concern` of this :class:`Database` is used. - `collation` (optional): An instance of :class:`~pymongo.collation.Collation`. - `**kwargs` (optional): additional keyword arguments will be passed as options for the create collection command .. versionchanged:: 3.4 Added the collation option. .. versionchanged:: 3.0 Added the codec_options, read_preference, and write_concern options. .. versionchanged:: 2.2 Removed deprecated argument: options """ if name in self.collection_names(): raise CollectionInvalid("collection %s already exists" % name) return Collection(self, name, True, codec_options, read_preference, write_concern, read_concern, **kwargs)
def get_user_list(): task_id_collection = Collection(db, 'user_info') return task_id_collection.find({})
def update_user(self, user): collection = Collection(self.db, 'douyin_user') collection.update_one({'share_id': user['share_id']}, {'$set': user})