def save_fans_info(task):
task_id_collection = Collection(db, "fans_info")
task_id_collection.update({'share_id': task['share_id']}, {'$set': task},
True)
def get_collection(db: Database, name: str) -> Collection:
collection = Collection(db, name)
return collection
def handle_get_task():
task_id_collection = Collection(db, 'task_id')
return task_id_collection.find_one_and_delete({})
def save_data(item):
"""
原始插入数据， 已被save_task()代替
"""
data_collections = Collection(db, 'douyin_data')
data_collections.insert(item)
def process_item(self, item, spider):
db_collections = Collection(self.db_data, 'mafengwo_article')
db_collections.update({'from_url': item['from_url']}, item, True)
return item
def save_task(task):
task_collections = Collection(db, 'task_id')
task_collections.update({'share_id': task['share_id']}, task, True)
def insert_item(self, item):
db_collection = Collection(self.db_data, 'dou_guo_mei_shi_item')
db_collection.insert(item)
def insert_item(self, item):
#表名
db_collection = Collection(self.db_data, 'douyin_chanmama_info_data')
db_collection.insert(item)
def get_collection(self):
return Collection(self.get_db(),self.collection)
# -*- coding:utf-8 -*-
import pymongo
import os
from pymongo.collection import Collection
# client = pymongo.MongoClient(host='192.168.209.128', port=27017)
client = pymongo.MongoClient(host='127.0.0.1', port=27017)
db = client['douyin']
collection = Collection(db, 'taskid')
collection_user_info = Collection(db, 'userinfo')
def init_task():
shareid = os.path.dirname(
os.path.realpath(__file__)) + "\\utils\\shareid.txt"
with open(shareid) as f:
for shareid in f.readlines():
task_id = {}
task_id['share_id'] = shareid.replace('\n', '')
collection.insert_one(task_id)
# print(task_id)
def save_task(task):
try:
collection.update({'share_id': task['share_id']}, task, True)
except:
pass
# collection.update_one({'share_id': task['share_id']}, {'$set': {'share_id': task['share_id']}})
def create_collection(
self,
name: str,
codec_options: Optional[CodecOptions] = None,
read_preference: Optional[_ServerMode] = None,
write_concern: Optional["WriteConcern"] = None,
read_concern: Optional["ReadConcern"] = None,
session: Optional["ClientSession"] = None,
**kwargs: Any,
) -> Collection[_DocumentType]:
"""Create a new :class:`~pymongo.collection.Collection` in this
database.
Normally collection creation is automatic. This method should
only be used to specify options on
creation. :class:`~pymongo.errors.CollectionInvalid` will be
raised if the collection already exists.
:Parameters:
- `name`: the name of the collection to create
- `codec_options` (optional): An instance of
:class:`~bson.codec_options.CodecOptions`. If ``None`` (the
default) the :attr:`codec_options` of this :class:`Database` is
used.
- `read_preference` (optional): The read preference to use. If
``None`` (the default) the :attr:`read_preference` of this
:class:`Database` is used.
- `write_concern` (optional): An instance of
:class:`~pymongo.write_concern.WriteConcern`. If ``None`` (the
default) the :attr:`write_concern` of this :class:`Database` is
used.
- `read_concern` (optional): An instance of
:class:`~pymongo.read_concern.ReadConcern`. If ``None`` (the
default) the :attr:`read_concern` of this :class:`Database` is
used.
- `collation` (optional): An instance of
:class:`~pymongo.collation.Collation`.
- `session` (optional): a
:class:`~pymongo.client_session.ClientSession`.
- `**kwargs` (optional): additional keyword arguments will
be passed as options for the `create collection command`_
All optional `create collection command`_ parameters should be passed
as keyword arguments to this method. Valid options include, but are not
limited to:
- ``size`` (int): desired initial size for the collection (in
bytes). For capped collections this size is the max
size of the collection.
- ``capped`` (bool): if True, this is a capped collection
- ``max`` (int): maximum number of objects if capped (optional)
- ``timeseries`` (dict): a document specifying configuration options for
timeseries collections
- ``expireAfterSeconds`` (int): the number of seconds after which a
document in a timeseries collection expires
- ``validator`` (dict): a document specifying validation rules or expressions
for the collection
- ``validationLevel`` (str): how strictly to apply the
validation rules to existing documents during an update. The default level
is "strict"
- ``validationAction`` (str): whether to "error" on invalid documents
(the default) or just "warn" about the violations but allow invalid
documents to be inserted
- ``indexOptionDefaults`` (dict): a document specifying a default configuration
for indexes when creating a collection
- ``viewOn`` (str): the name of the source collection or view from which
to create the view
- ``pipeline`` (list): a list of aggregation pipeline stages
- ``comment`` (str): a user-provided comment to attach to this command.
This option is only supported on MongoDB >= 4.4.
.. versionchanged:: 3.11
This method is now supported inside multi-document transactions
with MongoDB 4.4+.
.. versionchanged:: 3.6
Added ``session`` parameter.
.. versionchanged:: 3.4
Added the collation option.
.. versionchanged:: 3.0
Added the codec_options, read_preference, and write_concern options.
.. _create collection command:
https://docs.mongodb.com/manual/reference/command/create
"""
with self.__client._tmp_session(session) as s:
# Skip this check in a transaction where listCollections is not
# supported.
if (not s or not s.in_transaction
) and name in self.list_collection_names(filter={"name": name},
session=s):
raise CollectionInvalid("collection %s already exists" % name)
return Collection(
self,
name,
True,
codec_options,
read_preference,
write_concern,
read_concern,
session=s,
**kwargs,
)
def get_infsoync_collections(self, collection='form_answer', create=False):
database = get_infosync_connection()
return Collection(database['db'], collection, create)
def get_collections(self, collection='form_answer', create=False):
database = self.get_user_connection()
return Collection(database['db'], collection, create)
def create_collection(self, collection, user_connection):
if config['CREATE'] and collection in user_connection['db'].collection_names():
oldCollection = user_connection['db'][collection]
oldCollection.drop()
newCollection = Collection(user_connection['db'], collection, create=config['CREATE'])
return newCollection
for d in gd:
for n in d:
try:
if n == "SECCODE":
pass
else:
d[n] = float(d[n].replace(",", ""))
except:
pass
# print(d)
ol.append(d)
try:
# 建立连接
client = pymongo.MongoClient('localhost', 27017)
# 建立数据库
db = client["XinlangFinance"]
# 从原有的txt文件导入share_id：
# 表的对象化
mgtable = Collection(db, 'FinanceReport_data2')
mgtable.insert_many(ol)
except:
print("写入出错！！")
pass
# print(gd)
def delete_flag(self,collection_name):
db_collections = Collection(self.db_data,collection_name)
return db_collections.find_one_and_delete({})
def test_get_coll(self):
db = Database(self.client, "pymongo_test")
self.assertEqual(db.test, db["test"])
self.assertEqual(db.test, Collection(db, "test"))
self.assertNotEqual(db.test, Collection(db, "mike"))
self.assertEqual(db.test.mike, db["test.mike"])
def delete_all_flag(self):
db_collections = Collection(self.db_data,'didi_flag')
db_collections.remove({})
def handle_insert_douyin(douyin_info):
task_id_collections = Collection(db, 'douyin_info')
task_id_collections.insert(douyin_info)
def insert_data(self,collection_name,item):
db_collections = Collection(self.db_data,collection_name)
db_collections.update({'shop_id':item['shop_id']},item,True)
def save_task(task):
"""
更新，不存在则插入
"""
task_collections = Collection(db, 'douyin_task')
task_collections.update({'share_id': task['share_id']}, task, True)
def find_data(self,collection_name,item):
db_collections = Collection(self.db_data,collection_name)
shop_id_result = db_collections.find_one({'shop_id':item})
return shop_id_result
def __init__(self, db, collection_Name):
self._collection = Collection(db, name=collection_Name)
self._logger = logging.getLogger(f'StockCollection_{collection_Name}')
def save_flag(self,collection_name,item):
db_collections = Collection(self.db_data,collection_name)
db_collections.update({'flag':item['flag']},item,True)
#{'username':'******','pwd':'gengxiaochan'},
{'username':'******','pwd':'ppjnelv2'},
#{'username':'******','pwd':'xkexbta8'},
#{'username':'******','pwd':'liuchenggang1988'},
]
agent = 'Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0'
headers = {
'User-Agent': agent
}
session = requests.session()
mongo_client = MongoClient(host = '172.16.40.12', port = 27017)
mongo_oreo = mongo_client.get_database('oreo')
mongo_cookies_col = Collection(mongo_oreo,'cookies')
#mongo_oreo = mongo_client.get_database(settings.MONGO_DB_NAME)
def getCookies(accounts,mongo_cookies_col):
cookies=[]
loginURL=""
for e in accounts:
account=e['username']
password=e['pwd']
cookie=login(account,password)
cookie['userID'] = account
cookies.append(cookie)
mongo_cookies_col.update({'userID':cookie['userID']} , cookie , True)
print cookie
return cookies
from pymongo.connection import Connection
from pymongo.database import Database
from pymongo.collection import Collection
import FreebaseApi
import AlchemyApiCategory
import nerd
import DbPediaApi
# constants and globals
MONGO_CON = '107.22.242.25:27017'
mongo_con = Connection(MONGO_CON)
MONGO_DB = 'queries'
mongo_db = Database(mongo_con, MONGO_DB)
MONGO_COL = 'hott_rends'
mongo_col = Collection(mongo_db, MONGO_COL)
# for specific data
LOG_FILE = '/home/penser/tmp/specifics.txt'
# for simple data
STAT_PAPER = '/home/penser/tmp/stats.txt'
# number of queries
LIMIT = 420
# threshold to cut off unusual types
THRESHOLD = 2
class HotTrendsAnalysisAl:
''' docs '''
def __init__(self):
self.freebaseApi = FreebaseApi.FreebaseApi()
def insert_item(self, item):
#表名
db_collection = Collection(self.db_data, 'wx_info_data')
db_collection.insert(item)
def create_collection(self,
name,
codec_options=None,
read_preference=None,
write_concern=None,
read_concern=None,
**kwargs):
"""Create a new :class:`~pymongo.collection.Collection` in this
database.
Normally collection creation is automatic. This method should
only be used to specify options on
creation. :class:`~pymongo.errors.CollectionInvalid` will be
raised if the collection already exists.
Options should be passed as keyword arguments to this method. Supported
options vary with MongoDB release. Some examples include:
- "size": desired initial size for the collection (in
bytes). For capped collections this size is the max
size of the collection.
- "capped": if True, this is a capped collection
- "max": maximum number of objects if capped (optional)
See the MongoDB documentation for a full list of supported options by
server version.
:Parameters:
- `name`: the name of the collection to create
- `codec_options` (optional): An instance of
:class:`~bson.codec_options.CodecOptions`. If ``None`` (the
default) the :attr:`codec_options` of this :class:`Database` is
used.
- `read_preference` (optional): The read preference to use. If
``None`` (the default) the :attr:`read_preference` of this
:class:`Database` is used.
- `write_concern` (optional): An instance of
:class:`~pymongo.write_concern.WriteConcern`. If ``None`` (the
default) the :attr:`write_concern` of this :class:`Database` is
used.
- `read_concern` (optional): An instance of
:class:`~pymongo.read_concern.ReadConcern`. If ``None`` (the
default) the :attr:`read_concern` of this :class:`Database` is
used.
- `collation` (optional): An instance of
:class:`~pymongo.collation.Collation`.
- `**kwargs` (optional): additional keyword arguments will
be passed as options for the create collection command
.. versionchanged:: 3.4
Added the collation option.
.. versionchanged:: 3.0
Added the codec_options, read_preference, and write_concern options.
.. versionchanged:: 2.2
Removed deprecated argument: options
"""
if name in self.collection_names():
raise CollectionInvalid("collection %s already exists" % name)
return Collection(self, name, True, codec_options, read_preference,
write_concern, read_concern, **kwargs)
def get_user_list():
task_id_collection = Collection(db, 'user_info')
return task_id_collection.find({})
def update_user(self, user):
collection = Collection(self.db, 'douyin_user')
collection.update_one({'share_id': user['share_id']}, {'$set': user})
