def __init__ (self, database, config):
# Init the pymongo object
Collection.__init__(self, database,
config.get('db_phrases_collection','phrases_collection'))
# Create logger
self.logger = logging.getLogger(__name__)
# Ensures the collection of documents is indexed by _id and subject
self.logger.info("Ensuring indexes exist...")
self.ensure_index('_id', unique = True, drop_dups = True)
self.ensure_index('lc_subject')
# Check phrase count
try:
count = self.count()
except TypeError:
# Count is not set, set it to a sane value
self.database.info.save({'_id': self.counter_name, 'count': self.find_highest_id()})
finally:
self.logger.info("Collection {} loaded - {}: {}".format(
self.full_name, self.counter_name, self.count()))
# Initialise variables
self.last = None
def get_dict(self, name, strict=False, **kwargs):
""" 自定义创建collection,kwargs可以附带相关参数:
- "size": desired initial size for the collection (in
bytes). must be less than or equal to 10000000000. For
capped collections this size is the max size of the
collection.
- "capped": if True, this is a capped collection
- "max": maximum number of objects if capped (optional)
"""
#使用collection_names判断是否存在，还是用异常处理呢？
## if name in self._db.collection_names():
## item = Collection(self._db, name)
## else:
## item = Collection(self._db, name, **kwargs)
try:
item = Collection(self._db, name, **kwargs)
except OperationFailure as e:
if strict:
raise
if e.args and 'exists' in e.args[0]:
pass
else:
log.log_except()
item = Collection(self._db, name)
return _MongoDict(self, item)
class UserSession(object):
"""
* `_id` (str) - Session id
* `user_id` (str)
* `session` (str) *optional* - Serialized session dict data
* `s_type` (int) - Session type
* `update_time` (datetime)
* `openid` (str) - 微信openid
* `session_key` (str) - 微信session_key
---
"""
COL_NAME = 'user_session'
p_col = Collection(mongo_db,
COL_NAME,
read_preference=ReadPreference.PRIMARY_PREFERRED)
s_col = Collection(mongo_db,
COL_NAME,
read_preference=ReadPreference.SECONDARY_PREFERRED)
class Field(object):
_id = '_id'
user_id = 'user_id'
session = 'session'
s_type = 's_type'
update_time = 'update_time'
openid = 'openid'
session_key = 'session_key'
p_col.create_index(Field.update_time, expireAfterSeconds=60 * 60 * 24 * 30)
class Image(object):
"""
* `_id`
* `url` (str)
* `width` (int) - 宽
* `height` (int) - 高
---
"""
COL_NAME = 'image'
p_col = Collection(mongo_db,
COL_NAME,
read_preference=ReadPreference.PRIMARY_PREFERRED)
s_col = Collection(mongo_db,
COL_NAME,
read_preference=ReadPreference.SECONDARY_PREFERRED)
class Field(object):
_id = '_id'
url = 'url'
width = 'width'
height = 'height'
p_col.create_index(Field.url, unique=True)
class Schedule(object):
"""
* `_id` (string)
* `title` (string) - 事件名
* `type` (string) - 事件分类
* `userId` (string) - 用户对应的id
* `startDate` (string) - 事件开始时间
* `endDate` (string) - 事件结束时间
* `startMonth` (string) - 事件开始月份
* `endMonth` (string) - 事件结束月份
* `create_time` (date) - 事件创建时间
"""
COL_NAME = 'schedule'
p_col = Collection(db,
COL_NAME,
read_preference=ReadPreference.PRIMARY_PREFERRED)
s_col = Collection(db,
COL_NAME,
read_preference=ReadPreference.SECONDARY_PREFERRED)
class Field(object):
_id = '_id'
title = 'title'
type = 'type'
userId = 'userId'
startDate = 'startDate'
endDate = 'endDate'
startMonth = 'startMonth'
endMonth = 'endMonth'
create_time = 'create_time'
p_col.create_index(Field.userId, unique=False, sparse=False)
p_col.create_index(Field.startMonth, unique=False, sparse=False)
p_col.create_index(Field.endMonth, unique=False, sparse=False)
def load(data, client, name):
''' Add the observed weather to the corrosponding instant document and load it to the remote database
:param data: the dictionary created from the api calls
:type data: dict
:param client: the pymongo client object
:type client: MongoClient
:param name: the database collection to be used
:type name: str
'''
global uri
# connect to the local database and search for the matching instant document and update it
client = MongoClient(host=host, port=port)
database = client.OWM
col = Collection(database, name)
filters = {'zipcode': data['zipcode'], 'instant': data['instant']}
updates = {
'$set': data['Weather']
} # use only the weather object from the current weather created from the API call
try:
# check to see if there is a document that fits the parameters. If there is, update it, if there isn't, upsert it
update = col.find_one_and_update(filters,
updates,
upsert=True,
return_document=ReturnDocument.AFTER)
# switch tho the remote database and load the new document to it
client = MongoClient(uri)
database = client.OWM
col = Collection(database, name)
loaded = col.insert_one(update)
except DuplicateKeyError:
return (f'DuplicateKeyError, could not insert data into {name}.')
def _pymongo(cls, create=False, slave_ok_setting=None):
from iu_mongo.connection import get_db
database = None
collection = None
database = get_db(cls._meta['db_name'])
collection_name = cls._meta['collection']
if database:
try:
collection = Collection(database,
collection_name,
create=create)
except pymongo.errors.OperationFailure:
collection = Collection(database, collection_name)
if not collection or not database:
raise ConnectionError('No mongo connections for collection %s' %
cls.__name__)
# override default configuration if possible
read_preference = SlaveOkSetting.TO_PYMONGO.get(slave_ok_setting, None)
default_write_concern = collection.write_concern
w = cls._meta.get("write_concern",
default_write_concern.document.get('w', None))
wtimeout = cls._meta.get(
"wtimeout", default_write_concern.document.get('wtimeout', None))
return collection.with_options(read_preference=read_preference,
write_concern=WriteConcern(
w=w, wtimeout=wtimeout))
class Work(object):
"""
* `_id`
* `user_id` (str) - 用户id
* `image_ids` (list of str) - 图片id列表
* `description` (str) - 描述
* `create_time` (datetime utcnow) - 创建时间
* `like_no` (int) - 点赞数
---
"""
COL_NAME = 'work'
p_col = Collection(mongo_db,
COL_NAME,
read_preference=ReadPreference.PRIMARY_PREFERRED)
s_col = Collection(mongo_db,
COL_NAME,
read_preference=ReadPreference.SECONDARY_PREFERRED)
class Field(object):
_id = '_id'
user_id = 'user_id'
image_ids = 'image_ids'
description = 'description'
create_time = 'create_time'
like_no = 'like_no'
p_col.create_index(Field.user_id, unique=False, sparse=False)
def __init__(self, database, main_collection_name, audit_collection_name,
revision_field, **kwargs):
# type: (Database, str, str, str, dict) -> None
"""
:param database: A PyMongo Database
:type database: Database
:param main_collection_name: Name of the main collection
:type main_collection_name: str
:param audit_collection_name: Name of the audit collection
:type audit_collection_name: str
:param revision_field: Name of the field to be added to any
document in the main collection to reference the current
revision in the audit collection
:type revision_field: str
:param kwargs: additional arguments
:type kwargs: dict
"""
super(Collection, self).__init__(database, main_collection_name,
**kwargs)
self.main_collection = PyMongoCollection(database,
main_collection_name,
**kwargs)
self.audit_collection = PyMongoCollection(database,
audit_collection_name,
**kwargs)
self.audit_collection.create_index('document_id',
unique=False,
name='document_id')
self.revision_field = revision_field
def GetChangesByName(self, strName):
dResult = dict()
try:
oChangesCollection = PymongoCollection(self.m_oDatabaseMongodb, "changes", False)
dResult = oChangesCollection.find({'name': strName}, {'active':1, 'passive':1, '_id':0})[0]
except Exception, exc:
strErrorMsg = '%s.%s Error: %s - Line: %s' % (self.__class__.__name__, str(exc), stack()[0][3], sys.exc_traceback.tb_lineno) # give a error message
Utilities.WriteErrorLog(strErrorMsg, self.m_oConfig)
def test_get_collection(self):
eq_(
backend.get_collection('logs'),
Collection(Database(Connection(u'127.0.0.1', 27017), u'amon_test'),
u'logs'))
eq_(
backend.get_collection('exceptions'),
Collection(Database(Connection(u'127.0.0.1', 27017), u'amon_test'),
u'exceptions'))
def handle_mongodb_save_data(item):
# 抖音
if item['item_type'] == 'douyin_info':
douyin_data_collection = Collection(db, 'douyin_info')
douyin_data_collection.insert(item)
# 抖音
if item['item_type'] == 'douyin_video':
douyin_data_collection = Collection(db, 'douyin_video')
douyin_data_collection.insert(item)
def test_join():
con = pymongo.Connection('mongodb://dev.zl.efun.com/test')
db = getattr(con, 'jhy_new')
cl = Collection(db, u'setting.logon.telcom-广东区-八月十五.players')
print cl.count()
cl = Collection(db, u'setting.logon.telcom-广东区-八月十五.temp_world_boss')
r = cl.inline_map_reduce(sum_map, sum_reduce, limit=10)
print len(r)
print r
def ops(utcnow, oid, filename, data):
oid_cache.clear()
grid_cache.clear()
client = MongoClient()
db = Database(client, 'test')
files = Collection(db, 'fs.files')
chunks = Collection(db, 'fs.chunks')
files.drop()
chunks.drop()
fs = GridFS(db)
fs.put(data, _id=oid, filename=filename, upload_date=utcnow)
return GridFSOperations(client.host)
def SwitchChangesActive(self, strName, strActive, strPassive):
bResult = False;
try:
oChangesCollection = PymongoCollection(self.m_oDatabaseMongodb, "changes", False)
if oChangesCollection.find({'name': strName}).count() == 1:
oChangesCollection.update({ 'name': strName },
{ '$set': {'active': strPassive, 'passive': strActive} })
bResult = True
except Exception, exc:
strErrorMsg = '%s.%s Error: %s - Line: %s' % (self.__class__.__name__, str(exc), stack()[0][3], sys.exc_traceback.tb_lineno) # give a error message
Utilities.WriteErrorLog(strErrorMsg, self.m_oConfig)
def handle_save_data(item):
#抖音
if item['item_type'] == 'douyin_item':
print('3333333333333333333333')
douyin_data_collection = Collection(db, 'douyin')
douyin_data_collection.insert(item)
#快手
elif item['item_type'] == 'kuaishou_item':
kuaishou_data_collection = Collection(db, 'kuaishou')
kuaishou_data_collection.insert(item)
#今日头条
elif item['item_type'] == 'jinritoutiao_item':
jinritoutiao_data_collection = Collection(db, 'jintitoutiao')
jinritoutiao_data_collection.insert(item)
class User(UserMixin):
"""
* `_id`
* `nick_name` (str) - 用户名
* `avatar_url` (str) - 头像
* `description` (str) - 个人介绍
* `gender` (int) - 性别
* `0` - 未知
* `1` - 男
* `2` - 女
* `openid` (str) - 微信openid(该数据不能返回前端，用作用户的唯一标识)
* `session_key` (str) - 微信session key(该数据不能返回前端)
* `create_time` (datetime utcnow) - 创建时间
---
"""
COL_NAME = 'user'
p_col = Collection(mongo_db,
COL_NAME,
read_preference=ReadPreference.PRIMARY_PREFERRED)
s_col = Collection(mongo_db,
COL_NAME,
read_preference=ReadPreference.SECONDARY_PREFERRED)
class Field(object):
_id = '_id'
nick_name = 'nick_name'
avatar_url = 'avatar_url'
description = 'description'
gender = 'gender'
openid = 'openid'
session_key = 'session_key'
create_time = 'create_time'
class Gender(object):
""" 性别"""
unknown = 0
male = 1
female = 2
def __init__(self, **kwargs):
UserMixin.__init__(self)
for k, v in kwargs.items():
self.__setattr__(k, v)
def get_id(self):
return self._id
p_col.create_index(Field.openid, unique=True, sparse=True)
def send_linkinfo_back2_taskqueue(self,
linkinfo,
topic,
status=None,
url_resp=None):
#判断是否用跳转后的链接更新linkinfo里的url。在百度抓取过程中链接会重定向到真实链接，并对真实链接进行调度下载，真实链接下载失败回队列重试时应该用真是队列而非原始百度加密链接，避免重复跳转破解工作
linkinfo = self.update_schedule_time(linkinfo['spider_name'],
linkinfo['url'],
org_item=linkinfo)
if status is not None and url_resp is not None:
domain_org = self.get_mdomain_from_url(linkinfo.get('url', ''))
domain_jump = self.get_mdomain_from_url(url_resp)
logging.debug("尝试更新linkinfo-url。domain_org:%s domain_jump:%s" %
(domain_org, domain_jump))
if domain_org == 'baidu.com' and domain_org != domain_jump:
logging.info("破解百度链接，重入队列时更新url。org:[%s] new:[%s]" %
(linkinfo.get('url', ''), url_resp))
linkinfo['url'] = url_resp
if linkinfo == None:
logging.info('发送回消息队列时，因linkinfo为None，忽略')
return
logging.debug('发送回消息队列. now scheduler-cnt[%d] failed-cnt[%d] url:%s' %
(linkinfo.get('schedule_cnt', 0), linkinfo['failed_cnt'],
linkinfo['url']))
#调度次数或下载次数太多 则认为可能是死链 不在进行调度了
if int(linkinfo.get('schedule_cnt', 0)) > self.max_schedule_cnt or int(
linkinfo['failed_cnt']) > self.max_failed_cnt:
logging.warning('调度次数或下载次数太多 url:%s' % linkinfo['url'])
return
task_col = Collection(self.mongo_task, topic)
task_col.insert(linkinfo)
def handle_init_task():
task_id_collection = Collection(db, "task_id")
with open("douyin_zhanghao.txt", 'r') as f_share:
for f_share_task in f_share.readlines():
init_task = {}
init_task['share_id'] = f_share_task.replace("\n", '')
task_id_collection.insert(init_task)
def find(self, *args, **kwargs):
#if it's a single argument and a string, convert it to an objectid
if len(args) == 1 and isinstance(args[0], basestring):
args = ({u'_id': ObjectId(args[0])},)
result = Collection.find(self, *args, **kwargs)
return CSCursor(result)
def sql_create_model(self, model, *args, **kwargs):
"""Creates the collection for model. Mostly used for capped collections.
:param model: The model that should be created.
:param \*args: Extra args not used in this engine.
:param \*\*kwargs: Extra kwargs not used in this engine.
Example
>>> class TestFieldModel(Task):
...
... class MongoMeta:
... capped = True
... collection_max = 100000
... collection_size = 10
"""
opts = model._meta
kwargs = {}
kwargs["capped"] = getattr(opts, "capped", False)
if hasattr(opts, "collection_max") and opts.collection_max:
kwargs["max"] = opts.collection_max
if hasattr(opts, "collection_size") and opts.collection_size:
kwargs["size"] = opts.collection_size
col = Collection(self.connection.db_connection, model._meta.db_table,
**kwargs)
return [], {}
def test_mongodb_connection(db):
try:
test_collection = Collection(db, "auth_check", create=True)
test_collection.drop()
return True
except:
return False
def handle_init_task():
task_id_collection = Collection(db, 'task_id')
with open('douyin_hot_id.txt', 'r') as f_share:
for f_share_task in f_share.readlines():
init_task = {}
init_task['share_id'] = f_share_task.replace('\n', '')
task_id_collection.insert_one(init_task)
def _update(cls):
"""Runs validator & index update commands on database"""
db = cls.db()
name = cls.name()
validator = cls._validator()
indexes = cls._indexes()
collection = db[name]
# Create or update validator
try:
collection = Collection(db, name, validator=validator)
except OperationFailure:
db.command({
"collMod": name,
"validator": validator
})
# Update Indexes
i_names = [model.document['name'] for model in indexes]
existing = list(collection.index_information().keys())
for name in existing:
if name != '_id_' and name not in i_names:
collection.drop_index(name)
to_create = [model for model in indexes if model.document['name'] not in existing]
if len(to_create) > 0:
collection.create_indexes(indexes)
# Set as updated
cls._on_update()
def _get_or_create_collection(self, collection_name):
"""Returns the collection with the required indexes.
Args:
collection_name (pymongo.collection.Collection)
Returns:
pymongo.collection.Collection
"""
collection = self._client_master[
self._database_master['database']][collection_name]
if collection_name in self._indexed:
return collection
try:
indexes = collection.index_information()
# When the table does not exist it needs to be created.
except OperationFailure:
collection = Collection(
database=self._client_master[self._database_master],
name=collection_name,
create=True)
indexes = collection.index_information()
for index_name in indexes:
if str(index_name).startswith('time'):
self._indexed.append(collection_name)
return collection
# If index was found create one and update indexed list.
collection.create_index([('time', ASCENDING)], unique=True)
self._indexed.append(collection_name)
return collection
def handle_init_task():
task_id_collection = Collection(db, 'task_id')
with open('file/share_task.txt', 'r') as f_task:
for task_info in f_task.readlines():
task = {}
task['share_link'] = task_info.replace('\n', '')
task_id_collection.insert(task)
def dbncol(client, collection, database=database):
''' Make a connection to the database and collection given in the arguments.
:param client: a MongoClient instance
:type client: pymongo.MongoClient
:param database: the name of the database to be used. It must be a database
name present at the client
:type database: str
:param collection: the database collection to be used. It must be a
collection name present in the database
:type collection: str
:return col: the collection to be used
:type: pymongo.collection.Collection
'''
try:
db = Database(client, database)
print(f'dbncol created db without AttributeError using {client}.')
except AttributeError:
print(
f'dbncol caught AttributeError while trying to connect {client}.')
from config import uri
print(
'trying to connect using MongoClient rather than my own Client()')
client = MongoClient(uri)
db = Database(client, database)
print('did it without issue.')
col = Collection(db, collection)
return col
def task_zhishu360(self):
spider_name = 'zhishu360'
mongo_task_col = Collection(self.mongo_task_12, spider_name)
url_partten = {
1:'http://index.haosou.com/index/overviewJson?area=全国&q=query_partten',
2:'http://index.haosou.com/index/soIndexJson?area=全国&q=query_partten',
3:'http://index.haosou.com/index/soMediaJson?q=query_partten',
4:'http://index.haosou.com/index/indexqueryhour?q=query_partten&t=30',
5:'http://index.haosou.com/index/radarJson?t=30&q=query_partten',
6:'http://index.haosou.com/index/indexquerygraph?t=30&area=全国&q=query_partten',
}
i = 0
cursor = self.mongo_oreo_35.video_maoyan_mobile.find()
for item in cursor:
for key in url_partten.keys():
query = item['name']
url = url_partten[key].replace('query_partten',query)
print url
q_flag = key
linkinfo = {'url':url, 'q_flag':q_flag , 'query':query, 'from_url':url, 'spider_name':spider_name, 'item_type':'item_url', 'status':0, 'schedule_cnt':0, 'failed_cnt':0, 'in_time': math.floor(time.time()), 'update_time':math.floor(time.time())}
#print linkinfo
mongo_task_col.insert(linkinfo)
del linkinfo['_id']
linkinfo_str = json.dumps(linkinfo, ensure_ascii = False, encoding = 'utf-8')
self.linkbase.set(url, linkinfo_str)
i += 1
print ('%s send kafka massage number %d by spider name %s' % (datetime.datetime.now().strftime("%Y-%m-%d %H:%m:%S %A %B"),i,spider_name))
print 'happy work'
def get_from_url(self, item):
db_collections = Collection(self.db_data, 'mafengwo_article')
result = db_collections.find_one({'from_url': item})
if result:
return True
else:
return False
def __init__(self):
"""
Custom MongoDBPlugin
"""
super().__init__()
if USERNAME and PASSWORD:
mongoURI = "mongodb://{username}:{password}@{host}:{port}/{db}".format(
username=USERNAME,
password=PASSWORD,
host=HOST,
port=PORT,
db=DATABASE,
)
else:
mongoURI = "mongodb://{host}:{port}/{db}".format(host=HOST,
port=PORT,
db=DATABASE)
connect = pymongo.MongoClient(mongoURI, connect=False)
self.connection = connect[DATABASE]
self.user_store = connect[DATABASE]["mgmt_users"]
for each in self.connection.list_collection_names():
setattr(
self,
"store_{}".format(each),
Collection(database=self.connection, name=each),
)
def handler_init_task():
task_id_collection = Collection(db, 'task_id')
with open('douyin_id.txt') as f:
for id in f.readlines():
init_task = {}
init_task['share_id'] = id.strip()
task_id_collection.insert_one(init_task)
def get_record_field_by_field_value(gc_collection: collection.Collection, res_field: str, fields: List,
fields_values: List, is_field_array: List):
"""
Returns the given res_field of all the records in the given collection where the field with
the given name is one of the given field_values.
:param gc_collection: The collection to be queried
:param res_field: The desired field to be returned
:param fields: List of field to match by.
:param fields_values: List of Lists of desired values dor each field.
:param is_field_array: List of Booleans indicating whether the field is an array or not.
:return: List of ids in which the field is one of the given values
"""
query = {}
for field_num in range(len(fields)):
# ToDo: replace the is_field_array parameter with type checking of passed field_values
if is_field_array[field_num]:
query[fields[field_num]] = {"$in": fields_values[field_num]}
else:
query[fields[field_num]] = fields_values[field_num]
# query["is_on"] = True # uncomment if querying orchestration_vm
return gc_collection.distinct(res_field, query)
def __init__(self,
database,
name,
codec_options=None,
read_preference=None,
write_concern=None,
read_concern=None,
_delegate=None):
db_class = create_class_with_framework(AgnosticDatabase,
self._framework,
self.__module__)
if not isinstance(database, db_class):
raise TypeError("First argument to MotorCollection must be "
"MotorDatabase, not %r" % database)
delegate = _delegate or Collection(database.delegate,
name,
codec_options=codec_options,
read_preference=read_preference,
write_concern=write_concern,
read_concern=read_concern)
super(self.__class__, self).__init__(delegate)
self.database = database
def create_collection(self, name, codec_options=None,
read_preference=None, write_concern=None,
read_concern=None, **kwargs):
"""Create a new :class:`~pymongo.collection.Collection` in this
database.
Normally collection creation is automatic. This method should
only be used to specify options on
creation. :class:`~pymongo.errors.CollectionInvalid` will be
raised if the collection already exists.
Options should be passed as keyword arguments to this method. Supported
options vary with MongoDB release. Some examples include:
- "size": desired initial size for the collection (in
bytes). For capped collections this size is the max
size of the collection.
- "capped": if True, this is a capped collection
- "max": maximum number of objects if capped (optional)
See the MongoDB documentation for a full list of supported options by
server version.
:Parameters:
- `name`: the name of the collection to create
- `codec_options` (optional): An instance of
:class:`~bson.codec_options.CodecOptions`. If ``None`` (the
default) the :attr:`codec_options` of this :class:`Database` is
used.
- `read_preference` (optional): The read preference to use. If
``None`` (the default) the :attr:`read_preference` of this
:class:`Database` is used.
- `write_concern` (optional): An instance of
:class:`~pymongo.write_concern.WriteConcern`. If ``None`` (the
default) the :attr:`write_concern` of this :class:`Database` is
used.
- `read_concern` (optional): An instance of
:class:`~pymongo.read_concern.ReadConcern`. If ``None`` (the
default) the :attr:`read_concern` of this :class:`Database` is
used.
- `collation` (optional): An instance of
:class:`~pymongo.collation.Collation`.
- `**kwargs` (optional): additional keyword arguments will
be passed as options for the create collection command
.. versionchanged:: 3.4
Added the collation option.
.. versionchanged:: 3.0
Added the codec_options, read_preference, and write_concern options.
.. versionchanged:: 2.2
Removed deprecated argument: options
"""
if name in self.collection_names():
raise CollectionInvalid("collection %s already exists" % name)
return Collection(self, name, True, codec_options,
read_preference, write_concern,
read_concern, **kwargs)
def find_one(self, *args, **kwargs):
#if it's a single argument and a string, convert it to an objectid
if len(args) == 1 and isinstance(args[0], basestring):
args = ({u'_id': ObjectId(args[0])},)
result = Collection.find_one(self, *args, **kwargs)
if not result is None:
return CSObject(getattr(winter.objects, self.name)(result), self.name)
return result
def find_one(self, spec_or_id=None, *args, **kwargs):
"""Like :meth:`~pymongo.collection.Collection.find_one` but sets the
`as_class` attribute to the class of the calling model and spec_or_id
can optionally be a string.
"""
if isinstance(spec_or_id, basestring):
spec_or_id = ObjectId(spec_or_id)
kwargs['as_class'] = self.as_class
return BaseCollection.find_one(self, *args, **kwargs)
def insert(self, doc_or_docs, *args, **kwargs):
#tag incoming objects with the winter head revision if they don't
#already have a rev number
if isinstance(doc_or_docs, list):
for doc in doc_or_docs:
getattr(winter.objects, self.name).tag(doc)
else:
getattr(winter.objects, self.name).tag(doc_or_docs)
return Collection.insert(self, doc_or_docs, *args, **kwargs)
def UpdatePhysicalServerInterface(self):
try:
dServerInfo = dict()
# Connect to CMDBv2
if self.IsConnectedToCMDBv2() is False:
return 0
oServerCollection = PymongoCollection(self.m_oDatabaseMongodb, CLT_SERVER, False)
if oServerCollection is None:
strErrorMsg = "Cannot connect to %s " % self.m_oConfig.CMDBv2Host
Utilities.WriteErrorLog(strErrorMsg, self.m_oConfig)
return 0
dServerInfo = self.GetPhysicalServerInterfaceFromMDR()
if len(dServerInfo) > 0:
for strServerKey, dInterfaceInfo in dServerInfo.items():
oDataResult = oServerCollection.find({"code": strServerKey})
if Utilities.CheckExistence(oDataResult) is not False:
oServerCollection.update(
{"code": strServerKey},
{
"$set": {
"private_interface": dInterfaceInfo["private"],
"public_interface": dInterfaceInfo["public"],
}
},
)
except Exception, exc:
strErrorMsg = "%s.%s Error: %s - Line: %s" % (
self.__class__.__name__,
str(exc),
stack()[0][3],
sys.exc_traceback.tb_lineno,
) # give a error message
Utilities.WriteErrorLog(strErrorMsg, self.m_oConfig)
def test_with_many_logs_and_failed_one_with_exceptions_2(self):
now = datetime.today()
self._save_dummy(now, ERROR, None)
onesecond = timedelta(0, 1)
for i in range(0, 20):
for level in [DEBUG, INFO, AUDIT]:
self._save_dummy(now, level, str(i))
now = now + onesecond
self.mox.StubOutWithMock(Collection, 'save')
Collection.save(mox.IgnoreArg()).AndReturn(None)
Collection.save(mox.IgnoreArg()).AndRaise(Exception())
Collection.save(mox.IgnoreArg()).AndReturn(None)
Collection.save(mox.IgnoreArg()).AndReturn(None)
Collection.save(mox.IgnoreArg()).AndRaise(Exception())
Collection.save(mox.IgnoreArg()).AndReturn(None)
Collection.save(mox.IgnoreArg()).AndReturn(None)
Collection.save(mox.IgnoreArg()).AndReturn(None)
self.mox.ReplayAll()
archiver.main()
def __init__(self, *args, **kwargs):
self.as_class = kwargs.pop('as_class')
BaseCollection.__init__(self, *args, **kwargs)
def update(self, *args, **kwargs):
return _Collection.update(self, *args, **kwargs)
def remove(self, *args, **kwargs):
return _Collection.remove(self, *args, **kwargs)
def find_one(self, *args, **kwargs):
return _Collection.find(self, *args, **kwargs)
def __init__(self, database, collection_name, type_conversion):
self._type_conversion = type_conversion
Collection.__init__(self, database, collection_name)
