def get_count(coll,event):
count1=0
count2=0
print 'num_days_1', num_days_1
for i in xrange(num_days_1):
print i
y1 = main_y1 + datetime.timedelta(days=i)
t1 = main_y1 + datetime.timedelta(days=i+1)
print "dates are:"
print y1,t1
ts_y1 = int(time.mktime(y1.timetuple())*1000)
ts_t1 = int(time.mktime(t1.timetuple())*1000)
yutcd1 = datetime.datetime.utcfromtimestamp(ts_y1/1000)
tutcd1 = datetime.datetime.utcfromtimestamp(ts_t1/1000)
yutcoid1 = ObjectId.from_datetime(yutcd1)
tutcoid1 = ObjectId.from_datetime(tutcd1)
print yutcoid1, tutcoid1
count1 += len(coll.find({'_id': {'$gte': yutcoid1, '$lte': tutcoid1}, 'service': event['service'], 'device':event['device'], 'action':event['action']}).distinct('uid'))
print 'num_days_2', num_days_2
for i in xrange(num_days_2):
print i
y2 = main_y2 + datetime.timedelta(days=i)
t2 = main_y2 + datetime.timedelta(days=i+1)
print "dates are:"
print y2,t2
ts_y2 = int(time.mktime(y2.timetuple())*1000)
ts_t2 = int(time.mktime(t2.timetuple())*1000)
yutcd2 = datetime.datetime.utcfromtimestamp(ts_y2/1000)
tutcd2 = datetime.datetime.utcfromtimestamp(ts_t2/1000)
yutcoid2 = ObjectId.from_datetime(yutcd2)
tutcoid2 = ObjectId.from_datetime(tutcd2)
print yutcoid2, tutcoid2
count2 += len(coll.find({'_id': {'$gte': yutcoid2, '$lte': tutcoid2}, 'service': event['service'], 'device':event['device'], 'action':event['action']}).distinct('uid'))
return (count1,count2)
def get(self, itemId):
ffrom, to = self.get_argument('from', None), self.get_argument('to', None)
filt = {'item_id': ObjectId(itemId), '_id': {
'$gt': ObjectId.from_datetime(parse(ffrom) if ffrom else (self.now - datetime.timedelta(1))),
'$lte': ObjectId.from_datetime(parse(to) if to else self.now)
}}
self.write(dumps(alfred.db.values.find(filt)))
def datetime_to_objectId(input="", year=0, month=1, day=1, hour=0):
'''
Input -- input <datetime.datetime> <basestring>
year <int>
month <int>
day <int>
hour <int>
Output -- <ObjectId> or None or <basestring> if bson.objectid module does not exist
reference: http://api.mongodb.org/python/1.5.2/api/pymongo/objectid.html
'''
response_objectid = None #default to None to enable 'if' check
if "bson" not in IMPORTFAILS:
if input:
if isinstance(input, basestring): #need to parse
response_objectid = ObjectId.from_datetime(_parse_input(input))
else: #should only be a string or datetime, all else will fail here
response_objectid = ObjectId.from_datetime(input)
elif year:
response_objectid = ObjectId.from_datetime(datetime.datetime(int(year), int(month), int(day), int(hour)))
else:
pass
#TODO: build object ID without objectID library
return response_objectid # should be of ObjectID or None
def get_forecast(host, port, source, db_name, username, password,
collection_name, date):
# forecasts are always pushed after time that whole suite starts running
# so implementation below is correct, and will result in us
# being able to capture forecast and archive data used by operator in past
# when in debug mode in present
date_lb = date - pd.Timedelta(refresh_rate)
date_ub = date + pd.Timedelta(refresh_rate)
with pymongo.MongoClient(host=host, port=port) as conn:
conn[db_name].authenticate(username, password, source=source)
collection = conn[db_name][collection_name]
for data in collection.find({
"_id":
{
"$gte": ObjectId.from_datetime(
date_lb),
"$lte": ObjectId.from_datetime(
date_ub)
}
}).sort(
"_id", pymongo.ASCENDING):
reading = data['readings']
wfore = pd.DataFrame(reading)
if len(wfore) == 0:
raise ValueError("An appropriate forecast for the passed"
"date does not exist in the database. Please"
"pass another date, and check to make sure"
"the database is functioning correctly.")
else:
wfore.set_index('time', inplace=True)
wfore = wfore.sort_index()
wfore = wfore.tz_localize('UTC')
return wfore
def add_mongo_id(obj, objidkeys = ['_id', u'_id',]):
if isinstance(obj, str) or isinstance(obj, unicode):
try:
if len(obj) == 24:
obj = ObjectId(obj)
except:
pass
d = None
try:
d = datetime.datetime.strptime(obj, "%Y-%m-%d %H:%M:%S.%F")
except:
try:
d = datetime.datetime.strptime(obj, "%Y-%m-%d %H:%M:%S")
except:
try:
d = datetime.datetime.strptime(obj, "%Y-%m-%d")
except:
d = None
if d:
obj = d
return obj
elif isinstance(obj, dict):
for k in obj.keys():
if k in objidkeys and obj[k] is None:
obj[k] = ObjectId()
obj[k] = add_mongo_id(obj[k])
elif isinstance(obj, list):
for i in obj:
obj[obj.index(i)] = add_mongo_id(i)
return obj
def test_generation_time(self):
d1 = datetime.datetime.utcnow()
d2 = ObjectId().generation_time
self.assertEqual(utc, d2.tzinfo)
d2 = d2.replace(tzinfo=None)
self.assertTrue(d2 - d1 < datetime.timedelta(seconds=2))
def delete(self, request, *args, **kw): # deletes table
db_id = request.data.get("db_id", None)
table_id = request.data.get("table_id", None)
access_token = request.META.get("HTTP_ACCESS_TOKEN", None)
result = {}
result[RESPONSE_STATUS] = RESPONSE_STATUS_ERROR
if db_id and ObjectId.is_valid(db_id) and access_token:
mainDB = getDatabase()
token = mainDB.tokens.find_one({"token": access_token}) if mainDB else None
if token and access_token_valid(token):
if mainDB:
result[RESPONSE_STATUS] = RESPONSE_STATUS_SUCCESS
if table_id:
if ObjectId.is_valid(table_id):
result[DELETED_TABLE_COUNT] = mainDB.tables.delete_one(
{"_id": ObjectId(table_id), "author": token["author"], "database_id": ObjectId(db_id)}
).deleted_count
else:
result[RESPONSE_STATUS] = RESPONSE_STATUS_ERROR
result[RESPONSE_STATUS_ERROR_MSG] = TABLE_ID_INVALID
else:
result[DELETED_TABLE_COUNT] = mainDB.tables.delete_many(
{"author": token["author"], "database_id": ObjectId(db_id)}
).deleted_count
else:
result[RESPONSE_STATUS_ERROR_MSG] = MASTER_DB_UNINITIALIZED
else:
result[RESPONSE_STATUS_ERROR_MSG] = ACCESS_TOKEN_INVALID
else:
result[RESPONSE_STATUS_ERROR_MSG] = TOKEN_MISSING if not access_token else DATABASE_ID_INVALID
response = Response(result, status=status.HTTP_200_OK)
return response
def json(self, argv):
self.manager = MovieManager(self.site, self.module)
action = self.site.get_argument('action', None)
if action == 'setting':
post_id = self.site.get_argument('post', None)
if post_id:
if type(post_id) == str and re.match(r'[a-z0-9]{24}', post_id):
post_id = ObjectId(post_id)
mv_setting = self.site.get_argument('setting', None)
if mv_setting == "complete":
mv_setting = "mvc"
elif mv_setting == "remove":
mv_setting = "mvd"
result = yield self.manager.set_user_view_post(post_id, mv_setting)
if result:
return '{"error":0,"success":1}'
else:
post_id = self.site.get_argument('post', None)
if not post_id:
posts_view = yield self.manager.get_user_view_post()
if posts_view:
post_id = []
for p in posts_view:
if 'post_id' in p:
post_id.append(p['post_id'])
###
posts = yield self.post_json(post_id)
posts_argv = []
for post in posts:
posts_argv.append(self.post_argv(post))
return {"post": posts_argv}
return '{"error":1,"success":0}'
def copy(source_params, dest_params, start_date, end_date):
source_db = connect_mongodb(source_params)
source_tables = get_table_names(source_params)
dest_db = connect_mongodb(dest_params)
dest_tables = get_table_names(dest_params)
# records = []
# for record in source_db[source_tables['topics_table']].find():
# records.append(record)
# print("total records {}".format(len(records)))
# dest_db[dest_tables['topics_table']].insert_many(
# records)
#
# records = []
# for record in source_db[source_tables['meta_table']].find():
# records.append(record)
# print("total records {}".format(len(records)))
# dest_db[dest_tables['meta_table']].insert_many(
# records)
# This is probably the most inefficient way of doing a copying a subset
# of a
# collection to another database but this is the one that requires the
# minimum access
# Wish this feature request is closed soon
# https://jira.mongodb.org/browse/SERVER-13201
# Aggregation is the fastest way to get a subset of data from a collection,
# next would be map reduce. map reduce can write output to another db
# but it would only generate doc of schema
# id:<object id>, value:<search/mapreduce result>
dest_db[dest_tables['data_table']].create_index(
[('topic_id', pymongo.DESCENDING),
('ts', pymongo.DESCENDING)],
unique=True, background=False)
records = []
i = 0
print ("start obj:{}".format(ObjectId.from_datetime(start_date)))
print ("end obj:{}".format(ObjectId.from_datetime(end_date)))
cursor = source_db[source_tables['data_table']].find(
{'$and':
[{'_id': {'$gte': ObjectId.from_datetime(start_date)}},
{'_id': {'$lte': ObjectId.from_datetime(end_date)}}]})
print ("Record count from cursor {}".format(cursor.count()))
for record in cursor:
i += 1
records.append(
ReplaceOne(
{'ts':record['ts'], 'topic_id':record['topic_id']},
{'ts': record['ts'], 'topic_id': record['topic_id'],
'value': record['value']},
upsert=True))
if i == 2000:
print("total records {}".format(len(records)))
dest_db[dest_tables['data_table']].bulk_write(records)
i =0
records = []
def get_date_query(start_date, end_date):
if start_date is not None and end_date is not None:
oid_start = ObjectId.from_datetime(start_date)
oid_stop = ObjectId.from_datetime(end_date)
return {"_id": {"$gte": oid_start, "$lt": oid_stop}}
else:
return None
def validate(self, data, datatype="String", required=False, fieldname="no_fieldname"):
try:
if data == None or data == "null":
if required:
raise Exception()
else:
return [True, None, datatype, fieldname, required]
#String
if datatype=="String":
pass
#Integer
elif datatype=="Integer":
data = int(data)
#Float/Date
elif datatype=="Float":
data = self.isFloat(data, required=True)
elif datatype=="Date":
data = float(data)
#MongoID
elif datatype=="MongoID":
data = ObjectId(data)
elif datatype=="MongoIDs":
mongoIDs = []
for eachId in data.split(","):
mongoIDs.append(ObjectId(eachId))
data = mongoIDs
#Bool
elif datatype=="Bool":
if str(data).lower() in ["yes", "y", "true", "t", "1","True"]:
data = True
elif str(data).lower() in ["no", "n", "false", "f", "0", "0.0","False"]:
data = False
else:
raise Exception()
#Email
elif datatype=="Email":
if not re.match(r"[^@]+@[^@]+\.[^@]+", data):
raise Exception()
#Something went wrong
except Exception:
return [False, None, datatype, fieldname, required]
#All Ok
return [True, data, datatype, fieldname, required]
pass
def idslice(col, start_seconds, end_seconds=0):
start_delta = timedelta(seconds=start_seconds)
start_objid = ObjectId.from_datetime(utc_now() - start_delta)
end_delta = timedelta(seconds=end_seconds)
end_objid = ObjectId.from_datetime(utc_now() - end_delta)
for obj in col.find(
{'_id': {'$gte': start_objid, '$lt': end_objid}}).sort('_id'):
yield obj
def getDateQuery(start_date,end_date):
if not start_date is None and not end_date is None:
oid_start = ObjectId.from_datetime(start_date)
oid_stop = ObjectId.from_datetime(end_date)
return { "_id": { "$gte": oid_start, "$lt": oid_stop } }
else:
return None
def _updateStartingPanelID(self, comicID, firstPanelID):
"""This is a private method.
Add the firstPanelID to the given comic as the starting panel,
return true if successful, false otherwise"""
if (not ObjectId.is_valid(comicID)) or (not ObjectId.is_valid(firstPanelID)):
return False
if (self.comicsCollection.find_and_modify({self.idField: ObjectId(comicID)},
{"$set" : {self.firstPanelIDField : firstPanelID} }) != None):
return True
return False # update unsuccessful
def test_from_datetime(self):
d = datetime.datetime.utcnow()
d = d - datetime.timedelta(microseconds=d.microsecond)
oid = ObjectId.from_datetime(d)
self.assertEqual(d, oid.generation_time.replace(tzinfo=None))
self.assertEqual("0" * 16, str(oid)[8:])
aware = datetime.datetime(1993, 4, 4, 2, tzinfo=FixedOffset(555, "SomeZone"))
as_utc = (aware - aware.utcoffset()).replace(tzinfo=utc)
oid = ObjectId.from_datetime(aware)
self.assertEqual(as_utc, oid.generation_time)
def objectid (day):
start_date = str(day)
date_1 = datetime.datetime.strptime(start_date,"%Y-%m-%d %H:%M:%S")
end_date = date_1 + datetime.timedelta(hours=1)
s=datetime.datetime(date_1.year,date_1.month,date_1.day,date_1.hour)
d=datetime.datetime(end_date.year,end_date.month,end_date.day,end_date.hour)
objmin =ObjectId.from_datetime(s)
objmax =ObjectId.from_datetime(d)
#print str(date_1),str(end_date),objmin,objmax
return objmin,objmax,end_date
def update_mirroring(frame_id, mirroring_id):
"""
Set a frame to mirror another by id
"""
fid = frame_id if not ObjectId.is_valid(frame_id) else ObjectId(frame_id)
mid = frame_id if not ObjectId.is_valid(mirroring_id) else ObjectId(mirroring_id)
doc = {"mirroring": mid}
resp = Frames.collection.find_one_and_update({"_id": fid}, {"$set": doc}, return_document=ReturnDocument.AFTER)
return _unify_ids(resp)
def test_is_valid(self):
self.assertFalse(ObjectId.is_valid(None))
self.assertFalse(ObjectId.is_valid(4))
self.assertFalse(ObjectId.is_valid(175.0))
self.assertFalse(ObjectId.is_valid({"test": 4}))
self.assertFalse(ObjectId.is_valid(["something"]))
self.assertFalse(ObjectId.is_valid(""))
self.assertFalse(ObjectId.is_valid("12345678901"))
self.assertFalse(ObjectId.is_valid("1234567890123"))
self.assertTrue(ObjectId.is_valid(b("123456789012")))
self.assertTrue(ObjectId.is_valid("123456789012123456789012"))
def update_artifact_by_id(self,collection,artifact_id,field=None,value=None,document=None):
if document:
#Careful, this replaces the document entirely
if ObjectId.is_valid(artifact_id):
self.c[collection].update({"_id":ObjectId(artifact_id)}, document)
else:
self.c[collection].update({"_id":artifact_id}, document)
elif field and value:
if ObjectId.is_valid(artifact_id):
self.c[collection].update({"_id":ObjectId(artifact_id)}, {"$set" : {field:value}})
else:
self.c[collection].update({"_id":artifact_id}, {"$set" : {field:value}})
else:
raise WrongParametersExecption
def test_from_datetime(self):
if "PyPy 1.8.0" in sys.version:
# See https://bugs.pypy.org/issue1092
raise SkipTest("datetime.timedelta is broken in pypy 1.8.0")
d = datetime.datetime.utcnow()
d = d - datetime.timedelta(microseconds=d.microsecond)
oid = ObjectId.from_datetime(d)
self.assertEqual(d, oid.generation_time.replace(tzinfo=None))
self.assertEqual("0" * 16, str(oid)[8:])
aware = datetime.datetime(1993, 4, 4, 2, tzinfo=FixedOffset(555, "SomeZone"))
as_utc = (aware - aware.utcoffset()).replace(tzinfo=utc)
oid = ObjectId.from_datetime(aware)
self.assertEqual(as_utc, oid.generation_time)
def generate_date_based_blackboard(db, blackboard_name):
blackboard_name = blackboard_name.upper()
# Generate tags collection
tags_coll = db[blackboard_name + TagManager.tag_suffix]
tag_ids = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
tgs = [{'_id': x, 'Nm': 'Tag_{}'.format(x), 'Ctrl': 0} for x in tag_ids]
tgs.append({'_id': 11, 'Nm': 'FOR>Tag_11', 'Ctrl': 1})
tgs.append({'_id': 12, 'Nm': 'POST>Tag_12', 'Ctrl': 1})
tags_coll.insert(tgs)
# Generate counter collection
counter_coll = db[blackboard_name + CounterManager.counter_suffix]
counter_coll.insert({"_id" : CounterManager.counter_type, CounterManager.counter_type : CounterManager.counter_type_date_based})
counter_coll.insert({"_id" : CounterManager.counter_next, CounterManager.counter_tag : tags_coll.count() + 1})
counter_coll.insert({"_id" : CounterManager.counter_hash, CounterManager.counter_hash : 'HSH', CounterManager.counter_hash_fields : ['oID', 'T', 'D']})
counter_coll.insert({"_id" : CounterManager.counter_indexes, CounterManager.counter_indexes : [{"_id" : 1},{"oID" : 1, "_id" : 1}, {"Tg" : 1, "_id" : 1, "oID" : 1}, {"HSH" : 1}, { "Fds" : 1, "_id" : 1}, {"aID" : 1}, {"FOR" : 1, "_id" : 1}, {"TrOf" : 1}, {"Tg" : 1, "_id" : 1}]})
# Generate date_based collections
document_colls = {year: db['{}_{}'.format(blackboard_name, year)] for year in range(2009,2019)}
for tid, (year, coll) in zip(tag_ids, document_colls.items()):
obj_id = ObjectId.from_datetime(datetime(year, 1, 1))
coll.insert({'_id': obj_id, 'T': 'Title {}'.format(tid), 'oID' : tid, 'D' : 'Description', 'Tg' : [tid, tid-1], 'FOR' : [11, 12]})
coll.create_index([('_id', 1)], background = True)
coll.create_index([('oID', 1), ('_id', 1)], background = True)
coll.create_index([('Tg', 1), ('_id', 1), ('oID', 1)], background = True)
coll.create_index([('HSH', 1)], background = True)
def delData(request,res,db):
'''
Desc:
delete the schedule exercise by id
Args:
request: maybe useful
res: id stores in res["sid"], and update res with proper information
Err:
1. invalid objectId
2. fail to delete data
'''
# error handler for invalid objectid
if not ObjectId.is_valid(res["sid"]):
res["err"]["status"] = 1
res["err"]["msg"] = "wrong id"
return res
data = {"_id":ObjectId(res["sid"])}
# data = {"sid":{"$in":schedule_list}}
docs = db.removeData(data)
# error handler for getting data
if docs["status"]:
res["err"] = docs
return res
#
# normal process
#
res["content"]["status"] = "successful"
return res
def approveUser(ID, accept):
if not ObjectId.is_valid(ID): return False
if ID == str(session['user']['_id']): return False
result = db.users.update_one({'_id' : ObjectId(ID), 'confirmed' : True,
'clinic' : session['user']['clinic']}, {'$set' : {'approved' : accept}})
if result.modified_count == 1 or accept == False:
info = db.users.find_one({'_id' : ObjectId(ID)})
if not info: return False #cover all of our bases
if accept:
userMsg = Message('Account Approved', #app title
sender = ('Example Administrator', '*****@*****.**'),
recipients = [(info['first'] + ' ' + info['last'], info['email'])])
userMsg.body = template('finalUser.txt', user = info, accept = True)
userMsg.html = template('finalUser.html', user = info, accept = True)
else:
userMsg = Message('Account Removed', #app title
sender = ('Example Administrator', '*****@*****.**'),
recipients = [(info['first'] + ' ' + info['last'], info['email'])])
userMsg.body = template('finalUser.txt', user = info, accept = False)
userMsg.html = template('finalUser.html', user = info, accept = False)
deletion = db.users.delete_one({'_id' : ObjectId(ID)})
if deletion.deleted_count == 0: return False
mail.send(userMsg)
return True
#something messed up
return False
def get_doc_obj_by_dt(self, p_db_nm, p_collection_nm, p_query_dt):
v_query_dt_str = p_query_dt.strftime('%Y-%m-%d %H:%M.%S')
v_query_dt_utc = datetime.datetime.strptime(v_query_dt_str, '%Y-%m-%d %H:%M.%S')
v_query_obj_id = ObjectId.from_datetime(v_query_dt_utc)
db_obj = self.client_obj[p_db_nm]
v_doc_obj = db_obj[p_collection_nm].find({"_id": {"$gte": v_query_obj_id}})
return v_doc_obj
def getClinicName(ID):
if not ObjectId.is_valid(ID): return None
#do we want a more human-readable clinic code in the future?
#this is good because we don't need to f**k with autoincrement
clinic = db.clinics.find_one({'_id' : ObjectId(ID)})
if clinic: return clinic['name']
return None
def handleSellMaterial(self, rqst, owner): ses = self.sessions.get(owner, None) if ses != None: try: pid = ObjectId(rqst.data['pid']) if not ObjectId.is_valid(pid): return matData = self.database.fetchPlayerItemData(pid)['Material'] price = 0 for i in matData: for j in rqst.data['mat']: if j['type'] == i['type'] and j['level'] == i['level']: if j['count'] > i['count']: return else: price += self.getMaterialSetting(j['type'], j['level'])[1] * j['count'] i['count'] -= j['count'] # remove whose count is 0: rev = range(len(matData)) rev.reverse() for i in rev: if matData[i]['count'] == 0: del matData[i] if price == rqst.data['price']: pprop = self.database.fetchPlayerProperty(pid) pprop['Coins'] = pprop['Coins'] + price self.database.updatePlayerProperty(pid, pprop) self.database.updatePlayerMaterial(pid, matData) except: print '%d sell material failed'%(owner)
def delData(request,res,db):
'''
Desc:
fetch all data about the user
Args:
request: request with different data
res: result that we need to update and return
Err:
1. invalid objectId
2. fail to delete data
'''
# error handler for invalid objectid
if not ObjectId.is_valid(res["uid"]):
res["err"]["status"] = 1
res["err"]["msg"] = "wrong id"
return res
data = {"_id":ObjectId(res["uid"])}
# data = {"sid":{"$in":schedule_list}}
docs = db.removeData(data)
# error handler for getting data
if docs["status"]:
res["err"] = docs
return res
#
# normal process
#
return res
def putData(request,res,db):
'''
Desc:
update some fields of a schedule exercise
Args:
request: store the updated information of exercise within request.form
db: referrence to db object
res: store the status
Err:
1. fail to update data
'''
if not ObjectId.is_valid(res["sid"]):
#res["err"]["status"] = 1
#res["err"]["msg"] = "wrong id"
#return res
match_data = {"sid":int(res["sid"])}
else:
match_data = {"_id":ObjectId(res["sid"])}
docs = db.updateData(match_data,request.form)
# catch the error of updating
if docs["status"]:
res["err"] = docs
return res
# return the status
res["content"]["status"] = docs["status"]
return res
def test_jsanitize(self):
#clean_json should have no effect on None types.
d = {"hello": 1, "world": None}
clean = jsanitize(d)
self.assertIsNone(clean["world"])
self.assertEqual(json.loads(json.dumps(d)), json.loads(json.dumps(
clean)))
d = {"hello": GoodMSONClass(1, 2)}
self.assertRaises(TypeError, json.dumps, d)
clean = jsanitize(d)
self.assertIsInstance(clean["hello"], six.string_types)
clean_strict = jsanitize(d, strict=True)
self.assertEqual(clean_strict["hello"]["a"], 1)
self.assertEqual(clean_strict["hello"]["b"], 2)
d = {"dt": datetime.datetime.now()}
clean = jsanitize(d)
self.assertIsInstance(clean["dt"], six.string_types)
clean = jsanitize(d, allow_bson=True)
self.assertIsInstance(clean["dt"], datetime.datetime)
d = {"a": ["b", np.array([1, 2, 3])],
"b": ObjectId.from_datetime(datetime.datetime.now())}
clean = jsanitize(d)
self.assertEqual(clean["a"], ['b', [1, 2, 3]])
self.assertIsInstance(clean["b"], six.string_types)
def _search(self, query, search_term):
# TODO: Unfortunately, MongoEngine contains bug which
# prevents running complex Q queries and, as a result,
# Flask-Admin does not support per-word searching like
# in other backends
op, term = parse_like_term(search_term)
criteria = None
for field in self._search_fields:
flt = {}
if isinstance(field, (mongoengine.base.fields.ObjectIdField)):
if ObjectId.is_valid(term):
flt = {field.name: term}
else:
continue
elif field.name.startswith("_"):
# Don't process internal fields, such as _cls
# which will be used in cases of inherited document classes
continue
else:
flt = {'%s__%s' %
(field if isinstance(field, string_types) else field.name, op): term}
q = mongoengine.Q(**flt)
if criteria is None:
criteria = q
else:
criteria |= q
return query.filter(criteria)
def main():
# criação
zdb = ZeroTinyDB('./data/db_teste1.json',
sort_keys=True,
indent=4,
separators=(',', ': '))
zdb.log.info('Iniciado')
table_access = zdb.table_access('tabela01')
try:
with ZeroTransaction(table_access) as ztr:
# inserção dado
ztr.insert({
'id_data': str(ObjectId()),
'idade': 10,
'status': 0,
'nome': 'Eduardo Pagotto',
'sexo': True,
'last': datetime.timestamp(datetime.now())
})
ztr.insert({
'id_data': str(ObjectId()),
'status': 0,
'idade': 51,
'nome': 'Eduardo Pagotto',
'sexo': True,
'last': datetime.timestamp(datetime.now())
})
ztr.insert({
'id_data': str(ObjectId()),
'status': 0,
'idade': 55,
'nome': 'Eduardo Pagotto',
'sexo': True,
'last': datetime.timestamp(datetime.now())
})
ztr.insert({
'id_data': str(ObjectId()),
'status': 0,
'nome': 'Eduardo Pagotto',
'sexo': False,
'idade': 30,
'last': datetime.timestamp(datetime.now())
})
# query com where
result2 = ztr.search(where('sexo') == False)
zdb.log.debug(str(result2))
for item in result2:
ztr.update(increment('status'),
where('id_data') == item['id_data'])
# query
dados = Query()
result = ztr.search((dados.idade > 50) & (dados.sexo == True))
zdb.log.debug(str(result))
ultimo = None
for item in result:
# update
novo = {
'last': datetime.timestamp(datetime.now()),
'status': 3
}
ztr.update(novo, where('id_data') == item['id_data'])
ultimo = item
ztr.remove(dados.id_data == ultimo['id_data'])
lista_existe = ztr.search(where('id_data') == ultimo['id_data'])
# Mostra tudo
result = ztr.all()
zdb.log.debug(str(result))
except Exception as exp:
zdb.log.error('erro: %s', str(exp))
zdb.log.info('fim')
def get_recipe(recipe_id):
the_recipe = mongo.db.recipe_info.find_one({"_id": ObjectId(recipe_id)})
return render_template('get_recipe.html', recipe=the_recipe)
def delete(self, obj_id):
try:
return super(ServiceInstanceDao, self).delete(ObjectId(obj_id))
except InvalidId:
return False
station = 'theheat'
MONGODB_URI = os.environ.get('MONGODB_URI')
client = MongoClient(MONGODB_URI)
db = client.get_default_database()
collection_nowplaying = db[station + '_nowplaying']
collection_songs = db[station + '_songs']
scrub_manager.init(station)
#songs = db.nowplaying.find({'song': {"$regex": 'Break Up'}}).sort("startTime", direction=ASCENDING)
songs = collection_nowplaying.find({"$and":[{"song": "Wayback"}, {"spotify.url": ""}]}).sort("startTime", direction=ASCENDING)
counter = 1
for song in songs:
print(str(counter))
print(song)
song_id = song.get('_id')
song = scrub_manager.scrub_artist(station, song)
print(song)
song = scrub_manager.scrub_title(station, song)
print(song)
song = api_manager.get_spotify(song)
song.pop('_id', None)
print(song)
collection_nowplaying.replace_one({'_id': ObjectId(song_id)}, song)
# TODO: Delete old in songs
db_manager.save_in_songs(station, song)
counter += 1
time.sleep(0.5)
def edit_recipe(recipe_id):
_recipe = mongo.db.all_recipes.find_one({'_id': ObjectId(recipe_id)})
return render_template('edit_recipe.html', recipe=_recipe)
def delete_category(category_id):
mongo.db.categories.remove({"_id": ObjectId(category_id)})
flash("Category has been deleted for you hun!")
return redirect(url_for("get_categories"))
def update_category(category_id):
mongo.db.categories.update(
{'_id': ObjectId(category_id)},
{'category_name': request.form['category_name']})
return redirect(url_for('get_categories'))
def edit_category(category_id):
return render_template('editcategory.html',
category=mongo.db.categories.find_one({'_id': ObjectId(category_id)}))
def delete_task(task_id):
mongo.db.tasks.remove({'_id': ObjectId(task_id)})
return redirect(url_for('get_tasks'))
import copy
import unittest
from unittest.mock import MagicMock, Mock, call
import ddt
from bson.objectid import ObjectId
from opaque_keys.edx.locator import CourseLocator
from xmodule.modulestore.split_mongo.mongo_connection import MongoConnection
from xmodule.modulestore.split_mongo.split import SplitBulkWriteMixin
VERSION_GUID_DICT = {
'SAMPLE_VERSION_GUID': 'deadbeef1234' * 2,
'SAMPLE_UNICODE_VERSION_GUID': 'deadbeef1234' * 2,
'BSON_OBJECTID': ObjectId()
}
SAMPLE_GUIDS_LIST = [
'SAMPLE_VERSION_GUID', 'SAMPLE_UNICODE_VERSION_GUID', 'BSON_OBJECTID'
]
class TestBulkWriteMixin(unittest.TestCase): # lint-amnesty, pylint: disable=missing-class-docstring
def setUp(self):
super().setUp()
self.bulk = SplitBulkWriteMixin()
self.bulk.SCHEMA_VERSION = 1
self.clear_cache = self.bulk._clear_cache = Mock(name='_clear_cache')
self.conn = self.bulk.db_connection = MagicMock(name='db_connection',
spec=MongoConnection)
self.conn.get_course_index.return_value = {'initial': 'index'}
def delete_recipe(recipe_id):
mongo.db.all_recipes.remove({'_id': ObjectId(recipe_id)})
return redirect(url_for('get_meals'))
def show_recipe(recipe_id):
return render_template('recipe.html',
recipe=mongo.db.all_recipes.find_one(
{'_id': ObjectId(recipe_id)}))
def find_by_class_name_and_id(self, class_name, obj_id):
try:
instance_id = ObjectId(obj_id)
return self.dbcoll.find_one({'_id': instance_id, 'class_name': class_name})
except (InvalidId, TypeError):
return None
def make_object_id():
return str(ObjectId())
def delete_category(category_id):
mongo.db.categories.remove({'_id': ObjectId(category_id)})
return redirect(url_for("get_categories"))
def find(self, obj_id):
try:
instance_id = ObjectId(obj_id)
return super(ServiceInstanceDao, self).find(instance_id)
except (InvalidId, TypeError):
return None
def delete_task(task_id):
mongo.db.tasks.remove({"_id": ObjectId(task_id)})
flash("Task has been deleted for you hun!")
return redirect(url_for("get_tasks"))
def delete_animal(animal_id):
mongo.db.animals.remove({"_id": ObjectId(animal_id)})
return redirect(url_for("all_animals"))
def edit_animal(animal_id):
the_animal = mongo.db.animals.find_one({"_id": ObjectId(animal_id)})
all_animal_types = mongo.db.types.find()
all_diets = mongo.db.diets.find()
return render_template("editanimal.html", animal=the_animal,
types=all_animal_types, diets=all_diets)
def _topology_id(params):
_id = params.get('id')
return ({
'id': _id
}, None) if _id and ObjectId.is_valid(_id) else (
None, Error('Invalid topology\'s id'))
def post(self, request):
'''
任务结果列表
'''
taskid = request.data.get("search[value]", 0)
taskdate = request.data.get('taskdate', "")
length = request.data.get("number", 10)
start = request.data.get("start", 0)
conn = mongo.MongoConn()
result_list = []
vulcount = 0
lastscan = []
if taskid:
lastscan = conn.db["Result"].distinct(
'task_date', {'task_id': ObjectId(taskid)})
if len(lastscan) > 0:
lastscan.sort(reverse=True)
if taskdate: # 根据扫描批次查看结果
cursor = conn.db['Result'].find({
'task_id':
ObjectId(id),
'task_date':
datetime.strptime(taskdate, "%Y-%m-%d %H:%M:%S.%f")
}).sort('time', -1).limit(length).skip(start)
else: # 查看最新批次结果
cursor = conn.db['Result'].find({
'task_id': ObjectId(taskid),
'task_date': lastscan[0]
}).sort('time', -1).limit(length).skip(start)
vulcount = cursor.count()
for item in cursor:
result_list.append({
'ip':
item['ip'],
'port':
item['port'],
'info':
item['info'],
'vul_level':
item['vul_info']['vul_level'],
'time':
datetime.strftime(item['time'], '%Y-%m-%d %H:%M:%S')
})
# 速度优化，数据量多采取不同的方式查询
if len(result_list) > 100:
ip_hostname = {}
hostname = conn.db['Info'].aggregate([{
'$match': {
'hostname': {
'$ne': None
}
}
}, {
'$project': {
'_id': 0,
'ip': 1,
'hostname': 1
}
}])
for _ in hostname:
if 'hostname' in hostname:
ip_hostname[_["ip"]] = _["hostname"]
for _ in result_list:
if 'ip' in ip_hostname:
_['hostname'] = ip_hostname[_["ip"]]
else:
_['hostname'] = ''
else:
for _ in result_list:
hostname = conn.db['Info'].find_one({'ip': _['ip']})
if hostname and 'hostname' in hostname:
_['hostname'] = hostname['hostname']
else:
_['hostname'] = ''
context = {
'draw': 0,
'recordsTotal': vulcount,
'recordsFiltered': vulcount,
'data': result_list,
'result': 'ok'
}
return Response(context)
def playlists_delete(playlist_id):
"""Delete one playlist."""
playlists.delete_one({'_id': ObjectId(playlist_id)})
return redirect(url_for('playlists_index'))
def post(self, *args, **kwargs):
request_body = json.loads(self.request.body)
token = request_body.get('token', None)
if not token:
self.set_status(400)
self.write({'message': generate_missing_param_message('token')})
return
request_id = request_body.get('request_id', None)
if not request_id:
self.set_status(400)
self.write(
{'message': generate_missing_param_message('request_id')})
return
can_entry = request_body.get('can_entry', None)
if can_entry not in (True, False):
self.set_status(400)
self.write(
{'message': generate_missing_param_message('can_entry')})
return
users = self.settings['db'].users
tokens = self.settings['db'].tokens
requests = self.settings['db'].requests
token = yield tokens.find_one({'token': token})
if not token:
self.set_status(401)
self.write({'message': 'Invalid token'})
return
user_id = token['user_id']
user = yield users.find_one({'_id': user_id})
if not user['is_manager']:
self.set_status(403)
self.write({
'message':
'User {username} does not have permission for this action'.
format(username=user['username'])
})
return
if not ObjectId.is_valid(request_id):
self.set_status(401)
self.write({'message': 'Invalid request_id'})
return
request = yield requests.find_one({'_id': ObjectId(request_id)})
if not request:
self.set_status(401)
self.write({'message': 'Invalid request_id'})
return
result = requests.update_one({'_id': ObjectId(request_id)}, {
'$set': {
'status':
can_entry and RequestStatusIds.ACCEPTED
or RequestStatusIds.REJECTED,
'modification_date':
datetime.utcnow(),
}
})
if result:
result = users.update_one({'_id': request['user_id']}, {
'$set': {
'can_entry': can_entry,
'modification_date': datetime.utcnow(),
}
})
if result:
if can_entry:
self.write({'message': 'User has received a entry permit'})
else:
self.write({'message': 'User application was rejected'})
return
return
def find_category_by_id(category_id):
category = get_category_collection()
return category.find_one({'_id': ObjectId(category_id)})
def comments_delete(comment_id):
"""Action to delete a comment."""
comment = comments.find_one({'_id': ObjectId(comment_id)})
comments.delete_one({'_id': ObjectId(comment_id)})
return redirect(url_for('playlists_show', playlist_id=comment.get('playlist_id')))
def delete_user(id):
mongo.db.Student.delete_one({"_id": ObjectId(id)})
resp = jsonify("User deleted Successfully ")
resp.status_code = 200
return resp
def playlists_edit(playlist_id):
"""Show the edit form for a playlist."""
playlist = playlists.find_one({'_id': ObjectId(playlist_id)})
return render_template('playlists_edit.html', playlist=playlist, title='Edit Playlist')
def user(id):
users = mongo.db.Student.find_one({"_id": ObjectId(id)})
resp = dumps(users)
return resp
def edit_task(task_id):
the_task = mongo.db.tasks.find_one({"_id": ObjectId(task_id)})
all_categories = mongo.db.categories.find()
return render_template('edittask.html', task=the_task, categories=all_categories)
def delete_recipe(recipe_id):
mongo.db.recipe_info.remove({'_id': ObjectId(recipe_id)})
return redirect(url_for('index'))
