def get_count(coll,event): count1=0 count2=0 print 'num_days_1', num_days_1 for i in xrange(num_days_1): print i y1 = main_y1 + datetime.timedelta(days=i) t1 = main_y1 + datetime.timedelta(days=i+1) print "dates are:" print y1,t1 ts_y1 = int(time.mktime(y1.timetuple())*1000) ts_t1 = int(time.mktime(t1.timetuple())*1000) yutcd1 = datetime.datetime.utcfromtimestamp(ts_y1/1000) tutcd1 = datetime.datetime.utcfromtimestamp(ts_t1/1000) yutcoid1 = ObjectId.from_datetime(yutcd1) tutcoid1 = ObjectId.from_datetime(tutcd1) print yutcoid1, tutcoid1 count1 += len(coll.find({'_id': {'$gte': yutcoid1, '$lte': tutcoid1}, 'service': event['service'], 'device':event['device'], 'action':event['action']}).distinct('uid')) print 'num_days_2', num_days_2 for i in xrange(num_days_2): print i y2 = main_y2 + datetime.timedelta(days=i) t2 = main_y2 + datetime.timedelta(days=i+1) print "dates are:" print y2,t2 ts_y2 = int(time.mktime(y2.timetuple())*1000) ts_t2 = int(time.mktime(t2.timetuple())*1000) yutcd2 = datetime.datetime.utcfromtimestamp(ts_y2/1000) tutcd2 = datetime.datetime.utcfromtimestamp(ts_t2/1000) yutcoid2 = ObjectId.from_datetime(yutcd2) tutcoid2 = ObjectId.from_datetime(tutcd2) print yutcoid2, tutcoid2 count2 += len(coll.find({'_id': {'$gte': yutcoid2, '$lte': tutcoid2}, 'service': event['service'], 'device':event['device'], 'action':event['action']}).distinct('uid')) return (count1,count2)
def datetime_to_objectId(input="", year=0, month=1, day=1, hour=0): ''' Input -- input <datetime.datetime> <basestring> year <int> month <int> day <int> hour <int> Output -- <ObjectId> or None or <basestring> if bson.objectid module does not exist reference: http://api.mongodb.org/python/1.5.2/api/pymongo/objectid.html ''' response_objectid = None #default to None to enable 'if' check if "bson" not in IMPORTFAILS: if input: if isinstance(input, basestring): #need to parse response_objectid = ObjectId.from_datetime(_parse_input(input)) else: #should only be a string or datetime, all else will fail here response_objectid = ObjectId.from_datetime(input) elif year: response_objectid = ObjectId.from_datetime(datetime.datetime(int(year), int(month), int(day), int(hour))) else: pass #TODO: build object ID without objectID library return response_objectid # should be of ObjectID or None
def get(self, itemId): ffrom, to = self.get_argument('from', None), self.get_argument('to', None) filt = {'item_id': ObjectId(itemId), '_id': { '$gt': ObjectId.from_datetime(parse(ffrom) if ffrom else (self.now - datetime.timedelta(1))), '$lte': ObjectId.from_datetime(parse(to) if to else self.now) }} self.write(dumps(alfred.db.values.find(filt)))
def get_forecast(host, port, source, db_name, username, password, collection_name, date): # forecasts are always pushed after time that whole suite starts running # so implementation below is correct, and will result in us # being able to capture forecast and archive data used by operator in past # when in debug mode in present date_lb = date - pd.Timedelta(refresh_rate) date_ub = date + pd.Timedelta(refresh_rate) with pymongo.MongoClient(host=host, port=port) as conn: conn[db_name].authenticate(username, password, source=source) collection = conn[db_name][collection_name] for data in collection.find({ "_id": { "$gte": ObjectId.from_datetime( date_lb), "$lte": ObjectId.from_datetime( date_ub) } }).sort( "_id", pymongo.ASCENDING): reading = data['readings'] wfore = pd.DataFrame(reading) if len(wfore) == 0: raise ValueError("An appropriate forecast for the passed" "date does not exist in the database. Please" "pass another date, and check to make sure" "the database is functioning correctly.") else: wfore.set_index('time', inplace=True) wfore = wfore.sort_index() wfore = wfore.tz_localize('UTC') return wfore
def get_vitals(self, collection, dt_now, td_lookback): # TODO: Currently just going [6] hours back lstintPatientUID = [ int(x) for x in list(self.df_i_cohort['UID'].unique()) ] dt_end = dt_now dt_start = dt_end - td_lookback obj_start = ObjectId.from_datetime(dt_start) obj_end = ObjectId.from_datetime(dt_end) # TODO: Create aggregate group so not overwhelmed mg_query = { '_id': { '$gt': obj_start, '$lt': obj_end }, 'patient_id': { '$in': lstintPatientUID } } sortOrder = (('_id', -1), ('recorded_on', -1)) temp = collection.find(mg_query).sort(sortOrder) df_i_vitals = pd.DataFrame.from_dict(list(temp)) return df_i_vitals
def get_orders(self, collection, dt_now): df_i_cohort = self.df_i_cohort df_i_orders = pd.DataFrame() print("size: " + str(df_i_cohort.shape)) for index, row in df_i_cohort.iterrows(): if index % 50 == 0: print(str(datetime.datetime.now()) + '\t' + str(index)) dt_end = dt_now obj_end = ObjectId.from_datetime(dt_end) dt_start = row['dt_visitStart_UTC'] obj_start = ObjectId.from_datetime(dt_start) mg_query = {'UID': int(row['UID']), '_id': {'$lt': obj_end, '$gt': obj_start}, # 'recorded_on': {'$gt': nTimeStart}, } sortOrder = (('_id', -1), ('OrderDate', -1)) df_temp_ind = pd.DataFrame(list(collection.find(mg_query))) if df_temp_ind.shape[0] == 0: df_temp_ind = pd.DataFrame(columns=['CSN'], data=[row['CSN']]) else: df_temp_ind['CSN'] = row['CSN'] # df_rows = pd.concat([pd.DataFrame(row).T[['CSN']], df_temp_ind], axis='columns') df_i_orders = pd.concat([df_i_orders, df_temp_ind], axis='rows', sort=False) df_i_orders = df_i_orders.reset_index(drop=True) return df_i_orders
def count_task_state(begin_time, end_time=None): query_body = dict() query_body['_id'] = dict() query_body['_id']['$gte'] = ObjectId.from_datetime(begin_time) if end_time: query_body['_id']['$lte'] = ObjectId.from_datetime(end_time) _data = dict() _data['task_failure'] = 0 _data['task_success'] = 0 _data['task_total'] = 0 for item in mongo.db.task.aggregate([{ '$group': { '_id': '$state', 'count': { '$sum': 1 } } }]): if item['_id'] == 'FAILURE': _data['task_failure'] = item['count'] if item['_id'] == 'SUCCESS': _data['task_success'] = item['count'] _data['task_total'] = _data.get('task_failure', 0) + _data.get( 'task_success', 0) return _data
def query_by_user_id_ndays(self, db, collection, find, serial_no, user_id="", days=30): collect_lag = collection data = self.db_skynet['skynet_user_info'] if serial_no != "": factor = data.find({"serial_no": serial_no}).sort("_id", -1)[0] else: factor = data.find({"user_id": user_id}).sort("_id", -1)[0] date = factor['event_time'] #+ datetime.timedelta(hours=8) now_id = ObjectId.from_datetime(date) date = date - datetime.timedelta(days=days) pre30day = str(date)[0:10] pre30day = datetime.datetime.strptime(pre30day, "%Y-%m-%d") # print pre30day pre30_id = ObjectId.from_datetime(pre30day) collection = eval("self.db_{0}['{1}']".format(db, collection)) ret = [] find["_id"] = {"$gt": pre30_id, "$lt": now_id} for data in collection.find(find).sort("_id", -1): ret.append(data) ret = self.public_get_gzip_data(ret, collect_lag) return ret
def query_sms_list_beforeEvenTime_inXdays(self, db, collection, find, serial_no, days): collect_lag = collection collection = eval("self.db_{0}['{1}']".format(db, collection)) info = self.db_skynet['skynet_user_info'].find({ "serial_no": serial_no }).sort("_id", -1)[0] event_time = info['event_time'] now_id = ObjectId.from_datetime(event_time) find["_id"] = {"$lt": now_id} start_date = event_time - datetime.timedelta(days=days) start_id = ObjectId.from_datetime(start_date) ret = [] new_ret = [] find["_id"] = {"$gt": start_id, "$lt": now_id} for data in collection.find(find).sort("_id", -1): ret.append(data) ret = self.public_get_gzip_data(ret, collect_lag) if not ret: return 'No DATA' for row in ret: for sub_row in row.get('mobileSms'): if sub_row not in new_ret: new_ret.append(sub_row) return new_ret
def query_phone_max5_beforeEvenTime_inXdays(self, db, collection, find, serial_no, days, validTime): collect_lag = collection collection = eval("self.db_{0}['{1}']".format(db, collection)) info = self.db_skynet['skynet_user_info'].find({ "serial_no": serial_no }).sort("_id", -1)[0] event_time = info['event_time'] now_id = ObjectId.from_datetime(event_time) find["_id"] = {"$lt": now_id} start_date = event_time - datetime.timedelta(days=days) start_id = ObjectId.from_datetime(start_date) ret = [] find["_id"] = {"$gt": start_id, "$lt": now_id} for data in collection.find(find).sort("_id", -1): ret.append(data) ret = self.public_get_gzip_data(ret, collect_lag) phoneDic = {} if not ret: return 'No DATA' for everyId in ret: for line in everyId['actions']: phone = phoneClear(line["callNumber"]) phone = phone_clean(phone) if phone != None and line.get("callTime") > validTime: if phone not in phoneDic: phoneDic[phone] = [line["callTime"]] else: if line["callTime"] not in phoneDic[phone]: phoneDic[phone].append(line["callTime"]) temp = [] for k, v in phoneDic.items(): temp.append([len(set(v)), max(v), k]) temp.sort(reverse=True) return [str(phone[2]) for phone in temp[:5]]
def transaction_counts(): global interval_in_milliseconds global db current_timestamp = datetime.datetime.now() time_delta = timedelta(milliseconds=interval_in_milliseconds) current_timestamp_objectid = ObjectId.from_datetime(current_timestamp) previous_timestamp_objectid = ObjectId.from_datetime( current_timestamp - time_delta) # transaction_counts = db.abot_transaction_counters.aggregate([{'$match': {'_id': {'$lt': current_timestamp_objectid, '$gt': previous_timestamp_objectid}}}, { # '$group': {'_id': 'null', 'average': {'$avg': '$count'}, 'total': {'$sum': '$count'}}}]) s1ap_transaction_counts = db.abot_transaction_counters.aggregate([{'$match': {'_id': {'$lt': current_timestamp_objectid, '$gt': previous_timestamp_objectid}, 'counter': { '$regex': 'S1AP'}}}, {'$group': {'_id': 'null', 'average': {'$avg': '$count'}, 'total': {'$sum': '$count'}}}]) gtpv1u_transaction_counts = db.abot_transaction_counters.aggregate([{'$match': {'_id': {'$lt': current_timestamp_objectid, '$gt': previous_timestamp_objectid}, 'counter': { '$regex': 'GTPV1U'}}}, {'$group': {'_id': 'null', 'average': {'$avg': '$count'}, 'total': {'$sum': '$count'}}}]) gtpv2c_transaction_counts = db.abot_transaction_counters.aggregate([{'$match': {'_id': {'$lt': current_timestamp_objectid, '$gt': previous_timestamp_objectid}, 'counter': { '$regex': 'GTPV2C'}}}, {'$group': {'_id': 'null', 'average': {'$avg': '$count'}, 'total': {'$sum': '$count'}}}]) s1ap_tc = loads(dumps(s1ap_transaction_counts)) gtpv1u_tc = loads(dumps(gtpv1u_transaction_counts)) gtpv2c_tc = loads(dumps(gtpv2c_transaction_counts)) response = {} response['s1ap'] = 0 response['gtpv1u'] = 0 response['gtpv2c'] = 0 if len(s1ap_tc) > 0: response['s1ap'] = dumps(s1ap_tc[0]['average']) if len(gtpv1u_tc) > 0: response['gtpv1u'] = dumps(gtpv1u_tc[0]['average']) if len(gtpv2c_tc) > 0: response['gtpv2c'] = dumps(gtpv2c_tc[0]['average']) return dumps(response)
def worker(args): macsy_settings, liwc_dict, start_date, end_date = args root = logging.getLogger() root.setLevel(logging.DEBUG) bbapi, db = load(macsy_settings) with io.open(liwc_dict, 'r', encoding="utf-8") as liwc_file: liwc = LIWC(liwc_file) filter = { "_id": { "$gte": ObjectId.from_datetime(start_date), "$lt": ObjectId.from_datetime(end_date) } } p = pipeline(liwc, bbapi, db, filter) while True: _id, total_tweets = p.send(None) logging.debug("{}: {} tweets".format(_id['x'].isoformat(), total_tweets)) return None
def copy(source_params, dest_params, start_date, end_date): source_db = connect_mongodb(source_params) source_tables = get_table_names(source_params) dest_db = connect_mongodb(dest_params) dest_tables = get_table_names(dest_params) # records = [] # for record in source_db[source_tables['topics_table']].find(): # records.append(record) # print("total records {}".format(len(records))) # dest_db[dest_tables['topics_table']].insert_many( # records) # # records = [] # for record in source_db[source_tables['meta_table']].find(): # records.append(record) # print("total records {}".format(len(records))) # dest_db[dest_tables['meta_table']].insert_many( # records) # This is probably the most inefficient way of doing a copying a subset # of a # collection to another database but this is the one that requires the # minimum access # Wish this feature request is closed soon # https://jira.mongodb.org/browse/SERVER-13201 # Aggregation is the fastest way to get a subset of data from a collection, # next would be map reduce. map reduce can write output to another db # but it would only generate doc of schema # id:<object id>, value:<search/mapreduce result> dest_db[dest_tables['data_table']].create_index( [('topic_id', pymongo.DESCENDING), ('ts', pymongo.DESCENDING)], unique=True, background=False) records = [] i = 0 print ("start obj:{}".format(ObjectId.from_datetime(start_date))) print ("end obj:{}".format(ObjectId.from_datetime(end_date))) cursor = source_db[source_tables['data_table']].find( {'$and': [{'_id': {'$gte': ObjectId.from_datetime(start_date)}}, {'_id': {'$lte': ObjectId.from_datetime(end_date)}}]}) print ("Record count from cursor {}".format(cursor.count())) for record in cursor: i += 1 records.append( ReplaceOne( {'ts':record['ts'], 'topic_id':record['topic_id']}, {'ts': record['ts'], 'topic_id': record['topic_id'], 'value': record['value']}, upsert=True)) if i == 2000: print("total records {}".format(len(records))) dest_db[dest_tables['data_table']].bulk_write(records) i =0 records = []
def idslice(col, start_seconds, end_seconds=0): start_delta = timedelta(seconds=start_seconds) start_objid = ObjectId.from_datetime(utc_now() - start_delta) end_delta = timedelta(seconds=end_seconds) end_objid = ObjectId.from_datetime(utc_now() - end_delta) for obj in col.find( {'_id': {'$gte': start_objid, '$lt': end_objid}}).sort('_id'): yield obj
def get_date_query(start_date, end_date): if start_date is not None and end_date is not None: oid_start = ObjectId.from_datetime(start_date) oid_stop = ObjectId.from_datetime(end_date) return {"_id": {"$gte": oid_start, "$lt": oid_stop}} else: return None
def getDateQuery(start_date,end_date): if not start_date is None and not end_date is None: oid_start = ObjectId.from_datetime(start_date) oid_stop = ObjectId.from_datetime(end_date) return { "_id": { "$gte": oid_start, "$lt": oid_stop } } else: return None
def test_from_datetime(self): d = datetime.datetime.utcnow() d = d - datetime.timedelta(microseconds=d.microsecond) oid = ObjectId.from_datetime(d) self.assertEqual(d, oid.generation_time.replace(tzinfo=None)) self.assertEqual("0" * 16, str(oid)[8:]) aware = datetime.datetime(1993, 4, 4, 2, tzinfo=FixedOffset(555, "SomeZone")) as_utc = (aware - aware.utcoffset()).replace(tzinfo=utc) oid = ObjectId.from_datetime(aware) self.assertEqual(as_utc, oid.generation_time)
def objectid (day): start_date = str(day) date_1 = datetime.datetime.strptime(start_date,"%Y-%m-%d %H:%M:%S") end_date = date_1 + datetime.timedelta(hours=1) s=datetime.datetime(date_1.year,date_1.month,date_1.day,date_1.hour) d=datetime.datetime(end_date.year,end_date.month,end_date.day,end_date.hour) objmin =ObjectId.from_datetime(s) objmax =ObjectId.from_datetime(d) #print str(date_1),str(end_date),objmin,objmax return objmin,objmax,end_date
def worker(macsy_settings, liwc_dict, start_date, end_date): root = logging.getLogger() root.setLevel(logging.DEBUG) bbapi, db = load(macsy_settings) with io.open(liwc_dict, 'r', encoding="utf-8") as liwc_file: liwc = LIWC(liwc_file) locs_col = bbapi.load_blackboard( "LOCATION_PIPE_B").document_manager.get_collection() locs = [loc["_id"] for loc in locs_col.find({}, {"_id": 1})] _id_filter = { "$gte": ObjectId.from_datetime(start_date), "$lt": ObjectId.from_datetime(end_date) } filter = {"_id": _id_filter, "L": {"$in": locs}} p = pipeline(liwc, bbapi, db, filter) labels = [v for v, _ in liwc.categories.values()] + ["wc", "wc_dic"] tweets_col = bbapi.load_blackboard( "TWEET_PIPE_B").document_manager.get_collection() # pls symlink locations = {} for l in locs: count = tweets_col.find({"_id": _id_filter, "L": l}).count() locations[l] = { "fp": np.lib.format.open_memmap("brexit/loc_{}_tweets.npy".format( str(l)), mode="w+", dtype=np.int64, shape=(count, len(labels))), "i": 0 } for o in p: _id, location, vector, total, total_dic = o fp = locations[location]["fp"] i = locations[location]["i"] fp[i, :-2] = vector fp[i, -2:] = [total, total_dic] locations[location]["i"] += 1
def objectid(day): start_date = str(day) date_1 = datetime.datetime.strptime(start_date, "%Y-%m-%d %H:%M:%S") end_date = date_1 + datetime.timedelta(hours=1) s = datetime.datetime(date_1.year, date_1.month, date_1.day, date_1.hour) d = datetime.datetime(end_date.year, end_date.month, end_date.day, end_date.hour) objmin = ObjectId.from_datetime(s) objmax = ObjectId.from_datetime(d) #print str(date_1),str(end_date),objmin,objmax return objmin, objmax, end_date
def idslice(col, start_seconds, end_seconds=0): start_delta = timedelta(seconds=start_seconds) start_objid = ObjectId.from_datetime(utc_now() - start_delta) end_delta = timedelta(seconds=end_seconds) end_objid = ObjectId.from_datetime(utc_now() - end_delta) for obj in col.find({ '_id': { '$gte': start_objid, '$lt': end_objid } }).sort('_id'): yield obj
def query_all_by_userId_inXdays(self, db, collection, find, serial_no, start_days=0, start_time="", end_days=0, end_time='', s_start_id="", s_end_id=""): collect_lag = collection collection = eval("self.db_{0}['{1}']".format(db, collection)) info = self.db_skynet['skynet_user_info'].find({ "serial_no": serial_no }).sort("_id", -1)[0] event_time = info['event_time'] now_id = ObjectId.from_datetime(event_time) find["_id"] = {"$lt": now_id} if len(start_time) == 0: start_date = event_time - datetime.timedelta(days=start_days) else: event_time_add8 = event_time - datetime.timedelta( days=start_days) + datetime.timedelta(hours=8) start_day = str(event_time_add8)[0:10] + ' ' + start_time start_date_add8 = datetime.datetime.strptime( start_day, "%Y-%m-%d %H:%M:%S") start_date = start_date_add8 - datetime.timedelta(hours=8) if s_start_id: start_id = s_start_id else: start_id = ObjectId.from_datetime(start_date) if len(end_time) == 0: end_date = event_time + datetime.timedelta(days=end_days) else: event_time_add8 = event_time + datetime.timedelta( days=end_days) + datetime.timedelta(hours=8) end_day = str(event_time_add8)[0:10] + ' ' + end_time end_date_add8 = datetime.datetime.strptime(end_day, "%Y-%m-%d %H:%M:%S") end_date = end_date_add8 - datetime.timedelta(hours=8) if s_end_id: end_id = s_end_id else: end_id = ObjectId.from_datetime(end_date) ret = [] find["_id"] = {"$gt": start_id, "$lt": end_id} for data in collection.find(find).sort("_id", -1): ret.append(data) ret = self.public_get_gzip_data(ret, collect_lag) return ret
def worker(start_date, end_date, f): root = logging.getLogger() root.setLevel(logging.DEBUG) macsy_settings = f["tweets"].attrs["macsy_settings"] bbapi, db = load(macsy_settings) with io.open(f["tweets"].attrs["liwc_dict"], 'r', encoding="utf-8") as liwc_file: liwc = LIWC(liwc_file) _id_filter = { "$gte": ObjectId.from_datetime(start_date), "$lt": ObjectId.from_datetime(end_date) } filter = ast.literal_eval(f["tweets"].attrs["filter"]) filter["_id"] = _id_filter blackboard = f["tweets"].attrs["blackboard"] tweets_col = bbapi.load_blackboard( blackboard).document_manager.get_collection() f["tweets"].attrs["inserted"] = np.asarray( f["tweets"].attrs["inserted"].tolist() + ["{} to {}".format(start_date.isoformat(), end_date.isoformat())], dtype=h5py.string_dtype()) # times should contain UTC isoformated times times = f["tweets"]["times"] buckets_lookup = dict((dt, i) for i, dt in enumerate(times)) indicators = f["tweets"]["indicators"] wordcounts = f["tweets"]["wordcounts"] tweetcounts = f["tweets"]["tweetcounts"] # Make sure indicators are normalized p = pipeline(liwc, bbapi, db, filter, blackboard, f["tweets"].attrs["trim_rt"], f["tweets"].attrs["indicator_resolution"]) for _id, vector, wordcount, _ in p: i = buckets_lookup.get( find_bucket(_id.generation_time).isoformat(), None) if i is None: continue indicators[i, :] += vector wordcounts[i] += wordcount tweetcounts[i] += 1
def main(): client = MongoClient() db = client['convai-bot'] dialogs = db.dialogs now = datetime.datetime.now(tzlocal()) start_id = ObjectId.from_datetime(datetime.datetime(now.year, now.month, now.day, 0, 0, 0, 0, tzlocal()).astimezone(tzutc())) end_id = ObjectId.from_datetime(datetime.datetime(now.year, now.month, now.day, 23, 59, 59, 999, tzlocal()).astimezone(tzutc())) test_set = dialogs.find({'$and': [{'_id': {'$gte': start_id}}, {'_id': {'$lte': end_id}}]}).sort('_id', 1) train_set = dialogs.find().sort('_id', 1) print("\nExport test set for day %s\n" % now.date()) export_to_file('test_set', test_set, labeled=False, filtered=False) print("\nExport train set for day %s\n" % now.date()) export_to_file('train_set', train_set, labeled=True, filtered=True) client.close()
def object_id_day_range(start_time, end_time): """ Aim: Converts time to Mongo's ObjectId Input: start_time: The starting time, type: datetime.datetime object end_time: The end time, type: datetime.datetime object Output: start_object_id: The starting ObjectId, pymongo ObjectId object end_object_id: The ending ObjectId, pymongo ObjectId object """ start_object_id = ObjectId.from_datetime(start_time) end_object_id = ObjectId.from_datetime(end_time) return start_object_id, end_object_id
def test_bb_get_date(self): self.assertEqual( str( self.bb.get_date({ DateBasedDocumentManager.doc_id: ObjectId.from_datetime(dtparser.parse('21-10-2017')) })), '2017-10-21 00:00:00+00:00') with self.assertRaises(ValueError): self.bb.get_date({ 'different_id': ObjectId.from_datetime(dtparser.parse('21-10-2017')) }) with self.assertRaises(ValueError): self.bb.get_date({DateBasedDocumentManager.doc_id: 1})
def test_from_datetime(self): if "PyPy 1.8.0" in sys.version: # See https://bugs.pypy.org/issue1092 raise SkipTest("datetime.timedelta is broken in pypy 1.8.0") d = datetime.datetime.utcnow() d = d - datetime.timedelta(microseconds=d.microsecond) oid = ObjectId.from_datetime(d) self.assertEqual(d, oid.generation_time.replace(tzinfo=None)) self.assertEqual("0" * 16, str(oid)[8:]) aware = datetime.datetime(1993, 4, 4, 2, tzinfo=FixedOffset(555, "SomeZone")) as_utc = (aware - aware.utcoffset()).replace(tzinfo=utc) oid = ObjectId.from_datetime(aware) self.assertEqual(as_utc, oid.generation_time)
def test_from_datetime(self): d = datetime.datetime.utcnow() d = d - datetime.timedelta(microseconds=d.microsecond) oid = ObjectId.from_datetime(d) self.assertEqual(d, oid.generation_time.replace(tzinfo=None)) self.assertEqual("0" * 16, str(oid)[8:]) aware = datetime.datetime(1993, 4, 4, 2, tzinfo=FixedOffset(555, "SomeZone")) as_utc = (aware - aware.utcoffset()).replace(tzinfo=utc) oid = ObjectId.from_datetime(aware) self.assertEqual(as_utc, oid.generation_time)
def get(self, itemId): ffrom, to = self.get_argument('from', None), self.get_argument('to', None) filt = { 'item_id': ObjectId(itemId), '_id': { '$gt': ObjectId.from_datetime( parse(ffrom) if ffrom else (self.now - datetime.timedelta(1))), '$lte': ObjectId.from_datetime(parse(to) if to else self.now) } } self.write(dumps(alfred.db.values.find(filt)))
def __init__(self,env,product,startTime,endTime,step=30): """ :param env: 环境 :param mongodbSelect:mongodb数据库【skynet,lake,galaxy】 :param startTime: 开始时间 【样式:"2019-02-20 16:16:01"】 :param endTime: 结束时间 【样式:"2019-02-20 17:16:01"】 :param step: 时间间隔,默认10分钟,单位分钟 """ self.env=env self.product=product self.startTime=datetime.datetime.strptime(startTime,"%Y-%m-%d %H:%M:%S") self.endTime=datetime.datetime.strptime(endTime,"%Y-%m-%d %H:%M:%S") self.step=step self.startObjectId=ObjectId.from_datetime(self.startTime) self.endObjectId=ObjectId.from_datetime(self.endTime)
def procedure_completions(): global interval_in_milliseconds global db current_timestamp = datetime.datetime.now() time_delta = timedelta(milliseconds=interval_in_milliseconds) current_timestamp_objectid = ObjectId.from_datetime(current_timestamp) previous_timestamp_objectid = ObjectId.from_datetime( current_timestamp - time_delta) procedures_counts = db.abot_procedure_counters.aggregate([{'$match': {'_id': {'$lt': current_timestamp_objectid, '$gt': previous_timestamp_objectid}}}, { '$group': {'_id': 'null', 'total': {'$sum': '$count'}}}]) tc = loads(dumps(procedures_counts)) if len(tc) > 0: d = dumps(tc[0]['total']) return d return dumps(0)
def get(self): try: args = parse(items_parser_sett['get']) except Exception as e: return parse_error_response(e) subreddit = args['subreddit'] from_oid = ObjectId.from_datetime( datetime.datetime.utcfromtimestamp(args['from'])) to_oid = ObjectId.from_datetime( datetime.datetime.utcfromtimestamp(args['to'])) keyword = args.get('keyword', None) data = request.mongo.get_entities(subreddit, from_oid, to_oid, keyword) return ok_response(data=data)
def test_from_datetime(self): if 'PyPy 1.8.0' in sys.version: # See https://bugs.pypy.org/issue1092 raise SkipTest("datetime.timedelta is broken in pypy 1.8.0") d = datetime.datetime.utcnow() d = d - datetime.timedelta(microseconds=d.microsecond) oid = ObjectId.from_datetime(d) self.assertEqual(d, oid.generation_time.replace(tzinfo=None)) self.assertEqual("0" * 16, str(oid)[8:]) aware = datetime.datetime(1993, 4, 4, 2, tzinfo=FixedOffset(555, "SomeZone")) as_utc = (aware - aware.utcoffset()).replace(tzinfo=utc) oid = ObjectId.from_datetime(aware) self.assertEqual(as_utc, oid.generation_time)
def on_status(self, status): # Connect to MongoDB host client = MongoClient(MONGO_HOST, username=auth_data.username, password=auth_data.password) # Use defined database db = client.twitter_analytics collection = db.tweets gen_time = datetime.utcnow() - timedelta(days=days_to_keep) del_to_id = ObjectId.from_datetime(gen_time) if from_creator(status): try: if hasattr(status, "extended_tweet"): text = deEmojify(status.extended_tweet["full_text"]) else: text = deEmojify(status.text) collection.insert_one({ 'created_at': status.created_at, 'user_id': status.id, 'user_name': status.user.name, 'user_screen_name': status.user.screen_name, 'location': status.user.location, 'text': text, 'reply_count': status.reply_count, 'retweet_count': status.retweet_count }) print(text) except Exception: pass
def generate_date_based_blackboard(db, blackboard_name): blackboard_name = blackboard_name.upper() # Generate tags collection tags_coll = db[blackboard_name + TagManager.tag_suffix] tag_ids = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] tgs = [{'_id': x, 'Nm': 'Tag_{}'.format(x), 'Ctrl': 0} for x in tag_ids] tgs.append({'_id': 11, 'Nm': 'FOR>Tag_11', 'Ctrl': 1}) tgs.append({'_id': 12, 'Nm': 'POST>Tag_12', 'Ctrl': 1}) tags_coll.insert(tgs) # Generate counter collection counter_coll = db[blackboard_name + CounterManager.counter_suffix] counter_coll.insert({"_id" : CounterManager.counter_type, CounterManager.counter_type : CounterManager.counter_type_date_based}) counter_coll.insert({"_id" : CounterManager.counter_next, CounterManager.counter_tag : tags_coll.count() + 1}) counter_coll.insert({"_id" : CounterManager.counter_hash, CounterManager.counter_hash : 'HSH', CounterManager.counter_hash_fields : ['oID', 'T', 'D']}) counter_coll.insert({"_id" : CounterManager.counter_indexes, CounterManager.counter_indexes : [{"_id" : 1},{"oID" : 1, "_id" : 1}, {"Tg" : 1, "_id" : 1, "oID" : 1}, {"HSH" : 1}, { "Fds" : 1, "_id" : 1}, {"aID" : 1}, {"FOR" : 1, "_id" : 1}, {"TrOf" : 1}, {"Tg" : 1, "_id" : 1}]}) # Generate date_based collections document_colls = {year: db['{}_{}'.format(blackboard_name, year)] for year in range(2009,2019)} for tid, (year, coll) in zip(tag_ids, document_colls.items()): obj_id = ObjectId.from_datetime(datetime(year, 1, 1)) coll.insert({'_id': obj_id, 'T': 'Title {}'.format(tid), 'oID' : tid, 'D' : 'Description', 'Tg' : [tid, tid-1], 'FOR' : [11, 12]}) coll.create_index([('_id', 1)], background = True) coll.create_index([('oID', 1), ('_id', 1)], background = True) coll.create_index([('Tg', 1), ('_id', 1), ('oID', 1)], background = True) coll.create_index([('HSH', 1)], background = True)
def test_sql_insert(self): cursor = MagicMock() now = datetime.now() # Use ordereddict to ensure correct order in generated SQL request doc = OrderedDict() doc['_id'] = ObjectId.from_datetime(now) doc['field'] = 'val' sql.sql_insert(cursor, 'table', doc, '_id') doc['_creationDate'] = utils.extract_creation_date(doc, '_id') cursor.execute.assert_called_with( 'INSERT INTO table (_creationDate,_id,field) VALUES (%(_creationDate)s,%(_id)s,%(field)s) ON CONFLICT (_id) DO UPDATE SET (_creationDate,_id,field) = (%(_creationDate)s,%(_id)s,%(field)s) ', doc ) doc = { 'field': 'val' } sql.sql_insert(cursor, 'table', doc, '_id') cursor.execute.assert_called_with( 'INSERT INTO table (field) VALUES (%(field)s) ', doc )
def get(self): """Get Daily KPi.""" now = datetime.now() gen_time = datetime(now.year, now.month, now.day) dummy_id = ObjectId.from_datetime(gen_time) vdm_database = config.setup_mongo() cursor = vdm_database.booking.aggregate([ { "$project": { "_id": 1, "NbSpectateur": {"$size": "$Reservation"}, "TotalPrice": {"$sum": "$Reservation.prix"} }, }, {"$match": {"_id": {"$gte": dummy_id}}}, ]) data = [] for reservation in cursor: res_str = json.loads(dumps(reservation)) data.append(res_str) response = jsonify(data) response.headers.add('Access-Control-Allow-Origin', '*') response.status_code = 200 return response
def build_filters(self, **kwargs): ''' Break url parameters and turn into filters ''' filters = {} for param, value in kwargs.items(): # break each url parameter to key + operator (if exists) pl = dict(enumerate(param.split('__'))) key = pl[0] operator = pl.get(1, None) if key in self._meta.object_class.fields(): field = self._meta.object_class._fields[key] if field.is_composite: # composite keys require additional handling # currently covering cases for dbref and list if isinstance(field, DBRefField): key, value = self.process_dbref_filter(key, value) elif isinstance(value, list) and operator == 'in': value = [convert_value(v) for v in value] else: value = convert_value(value) # assign operator, if applicable filters[key] = {'${}'.format(operator): value} if operator else value elif key == 'created': # special case where we map `created` key to mongo's _id which also contains a creation timestamp dt = parser.parse(convert_value(value)) dummy_id = ObjectId.from_datetime(dt) filters['_id'] = {'${}'.format(operator): dummy_id} if operator else dummy_id return filters
def _remove_expired_publish_queue_items(self): expire_interval = app.config.get('PUBLISH_QUEUE_EXPIRY_MINUTES', 0) if expire_interval: expire_time = utcnow() - timedelta(minutes=expire_interval) logger.info('{} Removing publish queue items created before {}'.format(self.log_msg, str(expire_time))) get_resource_service('publish_queue').delete({'_id': {'$lte': ObjectId.from_datetime(expire_time)}})
def test_jsanitize(self): #clean_json should have no effect on None types. d = {"hello": 1, "world": None} clean = jsanitize(d) self.assertIsNone(clean["world"]) self.assertEqual(json.loads(json.dumps(d)), json.loads(json.dumps( clean))) d = {"hello": GoodMSONClass(1, 2)} self.assertRaises(TypeError, json.dumps, d) clean = jsanitize(d) self.assertIsInstance(clean["hello"], six.string_types) clean_strict = jsanitize(d, strict=True) self.assertEqual(clean_strict["hello"]["a"], 1) self.assertEqual(clean_strict["hello"]["b"], 2) d = {"dt": datetime.datetime.now()} clean = jsanitize(d) self.assertIsInstance(clean["dt"], six.string_types) clean = jsanitize(d, allow_bson=True) self.assertIsInstance(clean["dt"], datetime.datetime) d = {"a": ["b", np.array([1, 2, 3])], "b": ObjectId.from_datetime(datetime.datetime.now())} clean = jsanitize(d) self.assertEqual(clean["a"], ['b', [1, 2, 3]]) self.assertIsInstance(clean["b"], six.string_types)
def test_jsanitize(self): # clean_json should have no effect on None types. d = {"hello": 1, "world": None} clean = jsanitize(d) self.assertIsNone(clean["world"]) self.assertEqual(json.loads(json.dumps(d)), json.loads(json.dumps(clean))) d = {"hello": GoodMSONClass(1, 2, 3)} self.assertRaises(TypeError, json.dumps, d) clean = jsanitize(d) self.assertIsInstance(clean["hello"], str) clean_strict = jsanitize(d, strict=True) self.assertEqual(clean_strict["hello"]["a"], 1) self.assertEqual(clean_strict["hello"]["b"], 2) d = {"dt": datetime.datetime.now()} clean = jsanitize(d) self.assertIsInstance(clean["dt"], str) clean = jsanitize(d, allow_bson=True) self.assertIsInstance(clean["dt"], datetime.datetime) d = { "a": ["b", np.array([1, 2, 3])], "b": ObjectId.from_datetime(datetime.datetime.now()), } clean = jsanitize(d) self.assertEqual(clean["a"], ["b", [1, 2, 3]]) self.assertIsInstance(clean["b"], str) rnd_bin = bytes(np.random.rand(10)) d = {"a": bytes(rnd_bin)} clean = jsanitize(d, allow_bson=True) self.assertEqual(clean["a"], bytes(rnd_bin)) self.assertIsInstance(clean["a"], bytes)
def get_doc_obj_by_dt(self, p_db_nm, p_collection_nm, p_query_dt): v_query_dt_str = p_query_dt.strftime('%Y-%m-%d %H:%M.%S') v_query_dt_utc = datetime.datetime.strptime(v_query_dt_str, '%Y-%m-%d %H:%M.%S') v_query_obj_id = ObjectId.from_datetime(v_query_dt_utc) db_obj = self.client_obj[p_db_nm] v_doc_obj = db_obj[p_collection_nm].find({"_id": {"$gte": v_query_obj_id}}) return v_doc_obj
def dates_filter_query(field, start_time, end_time): collection = get_mongo_client(db_name=db_json, collection_name="json_logs") if (field == '_id'): filtered_rows = collection.find({ field: { '$gte': ObjectId.from_datetime(start_time), '$lt': ObjectId.from_datetime(end_time) } }) else: filtered_rows = collection.find( {field: { '$gte': start_time, '$lt': end_time }}) return (filtered_rows)
def from_metaweblog(cls, struct, post_type ='post', is_edit = False): # 从metaweblog RPC取得的结构,初始化一个post title = struct.get('title','') meta_description = struct.get('mt_excerpt', '') if len(meta_description) > 155: raise ValueError( "Description is %d chars, Max length is 155" % len(meta_description) ) if 'mt_keywords' in struct: tags = [ tag.strip() for tag in struct['mt_keywords'].split(',') if tag.strip() ] else: tags = None slug = ( slugify.slugify(struct['wp_slug']) if struct.get('wp_slug') else slugify.slugify(title)) description = struct.get('description', '') status = (struct.get('post_status') or struct.get('page_status') or 'publish') if "date_modified_gmt" in struct: tup = struct['date_modified_gmt'].timetuple() mod = utc_tz.localize(datetime.datetime(*tup[0:6])) else: mod = datetime.datetime.utcnow() body = markup.markup(description) img_src = struct[u'img_src'] if struct.has_key(u'img_src') else '' rv = cls( title=title, # Format for display body=body, plain=plain.plain(body), summary=summarize.summarize(body, 200), original=description, meta_description=meta_description, tags=tags, slug=slug, type=post_type, status=status, wordpress_id=struct.get('postid'), mod=mod, img_src=img_src ) if not is_edit and "data_created_gmt" in struct: date_created = datetime.datetime.strptime( struct['date_created_gmt'].value, "%Y%m%dT%H:%M:%S") rv.id = ObjectId.from_datetime(date_created) return rv
def from_metaweblog( cls, struct, post_type='post', is_edit=False ): """Receive metaWeblog RPC struct and initialize a Post. Used both by migrate_from_wordpress and when receiving a new or edited post from MarsEdit. """ title = struct.get('title', '') # We expect MarsEdit to set categories with mt_setPostCategories() assert 'categories' not in struct if 'mt_keywords' in struct: tags = [ tag.strip() for tag in struct['mt_keywords'].split(',') if tag.strip() ] else: tags = None slug = ( slugify.slugify(struct['wp_slug']) if struct.get('wp_slug') else slugify.slugify(title)) description = struct.get('description', '') status = struct.get('post_status', 'publish') if 'date_modified_gmt' in struct: tup = struct['date_modified_gmt'].timetuple() mod = utc_tz.localize(datetime.datetime(*tup[0:6])) else: mod = datetime.datetime.utcnow() body = markup.markup(description) rv = cls( title=title, # Format for display body=body, summary=summarize.summarize(body, 200), original=description, tags=tags, slug=slug, type=post_type, status=status, wordpress_id=struct.get('postid'), mod=mod ) if not is_edit and 'date_created_gmt' in struct: # TODO: can fail if two posts created in same second, add random # suffix to ObjectId date_created = datetime.datetime.strptime( struct['date_created_gmt'].value, "%Y%m%dT%H:%M:%S") rv.id = ObjectId.from_datetime(date_created) return rv
def remove_expired_results(self, collection): limit = datetime.datetime.now(pytz.utc) - datetime.timedelta(days=EXPIRE_AFTER) _id = ObjectId.from_datetime(limit) docs = self._db[collection].find({"_id": {"$lt": _id}, "sample_id": ""}) for d in docs: if "title" in d: print collection + ": removing " + d["title"] else: print collection + ": removing (no name)" self._db[collection].remove({"_id": {"$lt": _id}, "sample_id": ""})
def from_datetime(cls, *args, **kwargs): min = cls._pop_from_dict(kwargs, 'min') max = cls._pop_from_dict(kwargs, 'max') assert not (min and max), "Both min and max specified" oid = ObjectId(BaseObjectId.from_datetime(*args, **kwargs)) if min or max: oid = ObjectId(oid.__id[:2] + ('\x00' if min else '\xff') * 10) return oid
def admin(): # Auth Check is_admin = auth_check("ordering_admin") if request.form.get("action") == "tax" and request.form.get("tax") and is_admin: g.mongo.db.preferences.update({"_id": "ordering"}, {"$set": {"tax": float(request.form.get("tax", 0))}}, upsert=True) elif request.form.get("action") == "tax_corp" and request.form.get("tax") and is_admin: g.mongo.db.preferences.update({"_id": "ordering"}, {"$set": {"tax_corp": float(request.form.get("tax", 0))}}, upsert=True) tax_db = g.mongo.db.preferences.find_one({"_id": "ordering"}) tax = "{:.02f}".format(tax_db.get("tax", 0)) if tax_db else 0 tax_corp = "{:.02f}".format(tax_db.get("tax_corp", 0)) if tax_db else 0 # Invoice List one_month_oid = ObjectId.from_datetime(datetime.datetime.today() - datetime.timedelta(30)) invoice_table = [] marketeer_invoice_table = [] new_invoice_table = [] for invoice_db in g.mongo.db.invoices.find({"$or": [{"_id": {"$gt": one_month_oid}}, {"status": {"$not": re.compile("Completed")}}]}): invoice_status = invoice_db.get("status", "Not Processed") invoice_timestamp = ObjectId(invoice_db["_id"]).generation_time.strftime("%Y-%m-%d %H:%M:%S") invoice_color = "" if invoice_status == "Shipping - Completed": invoice_color = "primary" elif invoice_status == "Processing" or invoice_status.startswith("Shipping"): invoice_color = "warning" elif invoice_status in ["Failed", "Rejected", "Hold"]: invoice_color = "danger" elif invoice_status == "Completed": invoice_color = "success" finish_time = invoice_db.get("finish_time") if finish_time: time_to_delivery = finish_time - int(ObjectId(invoice_db["_id"]).generation_time.timestamp()) ttd_days = time_to_delivery // (60 * 60 * 24) ttd_hours = time_to_delivery % (60 * 60 * 24) // (60 * 60) ttd_minutes = time_to_delivery % (60 * 60 * 24) % (60 * 60) // 60 ttd_format = "{0}D{1}H{2}M".format(ttd_days, ttd_hours, ttd_minutes) else: ttd_format = "N/A" invoice_row = [invoice_color, invoice_timestamp, ttd_format, invoice_db["_id"], invoice_db["jf_end"], "{:,.02f}".format(invoice_db["order_total"]), invoice_db.get("character"), invoice_db.get("marketeer"), invoice_status] invoice_table.append(invoice_row) if invoice_db.get("marketeer") == session["CharacterName"]: marketeer_invoice_table.append(invoice_row) if invoice_status in ["Not Processed", "Submitted"]: new_invoice_table.append(invoice_row) return render_template("ordering_admin.html", invoice_table=invoice_table, tax=tax, is_admin=is_admin, marketeer_invoice_table=marketeer_invoice_table, new_invoice_table=new_invoice_table, tax_corp=tax_corp)
def oid_date_range_filter(dt_from=None, dt_upto=None, field_name='_id'): """ constructs a range query usefull to query an ObjectId field by date :Parameters: - dt_from (datetime or tuple): starting date_time if tuple a datetime is constucted from tuple - dt_upto (datetime or tuple): end date_time if tuple a datetime is constucted from tuple - field_name: (str): optional default to '_id' field to query or None if None returns range only else returns full query :Returns: - range query (due to objectId structure $gt includes dt_from) while $lt dt_upto (not included) """ def dt(dt_or_tuple): if isinstance(dt_or_tuple, datetime): return dt_or_tuple elif isinstance(dt_or_tuple, tuple): return datetime(*dt_or_tuple) else: raise TypeError('dt must be a date or tuple') q = SON() if dt_from is not None: q.update(SON([('$gte', ObjectId.from_datetime(dt(dt_from)))])) if dt_upto is not None: q.update(SON([('$lte', ObjectId.from_datetime(dt(dt_upto)))])) return q if field_name is None else SON([(field_name, q)])
def get_server_object_ids(**kwargs): """ # for a given timestamp_range, compute the corresponding ObjectIds of MongoDB """ # y = datetime.datetime.now().date() - datetime.timedelta(days=1) # t= datetime.datetime.now().date() if kwargs is not None: if 'time_range' in kwargs: prev, curr = kwargs['time_range'] print prev, curr prev_timestamp = int(time.mktime(prev.timetuple())*1000) curr_timestamp = int(time.mktime(curr.timetuple())*1000) elif 'timestamp_range' in kwargs: prev_timestamp, curr_timestamp = kwargs['timestamp_range'] else: raise Exception('Keyword Error') prev_utc_datetime = datetime.datetime.utcfromtimestamp(prev_timestamp/1000) curr_utc_datetime = datetime.datetime.utcfromtimestamp(curr_timestamp/1000) prev_utc_objectId = ObjectId.from_datetime(prev_utc_datetime) curr_utc_objectId = ObjectId.from_datetime(curr_utc_datetime) return prev_utc_objectId, curr_utc_objectId
def home(): with open("configs/base.json", "r") as base_config_file: base_config = json.load(base_config_file) # Missing APIs missing_apis = [] corp_ids = [] api_characters = set() # Determine accounts in corp for user_info in g.mongo.db.users.find({"corporation_id": base_config["corporation_id"]}): if g.mongo.db.security_characters.find_one({"_id": user_info["character_id"]}): corp_ids.append(user_info["_id"]) # Determine characters with an api for api_user in g.mongo.db.api_keys.find({"_id": {"$in": corp_ids}}): for key in api_user["keys"]: api_characters.add(key["character_name"]) # Determine characters in corp without an api from a corp account for corp_character in g.mongo.db.security_characters.find(): if corp_character["name"] not in api_characters: missing_apis.append(corp_character["name"]) # Personal Invoices one_month_oid = ObjectId.from_datetime(datetime.datetime.today() - datetime.timedelta(30)) invoice_table = [] for invoice in g.mongo.db.invoices.find({"_id": {"$gt": one_month_oid}, "user": session["CharacterOwnerHash"]}): invoice_status = invoice.get("status", "Not Processed") invoice_timestamp = ObjectId(invoice["_id"]).generation_time.strftime("%Y-%m-%d %H:%M:%S") invoice_color = "" if invoice_status == "Shipping - Completed": invoice_color = "primary" elif invoice_status == "Processing" or invoice_status.startswith("Shipping"): invoice_color = "warning" elif invoice_status in ["Failed", "Rejected"]: invoice_color = "danger" invoice_table.append([invoice_color, invoice_timestamp, invoice["_id"], invoice["jf_end"], "{:,.02f}".format(invoice["order_total"]), invoice.get("marketeer"), invoice_status]) # Away from EVE db_vacation = g.mongo.db.personals.find({"vacation": {"$exists": True}}) away_from_eve = [] for character in db_vacation: if character["corporation_id"] == base_config["corporation_id"]: away_from_eve.append([character["character_name"], character["vacation"], character["vacation_date"]]) return render_template("corp.html", away_from_eve=away_from_eve, invoice_table=invoice_table, missing_apis=missing_apis)
def history(self, after=True, fields=[], limit=10, since=None, sort=DESCENDING): """ Return historical versions of this :term:`context`. :param after: if since is provided, only return history items after `since`. False will return items before since. **Default: ``True``** :params fields: a ``list`` or ``dict`` of fields to return in the results. If a list it should be list of strings representing the fields to return i.e. ``['mtime', ]``. If a ``dict`` it should specify either fields to omit or include i.e.: ``{'_id': False}`` or {'_id': False, 'title': True, 'changed_by': True}. :param limit: The number of history records to fetch. **Default: 10** :param since: a :mod:``datetime.datetime`` object representing the date to use as a search point with ``after``. **Default: None** :param sort: The direction to sort the history items. ``DESCENDING`` provides the most recent change first. **Default: ``DESCENDING``** """ if limit is None: limit = 0 if not isinstance(limit, int): raise TypeError("expected int, recieved {}".format(type(limit))) query = {'orig_id': self.data['_id']} if since and isinstance(since, datetime.datetime): stamp = ObjectId.from_datetime(since) if after: operator = "$gt" else: operator = "$lt" query['_id'] = {operator: stamp} if fields: cursor = self._collection_history.find( query, fields).limit(limit).sort('_id', DESCENDING) else: cursor = self._collection_history.find( query).limit(limit).sort('_id', DESCENDING) return cursor
def questionGathering(self, usersToSendMsg=None): days = 5 gen_time = datetime.today() - timedelta(days=days) dummy_id = ObjectId.from_datetime(gen_time) if not usersToSendMsg: usersToSendMsg = list(self.db.users.find({"_id": {"$lte": dummy_id}})) question = self.db.questions.find_one({"order": (self.order % self.db.questions.count())}) self.order = +1 for user in usersToSendMsg: markup = ReplyKeyboardMarkup(keyboard= self.keyboard_two_col(question['keyboard'][user['language']]), resize_keyboard=True, one_time_keyboard=True) text_msg = question['content'][user['language']] self.sendMessage(user['chat_id'],self.data['question_info_msg'][user['language']]) user['question_msg'] = self.sendMessage(user['chat_id'], text_msg, reply_markup=markup) user['question'] = question self.users_to_send_msg[user['chat_id']] = user
def getDocsForToday(conColl): today = datetime.today() aggr = conColl.aggregate( [ {"$match": {"_id": {"$gte": ObjectId.from_datetime(datetime(today.year, today.month, today.day))}}}, { "$group": { "_id": None, "caloriesForDay": {"$sum": "$calories"}, "fatsForDay": {"$sum": "$fat"}, "carbsForDay": {"$sum": "$carbohydrates"}, "proteinsForDay": {"$sum": "$proteins"}, } }, ] ) aggrDocs = list(aggr) return aggrDocs
def on_get(self, req, resp): now = datetime.datetime.now(); querydate = now - datetime.timedelta(days=1) if req.params.get('date', None): arrdate = req.params['date'].split("-"); year = int(arrdate[0]); month = int(arrdate[1]); day = int(arrdate[2].split(" ")[0]); hour = int(arrdate[2].split(" ")[1].split(":")[0]); minute = int(arrdate[2].split(" ")[1].split(":")[1]); second = int(arrdate[2].split(" ")[1].split(":")[2]); querydate = datetime.datetime(year,month,day,hour,minute,second); msg = "Request ::: "+str(querydate); self.printMsg(msg) ; dummy_id = ObjectId.from_datetime(querydate) result = EventModel.objects(id__gt = dummy_id)[:100]; json_string = json.dumps([r.obj_dict() for r in result]) resp.body = json_string; resp.status = falcon.HTTP_200
def test_extract_creation_date(self): now = datetime.now() doc = { '_id': ObjectId.from_datetime(now) } got = utils.extract_creation_date(doc, '_id') expected = now if expected.utcoffset() is not None: expected -= expected.utcoffset() expected = timegm(expected.timetuple()) expected = datetime.fromtimestamp(expected, utc) self.assertEqual(expected, got) got = utils.extract_creation_date({}, '_id') self.assertIsNone(got)
def test_insert(self): from macsy import utils # Generate a doc, check # of docs, insert it, check it's incremented obj_id = ObjectId.from_datetime(dtparser.parse('21-10-2017')) hsh = utils.java_string_hashcode('515TitleDescription') expected = 11 self.assertEqual(self.bb.count(), expected-1) self.assertEqual(self.bb.insert({DateBasedDocumentManager.doc_id : obj_id, 'HSH' : hsh, 'oID' : 515, 'T' : 'Title', 'D' : 'Description', 'Overwritten' : False, 'Inserted' : True, 'Tg' : [1, 2, 3]}), obj_id) self.assertEqual(self.bb.count(), expected) self.assertEqual([x for x in self.bb.find(query={'Inserted' : True})][0]['_id'], obj_id) # Try to insert it again self.assertEqual(self.bb.insert({DateBasedDocumentManager.doc_id : obj_id, 'oID' : 515, 'T' : 'Title', 'D' : 'Description', 'Overwritten' : True, 'Updated' : True, 'Tg' : [4, 5]}), obj_id) self.assertEqual(self.bb.count(), expected) self.assertEqual([x for x in self.bb.find(query={'Updated' : True})][0]['_id'], obj_id) self.assertEqual(self.bb.count(query={'Overwritten' : False}), 0) self.assertEqual([x for x in self.bb.find(query={'Overwritten' : True})][0]['_id'], obj_id) self.assertEqual([x for x in self.bb.find(tags=[1, 2, 3, 4, 5])][0]['_id'], obj_id) # Insert a document without an id and generate one self.assertEqual(self.bb.insert({'Blank_id' : True}).generation_time.date(), datetime.now().date())