class Client: def __init__(self, host, db): self.client = MongoClient(host) self.db = self.client.get_database(db) self.lists = self.db.get_collection("Clients") def create_new_member(self, chat_id, full_name): self.lists.replace_one({"chat_id": chat_id}, { "chat_id": chat_id, "full name": full_name, "friends": [], }, upsert=True) def add_friend_to_list(self, chat_id, friend): self.lists.update_one({"chat_id": chat_id}, {"$push": { "friends": friend }}) def get_doc(self, chat_id): return self.lists.find_one({"chat_id": chat_id}) def get_all_friends(self, chat_id): client = self.lists.find_one({"chat_id": chat_id}) return client["friends"] def delete_friend(self, chat_id, friend): self.lists.delete_one({"chat_id": chat_id, 'friends'[0]: friend})
def test_read_with_failover(self): c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) wait_until(lambda: c.primary, "discover primary") wait_until(lambda: len(c.secondaries) == 2, "discover secondaries") def iter_cursor(cursor): for _ in cursor: pass return True w = len(c.secondaries) + 1 db = c.get_database("pymongo_test", write_concern=WriteConcern(w=w)) db.test.delete_many({}) # Force replication db.test.insert_many([{'foo': i} for i in xrange(10)]) self.assertEqual(10, db.test.count()) db.read_preference = SECONDARY_PREFERRED cursor = db.test.find().batch_size(5) next(cursor) self.assertEqual(5, cursor._Cursor__retrieved) self.assertTrue(cursor.address in c.secondaries) ha_tools.kill_primary() # Primary failure shouldn't interrupt the cursor self.assertTrue(iter_cursor(cursor)) self.assertEqual(10, cursor._Cursor__retrieved)
def __init__(self): connection = MongoClient( settings['MONGO_SERVER'], settings['MONGO_PORT'] ) db = connection.get_database(settings['MONGO_DB']) self.collection = db[settings['MONGO_COLLECTION']]
def test_read_with_failover(self): c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) wait_until(lambda: c.primary, "discover primary") wait_until(lambda: len(c.secondaries) == 2, "discover secondaries") def iter_cursor(cursor): for _ in cursor: pass return True w = len(c.secondaries) + 1 db = c.get_database("pymongo_test", write_concern=WriteConcern(w=w)) db.test.delete_many({}) # Force replication db.test.insert_many([{'foo': i} for i in xrange(10)]) self.assertEqual(10, db.test.count()) db.read_preference = SECONDARY_PREFERRED cursor = db.test.find().batch_size(5) next(cursor) self.assertEqual(5, cursor._Cursor__retrieved) self.assertTrue(cursor.address in c.secondaries) ha_tools.kill_primary() # Primary failure shouldn't interrupt the cursor self.assertTrue(iter_cursor(cursor)) self.assertEqual(10, cursor._Cursor__retrieved)
def __init__(self): connection = MongoClient( settings['MONGO_SERVER'], settings['MONGO_PORT'] ) db = connection.get_database(settings['MONGO_DB']) self.collection = db[settings['MONGO_COLLECTION']]
def updatePlayerStats(): client = MongoClient() db = client.get_database("dota") matches: Collection = db.match_details players: Collection = db.players {"$and": [{"skill": {"$ne": null}}, {"skill": {"$ne": 1}}]}
def stash(results): """ 暂存到mongo数据库中。 """ summary = {} mongo = MongoClient(**config.mongo) try: for item_model, objs in results: collection_name = item_model['name'] db = mongo.get_database('theforce') collection = db.get_collection(collection_name) collection.insert_many(objs) summary[collection_name] = len( objs) if collection_name not in summary else len( objs) + summary[collection_name] print print "=" * 40 print ' ' * 15, u'Stash' print "=" * 40 print print u"数据已成功保存到MongoDB的theforce库中,其中新增数据:" for name, length in summary.items(): print name, length finally: mongo.close()
class giftList(): def __init__(self, host, db): self.client = MongoClient(host) self.db = self.client.get_database(db) self.gifts = self.db.get_collection("gifts") initial() def add_gift(self, type, price, link): self.gifts.replace_one({"link": link}, { "type": type, "price": price, "link": link }, upsert=True) def get_gifts_by_cond(self, type, price): p = price.split() start = int(p[0]) - 1 end = int(p[1]) + 1 myRange = self.gifts.find({ "type": type, "price": { "$gt": start, "$lt": end } }) list = [] for x in myRange: list.append(x) return list
def mongo_text_test(): cli = MongoClient() test = cli.get_database("test").get_collection("test_search") test.create_index( [("super_type", ASCENDING), ("resource_state", ASCENDING), ("uuid", TEXT), ("name", TEXT), ("description", TEXT)], background=True ) docs = [{ "uuid": str(uuid.uuid4()), "name": str(i), "description": "Nature, \"time, and patience are " "the three great -physicians.", "super_type": "super_vol", "type": "vol", "create_time": int( time.time()), "resource_state": "inUse" } for i in range(1, 11)] test.insert_many(docs) text = [" -physicians"] print test.find({ "description": { "$regex": "|".join([re.sub( r"(\*|\.|\?|\+|\$|\^|\[|\]|\(|\)|\{|\}|\||\\|/)", r"\\\1", g ) for g in text]), "$options": "i" } }).count()
def init_db(app): app.teardown_appcontext(close_db) # Set up Mongo client for text indexing global client client = MongoClient(Config.MONGO_URL) db = client.get_database('rokwire') events = db['eventsmanager_events'] events.create_index([("title", pymongo.TEXT)])
def init_db(): global client client = MongoClient(cfg.APP_CONFIG_MONGO_URL) # Create indexes on api start db = client.get_database(name=cfg.APP_CONFIG_DB_NAME) app_configs = db[cfg.APP_CONFIGS_COLLECTION] app_configs.create_index([("mobileAppVersion", pymongo.DESCENDING)], unique=True)
def test_get_database(self): client = MongoClient(host, port, _connect=False) codec_options = CodecOptions( tz_aware=True, uuid_representation=JAVA_LEGACY) write_concern = WriteConcern(w=2, j=True) db = client.get_database( 'foo', codec_options, ReadPreference.SECONDARY, write_concern) self.assertEqual('foo', db.name) self.assertEqual(codec_options, db.codec_options) self.assertEqual(JAVA_LEGACY, db.uuid_subtype) self.assertEqual(ReadPreference.SECONDARY, db.read_preference) self.assertEqual([{}], db.tag_sets) self.assertEqual(write_concern.document, db.write_concern) pref = Secondary([{"dc": "sf"}]) db = client.get_database('foo', read_preference=pref) self.assertEqual(pref.mode, db.read_preference) self.assertEqual(pref.tag_sets, db.tag_sets) self.assertEqual({}, db.write_concern) self.assertEqual(CodecOptions(), db.codec_options) self.assertEqual(PYTHON_LEGACY, db.uuid_subtype)
def init_db(): global client client = MongoClient(cfg.EVENT_MONGO_URL) # Create indexes on app start db = client.get_database(name=cfg.EVENT_DB_NAME) events = db['events'] events.create_index([("title", pymongo.TEXT)]) events.create_index([("startDate", pymongo.DESCENDING)]) events.create_index([("endDate", pymongo.DESCENDING)]) events.create_index([("category", pymongo.ASCENDING)]) events.create_index([("categorymainsub", pymongo.ASCENDING)]) events.create_index([("coordinates", pymongo.GEOSPHERE)])
def test_last_error(self): c = MongoClient(self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) wait_until(lambda: c.primary, "discover primary") wait_until(lambda: c.secondaries, "discover secondary") ha_tools.stepdown_primary() db = c.get_database("pymongo_test", write_concern=WriteConcern(w=0)) db.test.insert_one({}) response = db.error() self.assertTrue('err' in response and 'not master' in response['err']) wait_until(lambda: len(c.secondaries) == 2, "discover two secondaries")
class win_gifDB: def __init__(self): self.client = MongoClient(settings.HOST) self.db = self.client.get_database(settings.DB) self.lists = self.db.get_collection(settings.WINNER_GIFS_COL) def add_all_to_DB(self): with open("win_gifs.txt", "r") as f: for url in f.readlines(): self.lists.replace_one({"url": url}, {"url": url}, upsert=True) def get_random_gif(self): count = self.lists.estimated_document_count() return self.lists.find()[random.randrange(count)]['url']
def test_last_error(self): c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) wait_until(lambda: c.primary, "discover primary") wait_until(lambda: c.secondaries, "discover secondary") ha_tools.stepdown_primary() db = c.get_database( "pymongo_test", write_concern=WriteConcern(w=0)) db.test.insert_one({}) response = db.error() self.assertTrue('err' in response and 'not master' in response['err']) wait_until(lambda: len(c.secondaries) == 2, "discover two secondaries")
class WordsDB: def __init__(self): self.client = MongoClient(settings.HOST) self.db = self.client.get_database(settings.DB) self.lists = self.db.get_collection(settings.WORD_COL) def add_all_to_DB(self): with open("common_words.txt", "r") as f: for word in f: self.lists.replace_one({"word": word.split()[0].lower()}, {"word": word.split()[0].lower()}, upsert=True) def get_word(self, word): return self.lists.find_one({"word": word})
class Event: def __init__(self, host, db): self.client = MongoClient(host) self.db = self.client.get_database(db) self.events = self.db.get_collection("events") def add_event(self, client_id, full_name, type, date, mark): self.events.insert_one({ "client_id": client_id, "name": full_name, "type": type, "date": date, "mark": mark, }) def delete_event(self, name, type, date): self.events.delete_one({"name": name, "type": type, "date": date}) def get_events_by_date(self, date): myCursor = self.events.find({"date": date}) list = [] for x in myCursor: list.append(x) return list def get_events_by_name(self, name): myCursor = self.events.find({"name": name}) list = [] for x in myCursor: list.append(x) return list def count_events(self): return self.events.count_documents({}) def get_all_events(self): myCursor = self.events.find({}) list = [] for x in myCursor: list.append(x) return list def get_upcoming_events(self, start, end): myCursor = self.events.find({"date": {"$gt": start, "$lt": end}}) list = [] for x in myCursor: list.append(x) return list
class Storage_store: def __init__(self, host, db): self.client = MongoClient(host) self.db = self.client.get_database(db) self.my_DB = self.db.get_collection("Matches") # self.user_list = self.db.get_collection("Users List") def add_my_db(self, face_id, ad_id, store_id, img, time): self.my_DB.replace_one({'db_id': "COUNTER DOCUMENT NUMBER 0"}, { 'ad_id': ad_id, 'store_id': store_id, 'face_id': face_id, 'time': time, 'image': img.tolist() }, upsert=True) def clear_table(self): self.my_DB.delete_many({})
def save(metas, batch_num=100): """ 读取配置,把Mongo数据同步到mysql中。 """ mongo = MongoClient(**config.mongo) db = MySQLdb.connect(**config.mysql) cursor = db.cursor() print print "=" * 40 print ' ' * 15, u'Mongo --> MySQL' print "=" * 40 print try: mongo_db = mongo.get_database('theforce') for meta in metas: for model_name, item_model in meta.iter_model(): collection_name = item_model['name'] table_name = item_model['table'] attrs = meta.get_model_persist_attr_names(item_model) collection = mongo_db.get_collection(collection_name) results = [obj for obj in collection.find({})] sql = "insert into {0}({1}) values({2})".format( table_name, ','.join(attrs), ','.join(itertools.repeat('%s', len(attrs)))) print print '-' * 40 print u'开始处理{0}@mongo --> {1}@mysql, 共{2}条数据,每批{3}条批量迁移:'.format( collection_name, table_name, len(results), batch_num) # 分组进行批量处理 results2 = itertools.izip(itertools.count(), results) for group_key, group_it in itertools.groupby( results2, lambda item: item[0] / batch_num): print '.', values = [[obj[attr] for attr in attrs] for index, obj in group_it] cursor.executemany(sql, values) print u'[完成]' finally: mongo.close() cursor.close() db.close()
class UsersDB: def __init__(self): self.client = MongoClient(settings.HOST) self.db = self.client.get_database(settings.DB) self.lists = self.db.get_collection(settings.USER_COL) def add_to_DB(self, id): if not self.lists.find_one({"user_id": id}): self.lists.insert_one({"user_id": id, "score": 0}) def get_score(self, id): user = self.lists.find_one({"user_id": id}) score = user['score'] return score def update_score(self, id, score): self.lists.replace_one({"user_id": id}, { "user_id": id, "score": self.get_score(id) + score }, upsert=True)
def test_writes_with_failover(self): c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) wait_until(lambda: c.primary, "discover primary") wait_until(lambda: len(c.secondaries) == 2, "discover secondaries") primary = c.primary w = len(c.secondaries) + 1 db = c.get_database("pymongo_test", write_concern=WriteConcern(w=w)) db.test.delete_many({}) db.test.insert_one({'foo': 'bar'}) self.assertEqual('bar', db.test.find_one()['foo']) killed = ha_tools.kill_primary(9) self.assertTrue(bool(len(killed))) # Wait past pool's check interval, so it throws an error from # get_socket(). time.sleep(1) # Verify that we only raise AutoReconnect, not some other error, # while we wait for new primary. for _ in xrange(10000): try: db.test.insert_one({'bar': 'baz'}) # No error, found primary. break except AutoReconnect: time.sleep(.01) else: self.fail("Couldn't connect to new primary") # Found new primary. self.assertTrue(c.primary) self.assertTrue(primary != c.primary) self.assertEqual('baz', db.test.find_one({'bar': 'baz'})['bar'])
def test_writes_with_failover(self): c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) wait_until(lambda: c.primary, "discover primary") wait_until(lambda: len(c.secondaries) == 2, "discover secondaries") primary = c.primary w = len(c.secondaries) + 1 db = c.get_database("pymongo_test", write_concern=WriteConcern(w=w)) db.test.delete_many({}) db.test.insert_one({'foo': 'bar'}) self.assertEqual('bar', db.test.find_one()['foo']) killed = ha_tools.kill_primary(9) self.assertTrue(bool(len(killed))) # Wait past pool's check interval, so it throws an error from # get_socket(). time.sleep(1) # Verify that we only raise AutoReconnect, not some other error, # while we wait for new primary. for _ in xrange(10000): try: db.test.insert_one({'bar': 'baz'}) # No error, found primary. break except AutoReconnect: time.sleep(.01) else: self.fail("Couldn't connect to new primary") # Found new primary. self.assertTrue(c.primary) self.assertTrue(primary != c.primary) self.assertEqual('baz', db.test.find_one({'bar': 'baz'})['bar'])
class PagesDB: def __init__(self): self.client = MongoClient(settings.HOST) self.db = self.client.get_database(settings.DB) self.lists = self.db.get_collection(settings.PAGE_COL) def add_all_to_DB(self): with open("minimal_data.csv", "r", encoding='utf-8') as f: reader = csv.reader(f, delimiter=",") for i, line in enumerate(reader): if re.search(r"\b" + re.escape(line[0].lower()) + r"\b", settings.WIKI_EXLUDE_VALS): continue self.lists.replace_one({"title": line[0].lower()}, {"title": line[0].lower()}, upsert=True) def get_page(self, word): return self.lists.find_one({"title": word}) def get_random_page(self): count = self.lists.estimated_document_count() return self.lists.find()[random.randrange(count)]['title']
from pprint import pprint from pymongo.mongo_client import MongoClient client = MongoClient() db = client.get_database("my_school") students = db.get_collection('students') students.insert_one({ 'name': 'Shira', 'age': 22, 'address': { 'city': 'Jerusalem', 'street': 'King George' } }) for d in students.find(): pprint(d) print() xx = students.find({}) print(type(xx))
class PickAnalysis: def __init__(self): self.client = MongoClient() self.db = self.client.get_database("dota") self.matches: Collection = self.db.match_details self.abilities: Collection = self.db.get_collection("abilities") self.ab_abilities: Collection = self.db.get_collection("ab_abilities") self.days = 90 self.ability_id = None self.pick_data = None def get_distinct_abilities(self, period: Union[Tuple[int, int], None] = None): if period is None: res = self.matches.distinct("ability_draft.skills") else: res = self.matches.distinct( "ability_draft.skills", {"start_time": { "$gt": period[0], "$lt": period[1] }}) return res def save_ab_abilities(self): abilities = self.get_distinct_abilities() self.ab_abilities.update_one({"_id": "latest"}, { "$set": { "abilities": abilities, "last_run": int(datetime.utcnow().timestamp()) } }, upsert=True) def __load_ability_picks__(self) -> pd.DataFrame: cursor: CommandCursor = self.matches.aggregate([{ "$match": { "ability_draft.skills": self.ability_id, "duration": { "$gt": 900 }, "start_time": { "$gt": time.time() - 60 * 60 * 24 * self.days } } }, { "$project": { "position": { "$indexOfArray": ["$ability_draft.drafts", self.ability_id] }, "radiant_win": 1 } }]) arr = np.asarray([[x['radiant_win'], x['position']] for x in cursor]) if arr.size == 0: raise NoSkillDataException arr = np.hstack( [arr, (arr[:, 1] % 2 == (arr[:, 0] + 1) % 2)[..., None]]) df = pd.DataFrame(arr, columns=['radiant_win', 'pick', 'win']) df.loc[df['pick'] == -1, 'pick'] = 40 return df def __load_ability_picks__(self) -> pd.DataFrame: cursor: CommandCursor = self.matches.aggregate([{ "$match": { "ability_draft.skills": self.ability_id, "duration": { "$gt": 900 }, "start_time": { "$gt": time.time() - 60 * 60 * 24 * self.days } } }, { "$project": { "position": { "$indexOfArray": ["$ability_draft.drafts", self.ability_id] }, "radiant_win": 1 } }]) arr = np.asarray([[x['radiant_win'], x['position']] for x in cursor]) if arr.size == 0: raise NoSkillDataException arr = np.hstack( [arr, (arr[:, 1] % 2 == (arr[:, 0] + 1) % 2)[..., None]]) df = pd.DataFrame(arr, columns=['radiant_win', 'pick', 'win']) df.loc[df['pick'] == -1, 'pick'] = 40 return df def combo_picks(self): # combo_dict = defaultdict(Counter) # wins, played, gold, damage, kills, deaths, assists, xp, tower combo_dict = defaultdict(lambda: np.zeros(9).astype(np.float64)) single_dict = defaultdict(lambda: np.zeros(9).astype(np.float64)) combos = self.db.get_collection('combos') cursor: CommandCursor = self.matches.find( { "duration": { "$gt": 900 }, "start_time": { "$gt": time.time() - 60 * 60 * 24 * self.days } }, { "players.ability_upgrades_arr": 1, "players.gold_per_min": 1, "players.hero_damage": 1, "players.kills": 1, "players.deaths": 1, "players.assists": 1, "players.xp_per_min": 1, "players.tower_damage": 1, "duration": 1, "radiant_win": 1 }) for match in tqdm(cursor): mmin = match['duration'] / 60 for i, player in enumerate(match['players']): win = 1 if (i % 2 == 0 and match['radiant_win']) or ( i % 2 == 1 and not match['radiant_win']) else 0 skills = np.unique(player['ability_upgrades_arr']) # wins, played, gold, damage, kills, deaths, assists, xp, tower new_stats = [ win, 1, player['gold_per_min'], player['hero_damage'] / mmin, player['kills'] / mmin, player['deaths'] / mmin, player['assists'] / mmin, player['xp_per_min'], player['tower_damage'] / mmin ] for skill in skills: single_dict[skill] += new_stats for combo in combinations( np.unique(player['ability_upgrades_arr']), 2): combo_dict[combo] += new_stats def get_average(skill1, skill2): skill1_win = single_dict[skill1][0] * 1.0 / single_dict[skill1][1] skill2_win = single_dict[skill2][0] * 1.0 / single_dict[skill2][1] return (skill1_win + skill2_win) / 2 combos.delete_many({}) combo_docs = [ { "_id": { "skill1": int(key[0]), "skill2": int(key[1]) }, "avg_win_pct": get_average(key[0], key[1]), "raw_win_pct": vals[0] / vals[1], "win_pct": vals[0] / vals[1] - .5 / np.sqrt(vals[1]), "synergy": vals[0] / vals[1] - .5 / np.sqrt(vals[1]) - get_average(key[0], key[1]), "gold": vals[2] / vals[1], "damage": vals[3] / vals[1], "kills": vals[4] / vals[1], "deaths": vals[5] / vals[1], "assists": vals[6] / vals[1], "xp": vals[7] / vals[1], "tower": vals[8] / vals[1], "matches": vals[1] } for key, vals in combo_dict.items() if vals[1] > 25 and vals[0] / vals[1] - .5 / np.sqrt(vals[1]) > .5 ] combos.insert_many(combo_docs) for skill_id, vals in single_dict.items(): try: if skill_id is None: continue self.abilities.update_one({"_id": int(skill_id)}, { "$set": { "win_rate": vals[0] / vals[1], "gold": vals[2] / vals[1], "damage": vals[3] / vals[1], "kills": vals[4] / vals[1], "deaths": vals[5] / vals[1], "assists": vals[6] / vals[1], "xp": vals[7] / vals[1], "tower": vals[8] / vals[1], } }) except WriteError as e: print(f"Failed to write {skill_id}") pass def save_pick_analytics(self): self.abilities.update_one({"_id": self.ability_id}, { "$set": dict(survival=list(self.get_pick_survival()), **self.get_pick_win_rates(), **self.get_pick_summary(), **self.get_pick_rates()) }, upsert=True) def get_pick_rates(self): df = self.pick_data picks = self.pick_data['pick'].count() * 1.0 return dict( pick_rate=df.loc[df['pick'].between(0, 39), 'pick'].count() / picks, pick_rate_rounds=[ df.loc[df['pick'].between(x * 10, x * 10 + 9), 'pick'].count() / picks for x in range(4) ]) def get_pick_win_rates(self): df = self.pick_data return dict(win_rate_rounds=[ df.loc[df['pick'].between(x * 10, x * 10 + 9), 'win'].mean() for x in range(4) ]) def get_pick_summary(self): return self.pick_data['pick'].describe().to_dict() def get_pick_median(self): return self.pick_data['pick'].median() def get_pick_survival(self): data = self.pick_data df = (data.groupby('pick').count().sort_index().reindex( pd.Index(range(data['pick'].min(), data['pick'].max() + 1)), fill_value=0).iloc[:, [0]].rename(columns={data.columns[0]: 'count'})) df['survival'] = ( (1 - df['count'] / (df['count'].sum() - df['count'].shift(1).cumsum())).cumprod()) df = df.reindex(range(40)) df = df.where(df.ffill().notna(), 1) df = df.where(df.bfill().notna(), 0) return df['survival'].values def set_pick_id(self, ability_id): self.ability_id = ability_id self.pick_data = self.__load_ability_picks__() return self def hero_stats(self): cursor: CommandCursor = self.matches.find( { "duration": { "$gt": 900 }, "start_time": { "$gt": time.time() - 60 * 60 * 24 * self.days } }, { "players.ability_upgrades_arr": 1, "players.hero_id": 1, "players.gold_per_min": 1, "players.hero_damage": 1, "players.kills": 1, "players.deaths": 1, "players.assists": 1, "players.xp_per_min": 1, "players.tower_damage": 1, "duration": 1, "players.is_radiant": 1, "radiant_win": 1 }) heros_coll = self.db.get_collection("heros") heros_coll.delete_many({}) hero_skill = defaultdict( lambda: defaultdict(lambda: np.zeros(2, np.int64))) hero_stats = defaultdict(lambda: np.zeros(2, np.int64)) for match in tqdm(cursor): for player in match["players"]: hero_id = player["hero_id"] win = 0 if (player["is_radiant"] ^ match["radiant_win"]) else 1 hero_stats[hero_id] += [win, 1] for skill in np.unique(player['ability_upgrades_arr']): hero_skill[hero_id][skill] += [win, 1] for hero_id, skills in tqdm(hero_skill.items()): h_stat = hero_stats[hero_id] hero_dict = { "_id": hero_id, "win_rate": float(h_stat[0] * 1.0 / h_stat[1]) } hero_skills = [] for skill_id, stats in skills.items(): win_pct = float(stats[0] * 1.0 / (stats[1])) if stats[1] > 15 and win_pct - .5 / np.sqrt(stats[1]) > .5: hero_skills.append({ "id": int(skill_id), "matches": int(stats[1]), "win_rate": float(stats[0] * 1.0 / stats[1]) }) hero_dict["skills"] = hero_skills heros_coll.insert_one(hero_dict)
def test_ship_of_theseus(self): c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) db = c.get_database( "pymongo_test", write_concern=WriteConcern(w=len(c.secondaries) + 1)) db.test.insert_one({}) find_one = db.test.find_one primary = ha_tools.get_primary() secondary1 = ha_tools.get_random_secondary() new_hosts = [] for i in range(3): new_hosts.append(ha_tools.add_member()) # RS closes all connections after reconfig. for j in xrange(30): try: if ha_tools.get_primary(): break except (ConnectionFailure, OperationFailure): pass time.sleep(1) else: self.fail("Couldn't recover from reconfig") # Wait for new members to join. for _ in xrange(120): if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4: break time.sleep(1) else: self.fail("New secondaries didn't join") ha_tools.kill_members([primary, secondary1], 9) time.sleep(5) wait_until(lambda: (ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 2), "fail over", timeout=30) time.sleep(2 * self.heartbeat_frequency) # No error. find_one() find_one(read_preference=SECONDARY) # All members down. ha_tools.kill_members(new_hosts, 9) self.assertRaises( ConnectionFailure, find_one, read_preference=SECONDARY) ha_tools.restart_members(new_hosts) # Should be able to reconnect to set even though original seed # list is useless. Use SECONDARY so we don't have to wait for # the election, merely for the client to detect members are up. time.sleep(2 * self.heartbeat_frequency) find_one(read_preference=SECONDARY) # Kill new members and switch back to original two members. ha_tools.kill_members(new_hosts, 9) self.assertRaises( ConnectionFailure, find_one, read_preference=SECONDARY) ha_tools.restart_members([primary, secondary1]) # Wait for members to figure out they're secondaries. wait_until(lambda: len(ha_tools.get_secondaries()) == 2, "detect two secondaries", timeout=30) # Should be able to reconnect to set again. time.sleep(2 * self.heartbeat_frequency) find_one(read_preference=SECONDARY)
def test_secondary_connection(self): self.c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) wait_until(lambda: len(self.c.secondaries), "discover secondary") # Wait for replication... w = len(self.c.secondaries) + 1 db = self.c.get_database("pymongo_test", write_concern=WriteConcern(w=w)) db.test.delete_many({}) db.test.insert_one({'foo': 'bar'}) # Test direct connection to a primary or secondary primary_host, primary_port = ha_tools.get_primary().split(':') primary_port = int(primary_port) (secondary_host, secondary_port) = ha_tools.get_secondaries()[0].split(':') secondary_port = int(secondary_port) arbiter_host, arbiter_port = ha_tools.get_arbiters()[0].split(':') arbiter_port = int(arbiter_port) # MongoClient succeeds no matter the read preference for kwargs in [ {'read_preference': PRIMARY}, {'read_preference': PRIMARY_PREFERRED}, {'read_preference': SECONDARY}, {'read_preference': SECONDARY_PREFERRED}, {'read_preference': NEAREST}, ]: client = MongoClient( primary_host, primary_port, serverSelectionTimeoutMS=self.server_selection_timeout, **kwargs) wait_until(lambda: primary_host == client.host, "connect to primary") self.assertEqual(primary_port, client.port) self.assertTrue(client.is_primary) # Direct connection to primary can be queried with any read pref self.assertTrue(client.pymongo_test.test.find_one()) client = MongoClient( secondary_host, secondary_port, serverSelectionTimeoutMS=self.server_selection_timeout, **kwargs) wait_until(lambda: secondary_host == client.host, "connect to secondary") self.assertEqual(secondary_port, client.port) self.assertFalse(client.is_primary) # Direct connection to secondary can be queried with any read pref # but PRIMARY if kwargs.get('read_preference') != PRIMARY: self.assertTrue(client.pymongo_test.test.find_one()) else: self.assertRaises( AutoReconnect, client.pymongo_test.test.find_one) # Since an attempt at an acknowledged write to a secondary from a # direct connection raises AutoReconnect('not master'), MongoClient # should do the same for unacknowledged writes. try: client.get_database( "pymongo_test", write_concern=WriteConcern(w=0)).test.insert_one({}) except AutoReconnect as e: self.assertEqual('not master', e.args[0]) else: self.fail( 'Unacknowledged insert into secondary client %s should' 'have raised exception' % (client,)) # Test direct connection to an arbiter client = MongoClient( arbiter_host, arbiter_port, serverSelectionTimeoutMS=self.server_selection_timeout, **kwargs) wait_until(lambda: arbiter_host == client.host, "connect to arbiter") self.assertEqual(arbiter_port, client.port) self.assertFalse(client.is_primary) # See explanation above try: client.get_database( "pymongo_test", write_concern=WriteConcern(w=0)).test.insert_one({}) except AutoReconnect as e: self.assertEqual('not master', e.args[0]) else: self.fail( 'Unacknowledged insert into arbiter client %s should' 'have raised exception' % (client,))
class TestReadPreference(HATestCase): # Speed up assertReadFrom() when no server is suitable. server_selection_timeout = 0.001 def setUp(self): super(TestReadPreference, self).setUp() members = [ # primary {'tags': {'dc': 'ny', 'name': 'primary'}}, # secondary {'tags': {'dc': 'la', 'name': 'secondary'}, 'priority': 0}, # other_secondary {'tags': {'dc': 'ny', 'name': 'other_secondary'}, 'priority': 0}, ] res = ha_tools.start_replica_set(members) self.seed, self.name = res primary = ha_tools.get_primary() self.primary = partition_node(primary) self.primary_tags = ha_tools.get_tags(primary) # Make sure priority worked self.assertEqual('primary', self.primary_tags['name']) self.primary_dc = {'dc': self.primary_tags['dc']} secondaries = ha_tools.get_secondaries() (secondary, ) = [ s for s in secondaries if ha_tools.get_tags(s)['name'] == 'secondary'] self.secondary = partition_node(secondary) self.secondary_tags = ha_tools.get_tags(secondary) self.secondary_dc = {'dc': self.secondary_tags['dc']} (other_secondary, ) = [ s for s in secondaries if ha_tools.get_tags(s)['name'] == 'other_secondary'] self.other_secondary = partition_node(other_secondary) self.other_secondary_tags = ha_tools.get_tags(other_secondary) self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']} self.c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) self.w = len(self.c.secondaries) + 1 self.db = self.c.get_database("pymongo_test", write_concern=WriteConcern(w=self.w)) self.db.test.delete_many({}) self.db.test.insert_many([{'foo': i} for i in xrange(10)]) self.clear_ping_times() def set_ping_time(self, host, ping_time_seconds): ServerDescription._host_to_round_trip_time[host] = ping_time_seconds def clear_ping_times(self): ServerDescription._host_to_round_trip_time.clear() def test_read_preference(self): # We pass through four states: # # 1. A primary and two secondaries # 2. Primary down # 3. Primary up, one secondary down # 4. Primary up, all secondaries down # # For each state, we verify the behavior of PRIMARY, # PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) wait_until(lambda: c.primary, "discover primary") wait_until(lambda: len(c.secondaries) == 2, "discover secondaries") def assertReadFrom(member, *args, **kwargs): utils.assertReadFrom(self, c, member, *args, **kwargs) def assertReadFromAll(members, *args, **kwargs): utils.assertReadFromAll(self, c, members, *args, **kwargs) def unpartition_node(node): host, port = node return '%s:%s' % (host, port) # To make the code terser, copy hosts into local scope primary = self.primary secondary = self.secondary other_secondary = self.other_secondary bad_tag = {'bad': 'tag'} # 1. THREE MEMBERS UP ------------------------------------------------- # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED # Trivial: mode and tags both match assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc) # Secondary matches but not primary, choose primary assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc) # Chooses primary, ignoring tag sets assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc) # Chooses primary, ignoring tag sets assertReadFrom(primary, PRIMARY_PREFERRED, bad_tag) assertReadFrom(primary, PRIMARY_PREFERRED, [bad_tag, {}]) # SECONDARY assertReadFromAll([secondary, other_secondary], SECONDARY) # SECONDARY_PREFERRED assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED) # Multiple tags assertReadFrom(secondary, SECONDARY_PREFERRED, self.secondary_tags) # Fall back to primary if it's the only one matching the tags assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'primary'}) # No matching secondaries assertReadFrom(primary, SECONDARY_PREFERRED, bad_tag) # Fall back from non-matching tag set to matching set assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED, [bad_tag, {}]) assertReadFrom(other_secondary, SECONDARY_PREFERRED, [bad_tag, {'dc': 'ny'}]) # NEAREST self.clear_ping_times() assertReadFromAll([primary, secondary, other_secondary], NEAREST) assertReadFromAll([primary, other_secondary], NEAREST, [bad_tag, {'dc': 'ny'}]) self.set_ping_time(primary, 0) self.set_ping_time(secondary, .03) # 30 ms self.set_ping_time(other_secondary, 10) # Nearest member, no tags assertReadFrom(primary, NEAREST) # Tags override nearness assertReadFrom(primary, NEAREST, {'name': 'primary'}) assertReadFrom(secondary, NEAREST, self.secondary_dc) # Make secondary fast self.set_ping_time(primary, .03) # 30 ms self.set_ping_time(secondary, 0) assertReadFrom(secondary, NEAREST) # Other secondary fast self.set_ping_time(secondary, 10) self.set_ping_time(other_secondary, 0) assertReadFrom(other_secondary, NEAREST) self.clear_ping_times() assertReadFromAll([primary, other_secondary], NEAREST, [{'dc': 'ny'}]) # 2. PRIMARY DOWN ----------------------------------------------------- killed = ha_tools.kill_primary() # Let monitor notice primary's gone time.sleep(2 * self.heartbeat_frequency) # PRIMARY assertReadFrom(None, PRIMARY) # PRIMARY_PREFERRED # No primary, choose matching secondary assertReadFromAll([secondary, other_secondary], PRIMARY_PREFERRED) assertReadFrom(secondary, PRIMARY_PREFERRED, {'name': 'secondary'}) # No primary or matching secondary assertReadFrom(None, PRIMARY_PREFERRED, bad_tag) # SECONDARY assertReadFromAll([secondary, other_secondary], SECONDARY) # Only primary matches assertReadFrom(None, SECONDARY, {'name': 'primary'}) # No matching secondaries assertReadFrom(None, SECONDARY, bad_tag) # SECONDARY_PREFERRED assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED) # Mode and tags both match assertReadFrom(secondary, SECONDARY_PREFERRED, {'name': 'secondary'}) # NEAREST self.clear_ping_times() assertReadFromAll([secondary, other_secondary], NEAREST) # 3. PRIMARY UP, ONE SECONDARY DOWN ----------------------------------- ha_tools.restart_members([killed]) ha_tools.wait_for_primary() ha_tools.kill_members([unpartition_node(secondary)], 2) time.sleep(5) ha_tools.wait_for_primary() time.sleep(2 * self.heartbeat_frequency) # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED assertReadFrom(primary, PRIMARY_PREFERRED) # SECONDARY assertReadFrom(other_secondary, SECONDARY) assertReadFrom(other_secondary, SECONDARY, self.other_secondary_dc) # Only the down secondary matches assertReadFrom(None, SECONDARY, {'name': 'secondary'}) # SECONDARY_PREFERRED assertReadFrom(other_secondary, SECONDARY_PREFERRED) assertReadFrom( other_secondary, SECONDARY_PREFERRED, self.other_secondary_dc) # The secondary matching the tag is down, use primary assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'}) # NEAREST assertReadFromAll([primary, other_secondary], NEAREST) assertReadFrom(other_secondary, NEAREST, {'name': 'other_secondary'}) assertReadFrom(primary, NEAREST, {'name': 'primary'}) # 4. PRIMARY UP, ALL SECONDARIES DOWN --------------------------------- ha_tools.kill_members([unpartition_node(other_secondary)], 2) # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED assertReadFrom(primary, PRIMARY_PREFERRED) assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc) # SECONDARY assertReadFrom(None, SECONDARY) assertReadFrom(None, SECONDARY, self.other_secondary_dc) assertReadFrom(None, SECONDARY, {'dc': 'ny'}) # SECONDARY_PREFERRED assertReadFrom(primary, SECONDARY_PREFERRED) assertReadFrom(primary, SECONDARY_PREFERRED, self.secondary_dc) assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'}) assertReadFrom(primary, SECONDARY_PREFERRED, {'dc': 'ny'}) # NEAREST assertReadFrom(primary, NEAREST) assertReadFrom(None, NEAREST, self.secondary_dc) assertReadFrom(None, NEAREST, {'name': 'secondary'}) # Even if primary's slow, still read from it self.set_ping_time(primary, 100) assertReadFrom(primary, NEAREST) assertReadFrom(None, NEAREST, self.secondary_dc) self.clear_ping_times()
class MONGODB: # 数据库链接对象 client = None # 某一个数据库操作对象 db = None # 数据库中的集合操作对象 collection = None ''' 连接mongodb数据库 ''' def connect(self): user = "******" pwd = "qiangbi123" server = "115.28.161.44" port = '27017' # 表示 用于授权的 db_name = "admin" uri = 'mongodb://' + user + ':' + pwd + '@' + server + ':' + port + '/' + db_name try: self.client = MongoClient(uri) except ServerSelectionTimeoutError as ex: pass ''' 获取数据库操作对象 dbname mxmanage ''' def getdb(self, dbname): self.db = self.client.get_database(dbname) ''' 获取指定集合的操作对象 ''' def getcollection(self, coll_name): self.collection = getattr(self.db, coll_name) ''' 多条数据插入集合操作 ''' def insert(self, data): self.collection.insert(data) ''' 单条数据插入集合 ''' def insertOne(self, data): self.collection.insert_one(data) ''' 集合中数据删除操作 ''' def delete(self): pass ''' 集合中数据更新操作 condition 要更新的查询条件 data 要更新成为什么字段 ''' def updateOne(self, condition, data): # 修改聚集内的记录 # self.collection.update({"UserName": "******"}, {"$set": {'AccountID': random.randint(20, 50)}}) # db.test.update_one({'x': 1}, {'$inc': {'x': 3}}) # print(data) try: self.collection.update_one(condition, data) except Exception as e: pass ''' 查询一个 ''' def findOne(self, condition): try: return self.collection.find_one(condition) except ServerSelectionTimeoutError as ex: return {} except Exception as e: return {} ''' 批量查找 ''' def findMany(self, queue, start, count): # exclude 某个字段 # db.Info.find_one({'$and': [{'$text': {'$search': "Hello"}, 'Username': '******'}]}, # {"Expenses.description": 1, "fullName_normal": 1}) cursor = self.collection.find().skip(start).limit(count) print(start) print(count) print(cursor) try: for data in cursor: queue.put(data) if queue.qsize() > 0: return True else: return False except Exception as ex: # print("failed get data" + ex.message) pass ''' 统计字段信息 ''' def count(self): return self.collection.count() def close(self): self.client.close()
class Connection: _graph_map: dict[str, Connection] = {} _initialized_map: dict[str, bool] = {} def __new__(cls: type[Connection], graph_name: str) -> Connection: if not cls._graph_map.get(graph_name): cls._graph_map[graph_name] = super(Connection, cls).__new__(cls) return cls._graph_map[graph_name] def __init__(self: Connection, graph_name: str) -> None: if self.__class__._initialized_map.get(graph_name): return self._graph_name: str = graph_name self._url: Optional[str] = None self._client: Optional[MongoClient] = None self._database: Optional[Database] = None self._collections: dict[str, Collection] = {} self._connection_callbacks: dict[str, ConnectedCallback] = {} self._connected: bool = False self.__class__._initialized_map[graph_name] = True return None @property def graph_name(self: Connection) -> str: return self._graph_name @property def url(self: Connection) -> str: if self._url: return self._url return self._generate_default_url() def set_url(self: Connection, url: str) -> None: self._url = url def _generate_default_url(self: Connection) -> str: if self.graph_name == 'default': user_url = uconf()['pymongo.url'] or uconf()['pymongo.default.url'] else: user_url = uconf()[f'pymongo.{self.graph_name}.url'] if user_url is not None: self._url = user_url return user_url base = 'mongodb://localhost:27017/' proj = camelize(parameterize(path.basename(getcwd()))).lower() self._url = base + proj return self._url @property def client(self: Connection) -> MongoClient: if self._client is not None: return self._client self.connect() return self._client @property def database(self: Connection) -> Database: if self._database is not None: return self._database self.connect() return self._database def connect(self: Connection) -> None: self._client = MongoClient(self.url) self._database = self._client.get_database() self._connected = True for name, callback in self._connection_callbacks.items(): self._call_callback(name, callback) def disconnect(self: Connection) -> None: if self._client is not None: self._client.close() self._client = None self._database = None self._collections = {} self._connected = False @property def connected(self: Collection) -> bool: return self._connected def collection(self: Connection, name: str, index_keys: list[str] | None = None) -> Collection: if self._collections.get(name) is not None: return self._collections[name] coll = self.database.get_collection(name) if index_keys is not None: ukeys = [(k, 1) for k in index_keys] coll.create_index(ukeys, name='ref', unique=True) self._collections[name] = coll return coll def add_connected_callback(self: Connection, name: str, callback: ConnectedCallback) -> None: self._connection_callbacks[name] = callback if self._client: self._call_callback(name, callback) def _call_callback(self: Connection, name: str, callback: ConnectedCallback) -> None: callback(self.collection(name)) def collection_from(self: Connection, cls: type[T]) -> Collection: coll_name = cls.pconf.collection_name return self.collection(coll_name) default: ClassVar[Connection] @classmethod def get_collection(cls: type[Connection], pmcls: type[T]) -> Collection: graph = pmcls.cdef.jconf.cgraph.name connection = Connection(graph) return connection.collection_from(pmcls) @classmethod def from_class(cls: type[Connection], pmcls: type[T]) -> Connection: return Connection(pmcls.cdef.jconf.cgraph.name)
class Storage: def __init__(self, host, db): self.client = MongoClient(host) self.db = self.client.get_database(db) self.event_data = self.db.get_collection("event_data") def get_event_by_password(self, password): return self.event_data.find_one({"password": password}) def get_event_by_chat_group(self, chat_group): return self.event_data.find_one({"group_chat_id": chat_group}) def set_manager(self, chat_id, password): event = self.get_event_by_password(password) logger.info(f"event {event['manager_id']}") if event["manager_id"] == "": logger.info("manager_id") self.event_data.update({"password": password}, {'$set': { "manager_id": chat_id }}) return True return False def set_user_id(self, event_password, chat_id): event = self.get_event_by_password(event_password) if chat_id not in event["users_chat_id"]: self.event_data.update_one({"password": event_password}, {"$push": { "users_chat_id": chat_id }}) def add_item_to_list(self, chat_id, item): event = self.get_event_by_chat_group(chat_id) if item not in event["items"]: self.event_data.update_one({"group_chat_id": chat_id}, {"$push": { "items": item }}) return True return False def remove_item_from_list(self, chat_id, item): logger.info(f"remove_item_from_list {item}") event = self.get_event_by_chat_group(chat_id) if item in event["items"]: self.event_data.update_one({"group_chat_id": chat_id}, {"$pull": { "items": item }}) return True return False def get_items(self, chat_id): event = self.get_event_by_chat_group(chat_id) return event['items'] def get_items_by_password(self, password): event = self.get_event_by_password(password) return event['items'] def get_taken_items(self, password): event = self.get_event_by_password(password) return event['taken_items'] def get_remaining_items(self, password): logger.info(f"get_remaining_items: {password}") remaining_items = [ item for item in self.get_items_by_password(password) if item not in self.get_taken_items(password) ] logger.info(f"remaining_items: {remaining_items}") return remaining_items def set_taken_item(self, password, item): if item in self.get_remaining_items(password): self.event_data.update_one({"password": password}, {"$push": { "taken_items": item }}) return True return False def set_costs(self, password, cost, chat_id): event = self.get_event_by_password(password) self.event_data.update_one( {"password": password}, {"$push": { "expenses": [str(chat_id), cost] }}) if chat_id not in event["responders"]: self.event_data.update_one({"password": password}, {"$push": { "responders": chat_id }}) def get_manager_id(self, password, chat_id): event = self.get_event_by_password(password) logger.info(f"event[manager_id] {event['manager_id']}") if event["manager_id"] == chat_id: logger.info(f"chat_id {chat_id}") return True return False def get_all_clumsys(self, password): event = self.get_event_by_password(password) logger.info( f"users_chat_id {event['users_chat_id']}, responders: {event['responders']}" ) clumsys = [ clumsy for clumsy in event["users_chat_id"] if clumsy not in event["responders"] ] logger.info(f"clumsys {clumsys}") return clumsys def set_balance(self, password): event = self.get_event_by_password(password) sum = 0 logger.info(event["expenses"]) for i, member in enumerate(event["expenses"]): sum += int(member[1]) logger.info(f"member sum {sum}") avg = sum / len(event["users_chat_id"]) for member in event["users_chat_id"]: sum = 0 for member_ex in event["expenses"]: if member == int(member_ex[0]): sum += int(member_ex[1]) logger.info(f"sum: {sum}") logger.info(f"sum: {sum} +-{avg}") self.event_data.update_one( {"password": password}, {"$push": { "balance": [member, sum - avg] }}) def get_balance(self, password, chat_id): event = self.get_event_by_password(password) for member in event["balance"]: logger.info(member) if member[0] == chat_id: logger.info(member) return member[1] def set_who_paid(self, password, chat_id): event = self.get_event_by_password(password) self.event_data.update_one({"password": password}, {"$push": { "who_paid": chat_id }}) def get_cheaps(self, password): event = self.get_event_by_password(password) cheaps = [ cheap for cheap in event["users_chat_id"] if cheap not in event["who_paid"] ] return cheaps def set_password(self, chat_id, password): logger.info(f"> set_password #{chat_id} #{password}") self.event_data.replace_one({"group_chat_id": chat_id}, { "group_chat_id": chat_id, "manager_id": "", "password": password, "items": [], "taken_items": [], "users_chat_id": [], "expenses": [], "balance": [], "responders": [], "who_paid": [] }, upsert=True)
import random import feedparser import dict_questions from pymongo.mongo_client import MongoClient from datetime import datetime client = MongoClient() db = client.get_database("Teachild") db.drop_collection('Parent') db.drop_collection('Child') db.drop_collection('Task') parent_collection = db.get_collection('Parent') child_collection = db.get_collection('Child') task_collection = db.get_collection('Task') ''' #0-39 dict_questions.dict_questions_math_level1 #0-29 dict_questions.dict_questions_math_level2 ''' NUMBER_Q_IN_TASK = 2 def make_task_for_level(dictt: dict) -> dict: l = [] for i in range(1, NUMBER_Q_IN_TASK + 1): l.append(str(i)) return dict(zip(l, random.choices(dictt, k=NUMBER_Q_IN_TASK)))
import bson, json, datetime from bson.objectid import ObjectId from bson.json_util import loads, dumps jobtemplate = { "url": None, "name": None, "target_commit": None, "target_commit_date": None, "status": "unclaimed", "worker": None, "source": None } dbclient = MongoClient() db = dbclient.get_database(name='ca-core') jobcollection = db.get_collection('jobs') workercollection = db.get_collection('workers') ### workercollection def get_worker_state(workername): try: result = workercollection.find_one({"name": workername}) if result is None: return 404 return result["state"] except Exception as e: log.err(e) raise e
import pandas as pd import lxml.html as html import requests import numpy as np from pymongo.mongo_client import MongoClient from pymongo.collection import Collection client = MongoClient() db = client.get_database("dota") abilities: Collection = db.get_collection("abilities") abilities.update_many({}, { "$unset": { "scepter_pickup_rate": 1, "scepter_win_wo": 1, "scepter_win_w": 1, "shard_pickup_rate": 1, "shard_win_wo": 1, "shard_win_w": 1 } }) format_pct = lambda xs: [ None if x == '-' else float(x.strip().strip('%')) / 100 for x in xs ] res = requests.get("https://windrun.io/ability-shifts") doc = html.fromstring(res.content) ids = [ int(x.split("/")[2]) for x in doc.xpath('//*[@id="ability-shift-stats"]/tbody/tr/td[2]/a/@href') ]
from pymongo.mongo_client import MongoClient client = MongoClient( "mongodb://*****:*****@aws-us-east-1-portal.25.dblayer.com:28209/ascension?ssl=true&ssl_cert_reqs=CERT_NONE&authMechanism=SCRAM-SHA-1" ) db = client.get_database() collection = db['metrics'] cursor = collection.find({}) print(cursor) output = [] for doc in cursor: output.append(doc) print(output)
class TestReadPreference(HATestCase): # Speed up assertReadFrom() when no server is suitable. server_selection_timeout = 0.001 def setUp(self): super(TestReadPreference, self).setUp() members = [ # primary {'tags': {'dc': 'ny', 'name': 'primary'}}, # secondary {'tags': {'dc': 'la', 'name': 'secondary'}, 'priority': 0}, # other_secondary {'tags': {'dc': 'ny', 'name': 'other_secondary'}, 'priority': 0}, ] res = ha_tools.start_replica_set(members) self.seed, self.name = res primary = ha_tools.get_primary() self.primary = partition_node(primary) self.primary_tags = ha_tools.get_tags(primary) # Make sure priority worked self.assertEqual('primary', self.primary_tags['name']) self.primary_dc = {'dc': self.primary_tags['dc']} secondaries = ha_tools.get_secondaries() (secondary, ) = [ s for s in secondaries if ha_tools.get_tags(s)['name'] == 'secondary'] self.secondary = partition_node(secondary) self.secondary_tags = ha_tools.get_tags(secondary) self.secondary_dc = {'dc': self.secondary_tags['dc']} (other_secondary, ) = [ s for s in secondaries if ha_tools.get_tags(s)['name'] == 'other_secondary'] self.other_secondary = partition_node(other_secondary) self.other_secondary_tags = ha_tools.get_tags(other_secondary) self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']} self.c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) self.w = len(self.c.secondaries) + 1 self.db = self.c.get_database("pymongo_test", write_concern=WriteConcern(w=self.w)) self.db.test.delete_many({}) self.db.test.insert_many([{'foo': i} for i in xrange(10)]) self.clear_ping_times() def set_ping_time(self, host, ping_time_seconds): ServerDescription._host_to_round_trip_time[host] = ping_time_seconds def clear_ping_times(self): ServerDescription._host_to_round_trip_time.clear() def test_read_preference(self): # We pass through four states: # # 1. A primary and two secondaries # 2. Primary down # 3. Primary up, one secondary down # 4. Primary up, all secondaries down # # For each state, we verify the behavior of PRIMARY, # PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) wait_until(lambda: c.primary, "discover primary") wait_until(lambda: len(c.secondaries) == 2, "discover secondaries") def assertReadFrom(member, *args, **kwargs): utils.assertReadFrom(self, c, member, *args, **kwargs) def assertReadFromAll(members, *args, **kwargs): utils.assertReadFromAll(self, c, members, *args, **kwargs) def unpartition_node(node): host, port = node return '%s:%s' % (host, port) # To make the code terser, copy hosts into local scope primary = self.primary secondary = self.secondary other_secondary = self.other_secondary bad_tag = {'bad': 'tag'} # 1. THREE MEMBERS UP ------------------------------------------------- # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED # Trivial: mode and tags both match assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc) # Secondary matches but not primary, choose primary assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc) # Chooses primary, ignoring tag sets assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc) # Chooses primary, ignoring tag sets assertReadFrom(primary, PRIMARY_PREFERRED, bad_tag) assertReadFrom(primary, PRIMARY_PREFERRED, [bad_tag, {}]) # SECONDARY assertReadFromAll([secondary, other_secondary], SECONDARY) # SECONDARY_PREFERRED assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED) # Multiple tags assertReadFrom(secondary, SECONDARY_PREFERRED, self.secondary_tags) # Fall back to primary if it's the only one matching the tags assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'primary'}) # No matching secondaries assertReadFrom(primary, SECONDARY_PREFERRED, bad_tag) # Fall back from non-matching tag set to matching set assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED, [bad_tag, {}]) assertReadFrom(other_secondary, SECONDARY_PREFERRED, [bad_tag, {'dc': 'ny'}]) # NEAREST self.clear_ping_times() assertReadFromAll([primary, secondary, other_secondary], NEAREST) assertReadFromAll([primary, other_secondary], NEAREST, [bad_tag, {'dc': 'ny'}]) self.set_ping_time(primary, 0) self.set_ping_time(secondary, .03) # 30 ms self.set_ping_time(other_secondary, 10) # Nearest member, no tags assertReadFrom(primary, NEAREST) # Tags override nearness assertReadFrom(primary, NEAREST, {'name': 'primary'}) assertReadFrom(secondary, NEAREST, self.secondary_dc) # Make secondary fast self.set_ping_time(primary, .03) # 30 ms self.set_ping_time(secondary, 0) assertReadFrom(secondary, NEAREST) # Other secondary fast self.set_ping_time(secondary, 10) self.set_ping_time(other_secondary, 0) assertReadFrom(other_secondary, NEAREST) self.clear_ping_times() assertReadFromAll([primary, other_secondary], NEAREST, [{'dc': 'ny'}]) # 2. PRIMARY DOWN ----------------------------------------------------- killed = ha_tools.kill_primary() # Let monitor notice primary's gone time.sleep(2 * self.heartbeat_frequency) # PRIMARY assertReadFrom(None, PRIMARY) # PRIMARY_PREFERRED # No primary, choose matching secondary assertReadFromAll([secondary, other_secondary], PRIMARY_PREFERRED) assertReadFrom(secondary, PRIMARY_PREFERRED, {'name': 'secondary'}) # No primary or matching secondary assertReadFrom(None, PRIMARY_PREFERRED, bad_tag) # SECONDARY assertReadFromAll([secondary, other_secondary], SECONDARY) # Only primary matches assertReadFrom(None, SECONDARY, {'name': 'primary'}) # No matching secondaries assertReadFrom(None, SECONDARY, bad_tag) # SECONDARY_PREFERRED assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED) # Mode and tags both match assertReadFrom(secondary, SECONDARY_PREFERRED, {'name': 'secondary'}) # NEAREST self.clear_ping_times() assertReadFromAll([secondary, other_secondary], NEAREST) # 3. PRIMARY UP, ONE SECONDARY DOWN ----------------------------------- ha_tools.restart_members([killed]) ha_tools.wait_for_primary() ha_tools.kill_members([unpartition_node(secondary)], 2) time.sleep(5) ha_tools.wait_for_primary() time.sleep(2 * self.heartbeat_frequency) # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED assertReadFrom(primary, PRIMARY_PREFERRED) # SECONDARY assertReadFrom(other_secondary, SECONDARY) assertReadFrom(other_secondary, SECONDARY, self.other_secondary_dc) # Only the down secondary matches assertReadFrom(None, SECONDARY, {'name': 'secondary'}) # SECONDARY_PREFERRED assertReadFrom(other_secondary, SECONDARY_PREFERRED) assertReadFrom( other_secondary, SECONDARY_PREFERRED, self.other_secondary_dc) # The secondary matching the tag is down, use primary assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'}) # NEAREST assertReadFromAll([primary, other_secondary], NEAREST) assertReadFrom(other_secondary, NEAREST, {'name': 'other_secondary'}) assertReadFrom(primary, NEAREST, {'name': 'primary'}) # 4. PRIMARY UP, ALL SECONDARIES DOWN --------------------------------- ha_tools.kill_members([unpartition_node(other_secondary)], 2) # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED assertReadFrom(primary, PRIMARY_PREFERRED) assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc) # SECONDARY assertReadFrom(None, SECONDARY) assertReadFrom(None, SECONDARY, self.other_secondary_dc) assertReadFrom(None, SECONDARY, {'dc': 'ny'}) # SECONDARY_PREFERRED assertReadFrom(primary, SECONDARY_PREFERRED) assertReadFrom(primary, SECONDARY_PREFERRED, self.secondary_dc) assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'}) assertReadFrom(primary, SECONDARY_PREFERRED, {'dc': 'ny'}) # NEAREST assertReadFrom(primary, NEAREST) assertReadFrom(None, NEAREST, self.secondary_dc) assertReadFrom(None, NEAREST, {'name': 'secondary'}) # Even if primary's slow, still read from it self.set_ping_time(primary, 100) assertReadFrom(primary, NEAREST) assertReadFrom(None, NEAREST, self.secondary_dc) self.clear_ping_times()
import sys import datetime from pprint import pprint import feedparser import pymongo from pymongo.mongo_client import MongoClient arg = sys.argv new_rss = feedparser.parse(arg[1]) client = MongoClient() db = client.get_database("my_rss") all_rss = db.get_collection('all_rss') ft = new_rss.feed.title for e in new_rss.entries: tt = e.published_parsed dt = datetime.datetime(*tt[:6], tzinfo=datetime.timezone.utc) if tt else None d = { 'published_parsed': dt, 'title': e.title, 'link': e.link, 'content': e.description, 'feed_title': ft } result = all_rss.replace_one({'link': e.link}, d, upsert=True) print('Feed added.')
def test_secondary_connection(self): self.c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) wait_until(lambda: len(self.c.secondaries), "discover secondary") # Wait for replication... w = len(self.c.secondaries) + 1 db = self.c.get_database("pymongo_test", write_concern=WriteConcern(w=w)) db.test.delete_many({}) db.test.insert_one({'foo': 'bar'}) # Test direct connection to a primary or secondary primary_host, primary_port = ha_tools.get_primary().split(':') primary_port = int(primary_port) (secondary_host, secondary_port) = ha_tools.get_secondaries()[0].split(':') secondary_port = int(secondary_port) arbiter_host, arbiter_port = ha_tools.get_arbiters()[0].split(':') arbiter_port = int(arbiter_port) # MongoClient succeeds no matter the read preference for kwargs in [ {'read_preference': PRIMARY}, {'read_preference': PRIMARY_PREFERRED}, {'read_preference': SECONDARY}, {'read_preference': SECONDARY_PREFERRED}, {'read_preference': NEAREST}, ]: client = MongoClient( primary_host, primary_port, serverSelectionTimeoutMS=self.server_selection_timeout, **kwargs) wait_until(lambda: primary_host == client.host, "connect to primary") self.assertEqual(primary_port, client.port) self.assertTrue(client.is_primary) # Direct connection to primary can be queried with any read pref self.assertTrue(client.pymongo_test.test.find_one()) client = MongoClient( secondary_host, secondary_port, serverSelectionTimeoutMS=self.server_selection_timeout, **kwargs) wait_until(lambda: secondary_host == client.host, "connect to secondary") self.assertEqual(secondary_port, client.port) self.assertFalse(client.is_primary) # Direct connection to secondary can be queried with any read pref # but PRIMARY if kwargs.get('read_preference') != PRIMARY: self.assertTrue(client.pymongo_test.test.find_one()) else: self.assertRaises( AutoReconnect, client.pymongo_test.test.find_one) # Since an attempt at an acknowledged write to a secondary from a # direct connection raises AutoReconnect('not master'), MongoClient # should do the same for unacknowledged writes. try: client.get_database( "pymongo_test", write_concern=WriteConcern(w=0)).test.insert_one({}) except AutoReconnect as e: self.assertEqual('not master', e.args[0]) else: self.fail( 'Unacknowledged insert into secondary client %s should' 'have raised exception' % (client,)) # Test direct connection to an arbiter client = MongoClient( arbiter_host, arbiter_port, serverSelectionTimeoutMS=self.server_selection_timeout, **kwargs) wait_until(lambda: arbiter_host == client.host, "connect to arbiter") self.assertEqual(arbiter_port, client.port) self.assertFalse(client.is_primary) # See explanation above try: client.get_database( "pymongo_test", write_concern=WriteConcern(w=0)).test.insert_one({}) except AutoReconnect as e: self.assertEqual('not master', e.args[0]) else: self.fail( 'Unacknowledged insert into arbiter client %s should' 'have raised exception' % (client,))
def worker(workerthread, numofsymbols): try: #Create an initial value for each security last_value = [] for i in range(0, numofsymbols): last_value.append(round(random.uniform(1, 100), 2)) #Wait until MongoDB Server is online and ready for data while True: print('Checking MongoDB Connection') if checkmongodbconnection() == False: print('Problem connecting to MongoDB, sleeping 10 seconds') time.sleep(10) else: break print('Successfully connected to MongoDB') c = MongoClient(MONGO_URI, server_api=ServerApi("1", strict=False)) db = c.get_database(name=args.database) txtime = dt.now() txtime_end = txtime + timedelta(minutes=args.duration) if args.drop: print('\nDropping collection ' + args.collection + '\n') db.drop_collection(args.collection) if args.timeseries: collection = db.create_collection(name=args.collection, timeseries={ "timeField": "tx_time", "granularity": "seconds" }) print('Create collection result=' + collection.full_name) print( 'Data Generation Summary:\n{:<12} {:<12}\n{:<12} {:<12}\n{:<12} {:<12}' .format('# symbols', args.symbols, 'Database', args.database, 'Collection', args.collection)) print('\n{:<8} {:<50}'.format('Symbol', 'Company Name')) for x in range(len(company_name)): print('{:<8} {:<50}'.format(company_symbol[x], company_name[x])) print('\n{:<12} {:<12}'.format('Start time', txtime.strftime('%Y-%m-%d %H:%M:%S'))) if args.duration: print('{:<12} {:<12}\n'.format( 'End time', txtime_end.strftime('%Y-%m-%d %H:%M:%S'))) else: print('No end time - run until user stop (control-Z)\n\n') counter = 0 bContinue = True while bContinue: for i in range(0, numofsymbols): #Get the last value of this particular security x = getvalue(last_value[i]) last_value[i] = x try: if args.AsString: result = db[args.collection].insert_one({ 'company_symbol': company_symbol[i], 'company_name': company_name[i], 'price': x, 'tx_time': txtime.strftime('%Y-%m-%dT%H:%M:%SZ') }) else: result = db[args.collection].insert_one({ 'company_symbol': company_symbol[i], 'company_name': company_name[i], 'price': x, 'tx_time': txtime }) counter = counter + 1 if counter % 100 == 0: if args.duration > 0: print('Generated ' + str(counter) + ' samples ({0:.0%})'.format( counter / (numofsymbols * args.duration * 60))) else: print('Generated ' + str(counter)) if args.duration > 0: if txtime > txtime_end: bContinue = False continue except Exception as e: print("error: " + str(e)) txtime += timedelta(seconds=1) duration = txtime - dt.now() print('\nFinished - ' + str(duration).split('.')[0]) except: print('Unexpected error:', sys.exc_info()[0]) raise
def test_ship_of_theseus(self): c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) db = c.get_database( "pymongo_test", write_concern=WriteConcern(w=len(c.secondaries) + 1)) db.test.insert_one({}) find_one = db.test.find_one primary = ha_tools.get_primary() secondary1 = ha_tools.get_random_secondary() new_hosts = [] for i in range(3): new_hosts.append(ha_tools.add_member()) # RS closes all connections after reconfig. for j in xrange(30): try: if ha_tools.get_primary(): break except (ConnectionFailure, OperationFailure): pass time.sleep(1) else: self.fail("Couldn't recover from reconfig") # Wait for new members to join. for _ in xrange(120): if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4: break time.sleep(1) else: self.fail("New secondaries didn't join") ha_tools.kill_members([primary, secondary1], 9) time.sleep(5) wait_until(lambda: (ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 2), "fail over", timeout=30) time.sleep(2 * self.heartbeat_frequency) # No error. find_one() find_one(read_preference=SECONDARY) # All members down. ha_tools.kill_members(new_hosts, 9) self.assertRaises( ConnectionFailure, find_one, read_preference=SECONDARY) ha_tools.restart_members(new_hosts) # Should be able to reconnect to set even though original seed # list is useless. Use SECONDARY so we don't have to wait for # the election, merely for the client to detect members are up. time.sleep(2 * self.heartbeat_frequency) find_one(read_preference=SECONDARY) # Kill new members and switch back to original two members. ha_tools.kill_members(new_hosts, 9) self.assertRaises( ConnectionFailure, find_one, read_preference=SECONDARY) ha_tools.restart_members([primary, secondary1]) # Wait for members to figure out they're secondaries. wait_until(lambda: len(ha_tools.get_secondaries()) == 2, "detect two secondaries", timeout=30) # Should be able to reconnect to set again. time.sleep(2 * self.heartbeat_frequency) find_one(read_preference=SECONDARY)