def drop_and_reset_database(dbName):
db = MongoClient("mongodb://localhost:27017/")[dbName]
db.command("dropDatabase")
db.prot.create_indexes(indexes_prot_gene)
db.gene.create_indexes(indexes_prot_gene)
db.goall.create_indexes(indexes_goall)
class EventRepository(object):
def __init__(self):
uri = os.environ.get('MONGO_URI')
self.db = MongoClient(uri).unity
def find_all(self, selector):
return self.db.events.find(selector)
def find(self, selector):
return self.db.events.find_one(selector)
def create(self, event):
return self.db.events.insert_one(event)
def update(self, selector, event):
return self.db.events.replace_one(selector, event).modified_count
def delete(self, selector):
return self.db.events.delete_one(selector).deleted_count
def status(self):
report = self.db.command('dbstats')
server_status = self.db.command('serverStatus')
events = self.db.command('collStats', 'events')
opcounters = server_status['opcounters']
parsedReport = {
'ok': report['ok'],
'database': str(report['db']),
'collections': {
'events': {
'count': events['count'],
'size': humanbytes(events['size'])
}
},
'objects': report['objects'],
'indexes': report['indexes'],
'metrics': {
'connections': server_status['connections'],
'opcounters': {
'insert': opcounters['insert'],
'query': opcounters['query'],
'update': opcounters['update'],
'delete': opcounters['delete']
},
'usage': {
'ratio':
format(report['dataSize'] / report['storageSize'], ".2%"),
'used':
humanbytes(report['dataSize']),
'total':
humanbytes(report['storageSize'])
}
},
}
return parsedReport
def mgrant_server():
# TODO: This is whacked code that starts a mongo server. How do we fix this?
_, config_path = tempfile.mkstemp()
_, mdlogpath = tempfile.mkstemp()
mdpath = tempfile.mkdtemp()
mdport = 27020
if not os.getenv("CONTINUOUS_INTEGRATION"):
basecmd = (
f"mongod --port {mdport} --dbpath {mdpath} --quiet --logpath {mdlogpath} "
"--bind_ip_all --auth")
mongod_process = subprocess.Popen(basecmd,
shell=True,
start_new_session=True)
time.sleep(5)
client = MongoClient(port=mdport)
client.admin.command("createUser",
"mongoadmin",
pwd="mongoadminpass",
roles=["root"])
client.close()
else:
pytest.skip("Disabling mongogrant tests on CI for now")
dbname = "test_" + uuid4().hex
db = MongoClient(
f"mongodb://*****:*****@127.0.0.1:{mdport}/admin"
)[dbname]
db.command("createUser", "reader", pwd="readerpass", roles=["read"])
db.command("createUser", "writer", pwd="writerpass", roles=["readWrite"])
db.client.close()
# Yields the fixture to use
yield config_path, mdport, dbname
if not (os.getenv("CONTINUOUS_INTEGRATION") and os.getenv("TRAVIS")):
os.killpg(os.getpgid(mongod_process.pid), signal.SIGTERM)
os.waitpid(mongod_process.pid, 0)
os.remove(config_path)
shutil.rmtree(mdpath)
os.remove(mdlogpath)
def run(self, count, limit=10):
db = MongoClient(self.mongo_url) \
.get_database(self.db_name)
# .get_collection(self.collection_name)
doc_count = 0
for ordinal in range(0, count):
query = self._generate_query()
cmd = self.create_command(ordinal, self.collection_name, query,
self.sort_spec, limit)
for d in db.command(cmd):
doc_count = doc_count + 1
logging.info((ordinal, doc_count, cmd['filter'],
cmd.get('sort', '_unsorted_')))
return doc_count
}
}
collection = 'Userinformation'
validator = {'$jsonSchema': {'bsonType': 'object', 'properties': {}}}
required = []
for field_key in user_schema:
field = user_schema[field_key]
properties = {'bsonType': field['type']}
minimum = field.get('minlength')
if type(minimum) == int:
properties['minimum'] = minimum
if field.get('required') is True: required.append(field_key)
validator['$jsonSchema']['properties'][field_key] = properties
if len(required) > 0:
validator['$jsonSchema']['required'] = required
query = [('collMod', collection), ('validator', validator)]
try:
db.create_collection(collection)
except CollectionInvalid:
pass
command_result = db.command(OrderedDict(query))
class SN_OneCollection(SocialNetwork):
"""
Simplest implementation of SocialNetwork:
1. All data (posts and comments) are in one collection
2. No indexes
3. Write Concern can be reset (https://docs.mongodb.com/manual/reference/write-concern/)
"""
favorites_id = [] ## This is list of most popular posts which will be read, commented and upvoted further
favorites_id_update_cycle = itertools.cycle([])
favorites_id_read_cycle = itertools.cycle([])
def __init__(self, connection_string, db_name):
self.db = MongoClient(connection_string)[db_name]
self._WriteConcern = { "w": 1, "j": True, "wtimeout": 10000 }
def sort_favorites(self, count = 1000):
SN_OneCollection.favorites_id = list(self.db.posts.aggregate([ { '$sample': { 'size': count } }, {'$project':{'_id':1}} ]))
SN_OneCollection.favorites_id_update_cycle = itertools.cycle(SN_OneCollection.favorites_id)
SN_OneCollection.favorites_id_read_cycle = itertools.cycle(SN_OneCollection.favorites_id)
def prepare(self):
self.sort_favorites()
# index creation to go here
@property
def WriteConcern(self):
return self._WriteConcern
@WriteConcern.setter
def WriteConcern(self, wc):
self._WriteConcern = wc
def get_doc_id_for_update(self):
if len(SN_OneCollection.favorites_id) == 0 :
self.sort_favorites()
return next(SN_OneCollection.favorites_id_update_cycle)
def get_doc_id_for_read(self):
if len(SN_OneCollection.favorites_id) == 0 :
self.sort_favorites()
return next(SN_OneCollection.favorites_id_read_cycle)
def post(self, score = 0, text = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."):
self.db.posts.insert_one({
"content": text,
"score": score,
"comments": [],
})
def _inflate_to(self, M, max_string_len=len(randomText())):
"""
Make full join of posts to posts and (not very) randomly combine content.
This will create more different 'words' for future experiments with indexes and text search.
"""
N = ceil(sqrt(M))
limit = {"$limit": N}
lookup = {
"$lookup":{
"from": "posts",
"localField": "score",
"foreignField": "score",
"as": "joined"
}
}
unwinding = {
"$unwind":
{
"path": "$joined"
}
}
split = randint(0, max_string_len-1)
project = {
'$project': {
'_id':0,
'score': 1,
'comments': 1,
'content': { '$concat': [ { '$substr': [ '$content', 0, split ] },
{ '$substr': [ '$joined.content' , split, max_string_len-split] }]
}
}
}
self.db.posts.aggregate([limit, lookup, unwinding, project, {'$limit': M}, { '$out' : "posts" }])
def generate_data(self, num):
"""
Used for initial data generation.
"""
self.db.posts.insert_many([{
"content": randomText(),
"score": 1, # this need to be constant for self._inflate_to() to work properly
"comments": [],
} for _ in range(min(10, num))]) ## Insert up to 10 initial posts
## Inflate exponentially until required number of posts
while(self.db.command("collstats", "posts")['count'] < num):
self._inflate_to(num)
self.sort_favorites()
def comment(self, text = "What is it? Twitter for ants?"):
doc_id = self.get_doc_id_for_update()
self.db.posts.update_one( doc_id, {"$push":{"comments": text}})
def upvote(self):
doc_id = self.get_doc_id_for_update()
self.db.posts.update_one( doc_id, {"$inc":{"score":1}})
#raise NotImplementedError
def read(self):
doc_id = self.get_doc_id_for_read()
return self.db.posts.find_one(doc_id)
class MongoDBPersister(object):
def __init__(self,
conn_url='localhost:27017',
dbname='skunkqueue'):
self.conn_url=conn_url
self.dbname=dbname
self.skunkdb = MongoClient(conn_url)[dbname]
self.access_collection = self.skunkdb['access']
self.jobs_collection = self.skunkdb['jobs']
self.worker_collection = self.skunkdb['workers']
self.result_collection = self.skunkdb['result']
### Basic information ###
def get_location(self):
return self.conn_url + '/' + self.dbname
def get_backend_type(self):
return 'mongodb'
def get_version(self):
return self.skunkdb.command({'buildInfo': 1})['version']
### Queue manipulation ###
def get_all_queues(self):
return self.access_collection.distinct('q')
def route_is_empty(self, queue_name, route):
ret = False
try:
res = self.access_collection.find_and_modify(
{'q': queue_name, 'locked': False},
update={'$set': {'locked': True}})
if res:
ret = self.jobs_collection.find({
'q': queue_name,
'route': route,
'now': {'$lte': datetime.utcnow()}
}).count() == 0
finally:
self.access_collection.update({'q': queue_name},
{'$set': {'locked': False}})
return ret
### Result access ###
def job_state(self, job_id):
ret = self.result_collection.find_one({'job_id': job_id})
if ret:
return ret['state']
else:
return 'pending'
def job_result(self, job_id):
ret = self.result_collection.find_one({'job_id': job_id})
if ret:
return ret['value']
def save_result(self, job_id, value, state):
self.result_collection.insert(
{'job_id': job_id, 'value': value, 'state': state})
### Job manipulation ###
def add_job_to_queue(self, job, route, ts=None):
queue_name = job.queue.name
job.job_id = str(ObjectId())
self.access_collection.find_and_modify(
{'q': queue_name}, {'q': queue_name, 'locked': False}, upsert=True)
job_flat = job.json()
# ts should be a datetime.timedelta object
if ts:
job_flat['now'] = datetime.utcnow() + ts
else:
job_flat['now'] = datetime.utcnow()
job_flat['route'] = route
if job.queue.queue_type == 'broadcast':
for worker in self.worker_collection.find():
job_flat['q'] = worker['worker_id']
job_flat['_id'] = ObjectId()
self.jobs_collection.insert(job_flat)
else:
self.jobs_collection.insert(job_flat)
"""Pop the first job in the queue that is schedule for work."""
def get_job_from_queue(self, queue_name, worker_id, route):
try:
res = self.access_collection.find_and_modify(
{'q': queue_name, 'locked': False},
update={'$set': {'locked': True}})
if res:
job = self.jobs_collection.find_and_modify(
{'$or': [{'q': queue_name},{'q': worker_id}],
'now': {'$lte': datetime.utcnow()},
'route': route},
remove=True, sort=[('now', -1)])
return job
finally:
self.access_collection.update({'q': queue_name},
{'$set': {'locked': False}})
def get_jobs_by_queue(self, queue):
return [c for c in self.jobs_collection.find({'q': queue})]
"""Delete a specific job from somewhere in the queue."""
def dequeue_job(self, queue_name, job_id):
try:
res = None
# this call should block on queue availability
while not res:
res = self.access_collection.find_and_modify(
{'q': queue_name, 'locked': False},
update={'$set': {'locked': True}})
time.sleep(0.1)
self.jobs_collection.remove({
'q': queue_name,
'job_id': job_id
})
finally:
self.access_collection.update({'q': queue_name},
{'$set': {'locked': False}})
### Worker manipulation ###
def add_worker(self, worker_id, host, port):
self.worker_collection.insert({
'worker_id': worker_id,
'host': host,
'port': port,
'state': 'waiting'
})
def delete_worker(self, worker_id):
self.worker_collection.remove(dict(worker_id=worker_id))
def add_monitor(self, host):
if not self.worker_collection.find_one({'monitor': 1, 'host': host}):
self.worker_collection.insert({
'host': host,
'monitor': 1
})
return True
return False
def delete_monitor(self, host):
self.worker_collection.remove({'monitor': 1, 'host': host})
def set_working(self, worker_id):
self.worker_collection.update(
dict(worker_id=worker_id),
{'$set': {
'state': 'working',
'start': datetime.utcnow()
}}, True)
def unset_working(self, worker_id):
self.worker_collection.update(
dict(worker_id=worker_id),
{'$set': {
'state': 'waiting',
}}, True)
def get_all_workers(self):
return [w for w in self.worker_collection.find()]
print trial
min_distance = 0.0
last_ids = []
seen_ids = set()
for skip in range(0, 10000, 100):
assert len(seen_ids) == skip, 'Skip = %d, got %d ids' % (
skip, len(seen_ids))
start = time.time()
result = db.command(
'geoNear', 'collection',
near={
'type': 'Point',
'coordinates': [
-73.991084,
40.735863]},
spherical=True,
minDistance=min_distance,
query={
'_id': {'$nin': last_ids}
},
num=100)
duration = time.time() - start
# if trial == 0:
# print('skip: %d' % skip)
# print('duration: %.2fms' % (duration * 1000))
# print('minDistance: %.2fms' % min_distance)
# pprint.pprint(result['stats'])
assert len(result['results']) == 100
import pymongo
from pymongo import MongoClient
from bson.son import SON
db = MongoClient().aggregation_example
#result = db.things.insert_many([{"x": 1, "tags": ["dog", "cat"]},
# {"x": 2, "tags": ["cat"]},
# {"x": 2, "tags": ["mouse", "cat", "dog"]},
# {"x": 3, "tags": []}])
#result.inserted_ids
pipeline = [
{"$unwind": "$tags"},
{"$group": {"_id": "$tags", "count": {"$sum": 1}}},
{"$sort": SON([("count", -1), ("_id", -1)])}
]
print ("List:", list(db.things.aggregate(pipeline)))
print ("plan for this aggregation:", db.command('aggregate', 'things', pipeline=pipeline, explain=True))
def install(args):
if sys.platform == "linux" or 1 == 1:
if step_enabled("dependencies", args):
# install MongoDB and some other things if they're not installed
try:
call(["cherrydoor-install"], shell=False) # nosec
except (PermissionError, FileNotFoundError):
print("unable to install dependencies")
if args.fail:
sys.exit(1)
# generate a configuration based on default config
if (
not os.path.exists(f"{Path.home()}/.config/cherrydoor/config.json")
or "config" in args.install_steps
):
config = {
"__comment__": "This is a default config for setuptools installation - it shouldn't be used if installed from GitHub",
"host": "127.0.0.1",
"port": 5000,
"mongo": {
"url": "localhost:27017",
"name": "cherrydoor",
"username": "******",
"password": "******",
},
"login-translation": {
"username": "******",
"password": "******",
"remember-me": "Pamiętaj mnie",
"log-in": "Zaloguj się",
"message": "Musisz się zalogować by uzyskać dostęp do tej strony",
},
"secret-key": "\\xd7w7\\x04\\r\\xfc/q\\x1a\\x9b&",
"https": {
"enabled": False,
"hsts-enabled": False,
"hsts-preload": False,
},
"interface": {
"type": "serial",
"baudrate": 115200,
"port": "/dev/serial0",
"encoding": "utf-8",
},
"manufacturer-code": "18",
}
else:
with open(
f"{Path.home()}/.config/cherrydoor/config.json", "r", encoding="utf-8"
) as f:
config = json.load(f)
if step_enabled("config", args):
# create a random secret key
config["secret-key"] = os.urandom(24).hex()
# let user choose a password for the database
if step_enabled("database", args):
config["mongo"]["password"] = getpass("Wprowadź hasło do bazy danych: ")
try:
# files configuration
if not os.path.exists(f"${Path.home()}"):
os.makedirs(f"{Path.home()}/.config/cherrydoor")
with open(
f"{Path.home()}/.config/cherrydoor/config.json",
"w",
encoding="utf-8",
) as f:
json.dump(config, f, ensure_ascii=False, indent=4)
except (IOError, PermissionError):
print(
f"Nie udało się stworzyć plików w {Path.home()}/.config/cherrydoor. Spróbuj stworzyć ten folder manualnie i nadać mu właściwe uprawnienia",
file=sys.stderr,
)
if args.fail:
sys.exit(1)
if step_enabled("service", args):
service_config = f"""\
[Unit]
Description=Cherrydoor Service
After=network.target
[Service]
ExecStart={os.path.realpath(__file__).replace("install.py", "__init__.py")} start
Environment=PYTHONUNBUFFERED=1
Restart=always
Type=simple
User=ubuntu
[Install]
WantedBy=multi-user.target
"""
try:
if not os.path.exists(f"{Path.home()}/.config/systemd/user"):
os.makedirs(f"{Path.home()}/.config/systemd/user")
with open(
f"{Path.home()}/.config/systemd/user/cherrydoor.service", "w"
) as f:
f.write(service_config)
print(
f"Plik konfiguracyjny znajduje się w folderze {Path.home()}/.config/cherrydoor"
)
except (IOError, PermissionError):
print(
f"Nie udało się stworzyć pliku usługi pod {Path.home()}/.config/systemd/user/cherrydoor.service - spróbuj uruchomić skrypt z właściwymi uprawnieniami lub stworzyć ten plik manualnie. Zawartość:",
file=sys.stderr,
)
print(service_config, file=sys.stderr)
if args.fail:
sys.exit(1)
hasher = PasswordHasher(
time_cost=4,
memory_cost=65536,
parallelism=8,
hash_len=16,
salt_len=16,
encoding="utf-8",
)
db = MongoClient(
f"mongodb://{config['mongo']['url']}/{config['mongo']['name']}"
)[config["mongo"]["name"]]
if step_enabled("database", args):
try:
db.command(
"createUser",
config["mongo"]["username"],
pwd=config["mongo"]["password"],
roles=[
{"role": "readWrite", "db": config["mongo"]["name"]},
{"role": "clusterMonitor", "db": "admin"},
],
)
db.create_collection("users")
db.create_collection(
"logs", options={"size": 1073742000, "capped": True}
)
db.create_collection("settings")
db.create_collection(
"terminal", options={"size": 1048576, "capped": True, "max": 10000}
)
except OperationFailure:
pass
user_indexes = db.users.index_information()
if "username_index" not in user_indexes.keys():
db.users.create_index("username", name="username_index", unique=True)
if "cards_index" not in user_indexes.keys():
db.users.create_index("cards", name="cards_index", sparse=True)
# nosec - it's python3, not 2, Bandit...
if step_enabled("user", args) and input(
"Czy chcesz stworzć nowego użytkownika-administratora? [y/n]"
).lower() in ["y", "yes", "tak", "t"]:
# nosec - it's python3, not 2, Bandit...
username = input("Wprowadź nazwę użytkownika: ")
password = hasher.hash(getpass("Hasło: "))
db.users.insert({"username": username, "password": password, "cards": []})
print("Instalacja skończona!")
try:
service_call_args = ["systemctl", "--user", "enable", "cherrydoor"]
call(service_call_args, shell=False) # nosec
except (IOError, PermissionError):
pass
else:
print("Ten system operacyjny nie jest obecnie obsługiwany")
sys.exit()
class Database:
def __init__(self, database_url: str, database_name: str = "username601"):
""" Database object. """
self.db = MongoClient(database_url)[database_name]
self.types = ModifyType
self.database_calls = {
'modify': {},
'eval': 0,
'add': 0,
'delete': 0,
'get': 0
}
del database_url # for safety purposes lel
def modify(self, collection_name: str, modify_type: ModifyType,
query: dict, payload: dict): # resembles a POST request
"""
Updates/modifies a single part of the database.
Example:
db.modify("economy", db.types.CHANGE, {"key_name": "key_value"}, {"key_name_to_change": "new_value"})
"""
self.database_calls['modify'][
modify_type.value] = self.database_calls['modify'].get(
modify_type.value, 0) + 1
self.db[collection_name].update_one(query,
{modify_type.value: payload})
return payload
def eval(self, command: str):
""" Evaluates stuff """
self.database_calls['eval'] += 1
try:
return self.db.command(command)
except:
return
def add(self, collection_name: str,
new_data: dict): # resembles a PUT request
""" Adds a document to the database. """
self.database_calls['add'] += 1
self.db[collection_name].insert_one(new_data)
return new_data
def delete(self, collection_name: str,
query: dict): # resembles a DELETE request
""" Removes a document from the database. """
self.database_calls['delete'] += 1
self.db[collection_name].delete_one(query)
def get(self, collection_name: str,
query: dict) -> dict: # resembles a GET request
""" Fetches a document from the database. """
self.database_calls['get'] += 1
return self.db[collection_name].find_one(query)
def exist(self, collection_name: str,
query: dict) -> bool: # resembles a GET request (sort of)
""" Checks if a specific query exists. """
return (self.get(collection_name, query) is not None)
def get_all(
self, collection_name: str
) -> list: # resembles a hell of a big GET request
""" Gets every single document from a specific database collection. """
return self.db[collection_name].find()
class MongoDBPersister(object):
def __init__(self, conn_url='localhost:27017', dbname='fwbots'):
self.conn_url = conn_url
self.dbname = dbname
self.fwdb = MongoClient(conn_url)[dbname]
self.worker_collection = self.fwdb['workers']
self.pool_collection = self.fwdb['pool']
### Basic information ###
def get_location(self):
return self.conn_url + '/' + self.dbname
def get_backend_type(self):
return 'mongodb'
def get_version(self):
return self.fwdb.command({'buildInfo': 1})['version']
### Pool manipulation ###
def add_pool(self, pool):
obj = {
'name': pool.name,
'manual': pool.manual.keys(),
'auto': pool.auto.keys(),
'insta': pool.insta.keys()
}
self.pool_collection.insert(obj)
def get_all_pools(self):
res = {}
for p in self.pool_collection.find():
res[p['name']] = p
return res
def delete_pool(self, name):
self.pool_collection.remove({'name': name})
### Worker manipulation ###
def add_worker(self, worker_id, host, port):
self.worker_collection.insert({
'worker_id': worker_id,
'host': host,
'port': port,
'state': 'waiting'
})
def delete_worker(self, worker_id):
self.worker_collection.remove(dict(worker_id=worker_id))
def delete_monitor(self, host):
self.worker_collection.remove({'monitor': 1, 'host': host})
def set_working(self, worker_id):
self.worker_collection.update(
dict(worker_id=worker_id),
{'$set': {
'state': 'working',
'start': datetime.utcnow()
}}, True)
def unset_working(self, worker_id):
self.worker_collection.update(dict(worker_id=worker_id),
{'$set': {
'state': 'waiting',
}}, True)
def get_all_workers(self):
return [w for w in self.worker_collection.find()]
def get_avail_workers(self):
return [w for w in self.worker_collection.find({'state': 'waiting'})]
## This script is to remove additional datasets from S3 that don't exist in Mongo
import boto3
import botocore
from pymongo import MongoClient
from pprint import pprint
from bson.objectid import ObjectId
import yaml
creds = yaml.safe_load(open("../creds.yaml", "r"))
dbHostUri = "mongodb+srv://" + creds["DB_USER"] + ":" + creds[
"DB_PASSWORD"] + "@cluster0-ollas.mongodb.net/test?retryWrites=true&w=majority&ssl=true&ssl_cert_reqs=CERT_NONE"
db = MongoClient(dbHostUri).test
serverStatusResult = db.command("serverStatus")
# pprint(serverStatusResult)
s3 = boto3.resource('s3')
bucketName = "agriworks-user-datasets"
bucket = s3.Bucket(bucketName)
s3Length = sum(1 for i in bucket.objects.all())
print('s3 Size =', s3Length)
for key in bucket.objects.all():
if not db.dataset.find_one({'_id': ObjectId(key.key.split('.')[0])}):
print(key.key)
obj = s3.Object(bucketName, key.key)
obj.delete()
print('________DELETED__________')
'type': 'text',
'analyzer': 'ik_max_word',
'search_analyzer': 'ik_max_word',
},
'en': {
'type': 'text',
'analyzer': 'english',
}
}
},
}
}
}
}
total = db.command('collstats', 'Movies')['count']
docs = db['Movies'].find({},
projection={
'_id': False,
'cover_x': False,
'cover_y': False,
'cover': False,
'star': False,
})
def clean_movie(m):
parenthesis_pat = re.compile(r'\(.*?\)')
release_date = '/'.join(m['release_date'])
release_date = parenthesis_pat.sub('', release_date)
m['release_date'] = release_date.split('/')
class DataBase:
def __init__(self, db_name):
# 设置登录IP，密码，数据库ip，数据库名
usr = '******'
passwd = 'xxxxxxxx'
ip = 'xxx.xxx.xxx.xxx'
# 建立连接
self.db = MongoClient('mongodb://%s:%s@%s' % (usr, passwd, ip))[db_name]
self.fs = gridfs.GridFS(self.db)
self.db_name = db_name
# 向表插入数据
# 注意：data原本的格式必须是json格式
def _insert(self, col_name, data):
# 转换符号
if data:
data = data.replace("'", '"').replace("true", "True").replace("false", "False").replace("None", "null")
order = '{"time": "%s", "data": %s}' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), str(data))
order = order.encode('utf-8') # 编码
up = self.fs.put(data=order, filename=col_name) # 上传
print(up)
print('inserted')
# 请求数据
def _query(self, col_name):
print('query begin')
print(col_name)
print(self.db_name)
down = self.fs.get_version(col_name).read() # 下载
down = json.loads(down.decode("utf-8").replace('None', 'null'), strict=False) # 转化为json格式
return down
# 查询数据库，返回数据库中保存的所有数据名
def _check(self):
print('check begin')
col_names = []
# 提取所有文件名
for grid_out in self.db['fs.files'].find({}, {"_id": 0, "filename": 1}): # 数据库中的fs.files表存储了文件名信息
col_names.append(grid_out['filename'])
# 去除重复值
return_list = []
for d in col_names:
if d not in return_list:
return_list.append(d)
print(return_list)
return return_list
# 删除数据库
def _delete(self):
self.db.command("dropDatabase")
print('database %s deleted' % str(self.db_name))
# 自定义数据库检索（格式转换可能转成dict，还存在bug）
def _custom(self, col_name, _request_order):
return_list = []
_request_order = _request_order.replace("'", '"').replace("true", "True").replace("false", "False").replace(
"Null", "none")
_request_order = ast.literal_eval(_request_order)
print(_request_order)
print(type(_request_order))
col = self.db[col_name]
data = col.find(_request_order)
for file in data:
return_list.append(file)
return return_list
def connect_to_mongodb_and_print(ds, **kwargs):
db = MongoClient().zips
buildinfo = db.command("buildinfo")
print(buildinfo)
return 'Whatever you return gets printed in the logs'
"tags": ["mouse", "cat", "dog"]
}, {
"x": 3,
"tags": []
}])
result.inserted_ids
'''
{ "_id" : ObjectId("576aaa973e5269020848cc7c"), "x" : 1, "tags" : [ "dog", "cat" ] }
{ "_id" : ObjectId("576aaa973e5269020848cc7d"), "x" : 2, "tags" : [ "cat" ] }
{ "_id" : ObjectId("576aaa973e5269020848cc7e"), "x" : 2, "tags" : [ "mouse", "cat", "dog" ] }
{ "_id" : ObjectId("576aaa973e5269020848cc7f"), "x" : 3, "tags" : [ ] }
'''
from bson.son import SON
# $unwind 解开-后面的变量
pipeline = [{
"$unwind": "$tags"
}, {
"$group": {
"_id": "$tags",
"count": {
"$sum": 1
}
}
}, {
"$sort": SON([("count", -1), ("_id", -1)])
}]
list(db.things.aggregate(pipeline))
# 使用聚合函数with command
db.command('aggregate', 'things', pipeline=pipeline, explain=True)
class MongoDBPersister(object):
def __init__(self,
conn_url='localhost:27017',
dbname='fwbots'):
self.conn_url=conn_url
self.dbname=dbname
self.fwdb = MongoClient(conn_url)[dbname]
self.worker_collection = self.fwdb['workers']
self.pool_collection = self.fwdb['pool']
### Basic information ###
def get_location(self):
return self.conn_url + '/' + self.dbname
def get_backend_type(self):
return 'mongodb'
def get_version(self):
return self.fwdb.command({'buildInfo': 1})['version']
### Pool manipulation ###
def add_pool(self,pool):
obj = {
'name':pool.name,
'manual':pool.manual.keys(),
'auto':pool.auto.keys(),
'insta':pool.insta.keys()
}
self.pool_collection.insert(obj)
def get_all_pools(self):
res = {}
for p in self.pool_collection.find():
res[p['name']] = p
return res
def delete_pool(self,name):
self.pool_collection.remove({'name':name})
### Worker manipulation ###
def add_worker(self, worker_id, host, port):
self.worker_collection.insert({
'worker_id': worker_id,
'host': host,
'port': port,
'state': 'waiting'
})
def delete_worker(self, worker_id):
self.worker_collection.remove(dict(worker_id=worker_id))
def delete_monitor(self, host):
self.worker_collection.remove({'monitor': 1, 'host': host})
def set_working(self, worker_id):
self.worker_collection.update(
dict(worker_id=worker_id),
{'$set': {
'state': 'working',
'start': datetime.utcnow()
}}, True)
def unset_working(self, worker_id):
self.worker_collection.update(
dict(worker_id=worker_id),
{'$set': {
'state': 'waiting',
}}, True)
def get_all_workers(self):
return [w for w in self.worker_collection.find()]
def get_avail_workers(self):
return [w for w in self.worker_collection.find({'state':'waiting'})]
