def create(self, doc=None, collection=None, db=None):
"""
Insert object(s).
:param doc: data to be inserted.
:param collection: to change collection/table
:param db: to change database
"""
self.cd(collection, db)
count = 0
r = None
c = 204
m = 'Nothing happened.'
try:
ins = None
if isinstance(doc, dict):
ins = self.collection.insert_one(doc)
r = self._decode_objectid(ins.inserted_id)
count = 1 if r else 0
elif isinstance(doc, list):
ins = self.collection.insert_many(doc)
r = ins.inserted_ids
count = len(r)
if r and ins:
c = 201
m = 'Data inserted.'
log.info('create_count: {}, create_ack: {}'.format(count, ins.acknowledged))
except Exception as e:
c = 500
m = 'Server Error: {}'.format(e)
return self._response(r, c, m)
def del_files(self, path, files=None, dir_flag=False, fn_pattern=None):
"""
Delete list of files provided.
:param path: directory (only)
:param files: list [] of files
:param fn_pattern: override [files] if provided
:return: True or False
"""
r = False
if self.exists(path):
if fn_pattern:
files = self.ls(path, fn_pattern=fn_pattern, fn_only=True)
if isinstance(files, list):
for fn in files:
try:
os.remove(os.path.join(path, fn))
log.info('deleted: {}/{}'.format(path, fn))
r = True
except Exception as e:
log.error("Couldn't remove file: {}".format(e))
else:
log.debug(
'Expected a list of files or a path with flag "dir=True"')
return r
def update(self, doc=None, collection=None, db=None, where=None, like=None, set=None):
"""
:param doc: new version of database object to update
:param collection: to change collection/table
:param db: to change database
:param where: criteria to locate database object i.e {'city': 'Atlanta}
:param like: use filter with $regex i.e. like={'employe_name': '^Ran'}
:param set: use $set to update field i.e. where={'_id': '5e1ab71ed4a0e6a7bdd5233f'}, set={'employe_name': 'Randoll'}
"""
log.debug('update: {}'.format(doc))
self.cd(collection, db)
r = []
c = 204
m = 'Nothing happened.'
try:
if isinstance(doc, dict):
self._encode_objectid(doc)
obj = self.collection.replace_one({'_id': doc['_id']}, doc)
log.info('update_match_count: {}, update_mod: {}, update_ack: {}'.format(
obj.matched_count, obj.modified_count, obj.acknowledged))
c = 200
m = 'Document(s) updated.'
except Exception as e:
r = doc
c = 500
m = 'Server Error: {}'.format(e)
return self._response(r, c, m)
def main():
"""
* Create backup directory structure.
* Run backup daily.
* Rotate on a daily, weekly, monthly basis.
"""
log.info('Starting mongodb backup script.')
if config:
# -- check directory and structure
path = False
if FileManager.dir_struct(config['backup']['path']):
path = True
# -- backup and rotate files
if path:
run = False
last_run = config['last_run']
if last_run:
log.info('checked last run: {}'.format(last_run))
if not last_run == time.strftime('%Y-%m-%d', m_today):
run = True
else:
log.info('No need to run a backup at this time.')
else:
run = True
if run:
if backup_mongo(config['backup']['databases'],
config['backup']['path'],
config['db-instance']):
log.info('Backup process completed successfully.')
else:
log.info('Backup process completed without errors.')
else:
log.error("Couln't retreive config file")
def _update_session(self, collection=None, db=None):
if collection:
if db:
self.collection = self.connector.db.client[db][collection]
else:
self.collection = self.connector.db[collection]
log.info('Using collection: {}.{}'.format(self.connector.db.name,
self.collection.name))
# bugfix at line 49, previously test wrong db object
def read(self,
where=None,
collection=None,
db=None,
projection=None,
aggr_cols=None,
aggr_type=None,
like=None):
"""
Read <database>.<collection>.
:param where: filter object to look for
:param collection: to change collection/table
:param db: to change database
:param projection: fields to return in response
:param aggr_cols: fields to group by ['name', 'department', 'salary']
:param aggr_type: 'count', 'sum' i.e. aggr_type='count' or aggr_type={'sum': 'salary'}
:param like: use find() with $regex i.e. like={'employe_name': '^Ran'}
"""
self._update_session(collection, db)
r = statement = None
c = 404
m = 'No data returned.'
doc_count = 0
try:
if where:
statement = where
else:
statement = {}
log.info('retrieving doc(s) like: {}'.format(statement))
if isinstance(aggr_cols, list):
if isinstance(aggr_type, dict):
pass # sum
else:
pass # count
data = None
elif isinstance(statement, dict):
data = self.collection.find(statement, projection=projection)
for _ in data:
doc_count += 1
if doc_count > 0:
data.rewind()
r = data
c = 200
m = 'OK'
log.info('data: {} doc(s).'.format(doc_count))
except Exception as e:
r = statement
c = 500
m = 'Server Error: {}'.format(e)
return self._response(r, c, m)
def setbucket(self, dirname):
"""
Add a "bucket" to directory structure.
:param dirname: name to set the subdirectory
"""
if not self.bucket:
self.bucket = str(dirname)
self.dir_struct()
log.info('Bucket is now set to: {}.'.format(self.bucket))
else:
log.info('Buckets cannot be reset to a different name ({}). '
'Currently set to "{}"'.format(dirname, self.bucket))
def cd(self, collection=None, db=None):
"""
Change collection and/or database.
:param collection: collection name to change
:param db: database name to change (collecion is required)
"""
if collection and self.connector.connected:
if db:
self.collection = self.connector.db.client[db][collection]
log.info('Using database: {}'.format(self.connector.db.name))
else:
self.collection = self.connector.db[collection]
log.info('Using collection: {}.{}'.format(self.connector.db.name, self.collection.name))
def do_move(fnum=0):
ret = False
new_fn = newfilename(fnum)
if not self.exists('{}/{}'.format(dst, new_fn)):
if self.exists('{}/{}'.format(src, fn)):
os.rename('{}/{}'.format(src, fn),
'{}/{}'.format(dst, new_fn))
log.info('moved: [{}/{}] to [{}/{}]'.format(
src, fn, dst, new_fn))
ret = True
else:
fnum += 1
ret = do_move(fnum)
return ret
def delete(self, where=None, collection=None, db=None):
"""
Remove document or object in document.
:param statement: document/object to query to remove
:param collection: to change collection/table
:param db: to change database
:example:
delete({'_id': '5e114ad941734d371c5f84b9'})
delete([{'_id': '5e114ad941734d371c5f84b9'}, {'age': 25}])
delete({'person.fname': 'Randoll'} # delete document where {'person': {'fname': 'Randoll'}}
delete({}) # delete all in collection, not allowed
"""
log.info('deleting doc(s) like: {}'.format(where))
self._update_session(collection, db)
r = []
c = 204
m = 'Nothing happened.'
try:
if where:
if isinstance(where, dict):
where = [where]
if isinstance(where, list):
statement = []
for f in where:
if not isinstance(f, dict):
statement += [self._encode_objectid({'_id': f})]
r += [str(f)]
else:
statement += [self._encode_objectid(f)]
for s in statement:
obj = self.collection.delete_one(s)
log.info('delete: ack: {}, delete_count: {}, doc: {}'.
format(obj.acknowledged, obj.deleted_count,
s))
r = statement
c = 410
m = 'Item(s) delete.'
except Exception as e:
r = where
c = 500
m = 'Server Error: {}'.format(e)
return self._response(r, c, m)
def del_dir(self, path):
"""
Delete directory provided.
:param path: directory (only)
:return: True or False
"""
r = False
if self.exists(path):
try:
if self.basedir == '{}/fm'.format(path):
os.rmdir(self.basedir)
if self.bucket and self.bucket in path:
os.rmdir(path)
path = path.replace(self.bucket, '')
os.rmdir(path)
r = True
log.info('remove dir: {}'.format(path))
except Exception as e:
log.error("Couldn't remove directory: {}".format(e))
return r
def __init__(self, db_config=None, collection=None, db=None):
global config
self.client = None
self.db = None
self.collection = None
self.connected = False
if db_config is None:
if os.environ.get('APP_RUNTIME_CONTEXT') == 'dev':
db_config = config['mongo.dev']
self.environ = 'dev'
elif os.environ.get('APP_RUNTIME_CONTEXT') == 'qa':
db_config = config['mongo.qa']
self.environ = 'qa'
else:
db_config = config['mongo.prod']
self.environ = 'prod'
log.info('Using mongo.{} configuration.'.format(self.environ))
else:
log.info('Using db_config provided: {}'.format(db_config))
if db_config:
self.client = MongoClient(
'mongodb+srv://{}:{}@{}/{}?retryWrites=true&w=majority'.format(
db_config['username'], db_config['password'],
db_config['host'], db_config['database']))
# -- setup database
if self.db:
self.db = self.client[db]
else:
if db_config.get('database'):
self.db = self.client[db_config['database']]
else:
self.db = self.client['admin']
# -- setup collection
if collection:
self.collection = self.db[collection]
else:
if db_config.get('collection'):
self.collection = self.db[db_config['collection']]
else:
self.collection = self.db['system.version']
self.connected = True
if self.connected:
log.info('CONNECTED to {}@{}'.format(self.db.name,
db_config['host']))
else:
log.info('NOT CONNECTED. (db={}, host={})'.format(
db_config['database'], db_config['host']))
def create(self, doc=None, collection=None, db=None):
"""
Insert object(s).
:param doc: data to be inserted.
:param collection: to change collection/table
:param db: to change database
"""
log.info('inserting doc: {}'.format(doc))
self._update_session(collection, db)
r = None
c = 204
m = 'Nothing happened.'
try:
ins = None
if isinstance(doc, dict):
ins = self.collection.insert_one(doc)
r = [str(ins.inserted_id)]
log.info('inserted: {}, {}'.format(ins.acknowledged,
ins.inserted_id))
elif isinstance(doc, list):
ins = self.collection.insert_many(doc)
r = [str(i) for i in ins.inserted_ids]
log.info('inserted: {}, {}'.format(ins.acknowledged, r))
if r:
c = 201
m = 'Data inserted.'
except Exception as e:
r = doc
c = 500
m = 'Server Error: {}'.format(e)
return self._response(r, c, m)
def retainer(self, directory, fn, retain):
"""
Function to apply retention policy.
:param directory: base path
:param fn: file names containing substring
:param ret_d: number of files to retain
"""
if retain > 0:
del_list = self._ts_sorted_file('list', directory, \
fn_pattern='.*{}.*'.format(fn))
if len(del_list) > retain:
# -- unpack filename
t = []
for f in del_list:
t += [f[1]]
del_list = t[:len(t) - retain]
log.debug('list of file to delete: {}'.format(del_list))
self.del_files(directory, del_list)
del del_list, t
log.info('applied retention policy: {}'.format(directory))
def backup_mongo(dbs_l, path, conf):
r = False
# -- decide on backup rotation directory
dt_label = time.strftime('%Y-%m-%d', m_today)
b_dir = 'daily'
week_days = {
'monday': 0,
'tuesday': 1,
'wednesday': 2,
'thrusday': 3,
'friday': 4,
'saturday': 5,
'sunday': 6
}
if m_today.tm_wday == week_days[config['backup']['weekly'][
'on']] and config['backup']['weekly']['retention'] > 0:
b_dir = 'weekly'
if m_today.tm_mday == config['backup']['monthly'][
'on'] and config['backup']['monthly']['retention'] > 0:
b_dir = 'monthly'
log.debug('Retention: {}'.format(config['backup'][b_dir]['retention']))
log.debug('Policy: {}'.format(b_dir))
# -- run backup for each database
if dbs_l and (type(dbs_l) is list):
for dbs in dbs_l:
log.info('Database name: {}'.format(dbs))
log.info('creating backup file: {}-{}.gz in {}/{}/'.format(
dbs, dt_label, path, b_dir))
cmd = ['mongodump']
if conf['username'] and conf['password']:
cmd += ['-u', conf['username'], '-p', conf['password']]
cmd += [
'--authenticationDatabase={}'.format(
conf["authenticationDatabase"]),
'--archive={}/{}/{}-{}.gz'.format(path, b_dir, dbs, dt_label),
'--gzip', '--db', dbs
]
log.debug('command: {}'.format(" ".join(cmd)))
out = subprocess.run(cmd,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
log.info(out.stdout)
r = True
# -- apply retention policy
FileManager.retainer(b_dir, dbs)
config['last_run'] = dt_label
config.save()
return r
def dir_struct(self, path=None, known_dir=None):
"""
Check directory structure. If not valid, create structure.
:param path: filesystem full path
:param known_dir: list of dir paths
:return: True or False
"""
r = False
def update_path(_p, _np):
if _p == 'archive':
self.ARCHIVE = _np
elif _p == 'errored':
self.ERRORED = _np
elif _p == 'input':
self.INPUT = _np
elif _p == 'output':
self.OUTPUT = _np
if path:
self.basedir = '{}/fm'.format(path)
u_path = False
if not self.known_dir:
self.known_dir = ['archive', 'errored', 'input', 'output']
u_path = True
if known_dir:
self.known_dir = known_dir
log.info('validate directory structure for: {}'.format(self.basedir))
try:
i = 0
for d in self.known_dir:
n_path = os.path.join(self.basedir, d, self.bucket)
if u_path:
update_path(d, n_path)
if not os.path.exists(n_path):
log.info('creating: {}'.format(n_path))
os.makedirs(n_path, exist_ok=True)
r = True
except Exception as e:
log.info("Couldn't setup directory structure.\n{}".format(e))
return r
def close(self):
if self.status():
self.client.close()
self.connected = False
log.info('DISCONNECTED.')
def __init__(self, db_config=None, collection=None, db=None, collection_obj=None, db_obj=None):
"""
Create database connection.
:param db_config: configuration object map
:param collection: existing pymongo collection object
:param db: existing pymongo db object
"""
self.collection = collection_obj
self.db = db_obj
collection_name = collection; del collection # -- to avoid ambiguity
db_name = db; del db # -- to avoid ambiguity
self.client = None
self.connected = False
# -- validate passing objects
if db_obj and isinstance(db_obj, Database):
self.client = db_obj.client
if db_obj and not isinstance(db_obj, Database):
log.error('db_obj: {} is not a Database object'.format(type(db_obj)))
return
if collection_obj and not isinstance(collection_obj, Collection):
log.error('collection_obj: {} is not a Collection object'.format(type(collection_obj)))
return
# -- database basic config
if db_config is None and not self.db and not self.collection:
if os.environ.get('APP_RUNTIME_CONTEXT') == 'dev':
db_config = config['mongo.dev']
self.environ = 'dev'
elif os.environ.get('APP_RUNTIME_CONTEXT') == 'qa':
db_config = config['mongo.qa']
self.environ = 'qa'
else:
db_config = config['mongo.prod']
self.environ = 'prod'
log.info('Using mongo.{} configuration.'.format(self.environ))
else:
log.info('Using db_config provided: {}'.format(db_config))
if db_config:
self.client = MongoClient(
'mongodb://{}:{}/'.format(db_config['host'], db_config['port']),
username=db_config['username'],
password=db_config['password'],
authSource=db_config['authenticationDatabase'])
# -- setup database
if not db_name and not self.db:
if db_config.get('database'):
self.db = self.client[db_config['database']]
else:
self.db = self.client['test_db']
elif db_name and not self.db:
self.db = self.client[db_name]
# -- setup collection
if not collection_name and self.db:
if db_config and db_config.get('collection'):
self.collection = self.db[db_config['collection']]
elif not self.collection:
self.collection = self.db['test']
elif collection_name and self.db:
self.collection = self.db[collection_name]
if self.status():
if collection_obj and db_obj:
log.info('Using existing connection: {}@{}'.format(self.db.name, self.client.address))
else:
log.info('CONNECTED to {}@{}'.format(self.db.name, self.client.address))
else:
log.info('NOT CONNECTED.')
def read(self, where=None, collection=None, db=None, projection=None, sort=None, aggr_cols=None, aggr_type=None, like=None):
"""
Read <database>.<collection>.
:param where: filter object to look for
:param collection: to change collection/table
:param db: to change database
:param projection: fields to return in response i.e. {'name': true, 'department': true}
:param aggr_cols: fields to group by ['name', 'department', 'salary']
:param aggr_type: 'count', 'sum' i.e. aggr_type={'count': 'salary'} or aggr_type={'sum': 'salary'}
:param like: use find() with $regex i.e. like={'employe_name': '^Ran'}
"""
self.cd(collection, db)
r = None
c = 404
m = 'No data returned.'
doc_count = 0
statement = []
try:
# -- build statement
if where:
self._encode_objectid(where)
for k in where:
if where[k] is None:
statement += [(k, {'$exist': False})]
else:
statement += [(k, where[k])]
else:
statement += [('_id', {'$exists': True})]
# if isinstance(aggr_cols, list):
# if isinstance(aggr_type, dict):
# pass # sum
# else:
# pass # count
log.info('retrieving doc(s) like: {}{}'.format(dict(statement), \
', {}'.format(projection) if projection else ''))
# -- execute statement
if isinstance(sort, dict):
s_list = []
for k in sort:
s_list += [(k, sort[k])]
data = self.collection.find(SON(statement), projection).sort(s_list)
del s_list
else:
data = self.collection.find(SON(statement), projection)
# -- collect result
for _ in data:
doc_count += 1
if doc_count > 0:
data.rewind()
r = data
c = 200
m = 'OK'
log.info('read_count: {}'.format(doc_count))
except Exception as e:
r = statement
c = 500
m = 'Server Error: {}'.format(e)
return self._response(r, c, m)