def update(self, doc=None, collection=None, db=None, where=None, like=None, set=None):
"""
:param doc: new version of database object to update
:param collection: to change collection/table
:param db: to change database
:param where: criteria to locate database object i.e {'city': 'Atlanta}
:param like: use filter with $regex i.e. like={'employe_name': '^Ran'}
:param set: use $set to update field i.e. where={'_id': '5e1ab71ed4a0e6a7bdd5233f'}, set={'employe_name': 'Randoll'}
"""
log.debug('update: {}'.format(doc))
self.cd(collection, db)
r = []
c = 204
m = 'Nothing happened.'
try:
if isinstance(doc, dict):
self._encode_objectid(doc)
obj = self.collection.replace_one({'_id': doc['_id']}, doc)
log.info('update_match_count: {}, update_mod: {}, update_ack: {}'.format(
obj.matched_count, obj.modified_count, obj.acknowledged))
c = 200
m = 'Document(s) updated.'
except Exception as e:
r = doc
c = 500
m = 'Server Error: {}'.format(e)
return self._response(r, c, m)
def del_files(self, path, files=None, dir_flag=False, fn_pattern=None):
"""
Delete list of files provided.
:param path: directory (only)
:param files: list [] of files
:param fn_pattern: override [files] if provided
:return: True or False
"""
r = False
if self.exists(path):
if fn_pattern:
files = self.ls(path, fn_pattern=fn_pattern, fn_only=True)
if isinstance(files, list):
for fn in files:
try:
os.remove(os.path.join(path, fn))
log.info('deleted: {}/{}'.format(path, fn))
r = True
except Exception as e:
log.error("Couldn't remove file: {}".format(e))
else:
log.debug(
'Expected a list of files or a path with flag "dir=True"')
return r
def newfilename(fnum): # -- add <filename>_1[+n]
if fnum == 0:
ret = fn
else:
try:
nfn, ext = fn.split('.')
ret = '{}_{}.{}'.format(nfn, fnum, ext)
except Exception as e:
ret = '{}_{}'.format(fn, fnum)
log.debug('Error: {}'.format(e))
return ret
def test_touch(_g):
fm = _g['fm']
pwd = _g['pwd']
log.debug(f"{fm.bucket}")
fm.touch(f"{pwd}/fm/test1/result.1")
fm.touch(f"{pwd}/fm/test1/result.2")
fm.touch(f"{pwd}/fm/test1/result.3")
fm.touch(f"{pwd}/fm/test1/result.4")
files_list = [x[1] for x in fm.ls(f"{pwd}/fm/test1")]
assert files_list == ['result.1', 'result.2', 'result.3', 'result.4']
def test_del_dir(_g):
fm = _g['fm']
pwd = _g['pwd']
assert not fm.del_dir(f"{pwd}/fm/test1")
assert fm.del_files(f"{pwd}/fm/test1", fn_pattern='result.*')
assert fm.del_dir(f"{pwd}/fm/test3")
assert fm.del_dir(f"{pwd}/fm/test2")
assert fm.del_dir(f"{pwd}/fm/test1")
log.debug(fm.ls(pwd))
assert fm.del_dir(f"{pwd}/fm")
assert fm.del_dir(f"{pwd}")
def _ts_sorted_file(self,
req='latest',
directory=None,
fn_pattern=None,
fn_only=False):
"""
Look for the latest or oldest modified date from files in directory.
:param action: 'latest' or 'oldest'
:param directory: provide path
:param fn_pattern: filter based on filename pattern
:param fn_only: returns list [] of filenames only
:return: filename, timestamp
"""
r = []
if not directory and self.INPUT:
directory = self.INPUT
# -- build file list
log.debug('directory: {}'.format(directory))
log.debug('directory exists: {}'.format(self.exists(directory)))
if directory and self.exists(directory):
t = []
l = sorted([[int(os.stat(os.path.join(directory, f)).st_ctime*1000), f] \
for f in os.listdir(directory)])
if fn_pattern:
for n in sorted(l):
if re.search(fn_pattern, n[1]):
t += [[n[0], n[1]]]
elif l:
t = l
del l
# -- apply filter
if req == 'list':
r = t
elif req == 'latest' and t:
r = [t[len(t) - 1][0], t[len(t) - 1][1]]
elif req == 'oldest' and t:
r = [t[0][0], t[0][1]]
del t
# -- return simplified list
if fn_only:
if req in ['latest', 'oldest']:
r = [r]
r = [x[1] for x in r]
return r
def _response(self, data=None, rcode=None, message=None):
r = {'data': []}
if data:
if type(data) in [int, str, dict]:
r['data'] += [self._decode_objectid(data)]
elif type(data) in [Cursor, list]:
for d in data:
r['data'] += [self._decode_objectid(d)]
else:
r['data'] = []
rcode = 500
message = 'Could not format data for response object ({}).'.format(type(data))
r['status'] = {'code': rcode, 'message': message, 'records': len(r['data'])}
log.debug('response: {}'.format(r))
return r
def _response(self, data=None, rcode=None, message=None):
r = {'data': []}
if data:
if isinstance(data, str):
r['data'] = list(self._decode_objectid(data))
else:
for d in data:
r['data'] += [self._decode_objectid(d)]
r['status'] = {
'code': rcode,
'message': message,
'records': len(r['data'])
}
log.debug('response: {}'.format(r))
return r
def delete(self, where=None, collection=None, db=None):
"""
Remove document or object in document.
:param statement: document/object to query to remove
:param collection: to change collection/table
:param db: to change database
:example:
delete({'_id': '5e114ad941734d371c5f84b9'})
delete([{'_id': '5e114ad941734d371c5f84b9'}, {'age': 25}])
delete({'person.fname': 'Randoll'} # delete document where {'person': {'fname': 'Randoll'}}
delete({}) # delete all in collection, not allowed
"""
self.cd(collection, db)
r = []
c = 204
m = 'Nothing happened.'
log.debug('delete doc(s) like: {}'.format(where))
try:
if where:
if isinstance(where, dict):
where = [where]
statement = []
if isinstance(where, list):
for f in where:
if not isinstance(f, dict):
statement += [self._encode_objectid({'_id': f})]
r += [str(f)]
else:
statement += [self._encode_objectid(f)]
for s in statement:
obj = self.collection.delete_one(s)
log.info('delete_count: {}, delete_ack: {}'.format(obj.deleted_count, obj.acknowledged))
r += [{'statement': s, 'delete_count': obj.deleted_count, 'delete_ack':obj.acknowledged}]
c = 410
m = 'Item(s) deletion have been executed.'
except Exception as e:
r = where
c = 500
m = 'Server Error: {}'.format(e)
return self._response(r, c, m)
def retainer(self, directory, fn, retain):
"""
Function to apply retention policy.
:param directory: base path
:param fn: file names containing substring
:param ret_d: number of files to retain
"""
if retain > 0:
del_list = self._ts_sorted_file('list', directory, \
fn_pattern='.*{}.*'.format(fn))
if len(del_list) > retain:
# -- unpack filename
t = []
for f in del_list:
t += [f[1]]
del_list = t[:len(t) - retain]
log.debug('list of file to delete: {}'.format(del_list))
self.del_files(directory, del_list)
del del_list, t
log.info('applied retention policy: {}'.format(directory))
def backup_mongo(dbs_l, path, conf):
r = False
# -- decide on backup rotation directory
dt_label = time.strftime('%Y-%m-%d', m_today)
b_dir = 'daily'
week_days = {
'monday': 0,
'tuesday': 1,
'wednesday': 2,
'thrusday': 3,
'friday': 4,
'saturday': 5,
'sunday': 6
}
if m_today.tm_wday == week_days[config['backup']['weekly'][
'on']] and config['backup']['weekly']['retention'] > 0:
b_dir = 'weekly'
if m_today.tm_mday == config['backup']['monthly'][
'on'] and config['backup']['monthly']['retention'] > 0:
b_dir = 'monthly'
log.debug('Retention: {}'.format(config['backup'][b_dir]['retention']))
log.debug('Policy: {}'.format(b_dir))
# -- run backup for each database
if dbs_l and (type(dbs_l) is list):
for dbs in dbs_l:
log.info('Database name: {}'.format(dbs))
log.info('creating backup file: {}-{}.gz in {}/{}/'.format(
dbs, dt_label, path, b_dir))
cmd = ['mongodump']
if conf['username'] and conf['password']:
cmd += ['-u', conf['username'], '-p', conf['password']]
cmd += [
'--authenticationDatabase={}'.format(
conf["authenticationDatabase"]),
'--archive={}/{}/{}-{}.gz'.format(path, b_dir, dbs, dt_label),
'--gzip', '--db', dbs
]
log.debug('command: {}'.format(" ".join(cmd)))
out = subprocess.run(cmd,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
log.info(out.stdout)
r = True
# -- apply retention policy
FileManager.retainer(b_dir, dbs)
config['last_run'] = dt_label
config.save()
return r
