def update(self, doc=None, collection=None, db=None, where=None, like=None, set=None): """ :param doc: new version of database object to update :param collection: to change collection/table :param db: to change database :param where: criteria to locate database object i.e {'city': 'Atlanta} :param like: use filter with $regex i.e. like={'employe_name': '^Ran'} :param set: use $set to update field i.e. where={'_id': '5e1ab71ed4a0e6a7bdd5233f'}, set={'employe_name': 'Randoll'} """ log.debug('update: {}'.format(doc)) self.cd(collection, db) r = [] c = 204 m = 'Nothing happened.' try: if isinstance(doc, dict): self._encode_objectid(doc) obj = self.collection.replace_one({'_id': doc['_id']}, doc) log.info('update_match_count: {}, update_mod: {}, update_ack: {}'.format( obj.matched_count, obj.modified_count, obj.acknowledged)) c = 200 m = 'Document(s) updated.' except Exception as e: r = doc c = 500 m = 'Server Error: {}'.format(e) return self._response(r, c, m)
def del_files(self, path, files=None, dir_flag=False, fn_pattern=None): """ Delete list of files provided. :param path: directory (only) :param files: list [] of files :param fn_pattern: override [files] if provided :return: True or False """ r = False if self.exists(path): if fn_pattern: files = self.ls(path, fn_pattern=fn_pattern, fn_only=True) if isinstance(files, list): for fn in files: try: os.remove(os.path.join(path, fn)) log.info('deleted: {}/{}'.format(path, fn)) r = True except Exception as e: log.error("Couldn't remove file: {}".format(e)) else: log.debug( 'Expected a list of files or a path with flag "dir=True"') return r
def newfilename(fnum): # -- add <filename>_1[+n] if fnum == 0: ret = fn else: try: nfn, ext = fn.split('.') ret = '{}_{}.{}'.format(nfn, fnum, ext) except Exception as e: ret = '{}_{}'.format(fn, fnum) log.debug('Error: {}'.format(e)) return ret
def test_touch(_g): fm = _g['fm'] pwd = _g['pwd'] log.debug(f"{fm.bucket}") fm.touch(f"{pwd}/fm/test1/result.1") fm.touch(f"{pwd}/fm/test1/result.2") fm.touch(f"{pwd}/fm/test1/result.3") fm.touch(f"{pwd}/fm/test1/result.4") files_list = [x[1] for x in fm.ls(f"{pwd}/fm/test1")] assert files_list == ['result.1', 'result.2', 'result.3', 'result.4']
def test_del_dir(_g): fm = _g['fm'] pwd = _g['pwd'] assert not fm.del_dir(f"{pwd}/fm/test1") assert fm.del_files(f"{pwd}/fm/test1", fn_pattern='result.*') assert fm.del_dir(f"{pwd}/fm/test3") assert fm.del_dir(f"{pwd}/fm/test2") assert fm.del_dir(f"{pwd}/fm/test1") log.debug(fm.ls(pwd)) assert fm.del_dir(f"{pwd}/fm") assert fm.del_dir(f"{pwd}")
def _ts_sorted_file(self, req='latest', directory=None, fn_pattern=None, fn_only=False): """ Look for the latest or oldest modified date from files in directory. :param action: 'latest' or 'oldest' :param directory: provide path :param fn_pattern: filter based on filename pattern :param fn_only: returns list [] of filenames only :return: filename, timestamp """ r = [] if not directory and self.INPUT: directory = self.INPUT # -- build file list log.debug('directory: {}'.format(directory)) log.debug('directory exists: {}'.format(self.exists(directory))) if directory and self.exists(directory): t = [] l = sorted([[int(os.stat(os.path.join(directory, f)).st_ctime*1000), f] \ for f in os.listdir(directory)]) if fn_pattern: for n in sorted(l): if re.search(fn_pattern, n[1]): t += [[n[0], n[1]]] elif l: t = l del l # -- apply filter if req == 'list': r = t elif req == 'latest' and t: r = [t[len(t) - 1][0], t[len(t) - 1][1]] elif req == 'oldest' and t: r = [t[0][0], t[0][1]] del t # -- return simplified list if fn_only: if req in ['latest', 'oldest']: r = [r] r = [x[1] for x in r] return r
def _response(self, data=None, rcode=None, message=None): r = {'data': []} if data: if type(data) in [int, str, dict]: r['data'] += [self._decode_objectid(data)] elif type(data) in [Cursor, list]: for d in data: r['data'] += [self._decode_objectid(d)] else: r['data'] = [] rcode = 500 message = 'Could not format data for response object ({}).'.format(type(data)) r['status'] = {'code': rcode, 'message': message, 'records': len(r['data'])} log.debug('response: {}'.format(r)) return r
def _response(self, data=None, rcode=None, message=None): r = {'data': []} if data: if isinstance(data, str): r['data'] = list(self._decode_objectid(data)) else: for d in data: r['data'] += [self._decode_objectid(d)] r['status'] = { 'code': rcode, 'message': message, 'records': len(r['data']) } log.debug('response: {}'.format(r)) return r
def delete(self, where=None, collection=None, db=None): """ Remove document or object in document. :param statement: document/object to query to remove :param collection: to change collection/table :param db: to change database :example: delete({'_id': '5e114ad941734d371c5f84b9'}) delete([{'_id': '5e114ad941734d371c5f84b9'}, {'age': 25}]) delete({'person.fname': 'Randoll'} # delete document where {'person': {'fname': 'Randoll'}} delete({}) # delete all in collection, not allowed """ self.cd(collection, db) r = [] c = 204 m = 'Nothing happened.' log.debug('delete doc(s) like: {}'.format(where)) try: if where: if isinstance(where, dict): where = [where] statement = [] if isinstance(where, list): for f in where: if not isinstance(f, dict): statement += [self._encode_objectid({'_id': f})] r += [str(f)] else: statement += [self._encode_objectid(f)] for s in statement: obj = self.collection.delete_one(s) log.info('delete_count: {}, delete_ack: {}'.format(obj.deleted_count, obj.acknowledged)) r += [{'statement': s, 'delete_count': obj.deleted_count, 'delete_ack':obj.acknowledged}] c = 410 m = 'Item(s) deletion have been executed.' except Exception as e: r = where c = 500 m = 'Server Error: {}'.format(e) return self._response(r, c, m)
def retainer(self, directory, fn, retain): """ Function to apply retention policy. :param directory: base path :param fn: file names containing substring :param ret_d: number of files to retain """ if retain > 0: del_list = self._ts_sorted_file('list', directory, \ fn_pattern='.*{}.*'.format(fn)) if len(del_list) > retain: # -- unpack filename t = [] for f in del_list: t += [f[1]] del_list = t[:len(t) - retain] log.debug('list of file to delete: {}'.format(del_list)) self.del_files(directory, del_list) del del_list, t log.info('applied retention policy: {}'.format(directory))
def backup_mongo(dbs_l, path, conf): r = False # -- decide on backup rotation directory dt_label = time.strftime('%Y-%m-%d', m_today) b_dir = 'daily' week_days = { 'monday': 0, 'tuesday': 1, 'wednesday': 2, 'thrusday': 3, 'friday': 4, 'saturday': 5, 'sunday': 6 } if m_today.tm_wday == week_days[config['backup']['weekly'][ 'on']] and config['backup']['weekly']['retention'] > 0: b_dir = 'weekly' if m_today.tm_mday == config['backup']['monthly'][ 'on'] and config['backup']['monthly']['retention'] > 0: b_dir = 'monthly' log.debug('Retention: {}'.format(config['backup'][b_dir]['retention'])) log.debug('Policy: {}'.format(b_dir)) # -- run backup for each database if dbs_l and (type(dbs_l) is list): for dbs in dbs_l: log.info('Database name: {}'.format(dbs)) log.info('creating backup file: {}-{}.gz in {}/{}/'.format( dbs, dt_label, path, b_dir)) cmd = ['mongodump'] if conf['username'] and conf['password']: cmd += ['-u', conf['username'], '-p', conf['password']] cmd += [ '--authenticationDatabase={}'.format( conf["authenticationDatabase"]), '--archive={}/{}/{}-{}.gz'.format(path, b_dir, dbs, dt_label), '--gzip', '--db', dbs ] log.debug('command: {}'.format(" ".join(cmd))) out = subprocess.run(cmd, check=True, stdout=subprocess.PIPE, universal_newlines=True) log.info(out.stdout) r = True # -- apply retention policy FileManager.retainer(b_dir, dbs) config['last_run'] = dt_label config.save() return r