def create(self, doc=None, collection=None, db=None): """ Insert object(s). :param doc: data to be inserted. :param collection: to change collection/table :param db: to change database """ self.cd(collection, db) count = 0 r = None c = 204 m = 'Nothing happened.' try: ins = None if isinstance(doc, dict): ins = self.collection.insert_one(doc) r = self._decode_objectid(ins.inserted_id) count = 1 if r else 0 elif isinstance(doc, list): ins = self.collection.insert_many(doc) r = ins.inserted_ids count = len(r) if r and ins: c = 201 m = 'Data inserted.' log.info('create_count: {}, create_ack: {}'.format(count, ins.acknowledged)) except Exception as e: c = 500 m = 'Server Error: {}'.format(e) return self._response(r, c, m)
def del_files(self, path, files=None, dir_flag=False, fn_pattern=None): """ Delete list of files provided. :param path: directory (only) :param files: list [] of files :param fn_pattern: override [files] if provided :return: True or False """ r = False if self.exists(path): if fn_pattern: files = self.ls(path, fn_pattern=fn_pattern, fn_only=True) if isinstance(files, list): for fn in files: try: os.remove(os.path.join(path, fn)) log.info('deleted: {}/{}'.format(path, fn)) r = True except Exception as e: log.error("Couldn't remove file: {}".format(e)) else: log.debug( 'Expected a list of files or a path with flag "dir=True"') return r
def update(self, doc=None, collection=None, db=None, where=None, like=None, set=None): """ :param doc: new version of database object to update :param collection: to change collection/table :param db: to change database :param where: criteria to locate database object i.e {'city': 'Atlanta} :param like: use filter with $regex i.e. like={'employe_name': '^Ran'} :param set: use $set to update field i.e. where={'_id': '5e1ab71ed4a0e6a7bdd5233f'}, set={'employe_name': 'Randoll'} """ log.debug('update: {}'.format(doc)) self.cd(collection, db) r = [] c = 204 m = 'Nothing happened.' try: if isinstance(doc, dict): self._encode_objectid(doc) obj = self.collection.replace_one({'_id': doc['_id']}, doc) log.info('update_match_count: {}, update_mod: {}, update_ack: {}'.format( obj.matched_count, obj.modified_count, obj.acknowledged)) c = 200 m = 'Document(s) updated.' except Exception as e: r = doc c = 500 m = 'Server Error: {}'.format(e) return self._response(r, c, m)
def main(): """ * Create backup directory structure. * Run backup daily. * Rotate on a daily, weekly, monthly basis. """ log.info('Starting mongodb backup script.') if config: # -- check directory and structure path = False if FileManager.dir_struct(config['backup']['path']): path = True # -- backup and rotate files if path: run = False last_run = config['last_run'] if last_run: log.info('checked last run: {}'.format(last_run)) if not last_run == time.strftime('%Y-%m-%d', m_today): run = True else: log.info('No need to run a backup at this time.') else: run = True if run: if backup_mongo(config['backup']['databases'], config['backup']['path'], config['db-instance']): log.info('Backup process completed successfully.') else: log.info('Backup process completed without errors.') else: log.error("Couln't retreive config file")
def _update_session(self, collection=None, db=None): if collection: if db: self.collection = self.connector.db.client[db][collection] else: self.collection = self.connector.db[collection] log.info('Using collection: {}.{}'.format(self.connector.db.name, self.collection.name)) # bugfix at line 49, previously test wrong db object
def read(self, where=None, collection=None, db=None, projection=None, aggr_cols=None, aggr_type=None, like=None): """ Read <database>.<collection>. :param where: filter object to look for :param collection: to change collection/table :param db: to change database :param projection: fields to return in response :param aggr_cols: fields to group by ['name', 'department', 'salary'] :param aggr_type: 'count', 'sum' i.e. aggr_type='count' or aggr_type={'sum': 'salary'} :param like: use find() with $regex i.e. like={'employe_name': '^Ran'} """ self._update_session(collection, db) r = statement = None c = 404 m = 'No data returned.' doc_count = 0 try: if where: statement = where else: statement = {} log.info('retrieving doc(s) like: {}'.format(statement)) if isinstance(aggr_cols, list): if isinstance(aggr_type, dict): pass # sum else: pass # count data = None elif isinstance(statement, dict): data = self.collection.find(statement, projection=projection) for _ in data: doc_count += 1 if doc_count > 0: data.rewind() r = data c = 200 m = 'OK' log.info('data: {} doc(s).'.format(doc_count)) except Exception as e: r = statement c = 500 m = 'Server Error: {}'.format(e) return self._response(r, c, m)
def setbucket(self, dirname): """ Add a "bucket" to directory structure. :param dirname: name to set the subdirectory """ if not self.bucket: self.bucket = str(dirname) self.dir_struct() log.info('Bucket is now set to: {}.'.format(self.bucket)) else: log.info('Buckets cannot be reset to a different name ({}). ' 'Currently set to "{}"'.format(dirname, self.bucket))
def cd(self, collection=None, db=None): """ Change collection and/or database. :param collection: collection name to change :param db: database name to change (collecion is required) """ if collection and self.connector.connected: if db: self.collection = self.connector.db.client[db][collection] log.info('Using database: {}'.format(self.connector.db.name)) else: self.collection = self.connector.db[collection] log.info('Using collection: {}.{}'.format(self.connector.db.name, self.collection.name))
def do_move(fnum=0): ret = False new_fn = newfilename(fnum) if not self.exists('{}/{}'.format(dst, new_fn)): if self.exists('{}/{}'.format(src, fn)): os.rename('{}/{}'.format(src, fn), '{}/{}'.format(dst, new_fn)) log.info('moved: [{}/{}] to [{}/{}]'.format( src, fn, dst, new_fn)) ret = True else: fnum += 1 ret = do_move(fnum) return ret
def delete(self, where=None, collection=None, db=None): """ Remove document or object in document. :param statement: document/object to query to remove :param collection: to change collection/table :param db: to change database :example: delete({'_id': '5e114ad941734d371c5f84b9'}) delete([{'_id': '5e114ad941734d371c5f84b9'}, {'age': 25}]) delete({'person.fname': 'Randoll'} # delete document where {'person': {'fname': 'Randoll'}} delete({}) # delete all in collection, not allowed """ log.info('deleting doc(s) like: {}'.format(where)) self._update_session(collection, db) r = [] c = 204 m = 'Nothing happened.' try: if where: if isinstance(where, dict): where = [where] if isinstance(where, list): statement = [] for f in where: if not isinstance(f, dict): statement += [self._encode_objectid({'_id': f})] r += [str(f)] else: statement += [self._encode_objectid(f)] for s in statement: obj = self.collection.delete_one(s) log.info('delete: ack: {}, delete_count: {}, doc: {}'. format(obj.acknowledged, obj.deleted_count, s)) r = statement c = 410 m = 'Item(s) delete.' except Exception as e: r = where c = 500 m = 'Server Error: {}'.format(e) return self._response(r, c, m)
def del_dir(self, path): """ Delete directory provided. :param path: directory (only) :return: True or False """ r = False if self.exists(path): try: if self.basedir == '{}/fm'.format(path): os.rmdir(self.basedir) if self.bucket and self.bucket in path: os.rmdir(path) path = path.replace(self.bucket, '') os.rmdir(path) r = True log.info('remove dir: {}'.format(path)) except Exception as e: log.error("Couldn't remove directory: {}".format(e)) return r
def __init__(self, db_config=None, collection=None, db=None): global config self.client = None self.db = None self.collection = None self.connected = False if db_config is None: if os.environ.get('APP_RUNTIME_CONTEXT') == 'dev': db_config = config['mongo.dev'] self.environ = 'dev' elif os.environ.get('APP_RUNTIME_CONTEXT') == 'qa': db_config = config['mongo.qa'] self.environ = 'qa' else: db_config = config['mongo.prod'] self.environ = 'prod' log.info('Using mongo.{} configuration.'.format(self.environ)) else: log.info('Using db_config provided: {}'.format(db_config)) if db_config: self.client = MongoClient( 'mongodb+srv://{}:{}@{}/{}?retryWrites=true&w=majority'.format( db_config['username'], db_config['password'], db_config['host'], db_config['database'])) # -- setup database if self.db: self.db = self.client[db] else: if db_config.get('database'): self.db = self.client[db_config['database']] else: self.db = self.client['admin'] # -- setup collection if collection: self.collection = self.db[collection] else: if db_config.get('collection'): self.collection = self.db[db_config['collection']] else: self.collection = self.db['system.version'] self.connected = True if self.connected: log.info('CONNECTED to {}@{}'.format(self.db.name, db_config['host'])) else: log.info('NOT CONNECTED. (db={}, host={})'.format( db_config['database'], db_config['host']))
def create(self, doc=None, collection=None, db=None): """ Insert object(s). :param doc: data to be inserted. :param collection: to change collection/table :param db: to change database """ log.info('inserting doc: {}'.format(doc)) self._update_session(collection, db) r = None c = 204 m = 'Nothing happened.' try: ins = None if isinstance(doc, dict): ins = self.collection.insert_one(doc) r = [str(ins.inserted_id)] log.info('inserted: {}, {}'.format(ins.acknowledged, ins.inserted_id)) elif isinstance(doc, list): ins = self.collection.insert_many(doc) r = [str(i) for i in ins.inserted_ids] log.info('inserted: {}, {}'.format(ins.acknowledged, r)) if r: c = 201 m = 'Data inserted.' except Exception as e: r = doc c = 500 m = 'Server Error: {}'.format(e) return self._response(r, c, m)
def retainer(self, directory, fn, retain): """ Function to apply retention policy. :param directory: base path :param fn: file names containing substring :param ret_d: number of files to retain """ if retain > 0: del_list = self._ts_sorted_file('list', directory, \ fn_pattern='.*{}.*'.format(fn)) if len(del_list) > retain: # -- unpack filename t = [] for f in del_list: t += [f[1]] del_list = t[:len(t) - retain] log.debug('list of file to delete: {}'.format(del_list)) self.del_files(directory, del_list) del del_list, t log.info('applied retention policy: {}'.format(directory))
def backup_mongo(dbs_l, path, conf): r = False # -- decide on backup rotation directory dt_label = time.strftime('%Y-%m-%d', m_today) b_dir = 'daily' week_days = { 'monday': 0, 'tuesday': 1, 'wednesday': 2, 'thrusday': 3, 'friday': 4, 'saturday': 5, 'sunday': 6 } if m_today.tm_wday == week_days[config['backup']['weekly'][ 'on']] and config['backup']['weekly']['retention'] > 0: b_dir = 'weekly' if m_today.tm_mday == config['backup']['monthly'][ 'on'] and config['backup']['monthly']['retention'] > 0: b_dir = 'monthly' log.debug('Retention: {}'.format(config['backup'][b_dir]['retention'])) log.debug('Policy: {}'.format(b_dir)) # -- run backup for each database if dbs_l and (type(dbs_l) is list): for dbs in dbs_l: log.info('Database name: {}'.format(dbs)) log.info('creating backup file: {}-{}.gz in {}/{}/'.format( dbs, dt_label, path, b_dir)) cmd = ['mongodump'] if conf['username'] and conf['password']: cmd += ['-u', conf['username'], '-p', conf['password']] cmd += [ '--authenticationDatabase={}'.format( conf["authenticationDatabase"]), '--archive={}/{}/{}-{}.gz'.format(path, b_dir, dbs, dt_label), '--gzip', '--db', dbs ] log.debug('command: {}'.format(" ".join(cmd))) out = subprocess.run(cmd, check=True, stdout=subprocess.PIPE, universal_newlines=True) log.info(out.stdout) r = True # -- apply retention policy FileManager.retainer(b_dir, dbs) config['last_run'] = dt_label config.save() return r
def dir_struct(self, path=None, known_dir=None): """ Check directory structure. If not valid, create structure. :param path: filesystem full path :param known_dir: list of dir paths :return: True or False """ r = False def update_path(_p, _np): if _p == 'archive': self.ARCHIVE = _np elif _p == 'errored': self.ERRORED = _np elif _p == 'input': self.INPUT = _np elif _p == 'output': self.OUTPUT = _np if path: self.basedir = '{}/fm'.format(path) u_path = False if not self.known_dir: self.known_dir = ['archive', 'errored', 'input', 'output'] u_path = True if known_dir: self.known_dir = known_dir log.info('validate directory structure for: {}'.format(self.basedir)) try: i = 0 for d in self.known_dir: n_path = os.path.join(self.basedir, d, self.bucket) if u_path: update_path(d, n_path) if not os.path.exists(n_path): log.info('creating: {}'.format(n_path)) os.makedirs(n_path, exist_ok=True) r = True except Exception as e: log.info("Couldn't setup directory structure.\n{}".format(e)) return r
def close(self): if self.status(): self.client.close() self.connected = False log.info('DISCONNECTED.')
def __init__(self, db_config=None, collection=None, db=None, collection_obj=None, db_obj=None): """ Create database connection. :param db_config: configuration object map :param collection: existing pymongo collection object :param db: existing pymongo db object """ self.collection = collection_obj self.db = db_obj collection_name = collection; del collection # -- to avoid ambiguity db_name = db; del db # -- to avoid ambiguity self.client = None self.connected = False # -- validate passing objects if db_obj and isinstance(db_obj, Database): self.client = db_obj.client if db_obj and not isinstance(db_obj, Database): log.error('db_obj: {} is not a Database object'.format(type(db_obj))) return if collection_obj and not isinstance(collection_obj, Collection): log.error('collection_obj: {} is not a Collection object'.format(type(collection_obj))) return # -- database basic config if db_config is None and not self.db and not self.collection: if os.environ.get('APP_RUNTIME_CONTEXT') == 'dev': db_config = config['mongo.dev'] self.environ = 'dev' elif os.environ.get('APP_RUNTIME_CONTEXT') == 'qa': db_config = config['mongo.qa'] self.environ = 'qa' else: db_config = config['mongo.prod'] self.environ = 'prod' log.info('Using mongo.{} configuration.'.format(self.environ)) else: log.info('Using db_config provided: {}'.format(db_config)) if db_config: self.client = MongoClient( 'mongodb://{}:{}/'.format(db_config['host'], db_config['port']), username=db_config['username'], password=db_config['password'], authSource=db_config['authenticationDatabase']) # -- setup database if not db_name and not self.db: if db_config.get('database'): self.db = self.client[db_config['database']] else: self.db = self.client['test_db'] elif db_name and not self.db: self.db = self.client[db_name] # -- setup collection if not collection_name and self.db: if db_config and db_config.get('collection'): self.collection = self.db[db_config['collection']] elif not self.collection: self.collection = self.db['test'] elif collection_name and self.db: self.collection = self.db[collection_name] if self.status(): if collection_obj and db_obj: log.info('Using existing connection: {}@{}'.format(self.db.name, self.client.address)) else: log.info('CONNECTED to {}@{}'.format(self.db.name, self.client.address)) else: log.info('NOT CONNECTED.')
def read(self, where=None, collection=None, db=None, projection=None, sort=None, aggr_cols=None, aggr_type=None, like=None): """ Read <database>.<collection>. :param where: filter object to look for :param collection: to change collection/table :param db: to change database :param projection: fields to return in response i.e. {'name': true, 'department': true} :param aggr_cols: fields to group by ['name', 'department', 'salary'] :param aggr_type: 'count', 'sum' i.e. aggr_type={'count': 'salary'} or aggr_type={'sum': 'salary'} :param like: use find() with $regex i.e. like={'employe_name': '^Ran'} """ self.cd(collection, db) r = None c = 404 m = 'No data returned.' doc_count = 0 statement = [] try: # -- build statement if where: self._encode_objectid(where) for k in where: if where[k] is None: statement += [(k, {'$exist': False})] else: statement += [(k, where[k])] else: statement += [('_id', {'$exists': True})] # if isinstance(aggr_cols, list): # if isinstance(aggr_type, dict): # pass # sum # else: # pass # count log.info('retrieving doc(s) like: {}{}'.format(dict(statement), \ ', {}'.format(projection) if projection else '')) # -- execute statement if isinstance(sort, dict): s_list = [] for k in sort: s_list += [(k, sort[k])] data = self.collection.find(SON(statement), projection).sort(s_list) del s_list else: data = self.collection.find(SON(statement), projection) # -- collect result for _ in data: doc_count += 1 if doc_count > 0: data.rewind() r = data c = 200 m = 'OK' log.info('read_count: {}'.format(doc_count)) except Exception as e: r = statement c = 500 m = 'Server Error: {}'.format(e) return self._response(r, c, m)