def create_backend(db_col_names,name_only=False,**kwargs): """ Guess what's inside 'db_col_names' and return the corresponding backend. - It could be a string (by default, will lookup a mongo collection in target database) - or a tuple("target|src","col_name") - or a ("mongodb://*****:*****@host","db","col_name") URI. - or a ("es_host:port","index_name","doc_type") If name_only is true, just return the name uniquely identifying the collection or index URI connection. """ col = None db = None is_mongo = True if type(db_col_names) == str: db = mongo.get_target_db() col = db[db_col_names] # normalize params db_col_names = ["%s:%s" % (db.client.HOST,db.client.PORT),db.name,col.name] elif db_col_names[0].startswith("mongodb://"): assert len(db_col_names) == 3, "Missing connection information for %s" % repr(db_col_names) conn = mongo.MongoClient(db_col_names[0]) db = conn[db_col_names[1]] col = db[db_col_names[2]] # normalize params db_col_names = ["%s:%s" % (db.client.HOST,db.client.PORT),db.name,col.name] elif len(db_col_names) == 3 and ":" in db_col_names[0]: is_mongo = False idxr = ESIndexer(index=db_col_names[1],doc_type=db_col_names[2],es_host=db_col_names[0],**kwargs) db = idxr col = db_col_names[1] else: assert len(db_col_names) == 2, "Missing connection information for %s" % repr(db_col_names) db = db_col_names[0] == "target" and mongo.get_target_db() or mongo.get_src_db() col = db[db_col_names[1]] # normalize params (0:host, 1:port) db_col_names = ["%s:%s" % (db.client.address[0],db.client.address[1]),db.name,col.name] assert not col is None, "Could not create collection object from %s" % repr(db_col_names) if name_only: if is_mongo: return "mongo_%s_%s_%s" % (db_col_names[0].replace(":","_"), db_col_names[1],db_col_names[2]) else: return "es_%s_%s_%s" % (db_col_names[0].replace(":","_"), db_col_names[1],db_col_names[2]) else: if is_mongo: return DocMongoBackend(db,col) else: return DocESBackend(db)
def create_backend(db_col_names, name_only=False, follow_ref=False, **kwargs): """ Guess what's inside 'db_col_names' and return the corresponding backend. - It could be a string (will first check for an src_build doc to check a backend_url field, if nothing there, will lookup a mongo collection in target database) - or a tuple("target|src","col_name") - or a ("mongodb://*****:*****@host","db","col_name") URI. - or a ("es_host:port","index_name","doc_type") If name_only is true, just return the name uniquely identifying the collection or index URI connection. """ col = None db = None is_mongo = True if type(db_col_names) == str: # first check build doc, if there's backend_url key, we'll use it instead of # direclty using db_col_names as target collection (see LinkDataBuilder) bdoc = get_src_build().find_one({"_id": db_col_names}) if follow_ref and bdoc and bdoc.get( "backend_url") and bdoc["backend_url"] != db_col_names: return create_backend(bdoc["backend_url"], name_only=name_only, follow_ref=follow_ref, **kwargs) else: db = mongo.get_target_db() col = db[db_col_names] # normalize params db_col_names = [ "%s:%s" % (db.client.HOST, db.client.PORT), db.name, col.name ] elif db_col_names[0].startswith("mongodb://"): assert len( db_col_names ) == 3, "Missing connection information for %s" % repr(db_col_names) conn = mongo.MongoClient(db_col_names[0]) db = conn[db_col_names[1]] col = db[db_col_names[2]] # normalize params db_col_names = [ "%s:%s" % (db.client.HOST, db.client.PORT), db.name, col.name ] elif len(db_col_names) == 3 and ":" in db_col_names[0]: is_mongo = False idxr = ESIndexer(index=db_col_names[1], doc_type=db_col_names[2], es_host=db_col_names[0], **kwargs) db = idxr col = db_col_names[1] else: assert len( db_col_names ) == 2, "Missing connection information for %s" % repr(db_col_names) db = db_col_names[0] == "target" and mongo.get_target_db( ) or mongo.get_src_db() col = db[db_col_names[1]] # normalize params (0:host, 1:port) db_col_names = [ "%s:%s" % (db.client.address[0], db.client.address[1]), db.name, col.name ] assert col is not None, "Could not create collection object from %s" % repr( db_col_names) if name_only: if is_mongo: return "mongo_%s_%s_%s" % (db_col_names[0].replace( ":", "_"), db_col_names[1], db_col_names[2]) else: return "es_%s_%s_%s" % (db_col_names[0].replace( ":", "_"), db_col_names[1], db_col_names[2]) else: if is_mongo: return DocMongoBackend(db, col) else: return DocESBackend(db)