def create_backend(db_col_names,name_only=False,**kwargs):
"""
Guess what's inside 'db_col_names' and return the corresponding backend.
- It could be a string (by default, will lookup a mongo collection in target database)
- or a tuple("target|src","col_name")
- or a ("mongodb://*****:*****@host","db","col_name") URI.
- or a ("es_host:port","index_name","doc_type")
If name_only is true, just return the name uniquely identifying the collection or index
URI connection.
"""
col = None
db = None
is_mongo = True
if type(db_col_names) == str:
db = mongo.get_target_db()
col = db[db_col_names]
# normalize params
db_col_names = ["%s:%s" % (db.client.HOST,db.client.PORT),db.name,col.name]
elif db_col_names[0].startswith("mongodb://"):
assert len(db_col_names) == 3, "Missing connection information for %s" % repr(db_col_names)
conn = mongo.MongoClient(db_col_names[0])
db = conn[db_col_names[1]]
col = db[db_col_names[2]]
# normalize params
db_col_names = ["%s:%s" % (db.client.HOST,db.client.PORT),db.name,col.name]
elif len(db_col_names) == 3 and ":" in db_col_names[0]:
is_mongo = False
idxr = ESIndexer(index=db_col_names[1],doc_type=db_col_names[2],es_host=db_col_names[0],**kwargs)
db = idxr
col = db_col_names[1]
else:
assert len(db_col_names) == 2, "Missing connection information for %s" % repr(db_col_names)
db = db_col_names[0] == "target" and mongo.get_target_db() or mongo.get_src_db()
col = db[db_col_names[1]]
# normalize params (0:host, 1:port)
db_col_names = ["%s:%s" % (db.client.address[0],db.client.address[1]),db.name,col.name]
assert not col is None, "Could not create collection object from %s" % repr(db_col_names)
if name_only:
if is_mongo:
return "mongo_%s_%s_%s" % (db_col_names[0].replace(":","_"),
db_col_names[1],db_col_names[2])
else:
return "es_%s_%s_%s" % (db_col_names[0].replace(":","_"),
db_col_names[1],db_col_names[2])
else:
if is_mongo:
return DocMongoBackend(db,col)
else:
return DocESBackend(db)
def create_backend(db_col_names, name_only=False, follow_ref=False, **kwargs):
"""
Guess what's inside 'db_col_names' and return the corresponding backend.
- It could be a string (will first check for an src_build doc to check
a backend_url field, if nothing there, will lookup a mongo collection
in target database)
- or a tuple("target|src","col_name")
- or a ("mongodb://*****:*****@host","db","col_name") URI.
- or a ("es_host:port","index_name","doc_type")
If name_only is true, just return the name uniquely identifying the collection or index
URI connection.
"""
col = None
db = None
is_mongo = True
if type(db_col_names) == str:
# first check build doc, if there's backend_url key, we'll use it instead of
# direclty using db_col_names as target collection (see LinkDataBuilder)
bdoc = get_src_build().find_one({"_id": db_col_names})
if follow_ref and bdoc and bdoc.get(
"backend_url") and bdoc["backend_url"] != db_col_names:
return create_backend(bdoc["backend_url"],
name_only=name_only,
follow_ref=follow_ref,
**kwargs)
else:
db = mongo.get_target_db()
col = db[db_col_names]
# normalize params
db_col_names = [
"%s:%s" % (db.client.HOST, db.client.PORT), db.name, col.name
]
elif db_col_names[0].startswith("mongodb://"):
assert len(
db_col_names
) == 3, "Missing connection information for %s" % repr(db_col_names)
conn = mongo.MongoClient(db_col_names[0])
db = conn[db_col_names[1]]
col = db[db_col_names[2]]
# normalize params
db_col_names = [
"%s:%s" % (db.client.HOST, db.client.PORT), db.name, col.name
]
elif len(db_col_names) == 3 and ":" in db_col_names[0]:
is_mongo = False
idxr = ESIndexer(index=db_col_names[1],
doc_type=db_col_names[2],
es_host=db_col_names[0],
**kwargs)
db = idxr
col = db_col_names[1]
else:
assert len(
db_col_names
) == 2, "Missing connection information for %s" % repr(db_col_names)
db = db_col_names[0] == "target" and mongo.get_target_db(
) or mongo.get_src_db()
col = db[db_col_names[1]]
# normalize params (0:host, 1:port)
db_col_names = [
"%s:%s" % (db.client.address[0], db.client.address[1]), db.name,
col.name
]
assert col is not None, "Could not create collection object from %s" % repr(
db_col_names)
if name_only:
if is_mongo:
return "mongo_%s_%s_%s" % (db_col_names[0].replace(
":", "_"), db_col_names[1], db_col_names[2])
else:
return "es_%s_%s_%s" % (db_col_names[0].replace(
":", "_"), db_col_names[1], db_col_names[2])
else:
if is_mongo:
return DocMongoBackend(db, col)
else:
return DocESBackend(db)