def __init__(self,
host,
index,
type=None,
alias=None,
name=None,
port=9200,
read_only=True,
typed=None,
settings=None):
Container.__init__(self, None)
if not containers.config.default:
containers.config.default.settings = settings
self.settings = settings
self.name = coalesce(name, alias, index)
if read_only:
self._es = elasticsearch.Alias(alias=coalesce(alias, index),
settings=settings)
else:
self._es = elasticsearch.Cluster(settings=settings).get_index(
read_only=read_only, settings=settings)
self.meta = FromESMetadata(settings=settings)
self.settings.type = self._es.settings.type
self.edges = Dict()
self.worker = None
if typed == None:
self._columns = self.get_columns(table_name=index)
# SWITCH ON TYPED MODE
self.typed = any(c.name in ("$value", "$object")
for c in self._columns)
else:
self.typed = typed
def __init__(
self, host, index, type=None, alias=None, name=None, port=9200, read_only=True, typed=None, settings=None
):
Container.__init__(self, None)
if not containers.config.default:
containers.config.default.settings = settings
self.settings = settings
self.name = coalesce(name, alias, index)
if read_only:
self._es = elasticsearch.Alias(alias=coalesce(alias, index), settings=settings)
else:
self._es = elasticsearch.Cluster(settings=settings).get_index(read_only=read_only, settings=settings)
self.meta = FromESMetadata(settings=settings)
self.settings.type = self._es.settings.type
self.edges = Dict()
self.worker = None
columns = self.get_columns(table_name=name)
self._schema = Schema(columns)
if typed == None:
# SWITCH ON TYPED MODE
self.typed = any(c.name in ("$value", "$object") for c in columns)
else:
self.typed = typed
def __init__(self, name, data, schema=None):
#TODO: STORE THIS LIKE A CUBE FOR FASTER ACCESS AND TRANSFORMATION
data = list(data)
Container.__init__(self, data, schema)
if schema == None:
self.schema = get_schema_from_list(data)
self.name = name
self.data = data
self.locker = Lock() # JUST IN CASE YOU WANT TO DO MORE THAN ONE THING
def __init__(self, name, data, schema=None):
#TODO: STORE THIS LIKE A CUBE FOR FASTER ACCESS AND TRANSFORMATION
data = list(unwrap(data))
Container.__init__(self, data, schema)
if schema == None:
self._schema = get_schema_from_list(data)
else:
self._schema = schema
self.name = name
self.data = data
self.locker = Lock() # JUST IN CASE YOU WANT TO DO MORE THAN ONE THING
def __init__(self, name, db=None, uid=UID_PREFIX+"id", exists=False):
"""
:param name: NAME FOR THIS TABLE
:param db: THE DB TO USE
:param uid: THE UNIQUE INDEX FOR THIS TABLE
:return: HANDLE FOR TABLE IN db
"""
Container.__init__(self, frum=None)
if db:
self.db = db
else:
self.db = db = Sqlite()
self.name = name
self.uid = listwrap(uid)
self.columns = {}
for u in uid:
if not self.columns.get(u, None):
cs = self.columns[u] = set()
if u.startswith(UID_PREFIX):
cs.add(Column(name=u, table=name, type="integer", es_column=typed_column(u, "integer"), es_index=name))
else:
cs.add(Column(name=u, table=name, type="text", es_column=typed_column(u, "text"), es_index=name))
self.uid_accessor = jx.get(self.uid)
self.nested_tables = {} # MAP FROM TABLE NAME TO Table OBJECT
if exists:
# LOAD THE COLUMNS
command = "PRAGMA table_info(" + quote_table(name) + ")"
details = self.db.query(command)
self.columns = {}
for r in details:
cname = untyped_column(r[1])
ctype = r[2].lower()
column = Column(
name=cname,
table=name,
type=ctype,
es_column=typed_column(cname, ctype),
es_index=name
)
cs = self.columns.get(name, Null)
if not cs:
cs = self.columns[name] = set()
cs.add(column)
else:
command = "CREATE TABLE " + quote_table(name) + "(" + \
(",".join(_quote_column(c) + " " + c.type for u, cs in self.columns.items() for c in cs)) + \
", PRIMARY KEY (" + \
(", ".join(_quote_column(c) for u in self.uid for c in self.columns[u])) + \
"))"
self.db.execute(command)
def __new__(cls, *args, **kwargs):
if (len(args) == 1 and args[0].get("index") == "meta") or kwargs.get("index") == "meta":
output = FromESMetadata.__new__(FromESMetadata, *args, **kwargs)
output.__init__(*args, **kwargs)
return output
else:
return Container.__new__(cls)
def __new__(cls, *args, **kwargs):
if (len(args) == 1 and args[0].get("index") == "meta") or kwargs.get("index") == "meta":
output = FromESMetadata.__new__(FromESMetadata, *args, **kwargs)
output.__init__(*args, **kwargs)
return output
else:
return Container.__new__(cls)
def __init__(
self,
host,
index,
type=None,
alias=None,
name=None,
port=9200,
read_only=True,
timeout=None, # NUMBER OF SECONDS TO WAIT FOR RESPONSE, OR SECONDS TO WAIT FOR DOWNLOAD (PASSED TO requests)
consistency="one", # ES WRITE CONSISTENCY (https://www.elastic.co/guide/en/elasticsearch/reference/1.7/docs-index_.html#index-consistency)
typed=None,
kwargs=None):
Container.__init__(self, None)
if not containers.config.default:
containers.config.default.settings = kwargs
self.settings = kwargs
self.name = coalesce(name, alias, index)
if read_only:
self._es = elasticsearch.Alias(alias=coalesce(alias, index),
kwargs=kwargs)
else:
self._es = elasticsearch.Cluster(kwargs=kwargs).get_index(
read_only=read_only, kwargs=kwargs)
self.meta = FromESMetadata(kwargs=kwargs)
self.settings.type = self._es.settings.type
self.edges = Data()
self.worker = None
columns = self.meta.get_columns(
table_name=coalesce(name, alias, index))
self._schema = Schema(coalesce(name, alias, index), columns)
if typed == None:
# SWITCH ON TYPED MODE
self.typed = any(c.names["."] in ("$value", "$object")
for c in columns)
else:
self.typed = typed
def __init__(
self,
host,
index,
type=None,
alias=None,
name=None,
port=9200,
read_only=True,
timeout=None, # NUMBER OF SECONDS TO WAIT FOR RESPONSE, OR SECONDS TO WAIT FOR DOWNLOAD (PASSED TO requests)
consistency="one", # ES WRITE CONSISTENCY (https://www.elastic.co/guide/en/elasticsearch/reference/1.7/docs-index_.html#index-consistency)
typed=None,
settings=None,
):
Container.__init__(self, None)
if not containers.config.default:
containers.config.default.settings = settings
self.settings = settings
self.name = coalesce(name, alias, index)
if read_only:
self._es = elasticsearch.Alias(alias=coalesce(alias, index), settings=settings)
else:
self._es = elasticsearch.Cluster(settings=settings).get_index(read_only=read_only, settings=settings)
self.meta = FromESMetadata(settings=settings)
self.settings.type = self._es.settings.type
self.edges = Data()
self.worker = None
columns = self.get_columns(table_name=name)
self._schema = Schema(columns)
if typed == None:
# SWITCH ON TYPED MODE
self.typed = any(c.name in ("$value", "$object") for c in columns)
else:
self.typed = typed
def __init__(self, name, db=None, uid=GUID, exists=False, kwargs=None):
"""
:param name: NAME FOR THIS TABLE
:param db: THE DB TO USE
:param uid: THE UNIQUE INDEX FOR THIS TABLE
:return: HANDLE FOR TABLE IN db
"""
global _config
Container.__init__(self, frum=None)
if db:
self.db = db
else:
self.db = db = Sqlite()
if not _config:
from pyLibrary.queries.containers import config as _config
if not _config.default:
_config.default = {"type": "sqlite", "settings": {"db": db}}
self.name = name
self.uid = listwrap(uid)
self._next_uid = 1
self._make_digits_table()
self.uid_accessor = jx.get(self.uid)
self.nested_tables = OrderedDict(
) # MAP FROM NESTED PATH TO Table OBJECT, PARENTS PROCEED CHILDREN
self.nested_tables["."] = self
self.columns = Index(
keys=[join_field(["names", self.name])]
) # MAP FROM DOCUMENT ABS PROPERTY NAME TO THE SET OF SQL COLUMNS IT REPRESENTS (ONE FOR EACH REALIZED DATATYPE)
if not exists:
for u in self.uid:
if u == GUID:
pass
else:
c = Column(names={name: u},
type="string",
es_column=typed_column(u, "string"),
es_index=name)
self.add_column_to_schema(self.nested_tables, c)
command = ("CREATE TABLE " + quote_table(name) + "(" +
(",".join([quoted_UID + " INTEGER"] + [
_quote_column(c) + " " + sql_types[c.type]
for u, cs in self.columns.items() for c in cs
])) + ", PRIMARY KEY (" + (", ".join([quoted_UID] + [
_quote_column(c) for u in self.uid
for c in self.columns[u]
])) + "))")
self.db.execute(command)
else:
# LOAD THE COLUMNS
command = "PRAGMA table_info(" + quote_table(name) + ")"
details = self.db.query(command)
for r in details:
cname = untyped_column(r[1])
ctype = r[2].lower()
column = Column(names={name: cname},
type=ctype,
nested_path=['.'],
es_column=typed_column(cname, ctype),
es_index=name)
self.add_column_to_schema(self.columns, column)
def __init__(self, name, db=None, uid=UID_PREFIX + "id", exists=False):
"""
:param name: NAME FOR THIS TABLE
:param db: THE DB TO USE
:param uid: THE UNIQUE INDEX FOR THIS TABLE
:return: HANDLE FOR TABLE IN db
"""
Container.__init__(self, frum=None)
if db:
self.db = db
else:
self.db = db = Sqlite()
self.name = name
self.uid = listwrap(uid)
self.columns = {}
for u in uid:
if not self.columns.get(u, None):
cs = self.columns[u] = set()
if u.startswith(UID_PREFIX):
cs.add(
Column(name=u,
table=name,
type="integer",
es_column=typed_column(u, "integer"),
es_index=name))
else:
cs.add(
Column(name=u,
table=name,
type="text",
es_column=typed_column(u, "text"),
es_index=name))
self.uid_accessor = jx.get(self.uid)
self.nested_tables = {} # MAP FROM TABLE NAME TO Table OBJECT
if exists:
# LOAD THE COLUMNS
command = "PRAGMA table_info(" + quote_table(name) + ")"
details = self.db.query(command)
self.columns = {}
for r in details:
cname = untyped_column(r[1])
ctype = r[2].lower()
column = Column(name=cname,
table=name,
type=ctype,
es_column=typed_column(cname, ctype),
es_index=name)
cs = self.columns.get(name, Null)
if not cs:
cs = self.columns[name] = set()
cs.add(column)
else:
command = "CREATE TABLE " + quote_table(name) + "(" + \
(",".join(_quote_column(c) + " " + c.type for u, cs in self.columns.items() for c in cs)) + \
", PRIMARY KEY (" + \
(", ".join(_quote_column(c) for u in self.uid for c in self.columns[u])) + \
"))"
self.db.execute(command)