コード例 #1
0
    def __init__(self,
                 host,
                 index,
                 type=None,
                 alias=None,
                 name=None,
                 port=9200,
                 read_only=True,
                 typed=None,
                 settings=None):
        Container.__init__(self, None)
        if not containers.config.default:
            containers.config.default.settings = settings
        self.settings = settings
        self.name = coalesce(name, alias, index)
        if read_only:
            self._es = elasticsearch.Alias(alias=coalesce(alias, index),
                                           settings=settings)
        else:
            self._es = elasticsearch.Cluster(settings=settings).get_index(
                read_only=read_only, settings=settings)

        self.meta = FromESMetadata(settings=settings)
        self.settings.type = self._es.settings.type
        self.edges = Dict()
        self.worker = None
        if typed == None:
            self._columns = self.get_columns(table_name=index)
            # SWITCH ON TYPED MODE
            self.typed = any(c.name in ("$value", "$object")
                             for c in self._columns)
        else:
            self.typed = typed
コード例 #2
0
ファイル: jx_usingES.py プロジェクト: klahnakoski/esReplicate
    def __init__(
        self, host, index, type=None, alias=None, name=None, port=9200, read_only=True, typed=None, settings=None
    ):
        Container.__init__(self, None)
        if not containers.config.default:
            containers.config.default.settings = settings
        self.settings = settings
        self.name = coalesce(name, alias, index)
        if read_only:
            self._es = elasticsearch.Alias(alias=coalesce(alias, index), settings=settings)
        else:
            self._es = elasticsearch.Cluster(settings=settings).get_index(read_only=read_only, settings=settings)

        self.meta = FromESMetadata(settings=settings)
        self.settings.type = self._es.settings.type
        self.edges = Dict()
        self.worker = None

        columns = self.get_columns(table_name=name)
        self._schema = Schema(columns)

        if typed == None:
            # SWITCH ON TYPED MODE
            self.typed = any(c.name in ("$value", "$object") for c in columns)
        else:
            self.typed = typed
コード例 #3
0
ファイル: lists.py プロジェクト: klahnakoski/MoDevETL
 def __init__(self, name, data, schema=None):
     #TODO: STORE THIS LIKE A CUBE FOR FASTER ACCESS AND TRANSFORMATION
     data = list(data)
     Container.__init__(self, data, schema)
     if schema == None:
         self.schema = get_schema_from_list(data)
     self.name = name
     self.data = data
     self.locker = Lock()  # JUST IN CASE YOU WANT TO DO MORE THAN ONE THING
コード例 #4
0
 def __init__(self, name, data, schema=None):
     #TODO: STORE THIS LIKE A CUBE FOR FASTER ACCESS AND TRANSFORMATION
     data = list(unwrap(data))
     Container.__init__(self, data, schema)
     if schema == None:
         self._schema = get_schema_from_list(data)
     else:
         self._schema = schema
     self.name = name
     self.data = data
     self.locker = Lock()  # JUST IN CASE YOU WANT TO DO MORE THAN ONE THING
コード例 #5
0
    def __init__(self, name, db=None, uid=UID_PREFIX+"id", exists=False):
        """
        :param name: NAME FOR THIS TABLE
        :param db: THE DB TO USE
        :param uid: THE UNIQUE INDEX FOR THIS TABLE
        :return: HANDLE FOR TABLE IN db
        """
        Container.__init__(self, frum=None)
        if db:
            self.db = db
        else:
            self.db = db = Sqlite()
        self.name = name
        self.uid = listwrap(uid)

        self.columns = {}
        for u in uid:
            if not self.columns.get(u, None):
                cs = self.columns[u] = set()
            if u.startswith(UID_PREFIX):
                cs.add(Column(name=u, table=name, type="integer", es_column=typed_column(u, "integer"), es_index=name))
            else:
                cs.add(Column(name=u, table=name, type="text", es_column=typed_column(u, "text"), es_index=name))

        self.uid_accessor = jx.get(self.uid)
        self.nested_tables = {}  # MAP FROM TABLE NAME TO Table OBJECT
        if exists:
            # LOAD THE COLUMNS
            command = "PRAGMA table_info(" + quote_table(name) + ")"
            details = self.db.query(command)
            self.columns = {}
            for r in details:
                cname = untyped_column(r[1])
                ctype = r[2].lower()
                column = Column(
                    name=cname,
                    table=name,
                    type=ctype,
                    es_column=typed_column(cname, ctype),
                    es_index=name
                )

                cs = self.columns.get(name, Null)
                if not cs:
                    cs = self.columns[name] = set()
                cs.add(column)
        else:
            command = "CREATE TABLE " + quote_table(name) + "(" + \
                      (",".join(_quote_column(c) + " " + c.type for u, cs in self.columns.items() for c in cs)) + \
                      ", PRIMARY KEY (" + \
                      (", ".join(_quote_column(c) for u in self.uid for c in self.columns[u])) + \
                      "))"

            self.db.execute(command)
コード例 #6
0
 def __new__(cls, *args, **kwargs):
     if (len(args) == 1 and args[0].get("index") == "meta") or kwargs.get("index") == "meta":
         output = FromESMetadata.__new__(FromESMetadata, *args, **kwargs)
         output.__init__(*args, **kwargs)
         return output
     else:
         return Container.__new__(cls)
コード例 #7
0
ファイル: qb_usingES.py プロジェクト: klahnakoski/MoDevETL
 def __new__(cls, *args, **kwargs):
     if (len(args) == 1 and args[0].get("index") == "meta") or kwargs.get("index") == "meta":
         output = FromESMetadata.__new__(FromESMetadata, *args, **kwargs)
         output.__init__(*args, **kwargs)
         return output
     else:
         return Container.__new__(cls)
コード例 #8
0
    def __init__(
            self,
            host,
            index,
            type=None,
            alias=None,
            name=None,
            port=9200,
            read_only=True,
            timeout=None,  # NUMBER OF SECONDS TO WAIT FOR RESPONSE, OR SECONDS TO WAIT FOR DOWNLOAD (PASSED TO requests)
            consistency="one",  # ES WRITE CONSISTENCY (https://www.elastic.co/guide/en/elasticsearch/reference/1.7/docs-index_.html#index-consistency)
            typed=None,
            kwargs=None):
        Container.__init__(self, None)
        if not containers.config.default:
            containers.config.default.settings = kwargs
        self.settings = kwargs
        self.name = coalesce(name, alias, index)
        if read_only:
            self._es = elasticsearch.Alias(alias=coalesce(alias, index),
                                           kwargs=kwargs)
        else:
            self._es = elasticsearch.Cluster(kwargs=kwargs).get_index(
                read_only=read_only, kwargs=kwargs)

        self.meta = FromESMetadata(kwargs=kwargs)
        self.settings.type = self._es.settings.type
        self.edges = Data()
        self.worker = None

        columns = self.meta.get_columns(
            table_name=coalesce(name, alias, index))
        self._schema = Schema(coalesce(name, alias, index), columns)

        if typed == None:
            # SWITCH ON TYPED MODE
            self.typed = any(c.names["."] in ("$value", "$object")
                             for c in columns)
        else:
            self.typed = typed
コード例 #9
0
    def __init__(
        self,
        host,
        index,
        type=None,
        alias=None,
        name=None,
        port=9200,
        read_only=True,
        timeout=None,  # NUMBER OF SECONDS TO WAIT FOR RESPONSE, OR SECONDS TO WAIT FOR DOWNLOAD (PASSED TO requests)
        consistency="one",  # ES WRITE CONSISTENCY (https://www.elastic.co/guide/en/elasticsearch/reference/1.7/docs-index_.html#index-consistency)
        typed=None,
        settings=None,
    ):
        Container.__init__(self, None)
        if not containers.config.default:
            containers.config.default.settings = settings
        self.settings = settings
        self.name = coalesce(name, alias, index)
        if read_only:
            self._es = elasticsearch.Alias(alias=coalesce(alias, index), settings=settings)
        else:
            self._es = elasticsearch.Cluster(settings=settings).get_index(read_only=read_only, settings=settings)

        self.meta = FromESMetadata(settings=settings)
        self.settings.type = self._es.settings.type
        self.edges = Data()
        self.worker = None

        columns = self.get_columns(table_name=name)
        self._schema = Schema(columns)

        if typed == None:
            # SWITCH ON TYPED MODE
            self.typed = any(c.name in ("$value", "$object") for c in columns)
        else:
            self.typed = typed
コード例 #10
0
    def __init__(self, name, db=None, uid=GUID, exists=False, kwargs=None):
        """
        :param name: NAME FOR THIS TABLE
        :param db: THE DB TO USE
        :param uid: THE UNIQUE INDEX FOR THIS TABLE
        :return: HANDLE FOR TABLE IN db
        """
        global _config
        Container.__init__(self, frum=None)
        if db:
            self.db = db
        else:
            self.db = db = Sqlite()

        if not _config:
            from pyLibrary.queries.containers import config as _config
            if not _config.default:
                _config.default = {"type": "sqlite", "settings": {"db": db}}

        self.name = name
        self.uid = listwrap(uid)
        self._next_uid = 1
        self._make_digits_table()

        self.uid_accessor = jx.get(self.uid)
        self.nested_tables = OrderedDict(
        )  # MAP FROM NESTED PATH TO Table OBJECT, PARENTS PROCEED CHILDREN
        self.nested_tables["."] = self
        self.columns = Index(
            keys=[join_field(["names", self.name])]
        )  # MAP FROM DOCUMENT ABS PROPERTY NAME TO THE SET OF SQL COLUMNS IT REPRESENTS (ONE FOR EACH REALIZED DATATYPE)

        if not exists:
            for u in self.uid:
                if u == GUID:
                    pass
                else:
                    c = Column(names={name: u},
                               type="string",
                               es_column=typed_column(u, "string"),
                               es_index=name)
                    self.add_column_to_schema(self.nested_tables, c)

            command = ("CREATE TABLE " + quote_table(name) + "(" +
                       (",".join([quoted_UID + " INTEGER"] + [
                           _quote_column(c) + " " + sql_types[c.type]
                           for u, cs in self.columns.items() for c in cs
                       ])) + ", PRIMARY KEY (" + (", ".join([quoted_UID] + [
                           _quote_column(c) for u in self.uid
                           for c in self.columns[u]
                       ])) + "))")

            self.db.execute(command)
        else:
            # LOAD THE COLUMNS
            command = "PRAGMA table_info(" + quote_table(name) + ")"
            details = self.db.query(command)

            for r in details:
                cname = untyped_column(r[1])
                ctype = r[2].lower()
                column = Column(names={name: cname},
                                type=ctype,
                                nested_path=['.'],
                                es_column=typed_column(cname, ctype),
                                es_index=name)

                self.add_column_to_schema(self.columns, column)
コード例 #11
0
    def __init__(self, name, db=None, uid=UID_PREFIX + "id", exists=False):
        """
        :param name: NAME FOR THIS TABLE
        :param db: THE DB TO USE
        :param uid: THE UNIQUE INDEX FOR THIS TABLE
        :return: HANDLE FOR TABLE IN db
        """
        Container.__init__(self, frum=None)
        if db:
            self.db = db
        else:
            self.db = db = Sqlite()
        self.name = name
        self.uid = listwrap(uid)

        self.columns = {}
        for u in uid:
            if not self.columns.get(u, None):
                cs = self.columns[u] = set()
            if u.startswith(UID_PREFIX):
                cs.add(
                    Column(name=u,
                           table=name,
                           type="integer",
                           es_column=typed_column(u, "integer"),
                           es_index=name))
            else:
                cs.add(
                    Column(name=u,
                           table=name,
                           type="text",
                           es_column=typed_column(u, "text"),
                           es_index=name))

        self.uid_accessor = jx.get(self.uid)
        self.nested_tables = {}  # MAP FROM TABLE NAME TO Table OBJECT
        if exists:
            # LOAD THE COLUMNS
            command = "PRAGMA table_info(" + quote_table(name) + ")"
            details = self.db.query(command)
            self.columns = {}
            for r in details:
                cname = untyped_column(r[1])
                ctype = r[2].lower()
                column = Column(name=cname,
                                table=name,
                                type=ctype,
                                es_column=typed_column(cname, ctype),
                                es_index=name)

                cs = self.columns.get(name, Null)
                if not cs:
                    cs = self.columns[name] = set()
                cs.add(column)
        else:
            command = "CREATE TABLE " + quote_table(name) + "(" + \
                      (",".join(_quote_column(c) + " " + c.type for u, cs in self.columns.items() for c in cs)) + \
                      ", PRIMARY KEY (" + \
                      (", ".join(_quote_column(c) for u in self.uid for c in self.columns[u])) + \
                      "))"

            self.db.execute(command)