Ejemplo n.º 1
0
    def obj_get(self, bundle, **kwargs):

        gdb = GraphDatabase(GRAPH_DATABASE_REST_URL)
        document = gdb.nodes.get(GRAPH_DATABASE_REST_URL + "node/" +
                                 kwargs['pk'] + '/')

        new_obj = DataObject(kwargs['pk'])
        new_obj.__dict__['_data'] = document.properties
        new_obj.__dict__['_data']['id'] = kwargs['pk']

        sentences = gdb.query(
            """MATCH (d:`Document`)-[:sentences]->(s:`Sentence`) WHERE d.CTS='"""
            + document.properties['CTS'] +
            """' RETURN DISTINCT s ORDER BY ID(s)""")
        sentenceArray = []
        for s in sentences:
            sent = s[0]
            url = sent['self'].split('/')
            # this might seems a little hacky, but API resources are very decoupled,
            # which gives us great performance instead of creating relations amongst objects and referencing/dereferencing foreign keyed fields
            sent['data']['resource_uri'] = API_PATH + 'sentence/' + url[
                len(url) - 1] + '/'
            sentenceArray.append(sent['data'])

        if ENABLE_DISPLAYING_LONG_DOCUMENTS:
            if len(sentenceArray) > 500:
                new_obj.__dict__['_data']['sentences'] = sentenceArray
            else:
                new_obj.__dict__['_data']['sentences'] = sort_sentences(
                    sentenceArray)
        else:
            new_obj.__dict__['_data']['sentences'] = sort_sentences(
                sentenceArray)

        # get a dictionary of related translations of this document
        relatedDocuments = gdb.query(
            """MATCH (d:`Document`)-[:sentences]->(s:`Sentence`)-[:words]->(w:`Word`)-[:translation]->(t:`Word`)<-[:words]-(s1:`Sentence`)<-[:sentences]-(d1:`Document`) WHERE HAS (d.CTS) AND d.CTS='"""
            + document.properties['CTS'] +
            """' RETURN DISTINCT d1 ORDER BY ID(d1)""")

        new_obj.__dict__['_data']['translations'] = {}
        for rd in relatedDocuments:
            doc = rd[0]
            url = doc['self'].split('/')
            if doc['data']['lang'] in CTS_LANG:
                new_obj.__dict__['_data']['translations'][
                    doc['data']['lang']] = doc['data']
                new_obj.__dict__['_data']['translations'][doc['data']['lang']][
                    'resource_uri'] = API_PATH + 'document/' + url[len(url) -
                                                                   1] + '/'

        return new_obj
Ejemplo n.º 2
0
    def obj_get(self, bundle, **kwargs):
        
        gdb = GraphDatabase(GRAPH_DATABASE_REST_URL)
        document = gdb.nodes.get(GRAPH_DATABASE_REST_URL + "node/" + kwargs['pk'] + '/')
        
        new_obj = DataObject(kwargs['pk'])
        new_obj.__dict__['_data'] = document.properties
        new_obj.__dict__['_data']['id'] = kwargs['pk']
        
        sentences = gdb.query("""MATCH (d:`Document`)-[:sentences]->(s:`Sentence`) WHERE d.CTS='""" +document.properties['CTS']+ """' RETURN DISTINCT s ORDER BY ID(s)""")
        sentenceArray = []
        for s in sentences:
            sent = s[0]
            url = sent['self'].split('/')
            # this might seems a little hacky, but API resources are very decoupled,
            # which gives us great performance instead of creating relations amongst objects and referencing/dereferencing foreign keyed fields
            sent['data']['resource_uri'] = API_PATH + 'sentence/' + url[len(url)-1] + '/'
            sentenceArray.append(sent['data'])

        new_obj.__dict__['_data']['sentences'] = sort_sentences(sentenceArray)
            
        
        # get a dictionary of related translations of this document
        relatedDocuments = gdb.query("""MATCH (d:`Document`)-[:sentences]->(s:`Sentence`)-[:words]->(w:`Word`)-[:translation]->(t:`Word`)<-[:words]-(s1:`Sentence`)<-[:sentences]-(d1:`Document`) WHERE HAS (d.CTS) AND d.CTS='"""+ document.properties['CTS'] +"""' RETURN DISTINCT d1 ORDER BY ID(d1)""")
        
        new_obj.__dict__['_data']['translations']={}
        for rd in relatedDocuments:
            doc = rd[0]
            url = doc['self'].split('/')
            if doc['data']['lang'] in CTS_LANG:
                new_obj.__dict__['_data']['translations'][doc['data']['lang']] = doc['data']
                new_obj.__dict__['_data']['translations'][doc['data']['lang']]['resource_uri']= API_PATH + 'document/' + url[len(url)-1] +'/'


        return new_obj
Ejemplo n.º 3
0
    def analyze(self, text,
                lower=False,
                source='no_filter',
                sim_func=utils.get_similarity,
                pagerank_config={'alpha': 0.85}):

        self.key_sentences = []
        result = self.seg.segment(text=text, lower=lower)
        self.sentences = result.sentences
        self.words_no_filter = result.words_no_filter
        _source = result['words_' + source]

        self.key_sentences = utils.sort_sentences(
            sentences=self.sentences, words=_source)
Ejemplo n.º 4
0
    def analyze(self, text, lower = False,
                source = 'no_stop_words',
                sim_func = utils.get_similarity,
                damping_coeffi=0.85):
        self.key_sentences = list()
        result = self.seg.segment(text=text, lower=lower)

        self.sentences = result.sentences

        self.words_no_filter = result.words_no_filter
        self.words_no_stop_words = result.words_no_stop_words
        self.words_all_filters = result.words_all_filters
        options = ['no_filter', 'no_stop_words', 'all_filters']

        if source in options:
            source_ = result['words_' + source]
        else:
            source_ = result['words_no_stop_words']

        self.key_sentences = utils.sort_sentences(sentences=self.sentences, words=source_, sim_func=sim_func, damping_coeffi=damping_coeffi)
Ejemplo n.º 5
0
    def get_object_list(self, request):

        gdb = GraphDatabase(GRAPH_DATABASE_REST_URL)
        attrlist = ['CTS', 'length', 'sentence']
        sentences = []

        query_params = {}
        for obj in request.GET.keys():
            if obj in attrlist and request.GET.get(obj) is not None:
                query_params[obj] = request.GET.get(obj)
            elif obj.split('__')[0] in attrlist and request.GET.get(
                    obj) is not None:
                query_params[obj] = request.GET.get(obj)

        # implement filtering
        if len(query_params) > 0:

            # generate query
            q = """MATCH (d:`Document`)-[:sentences]->(s:`Sentence`) WHERE """

            # filter word on parameters
            for key in query_params:
                if len(key.split('__')) > 1:
                    if key.split('__')[1] == 'contains':
                        q = q + """HAS (s.""" + key.split(
                            '__')[0] + """) AND s.""" + key.split(
                                '__')[0] + """=~'.*""" + query_params[
                                    key] + """.*' AND """
                    elif key.split('__')[1] == 'startswith':
                        q = q + """HAS (s.""" + key.split(
                            '__')[0] + """) AND s.""" + key.split(
                                '__')[0] + """=~'""" + query_params[
                                    key] + """.*' AND """
                    elif key.split('__')[1] == 'endswith':
                        q = q + """HAS (s.""" + key.split(
                            '__')[0] + """) AND s.""" + key.split(
                                '__')[0] + """=~'.*""" + query_params[
                                    key] + """' AND """
                    elif key.split('__')[1] == 'gt':
                        q = q + """HAS (s.""" + key.split(
                            '__')[0] + """) AND s.""" + key.split('__')[
                                0] + """>""" + query_params[key] + """ AND """
                    elif key.split('__')[1] == 'lt':
                        q = q + """HAS (s.""" + key.split(
                            '__')[0] + """) AND s.""" + key.split('__')[
                                0] + """<""" + query_params[key] + """ AND """
                    elif key.split('__')[1] == 'isnot':
                        if key.split('__')[0] == 'length':
                            q = q + """HAS (s.""" + key.split(
                                '__')[0] + """) AND s.""" + key.split(
                                    '__')[0] + """<>""" + query_params[
                                        key] + """ AND """
                        else:
                            q = q + """HAS (s.""" + key.split(
                                '__')[0] + """) AND s.""" + key.split(
                                    '__')[0] + """<>'""" + query_params[
                                        key] + """' AND """
                else:
                    if key == 'length':
                        q = q + """HAS (s.""" + key + """) AND s.""" + key + """=""" + query_params[
                            key] + """ AND """
                    else:
                        q = q + """HAS (s.""" + key + """) AND s.""" + key + """='""" + query_params[
                            key] + """' AND """
            q = q[:len(q) - 4]
            q = q + """RETURN s, d ORDER BY ID(s)"""

            table = gdb.query(q)

        # default querying
        else:
            table = gdb.query(
                """MATCH (d:`Document`)-[:sentences]->(s:`Sentence`) WHERE HAS (s.CTS) RETURN s, d ORDER BY ID(s)"""
            )

        # create the objects which was queried for and set all necessary attributes
        for t in table:
            sentence = t[0]
            document = t[1]
            url = sentence['self'].split('/')
            urlDoc = document['self'].split('/')

            new_obj = DataObject(url[len(url) - 1])
            new_obj.__dict__['_data'] = sentence['data']
            new_obj.__dict__['_data']['id'] = url[len(url) - 1]
            new_obj.__dict__['_data'][
                'document_resource_uri'] = API_PATH + 'document/' + urlDoc[
                    len(urlDoc) - 1] + '/'
            sentences.append(new_obj)

        if ENABLE_DISPLAYING_LONG_DOCUMENTS:
            if len(sentences) > 500:
                return sentences
            else:
                return sort_sentences(sentences)
        else:
            return sort_sentences(sentences)

        return sort_sentences(sentences)
Ejemplo n.º 6
0
 def get_object_list(self, request):
     
     gdb = GraphDatabase(GRAPH_DATABASE_REST_URL)    
     attrlist = ['CTS', 'name', 'name_eng', 'lang', 'author']
     documents = []
     
     query_params = {}
     for obj in request.GET.keys():
         if obj in attrlist and request.GET.get(obj) is not None:
             query_params[obj] = request.GET.get(obj)
         elif obj.split('__')[0] in attrlist and request.GET.get(obj) is not None:
             query_params[obj] = request.GET.get(obj)
     
     # implement filtering
     if len(query_params) > 0:
         
         # generate query
         q = """MATCH (d:`Document`)-[:sentences]->(s:`Sentence`) WHERE """
         
         # filter word on parameters
         for key in query_params:
             if len(key.split('__')) > 1:
                 if key.split('__')[1] == 'contains':
                     q = q + """HAS (d.""" +key.split('__')[0]+ """) AND d.""" +key.split('__')[0]+ """=~'.*""" +query_params[key]+ """.*' AND """
                 elif key.split('__')[1] == 'startswith':
                     q = q + """HAS (d.""" +key.split('__')[0]+ """) AND d.""" +key.split('__')[0]+ """=~'""" +query_params[key]+ """.*' AND """
                 elif key.split('__')[1] == 'endswith':
                     q = q + """HAS (d.""" +key.split('__')[0]+ """) AND d.""" +key.split('__')[0]+ """=~'.*""" +query_params[key]+ """' AND """
                 elif key.split('__')[1] == 'isnot':
                     q = q + """HAS (d.""" +key.split('__')[0]+ """) AND d.""" +key.split('__')[0]+ """<>'""" +query_params[key]+ """' AND """
             else:
                 q = q + """HAS (d.""" +key+ """) AND d.""" +key+ """='""" +query_params[key]+ """' AND """
         q = q[:len(q)-4]
         q = q + """RETURN DISTINCT d ORDER BY ID(d)"""
         
         table = gdb.query(q)
     
     # default querying    
     else:    
         table = gdb.query("""MATCH (d:`Document`) RETURN DISTINCT d ORDER BY ID(d)""")
         
     # create the objects which was queried for and set all necessary attributes
     for t in table:
         document = t[0]        
         urlDoc = document['self'].split('/')        
             
         new_obj = DataObject(urlDoc[len(urlDoc)-1])
         new_obj.__dict__['_data'] = document['data']        
         new_obj.__dict__['_data']['id'] = urlDoc[len(urlDoc)-1]
     
         sentences = gdb.query("""MATCH (d:`Document`)-[:sentences]->(s:`Sentence`) WHERE d.CTS='""" +document['data']['CTS']+ """' RETURN DISTINCT s ORDER BY ID(s)""")
         sentenceArray = []
         for s in sentences:
             
             sent = s[0]
             url = sent['self'].split('/')
             sent_cts = sent['data']['CTS']
             sent['data'] = {}
             sent['data']['resource_uri'] = API_PATH + 'sentence/' + url[len(url)-1] + '/'
             sent['data']['CTS'] = sent_cts
             sentenceArray.append(sent['data'])
             
         if ENABLE_DISPLAYING_LONG_DOCUMENTS:
             if len(sentenceArray) > 500:
                 new_obj.__dict__['_data']['sentences'] = sentenceArray
             else:
                 new_obj.__dict__['_data']['sentences'] = sort_sentences(sentenceArray)
         else:
             new_obj.__dict__['_data']['sentences'] = sort_sentences(sentenceArray)
         
         documents.append(new_obj)        
             
     return documents
Ejemplo n.º 7
0
    def get_object_list(self, request):

        gdb = GraphDatabase(GRAPH_DATABASE_REST_URL)
        attrlist = ['CTS', 'name', 'name_eng', 'lang', 'author']
        documents = []

        query_params = {}
        for obj in request.GET.keys():
            if obj in attrlist and request.GET.get(obj) is not None:
                query_params[obj] = request.GET.get(obj)
            elif obj.split('__')[0] in attrlist and request.GET.get(
                    obj) is not None:
                query_params[obj] = request.GET.get(obj)

        # implement filtering
        if len(query_params) > 0:

            # generate query
            q = """MATCH (d:`Document`)-[:sentences]->(s:`Sentence`) WHERE """

            # filter word on parameters
            for key in query_params:
                if len(key.split('__')) > 1:
                    if key.split('__')[1] == 'contains':
                        q = q + """HAS (d.""" + key.split(
                            '__')[0] + """) AND d.""" + key.split(
                                '__')[0] + """=~'.*""" + query_params[
                                    key] + """.*' AND """
                    elif key.split('__')[1] == 'startswith':
                        q = q + """HAS (d.""" + key.split(
                            '__')[0] + """) AND d.""" + key.split(
                                '__')[0] + """=~'""" + query_params[
                                    key] + """.*' AND """
                    elif key.split('__')[1] == 'endswith':
                        q = q + """HAS (d.""" + key.split(
                            '__')[0] + """) AND d.""" + key.split(
                                '__')[0] + """=~'.*""" + query_params[
                                    key] + """' AND """
                    elif key.split('__')[1] == 'isnot':
                        q = q + """HAS (d.""" + key.split(
                            '__')[0] + """) AND d.""" + key.split(
                                '__'
                            )[0] + """<>'""" + query_params[key] + """' AND """
                else:
                    q = q + """HAS (d.""" + key + """) AND d.""" + key + """='""" + query_params[
                        key] + """' AND """
            q = q[:len(q) - 4]
            q = q + """RETURN DISTINCT d ORDER BY ID(d)"""

            table = gdb.query(q)

        # default querying
        else:
            table = gdb.query(
                """MATCH (d:`Document`) RETURN DISTINCT d ORDER BY ID(d)""")

        # create the objects which was queried for and set all necessary attributes
        for t in table:
            document = t[0]
            urlDoc = document['self'].split('/')

            new_obj = DataObject(urlDoc[len(urlDoc) - 1])
            new_obj.__dict__['_data'] = document['data']
            new_obj.__dict__['_data']['id'] = urlDoc[len(urlDoc) - 1]

            sentences = gdb.query(
                """MATCH (d:`Document`)-[:sentences]->(s:`Sentence`) WHERE d.CTS='"""
                + document['data']['CTS'] +
                """' RETURN DISTINCT s ORDER BY ID(s)""")
            sentenceArray = []
            for s in sentences:

                sent = s[0]
                url = sent['self'].split('/')
                sent_cts = sent['data']['CTS']
                sent['data'] = {}
                sent['data']['resource_uri'] = API_PATH + 'sentence/' + url[
                    len(url) - 1] + '/'
                sent['data']['CTS'] = sent_cts
                sentenceArray.append(sent['data'])

            if ENABLE_DISPLAYING_LONG_DOCUMENTS:
                if len(sentenceArray) > 500:
                    new_obj.__dict__['_data']['sentences'] = sentenceArray
                else:
                    new_obj.__dict__['_data']['sentences'] = sort_sentences(
                        sentenceArray)
            else:
                new_obj.__dict__['_data']['sentences'] = sort_sentences(
                    sentenceArray)

            documents.append(new_obj)

        return documents
Ejemplo n.º 8
0
 def get_object_list(self, request):
     
     gdb = GraphDatabase(GRAPH_DATABASE_REST_URL)    
     attrlist = ['CTS', 'length', 'sentence']
     sentences = []
     
     query_params = {}
     for obj in request.GET.keys():
         if obj in attrlist and request.GET.get(obj) is not None:
             query_params[obj] = request.GET.get(obj)
         elif obj.split('__')[0] in attrlist and request.GET.get(obj) is not None:
             query_params[obj] = request.GET.get(obj)
     
     # implement filtering
     if len(query_params) > 0:
         
         # generate query
         q = """MATCH (d:`Document`)-[:sentences]->(s:`Sentence`) WHERE """
         
         # filter word on parameters
         for key in query_params:                    
             if len(key.split('__')) > 1:
                 if key.split('__')[1] == 'contains':
                     q = q + """HAS (s.""" +key.split('__')[0]+ """) AND s.""" +key.split('__')[0]+ """=~'.*""" +query_params[key]+ """.*' AND """
                 elif key.split('__')[1] == 'startswith':
                     q = q + """HAS (s.""" +key.split('__')[0]+ """) AND s.""" +key.split('__')[0]+ """=~'""" +query_params[key]+ """.*' AND """
                 elif key.split('__')[1] == 'endswith':
                     q = q + """HAS (s.""" +key.split('__')[0]+ """) AND s.""" +key.split('__')[0]+ """=~'.*""" +query_params[key]+ """' AND """
                 elif key.split('__')[1] == 'gt':
                     q = q + """HAS (s.""" +key.split('__')[0]+ """) AND s.""" +key.split('__')[0]+ """>""" +query_params[key]+ """ AND """
                 elif key.split('__')[1] == 'lt':
                     q = q + """HAS (s.""" +key.split('__')[0]+ """) AND s.""" +key.split('__')[0]+ """<""" +query_params[key]+ """ AND """    
                 elif key.split('__')[1] == 'isnot':
                     if key.split('__')[0] == 'length':
                         q = q + """HAS (s.""" +key.split('__')[0]+ """) AND s.""" +key.split('__')[0]+ """<>""" +query_params[key]+ """ AND """
                     else:
                         q = q + """HAS (s.""" +key.split('__')[0]+ """) AND s.""" +key.split('__')[0]+ """<>'""" +query_params[key]+ """' AND """
             else:
                 if key == 'length':
                     q = q + """HAS (s.""" +key+ """) AND s.""" +key+ """=""" +query_params[key]+ """ AND """
                 else:
                     q = q + """HAS (s.""" +key+ """) AND s.""" +key+ """='""" +query_params[key]+ """' AND """
         q = q[:len(q)-4]
         q = q + """RETURN s, d ORDER BY ID(s)"""
         
         table = gdb.query(q)
     
     # default querying    
     else:    
         table = gdb.query("""MATCH (d:`Document`)-[:sentences]->(s:`Sentence`) WHERE HAS (s.CTS) RETURN s, d ORDER BY ID(s)""")
         
     # create the objects which was queried for and set all necessary attributes
     for t in table:
         sentence = t[0]
         document = t[1]        
         url = sentence['self'].split('/')
         urlDoc = document['self'].split('/')        
             
         new_obj = DataObject(url[len(url)-1])
         new_obj.__dict__['_data'] = sentence['data']        
         new_obj.__dict__['_data']['id'] = url[len(url)-1]
         new_obj.__dict__['_data']['document_resource_uri'] = API_PATH + 'document/' + urlDoc[len(urlDoc)-1] +'/'
         sentences.append(new_obj)
             
     return sort_sentences(sentences)