Beispiel #1
0
def main():
    start_time = time()

    wiki_info_file = 'e:/el/tmpres/demo/wiki-all/wiki-info.pkl'
    links_file = 'e:/el/tmpres/demo/wiki-all/links.txt'
    description_file = 'e:/el/tmpres/demo/wiki-all/text.txt'
    wiki_candidates_file = 'e:/el/tmpres/wiki/dict/name_candidates.pkl'
    word_idf_file = 'e:/el/tmpres/demo/word_idf.txt'

    tfidf = TfIdf(word_idf_file)

    wiki_info = WikiInfo(wiki_info_file, links_file, description_file)
    wiki_link = WikiLink(wiki_candidates_file, wiki_info, tfidf)

    input_file = 'input/00000001.txt'
    fin = open(input_file, 'rb')
    doc_text = fin.read()
    doc_text = doc_text.decode('utf-8')
    fin.close()

    pos = input_file.rfind('/')
    file_name = input_file[pos + 1:]
    ner_result_file = os.path.join('output', file_name + '.ner')
    merged_mention_list = mentiondetection.clean_ner_result(ner_result_file)

    merged_mention_list.sort(key=lambda x: x.span[0])
    wiki_link.link_all(doc_text, merged_mention_list)
    for mention in merged_mention_list:
        if (not mention.mesh_id) and mention.chebi_id < 0 < mention.wid:
            cur_name = doc_text[mention.span[0]:mention.span[1] + 1].lower()
            print cur_name, mention.wid, wiki_info.get_info(mention.wid)[0]

    print time() - start_time
Beispiel #2
0
def test():
    start_time = time()

    text = 'last opportunities Texas senator Cruz'

    word_idf_file = 'e:/el/tmpres/demo/merge/word_idf.txt'
    tfidf = TfIdf(word_idf_file)

    wiki_info_file = 'e:/el/tmpres/demo/wiki-all/wiki-info.pkl'
    links_file = 'e:/el/tmpres/demo/wiki-all/links.txt'
    description_file = 'e:/el/tmpres/demo/wiki-all/text.txt'
    wiki_info = WikiInfo(wiki_info_file, links_file, description_file)
    wiki_link = WikiLink('e:/el/tmpres/wiki/dict/name_candidates.pkl', wiki_info, tfidf)
    context_tfidf = tfidf.get_tfidf_from_text(text)
    print wiki_link.link_with_context('cruz', context_tfidf)

    print time() - start_time
Beispiel #3
0
def test():
    start_time = time()

    text = 'last opportunities Texas senator Cruz'

    word_idf_file = 'e:/el/tmpres/demo/merge/word_idf.txt'
    tfidf = TfIdf(word_idf_file)

    wiki_info_file = 'e:/el/tmpres/demo/wiki-all/wiki-info.pkl'
    links_file = 'e:/el/tmpres/demo/wiki-all/links.txt'
    description_file = 'e:/el/tmpres/demo/wiki-all/text.txt'
    wiki_info = WikiInfo(wiki_info_file, links_file, description_file)
    wiki_link = WikiLink('e:/el/tmpres/wiki/dict/name_candidates.pkl',
                         wiki_info, tfidf)
    context_tfidf = tfidf.get_tfidf_from_text(text)
    print wiki_link.link_with_context('cruz', context_tfidf)

    print time() - start_time
Beispiel #4
0
def init_model():
    # extra_wiki_desc_file = 'e:/el/tmpres/demo/merge/wiki_extra_sentences.txt'
    # extra_parents_file = 'e:/el/tmpres/demo/extra_parents.txt'
    #
    # name_wid_file = 'e:/el/tmpres/demo/dict/single_candidates_wid_dict.txt'
    # record_file = 'd:/data/lab_demo/med_edl_data/records_info_with_wiki.txt'
    # dict_file = 'd:/data/lab_demo/med_edl_data/med_dict_ascii_with_ids_edited.txt'
    # tree_number_file = 'd:/data/lab_demo/med_edl_data/id_tn.txt'

    # res_dir = '/media/dhl/Data/el/tmpres/demo/del-data/'

    # input_file = '/media/dhl/Data/el/tmpres/NER/NER/00000001.txt.bak'
    # output_file = '/media/dhl/Data/el/tmpres/demo/result/result-linux.json'

    res_dir = 'e:/data/el/tmpres/demo/del-data/'
    extra_wiki_desc_file = res_dir + 'wiki_extra_sentences.txt'
    extra_parents_file = res_dir + 'extra_parents.txt'
    mesh_record_file = res_dir + 'records_info_with_wiki.txt'
    mesh_dict_file = res_dir + 'med_dict_ascii_with_ids_edited.txt'
    exclude_words_file = res_dir + 'exclude_words.txt'
    tree_number_file = res_dir + 'id_tn.txt'
    obo_file = res_dir + 'chebi.obo'

    word_idf_file = 'e:/data/el/tmpres/demo/word_idf.txt'

    # wiki_candidates_file = 'e:/el/tmpres/wiki/dict/name_candidates.txt'
    wiki_candidates_file = 'e:/data/el/tmpres/wiki/dict/name_candidates.pkl'

    # wiki_info_file = r'E:\el\tmpres\demo\wiki-med\new\wiki-info.txt'
    # links_file = r'E:\el\tmpres\demo\wiki-med\new\links.txt'
    # description_file = r'E:\el\tmpres\demo\wiki-med\new\text.txt'

    wiki_info_file = 'e:/data/el/tmpres/demo/wiki-all/wiki-info.pkl'
    links_file = 'e:/data/el/tmpres/demo/wiki-all/links.txt'
    description_file = 'e:/data/el/tmpres/demo/wiki-all/text.txt'

    mesh_extra_description_file = 'e:/data/el/tmpres/demo/extra_description_for_mesh.txt'

    chebi_terms = ChebiTerm.load_obo_file(obo_file)
    print '%d chebi terms' % len(chebi_terms)

    mesh_match = MeshMatch(mesh_dict_file, exclude_words_file)
    mesh_records = MeshRecord.load_mesh_records(mesh_record_file)
    mesh_tree = MeshTree(tree_number_file, mesh_records)

    wiki_info = WikiInfo(wiki_info_file, links_file, description_file)
    tfidf = TfIdf(word_idf_file)
    wiki_link = WikiLink(wiki_candidates_file, wiki_info, tfidf)
    extra_wiki_desc = ioutils.load_wiki_extra_descriptions(
        mesh_extra_description_file)
    # extra_wiki_desc = ioutils.load_wiki_extra_sentences(extra_wiki_desc_file)

    med_link = MedLink(extra_parents_file, mesh_match, mesh_records, mesh_tree,
                       chebi_terms, wiki_info, extra_wiki_desc, wiki_link)
    return med_link
Beispiel #5
0
def init_model():
    res_dir = 'e:/data/el/tmpres/'
    # res_dir = '/home/dhl/data/el/tmpres/'
    del_res_dir = os.path.join(res_dir, 'demo/del-data/')
    # extra_wiki_desc_file = del_res_dir + 'wiki_extra_sentences.txt'
    # extra_parents_file = del_res_dir + 'extra_parents.txt'
    # mesh_record_file = del_res_dir + 'records_info_with_wiki.txt'
    # mesh_dict_file = del_res_dir + 'med_dict_ascii_with_ids_edited.txt'
    # exclude_words_file = del_res_dir + 'exclude_words.txt'
    # tree_number_file = del_res_dir + 'id_tn.txt'
    # obo_file = del_res_dir + 'chebi.obo'

    extra_wiki_desc_file = os.path.join(del_res_dir,
                                        'wiki_extra_sentences.txt')
    extra_parents_file = os.path.join(del_res_dir, 'extra_parents.txt')
    mesh_record_file = os.path.join(del_res_dir, 'records_info_with_wiki.txt')
    mesh_dict_file = os.path.join(del_res_dir,
                                  'med_dict_ascii_with_ids_edited.txt')
    exclude_words_file = os.path.join(del_res_dir, 'exclude_words.txt')
    tree_number_file = os.path.join(del_res_dir, 'id_tn.txt')
    obo_file = os.path.join(del_res_dir, 'chebi.obo')

    word_idf_file = os.path.join(res_dir, 'demo/word_idf.txt')
    wiki_candidates_file = os.path.join(res_dir,
                                        'wiki/dict/name_candidates.pkl')
    wiki_info_file = os.path.join(res_dir, 'demo/wiki-all/wiki-info.pkl')
    links_file = os.path.join(res_dir, 'demo/wiki-all/links.txt')
    description_file = os.path.join(res_dir, 'demo/wiki-all/text.txt')
    mesh_extra_description_file = os.path.join(
        res_dir, 'demo/extra_description_for_mesh.txt')

    # wiki_candidates_file = 'e:/el/tmpres/wiki/dict/name_candidates.txt'
    # wiki_candidates_file = 'e:/data/el/tmpres/wiki/dict/name_candidates.pkl'
    # wiki_info_file = 'e:/data/el/tmpres/demo/wiki-all/wiki-info.pkl'
    # links_file = 'e:/data/el/tmpres/demo/wiki-all/links.txt'
    # description_file = 'e:/data/el/tmpres/demo/wiki-all/text.txt'
    # mesh_extra_description_file = 'e:/data/el/tmpres/demo/extra_description_for_mesh.txt'

    chebi_terms = ChebiTerm.load_obo_file(obo_file)

    mesh_match = MeshMatch(mesh_dict_file, exclude_words_file)
    mesh_records = MeshRecord.load_mesh_records(mesh_record_file)
    mesh_tree = MeshTree(tree_number_file, mesh_records)

    wiki_info = WikiInfo(wiki_info_file, links_file, description_file)
    tfidf = TfIdf(word_idf_file)
    wiki_link = WikiLink(wiki_candidates_file, wiki_info, tfidf)
    extra_wiki_desc = ioutils.load_wiki_extra_descriptions(
        mesh_extra_description_file)
    # extra_wiki_desc = ioutils.load_wiki_extra_sentences(extra_wiki_desc_file)

    tmp_med_link = MedLink(extra_parents_file, mesh_match, mesh_records,
                           mesh_tree, chebi_terms, wiki_info, extra_wiki_desc,
                           wiki_link)
    return tmp_med_link
Beispiel #6
0
def __init_mellink():
    word_idf_file = 'e:/data/el/tmpres/demo/word_idf.txt'
    wiki_candidates_file = 'e:/data/el/tmpres/wiki/dict/name_candidates.pkl'
    wiki_info_file = 'e:/data/el/tmpres/demo/wiki-all/wiki-info.pkl'
    links_file = 'e:/data/el/tmpres/demo/wiki-all/links.txt'
    description_file = 'e:/data/el/tmpres/demo/wiki-all/text.txt'

    wiki_info = WikiInfo(wiki_info_file, links_file, description_file)
    tfidf = TfIdf(word_idf_file)
    wiki_link = WikiLink(wiki_candidates_file, wiki_info, tfidf)
    return MedLink(wiki_info=wiki_info, wiki_link=wiki_link)
Beispiel #7
0
    def wiki_merge(self, push_to_wiki=False, lifetime=False, overwrite_wiki=False):
        """
        Merge local collection with table on wiki
        """
        pagename=None
        if not lifetime:
            pagename=self._get_filename(nameonly=True)
        else:
            pagename='livstidsmedlemmer_fra_script'
        self.wl=WikiLink(pagename=pagename)
        ok, msg=self.wl.login()
        if not ok:
            return{u'status':u'Failure while accessing wiki. error: {}'.format(msg)}

        val={}
        if overwrite_wiki:
            self.wl.clear_page()
            val['pre']='== Medlemsliste ==\n'
            val['post']=''
        else:
            ok, val=self._merge()
            if not ok:
                return val

        if push_to_wiki:
            ok, table=self.wl.dict_to_table(self.storage, ['id', 'name', 'date', 'lifetime'], lifetime=lifetime)
            if not ok:
                return {u'status':table}
            text=u'{}{}{}'.format(val['pre'], table, val['post'])
            if overwrite_wiki:
                summary='Overwritten from script'
            else:
                summary='Merge from script'

            self.wl.write(text=text, summary=summary)

            return {u'status':u'Merged local storage with wiki and updated changes on wiki'}
        return {u'status':u'wiki merged with local storage'}
Beispiel #8
0
class Storage:
    """
    A simple class used to save data, and back it up.

    TODO: add sync to our `wiki`_. - Update! Almost done!

    .. _wiki: http://wiki.cyb.no
    """
    def __init__(self, **kwargs):
        """
        Constructor. Takes a python-dict `kwargs` as argument. kwargs must contain
        a list/dict `fields` where all values in `fields` will be required to create
        new objects and a single value `ident` used to fetch a specific object instead of an `id`.

        if kwargs contains boolean 'uniqueident' and it is set to True, then 'ident' must be unique
        on create.

        :param \*\*kwargs: dict containing valid fields for an object, and ident-field.
        """
        self.storage={}
        self.uniqueident=False
        self.objectname='Object'
        if 'fields' in kwargs:
            self.fields=kwargs['fields']
        if 'ident' in kwargs:
            self.ident=kwargs['ident']
        if 'uniqueident' in kwargs:
            self.uniqueident=kwargs['uniqueident']
        if 'objectname' in kwargs:
            self.objectname=kwargs['objectname']
        if 'gui' in kwargs:
            self.gui=kwargs['gui']
        self.wks=None
        self.wl=None

        self.filename=self._get_filename()


    def create(self, **kwargs):
        """
        Adds an object to collection with values from `kwargs`. If `self.fields` is used, only
        values from `fields` will be used, and all values from `fields` are required.

        All objects will automatically be given an `id` using :func:`_unique_id()<storage.Storage._unique_id>`.

        :param \*\*kwargs: dict with all fields for a new object.
        :returns: a dict with either 'success' containing the new object, or 'error' with errormessage.
        """
        if 'name' in kwargs:
            name=kwargs['name'].strip()
            if len(name.split())<2:
                return {u'error':u'You have to enter a full name!'}

            if len(name)<5:
                return {u'error':u'A name has to be more than 5 letters'}

        if self.uniqueident:
            if 'success' in self.read(**{self.ident:kwargs[self.ident]}):
                return {u'error':u'{} with {} {} already exists!'.format(self.objectname, self.ident, kwargs[self.ident])}

        args={}
        for f in self.fields:
            if not f in kwargs:
                return {u'error':u'missing field {} in arguments!'.format(f)}
            if kwargs[f]=='':
                return {u'error':u'missing value for field: {}!'.format(f)}
            value=u'{}'.format(kwargs[f]).strip().lower()
            args[f]=value#unicode(kwargs[f])

        key=self._unique_id()
        self.storage[key]=args
        return {u'success':key, 'object':args}

    def read(self, **kwargs):
        """
        Find and return a spesific object from storage. Either based on `id` or `self.ident`

        :param kwargs: dict containing either `id` or `self.ident`
        :returns: a dict with either 'success' containing the object, or 'error' with errormessage.
        """
        if len(self.storage)==0:
            return {u'error':u'empty collection'}

        if 'id' in kwargs:
            if kwargs['id'] in self.storage:
                return {u'success':self.storage[kwargs['id']]}

        match=0
        if self.ident in kwargs:
            for k, v in self.storage.iteritems():
                if v[self.ident]:
                    if v[self.ident]==kwargs[self.ident]:
                        retval=v
                        match+=1
        if match>1:
            return {u'error':u'multiple {}s matching name'.format(self.objectname)}
        if match==1:
            return {u'success':retval}
        return {u'error':u'no {}s found'.format(self.objectname)}

    def update(self, **kwargs):
        """
        find an element based on `id` and update with values in `kwargs`. Any values in kwargs that
        are not in self.fields are ignored.

        :param kwargs: dict containing `id` of object as well as updated fields.
        :returns: a dict with either 'success' containing the updated object, or 'error' with errormessage.
        """
        if len(self.storage)==0:
            return {u'error':u'empty collection'}

        if not 'id' in kwargs:
            return {u'error':u'No id provided'}

        if not kwargs['id'] in self.storage:
            return {u'error':u'Provided id does not exist'}

        if 'life' in kwargs:
            if kwargs['life']:
                return self._set_lifetime(kwargs['id'])
            else:
                return self._unset_lifetime(kwargs['id'])

        id=kwargs['id']
        del kwargs['id']
        obj=self.storage[id]

        for k, v in kwargs.iteritems():
            if k in self.fields:
                obj[k]=v

        return {u'success':self.storage[id]}


    def delete(self, **kwargs):
        """
        finds and deletes an object based on `id`

        :param kwargs: dict containing id of object.
        :returns: a dict with either 'success' containing the deleted `id`, or 'error' with errormessage.
        """

        if len(self.storage)==0:
            return {u'error':u'empty collection'}

        if not 'id' in kwargs:
            return {u'error':u'No id provided'}

        if not kwargs['id'] in self.storage:
            return {u'error':u'No {} matching id: {}'.format(self.objectname, kwargs['id'])}

        del self.storage[kwargs['id']]
        return {u'success':u'{} with id {} is deleted'.format(self.objectname, kwargs['id'])}


    def search(self, **kwargs):
        """
        search for any object that contains any values from `kwargs`.

        :param kwargs: dict containing search-fields. any field not in self.fields are ignored.
        :retval: returns a dictionary containing all objects that match search-fields.
        """
        retval={}
        for k, v in self.storage.iteritems():
            match=False
            for ak, av in kwargs.iteritems():
                if not match and ak in self.fields and not av=='':
                    value=v[ak]
                    if av in value:
                        retval[k]=v
                        match=True

        return retval

    def _set_lifetime(self, id):
        """
        Set a user as lifetime member.

        Give the user a lifetime ID: Lx where x is number.
        """
        if self.storage[id]['lifetime']=='y':
            return {u'error':u'{} already has a lifetime membership'.format(self.storage[id]['name'].title())}

        lid=self._unique_id(True)
        self.storage[lid]=self.storage[id]
        self.storage[lid]['lifetime']='y'
        del self.storage[id]
        return {u'success':u'{} now has id {} and a lifetime membership'.format(self.storage[lid]['name'].title(), lid)}

    def _unset_lifetime(self, lid):
        """
        Unset lifetime flag from a user.

        Give the user a normal id, instead of lifetime id.
        """
        if self.storage[lid]['lifetime']=='n':
            return {u'error':u'{} does not have a lifetime membership'.format(self.storage[id]['name'].title())}

        id=self._unique_id()
        self.storage[id]=self.storage[lid]
        self.storage[id]['lifetime']='n'
        del self.storage[lid]
        return {u'success':u'{} now has id {}, and have lost the lifetime membership'.format(self.storage[id]['name'].title(), id)}

    def update_id(self, oldid, newid):
        """
        Change/update the id of a person.
        """
        tmp=self.storage[oldid]
        if tmp['lifetime']=='y':
            if not 'L' in newid:
                newid='L{}'.format(newid)

        if newid in self.storage:
            return {'status':'FAILURE: Chosen id is already in use.'}

        self.storage[newid]=tmp

        del self.storage[oldid]

        return {'status':'Success! Updated the users id!'}

    def _unique_id(self, life=False):
        """
        Finds the first availible id and returns it.

        :returns: the first availible id.
        """
        if life:
            i=0
            val='L0'
            while val in self.storage:
                i+=1
                val='L{}'.format(i)
            return val

        i=0
        while i in self.storage:
            i+=1
        return i

    def save(self):
        """
        Saves entire collection to file. Filename is provided by :func:`_get_filename<storage.Storage._get_filename>`

        The collection is stored as JSON.
        """
        try:
            f=open(self.filename, 'w')
            js=json.dumps(self.storage, indent=3, sort_keys=True)
            f.write(js)
            f.close()
        except:
            return {'msg':'Could not write to file!', 'status':False}
        return {'msg':'Storage written to file.', 'status':True}

    def load(self):
        """
        Load from file if file exists. Filename is provided by :func:`_get_filename<storage.Storage._get_filename>`

        This function assumes that the collection is stored as JSON, created by :func:`save<storage.Storage.save>`.
        """
        js=''
        if os.path.exists(self.filename):
            f=open(self.filename, 'r')
            js=f.read()
            f.close()

        if js=='' or js=='{}':
            retval=self._load_lifetime()
            if 'success' in retval:
                retval['success']='No current file found. {}'.format(retval['success'])
            elif 'error' in retval:
                retval['error']='No current file found. {}'.format(retval['error'])

            return retval

        #bugfix for keys..
        tmp=json.loads(js)
        for k, v in tmp.iteritems():
            if not 'L' in k:
                self.storage[int(k)]=v
            else:
                self.storage[k]=v

        return {u'success':u'Loaded all data from file'}

    def _load_lifetime(self):
        """
        load lifetime-members from past periods. gets the filename for the current period and adds
        all lifetime-members from the last period to storage.

        :param filename: the filename of the current period.
        """
        m=re.search('(.+)medlemmer_(\S)(\d\d)\.json', self.filename)
        path=m.group(1)
        s=m.group(2)
        y=m.group(3)

        old_fname=''
        if s=='h':
            old_fname='{}medlemmer_{}{}.json'.format(path, 'v', y)
        elif s=='v':
            old_fname='{}medlemmer_{}{}.json'.format(path, 'h', (y-1))

        if not os.path.exists(old_fname):
            return {u'error':u'Previous file could not be found, no lifetime members loaded.'}

        f=open(old_fname, 'r')
        old_storage=json.loads(f.read())

        for k, v in old_storage.iteritems():
            if 'L' in k and v['lifetime']=='y':
                self.storage[k]=v

        return {u'success':u'loaded all lifetime members from previous file'}

    def _get_filename(self, nameonly=False):
        """
        Calculates and returns current filename, e.g. 'medlemmer_h12' for autumn 2012.

        :returns: current filename.
        """
        month=dt.now().strftime("%m")
        year=dt.now().strftime("%y")
        path=os.path.abspath(__file__).rsplit('/', 1)[0]

        if month<8:
            if nameonly:
                return 'medlemmer_v{}'.format(year)
            return '{}/medlemslister/medlemmer_v{}.json'.format(path, year)

        if nameonly:
            return 'medlemmer_h{}'.format(year)
        return '{}/medlemslister/medlemmer_h{}.json'.format(path, year)

    def size(self):
        """
        returns the size of the collection
        """
        return len(self.storage)

    def _testfile(self):
        try:
            open(self.filename, 'a')
        except(IOError):
            return False
        return True

    def google_write(self):
        """
        backup collection to a google spreadsheet.

        Note: lazy backup, does not merge.. overwrites google-data.
        """
        if not self._get_google_sheet():
            return {u'error':u'could not connect to google spreadsheet'}

        try:
            val=self.wks.col_values(1)
        except ValueError:
            val=[]

        for i in xrange(1, len(val)+1):
            self.wks.update_cell(i, 1, '')
            self.wks.update_cell(i, 2, '')

        x=1
        for k, v in self.storage.iteritems():
            self.wks.update_cell(x, 1, '{}'.format(k))
            self.wks.update_cell(x, 2, json.dumps(v))
            x+=1

        return {u'success':u'backed up to google'}

    def google_read(self):
        """
        read backup of collection from a google spreadsheet, merge values based on id and date.
        """
        if not self._get_google_sheet():
            return {u'error':u'could not connect to google spreadsheet'}

        empty=False
        try:
            keys=self.wks.col_values(1)
            vals=self.wks.col_values(2)
        except ValueError:
            empty=True

        if empty:
            return {u'error':u'spreadsheet is empty!'}

        gdoc={}
        for i in xrange(0, len(keys)):
            gdoc[keys[i]]=json.loads(vals[i])

        for k, v in gdoc.iteritems():
            if not k in self.storage:
                self.storage[k]=v
            else:
                if v['date']>self.storage[k]['date']:
                    self.storage[k]=v

        return {u'success':u'read and merged all values from google doc.'}

    def wiki_merge(self, push_to_wiki=False, lifetime=False, overwrite_wiki=False):
        """
        Merge local collection with table on wiki
        """
        pagename=None
        if not lifetime:
            pagename=self._get_filename(nameonly=True)
        else:
            pagename='livstidsmedlemmer_fra_script'
        self.wl=WikiLink(pagename=pagename)
        ok, msg=self.wl.login()
        if not ok:
            return{u'status':u'Failure while accessing wiki. error: {}'.format(msg)}

        val={}
        if overwrite_wiki:
            self.wl.clear_page()
            val['pre']='== Medlemsliste ==\n'
            val['post']=''
        else:
            ok, val=self._merge()
            if not ok:
                return val

        if push_to_wiki:
            ok, table=self.wl.dict_to_table(self.storage, ['id', 'name', 'date', 'lifetime'], lifetime=lifetime)
            if not ok:
                return {u'status':table}
            text=u'{}{}{}'.format(val['pre'], table, val['post'])
            if overwrite_wiki:
                summary='Overwritten from script'
            else:
                summary='Merge from script'

            self.wl.write(text=text, summary=summary)

            return {u'status':u'Merged local storage with wiki and updated changes on wiki'}
        return {u'status':u'wiki merged with local storage'}

    def _merge(self):
        ok, msg=self.wl.read()
        if not ok:
            return{u'status':u'Failure while accessing wiki. error: {}'.format(msg)}

        wiki=self.wl.wikitable_to_dict()
        ok, values=wiki['dict']
        if not ok:
            return false, {u'status':values}

        tmp=self.storage

        for wk, wv in values.iteritems():
            if wk in tmp:
                sv=tmp[wk]
                if wv['name'].strip()!=sv['name'].strip():
                    return {u'status':u'Failure while merging. Conflict! {} in wiki and {} in local storage have the same id! Please inform sys-admin.'.format(wv['name'], sv['name'])}
            else:
                match=False
                for sk, sv in tmp.iteritems():
                    if v['name']==sv['name']:
                        match=True
                if match:
                    return {u'status':u'Failure while merging. Conflict! {} is in wiki and local storage with different ids! Please inform sys-admin.'.format(v['name'])}
                tmp[k]=v

        self.storage=tmp
        return True, values

    def _get_google_sheet(self):
        """
        Connect to a google spreadsheet

        :returns: True if success, False if not
        """
        if self.wks:
            return True

        name=self._get_filename(nameonly=True)
        gc=gspread.login(username, passwd)
        try:
            wks=gc.open(name).sheet1
        except SpreadsheetNotFound:
            return False

        self.wks=wks
        return True