tweets = ""

tweetids = []

if language == 'english':

st = StanfordNERTagger(BASEPATH+'/classifiers/english.all.3class.distsim.crf.ser.gz', BASEPATH+'/classifiers/stanford-ner.jar', encoding='utf8')

elif language == 'spanish':

st = StanfordNERTagger(BASEPATH+'/classifiers/spanish.ancora.distsim.s512.crf.ser.gz', BASEPATH+'/classifiers/stanford-ner.jar', encoding='utf8')

if format == 'xml':

dataset = etree.parse(datasetfile)

for tweet in dataset.xpath('//Tweet'):

tweetText = tweet.xpath('./TweetText/text()')[0]

tweets += ' '.join(re.findall(r"[\w:/!#$%&*+,\-:;?@^_`{|}~.]+|[\"'()[\]<=>]", tweetText))+"\n"

tweetids.append(tweet.xpath('./TweetId/text()')[0])

tweets = tweets.encode('utf-8')

elif format == "nif":

tweetdict = {}

a = rdflib.Graph()

a.parse(datasetfile, format='n3')

for s, p, o in a:

if s.endswith(',') and p.endswith('isString'):

tweetid = s.split('#')[0].split('.xml/')[1]

tweetdict[tweetid] = ' '.join(re.findall(r"[\w:/!#$%&*+,\-:;?@^_`{|}~.]+|[\"'()[\]<=>]", o))

for key in sorted(tweetdict):

tweetids.append(key)

tweets += tweetdict[key]+'\n'

tweets = tweets.encode('utf-8')

#print tweets

elif format == "text":

tweets = datasetfile

tweetlist = []

for t in tweets.splitlines():

newtweet = []

for word in t.split():

newword = u''

if word.endswith(",") or word.endswith(".") or word.endswith(")") or word.endswith("\'"):

newtweet.append(word[:-1])

newtweet.append(word[-1])

else:

newtweet.append(word)

#print newtweet

tweetlist.append(newtweet)

results = ''

tagged = []

for tweet in tweetlist:

tagged.append(st.tag(tweet))

#print tagged[-1]

#print len(tagged)

inEntity = False

for line in tagged:

#print line

for (word, entity) in line:

if entity != 'O' and inEntity:

entity = 'I-'+entity

elif entity != 'O' and inEntity == False:

entity = 'B-'+entity

inEntity = True

else:

inEntity = False

results += word + '/' + entity + ' '

if tweetids:

results += "||"+tweetids[x]

results += "\n"

#print results