def main(argv): parameters = Parameters(argv) hostname = parameters.getHostname() port = parameters.getPort() dbname = parameters.getDBName() language_1, language_2 = parameters.getLanguage() collection = parameters.getCollection() fileinput_1 = parameters.getInputFile_1() fileinput_2 = parameters.getInputFile_2() type_corpus = parameters.getType() print 'Using parameters of configuration: ' print '- Host : ',hostname print '- Port : ',port print '- Coll : ',collection print '- DBase: ',dbname print '- File1: ',fileinput_1 print '- File2: ',fileinput_2 database = Mongo(hostname, dbname, collection) id_file_1 = (fileinput_1.split('/'))[-1] id_file_2 = (fileinput_2.split('/'))[-1] try: file_1 = codecs.open(fileinput_1, 'r', 'utf-8') except IOError: print 'ERROR: System cannot open the '+fileinput_1+' file' sys.exit(2) try: file_2 = codecs.open(fileinput_2, 'r', 'utf-8') except IOError: print 'ERROR: System cannot open the '+fileinput_2+' file' sys.exit(2) #Sentences indexed by the number of the line : number_line = _id (sentence) line_number = 1 lines_2 = file_2.readlines() for counter, content_1 in enumerate(file_1): content_2 = lines_2[counter] if not database.exists(language_1, id_file_1) and not database.exists(language_2, id_file_2): database.insertNewData(language_1, id_file_1, content_1, language_2, id_file_2, content_2, type_corpus, line_number) else: if database.existsSentence(language_1, id_file_1, line_number): if not database.existsSentence(language_2, id_file_2, line_number): database.insertInExistingSentence(language_1, id_file_1, language_2, id_file_2, content_2, line_number) else: if database.existsSentence(language_2, id_file_2, line_number): database.insertInExistingSentence(language_2, id_file_2, language_1, id_file_1, content_1, line_number) else: database.insertNewSentence(language_1, id_file_1, content_1, language_2, id_file_2, content_2, line_number) if (line_number % 1000 == 0): print 'Indexing line: ',line_number line_number += 1
def main(argv): parameters = Parameters(argv) hostname = parameters.getHostname() port = parameters.getPort() dbname = parameters.getDBName() language_1, language_2 = parameters.getLanguage() collection = parameters.getCollection() input_folder = parameters.getInputFolder() type_corpus = parameters.getType() print 'Using parameters of configuration: ' print '- Host : ',hostname print '- Port : ',port print '- Coll : ',collection print '- DBase: ',dbname print '- Input: ',input_folder database = Mongo(hostname, dbname, collection) try: root, dirs, files = os.walk(input_folder+''+language_1+'/').next()[:3] except IOError: print 'ERROR: It was not possible to open the '+input_folder+'en/ folder' sys.exit(2) for corpus_file in files: #if (corpus_file ~ "/~/$"): if not '.txt~' in corpus_file: print 'Working on file: '+corpus_file id_file_1 = language_1+'_'+corpus_file[0:-4] id_file_2 = language_2+'_'+corpus_file[0:-4] try: file_1 = codecs.open(input_folder+''+language_1+'/'+corpus_file, 'r', 'utf-8') except IOError: print 'ERROR: System cannot open the '+root+''+corpus_file+' file' sys.exit(2) try: file_2 = codecs.open(input_folder+''+language_2+'/'+corpus_file, 'r', 'utf-8') except IOError: print 'ERROR: System cannot open the '+root+'../'+language_2+'/'+corpus_file+' file' sys.exit(2) #Sentences indexed by the number of the line : number_line = _id (sentence) line_number = 1 lines_2 = file_2.readlines() content_1 = '' content_2 = '' for counter, line in enumerate(file_1): if re.match('(^<)', line): if content_1 != '' and content_2 != '': if not database.exists(language_1, id_file_1) and not database.exists(language_2, id_file_2): database.insertNewData(language_1, id_file_1, content_1, language_2, id_file_2, content_2, type_corpus, line_number) else: if database.existsSentence(language_1, id_file_1, line_number): if not database.existsSentence(language_2, id_file_2, line_number): database.insertInExistingSentence(language_1, id_file_1, language_2, id_file_2, content_2, line_number) else: if database.existsSentence(language_2, id_file_2, line_number): database.insertInExistingSentence(language_2, id_file_2, language_1, id_file_1, content_1, line_number) else: database.insertNewSentence(language_1, id_file_1, content_1, language_2, id_file_2, content_2, line_number) line_number += 1 content_1 = '' content_2 = '' if (line_number % 100 == 0): print 'Indexing line: ',line_number else: content_1 += line content_2 += lines_2[counter] file_1.close() file_2.close()