def main():
"""
This file is the main method call. Execution starts from here.
It accepts input data either correct or incorrect data.
:return: Output of the data provided.
:param: It accepts file directory,which contains input_data.txt file.
"""
simulation = RobotSimulation()
execute_directory = sys.argv[1]
if len(execute_directory) == len("input_data"):
correct_files = fm.load("input_data", "r")
for correct_file in correct_files:
print("\n")
print("FIle executed from Input Directory - ", correct_file)
for values in fm.read(correct_file):
for line in values.split("\n"):
if len(line):
simulation.run(line)
elif len(execute_directory) == len("incorrect_data"):
incorrect_files = fm.load("incorrect_data")
for incorrect_file in incorrect_files:
print("\n")
print("FIle executed from Incorrect Directory - ", incorrect_file)
for values in fm.read(incorrect_file):
for line in values.split("\n"):
if len(line):
simulation.run(line)
else:
print("Please provide the exact directory")
def main(argv):
print "\n"
sequence = []
all_files = fm.load('Edge1_')
for f in all_files:
Sort_file(f)
all_files = fm.load('Edge1_')
for f in all_files:
sampling_time = 5 # in mins
file_ = open("Edge1_/" + f, 'r')
line = file_.readline()
data = line.split(' ')
prev_time = data[1] + ' ' + data[2]
date_format = "%Y-%m-%d %H:%M:%S"
prev_time_ = datetime.strptime(prev_time, date_format)
#print prev_time_
no_cars = 0
avg_velocity = 0
for line in fm.read(f):
line = line.replace('\n', '')
data = line.split(' ')
current_time = data[1] + ' ' + data[2]
current_time_ = datetime.strptime(current_time, date_format)
time_interval = current_time_ - prev_time_
print time_interval
if time_interval.seconds < sampling_time * 60:
no_cars = no_cars + 1
avg_velocity = avg_velocity + float(data[9])
#print no_cars
else:
dd = str(data[5]) + ' ' + str(
data[6]) + ' ' + str(no_cars) + ' ' + str(
avg_velocity / no_cars) + '\n'
print dd
prev_time_ = current_time_
avg_velocity = float(data[9])
no_cars = 1
with open('feature_extracted.txt', 'a') as outfile:
outfile.write(dd)
return 0
def test_setup():
print(fm.create('resources'))
print(fm.load('resources'))
for i in fm.read('adverbs.dat'):
print(i)
print(fm.close('nouns.dat'))
print(fm.close())
def learnFromFiles():
path = fm.load('resources')
for f in path:
isFirst = True
lettercount = 0
for i in fm.read(f):
if isFirst:
for x in langs:
if i.replace("\n", "") == x.get("language"):
currentdict = x
break
else:
langs.append({"language": i.replace("\n", "")})
currentdict = langs[len(langs) - 1]
isFirst = False
else:
for singleletter in i:
if (ord(singleletter) >= 97 and ord(singleletter) <= 122
) or (ord(singleletter) >= 65
and ord(singleletter) <= 90):
lettercount = lettercount + 1
if singleletter.lower() in currentdict:
currentdict[singleletter.lower(
)] = currentdict[singleletter.lower()] + 1
else:
currentdict[singleletter.lower()] = 1
if "letternumber" in currentdict:
currentdict["letternumber"] = currentdict["letternumber"] + float(
lettercount)
else:
currentdict["letternumber"] = float(lettercount)
for i in langs:
handleFile(i)
def test_setup():
print(fm.create('resources'))
print(fm.load('resources'))
for i in fm.read('adverbs.dat'):
print(i)
print(fm.close('nouns.dat'))
print(fm.close())
def get_query_index(requete):
"""
fonction qui prend une requête écrite en entrée et donne son indice parmi les 64 requêtes proposées.
"""
index = 0
queries = fm.load("Requetes")
for query in queries:
for line in fm.read(query):
if line == requete:
index = query.split()[1]
return index
def main():
#opens all the files in the folder
all_files = fm.load('test')
docs = []
#instantiating Porter Stemming
stemmer = PorterStemmer()
for f in all_files:
docStr = ""
for i in fm.read(f):
docStr += i
# docStr.replace('\n', ' ')
# docStr.replace('\\', ' ')
# docStr.replace('.', ' ')
# docStr.replace(',', ' ')
docStr = remove_all("\n", docStr)
docStr = remove_all("\\", docStr)
docStr = remove_all('"', docStr)
docStr = remove_all(':', docStr)
docStr = remove_all(".", docStr)
docStr = remove_all(",", docStr)
#unicode encoding for stemming
p = docStr.decode('cp850').replace(u"\u2019", u"\x27")
res = " ".join([stemmer.stem(kw) for kw in p.split(" ")])
k = unicodedata.normalize('NFKD', res).encode('ascii', 'ignore')
docs.append(k)
docOfInterest = docs[5]
#testing on article
a = ArticleScorer(docOfInterest, docs)
print(a.represent())
docs_and_similarities = []
for docOfInterest in docs:
# print docOfInterest
a = ArticleScorer(docOfInterest, docs)
# print(a.represent())
docs_and_similarities.append((docOfInterest, a.represent()))
print docs_and_similarities
best_matches = queue.PriorityQueue()
article = docs_and_similarities[0]
for other in docs_and_similarities[1:]:
print "score", cosine(article, other)
best_matches.put((-1 * cosine(article, other), other[0]))
for i in xrange(5):
print extract_title(best_matches.get()[1])
def test_open():
print("begin test_open")
try:
print(fm.reset())
except:
pass
print(fm.load('resources'))
try:
print(fm.close('nouns.dat', 'verbs.dat', 'test.dat'))
raise ValueError("test_open failed") # pragma: no cover
except IOError:
print("test_open passed")
def test_open():
print("begin test_open")
try:
print(fm.reset())
except:
pass
print(fm.load('resources'))
try:
print(fm.close('nouns.dat', 'verbs.dat', 'test.dat'))
raise ValueError("test_open failed") # pragma: no cover
except IOError:
print("test_open passed")
def create_document_pairs(folder_name):
"""
cree un dictionnaire de paires de doc: doc_id a partir de la collection
"""
documents = fm.load(folder_name)
documents_pairs = {}
for document in documents:
document_id = [int(s) for s in document.split() if s.isdigit()][0]
try:
documents_pairs[document]
except KeyError:
documents_pairs[document] = document_id
return documents_pairs
def open_fun(sub, unit):
sub_dir = fm.load(sub,'w') # fm.load('sub','w') will open in write mode, loads all files in the directory
#print sub_dir --shows the files from the given directory
for f in sub_dir:
print f
#print os.path.realpath(f)
to_open = open(f)
to_read = to_open.read()
regex = re.escape(unit) + ur"[a-z]\..*" #specifies which unit to select
unit_list = re.findall(regex, to_read) #returns the list containing the regex
for i in range(len(unit_list)):
print unit_list[i]
print "\n"
print "*"*130
def test_setup():
print("begin test_setup")
print(fm.create('resources'))
if 'words.dat' and 'nouns.dat' in fm.create('resources'):
print('Files Missing')
print(fm.load('resources'))
for i in fm.read('adverbs.dat'):
print(i)
print(fm.close('words.dat'))
print(fm.close())
try:
print(fm.close('words.dat'))
raise ValueError("test_setup failed") # pragma: no cover
except IOError:
print("test_setup passed")
def test_setup():
print("begin test_setup")
print(fm.create('resources'))
if 'words.dat' and 'nouns.dat' in fm.create('resources'):
print('Files Missing')
print(fm.load('resources'))
for i in fm.read('adverbs.dat'):
print(i)
print(fm.close('words.dat'))
print(fm.close())
try:
print(fm.close('words.dat'))
raise ValueError("test_setup failed") # pragma: no cover
except IOError:
print("test_setup passed")
def create_terms_pairs(folder_name):
"""
cree un dictionnaire de paires de term: term_id a partir de la collection
"""
documents = fm.load(folder_name)
terms_pairs = {}
id = 1
for document in documents:
for line in fm.read(document):
for word in line.split():
try:
terms_pairs[word]
except KeyError:
terms_pairs[word] = id
id += 1
return terms_pairs
def create_terms_doc_pairs(folder_name):
"""
cree une liste de paires de (terme_id, doc_id)
"""
documents_pairs = create_document_pairs(folder_name)
terms_pairs = create_terms_pairs(folder_name)
documents = fm.load(folder_name)
terms_doc_pairs_list = []
for document in documents:
for line in fm.read(document):
for word in line.split():
term_id = terms_pairs[word]
document_id = documents_pairs[document]
pair = (term_id, document_id)
terms_doc_pairs_list.append(pair)
return terms_doc_pairs_list
def moyenne_precisions():
"""
fonction qui ne prend rien en entrée
et qui fait la moyenne des précisions pour les 64 requêtes de query.text
"""
queries = fm.load("Requetes")
written_queries = []
precisions = []
rappel = [0] * (101)
precision = [0] * (101)
for query in queries:
for line in fm.read(query):
written_queries.append(line)
for i in range(64):
precisions.append(data_courbe(written_queries[i])[1])
for i in range(101):
rappel[i] = i / 100
for j in range(64):
precision[i] += precisions[j][i] / 64
return rappel, precision
def calculateDistance(text, lang, i, max, isLearning, isWiki):
path = fm.load('constlearnt')
dicts = []
distances = {}
for f in path:
fromFileInfo = {}
isFirstLine = True
for y in fm.read(f):
if isFirstLine:
isFirstLine = False
else:
arr = y.split('/')
fromFileInfo[arr[0]] = arr[1].replace("\n", "")
dicts.append(fromFileInfo)
(dicts[len(dicts) - 1])["language"] = f.replace(".txt", "").replace(
"learned_", "")
newtextdict = getDictFromText(text, True)
for dic in dicts:
for ch in alphabet:
if not ch in dic:
dic[ch] = 0.0
if not ch in newtextdict:
newtextdict[ch] = 0.0
distances[dic["language"]] = euclideanDistance(newtextdict, dic)
print("Zgaduje, iz artykul jest w jezyku " +
min(distances, key=lambda k: distances[k]))
global tries
global correct
tries = tries + 1
if min(distances, key=lambda k: distances[k]) == lang:
correct = correct + 1
print("Jezyk sie zgadza")
else:
print("Jezyk sie nie zgadza")
print(("Skutecznosc zgadywania to w tym momencie " +
str(float(correct) / float(tries) * 100) + " proby: " + str(tries) +
" poprawne: " + str(correct)))
print("\n")
if isWiki:
testAlgorithm(i, max, isLearning, isWiki)
# -*- coding: utf-8 -*-
"""
Created on Mon Aug 15 17:19:22 2016
@author: SRINIVAS
"""
import filemapper as fm
files=fm.load('Mom Proj')
fileslist=[]
for elem in files:
fileslist.append(elem.split('_'))
fileslistr=[]
for elem in fileslist:
elem[-1]=elem[-1].split('.')[0]
fileslistr.append(elem)
print(fileslistr)
loop=-1
final=[]
for f in files:
e=[]
loop=loop+1
for i in fm.read(f):
d=','.join(fileslistr[loop])
e.append(d+','+i)
final.append(e)
import csv
loop=-1
for elem in final:
loop=loop+1
results = {'cars': {}, 'bikes': {}}
for t in types:
results[t] = {}
for r in routers:
results[t][r] = {}
for bs in bufferSize:
results[t][r][bs] = {}
for n in nodes:
results[t][r][bs][n] = {}
# Read data
for t in types:
res = results[t]
files = fm.load(t)
for f in files:
r, bs, n, _ = f.split('_') # Getting the result's parameters
for line in fm.read(f):
try:
stat, value = line.split(':')
value = float(value)
res[r][bs][n][stat] = value
except ValueError:
pass
# Router Impact
for t in types:
for n in nodes:
delivery_probs = []
latency_avgs = []
import pandas as pd
import filemapper as fm
import xlrd
all_files = fm.load('./data_excel')
for f in all_files:
print(f)
#xls = pd.ExcelFile('./data_excel/'+f)
data_sheets = xlrd.open_workbook('./data_excel/' + f)
df = pd.read_excel('GRIP-IRA 2009-2010.xls')
# this will read the first sheet into df
xls = pd.ExcelFile('GRIP-IRA 2009-2010.xls')
# Now you can list all sheets in the file
#print(xls.sheet_names)
sheet_to_df_map = {}
for sheet_name in xls.sheet_names:
#print(sheet_name)
sheet_to_df_map[sheet_name] = xls.parse(sheet_name)
#print(sheet_to_df_map[sheet_name])
sheet_to_df_map[sheet_name]['annee'] = 2020
sheet_to_df_map[sheet_name].to_csv(sheet_name + '.csv',
sep='\t',
encoding='utf-8')
map_pos_file = configParser.get("pos", "map_pos")
if os.stat(map_pos_file).st_size > 0:
reader = unicode_csv_reader(open(map_pos_file))
is_header = True
for fields in reader:
# print(fields)
if is_header == True:
is_header = False
continue
else:
word = fields[0]
map_to = fields[1]
map_pos[word] = map_to
input_dir = configParser.get("file", "input_dir")
all_files = filemapper.load(configParser.get("file", "input_dir"))
output_dir = configParser.get("file", "output_dir")
def in_string(str1, str2):
try:
i = str1.index(str2)
return True
except ValueError:
try:
i = str2.index(str1)
return True
except ValueError:
return False
import filemapper as fm
all_files = fm.load('d065j')
for i in range(len(all_files)):
f1 = all_files[i]
f2 = all_files[i] + ".txt"
with open(f1) as infile, open(f2, 'w') as outfile:
copy = False
for line in infile:
if line.strip() == "<TEXT>":
copy = True
elif line.strip() == "</TEXT>":
copy = False
elif copy:
outfile.write(line)
import filemapper as fm
all_files = fm.load('Pics')
for f in all_files:
for i in fm.read(f):
for f in all_files:
for i in fm.write
import filemapper as fm
import os.path
import wikipedia
import math
from random import randint
from urllib.request import urlopen
import json
import threading
from wikipedia.exceptions import DisambiguationError, PageError
tries = 0
correct = 0
langs = []
percentRatio = []
path = fm.load('constlearnt')
currentdict = {}
alphabet = [chr(i) for i in range(ord('a'), ord('z') + 1)]
def testAlgorithm(i, max, isLearning, isWiki):
arr = ["pl", "en", "fr", "de", "es"]
lang = arr[randint(0, 4)]
wikipedia.set_lang(lang)
url = "https://" + lang + ".wikipedia.org/w/api.php?action=query&list=random&format=json&rnnamespace=0&rnlimit=1"
response = urlopen(url)
data = json.loads(response.read())
title = (((data['query'])['random'])[0])['title']
t1 = threading.Thread(
handleTestThread(title, lang, i, max, isLearning, isWiki))
t1.start()