def __init__(self, df_data):
self.df_data = df_data
self.network = None
self.network_adjusted = None
self.df_types = None
self.data_df = None
self.d_id_title = None
self.d_title_id = None
self.views_df = None
self.extract_data = ExtractData()
self.logger = logging.getLogger(__name__)
def execute(self):
all_files = [
splitext(f)[0] for f in os.listdir(self.INPUT_FOLDER_PATH)
]
all_files.sort()
print(all_files)
for filename in all_files:
if filename != '.DS_Store':
PDFtoText(filename).execute()
ExtractData(filename).execute()
self.remove_file(filename)
def details():
payload = {
'email': '*****@*****.**',
'password': '******'
}
res_1 = requests.post(
'http://api.passivereferral.com/index.php/api/authenticate/',
json=payload)
tok = res_1.json()
token = tok['token']
url = r'http://api.passivereferral.com/index.php/api/getsmtp/?token=' + token
res = requests.get(url, json=payload)
response = res.json()[0]
user_name = response['user_name']
password = response['password']
mail_server = response['smtpname']
ssl_enabling = response['ssl_enabled']
token = response['token']
if ssl_enabling == '0':
port = None
y = ExtractData(mail_server, None, False, user_name, password, token)
y.convert_into_html()
class FileHandler:
def __init__(self, df_data):
self.df_data = df_data
self.network = None
self.network_adjusted = None
self.df_types = None
self.data_df = None
self.d_id_title = None
self.d_title_id = None
self.views_df = None
self.extract_data = ExtractData()
self.logger = logging.getLogger(__name__)
def organize_data(self):
self.logger.info("organize data")
try:
data_preprocessing = DataPreprocessing(self.df_data)
network, network_adjusted, df_types = data_preprocessing.preprocessing(
)
data_df = self.extract_data.create_final_df(network_adjusted)
# data_df.to_csv("data_df.csv")
d_id_title, d_title_id = self.extract_data.find_dicts(data_df)
views_df = self.extract_data.create_views_df(data_df)
return network, network_adjusted, df_types, data_df, d_id_title, d_title_id, views_df
except Exception as err:
self.logger.info(f"encounter error: {str(err), err.args}")
def process_files(self):
self.logger.info("process_files")
network, network_adjusted, df_types, data_df, d_id_title, d_title_id, views_df = \
self.organize_data()
self.network = network
self.network_adjusted = network_adjusted
self.df_types = df_types
self.data_df = data_df
self.d_id_title = d_id_title
self.d_title_id = d_title_id
self.views_df = views_df
# -*- coding: utf-8 -*-
from extract_data import ExtractData
from math import degrees
parser = ExtractData()
parser.ExtractError()
def parse():
error = open("errors.csv")
X = []
Y = []
ANGLE = []
for line in error:
data = line.split(",")
X.append(float(data[0]))
Y.append(float(data[1]))
ANGLE.append(float(data[2]))
return X, Y, ANGLE
x_error, y_error, angle_error = parse()
import pylab
t = [i * 0.2 for i in range(0, len(x_error))]
pylab.plot(t, x_error, 'r')
pylab.plot(t, y_error, 'g')
ax1 = pylab.gca()
ax1.set_xlabel("Time [s]")
ax1.set_ylabel("Distance error [cm]")
pylab.legend((r'$x$', r'$y$'), shadow=True, loc=(0.84, 0.84))
import sys
from extract_data import ExtractData
from preprocess_data import PreprocessData
from utils import read_json
if __name__ == '__main__':
path_documents = ""
list_documents = glob.glob(path_documents + '*.json')
# Path of each document
for path_doc in list_documents:
# Load json
data = read_json(path_doc)
# object
obj_data = ExtractData(data)
# call get_paper_id method
# call get_title method
# call get_text method
# Object to pre-process the text
obj_preprocess = PreprocessData(text)
# Convert the text to lower case
# Remove punctuation
# Remove numbers
# Remove stop words
from preprocess_data import PreprocessData
from utils import read_json
from utils import write_file
language = 'en'
if __name__ == '__main__':
path_documents = sys.argv[1]
list_documents = glob.glob(path_documents + '*.json')
print("nombre de documents traitées :", len(list_documents))
# Path of each document
for path_doc in list_documents:
# Load json
data = read_json(path_doc)
# object
obj_data = ExtractData(data)
# call get_text method
text = obj_data.get_text()
# make sure the article is in english
if detect(text) == language:
# call get_paper_id method
paper_id = obj_data.get_paper_id()
# call get_title method
title = obj_data.get_title()
# Object to pre-process the text
obj_preprocess = PreprocessData(text)
# Convert the text to lower case
from utils import *
from preprocess_data import PreprocessData
from extract_data import ExtractData
dic = {'paper_id': 'ABC00001', 'metadata': {'title': 'Covid', 'authors': []}, 'body_text': [{'text': 'Covid is coron virus', 'cite_spans': [], 'section': '', 'ref_spans': []}], 'ref_entries': {}, 'back_matter': [], 'bib_entries': {'BIBREF0': {'title': 'The possible macroeconomic effect on the UK of an influenza pandemic', 'authors': [], 'year': 2009, 'venue': '', 'volume': '', 'issn': '', 'pages': None, 'other_ids': {'DOI': []}}}}
path_file = "/Users/youssefbencheikh/Desktop/ABC00001.json"
class Test_utils(unittest.TestCase):
def test_isnumber(self):
self.assertEqual(is_number(12), True, "Should be True")
self.assertEqual(is_number("Covid"), False, "Should be False")
def test_read_json(self):
self.assertEqual(read_json(path_file), dic, "Should be a dictionnary")
data = ExtractData(dic)
class Test_extract_data(unittest.TestCase):
def test_get_paper_id(self):
self.assertEqual(data.get_paper_id(), 'ABC00001', "Should be 'ABC00001'")
def test_get_title(self):
self.assertEqual(data.get_title(), 'covid', "Should be Covid")
def test_get_text(self):
self.assertEqual(data.get_text(), 'Covid is coron virus', "Should be Covid is coron virus")
preprocess = PreprocessData("test")
class Test_preprocess_data(unittest.TestCase):
def test_remove_number(self):
self.assertEqual(preprocess.remove_number("hello number 1 and 2"), 'hello number and', "Should be 'hello number and'")
def test_lower_case(self):
def requires(self):
return ExtractData() # tarea(s) de las que depende el Filter
def test_extract_data(self):
from extract_data import ExtractData
e = ExtractData(yaml_file="../ICML2011.yaml")
e.extract_data()
def test_extract_data(self):
from extract_data import ExtractData
e = ExtractData(yaml_file="../ICML2011.yaml")
e.extract_data()
# -*- coding: utf-8 -*-
from extract_data import ExtractData
parser = ExtractData()
parser.ExtractPlanSpeeds()
parser.ExtractOdomSpeeds()
def parse(filename):
error = open(filename)
V = []
W = []
for line in error:
data = line.split(",")
V.append(float(data[0]))
W.append(float(data[1]))
return V, W
v, w = parse("plan_speeds.csv")
vr, wr = parse("odom_speeds.csv")
import pylab
t = [i * 0.2 for i in range(0, len(v))]
pylab.plot(t, v)
pylab.plot(t, vr)
pylab.grid(True)
fig = pylab.figure()
pylab.plot(t, w)
pylab.plot(t, wr)
pylab.grid(True)
pylab.show()