def __get_cypher_line_if_loads_csv(file_name: str) -> str:
FileUtils.file_exists(file_name)
with open(file_name, 'r') as cypher_file:
for line in cypher_file.readlines():
if ('file:///' in line):
return line.replace('\n', '')
return None
def create_files_with_attributes_json() -> None:
try:
final_path = FileUtils.get_full_path(
UsedFileUtils.FILES_WITH_ATTRIBUTES_JSON)
FileUtils.file_exists(final_path)
except FileNotFoundError:
files_with_attributes = UsedFileUtils.initialize_used_files()
json_file = JsonUtils.dict_to_json(files_with_attributes)
JsonUtils.save_json(json_file,
UsedFileUtils.FILES_WITH_ATTRIBUTES_JSON)
def get_header(file_name: str) -> list:
if (FileUtils.file_exists(file_name)):
with open(file_name, 'r') as csv_file:
reader = csv.reader(csv_file)
header = next(reader, None)
HeaderUtils.validate_header(file_name, header)
delimiter = FileUtils.detect_csv_delimiter(str(header))
with open(file_name, 'r') as csv_file:
reader = csv.reader(csv_file, delimiter=delimiter)
return next(reader, None)
else:
return None
def get_cypher_file_if_loads_csv(file_name: str) -> list:
"""
Método que retorna um array com as linhas de um arquivo *.cypher
caso ele carregue um CSV
"""
FileUtils.file_exists(file_name)
file_loads_csv = CypherUtils.__get_cypher_line_if_loads_csv(file_name)
if (file_loads_csv):
return FileUtils.read_file(file_name)
else:
return None
def fill_header_and_used_attributes(key_name: str) -> None:
"""
Método que preenche o header e quais atributos dos *.csv são utilizados,
a partir de umas das chaves presentes no arquivo `used_files.json`
"""
UsedFileUtils.create_files_with_attributes_json()
file_names = UsedFileUtils.get_used_files_list_from_key_name(key_name)
csv_with_used_attributes = headers.cypher_utils.CypherUtils.get_used_attributes_from_cypher_files(
key_name)
if file_names:
for file_name in file_names:
prefix = FileUtils.get_full_path(HeaderUtils.PREFIX)
prefix += '/'
header = HeaderUtils.get_header(prefix + file_name)
used_attributes = [
aux.get('used_attributes')
for aux in csv_with_used_attributes
if aux.get('csv_name') == file_name
]
used_attributes = used_attributes[0]
UsedFileUtils.update_used_file(file_name, 'header', header)
UsedFileUtils.update_used_file(file_name, 'used_attributes',
used_attributes)
def get_used_attributes_from_cypher_files(csv_key_name: str) -> list:
"""
Método que retorna uma lista que contém o nome do CSV e quais atributos são utilizados
nos arquivos *.cypher, a partir de uma das chaves (tce, tse, receita_federal, sancoes)
que estão no arquivo `used_files.json`
"""
csv_files = FileUtils.get_used_files_list_from_key_name(csv_key_name)
csv_with_cypher_files = CypherUtils.get_csv_with_cypher_files(
csv_files)
csv_with_cypher_files = csv_with_cypher_files.get('csv_with_cypher')
csv_with_attributes = []
for csv_with_cypher_file in csv_with_cypher_files:
csv_name = csv_with_cypher_file.csv_name
cypher_array = csv_with_cypher_file.cypher_array
dict_tmp = {'csv_name': csv_name, 'used_attributes': set()}
for cypher_file in cypher_array:
for used_attribute in cypher_file.used_attributes:
dict_tmp['used_attributes'].add(
used_attribute.attribute_name)
dict_tmp['used_attributes'] = [
attribute for attribute in dict_tmp['used_attributes']
]
csv_with_attributes.append(dict_tmp)
return csv_with_attributes
def __get_all_cypher_files(path_name: str) -> list:
all_files = FileUtils.get_all_files_from_path(path_name)
cypher_files = []
for file in all_files:
if ('.cypher' in file):
cypher_files.append(file)
return cypher_files
def get_csv_with_cypher_files(csv_files: list) -> dict:
"""
Método que retorna um JSON que contém o nome do csv, o nome do arquivo cypher onde o CSV
é utilizado, quais o atributos utilizados e em quais linhas eles ocorrem,
a partir de uma lista de CSV's
"""
csv_with_cypher_file = CypherUtils.get_dict_with_csv_and_cypher_file()
final_json = {"csv_with_cypher": []}
for csv_key in csv_files:
cypher_files = csv_with_cypher_file.get(csv_key)
csv_with_cypher = CsvWithCypher(csv_name=csv_key, cypher_array=[])
for cypher_file in cypher_files:
file = FileUtils.read_file(CypherUtils.get_source_path() +
cypher_file)
cypher_object = CypherFile(file_name=cypher_file,
used_attributes=[])
used_attributes = SearchUtils.get_used_attributes_from_cypher_file(
file)
for attribute in used_attributes:
line_numbers = [
number + 1 for number, line in enumerate(file)
if ('line.' + attribute) in line
]
if (line_numbers):
attribute = attribute.replace('`', '')
used_attribute = UsedAttribute(
attribute_name=attribute,
line_numbers=line_numbers)
cypher_object.used_attributes.append(used_attribute)
csv_with_cypher.cypher_array.append(cypher_object)
final_json['csv_with_cypher'].append(csv_with_cypher)
return final_json