def get_subject( url:str, data:str ):
data_base_control = lib_data_base_control.DataBaseControl()
id_url = data_base_control.find_id_of_url( url )
id_data = data_base_control.find_id_of_data_in_url( url, data )
zFile = zipfile.ZipFile(PATHS.DATA_BY_URL_BY_DATA( id_url , id_data , __SUBJECT__ ) , 'r' )
out = json.loads( zFile.read(PATHS.IN_ZIP_NAME_FILE_BASIC() ) )
zFile.close()
return out
def contains_frequency(url, data) -> bool:
data_base_control = lib_data_base_control.DataBaseControl()
id_url = data_base_control.find_id_of_url(url)
id_data = data_base_control.find_id_of_data_in_url(url, data)
return os.path.exists(PATHS.DATA_BY_URL_BY_DATA(
id_url, id_data, __FREQUENCY__),
compresslevel=9)
def save_subject( url:str , data:str , subjects:dict )->None:
data_base_control = lib_data_base_control.DataBaseControl()
id_url = data_base_control.find_id_of_url( url )
id_data = data_base_control.find_id_of_data_in_url( url, data )
os.makedirs(PATHS.DATA_BY_URL_BY_DATA_FATHER( id_url , id_data ) , exist_ok = True )
zFile = zipfile.ZipFile(PATHS.DATA_BY_URL_BY_DATA( id_url , id_data , __SUBJECT__ ) , 'w' , compresslevel=9)
zFile.writestr(PATHS.IN_ZIP_NAME_FILE_BASIC() , json.dumps( subjects , indent= 4) )
zFile.close()
DataBaseSubject.update_keys( url , data )
def get_frequency(url, data):
data_base_control = lib_data_base_control.DataBaseControl()
id_url = data_base_control.find_id_of_url(url)
id_data = data_base_control.find_id_of_data_in_url(url, data)
zFile = zipfile.ZipFile(PATHS.DATA_BY_URL_BY_DATA(
id_url, id_data, __FREQUENCY__),
'r',
compression=zipfile.ZIP_BZIP2,
compresslevel=9)
out = json.loads(zFile.read(PATHS.IN_ZIP_NAME_FILE_BASIC()))
zFile.close()
return out
def save_frequency(url, data, frequency):
data_base_control = lib_data_base_control.DataBaseControl()
id_url = data_base_control.find_id_of_url(url)
id_data = data_base_control.find_id_of_data_in_url(url, data)
os.makedirs(PATHS.DATA_BY_URL_BY_DATA_FATHER(id_url, id_data),
exist_ok=True)
zFile = zipfile.ZipFile(PATHS.DATA_BY_URL_BY_DATA(
id_url, id_data, __FREQUENCY__),
'w',
compression=zipfile.ZIP_BZIP2,
compresslevel=9)
zFile.writestr(PATHS.IN_ZIP_NAME_FILE_BASIC(),
json.dumps(frequency, indent=4))
zFile.close()
#!/usr/bin/python3.8
import sys
sys.path.append("../libs")
import lib_driver_controler
import lib_data_base_control
FILE_URL = "../config/url.list"
if __name__ == "__main__":
objDBC = lib_data_base_control.DataBaseControl()
objDC = lib_driver_controler.DriverControler()
with open(FILE_URL, 'r') as arq:
for line in arq:
line = line.strip("\n")
print('[URL ]', 'Download:', line)
if objDBC.contem_no_sistema(line) == False:
try:
html = objDC.get(line)
objDBC.add_code(line, html)
print('[URL ]', "Download Ok")
except Exception as e:
print(
'[URL ]',
'Falha em conseguir o retorno da página ou em salvar os dados',
e)
objDC.exit()
def contains_subject( url , data )->bool:
data_base_control = lib_data_base_control.DataBaseControl()
id_url = data_base_control.find_id_of_url( url )
id_data = data_base_control.find_id_of_data_in_url( url, data )
return os.path.exists( PATHS.DATA_BY_URL_BY_DATA( id_url , id_data , __SUBJECT__ ) )