def get_geographical_areas(self):
# Get all geographical areas
geographical_areas = Request(
"geographical_areas",
"json/[scope]/geographical_areas/geographical_areas.json",
self.scope, self.as_of).json
countries = Request("geographical_areas/countries",
"json/[scope]/geographical_areas/countries.json",
self.scope, self.as_of).json
regions = Request("geographical_areas/regions",
"json/[scope]/geographical_areas/regions.json",
self.scope, self.as_of).json
def get_chapters(self):
# Get all chapters
chapters = Request("chapters", "json/[scope]/chapters/chapters.json",
self.scope, self.as_of).json
for item in chapters["data"]:
# Get chapters
chapter_id = item["attributes"]["goods_nomenclature_item_id"][0:2]
url = 'chapters/{}'.format(chapter_id)
path = 'json/[scope]/chapters/chapter_{:0>2}.json'.format(
chapter_id)
chapter = Chapter(
Request(url, path, self.scope, self.as_of).json, self.scope,
self.as_of)
def main():
from classes.request import Request
from classes.data import Data
import sys
name = sys.argv[1]
req = Request(name=name)
json = req.get_json_by_name()
d = Data(json)
d.parse_json()
print d.get_needed_data()
def push():
associados = selecionar()
options = {
"host": config['api']['host'],
"port": config['api']['port'],
"method": 'POST',
"route": '/ping',
"headers": { "token": 'xxx', "Content-Type": 'application/json' },
"body": json.dumps(associados)
}
req = Request(options)
response = req.request()
# print(response['status'])
# print(response['content'])
def get_additional_codes(self):
types = "2346789ABCDPVX"
types = [char for char in types]
for type in types:
url = 'additional_codes/search?type={}'.format(type)
path = 'json/[scope]/additional_codes/additional_codes_{}.json'.format(
type)
additional_codes = Request(url, path, self.scope, self.as_of).json
def get_new_requests():
query = "SELECT reqid, uid, reqname, reqstatus, num_containers FROM request WHERE reqstatus = 'NEW' ORDER BY reqid"
req_list = []
try:
conn = get_connection()
if conn:
cursor = conn.cursor()
cursor.execute(query)
for item in cursor:
request = Request()
request.reqid = item[0]
request.user = item[1]
request.name = item[2]
request.status = item[3]
request.num_containers = item[4]
print('Request: ', vars(request))
req_list.append(request)
cursor.close()
conn.close()
except mysql.connector.Error as err:
logging.error('Getting New Requests on Database Error: %s', err)
finally:
if not req_list:
logging.debug('Not find any New Request on Database')
return req_list
def download_commodities(self, minimum_code):
# Look for the downloaded goods_nomenclatures files in the sections folder
root = os.path.dirname(os.path.realpath(__file__))
root = os.path.join(root, "..")
root = os.path.realpath(root)
subfolder = os.path.join(root, "json")
subfolder = os.path.join(subfolder, self.scope)
subfolder = os.path.join(subfolder, "goods_nomenclatures")
try:
os.mkdir(subfolder)
except:
pass
if not (os.path.isdir(subfolder)):
print("Subfolder cannot be found")
sys.exit()
else:
file_list = os.listdir(subfolder) # dir is your directory path
file_list.sort()
if len(file_list) != 21:
print(
"There need to be 21 files in the json/goods_nomenclatures subfolder: one for each section"
)
sys.exit()
else:
for file in file_list:
if "04" in file:
full_path = os.path.join(subfolder, file)
looper = 0
with open(full_path, "r") as f:
data = json.load(f)
for item in data["data"]:
href = item["attributes"]["href"]
goods_nomenclature_item_id = item[
"attributes"]["goods_nomenclature_item_id"]
if goods_nomenclature_item_id[0:2] == "22":
# if goods_nomenclature_item_id >= minimum_code:
producline_suffix = item["attributes"][
"producline_suffix"]
chapter_id = goods_nomenclature_item_id[
0:2]
if producline_suffix == "80":
if "commodities" in href:
looper += 1
looper = looper % 4
if looper != 999:
url = 'commodities/{}'.format(
goods_nomenclature_item_id)
path = 'json/[scope]/commodities/{}/{}.json'.format(
chapter_id,
goods_nomenclature_item_id)
# commodity = Request(url, path, self.scope, "2021-01-07").json
commodity = Request(
url, path, self.scope,
"").json
def download(self):
# Get all headings
try:
for item in self.chapter["included"]:
if item["type"] == "heading":
heading_id = item["attributes"][
"goods_nomenclature_item_id"][0:4]
url = 'headings/{}'.format(heading_id)
path = 'json/headings/heading_{}.json'.format(heading_id)
heading = Request(url, path, self.scope, self.as_of).json
except:
pass
def requestMenu(session):
print("""
1. Create a New Request
2. Exit
""")
menu = input("Enter the Selected Option: ")
if menu == '1':
request = Request()
print('\n')
request.name = input('Enter Request Name: ')
request.user = session.uid
request.num_containers = int(input('Enter the Number of Containers: '))
print("""
1. Executing equal jobs?
2. Executing different jobs?
3. Exit
""")
menu2 = input('Enter the Selected Option: ')
if menu2 == '1':
request.listcontainers = containerMenuType2(
session, request.num_containers)
elif menu2 == '2':
request.listcontainers = containerMenuType1(
session, request.num_containers)
elif menu2 == '3':
print('Get Out...')
sys.exit()
else:
print('Invalid Option!')
session.reqid = database.create_request(request)
print('Request ID: ', session.reqid)
i = 0
for container in request.listcontainers:
container.name = 'rqst' + str(session.reqid) + 'cntnr' + str(i)
database.create_container(session.reqid, container.appid,
container.name, container.command,
container.estimated_time)
i += 1
elif menu == '2':
print('Get Out...')
sys.exit()
else:
print('Invalid Option!')
def main(): from classes.request import Request from classes.data import Data import sys from datetime import datetime import time try: lat = sys.argv[1] lon = sys.argv[2] except: # Default NYC coordinates. lat = 40 lon = -74 req = Request(lat=lat, lon=lon) print "Uploading coordinates.." while True: now = datetime.now() hour = now.hour minute = now.minute if not hour % 3 and not minute: # If 0, 3, 6, 9, 12 hours and 0 minute res = req.get_json_by_coords() data = Data(json=res) data.parse_json() print data.get_needed_data() else: print "Waiting for next data.." time.sleep(60)
def get_sections(self):
# Get all sections and section notes and commodities
url = "json/[scope]/sections/sections.json"
sections = Request("sections", "json/[scope]/sections/sections.json",
self.scope, self.as_of).json
for item in sections["data"]:
# Get sections
url = 'sections/{}'.format(item["id"])
path = 'json/[scope]/sections/section_{:0>2}.json'.format(
item["id"])
section = Request(url, path, self.scope, self.as_of).json
# Get section notes
# url = 'sections/{}/section_note'.format(item["id"])
# path = 'json/[scope]/section_notes/section_note_{:0>2}.json'.format(
# item["id"])
# section_note = Request(url, path).json
# Get commodities
url = 'goods_nomenclatures/section/{}'.format(item["id"])
path = 'json/[scope]/goods_nomenclatures/goods_nomenclature_{:0>2}.json'.format(
item["id"])
self.goods_nomenclatures = Request(url, path, self.scope,
self.as_of).json
from classes.request import Request
from classes.router import Router
import controller
import socket
# create a server listening on port 8888
HOST, PORT = 'localhost', 8888
listen_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
listen_socket.bind((HOST, PORT))
listen_socket.listen(1)
print(f'Serving HTTP on http://{HOST}:{PORT}')
while True:
client_connection, client_address = listen_socket.accept()
request = Request(client_connection)
response = Router.process(request)
client_connection.send(str(response).encode())
client_connection.close()
num_reps = 1
estimated_time = 500
rqst_interval = 60
reps_interval = 500
if __name__ == '__main__':
logging.basicConfig(filename='./log/auto-submit-script.log', filemode='a', format='%(asctime)s %(levelname)s:%(message)s',
datefmt='%d/%m/%Y %H:%M:%S',level=logging.INFO)
logging.info('Start Submission Test:')
for i in range(num_reps):
logging.info('Test %d', (i + 1))
for j in range(num_requests):
#Creating Request
request = Request()
request.name = req_name.replace('%', str(j + 1))
request.user = uid
request.num_containers = num_containers
reqid = database.create_request(request)
logging.info('Request Created: %d', reqid)
#Creating Container
for k in range(num_containers):
container = Container()
container.name = 'rqst' + str(reqid) + 'cntnr' + str(k)
container.appid = appid
container.command = command
container.estimated_time = timedelta(seconds = estimated_time)
request.listcontainers.append(container)
database.create_container(reqid, container.appid, container.name, container.command, container.estimated_time)
def test_content_page():
request = Request('http://www.vagalume.com.br', 'michael jackson')
assert type(request.content_page()) == type(str())
def get_additional_code_types(self):
# Get all additional code types
additional_code_types = Request(
"additional_code_types",
"json/[scope]/additional_code_types/additional_code_types.json",
self.scope, self.as_of).json
def get_certificate_types(self):
# Get all certificate types
certificate_types = Request(
"certificate_types",
"json/[scope]/certificate_types/certificate_types.json",
self.scope, self.as_of).json
def get_footnote_types(self):
# Get all footnote types
footnote_types = Request(
"footnote_types",
"json/[scope]/footnote_types/footnote_types.json", self.scope,
self.as_of).json
def test_content_page_error():
request = Request('http://www.testee.com.br', 'michael jackson')
assert request.content_page() == False
def get_search_references(self):
# Get all search refereçnces (green pages)
search_references = Request(
"search_references",
"json/[scope]/search_references/search_references.json",
self.scope, self.as_of).json
def test_find_musics():
request = Request('http://www.vagalume.com.br', 'michael jackson')
crawler = Crawler(request.content_page(), 20)
assert type(crawler.find_musics()) == type(list())
def test_number_songs():
request = Request('http://www.vagalume.com.br', 'michael jackson')
crawler = Crawler(request.content_page(), 20)
musics = crawler.find_musics()
total = crawler.number_songs(musics)
assert type(total) == type(int())
print('\nInforme um número inteiro para escolher o site')
sys.exit()
# Tratamento de exceção se o usuário informar o número de um site que não exista
try:
sites[site_key]
url = site.find_url_site(site_key) # pesquisa a url do site selecionado
artist = input('Digite o nome do artista: ')
# Tratamento de input de usuário que deve ser um número inteiro
try:
number_songs = int(input('Digite a quantidade de músicas que deseja pesquisar o default são 15: '))
except ValueError:
number_songs = 15
print('\nComo não foi informado um número inteiro será retornado 15 músicas do artista/banda')
request = Request(url, artist)
content = request.content_page() # retorna o texto html da página do artista/banda escolhido
# Condição que faz a verificação se a página foi encontrada
if content == False:
print('\nA página do cantor escolhido não foi encontrada')
sys.exit()
crawler = Crawler(content, number_songs)
musics = crawler.find_musics() # retorna uma lista com as músicas
i = 0
print('\nForam encontradas %d músicas da banda/artista %s' % (crawler.number_songs(musics), artist.upper()))
# Faz a iteração na lista de músicas
for music in musics:
print('%d) %s' % (i, music))
i = i + 1
except IndexError: