def insert_bulk(data, i):
session_t = Session()
logger.info(f"Inserting {len(data)} profiles in bulk ({i})")
session_t.bulk_insert_mappings(ProfileModel, data)
session_t.commit()
session_t.close()
engine.dispose()
def attach_zones_to_mayors():
logger.info("STARTING attach_zones_to_mayors")
"""
A razão dessa função ser executada em outro momento é por causa da frequencia que o governo libera os dados.
Como a informação de zonas eleitorias é liberada a cada 1 minuto e os prefeitos a cada 10 segundos não achei prudente
executar essa funcionalidade a cada 10 segundos.
"""
try:
cities = get_cities()
for city in cities.values():
city_name, state = slugify(city['n']), city['e'].lower()
zone = read_from_mock_s3(f"/mock_s3/zone-{state}-{city_name}.json")
"""
Not will keep >5000 files in the github repo, just keep the major one zone-sp-sao-paulo.json
"""
if zone:
mayor_key = f"/cities/mayors-{state}-{city_name}.json"
mayors_per_city = json.loads(redis_mayors.get(mayor_key))
mayors_per_city["zone"] = zone
redis_mayors.set(mayor_key, json.dumps(mayors_per_city))
except Exception as ex:
logger.exception(ex)
def populate_model(count=1000000):
data = []
for i in range(count):
data.append(ScenarioBuilder.build_fake())
if i % 10000 == 0:
ScenarioBuilder.insert_bulk(data, i)
data = []
logger.info(f"Loading fake data index({i})")
session.bulk_insert_mappings(ProfileModel, data)
def create_database():
# remove database name from string connection before create the new schema
tmp_engine = create_engine(os.path.join(os.getenv("POSTGRES_URL"),
"postgres"),
echo=False)
with tmp_engine.connect() as conn:
conn.execute("commit")
try:
conn.execute(
f"CREATE DATABASE {os.getenv('DATABASE_NAME')} WITH OWNER postgres TABLESPACE pg_default"
)
logger.info(f"Database {os.getenv('DATABASE_NAME')} created")
except ProgrammingError as ex:
logger.error(ex)
def subscriber():
if os.getenv("QUEUE") == "REDIS":
while True:
message = redis_connection.lpop(os.getenv("CHANNEL_NAME"))
if message:
message = json.loads(message.decode("utf-8"))
if 'p' in message and message[
'p'] and 'm' in message and message['m']:
logger.info(
f"RECEBENDO REDIS({redis_connection.llen(os.getenv('CHANNEL_NAME'))}) {message['p']}, {message['m']}"
)
push_data.delay(message['p'], message['m'])
else:
logger.info(f"Waiting...")
elif os.getenv("QUEUE") == "SQS":
while True:
for message in sqs_queue.receive_messages(MaxNumberOfMessages=10):
message_dict = json.loads(message.body)
if 'i' in message_dict:
logger.info(
f"RECEBENDO SQS ID: {message_dict['i']}, {message_dict['p']}, {message_dict['m']}"
)
else:
logger.info(
f"RECEBENDO SQS {message_dict['p']}, {message_dict['m']}"
)
push_data.delay(message_dict['p'], message_dict['m'])
message.delete()
def getAllSavedSearchesByEmail():
query_params = request.args;
logger.info('Query Params: %s', query_params);
email = query_params.get('email');
if ((email is None) or (len(email) == 0)):
msg = 'Please specify valid Email.';
logger.error(msg);
return jsonify({'message': msg}), 400;
searches = t_advance_search.objects(email=email).all();
logger.info('Successfully fetched all records for email=%s. Response size = %s', email, len(searches));
searches = sorted(searches, key=lambda search: search.last_modified_time, reverse=True);
return jsonify({'results': [search.toJson() for search in searches]});
def delete_saved_search():
query_params = request.args;
logger.info('Query Params: %s', query_params);
id = query_params.get('id');
email = query_params.get('email');
if ((id is None) or (email is None)):
msg = 'Please specify ' + ('Email' if id else 'Id') +' in the request.';
logger.error(msg);
return jsonify({'message': msg}), 400;
savedSearch = None;
try:
savedSearch = t_advance_search.objects(id=id, email=email).first();
if savedSearch is None:
logger.info('No record found for id=%s and email=%s', id, email);
return jsonify({'message': 'No records found.'}), 404;
savedSearch.delete();
except Exception as ex:
logger.error(ex);
return jsonify({'message': str(ex)}), 500;
logger.info('Successfully deleted the record by id=%s and email=%s', savedSearch.id, savedSearch.email);
return jsonify({'ID': savedSearch.id}), 204;
def save_search():
request_body = request.get_json();
logger.info('Request Body: %s', request_body);
if 'email' not in request_body.keys():
msg = 'email is missing in the request body.';
logger.error(msg);
return jsonify({'message': msg}), 400;
if 'search_name' not in request_body.keys():
msg = 'search_name is missing in the request body.';
logger.error(msg);
return jsonify({'message': msg}), 400;
if 'search_query' not in request_body.keys():
msg = 'search_query is missing in the request body.';
logger.error(msg);
return jsonify({'message': msg}), 400;
email = request_body.get('email');
search_name = request_body.get('search_name');
search_query = request_body.get('search_query');
logger.info('Parsed data: email=%s, search_name=%s, search_query=%s', email, search_name, search_query);
savedSearch = None;
try:
savedSearch = t_advance_search.create(email=email, search_query=search_query, search_name=search_name);
except Exception as ex:
logger.error(ex);
return jsonify({'message': str(ex)}), 500;
logger.info('Successfully saved the data.');
return jsonify({'ID': savedSearch.id}), 201;
def getSavedSearchByIdAndEmail():
query_params = request.args;
logger.info('Query Params: %s', query_params);
id = query_params.get('id');
email = query_params.get('email');
if ((id is None) or (email is None)):
msg = 'Please specify ' + ('Email' if id else 'Id') +' in the request.';
logger.error(msg);
return jsonify({'message': msg}), 400;
try:
savedSearch = t_advance_search.objects(id=id, email=email).first();
except Exception as ex:
logger.error(ex);
return jsonify({'message': str(ex)}), 500;
if savedSearch is None:
logger.info('No record found for id=%s and email=%s', id, email);
#return Response(status=404); # Another way to return response
#abort(404, 'No Saved Search found.');
return jsonify({'message': 'No records found.'}), 404;
logger.info('Successfully fetched data by id=%s and email=%s. response = %s', id, email, savedSearch);
return jsonify(savedSearch.toJson());
def slice_mayors_by_city():
"""
Essa função deverá fatiar um arquivo de 2mb que contém todas as cidade do país e salvar no REDIS utilizando
uma parte da URL que o client side irá acessar como chave do cache.
Os arquivos são enviados pelo governo para uma central que faz o upload no S3 em tempo real
High frequency offen 10 seconds during the votes counting
"""
logger.info("STARTING slice_mayors_by_city")
try:
released_votes = read_from_mock_s3('/mock_s3/mayors.json')
cities = get_cities()
last_update = date_parse(
f"{released_votes['last_updated']['date']} {released_votes['last_updated']['time']}"
)
if redis_mayors.get("last_mayors_update"):
last_update_redis = date_parse(
redis_mayors.get("last_mayors_update").decode("utf-8"))
"""
For the testing purpose the mayors.json never will be updated by the government, so will be needed to bypass
"""
if last_update_redis <= last_update and False:
logger.info("UPDATE NOT REQUIRED")
return
mayors = {}
for id_city, city in released_votes["municipios"].items():
state, city_name = cities[id_city]["e"].lower(), slugify(
city["nm"])
mayors[f"/cities/mayors-{state}-{city_name}.json"] = json.dumps(
city)
redis_mayors.set("last_mayors_update", last_update.isoformat())
redis_mayors.mset(mayors)
except Exception as ex:
logger.exception(ex)
def build_redis_cache():
if os.getenv("CLEAR_GEO_REDIS_BEFORE_INSERT", "1") == "1":
redis_conn.delete(os.getenv("REDIS_GEOCACHE_NAME"))
logger.info(f"quering database")
profiles = session.query(ProfileModel).all()
logger.info(f"Copy data to REDIS")
for profile in profiles:
redis_conn.geoadd(os.getenv("REDIS_GEOCACHE_NAME"),
float(profile.longitude), float(profile.latitude),
profile.to_json())
logger.info(f"Added {len(profiles)}")
def create_model():
try:
base.metadata.create_all(engine)
logger.info(f"Models profile created")
except Exception as ex:
logger.info(f"Models profile already created")
def create_extension_postgis():
with engine.connect() as conn:
conn.execute(f"CREATE EXTENSION IF NOT EXISTS postgis")
logger.info(f"Instalação da extensão postgis")
def __init__(self):
logger.info(f"Scenario Builder execute")
ScenarioBuilder.create_database()
ScenarioBuilder.create_extension_postgis()
ScenarioBuilder.create_model()
ScenarioBuilder.populate_model()
import json
import os
import random
from time import sleep
from app_config import fake, redis_connection, logger, sqs_queue
for i in range(100):
phone_number = fake.phone_number()
message = fake.sentence()
if os.getenv("QUEUE") == "REDIS":
redis_connection.rpush(
os.getenv("CHANNEL_NAME"),
json.dumps(dict(p=phone_number, m=message, i=i)))
logger.info(
f"PUBLICACAO ID: {i} FILA REDIS {redis_connection.llen(os.getenv('CHANNEL_NAME'))}"
)
elif os.getenv("QUEUE") == "SQS" and sqs_queue:
response = sqs_queue.send_message(
MessageBody=json.dumps(dict(p=phone_number, m=message, i=i)),
MessageGroupId=f"Group_{os.getenv('QUEUE')}")
logger.info(f"PUBLICACAO ID: {i} FILA SQS")
else:
raise NotImplemented()
if os.getenv("PUBLISHER_DELAY"):
sleep(random.uniform(0, float(os.getenv("DELAY"))))
def push_data(phone, message):
sleep(1)
logger.info(f"Pushed {phone} - {message}")
