def webhook_elk():
print("/ [POST]-> " + str(sys.stdout.flush()))
data = request.data
data_parse = bytesELK2json(data)
print_json(data_parse)
print("/heartbeat [POST] | len(list_groups):{0}".format(len(list_groups)))
return '', 200
def get(url, pretty):
"""retrieve an entity by its ID."""
if url is None:
utils.error("You must give the URL (ID) of the entity to retrieve")
json = nexus_utils.get_by_id(url)
utils.print_json(json, colorize=pretty)
def webhook_twilio_callback_url():
print("[alert_elk] twilio_callback_url()-> " + str(sys.stdout.flush()))
#print_json( bytesELK2json( request.data ))
print_json(request.json)
print_parameters(request)
rpt = req_post(URL_NGROK, data=request.data, timeout=None)
return '', 200
def set_user(config, id, email, name, org, title, clearance):
# validate has some data to update
if email == None and name == None and org == None and title == None and clearance == None:
click.echo(
"Must specify at least one of --email, --name, --org, --title, or --clearance."
)
return
# make update map
update_map = {}
if email:
update_map["email"] = email
if name:
update_map["recipient_name"] = name
if org:
update_map["org_name"] = org
if title:
update_map["title"] = title
if clearance:
update_map["clearance"] = clearance
# open grpc channel
channel, key = conn.get_connection(config)
# connect to grpc stub
stub = recipient_pb2_grpc.RecipientServiceStub(channel)
# make request
request = recipient_pb2.UpdateRecipientRequest(recipient_id=id,
update_map=update_map)
# send request
try:
response = stub.UpdateRecipient(request, metadata=[('biapikey', key)])
except grpc.RpcError as e:
utils.print_grpc_errors(e)
else:
utils.print_json(response)
def webhook_elk():
print("/ [post]]-> "+ str(sys.stdout.flush()) )
data = request.data
data_parse = bytesELK2json(data)
print_json( data_parse )
save_yml( data_parse , nameFile="alertas_elk.yml")
return '', 200
def put_org_tag(config, org, tags):
# validate org
org_id = utils.get_org_id_by_name(config, org)
if org_id == -1:
return
# open grpc channel
channel, key = conn.get_connection(config)
# connect to grpc stub
stub = org_pb2_grpc.BimetaOrgServiceStub(channel)
# break out tags
tag_list = utils.split_tags(tags)
for tag_name in tag_list:
# make request
org_tag = org_pb2.BimetaOrgTag(org_id=org_id,
tag_name=tag_name,
tag_value=tag_list[tag_name])
request = org_pb2.CreateBimetaOrgTagRequest(bimeta_org_tag=org_tag)
# send request
try:
response = stub.CreateBimetaOrgTag(request,
metadata=[('biapikey', key)])
except grpc.RpcError as e:
utils.print_grpc_errors(e)
else:
utils.print_json(response.bimeta_org_tag)
def _insert_and_verify_cmd(parse_data, db_session, queries):
if 'insert_data' in parse_data and parse_data['insert_data'] == True:
insert_data(db_session, parse_data['data_to_insert'])
query_results = []
verification_db = SQLiteDB()
verification_db.drop_table()
verification_db.setup()
query_id = 0
hash_data = False
if 'files' in parse_data['data_to_insert']:
hash_data = True
for query in queries:
print "Querying: {}".format(query)
query_res = db_session.query_db(query, hash_files=hash_data, time_out=300)
query_res_count = 0
# Row result is expected to contain:
# [[ID, file hashed by query_db function, file name], ...]
for row in query_res['result']:
verification_db.insert(query_id, query_res_count, row[-1], str(row[1]))
query_res_count += 1
if query_res_count == 0:
warning = color_str("WARNING: ", color='y')
print warning, "Invalid query occurred, no results retrieved (may be intentional)."
print warning, "QUERY {}".format(query)
print warning, "RESULT {}".format(query_res)
query_results.append(query_res)
query_id += 1
print print_json({"results": query_results})
def put_recipient_tag(config, email, tags):
# validate email
recipient_id = utils.get_recipient_id_by_email(config, email)
if recipient_id == -1:
return
# open grpc channel
channel, key = conn.get_connection(config)
# connect to grpc stub
stub = recipient_pb2_grpc.RecipientServiceStub(channel)
# break out tags
tag_list = utils.split_tags(tags)
for tag_name in tag_list:
# make request
recipient_tag = recipient_pb2.RecipientTag(
recipient_id=recipient_id,
tag_name=tag_name,
tag_value=tag_list[tag_name])
request = recipient_pb2.CreateRecipientTagRequest(
recipient_tag=recipient_tag)
# send request
try:
response = stub.CreateRecipientTag(request,
metadata=[('biapikey', key)])
except grpc.RpcError as e:
utils.print_grpc_errors(e)
else:
utils.print_json(response.recipient_tag)
def put_user(config, email, name, org, title, clearance):
# get the org id
org_id = utils.get_org_id_by_name(config, org)
if org_id == -1:
return
# open grpc channel
channel, key = conn.get_connection(config)
# connect to grpc stub
stub = recipient_pb2_grpc.RecipientServiceStub(channel)
# make request
recipient = recipient_pb2.Recipient(email=email,
recipient_name=name,
org_id=org_id,
title=title,
clearance=clearance)
request = recipient_pb2.CreateRecipientRequest(recipient=recipient)
# send request
try:
response = stub.CreateRecipient(request, metadata=[('biapikey', key)])
except grpc.RpcError as e:
utils.print_grpc_errors(e)
else:
utils.print_json(response.recipient)
def show(args):
dynamodb = boto3.resource('dynamodb', endpoint_url=args.endpoint)
table = dynamodb.Table(args.table)
response = table.scan()
show_any = False
for item in response['Items']:
gym = item['gym']
user = item['user_id_key']
item_bytes = base64.standard_b64decode(gym)
gym = Gym()
gym.ParseFromString(item_bytes)
should_show = False
if args.user:
if args.user == user:
should_show = True
elif args.name:
if re.search(args.name, gym.name, re.IGNORECASE):
should_show = True
else:
should_show = True
if should_show:
show_any = True
print_json(gym, args.depth, user)
if not show_any:
print("Nothing to show.")
return 0
def print_best_model():
space_best_model = load_best_hyperspace()
if space_best_model is None:
print("No best model to plot. Continuing...")
return
print("Best hyperspace yet:")
print_model_data(['model_name','train_accuracy','test_accuracy','model_param_num'])
print_json(space_best_model)
def plot_best_model():
"""Plot the best model found yet."""
space_best_model = load_best_hyperspace()
if space_best_model is None:
tf.logging.info("No best model to plot. Continuing...")
return
tf.logging.info("Best hyperspace yet:")
print_json(space_best_model)
plot(space_best_model, "model_best")
def plot_best_model():
"""Plot the best model found yet."""
space_best_model = load_best_hyperspace_retrained()
if space_best_model is None:
print("No best model to plot. Continuing...")
return
# Print best hyperspace and save model png
print("Best hyperspace yet:")
print_json(space_best_model)
plot(space_best_model, "retrained_model_best")
def process_data(list_json_old_data):
list_json_new_data = {}
list_json_new_data = calc_status_vers_product_agent(list_json_old_data)
#print_json(list_json_new_data)
list_json_new_data = calc_tipo_product(list_json_new_data)
#print_json(list_json_new_data)
list_json_new_data = calc_status_vers_VST_EST(list_json_new_data)
#print_json(list_json_new_data)
list_json_new_data = calc_status_data_amc(list_json_new_data)
#print_json(list_json_new_data)
list_json_new_data = calc_status_final(list_json_new_data)
print_json(list_json_new_data)
return list_json_new_data
def process_buckets_documents(bucket_documents, index_name):
elk = elasticsearch()
#Sending Bucket to Elasticsearch
rpt = elk.post_bulk(bucket_documents,header_json={"index":{"_index":index_name ,"_type":"_doc"}})
try:
if 'errors' in rpt:
if rpt['errors'] :
print(" INFO | process_buckets_documents | Response from ElasticSearch | Errors: {0}".format( rpt['errors'] ))
else:
print(" ERROR | process_buckets_documents | Response from ElasticSearch | error_field_not_found")
print_json(rpt)
input(" INFO | process_buckets_documents | error_field_not_found | Press any key to continue ...")
finally:
input(" INFO | process_buckets_documents | Data loaded into index={0}".format( index_name ) )
except:
def del_user(config, email):
# open grpc channel
channel, key = conn.get_connection(config)
# connect to grpc stub
stub = recipient_pb2_grpc.RecipientServiceStub(channel)
# make request
request = recipient_pb2.DeleteRecipientByEmailRequest(email=email)
# send request
try:
response = stub.DeleteRecipientByEmail(request,
metadata=[('biapikey', key)])
except grpc.RpcError as e:
utils.print_grpc_errors(e)
else:
utils.print_json(response)
def engine_facebook(data_json):
#Handle messages sent by facebook messenger to the application
if data_json["object"] == "page":
for entry in data_json["entry"]:
for messaging_event in entry["messaging"]:
if messaging_event.get("message"):
sender_id = messaging_event["sender"]["id"]
recipient_id = messaging_event["recipient"]["id"]
message_text = messaging_event["message"]["text"]
#send_message_split(sender_id, parse_natural_text(message_text))
response_text = process_mssg(message_text)
send_message_split(sender_id, response_text)
else:
print("{0}|WARN | engine_facebook ".format(
datetime.utcnow().isoformat()))
print_json(data_json)
return "ok"
def get_emails(config, email, num):
# validate email
# open grpc channel
channel, key = conn.get_connection(config)
# connect to grpc stub
stub = badapicreds_pb2_grpc.CredentialServiceStub(channel)
# make request
request = badapicreds_pb2.ListEmailCredentialRequest(email=email,
maxResults=num)
# send request
try:
response = stub.ListEmailCredentials(request,
metadata=[('biapikey', key)])
for cred in response:
utils.print_json(cred)
except grpc.RpcError as e:
utils.print_grpc_errors(e)
def add(args):
dynamodb = boto3.resource('dynamodb', endpoint_url=args.endpoint)
table = dynamodb.Table(args.table)
gym = Gym()
if args.file:
file_bytes = open(args.file, 'rb').read()
Parse(file_bytes, gym)
elif args.data:
data = args.data.replace("'", "\"")
Parse(data, gym)
gym_bytes = gym.SerializeToString()
gym_encoded = base64.standard_b64encode(gym_bytes)
response = table.scan()
already_exists = False
itr_gym = "{ no gym }"
for item in response['Items']:
itr_gym_encoded = item['gym']
itr_bytes = base64.standard_b64decode(itr_gym_encoded)
itr_gym = Gym()
itr_gym.ParseFromString(itr_bytes)
if itr_gym_encoded == gym_encoded:
already_exists = True
break
elif itr_gym.uuid == gym.uuid:
already_exists = True
break
if already_exists:
print("Item already exists:")
print_json(itr_gym, args.depth, args.user)
else:
table.put_item(Item={'gym': gym_encoded, 'user_id_key': args.user})
print('Successfully added item')
print_json(gym, args.depth, args.user)
return 0
def get_org_tag(config, org):
# validate org
org_id = utils.get_org_id_by_name(config, org)
if org_id == -1:
return
# open grpc channel
channel, key = conn.get_connection(config)
# connect to grpc stub
stub = org_pb2_grpc.BimetaOrgServiceStub(channel)
# make request
request = org_pb2.ListBimetaOrgTagRequest(org_id=org_id)
# send request
try:
response = stub.ListBimetaOrgTag(request, metadata=[('biapikey', key)])
for tag in response:
utils.print_json(tag.bimeta_org_tag)
except grpc.RpcError as e:
utils.print_grpc_errors(e)
def get_recipient_tag(config, email):
# validate email
recipient_id = utils.get_recipient_id_by_email(config, email)
if recipient_id == -1:
return
# open grpc channel
channel, key = conn.get_connection(config)
# connect to grpc stub
stub = recipient_pb2_grpc.RecipientServiceStub(channel)
# make request
request = recipient_pb2.ListRecipientTagRequest(recipient_id=recipient_id)
# send request
try:
response = stub.ListRecipientTag(request, metadata=[('biapikey', key)])
for tag in response:
utils.print_json(tag.recipient_tag)
except grpc.RpcError as e:
utils.print_grpc_errors(e)
def remove(args):
dynamodb = boto3.resource('dynamodb', endpoint_url=args.endpoint)
table = dynamodb.Table(args.table)
response = table.scan()
remove_any = False
for item in response['Items']:
gym_encoded = item['gym']
user = item['user_id_key']
item_bytes = base64.standard_b64decode(gym_encoded)
gym = Gym()
gym.ParseFromString(item_bytes)
should_remove = False
if not args.uuid and not args.user and args.all:
should_remove = True
print("removed by all:")
if gym.uuid == args.uuid:
should_remove = True
print("removed by uuid:")
if user == args.user and (gym.name == args.name or args.all):
should_remove = True
print("removed by user and name")
if args.name and re.search(args.name, gym.name, re.IGNORECASE):
should_remove = True
print("removed by regex name")
if should_remove:
remove_any = True
if not args.dry_run:
table.delete_item(Key={'gym': gym_encoded})
else:
print("[dry run, not actually removed]")
print_json(gym, args.depth, args.user)
if not remove_any:
print("No items matching criteria")
return 0
def put_config(config, org, name, file):
contents = None
# validate file
if file == "stdin":
# read file from std in
contents = sys.stdin.readlines()
else:
if os.path.exists(file):
f = open(file)
contents = f.read()
f.close()
if contents == None:
click.echo("Unable to read file")
return
# validate org
org_id = utils.get_org_id_by_name(config, org)
if org_id == -1:
return
# open grpc channel
channel, key = conn.get_connection(config)
# connect to grpc stub
stub = org_pb2_grpc.BimetaOrgServiceStub(channel)
# make request
org_config = org_pb2.BimetaOrgConfig(org_id=org_id,
config_name=name,
content=contents)
request = org_pb2.CreateBimetaOrgConfigRequest(
bimeta_org_config=org_config)
# send request
try:
response = stub.CreateBimetaOrgConfig(request,
metadata=[('biapikey', key)])
except grpc.RpcError as e:
utils.print_grpc_errors(e)
else:
utils.print_json(response.bimeta_org_config)
def del_org_tag(config, org, tags):
# validate org
org_id = utils.get_org_id_by_name(config, org)
if org_id == -1:
return
# open grpc channel
channel, key = conn.get_connection(config)
# connect to grpc stub
stub = org_pb2_grpc.BimetaOrgServiceStub(channel)
for tag_name in tags:
# make request
request = org_pb2.DeleteBimetaOrgTagRequest(org_id=org_id,
tag_name=tag_name)
# send request
try:
response = stub.DeleteBimetaOrgTag(request,
metadata=[('biapikey', key)])
except grpc.RpcError as e:
utils.print_grpc_errors(e)
else:
utils.print_json(response)
def del_recipient_tag(config, email, tags):
# validate email
recipient_id = utils.get_recipient_id_by_email(config, email)
if recipient_id == -1:
return
# open grpc channel
channel, key = conn.get_connection(config)
# connect to grpc stub
stub = recipient_pb2_grpc.RecipientServiceStub(channel)
for tag_name in tags:
# make request
request = recipient_pb2.DeleteRecipientTagRequest(
recipient_id=recipient_id, tag_name=tag_name)
# send request
try:
response = stub.DeleteRecipientTag(request,
metadata=[('biapikey', key)])
except grpc.RpcError as e:
utils.print_grpc_errors(e)
else:
utils.print_json(response)
def get_github(config, org, begin, end, viewed):
# set default start/end date
to_tm = datetime.datetime.utcnow()
from_tm = to_tm - datetime.timedelta(hours=24)
# if date provided, time to work with them
if begin != None:
try:
from_tm = datetime.datetime.fromisoformat(begin)
except:
click.echo(
f"Unable to convert {begin} (does it match ISO format?)")
return
if end != None:
try:
to_tm = datetime.datetime.fromisoformat(end)
except:
click.echo(
f"Unable to convert {begin} (does it match ISO format?)")
return
# open grpc channel
channel, key = conn.get_connection(config)
# connect to grpc stub
stub = github_pb2_grpc.GithubServiceStub(channel)
# make request
request = github_pb2.ListAlertsRequest(
org=org,
viewed=viewed,
startdate=from_tm.strftime("%Y-%m-%d %H:%M:%S.%f"),
enddate=to_tm.strftime("%Y-%m-%d %H:%M:%S.%f"))
utils.print_json(request)
# send request
try:
response = stub.ListAlerts(request, metadata=[('biapikey', key)])
for alert in response:
utils.print_json(alert)
except grpc.RpcError as e:
utils.print_grpc_errors(e)
def replace(args):
dynamodb = boto3.resource('dynamodb', endpoint_url=args.endpoint)
table = dynamodb.Table(args.table)
gym = Gym()
if args.file:
file_bytes = open(args.file, 'rb').read()
Parse(file_bytes, gym)
elif args.data:
data = args.data.replace("'", "\"")
Parse(data, gym)
gym_bytes = gym.SerializeToString()
gym_encoded = base64.standard_b64encode(gym_bytes)
response = table.scan()
for item in response['Items']:
itr_gym_encoded = item['gym']
itr_user = item['user_id_key']
itr_bytes = base64.standard_b64decode(itr_gym_encoded)
itr_gym = Gym()
itr_gym.ParseFromString(itr_bytes)
if itr_gym.uuid == args.uuid:
print('Successfully replaced item')
print_json(itr_gym, args.depth, itr_user)
print("With item")
print_json(gym, args.depth, itr_user)
table.delete_item(Key={'gym': itr_gym_encoded})
table.put_item(Item={'gym': gym_encoded, 'user_id_key': itr_user})
return 0
def get_creds(config, username, fqdn, num, startdate, enddate):
# open grpc channel
channel, key = conn.get_connection(config)
# connect to grpc stub
stub = badapicreds_pb2_grpc.CredentialSearchServiceStub(channel)
# make request
start = ""
end = ""
if startdate != None:
start = startdate.strftime("%Y-%m-%dT%H:%M:%SZ")
if enddate != None:
end = enddate.strftime("%Y-%m-%dT%H:%M:%SZ")
request = badapicreds_pb2.SearchTerm(username=username,
fqdn=fqdn,
maxResults=num,
start=start,
end=end)
# send request
try:
response = stub.ListCredentials(request, metadata=[('biapikey', key)])
for cred in response:
utils.print_json(cred)
except grpc.RpcError as e:
utils.print_grpc_errors(e)
def _get_tar_file_list(backup_file_list, backup_file):
tar_file_list = {}
# Create a {file_name : checksum} dictionary of all backup files.
# This has only to be created once. Else all data can just be read again.
if os.path.exists(backup_file_list):
with io.open(backup_file_list, 'r') as f:
tar_file_list = json.loads(f.read())
else:
for member in backup_file:
if member.isfile():
f = backup_file.extractfile(member)
tar_file_list[unicode(member.name)] = gen_checksum_from_file(f, use_file=True)
f.close()
# Write the entire dictionary.
with io.open(backup_file_list, 'w+') as f:
f.write(unicode(print_json(tar_file_list)))
return tar_file_list
def print_parameters(request):
rpt = None
print("[INFO ] print_parameters | len = {0}".format(request))
#try:
multi_dict = request.args
for key in multi_dict:
print("KEY\t:{0}".format(multi_dict.get(key)))
print(multi_dict.getlist(key))
print("[INFO ] printing form data: ")
data_form = request.form
print("[INFO ] printing headers: ")
print_json(dict(request.headers))
print("[INFO ] printing form: ")
print_json(dict(request.form))
print("[INFO ] printing args: ")
print_json(dict(request.args))
print("[INFO ] printing content: ")
print_json(request.json)
#for key in data_form:/ print ( "{0}\t: {1}".format( key, data_form[key] ))
return rpt
def print_debug_message(message):
if isinstance(message, dict) or isinstance(message, list):
print_json(message)
else:
print(Logger.TPL_DEBUG_MESSAGE.format(message))
