def test_export_recent_uploads():
loop.run_until_complete(export_recent_uploads(conf))
rec_upl = load_json(WEB_ROOT + '/json/recent_uploads.json')
assert rec_upl
assert rec_upl[0]['uploadDate']
assert rec_upl[0]['rda']
def test_export_rankings():
loop.run_until_complete(export_rankings(conf))
rankings = load_json(WEB_ROOT + '/json/rankings.json')
act = rankings['activator']['total']['total'][0]
assert act['rank'] == 1
assert act['callsign']
hunt = rankings['hunter']['total']['total'][0]
assert hunt['rank'] == 1
assert hunt['callsign']
def test_ext_loggers():
callsign = 'R7AB'
def request(update=None, delete=None):
data = {'token': create_token({'callsign': callsign})}
if update:
data['update'] = update
if delete:
data['delete'] = delete
return requests.post(API_URI + '/loggers',\
data=json.dumps(data))
logging.debug('ext_loggers insert')
lotw_login = load_json(path.dirname(path.abspath(__file__)) + '/lotw_login.json')
update = {'logger': 'LoTW', 'loginData': {}}
update['loginData'].update(lotw_login)
rsp = request(update)
assert rsp.status_code == 200
data = json.loads(rsp.text)
logging.debug(data)
assert not data['state']
assert data['id']
id = data['id']
logging.debug('ext_loggers list')
rsp = request()
assert rsp.status_code == 200
data = json.loads(rsp.text)
assert data
assert 'loggers' in data
assert 'accounts' in data
assert data['accounts'][0]['logger'] == 'LoTW'
assert data['accounts'][0]['loginData']
for k in update['loginData']:
assert data['accounts'][0]['loginData'][k] == update['loginData'][k]
logging.debug('ext_loggers update bad login')
update['loginData']['password'] += '_'
update['id'] = id
rsp = request(update)
assert rsp.status_code == 200
data = json.loads(rsp.text)
assert data['state']
logging.debug('ext_loggers delete')
rsp = request(delete=id)
assert rsp.status_code == 200
assert data['state']
def _json_validator(schemas_path):
schemas = load_json(schemas_path)
def _validate_dict(data, schema):
"""validates dict data with one of predefined jsonschemas
return true/false"""
try:
jsonschema.validate(data, schemas[schema])
return (True, None)
except jsonschema.exceptions.ValidationError as exc:
logging.error('Error validating json data. Schema: ' + schema)
logging.error(data)
logging.error(exc.message)
return (False, exc.message)
return _validate_dict
def scrape_slack(token, slack_args, filter_func=lambda x: x):
sc = SlackClient(token)
results = get_messages(sc, slack_args, [], filter_func)
while results['last_ts']:
slack_args['latest'] = results['last_ts']
results = get_messages(sc, slack_args, results['messages'],
filter_func)
print('Done fetching messages. Found {} in total.'.format(
len(results['messages'])))
return results['messages']
if __name__ == '__main__':
config = load_json('./env.json')
ap = argparse.ArgumentParser()
ap.add_argument('-c', '--channel', help='channel id to scrape')
ap.add_argument('-o', '--output', help='file to save out')
args = vars(ap.parse_args())
channel = args['channel']
output = args['output']
try:
old_json = load_json(output)
except Exception as e:
old_json = []
print('No existing messages, starting from scratch...')
slack_args = {
#!/usr/bin/env python
from json_utils import load_json, dump_json
from slackscrape import scrape_slack
from slackclient import SlackClient
import operator
import argparse
import os
config = load_json('./env.json')
def ensure_dir(directory):
if not os.path.exists(directory):
os.makedirs(directory)
return directory
if __name__ == '__main__':
ap = argparse.ArgumentParser()
ap.add_argument('-u',
'--update',
help='update channels',
action="store_true")
args = vars(ap.parse_args())
channel_args = {
'exclude_archived': 0,
}
sc = SlackClient(config['token'])
#!/usr/bin/env python
from json_utils import load_json, dump_json
from slackclient import SlackClient
import operator
import os
import argparse
def ensure_dir(directory):
if not os.path.exists(directory):
os.makedirs(directory)
return directory
config = load_json('./env.json')
if __name__ == '__main__':
ap = argparse.ArgumentParser()
ap.add_argument('-u',
'--update',
help='update channels',
action="store_true")
args = vars(ap.parse_args())
slack_args = {'presence': 1}
sc = SlackClient(config['token'])
response = sc.api_call('users.list', **slack_args)
users = response['members']
#!/usr/bin/python3
#coding=utf-8
from json_utils import load_json, save_json
rda = load_json('/var/www/cfmrda-dev/public/json/rda.json')
new = []
for group in rda:
new_group = {'group': group['group']}
last_value = int(group['values'][-1]['displayValue'])
if last_value != len(group['values']):
skip = []
c_el = 1
c_no = 1
while c_no < last_value:
if int(group['values'][c_el - 1]['displayValue']) != c_no:
skip.append(c_no)
else:
c_el += 1
c_no += 1
new_group['skip'] = skip
new_group['last'] = last_value
new.append(new_group)
save_json(new, '/var/www/cfmrda-dev/src/rdaShort.json')
def apply_range_modifier(container, modifier_ele):
plLogger = PLLogger.GetLogger("methodology")
if container is None:
return "template_processing_functions.apply_range_modifier " + \
"requires an StmTemplateConfig. Received invalid element (None)."
if not container.IsTypeOf("StmTemplateConfig"):
return "template_processing_functions.apply_range_modifier " + \
"requires an StmTemplateConfig. Received an object of type: " + \
container.GetType() + "."
if modifier_ele is None:
return "template_processing_functions.apply_range_modifier " + \
"requires an ElementTree modifier_ele (ModifierInfo). " + \
"Received invalid element (None)."
tag_name = modifier_ele.get("TagName")
md_json = modifier_ele.get("ModifierInfo")
if tag_name is None:
return "TagName attribute is required in the ModifierInfo"
if md_json is None:
err_str = "ModifierInfo is required in the StmPropertyModifier: " + \
str(modifier_ele)
return err_str
# Validate the ModifierInfo against the schema
res = json_utils.validate_json(md_json,
get_range_modifier_json_schema())
if res != "":
err_str = "ModifierInfo JSON is invalid or does not conform " + \
"to the schema: " + res
return err_str
# Load the JSON
err_str, md_dict = json_utils.load_json(md_json)
if err_str != "":
return err_str
plLogger.LogDebug("md_dict: " + str(md_dict))
obj_name = md_dict.get("objectName")
prop_name = md_dict.get("propertyName")
if not CMeta.ClassExists(obj_name):
err_str = "Class " + obj_name + " does not exist."
return err_str
if prop_name not in CMeta.GetProperties(obj_name):
err_str = "Property " + prop_name + " does not exist on " + \
obj_name
return err_str
# prop_range = CMeta.GetPropertyRange(obj_name, prop_name)
is_collection = dm_utils.is_property_collection(obj_name, prop_name)
plLogger.LogDebug("obj_name: " + obj_name)
plLogger.LogDebug("prop_name: " + prop_name)
# Determine the property type
prop_dict = CMeta.GetPropertyMeta(obj_name, prop_name)
plLogger.LogDebug("prop_dict: " + str(prop_dict))
prop_type = prop_dict["type"]
# Process the propertyValueDict
prop_val_dict = md_dict.get("propertyValueDict", {})
# start (required)
start = prop_val_dict["start"]
if isinstance(start, basestring):
start_list = [start]
else:
start_list = start
# Truncate the list of start values based on whether the
# property is a collection.
if not is_collection:
start_list = [start_list[0]]
# Count for all lists will depend on how many items are
# in the start_list
count = len(start_list)
# step (required)
step = prop_val_dict["step"]
if isinstance(step, basestring):
step_list = [step]
else:
step_list = step
# repeat
repeat = prop_val_dict.get("repeat", None)
if repeat is None:
repeat = [0]
if isinstance(repeat, int):
repeat_list = [repeat]
else:
repeat_list = repeat
# recycle
recycle = prop_val_dict.get("recycle", None)
if recycle is None:
recycle = [0]
if isinstance(recycle, int):
recycle_list = [recycle]
else:
recycle_list = recycle
# targetObjectStep (use default value if nothing specified)
target_step = prop_val_dict.get("targetObjectStep", None)
if target_step is None:
err_str, zero_val = get_zero_value(prop_type)
if err_str != "":
plLogger.LogWarn(err_str)
target_step = [str(zero_val)]
if isinstance(target_step, basestring):
target_step_list = [target_step]
else:
target_step_list = target_step
# Default for reset is True
reset = prop_val_dict.get("resetOnNewTargetObject", True)
# Turn everything into lists and even them out based on start_list
# prop_type takes precedence unless the required list/scalar is
# unspecified.
# len(start_list) determines list length; shorter lists will be
# extended. Longer lists will be truncated.
# (start_list and start are processed above)
step_list = build_range_parameter_list(is_collection,
step_list, count=count)
repeat_list = build_range_parameter_list(is_collection,
repeat_list, count=count)
recycle_list = build_range_parameter_list(is_collection,
recycle_list, count=count)
target_step_list = build_range_parameter_list(is_collection,
target_step_list,
count=count)
# Check the values
for val in start_list:
res = check_value_type(prop_type, val)
if res != "":
return "Invalid value " + str(val) + \
" in start: " + res
for val in step_list:
res = check_value_type(prop_type, val)
if res != "":
return "Invalid value " + str(val) + \
" in step: " + res
for val in target_step_list:
res = check_value_type(prop_type, val)
if res != "":
return "Invalid value " + str(val) + \
" in targetObjectStep: " + res
plLogger.LogDebug("count: " + str(count))
plLogger.LogDebug("is_collection: " + str(is_collection))
plLogger.LogDebug("start_list: " + str(start_list))
plLogger.LogDebug("step_list: " + str(step_list))
plLogger.LogDebug("repeat_list: " + str(repeat_list))
plLogger.LogDebug("recycle_list: " + str(recycle_list))
plLogger.LogDebug("target_step_list: " + str(target_step_list))
plLogger.LogDebug("reset: " + str(reset))
target_dict = get_modify_objects(container, obj_name, tag_name)
if target_dict == {}:
plLogger.LogWarn("No objects to modify")
return ""
if prop_type == "u8" or \
prop_type == "u16" or \
prop_type == "u32" or \
prop_type == "u64":
handle_uint_range_update(
target_dict, start_list, step_list, repeat_list,
recycle_list, target_step_list, reset,
obj_name, prop_name)
elif prop_type == "ip" or prop_type == "ipv6":
handle_ip_range_update(
target_dict, start_list, step_list, repeat_list,
recycle_list, target_step_list, reset,
obj_name, prop_name)
elif prop_type == "mac":
handle_mac_range_update(
target_dict, start_list, step_list, repeat_list,
recycle_list, target_step_list, reset,
obj_name, prop_name)
else:
return "Unsupported property type: " + prop_type + \
". Please fix the template_processing_functions." + \
"apply range modifier."
return ""
def get_range_modifier_json_schema():
info = {}
info["objectName"] = "Name of the object type that contains " + \
"the property to be modified."
info["propertyName"] = "Name of the property to be modified."
info["modifierType"] = "Type of modifier to apply. Currently, " + \
"only RANGE is supported."
info["propertyValueDict"] = "Properties specific to the type " + \
"of modifier being applied."
info["start"] = "Scalar start value or list of start values. " + \
"When a list is used, the number of values in the list " + \
"indicate how long the generated list will be."
info["step"] = "Scalar step value or list of step values. " + \
"When a list is specified, these are applied to elements " + \
"in the same index in the start list. If there are not " + \
"enough values, the values are repeated."
info["repeat"] = "Scalar repeat value or list of repeat values. " + \
"When a list is specified, these are applied to elements " + \
"in the same index in the start list. If there are not " + \
"enough values, the values are repeated."
info["recycle"] = "Scalar repeat value or list of repeat values. " + \
"When a list is specified, these are applied to elements " + \
"in the same index in the start list. If there are not " + \
"enough values, the values are repeated."
info["targetObjectStep"] = "Scalar target object step value or " + \
"list of target object step values. This step will be applied " + \
"when a new target (usually parent) object of the objects " + \
"being modified is encountered. Usually used with " + \
"resetOnNewTargetObject. When a list is specified, these " + \
"are applied to elements in the same index in the " + \
"start list. If there are not enough values, the values " + \
"are repeated."
info["resetOnNewTargetObject"] = "Reset the start value each time " + \
"a new target (usually parent) object of the objects being " + \
"modified is encountered."
schema_str = """
{
"title": "Schema for range-type ModifierInfo of the StmPropertyModifier.",
"type": "object",
"properties": {
"objectName": {
"type": "string"
},
"propertyName": {
"type": "string"
},
"modifierType": {
"enum": [
"RANGE"
]
},
"propertyValueDict": {
"type": "object",
"properties": {
"start": {
"oneOf": [
{
"type": "string"
},
{
"type": "array",
"items": {
"type": "string"
}
}
]
},
"step": {
"oneOf": [
{
"type": "string"
},
{
"type": "array",
"items": {
"type": "string"
}
}
]
},
"repeat": {
"oneOf": [
{
"type": "integer"
},
{
"type": "array",
"items": {
"type": "integer"
}
}
]
},
"recycle": {
"oneOf": [
{
"type": "integer"
},
{
"type": "array",
"items": {
"type": "integer"
}
}
]
},
"targetObjectStep": {
"oneOf": [
{
"type": "string"
},
{
"type": "array",
"items": {
"type": "string"
}
}
]
},
"resetOnNewTargetObject": {
"type": "boolean"
}
},
"required": [
"start",
"step"
]
}
},
"required": [
"objectName",
"propertyName",
"modifierType",
"propertyValueDict"
]
}
"""
err_str, j_schema = json_utils.load_json(schema_str)
prop_dict = j_schema["properties"]
for key in ["propertyName", "objectName", "modifierType",
"propertyValueDict"]:
prop_dict[key]["description"] = info[key]
pv_prop_dict = prop_dict["propertyValueDict"]["properties"]
for key in ["start", "step", "repeat", "recycle",
"targetObjectStep", "resetOnNewTargetObject"]:
pv_prop_dict[key]["description"] = info[key]
return json.dumps(j_schema)
def test_export_msc():
loop.run_until_complete(export_msc(conf))
data = load_json(WEB_ROOT + '/json/msc.json')
assert data['qsoCount']
#!/usr/bin/python3
#coding=utf-8
import logging
from os import path
import pytest
from ext_logger import ExtLogger, ExtLoggerException
from json_utils import load_json
LOGINS = load_json(
path.dirname(path.abspath(__file__)) + '/loggers_logins.json')
def test_lotw_login():
lotw = ExtLogger('LoTW')
logging.warning('test LOTW login')
login_ssn = lotw.login(LOGINS['LoTW'])
assert login_ssn
logging.warning('test LOTW bad login')
lotw = ExtLogger('LoTW')
bad_login = {}
bad_login.update(LOGINS['LoTW'])
bad_login['password'] += '_'
try:
bad_login_ssn = lotw.login(bad_login)
except ExtLoggerException as e:
assert str(e) == 'Login failed.'
def test_hamlog_login():
lotw = ExtLogger('HamLOG')
PARSER = argparse.ArgumentParser(
description="lenfer device software index builder")
PARSER.add_argument('path')
ARGS = PARSER.parse_args()
if not ARGS.path:
sys.exit('Path is required.')
if not os.path.isdir(ARGS.path):
sys.exit('Path does not exist')
INDEX_PATH = os.path.join(ARGS.path, 'index.json')
SOFTWARE_ROOT_PATH = os.path.join(ARGS.path, 'software')
INDEX = {}
INDEX_PREV = load_json(INDEX_PATH)
if not INDEX_PREV:
INDEX_PREV = {}
DEVICES_TYPES = {}
for root, dirs, files in os.walk(SOFTWARE_ROOT_PATH):
for file_name in files:
file_path = os.path.join(root, file_name)
file_web_path = os.path.relpath(file_path, SOFTWARE_ROOT_PATH)
print(file_web_path)
file_hash = None
with open(file_path, 'rb') as file_handle:
file_hash = hashlib.md5(file_handle.read()).hexdigest()
INDEX[file_web_path] = INDEX_PREV[
file_web_path] if file_web_path in INDEX_PREV else {}
INDEX[file_web_path]['hash'] = file_hash
'emojis': {},
'emojis_reactions': {},
}
if not master_count.get(channel).get(hit_type).get(hit):
master_count[channel][hit_type][hit] = 0
master_count[channel][hit_type][hit] += 1
def filter_emojis(text):
return Set(text.replace(':', '')).issubset(allowed_chars) and len(text) > 2
for idx, dump in enumerate(os.listdir('./output/channels')):
try:
messages = load_json('./output/channels/{}/messages/{}.json'.format(dump, dump))
print('ANALYZING {}'.format(dump))
except Exception as e:
print('ERROR GETTING MESSSAGES IN {}'.format(dump))
print(e)
continue
for message in messages:
emojis = []
emojis_reactions = []
if message.get('text'):
text_emojis = re.findall('\:.*?\:', message['text'])
text_emojis = list(filter(lambda x: filter_emojis(x), text_emojis))
if len(text_emojis):
text_emojis = list(map(lambda x: x.replace(':', ''), text_emojis))
def test_chat():
chat_path = WEB_ROOT + '/json/chat.json'
active_users_path = WEB_ROOT + '/json/active_users.json'
logging.debug('chat -- post from logged user')
data = {\
'token': create_token({'callsign': 'R7CL'}),\
'message': 'blah 0',\
'name': 'Name 0'}
rsp = requests.post(API_URI + '/chat',\
data=json.dumps(data))
logging.debug(rsp.text)
assert rsp.status_code == 200
chat = load_json(chat_path)
assert chat
assert chat[0]
assert chat[0]['callsign'] == 'R7CL'
assert chat[0]['admin']
assert chat[0]['text'] == data['message']
assert chat[0]['name'] == data['name']
assert chat[0]['ts']
assert chat[0]['date']
assert chat[0]['time']
au = load_json(active_users_path)
assert au
assert au['R7CL']
assert int(au['R7CL']['ts']) - int(chat[0]['ts']) < 2
logging.debug('chat -- post from not logged user')
data = {\
'callsign': 'B1AH',\
'message': 'blah 0',\
'name': 'Name 1'}
rsp = requests.post(API_URI + '/chat',\
data=json.dumps(data))
logging.debug(rsp.text)
assert rsp.status_code == 200
chat = load_json(chat_path)
assert chat
assert chat[0]
assert chat[0]['callsign'] == data['callsign']
assert not chat[0]['admin']
assert chat[0]['name'] == data['name']
assert chat[0]['text'] == data['message']
assert chat[0]['ts']
assert chat[0]['date']
assert chat[0]['time']
au = load_json(active_users_path)
assert au
assert au[data['callsign']]
assert int(au[data['callsign']]['ts']) - int(chat[0]['ts']) < 2
del_ts = chat[0]['ts']
logging.debug('chat -- admin deletes post')
data = {\
'token': create_token({'callsign': 'R7CL'}),\
'delete': del_ts,}
rsp = requests.post(API_URI + '/chat',\
data=json.dumps(data))
logging.debug(rsp.text)
assert rsp.status_code == 200
chat = load_json(chat_path)
assert chat
assert chat[0]['ts'] != del_ts
logging.debug('chat -- user leaves chat')
data = {\
'callsign': 'B1AH',\
'exit': True}
rsp = requests.post(API_URI + '/chat',\
data=json.dumps(data))
logging.debug(rsp.text)
assert rsp.status_code == 200
au = load_json(active_users_path)
assert au
assert data['callsign'] not in au
logging.debug('chat -- user status update')
data = {\
'callsign': 'B1AH',\
'typing': True}
rsp = requests.post(API_URI + '/chat',\
data=json.dumps(data))
logging.debug(rsp.text)
assert rsp.status_code == 200
au = load_json(active_users_path)
assert au
assert au[data['callsign']]['typing']
from validator import validate, bad_request
from db import DBConn, splice_params
from conf import CONF, APP_NAME, APP_ROOT, start_logging
from secret import get_secret, create_token
import send_email
from hash import data_hash
from json_utils import load_json
APP = Flask(APP_NAME)
APP.config.update(CONF['flask'])
APP.secret_key = get_secret(CONF['files']['secret'])
APP.json_encoder = JSONEncoder
APP.json_decoder = JSONDecoder
HASHIDS = Hashids(salt=APP.secret_key.decode('utf-8'), min_length=6)
TIMEZONES = load_json(APP_ROOT + '/timezones.json')
with APP.app_context():
start_logging('srv', CONF['logs']['srv_level'])
logging.debug('starting in debug mode')
DB = DBConn(CONF.items('db'))
DB.connect()
DB.verbose = False
APP.db = DB
def tz_shift(tzone):
return TIMEZONES[tzone] if tzone in TIMEZONES else ''