def test_config(client):
response = client.get('/api/config')
assert_that(response.status_code).is_equal_to(200)
assert_that(response.json).is_equal_to({
'github': {
'client_id': get_config()['github_app']['client_id'],
'redirect_uri': get_url(get_config()['github_app']['redirect_uri'])
}
})
def setUp(self):
kafka_config = config.get_config('kafka')
self.host = kafka_config['host']
self.port = kafka_config['port']
self.topic = kafka_config['topic']
self.schema = json.loads(kafka_config['schema'])
web_config = config.get_config('website')
self.url, self.expected_content = json.loads(web_config['urls'])[0]
def setUp(self):
kafka_config = config.get_config('kafka')
self.host = kafka_config['host']
self.port = kafka_config['port']
self.topic = kafka_config['topic']
self.schema = json.loads(kafka_config['schema'])
web_config = config.get_config('website')
self.urls = json.loads(web_config['urls'])
self.url, self.expected_content = self.urls[0]
self.waiting_time = 5 # seconds
self.conn_pool, _ = db.create_conn_pool(env='dev')
async def events(client: TelegramClient, status: object) -> None:
user = await client.get_me()
last_name = ""
if status['gaming']:
last_name = config.get_config().playing.get_value().format(
status['game'])
else:
last_name = config.get_config().non_playing.get_value()
if user.last_name != last_name:
logger.info('changing lastname: {}'.format(last_name))
await client(
UpdateProfileRequest(first_name=user.first_name,
last_name=last_name))
def cvalid(**kwargs):
inp_params = kwargs['params']
if 'which_run' not in inp_params.keys():
raise ValueError('You should provide the name of RUN')
if 'img_type' not in inp_params.keys():
raise ValueError('You should provide a valid image type to create batches')
if 'which_net' not in inp_params.keys():
raise ValueError('You should provide, which net to use')
conf = get_config(which_run=inp_params['which_run'], img_type=inp_params['img_type'])
tsoj = Test(conf,
params=dict(pprocessor_inp_img_shape=[224,224,3],
pprocessor_inp_crop_shape=[],
model_inp_img_shape=[224, 224, 3]),
device_type = 'gpu',
which_net=inp_params['which_net'])
tsoj.run(use_checkpoint_for_run=inp_params['which_run'],
use_checkpoint_for_imageType=inp_params['img_type'],
optional_batch_name='cvalid',
which_checkpoint='all',
which_data='cvalid',
dump_stats=True)
return 'COMPLETE'
def predictions(**kwargs):
inp_params = kwargs['params']
if 'which_run' not in inp_params.keys():
raise ValueError('You should provide the name of RUN')
if 'img_type' not in inp_params.keys():
raise ValueError('You should provide a valid image type to create batches')
if 'use_checkpoint_for_run' not in inp_params.keys():
raise ValueError('Provide the RUN name whose model you would wanna use')
if 'use_checkpoint_for_imageType' not in inp_params.keys():
raise ValueError('Provide the image_type name whose model you would wanna use')
if 'which_net' not in inp_params.keys():
raise ValueError('You should provide, which net to use')
if 'classification_threshold' not in inp_params.keys():
raise ValueError('Provide the Classification threshold for classification range: (0 to 1), best : 0.68')
if 'use_checkpoint_for_prediction' not in inp_params.keys():
raise ValueError('You should provide, whether you want prediction based on "all" checkpoints or "max"')
conf = get_config(which_run=inp_params['which_run'], img_type=inp_params['img_type'])
checkpoint_path = os.path.join(conf['pathDict']['parent_checkpoint_path'], inp_params['use_checkpoint_for_run'],
inp_params['use_checkpoint_for_imageType'], inp_params['which_net'])
get_predictions_using_multiple_checkpoints(conf, which_data='test_new', checkpoint_path=checkpoint_path,
use_checkpoint_for_prediction='all',
threshold=inp_params['classification_threshold'])
return 'COMPLETE'
def commands(args):
"""Return a list of all enabled commands for the channel."""
#usage = "usage: s!commands"
cmds = "[prefix: s!] - "
config = get_config()
custom_commands = None
try:
with open('custom.json') as custom_file:
custom_commands = json.load(custom_file)
except Exception:
queue = args[0]
queue.put(("{0}".format(sys.exc_info()[0]), 'BG_error'))
queue.put(("commands() - Could not load json!", 'BG_error'))
return None
for cmd in config['commands']:
if config['commands'][cmd]['enabled']:
cmds = cmds + cmd + ", "
if custom_commands != None:
for cmd in custom_commands:
cmds = cmds + cmd + ", "
return cmds[:-2]
def get_user_profile(user_id, org_id):
"""
Gets the user profile
:param user_id: user id
:param org_id: org id
:return: dict
"""
config_settings = config.get_config()
db_client = boto3.resource('dynamodb',
region_name=config_settings['region'])
table = db_client.Table(config_settings['table_name'])
try:
response = table.get_item(Key={
'user_id': str(user_id),
'org_id': str(org_id)
})
except ClientError as e:
logger.error(
"Failed to retrieve profile for user {} and organization {}:{}".
format(user_id, org_id, e.response['Error']['Message']))
else:
if "Item" in response:
return response["Item"]
return None
def login(self):
conf = get_config()
self.user = conf['MAIL_USER']
self.user_key = conf['MAIL_PASS']
self.server.ehlo()
self.server.starttls()
self.server.login(self.user, self.user_key)
def load_csv(sort):
#Switcher to convert text to index in array
switcher = {
'id': 0,
'file': 1,
'description': 2,
'date': 3,
'author': 4,
'type': 5,
'platform': 6,
'port': 7
}
#Get CSV data from exploitdb git repo
#To see the available data check the CSV url
config = get_config('CSV')
response = urlopen(config['csvurl'])
csvData = csv.reader(codecs.iterdecode(response, 'utf-8'))
#skip the headers
next(csvData)
#Data is sorted by date which is row 4
#This way the updater can go by the latest added date
#Get the index from the key name
csvData = sorted(csvData, key=lambda row: row[switcher.get(sort, 3)])
return list(csvData)
def opsgenie_alert():
config = get_config()
results = execute_sp('MWH.Ops_Gene_Alert_Check', {})
if len(results) < 1 or len(results[0]) < 1:
log('No Alert found.')
return
error = results[0][0]
if not error or 'error_message' not in error or error[
'error_message'] == '':
log('No Alert found.')
return
init_opsgenie({
'OPSGENIE_API_KEY': config['OPSGENIE_API_KEY'],
'OPSGENIE_GENIE_KEY': config['OPSGENIE_GENIE_KEY'],
'IS_PRODUCTION': config['IS_PRODUCTION']
})
alert = send_alert_for_error(error)
if not alert:
log('No OpsGenie alert sent.')
else:
log('OpsGenie alert sent:')
for key in error:
log(f'{key}: {error[key]}')
log('')
log('OpsGenie alert response:')
log(f'Request ID: {alert.request_id}')
log(f'Result: {alert.result}')
log(f'Took: {alert.took}')
def run(self):
while True:
if terminate:
config.update("running", False)
break
if not config.is_scheduled():
config.update("running", False)
break
editable = config.get_config()
theme = config.get("theme")
if should_be_light():
if theme == "light":
time.sleep(30)
continue
else:
switch_to_light()
else:
if theme == "dark":
time.sleep(30)
continue
else:
switch_to_dark()
time.sleep(30)
def data_check(in_filename):
filename = src.utils.get_configuration_file_path(in_filename)
configs = src.data_check_utils.get_excel_preference_file_data(filename)
sheets = []
data = []
for i, config in enumerate(configs):
try:
sheet_data = src.data_check_utils.compare_tables(config)
except Exception as e:
print(f"Error on row #{i + 1}:")
print(str(e))
continue
if len(sheet_data) > 0:
sheets.append(config['SOURCE_TABLE'])
data.append(sheet_data)
if len(data) < 1:
print("")
print('Nothing to do. aborting.')
exit(0)
filename, file_extension = os.path.splitext(in_filename)
out_filename = os.path.join(get_config()['ETL_CONFIG_OUT_DIR'], filename + '_DIFFS' + file_extension)
src.excel_utils.write_workbook_data(out_filename, sheets, data)
print("")
print('The following diffs file was created:')
print(out_filename)
print("")
def oauthRedirect():
github = get_config()['github_app']
access_token_response = requests.post(
f'https://github.com/login/oauth/access_token',
params={
'client_id': github['client_id'],
'client_secret': github['client_secret'],
'code': request.args.get('code'),
},
headers={
'Accept': 'application/json',
},
)
access_token = access_token_response.json()['access_token']
user_response = requests.get(
'https://api.github.com/user',
headers={
'Authorization': f'Bearer {access_token}',
'Accept': 'application/json',
},
)
email, github_login, name, avatar_url = itemgetter('email', 'github_login', 'name', 'avatar_url')(user_response.json())
user = get_model('User')(email=email, github_login=github_login, name=name, avatar_url=avatar_url)
db = get_database()
db.session.add(user)
db.session.commit()
return redirect(get_url(''))
def test_should_return_config_class_if_config_class(self, import_module_mock):
config_module_mock = self.mock.MagicMock()
testing_config_mock = self.mock.MagicMock()
config_module_mock.TestingConfig = testing_config_mock
import_module_mock.return_value = config_module_mock
testing_config = config.get_config()
self.assertEqual(testing_config, testing_config_mock)
def test_fetch_wrong_config_try_except():
# fetch invalid config
# this will fail due to KeyError
try:
invalid_config_val = get_config('my_secret', 'val_1')
except KeyError as e:
invalid_config_val = ''
assert invalid_config_val == ''
def config():
github_data = get_config()['github_app']
return {
'github': {
'client_id': github_data['client_id'],
'redirect_uri': get_url(github_data['redirect_uri']),
},
}
def get_table_definition(db_name,
schema_name,
table_name,
server_name,
data_partition_column_name='',
excluded_columns=()):
"""
Returns the table definition.
:param str db_name: Database name
:param str schema_name: Schema name
:param str table_name: Database table name
:param str server_name: Server name
:param str data_partition_column_name: Data partition column name
:param list excluded_columns: Columns to exclude
:return list
"""
server_name = '' if server_name == '127.0.0.1' or server_name == 'localhost' else server_name
server_name = f'[{server_name}].' if server_name else ''
sql = (
"SELECT T.name AS TABLE_NAME, C.name AS COLUMN_NAME, P.name AS DATA_TYPE, "
"P.max_length AS SIZE, CAST(P.precision AS VARCHAR) + '/' + CAST(P.scale AS VARCHAR) AS PRECISION_SCALE, "
"c.* FROM {0}[{1}].sys.objects AS T JOIN {0}[{1}].sys.columns AS C ON T.object_id = C.object_id "
"JOIN {0}[{1}].sys.types AS P ON C.system_type_id = P.system_type_id "
"JOIN sys.schemas ss ON (T.schema_id = ss.schema_id) "
" WHERE T.type_desc = 'USER_TABLE' and ss.name = ? "
"and T.name = ? and P.name != 'timestamp' and P.name != 'sysname' order by column_id asc"
).format(server_name, db_name)
columns = fetch_rows(sql, [schema_name, table_name])
target_table_column_prefix = get_config()['TARGET_TABLE_COLUMN_PREFIX']
out_columns = {}
for column in columns:
column['original_data_type'] = column['data_type']
if column['column_name'].upper() in default_columns:
column[
'target_table_column_name'] = target_table_column_prefix + column[
'column_name']
else:
column['target_table_column_name'] = column['column_name']
# Update the data type for the data partition column
if data_partition_column_name != '' and column['column_name'].upper(
) == data_partition_column_name.upper():
column['data_type'] = 'datetime'
out_columns[column['column_name'].upper()] = column
if len(excluded_columns) > 0:
for excluded_column in excluded_columns:
out_columns.pop(excluded_column)
return out_columns
def get_currency(queue):
"""Return the name of the currency if it's enabled."""
config = get_config()
try:
if not config['currency']['enabled']:
return None
return config['currency']['name']
except KeyError:
queue.put(("validate_currency() - Currency config is corrupted!", 'BG_error'))
return None
def test_populate_status_codes(self):
expected_rows = [(int(code), desc) for code, desc in sorted(
config.get_config('status_codes').items())]
db.execute_sql(db.create_tables_sql(), self.db_conn)
db.executemany_sql(*db.populate_status_codes_sql(), self.db_conn)
with self.db_conn.cursor() as curs:
curs.execute("SELECT * FROM public.status_codes")
rows = curs.fetchall()
self.assertEqual(rows, expected_rows)
async def main():
steam_client = steam.Steam(get_config())
bot_client = await bot.make_client()
await bot_client.connect()
while True:
status = await steam_client.pull_status()
await bot.events(bot_client, status)
time.sleep(10)
def train(**kwargs):
inp_params = kwargs['params']
print(inp_params)
if 'which_run' not in inp_params.keys():
raise ValueError('You should provide the name of RUN')
if 'img_type' not in inp_params.keys():
raise ValueError('You should provide a valid image type to create batches')
if 'use_checkpoint' not in inp_params.keys():
raise ValueError('You should provide whether to train using previous checkpoints')
if 'save_checkpoint' not in inp_params.keys():
raise ValueError('You should provide, whether to save checkpoint or not')
if 'write_tensorboard_summary' not in inp_params.keys():
raise ValueError('You should provide, whether to write tensorboard summary or not')
if 'which_net' not in inp_params.keys():
raise ValueError('You should provide, which net to use')
conf = get_config(which_run=inp_params['which_run'], img_type=inp_params['img_type'])
train_batches = [file
for file in os.listdir(conf['pathDict']['batch_path'])
if file.split('.')[1] == 'h5']
max_batches = len(train_batches) - 2
tr_obj = Train(conf,
dict(pprocessor_inp_img_shape=[224, 224, 3],
pprocessor_inp_crop_shape=[],
model_inp_img_shape=[224, 224, 3],
use_checkpoint=inp_params['use_checkpoint'],
save_checkpoint=inp_params['save_checkpoint'],
write_tensorboard_summary=inp_params['write_tensorboard_summary']
),
device_type='gpu',
which_net=inp_params['which_net'])
# # (tr_loss_arr, tr_acc_arr, tr_precision_arr, tr_recall_arr,
# # cv_loss_arr, cv_acc_arr, cv_precision_arr, cv_recall_arr,
# # l_rate_arr) = \
(tr_loss_arr, tr_acc_arr, tr_precision_arr, tr_recall_arr,
cv_loss_arr, cv_acc_arr, cv_precision_arr, cv_recall_arr,
l_rate_arr) = tr_obj.run(num_epochs=1, num_batches=max_batches, cv_num_batches=1, get_stats_at=2)
print(
'Mean Values: train_loss = %s, train_acc = %s, train_precision = %s, train_recall = %s, cv_loss = %s, '
'cv_acc = %s, cv_precision = %s, cv_recall = %s' % (
np.mean(tr_loss_arr), np.mean(tr_acc_arr), np.mean(tr_precision_arr), np.mean(tr_recall_arr),
np.mean(cv_loss_arr), np.mean(cv_acc_arr), np.mean(cv_precision_arr), np.mean(cv_recall_arr)))
return 'COMPLETE'
def get_base_sql_code(file_name):
"""
Returns the file contents in a string.
:param str file_name:
:return: str
"""
file_path = os.path.join(get_config()['CODE_GENERATION_IN_DIR'], ntpath.basename(file_name))
with open(file_path) as fp:
contents = fp.read()
return contents
def __init__(self, camera, perspective_transform_mat, detector_name):
self._camera = camera
self._perspective_transform_mat = perspective_transform_mat
self._perspective_transform_mat_inv = np.linalg.inv(perspective_transform_mat)
cfg = config.get_config(detector_name)
self._thresholder = thresholding.Thresholder(cfg["thresholds"])
self._img_shape = cfg["image-shape"]
self._sliding_window = cfg["sliding-window"]
self._polygon_window = cfg["polygon-window"]
self._left_lane = Lane('left', self._img_shape)
self._right_lane = Lane('right', self._img_shape)
def get_configuration_file_path(file_name):
"""
Returns the path for the specified configuration file.
:param str file_name:
:return: str
"""
file_path = os.path.join(get_config()['ETL_CONFIG_IN_DIR'], ntpath.basename(file_name))
if os.path.exists(file_path) is False:
raise FileExistsError(f"{file_name} is an invalid file.")
return file_path
def create_json_preference_file(file_name, contents):
"""
Creates a JSON preference file from a text file.
:param str file_name: File name
:param str contents: The file's contents
:return: str New file path
"""
file_path = os.path.join(get_config()['ETL_CONFIG_IN_DIR'], file_name)
with open(file_path, "w+") as fp:
fp.write(contents)
return file_path
def create_sql_file(file_name, sql_contents):
"""
Writes and creates a sql file.
:param str file_name: File name
:param str sql_contents: The file's SQL contents
:return: str New file path
"""
file_path = os.path.join(get_config()['ETL_CONFIG_OUT_DIR'], ntpath.basename(file_name))
with open(file_path, "w+") as fp:
fp.write(sql_contents)
return file_path
def main():
help_args = ['--help', '-h']
if len(sys.argv) >= 2 and sys.argv[1] not in help_args: # Check for help argument or if path args is greater then 2
conf = get_config()
#Join path argument from bash script with a space
path_arg = ' '.join(sys.argv[1:])
dirs = get_dirs(path_arg, conf)
start_display(dirs, conf)
else:
show_usage()
sys.exit(1)
return
def set_correct_time(self):
new_config = config.get_config()
d_hour = new_config["switchToDark"].split(":")[0]
d_minute = new_config["switchToDark"].split(":")[1]
l_hour = new_config["switchToLight"].split(":")[0]
l_minute = new_config["switchToLight"].split(":")[1]
# giving the time widget the values of the config
dark_time = QTime(int(d_hour), int(d_minute))
light_time = QTime(int(l_hour), int(l_minute))
self.ui.dark_time.setTime(dark_time)
self.ui.light_time.setTime(light_time)
def __init__(self):
config = get_config('DATABASE')
self.ip = config['mongo_ip']
self.user = config['mongo_user']
self.pw = config['mongo_pw']
self.database = config['mongo_database']
self.collection = config['mongo_collection']
#load database
client = MongoClient('mongodb://' + self.user + ':' + self.pw + '@' +
self.ip)
db = client[self.database]
self.db = db[self.collection]
sort_inds = np.argsort(extreme_errors)[::-1]
for i in xrange(30):
bad_ind = sort_inds[i]
bad_error = extreme_errors[bad_ind]
bad_img = draw_content(contents[bad_ind], config)
plt.figure(1)
plt.clf()
plt.imshow(bad_img)
plt.axis('off')
plt.title('%d-th worst, mean error %.2fmm' % (i, bad_error))
good_ind = sort_inds[-(i+1)]
good_error = extreme_errors[good_ind]
good_img = draw_content(contents[good_ind], config)
plt.figure(2)
plt.clf()
plt.imshow(good_img)
plt.axis('off')
plt.title('%d-th best, mean error %.2fmm' % (i, good_error))
import ipdb; ipdb.set_trace()
if __name__ == '__main__':
config = get_config()
if not config.load_path:
raise Exception('Must specify a model to use to predict!')
if 'model.ckpt' not in config.load_path:
raise Exception('Must specify a model checkpoint!')
main(config)
