def destroy_atlas_resources():
"""
Destroys Atlas resources using atlas_client REST call.
"""
# The following will work if run from a work directory, such as during an Evergreen task.
# It will cause ImportError when run from an Evergreen teardown hook.
# This means Atlas clusters must be shut down at the end of the task, they are not reused.
try:
import common.config
import common.atlas_setup as atlas_setup
except ImportError as error:
LOG.info(error)
LOG.info(
"Cannot import ConfigDict or AtlasSetup. Skipping Atlas teardown.")
LOG.info("(This is benign inside evergreen teardown hook.)")
return True
# AtlasSetup.destroy() will write to mongodb_setup.out.yml
config = common.config.ConfigDict('mongodb_setup')
config.load()
# start a mongodb configuration using config
atlas = atlas_setup.AtlasSetup(config)
atlas.destroy()
return True
def refresh(token):
cfg = config.load()
session = librusSession.create_with_token(token)
r = session.get(cfg['homework_url'])
if (r.status_code != 200):
return None
tasks = hp.parse(session, r.text)
return tasks
def main():
"""
Load an existing v20.conf file, update it interactively, and save it
back to a file.
"""
config = common.config.Config()
filename = common.input.get_string(
"Enter existing v20.conf filename to load",
common.config.default_config_path()
)
try:
config.load(filename)
except:
print("Config file '{}' doesn't exist, starting with defaults.".format(
filename
))
print
print("")
print("------------ Intitial v20 configuration ------------")
print(str(config))
print("----------------------------------------------------")
print("")
config.update_from_input()
print("")
print("-------------- New v20 configuration --------------")
print(str(config))
print("---------------------------------------------------")
print("")
dump = common.input.get_yn(
"Dump v20 configuration to {}?".format(filename),
True
)
if dump:
config.dump(filename)
def get(username, password):
cfg = config.load()
res = librusSession.create(username, password)
if res is None:
return None
session = res['session']
r = session.get(cfg['homework_url'])
if (r.status_code != 200):
return None
tasks = hp.parse(session, r.text)
tasks['token'] = res['token']
return tasks
from locust import HttpUser, task, tag
from common import config, data, errors, test_setup as setup
server_public_key = setup.loadServerPublicKey()
'''
If there is no server cert, the warnings are disabled because thousands will
appear in the logs and make it difficult to see anything else.
'''
if not server_public_key:
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
mbis = data.load_pa_mbis()
client_cert = setup.getClientCert()
setup.set_locust_env(config.load())
class BFDUser(HttpUser):
SERVICE_DATE = 'service-date=gt2020-01-05'
LAST_UPDATED = '_lastUpdated=gt2020-05-05'
def _mbi(self):
if len(mbis) == 0:
errors.no_data_stop_test(self)
return mbis.pop()
def _get(self, query, name):
self.client.get(query, cert=client_cert, verify=server_public_key, name=name)
@tag('claim')
from bs4 import BeautifulSoup
import common.config as config
import datetime
import time
cfg = config.load()
def parse(session, html):
soup = BeautifulSoup(html, 'lxml')
table = soup.find_all('table')[1]
tasks = {'zadania': {'na_dzisiaj': [], 'pozostale': []}}
today = datetime.datetime.today()
index = 1
for row in table.find_all('tr'):
columns = row.find_all('td')
if (len(columns) == 10):
button = columns[9].find_all('input')
onclick_attr = button[0]['onclick']
url = get_view_url(onclick_attr)
desc = get_task_desc(session, url)
task = {
'przedmiot': columns[0].get_text().strip(),
'tytul': columns[2].get_text().strip(),
'tresc': desc,
'data_zadania': columns[4].get_text().strip(),
'termin_oddania': columns[6].get_text().strip()
}
task_end = datetime.datetime.strptime(task['termin_oddania'],
'%Y-%m-%d')
def run_with_params(argv):
'''Runs a specified test via the input args.
'''
## Dictionary that holds the default values of each config value
default_config_data = {
'homePath': '',
'clientCertPath': '',
'databaseUri': '',
'testHost': '',
'configPath': 'config.yml',
'serverPublicKey': '',
'tableSamplePct': '0.25',
'testRunTime': "1m",
'testNumTotalClients': "100",
'testCreatedClientsPerSecond': "5",
'resetStatsAfterClientSpawn': False,
'storeStats': None
}
# Dictionary to hold data passed in via the CLI that will be stored in the root config.yml file
config_data = {}
test_file = ''
worker_threads = "1"
help_string = ('runtests.py \n--homePath="<path/to/home/directory>" (Required) '
'\n--clientCertPath="<path/to/client/pem/file>" (Required)'
'\n--databaseUri="postgres://<username:password>@<database-aws-node>.rds.amazonaws.com:port/'
'<dbname>" (Required)'
'\n--testHost="https://<nodeIp>:7443 or https://<environment>.bfd.cms.gov" (Required)'
'\n--test_file="/<v1/v2>/test_to_run.py" (Required)'
'\n--configPath="<path to a YAML configuration that will be read for CLI values but _not_ '
'written to>" (Optional, Default: "./config.yml")'
'\n--serverPublicKey="<server public key>" (Optional, Default: "")'
'\n--tableSamplePct=<% of table to sample> (Optional, Default: 0.25)'
'\n--testRunTime="<Test run time, ex. 30s, 1m, 2d 1h>" (Optional, Default 1m)'
'\n--maxClients="<Max number of clients to create at once, int>" (Optional, Default 100)'
'\n--clientsPerSecond="<Clients to create per second until maxClients is reached, int>" '
'(Optional, Default 5)'
'\n--worker_threads="<If >1 the test is run as distributed, and expects this many worker '
'processes to start, int>" (Optional, Default 1 - non distributed mode)'
'\n--storeStats="<If set, stores stats in JSON to S3 or local file. Must follow format: <STORAGE_TYPE>:<RUNNING_ENVIRONMENT>:<TAG>:<PATH_OR_BUCKET>" (Optional)'
'\n--resetStats (Optional)')
try:
opts, _args = getopt.getopt(argv, "h", ["homePath=", "clientCertPath=", "databaseUri=",
"testHost=", "serverPublicKey=", 'tableSamplePct=', "configPath=", "testRunTime=",
"maxClients=", "clientsPerSecond=", "testFile=", "workerThreads=", "storeStats=", "resetStats"])
except getopt.GetoptError as err:
print(err)
print(help_string)
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print(help_string)
sys.exit()
elif opt == "--homePath":
config_data["homePath"] = arg
elif opt == "--clientCertPath":
config_data["clientCertPath"] = arg
elif opt == "--databaseUri":
config_data["dbUri"] = arg
elif opt == "--testHost":
config_data["testHost"] = arg
elif opt == "--configPath":
config_data["configPath"] = arg
elif opt == "--serverPublicKey":
config_data["serverPublicKey"] = arg
elif opt == '--tableSamplePct':
config_data['tableSamplePct'] = arg
elif opt == "--testRunTime":
config_data["testRunTime"] = arg
elif opt == "--maxClients":
config_data["testNumTotalClients"] = arg
elif opt == "--clientsPerSecond":
config_data["testCreatedClientsPerSecond"] = arg
elif opt == "--testFile":
test_file = arg
elif opt == "--workerThreads":
worker_threads = arg
elif opt == "--storeStats":
try:
config_data["storeStats"] = StatsStorageConfig.from_arg_str(arg)
except ValueError as err:
print(f'--storeStats was invalid: {err}\n')
print(help_string)
sys.exit()
elif opt == "--resetStats":
config_data["resetStatsAfterClientSpawn"] = True
else:
print(help_string)
sys.exit()
## Read the specified configuration file
yaml_config = config.load_from_path(config_data.get("configPath",
default_config_data["configPath"])) or {}
## Merge the stored data with data passed in via the CLI, with the
## CLI data taking priority
config_data = {**yaml_config, **config_data}
## Finally, merge the merged configuration values with the defaults,
## in case any optional arguments were not set via the CLI or the specified
## YAML configuration file
config_data = {**default_config_data, **config_data}
## Add on extra time to the run-time to account for ramp-up of clients.
adjusted_time = adjusted_run_time(config_data["testRunTime"],
config_data["testNumTotalClients"], config_data["testCreatedClientsPerSecond"])
if adjusted_time is None:
print("Could not determine adjusted run time. Please use a format " +
"like \"1m 30s\" for the --testRunTime option")
sys.exit(1)
config_data["testRunTime"] = f"{adjusted_time}s"
print('Run time adjusted to account for ramp-up time. New run time: '
f'{timedelta(seconds=adjusted_time)}')
## Check if all required params are set
if not all([config_data["homePath"], config_data["clientCertPath"], config_data["dbUri"],
config_data["testHost"], test_file]):
print("Missing required arg (See -h for help on params)")
sys.exit(2)
## write out to repository root config file (_NOT_ the file specified by "configPath")
config.save(config_data)
setup.set_locust_test_name(test_file)
## strip off extra command line params for locust, or else it tries to parse them
sys.argv = sys.argv[:1]
if int(worker_threads) > 1:
## Spawn worker threads to connect to the main thread
for i in range(int(worker_threads)):
print(f"Creating worker #{i}")
process = Process(target=setup.run_worker_test, args=(i,worker_threads,))
process.start()
## Run the master test
setup.set_locust_env(config.load())
setup.run_master_test(worker_threads)
else:
# Reset the distributed values
setup.reset_distributed_values()
# call locust to run the master test
main()