def firm__get_parameters_from_file(self, args):
import os
from src.bank import Bank
from src.household import Household
from src.firm import Firm
from src.environment import Environment # needed for the bankDirectory
text = "This test checks firm.get_parameters_from_file \n"
self.print_info(text)
#
# INITIALIZATION
#
environment_directory = str(args[0])
identifier = str(args[1])
log_directory = str(args[2])
# Configure logging parameters so we get output while the program runs
logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %H:%M:%S',
filename=log_directory + identifier + ".log", level=logging.INFO)
logging.info('START logging for test firm__get_parameters_from_file in run: %s',
environment_directory + identifier + ".xml")
# Construct firm filename
environment = Environment(environment_directory, identifier)
# get the firm_directory from the environment
firm_directory = environment.firm_directory
# and loop over all firms in the directory
listing = os.listdir(firm_directory)
firmFilename = firm_directory + listing[0]
# generate a household
household = Household()
household.identifier = "test_household"
environment.households.append(household)
# generate a bank
bank = Bank()
bank.identifier = "test_bank"
environment.banks.append(bank)
# generate a firm
firm = Firm()
environment.firms.append(firm)
helper = Helper()
helper.initialize_standard_firm(firm, environment)
firm.get_parameters_from_file(firmFilename, environment)
#
# TESTING
#
# test whether the parameters are read properly
text = "Identifier has been read as follows: \n"
text = text + "Identifier: "
text = text + firm.identifier
text = text + "\n"
text = text + "Productivity: "
text = text + str(firm.parameters["productivity"])
self.print_info(text)
class Main:
def __init__(self):
current_dir_path = path.dirname(__file__)
config_path = path.join(current_dir_path, '../config/production.json')
self.__config = read_json(config_path)
self.loop = asyncio.get_event_loop()
asyncio.set_event_loop(self.loop)
self.log = Logger.get_logger_instance()
self.__helper = Helper()
self.__taskHandler = Handler(self.loop)
probe.readiness = True
probe.liveness = True
def _start_service(self):
self.log.info(f'Service is listening')
try:
self.__helper.create_folder_if_not_exists(
self.__config["fs"]["internal_outputs_path"])
keys = ("fs", "external_physical_path")
self.__helper.valid_configuration(keys)
self.loop.run_until_complete(self.__taskHandler.handle_tasks())
except Exception as e:
self.log.error(f'Error occurred during running service: {e}')
probe.liveness = False
finally:
self.loop.close()
def __init__(self, client):
self.client = client
self.h = Helper()
self.b = BucketHandler()
self.coins_folder_name = 'coins'
self.SUCCESS_CHANNEL_ID = self.h.get_success_channel_id()
class Success(commands.Cog):
def __init__(self, client):
self.client = client
self.h = Helper()
self.b = BucketHandler()
self.coins_folder_name = 'coins'
self.SUCCESS_CHANNEL_ID = self.h.get_success_channel_id()
@commands.Cog.listener()
async def on_message(self, message: discord.Message):
if message.channel.id != int(self.SUCCESS_CHANNEL_ID):
# not in the success channel
return
if not message.attachments:
# no images
return
mention = self.h.get_twitter_mention()
if '@' not in mention:
mention = f'@{mention}'
current_coin_count = self.b.add_coins(filename=f'{self.coins_folder_name}/{message.author.id}.json', coin_count=self.h.get_coins_for_picture())
embed = self.h.initialize_embed(f"Coins added!")
if current_coin_count == 1:
embed.description = f"You now have **{current_coin_count} coin**!\n\nMake sure to post on Twitter and tag us (`{mention}`)!"
else:
embed.description = f"You now have **{current_coin_count} coins**!\n\nMake sure to post on Twitter and tag us (`{mention}`)!"
await self.client.get_channel(id=self.SUCCESS_CHANNEL_ID
).send(embed=embed)
def get_exchange_rate(cls) -> dict:
data = Helper.get_bank_store(cls.name, cls.interval)
if len(data) == 0:
data = cls._get_bank_api()
print('save into store')
Helper.set_bank_store(cls.name, cls.interval, data)
return data
def __init__(self, client):
self.client = client
self.coins_folder_name = 'coins'
self.BucketHandler = BucketHandler()
self.h = Helper()
def firm__set_identifier(self, args):
import os
from src.bank import Bank
from src.household import Household
from src.firm import Firm
from src.environment import Environment # needed for the bankDirectory
text = "This test checks firm.set_identifier \n"
self.print_info(text)
#
# INITIALIZATION
#
environment_directory = str(args[0])
identifier = str(args[1])
log_directory = str(args[2])
# Configure logging parameters so we get output while the program runs
logging.basicConfig(format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %H:%M:%S',
filename=log_directory + identifier + ".log",
level=logging.INFO)
logging.info('START logging for test firm__set_identifier in run: %s',
environment_directory + identifier + ".xml")
# Construct firm filename
environment = Environment(environment_directory, identifier)
# get the firm_directory from the environment
firm_directory = environment.firm_directory
# and loop over all firms in the directory
listing = os.listdir(firm_directory)
firmFilename = firm_directory + listing[0]
# generate a household
household = Household()
household.identifier = "test_household"
environment.households.append(household)
# generate a bank
bank = Bank()
bank.identifier = "test_bank"
environment.banks.append(bank)
# generate a firm
firm = Firm()
environment.firms.append(firm)
helper = Helper()
helper.initialize_standard_firm(firm, environment)
#
# TESTING
#
text = "Original identifier: "
text = text + firm.get_identifier()
print(text)
firm.set_identifier("new_ident")
text = "New identifier: "
text = text + firm.get_identifier()
print(text)
def fillPdfSet(self):
Helper.print("Detect files:")
root = self.input_path
for dir_, _, files in os.walk(root):
for file_name in files:
self.pdf_set.add(file_name)
Helper.print(file_name)
def helper__translog(self, args):
import os
from src.bank import Bank
from src.household import Household
from src.firm import Firm
from src.environment import Environment
from src.transaction import Transaction
from src.helper import Helper
text = "This test checks helper.translog \n"
self.print_info(text)
#
# INITIALIZATION
#
environment_directory = str(args[0])
identifier = str(args[1])
log_directory = str(args[2])
# Configure logging parameters so we get output while the program runs
logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %H:%M:%S',
filename=log_directory + identifier + ".log", level=logging.INFO)
logging.info('START logging for test helper__translog in run: %s',
environment_directory + identifier + ".xml")
# Construct household filename
environment = Environment(environment_directory, identifier)
#
# TESTING
#
helper = Helper()
production = helper.translog(3, 2, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5)
print("Calculating production in translog:")
print(production)
def bank__set_parameters(self, args):
import os
from src.bank import Bank
from src.household import Household
from src.firm import Firm
from src.environment import Environment # needed for the bankDirectory
text = "This test checks bank.set_parameters \n"
self.print_info(text)
#
# INITIALIZATION
#
environment_directory = str(args[0])
identifier = str(args[1])
log_directory = str(args[2])
# Configure logging parameters so we get output while the program runs
logging.basicConfig(format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %H:%M:%S',
filename=log_directory + identifier + ".log",
level=logging.INFO)
logging.info('START logging for test bank__set_parameters in run: %s',
environment_directory + identifier + ".xml")
# Construct bank filename
environment = Environment(environment_directory, identifier)
# get the bank_directory from the environment
bank_directory = environment.bank_directory
# and loop over all banks in the directory
listing = os.listdir(bank_directory)
bank_filename = bank_directory + listing[0]
# generate a household
household = Household()
household.identifier = "test_household"
environment.households.append(household)
# generate a firm
firm = Firm()
firm.identifier = "test_firm"
environment.firms.append(firm)
# generate the bank
bank = Bank()
environment.banks.append(bank)
helper = Helper()
helper.initialize_standard_bank(bank, environment)
#
# TESTING
#
text = "Original parameters:"
print(text)
print(bank.get_parameters())
text = "New parameters:"
print(text)
bank.set_parameters({'rd': 0.44, 'rl': 0.55, 'active': 1})
print(bank.get_parameters())
def create_and_train_model():
helper = Helper()
(x_train, y_train), (x_test, y_test) = helper.get_cifar10_prepared()
model = create_model()
helper.fit(
model, x_train, y_train, batch_size=1024, epochs=100, validation_data=(x_test, y_test), process_name="linear"
)
def firm__set_state_variables(self, args):
import os
from src.bank import Bank
from src.household import Household
from src.firm import Firm
from src.environment import Environment # needed for the bankDirectory
text = "This test checks firm.set_state_variables \n"
self.print_info(text)
#
# INITIALIZATION
#
environment_directory = str(args[0])
identifier = str(args[1])
log_directory = str(args[2])
# Configure logging parameters so we get output while the program runs
logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %H:%M:%S',
filename=log_directory + identifier + ".log", level=logging.INFO)
logging.info('START logging for test firm__set_state_variables in run: %s',
environment_directory + identifier + ".xml")
# Construct firm filename
environment = Environment(environment_directory, identifier)
# get the firm_directory from the environment
firm_directory = environment.firm_directory
# and loop over all firms in the directory
listing = os.listdir(firm_directory)
firmFilename = firm_directory + listing[0]
# generate a household
household = Household()
household.identifier = "test_household"
environment.households.append(household)
# generate a bank
bank = Bank()
bank.identifier = "test_bank"
environment.banks.append(bank)
#
# TESTING
#
# generate a firm
firm = Firm()
environment.firms.append(firm)
helper = Helper()
helper.initialize_standard_firm(firm, environment)
text = "Original state variables:"
print(text)
print(firm.get_state_variables())
text = "New state variables:"
print(text)
firm.set_state_variables({'test': 0.66})
print(firm.get_state_variables())
def bank__set_identifier(self, args):
import os
from src.bank import Bank
from src.household import Household
from src.firm import Firm
from src.environment import Environment # needed for the bankDirectory
text = "This test checks bank.set_identifier \n"
self.print_info(text)
#
# INITIALIZATION
#
environment_directory = str(args[0])
identifier = str(args[1])
log_directory = str(args[2])
# Configure logging parameters so we get output while the program runs
logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %H:%M:%S',
filename=log_directory + identifier + ".log", level=logging.INFO)
logging.info('START logging for test bank__set_identifier in run: %s',
environment_directory + identifier + ".xml")
# Construct bank filename
environment = Environment(environment_directory, identifier)
# get the bank_directory from the environment
bank_directory = environment.bank_directory
# and loop over all banks in the directory
listing = os.listdir(bank_directory)
bank_filename = bank_directory + listing[0]
# generate a household
household = Household()
household.identifier = "test_household"
environment.households.append(household)
# generate a firm
firm = Firm()
firm.identifier = "test_firm"
environment.firms.append(firm)
# generate the bank
bank = Bank()
environment.banks.append(bank)
helper = Helper()
helper.initialize_standard_bank(bank, environment)
#
# TESTING
#
text = "Original identifier: "
text = text + bank.get_identifier()
print(text)
bank.set_identifier("new_ident")
text = "New identifier: "
text = text + bank.get_identifier()
print(text)
def __init__(self, dim=1):
self.helper = Helper()
if 1 <= dim <= 3:
(self.train_generator, self.valid_generator,
self.test_generator) = self.helper.get_plant_pathology_prepared()
else:
raise Exception(
"Plant Pathology dataset couldn't be initialized with dims != 3 or != 1"
)
def convertPdfs(self):
Helper.print("Start conversion of images...")
for pdf in self.pdf_set:
pdf_path = os.path.join(self.input_path, pdf)
rel_dir = os.path.splitext(pdf)[0]
output_path = os.path.join(self.pdf_images_path, rel_dir)
FileHelper.createSubdirectoriesIfNecessary(self.pdf_images_path,
rel_dir)
self.pdf2image(pdf_path, output_path)
Helper.print("Conversion of images done...")
def stacks(self):
"""
Deletes CloudFormation Stacks.
"""
self.logging.debug("Started cleanup of CloudFormation Stacks.")
is_cleaning_enabled = Helper.get_setting(
self.settings, "services.cloudformation.stack.clean", False
)
resource_maximum_age = Helper.get_setting(
self.settings, "services.cloudformation.stack.ttl", 7
)
resource_whitelist = Helper.get_whitelist(
self.whitelist, "cloudformation.stack"
)
semaphore = threading.Semaphore(value=1)
if is_cleaning_enabled:
try:
paginator = self.client_cloudformation.get_paginator("describe_stacks")
resources = paginator.paginate().build_full_result().get("Stacks")
except:
self.logging.error("Could not list all CloudFormation Stacks.")
self.logging.error(sys.exc_info()[1])
return False
# threads list
threads = []
for resource in resources:
threads.append(
threading.Thread(
target=self.delete_stack,
args=(
semaphore,
resource,
resource_whitelist,
resource_maximum_age,
),
)
)
# start all threads
for thread in threads:
thread.start()
# make sure that all threads have finished
for thread in threads:
thread.join()
self.logging.debug("Finished cleanup of CloudFormation Stacks.")
else:
self.logging.info("Skipping cleanup of CloudFormation Stacks.")
return True
def household__getattr(self, args):
import os
from src.bank import Bank
from src.household import Household
from src.firm import Firm
from src.environment import Environment # needed for the bankDirectory
text = "This test checks household.getattr \n"
self.print_info(text)
#
# INITIALIZATION
#
environment_directory = str(args[0])
identifier = str(args[1])
log_directory = str(args[2])
# Configure logging parameters so we get output while the program runs
logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %H:%M:%S',
filename=log_directory + identifier + ".log", level=logging.INFO)
logging.info('START logging for test household__getattr in run: %s',
environment_directory + identifier + ".xml")
# Construct household filename
environment = Environment(environment_directory, identifier)
# get the firm_directory from the environment
household_directory = environment.household_directory
# and loop over all firms in the directory
listing = os.listdir(household_directory)
householdFilename = household_directory + listing[0]
# generate a bank
bank = Bank()
bank.identifier = "test_bank"
environment.banks.append(bank)
# generate a firm
firm = Firm()
firm.identifier = "test_firm"
environment.firms.append(firm)
# generate a household
household = Household()
environment.households.append(household)
helper = Helper()
helper.initialize_standard_household(household, environment)
#
# TESTING
#
print('Accessing rates through household.parameters["propensity_to_save"] :')
print(household.parameters["propensity_to_save"])
print("Accessing rates through household.propensity_to_save:")
print(household.propensity_to_save)
def __init__(self):
current_dir_path = path.dirname(__file__)
config_path = path.join(current_dir_path, '../config/production.json')
self.__config = read_json(config_path)
self.loop = asyncio.get_event_loop()
asyncio.set_event_loop(self.loop)
self.log = Logger.get_logger_instance()
self.__helper = Helper()
self.__taskHandler = Handler(self.loop)
probe.readiness = True
probe.liveness = True
def buckets(self):
"""
Deletes Buckets. All Bucket Objects, Versions and Deleted Markers
are first deleted before the Bucket can be deleted.
"""
self.logging.debug("Started cleanup of S3 Buckets.")
is_cleaning_enabled = Helper.get_setting(
self.settings, "services.s3.bucket.clean", False
)
resource_maximum_age = Helper.get_setting(
self.settings, "services.s3.bucket.ttl", 7
)
resource_whitelist = Helper.get_whitelist(self.whitelist, "s3.bucket")
semaphore = threading.Semaphore(value=5)
if is_cleaning_enabled:
try:
resources = self.client_s3.list_buckets().get("Buckets")
except:
self.logging.error("Could not list all S3 Buckets.")
self.logging.error(sys.exc_info()[1])
return False
# threads list
threads = []
for resource in resources:
threads.append(
threading.Thread(
target=self.delete_bucket,
args=(
semaphore,
resource,
resource_whitelist,
resource_maximum_age,
),
)
)
# start all threads
for thread in threads:
thread.start()
# make sure that all threads have finished
for thread in threads:
thread.join()
self.logging.debug("Finished cleanup of S3 Buckets.")
return True
else:
self.logging.info("Skipping cleanup of S3 Buckets.")
return True
def helper__initialize_standard_bank(self, args):
import os
from src.bank import Bank
from src.household import Household
from src.firm import Firm
from src.environment import Environment
from src.transaction import Transaction
from src.helper import Helper
text = "This test checks helper.initialize_standard_bank \n"
self.print_info(text)
#
# INITIALIZATION
#
environment_directory = str(args[0])
identifier = str(args[1])
log_directory = str(args[2])
# Configure logging parameters so we get output while the program runs
logging.basicConfig(format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %H:%M:%S',
filename=log_directory + identifier + ".log",
level=logging.INFO)
logging.info(
'START logging for test helper__initialize_standard_bank in run: %s',
environment_directory + identifier + ".xml")
# Construct household filename
environment = Environment(environment_directory, identifier)
# generate a bank
bank = Bank()
# bank.identifier = "test_bank"
environment.banks.append(bank)
# generate a firm
firm = Firm()
firm.identifier = "test_firm"
environment.firms.append(firm)
# generate a household
household = Household()
household.identifier = "test_household"
environment.households.append(household)
#
# TESTING
#
helper = Helper()
helper.initialize_standard_bank(bank, environment)
print("Initialized standard bank")
print(bank)
class Handler:
def __init__(self, asyncio_event_loop):
self.log = Logger.get_logger_instance()
self.__helper = Helper()
self.loop = asyncio_event_loop
current_dir_path = path.dirname(__file__)
config_path = path.join(current_dir_path, '../config/production.json')
self.__config = read_json(config_path)
self.queue_handler = TaskHandler(
self.__config["queue"]["job_type"],
self.__config["queue"]["task_type"],
self.__config["queue"]["job_manager_url"],
self.__config["queue"]["heartbeat_manager_url"],
self.__config["queue"]["heartbeat_interval_ms"], self.log)
self.__exportImage = ExportImage(self.queue_handler, self.loop)
async def handle_tasks(self):
try:
while True:
task = await self.queue_handler.dequeue(
self.__config["queue"]["dequeue_interval_ms"])
if task:
await self.execute_task(task)
except Exception as e:
self.log.error(f'Error occurred consuming: {e}.')
async def execute_task(self, task_values):
try:
# get the job's and task's id/status/attempts from the consumed task
job_id = task_values['jobId']
task_id = task_values['id']
status = task_values['status']
attempts = task_values['attempts']
self.log.info(f'Task Id "{task_id}" received.')
# get and validate the export params from the consumed task
parameters = task_values['parameters']
self.__helper.json_fields_validate(parameters)
bbox = parameters['bbox']
filename = parameters['fileName']
url = parameters['url']
directory_name = parameters['directoryName']
max_zoom = parameters['maxZoom']
# send the consumed task to work
return await self.__exportImage.export(bbox, filename, url,
task_id, job_id,
directory_name, max_zoom,
status, attempts)
except Exception as e:
self.log.error(f'Error occurred while exporting: {e}.')
def _get_bank_api(cls) -> dict:
response = Helper.http_request(cls.api_url)
data = {}
bank = {}
unit_bank_fs = {}
for bank_unit in response['organizations']:
for currency in bank_unit['currencies']:
if currency in CURRENCIES:
exchange_rate = ExchangeRate(
f"<{cls}> {bank_unit['title']}", currency,
bank_unit['currencies'][currency]['bid'],
bank_unit['currencies'][currency]['ask'],
response['date'])
Helper.db().insert(exchange_rate)
bank[currency] = {
'buy': exchange_rate.buy,
'sale': exchange_rate.sale,
'date_expired': exchange_rate.date_expired
}
data[str(bank_unit['title'])] = bank
unit_bank_f = UnitBankFinance(
bank_unit['id'],
response['orgTypes'][str(bank_unit['orgType'])],
bank_unit['oldId'], bank_unit['title'],
response['regions'][bank_unit['regionId']],
response['cities'][bank_unit['cityId']], bank_unit['phone'],
bank_unit['address'], response['date'])
if Helper.db().find_by(UnitBankFinance.tb_name(), unit_bank_f.hash,
'hash') is None:
Helper.db().insert(unit_bank_f)
unit_bank_fs[unit_bank_f.__dict__['hash']] = unit_bank_f.__dict__
Helper.save_bank_units('banks_and_kantors', unit_bank_fs)
return data
def __init__(self, asyncio_event_loop):
self.log = Logger.get_logger_instance()
self.__helper = Helper()
self.loop = asyncio_event_loop
current_dir_path = path.dirname(__file__)
config_path = path.join(current_dir_path, '../config/production.json')
self.__config = read_json(config_path)
self.queue_handler = TaskHandler(
self.__config["queue"]["job_type"],
self.__config["queue"]["task_type"],
self.__config["queue"]["job_manager_url"],
self.__config["queue"]["heartbeat_manager_url"],
self.__config["queue"]["heartbeat_interval_ms"], self.log)
self.__exportImage = ExportImage(self.queue_handler, self.loop)
def helper__initialize_standard_bank(self, args):
import os
from src.bank import Bank
from src.household import Household
from src.firm import Firm
from src.environment import Environment
from src.transaction import Transaction
from src.helper import Helper
text = "This test checks helper.initialize_standard_bank \n"
self.print_info(text)
#
# INITIALIZATION
#
environment_directory = str(args[0])
identifier = str(args[1])
log_directory = str(args[2])
# Configure logging parameters so we get output while the program runs
logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %H:%M:%S',
filename=log_directory + identifier + ".log", level=logging.INFO)
logging.info('START logging for test helper__initialize_standard_bank in run: %s',
environment_directory + identifier + ".xml")
# Construct household filename
environment = Environment(environment_directory, identifier)
# generate a bank
bank = Bank()
# bank.identifier = "test_bank"
environment.banks.append(bank)
# generate a firm
firm = Firm()
firm.identifier = "test_firm"
environment.firms.append(firm)
# generate a household
household = Household()
household.identifier = "test_household"
environment.households.append(household)
#
# TESTING
#
helper = Helper()
helper.initialize_standard_bank(bank, environment)
print("Initialized standard bank")
print(bank)
def __init__(self, dim=1):
self.helper = Helper()
if dim == 1:
(self.x_train,
self.y_train), (self.x_test,
self.y_test) = self.helper.get_cifar10_prepared()
elif dim == 3:
(self.x_train,
self.y_train), (self.x_test,
self.y_test) = self.helper.get_cifar10_prepared(
dim=3)
else:
raise Exception(
"Cifar 10 couldn't be initialized with dims != 3 or != 1")
def launch_linear_model():
from src.models.processes.linear import create_model_pp
helper = Helper()
pp = PlantPathology()
# pp.plot_image(5)
x_train, y_train = pp.train_generator.next()
x_test, y_test = pp.valid_generator.next()
model = create_model_pp()
helper.fit(model,
x_train,
y_train,
batch_size=1024,
epochs=100,
validation_data=(x_test, y_test),
process_name="linear")
def get_allowlist(self):
allowlist = defaultdict(lambda: defaultdict(set))
try:
paginator = boto3.client("dynamodb").get_paginator("scan")
items = (
paginator.paginate(TableName=os.environ.get("ALLOWLIST_TABLE"))
.build_full_result()
.get("Items")
)
except:
self.logging.error(
f"""Could not read DynamoDB table '{os.environ.get("ALLOWLIST_TABLE")}'."""
)
self.logging.error(sys.exc_info()[1])
else:
for item in items:
item_json = dynamodb_json.loads(item, True)
parsed_resource_id = Helper.parse_resource_id(
item_json.get("resource_id")
)
allowlist[parsed_resource_id["service"]][
parsed_resource_id["resource_type"]
].add(parsed_resource_id["resource"])
return allowlist
class TestHelper(object):
ASSUME_PATH = os.path.expanduser("~/.assume")
helper = Helper()
def remove_file(self, file_):
os.remove(f"{os.path.expanduser('~/.assume')}/{file_}")
def test_read_file(self):
content = {"this": "is", "a": "test"}
self.helper.write_file("test", content)
assert isinstance(self.helper.read_file("test"), dict)
assert self.helper.read_file("test") == content
self.remove_file("test")
def test_write_file(self):
content = {"this": "is", "a": "test"}
self.helper.write_file("test", content)
written_content = self.helper.read_file("test")
assert written_content == content
self.remove_file("test")
def test_get_profiles_zero_prof(self):
profiles = self.helper.get_profiles()
assert len(profiles) == 0
def test_get_profiles_one_prof(self):
content = {"this": "is", "a": "test"}
self.helper.write_file("myprof.prof", content)
profiles = self.helper.get_profiles()
assert len(profiles) == 1
self.remove_file("myprof.prof")
class PlantPathology:
"""
Ease CIFAR-10 creation with indicated shape
"""
def __init__(self, dim=1):
self.helper = Helper()
if 1 <= dim <= 3:
(self.train_generator, self.valid_generator,
self.test_generator) = self.helper.get_plant_pathology_prepared()
else:
raise Exception(
"Plant Pathology dataset couldn't be initialized with dims != 3 or != 1"
)
def plot_image(self, nb_of_images):
sample_training, _ = next(self.train_generator)
images = sample_training[:nb_of_images]
fig, axes = plt.subplots(1, 5, figsize=(20, 20))
axes = axes.flatten()
for img, ax in zip(images, axes):
print(img.shape)
ax.imshow(img)
ax.axis('off')
plt.tight_layout()
plt.show()
def step_train(self):
return self.train_generator.n // self.train_generator.batch_size
def step_valid(self):
return self.valid_generator.n // self.valid_generator.batch_size
def step_test(self):
return self.test_generator.n // self.test_generator.batch_size
def _get_bank_api(cls) -> dict:
data = {
currency['ccy']: {
'buy': currency['buy'],
'sale': currency['sale']
}
for currency in Helper.http_request(cls.api_url)
if currency['ccy'] in CURRENCIES
}
print('_get_bank_api', data)
for currency in data:
exchange_rate = ExchangeRate(str(cls), currency,
data[currency]['buy'],
data[currency]['sale'])
Helper.db().insert(exchange_rate)
return data
def _get_bank_api(cls) -> dict:
data = {}
for code in CURRENCIES:
response = Helper.http_request(cls.api_url.format(code))[0]
data[code] = {
'buy': response['rate'],
'sale': None,
'date_expired': response['exchangedate']
}
print('_get_bank_api', data)
for currency in data:
exchange_rate = ExchangeRate(str(cls), currency,
data[currency]['buy'],
data[currency]['sale'],
data[currency]['date_expired'])
Helper.db().insert(exchange_rate)
return data
def __init__(self, client: discord.Client):
self.h = Helper()
self.b = BucketHandler()
self.SUCCESS_CHANNEL_ID = self.h.get_success_channel_id()
self.VERIFIED_ROLE_ID = self.h.get_verified_role_id()
# Set up Twitter API
with open("configuration/twitter.json") as f:
j = json.load(f)
auth = tweepy.OAuthHandler(str(j["twitter_consumer_key"]),
str(j["twitter_consumer_secret"]))
auth.set_access_token(str(j["twitter_access_token"]),
str(j["twitter_access_token_secret"]))
self.twitter_api = tweepy.API(auth)
print('[VERIFY] Logged into Twitter.')
self.client = client
def __init__(self, logging, allowlist, settings, execution_log, region):
self.logging = logging
self.allowlist = allowlist
self.settings = settings
self.execution_log = execution_log
self.region = region
self._client_elasticsearch = None
self.is_dry_run = Helper.get_setting(self.settings, "general.dry_run", True)
def __init__(self, logging, whitelist, settings, execution_log, region):
self.logging = logging
self.whitelist = whitelist
self.settings = settings
self.execution_log = execution_log
self.region = region
self._client_glue = None
self.is_dry_run = Helper.get_setting(self.settings, "general.dry_run", True)
def __init__(self, logging, allowlist, settings, execution_log):
self.logging = logging
self.allowlist = allowlist
self.settings = settings
self.execution_log = execution_log
self.region = "global"
self._client_iam = None
self.is_dry_run = Helper.get_setting(self.settings, "general.dry_run",
True)
def firm__purge_accounts(self, args):
import os
from src.bank import Bank
from src.household import Household
from src.firm import Firm
from src.environment import Environment # needed for the bankDirectory
text = "This test checks firm.purge_accounts \n"
text = text + " Checking if after the purge_accounts the total amount \n"
text = text + " of transactions in the firm stays the same. \n"
self.print_info(text)
#
# INITIALIZATION
#
environment_directory = str(args[0])
identifier = str(args[1])
log_directory = str(args[2])
# Configure logging parameters so we get output while the program runs
logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %H:%M:%S',
filename=log_directory + identifier + ".log", level=logging.INFO)
logging.info('START logging for test firm__purge_accounts in run: %s',
environment_directory + identifier + ".xml")
# Construct firm filename
environment = Environment(environment_directory, identifier)
# get the firm_directory from the environment
firm_directory = environment.firm_directory
# and loop over all firms in the directory
listing = os.listdir(firm_directory)
firmFilename = firm_directory + listing[0]
# generate a household
household = Household()
household.identifier = "test_household"
environment.households.append(household)
# generate a bank
bank = Bank()
bank.identifier = "test_bank"
environment.banks.append(bank)
# generate a firm
firm = Firm()
environment.firms.append(firm)
helper = Helper()
helper.initialize_standard_firm(firm, environment)
#
# TESTING
#
account = 0.0
tranx = 0
for transaction in firm.accounts:
account = account + transaction.amount
tranx = tranx + 1
print(tranx)
print(account)
firm.add_transaction("deposits", "", environment.households[0:1][0],
firm.identifier, 0.0, 0.09, 0, -1, environment)
# environment.households[0:1][0] is only for testing purposes DO NOT USE IN PRODUCTION
# what it does is is takes the first household in environment, but if there are no
# households (which happens in testing) it doesn't break down
account = 0.0
tranx = 0
for transaction in firm.accounts:
account = account + transaction.amount
tranx = tranx + 1
print(tranx)
print(account)
firm.accounts[0].purge_accounts(environment)
account = 0.0
tranx = 0
for transaction in firm.accounts:
account = account + transaction.amount
tranx = tranx + 1
print(tranx)
print(account)
def consume_rationed(self, environment, time):
# We want the consumption to be done in random pairs
# We use rationing from market clearing class to do that
# Price is static for this example, otherwise we can't use rationing
# and need some other market clearing
price = 10.0
environment.variable_parameters["price_of_goods"] = price
# We need a list of agents and their demand or supply
# Supply is denoted with positive float, demand with negative float
for_rationing = []
# Firms give us their supply, we assume that since the goods are
# perishable their supply is all they have in stock
from src.helper import Helper
helper = Helper()
for firm in environment.firms:
# amount = round(helper.leontief([firm.get_account("labour")], [1/firm.productivity]), 0)
amount = helper.cobb_douglas(firm.get_account("labour"), firm.get_account("capital"),
firm.total_factor_productivity, firm.labour_elasticity, firm.capital_elasticity)*price
for_rationing.append([firm, amount])
# Households give use their demand, we assume that they want to
# consume the part of their wealth (cash and deposits) that they
# do not want to save (determined through propensity to save)
# We denote demand in units of the goods, so we divide the cash
# households want to spend by price to get the demand
for household in environment.households:
demand = 0.0
# demand = -round(((household.get_account("deposits") * (1 - household.propensity_to_save)) / price), 0)
demand = -((household.get_account("deposits") * (1 - household.propensity_to_save)) / price)
# demand = -household.get_account("deposits")/price
for_rationing.append([household, demand])
# We import the market clearing class
from market import Market
# Put the appropriate settings, i.e.
# tolerance of error, resolution of search
# and amplification for exponential search
# This does not matter for rationing
# But in principle we need to initialize
# with these values
market = Market("market")
# And we find the rationing, ie the amounts
# of goods sold between pairs of agents
# TESTING THE ABSTRACT RATIONING
# The matching function means that all pairs will have the same priority
def matching_agents_basic(agent_one, agent_two):
return 1.0
# The below function means that all pairs are allowed
def allow_match_basic(agent_one, agent_two):
return True
# We find the actual trades
rationed = market.rationing_abstract(for_rationing, matching_agents_basic, allow_match_basic)
# Then we go through the rationing
# and move the goods and cash appropriately
for ration in rationed:
#
# A (from) L (to)
# bank loan deposit
# household goods loan
# firm deposit goods
#
environment.new_transaction("goods", "", ration[1].identifier, ration[0].identifier,
ration[2], 0, 0, -1)
random_bank = random.choice(environment.banks)
environment.new_transaction("deposits", "", ration[0].identifier, random_bank.identifier,
ration[2]*price, random_bank.interest_rate_deposits, 0, -1)
environment.new_transaction("loans", "", random_bank.identifier, ration[1].identifier,
ration[2]*price, random_bank.interest_rate_loans, 0, -1)
# We print the action of selling to the screen
print("%s sold %d units of goods at a price %f to %s at time %d.") % (ration[0].identifier,
ration[2], price, ration[1].identifier, time)
logging.info(" goods consumed on step: %s", time)
def bank__check_consistency(self, args):
import os
from src.bank import Bank
from src.household import Household
from src.firm import Firm
from src.environment import Environment # needed for the bankDirectory
text = "This test checks bank.check_consitency \n"
self.print_info(text)
#
# INITIALIZATION
#
environment_directory = str(args[0])
identifier = str(args[1])
log_directory = str(args[2])
# Configure logging parameters so we get output while the program runs
logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %H:%M:%S',
filename=log_directory + identifier + ".log", level=logging.INFO)
logging.info('START logging for test bank__check_consistency in run: %s',
environment_directory + identifier + ".xml")
# Construct bank filename
environment = Environment(environment_directory, identifier)
# get the bank_directory from the environment
bank_directory = environment.bank_directory
# and loop over all banks in the directory
listing = os.listdir(bank_directory)
bank_filename = bank_directory + listing[0]
# generate a household
household = Household()
household.identifier = "test_household"
environment.households.append(household)
# generate a firm
firm = Firm()
firm.identifier = "test_firm"
environment.firms.append(firm)
# generate the bank
bank = Bank()
environment.banks.append(bank)
helper = Helper()
helper.initialize_standard_bank(bank, environment)
#
# TESTING
#
print("Checking consistency of the standard bank: ")
print(bank.check_consistency())
print("Adding additional deposits without adding appropriate cash/loans.")
bank.add_transaction("deposits", "", environment.get_agent_by_id("test_household"),
bank, 150, bank.interest_rate_deposits, 0, -1, environment)
# environment.households[0:1][0] is only for testing purposes DO NOT USE IN PRODUCTION
# what it does is is takes the first household in environment, but if there are no
# households (which happens in testing) it doesn't break down
print("Checking consistency of the standard bank: ")
print(bank.check_consistency())
def consume_rationed(self, environment, time):
# We want the consumption to be done in random pairs
# We use rationing from market clearing class to do that
# Price is static for this example, otherwise we can't use rationing
# and need some other market clearing
price = 10.0
environment.variable_parameters["price_of_goods"] = price
# We need a list of agents and their demand or supply
# Supply is denoted with positive float, demand with negative float
for_rationing = []
# Firms give us their supply, we assume that since the goods are
# perishable their supply is all they have in stock
from src.helper import Helper
helper = Helper()
for firm in environment.firms:
# Firms produce based on their capital, for generality
# we use their net capital, as in their capital stock
# minus the capital owned of other agents
capital = 0.0
for tranx in firm.accounts:
# This is own capital stock
if tranx.type_ == "capital" and tranx.from_ == firm:
capital = capital + tranx.amount
# And here is the ownership of other agents' stock
if tranx.type_ == "capital" and tranx.to == firm:
capital = capital - tranx.amount
# We find the amount produced through the Cobb-Douglas function
amount = (
helper.cobb_douglas(
firm.get_account("labour"),
capital,
firm.total_factor_productivity,
firm.labour_elasticity,
firm.capital_elasticity,
)
* price
)
# And assume firm wants to sell whole production given the perishable nature of the goods
for_rationing.append([firm, amount])
# Households give use their demand, we assume that they want to
# consume the part of their wealth (cash and deposits) that they
# do not want to save (determined through propensity to save)
# We denote demand in units of the goods, so we divide the cash
# households want to spend by price to get the demand
for household in environment.households:
demand = 0.0
wealth = 0.0
# For generality we calculate net wealth for this, that is the
# amount of deposits they carry minus the amount of loans
for tranx in household.accounts:
if tranx.type_ == "deposits" and tranx.from_ == household:
wealth = wealth + tranx.amount
if tranx.type_ == "loans" and tranx.to == household:
wealth = wealth - tranx.amount
# Then the demand is determined by the agent's propensity to save
# and the wealth calculated above
demand = -((wealth * (1 - household.propensity_to_save)) / price)
for_rationing.append([household, demand])
# We import the market clearing class
from market import Market
# Put the appropriate settings, i.e.
# tolerance of error, resolution of search
# and amplification for exponential search
# This does not matter for rationing
# But in principle we need to initialize
# with these values
market = Market("market")
# And we find the rationing, ie the amounts
# of goods sold between pairs of agents
# We find the actual trades
rationed = market.rationing_proportional(for_rationing)
# Then we go through the rationing
# and move the goods and cash appropriately
for ration in rationed:
#
# A (from) L (to)
# bank loan deposit
# household goods loan
# firm deposit goods
#
# TODO: in the new version this may be irrelevant
environment.new_transaction("goods", "", ration[1].identifier, ration[0].identifier, ration[2], 0, 0, -1)
# The below makes sure the allocations of loans are correct
# That is the banks don't allow overdraft for buying
# consumption goods by the households
to_finance = ration[2] * price
itrange = list(range(0, len(environment.banks)))
# And randomise this list for the purposes of iterating randomly
random.shuffle(itrange)
# And we iterate over the agents randomly by proxy of iterating
# through their places on the list [agents]
for i in itrange:
current_bank = self.environment.banks[i]
# We find how much in deposits the household has
deposits_available = 0.0
for tranx in ration[1].accounts:
if tranx.type_ == "deposits" and tranx.to == current_bank:
deposits_available = deposits_available + tranx.amount
# This should be irrelevant, but for completeness:
if tranx.type_ == "loans" and tranx.from_ == current_bank:
deposits_available = deposits_available - tranx.amount
# We find the amount of deposits the household can spend for this particular bank
current_amount = min(to_finance, deposits_available)
# And add the appropriate transactions
environment.new_transaction(
"deposits",
"",
ration[0].identifier,
current_bank.identifier,
current_amount,
current_bank.interest_rate_deposits,
0,
-1,
)
environment.new_transaction(
"loans",
"",
current_bank.identifier,
ration[1].identifier,
current_amount,
current_bank.interest_rate_loans,
0,
-1,
)
to_finance = to_finance - current_amount
# We print the action of selling to the screen
print("%s sold %d units of goods at a price %f to %s at time %d.") % (
ration[0].identifier,
ration[2],
price,
ration[1].identifier,
time,
)
logging.info(" goods consumed on step: %s", time)
def bank__clear_accounts(self, args):
import os
from src.bank import Bank
from src.household import Household
from src.firm import Firm
from src.environment import Environment # needed for the bankDirectory
text = "This test checks bank.clear_accounts \n"
text = text + " Checking if after the clear_accounts the total amount \n"
text = text + " of transactions in zero. \n"
self.print_info(text)
#
# INITIALIZATION
#
environment_directory = str(args[0])
identifier = str(args[1])
log_directory = str(args[2])
# Configure logging parameters so we get output while the program runs
logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %H:%M:%S',
filename=log_directory + identifier + ".log", level=logging.INFO)
logging.info('START logging for test bank__clear_accounts in run: %s',
environment_directory + identifier + ".xml")
# Construct bank filename
environment = Environment(environment_directory, identifier)
# get the bank_directory from the environment
bank_directory = environment.bank_directory
# and loop over all banks in the directory
listing = os.listdir(bank_directory)
bank_filename = bank_directory + listing[0]
# generate a household
household = Household()
household.identifier = "test_household"
environment.households.append(household)
# generate a firm
firm = Firm()
firm.identifier = "test_firm"
environment.firms.append(firm)
# generate the bank
bank = Bank()
environment.banks.append(bank)
helper = Helper()
helper.initialize_standard_bank(bank, environment)
#
# TESTING
#
account = 0.0
tranx = 0
for transaction in bank.accounts:
account = account + transaction.amount
tranx = tranx + 1
print(tranx)
print(account)
bank.add_transaction("deposits", "", "test_household",
bank.identifier, 0.0, 0.09, 0, -1, environment)
account = 0.0
tranx = 0
for transaction in bank.accounts:
account = account + transaction.amount
tranx = tranx + 1
print(tranx)
print(account)
for bank in environment.banks:
print(bank)
for firm in environment.firms:
print(firm)
for household in environment.households:
print(household)
bank.clear_accounts()
account = 0.0
tranx = 0
for transaction in bank.accounts:
account = account + transaction.amount
tranx = tranx + 1
print(tranx)
print(account)
def firm__add_transaction(self, args):
import os
from src.bank import Bank
from src.household import Household
from src.firm import Firm
from src.environment import Environment # needed for the bankDirectory
text = "This test checks firm.add_transaction \n"
text = text + " The most simple way to test this function is to assign an new \n"
text = text + " transaction to our firm. Therefore, lets just assign the following \n"
text = text + " transaction and check whether it has been added: \n"
text = text + ' (type = "deposits", fromID = -1, toID = firm.identifier, amount = 10, \n'
text = text + " interest = 0.09, maturity = 0, timeOfDefault = -1) \n"
self.print_info(text)
#
# INITIALIZATION
#
environment_directory = str(args[0])
identifier = str(args[1])
log_directory = str(args[2])
# Configure logging parameters so we get output while the program runs
logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %H:%M:%S',
filename=log_directory + identifier + ".log", level=logging.INFO)
logging.info('START logging for test firm__add_transaction in run: %s',
environment_directory + identifier + ".xml")
# Construct firm filename
environment = Environment(environment_directory, identifier)
# get the firm_directory from the environment
firm_directory = environment.firm_directory
# and loop over all firms in the directory
listing = os.listdir(firm_directory)
firmFilename = firm_directory + listing[0]
# generate a household
household = Household()
household.identifier = "test_household"
environment.households.append(household)
# generate a bank
bank = Bank()
bank.identifier = "test_bank"
environment.banks.append(bank)
# generate a firm
firm = Firm()
environment.firms.append(firm)
helper = Helper()
helper.initialize_standard_firm(firm, environment)
#
# TESTING
#
print(firm)
print("Adding new transaction: \n")
firm.add_transaction("deposits", "", environment.households[0:1][0],
firm.identifier, 10, 0.09, 0, -1, environment)
# environment.households[0:1][0] is only for testing purposes DO NOT USE IN PRODUCTION
# what it does is is takes the first household in environment, but if there are no
# households (which happens in testing) it doesn't break down
print(firm)
def firm__get_account_num_transactions(self, args):
import os
from src.bank import Bank
from src.household import Household
from src.firm import Firm
from src.environment import Environment # needed for the bankDirectory
text = "This test checks firm.get_account_num_transactions \n"
text = text + " The purpose of this method is to count the numbers of transaction for \n"
text = text + " accounts firms hold. Our standard frm has 3 transactions by default. \n"
self.print_info(text)
#
# INITIALIZATION
#
environment_directory = str(args[0])
identifier = str(args[1])
log_directory = str(args[2])
# Configure logging parameters so we get output while the program runs
logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %H:%M:%S',
filename=log_directory + identifier + ".log", level=logging.INFO)
logging.info('START logging for test firm__get_account_num_transactions in run: %s',
environment_directory + identifier + ".xml")
# Construct firm filename
environment = Environment(environment_directory, identifier)
# get the firm_directory from the environment
firm_directory = environment.firm_directory
# and loop over all firms in the directory
listing = os.listdir(firm_directory)
firmFilename = firm_directory + listing[0]
# generate a household
household = Household()
household.identifier = "test_household"
environment.households.append(household)
# generate a bank
bank = Bank()
bank.identifier = "test_bank"
environment.banks.append(bank)
# generate a firm
firm = Firm()
environment.firms.append(firm)
helper = Helper()
helper.initialize_standard_firm(firm, environment)
#
# TESTING
#
num_transactions = 0.0 # counting all types in account together
print(firm)
# and checking if the number of transaction
# is increasing by one
for type in ["loans", "cash", "goods"]:
if type == "loans":
num_transactions = num_transactions + firm.get_account_num_transactions(type)
print("L = " + str(num_transactions))
if type == "cash":
num_transactions = num_transactions + firm.get_account_num_transactions(type)
print("L+M = " + str(num_transactions))
if type == "goods":
num_transactions = num_transactions + firm.get_account_num_transactions(type)
print("L+M+G = " + str(num_transactions))
def runner__init__(self, args):
import os
from src.bank import Bank
from src.household import Household
from src.firm import Firm
from src.environment import Environment # needed for the Directory
from src.runner import Runner
text = "This test checks runner.__init__ \n"
self.print_info(text)
#
# INITIALIZATION
#
environment_directory = str(args[0])
identifier = str(args[1])
log_directory = str(args[2])
# Configure logging parameters so we get output while the program runs
logging.basicConfig(
format="%(asctime)s %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
filename=log_directory + identifier + ".log",
level=logging.INFO,
)
logging.info("START logging for test runner__init__ in run: %s", environment_directory + identifier + ".xml")
# Construct bank filename
environment = Environment(environment_directory, identifier)
# get the bank_directory from the environment
bank_directory = environment.bank_directory
# and loop over all banks in the directory for testing purpose (just one bank)
listing = os.listdir(bank_directory)
bank_filename = bank_directory + listing[0]
# generate a household
household = Household()
household.identifier = "test_household"
environment.households.append(household)
# generate a firm
firm = Firm()
firm.identifier = "test_firm"
environment.firms.append(firm)
# generate the bank
bank = Bank()
environment.banks.append(bank)
helper = Helper()
helper.initialize_standard_bank(bank, environment)
# making an instance of the Runner class
runner = Runner(environment)
#
# TESTING
#
text = "Identifier: "
text = text + runner.get_identifier()
print(text)
text = "Number of sweeps:"
text = text + str(runner.num_sweeps)
print(text)
text = "Updater"
text = text + str(runner.updater)
print(text)
def firm__get_account(self, args):
import os
from src.bank import Bank
from src.household import Household
from src.firm import Firm
from src.environment import Environment # needed for the bankDirectory
text = "This test checks firm.get_account \n"
text = text + " The purpose of this method is to establish an account for our firm which contains \n"
text = text + " all kinds of assets and liabilities. The method simply adds all kinds of assets \n"
text = text + " and stores them in one volume. As our firms holds 250.0 assets \n"
text = text + " and 250 liabilites the total volume of our account should be 500.0 \n"
self.print_info(text)
#
# INITIALIZATION
#
environment_directory = str(args[0])
identifier = str(args[1])
log_directory = str(args[2])
# Configure logging parameters so we get output while the program runs
logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %H:%M:%S',
filename=log_directory + identifier + ".log", level=logging.INFO)
logging.info('START logging for test firm__get_account in run: %s',
environment_directory + identifier + ".xml")
# Construct firm filename
environment = Environment(environment_directory, identifier)
# get the firm_directory from the environment
firm_directory = environment.firm_directory
# and loop over all firms in the directory
listing = os.listdir(firm_directory)
firmFilename = firm_directory + listing[0]
# generate a household
household = Household()
household.identifier = "test_household"
environment.households.append(household)
# generate a bank
bank = Bank()
bank.identifier = "test_bank"
environment.banks.append(bank)
# generate a firm
firm = Firm()
environment.firms.append(firm)
helper = Helper()
helper.initialize_standard_firm(firm, environment)
#
# TESTING
#
account = 0.0 # counting all types in account together
print(firm) # and checking how much is the total
# volume of the account
for type in ["loans", "cash", "goods"]:
if type == "loans":
account = account + firm.get_account(type)
print("L = " + str(account))
if type == "cash":
account = account + firm.get_account(type)
print("L+M = " + str(account))
if type == "goods":
account = account + firm.get_account(type)
print("L+M+G = " + str(account))
