def build_model(self):
input_placeholder = Input(shape=self.input_shape)
x = Conv2D(filters=64, kernel_size=9, strides=1,
padding='same')(input_placeholder)
x = PReLU(alpha_initializer='zeros',
alpha_regularizer=None,
alpha_constraint=None,
shared_axes=[1, 2])(x)
skip = x
for _ in range(16):
x = Utils.residual_block(x, 64, 3, 1)
x = Conv2D(filters=64, kernel_size=3, strides=1, padding='same')(x)
x = BatchNormalization(momentum=0.5)(x)
x = add([skip, x])
for _ in range(2):
x = Utils.upsampling_block(x, 256, 3, 1)
x = Conv2D(filters=3, kernel_size=9, strides=1, padding='same')(x)
output = Activation('tanh')(x)
self.generator = Model(input_placeholder, output)
def main():
path = os.path.dirname(os.path.realpath(__file__))
utils = Utils()
settings = utils.load_settings()
##
# add tools
##
admin = Admin(settings['admin'])
parser = argparse.ArgumentParser(prog='pool-cli')
subparsers = parser.add_subparsers()
##
# add subparser for tools
##
admin_parser = admin.parser(subparsers)
args = parser.parse_args()
##
# execute tools
##
if 'which' in args:
if args.which == 'admin':
admin.execute(args, admin_parser)
else:
parser.print_help()
else:
parser.print_help()
def get_files(self):
urls = self.get_urls()
with open("files.csv", "w+") as f:
for i, url in enumerate(urls):
Utils.print_progress_bar(i,
len(urls),
prefix="Retrieving File Names")
f.writelines(url[0] + "\n")
def _clamp_camera(self):
left = self.ws_crop_w // 2
right = self.world_w - int(math.ceil(self.ws_crop_w / 2.0))
self.ws_camera_x = Utils.clamp(self.ws_camera_x, left, right)
top = self.ws_crop_h // 2
bottom = self.world_h - int(math.ceil(self.ws_crop_h / 2.0))
self.ws_camera_y = Utils.clamp(self.ws_camera_y, top, bottom)
def __init__(self):
print("Initializing Sample Class")
self.O1_IND = 1
self.O2_IND = 0
self.NUM_OBJ = 2
self.O1_COST = 1
self.O2_COST = 17.6 * self.O1_COST
self.utils = Utils()
def calcError(self, i):
# calcola l'errore sul training set
self.feedforward()
self.errorShow[i] = ut.error(self.outputNN, self.output, self.type)
# calcola l'errore sul validation set
if (self.validationInput is not None):
self.feed(self.validationInput)
self.errorValidShow[i] = ut.error(self.outputNN,
self.validationOutput, self.type)
def create_directories(self, dir_set):
acc = ""
for dir in dir_set:
acc += dir + FILE_DEL
new_dir = self.OUTPUT_FILE \
+ FILE_DEL + (self.dp if self.dp else "") \
+ acc
Utils.create_dir(new_dir)
self.created_dirs[acc] = True
def generate_email(user_email, user_name, context):
sender = user_email
to = context['destination']
subject = Utils.rm_subject(context['subject'])
decoded_msg = Utils.decode_msg(context['emailmessage'])
message = Utils.rm_quotes(decoded_msg) + __signature(user_name)
info = 'To: ' + to + '\n\n' + message
mail = __create_mail(sender, to, subject, message)
return (mail, subject, info)
def __init__(self):
aws_host = Utils.read_properties().get('DATABASE','aws_host')
aws_database = Utils.read_properties().get('DATABASE','aws_database')
aws_user = Utils.read_properties().get('DATABASE','aws_user')
aws_password = Utils.read_properties().get('DATABASE', 'aws_password')
self.aws_db = mysql.connector.connect(host=aws_host, user=aws_user, passwd=aws_password, database=aws_database)
self.insert_data_statement = "INSERT INTO {} (symbol, company_name, trade_date, " \
"RSI_rsi, MACD_macd_signal, MACD_macd_histogram, MACD_macd," \
"BB_real_upper_band, BB_real_middle_band, BB_real_lower_band, OBV_obv, open_price, low_price, high_price, close_price, volume) " \
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
def __init__(self, index, query, language):
self.results = list()
self.query = dict()
util = Utils(language)
self.index = index
self.original_query = util.get_cleaned_tokens(query)
cleaned_query = Utils.stemmer(self.original_query, language)
self.add_query_words(cleaned_query)
self.normalize_frequencies()
def execute():
files = list(
filter(lambda x: not os.path.isdir(x),
glob(path, recursive=True)))
if files:
print("New Files Found")
for index, file in enumerate(files):
Utils.print_progress_bar(index + 1, len(files),
"Organizing Files")
fsplit = file.split(FILE_DEL)
old_name = fsplit[-1]
if self.org_type == "DIMENSION":
vid = cv2.VideoCapture(file)
height = vid.get(cv2.CAP_PROP_FRAME_HEIGHT)
width = vid.get(cv2.CAP_PROP_FRAME_WIDTH)
if height > width:
Utils.create_dir(self.dest_dir + "/vertical/")
Utils.create_dir(self.dest_dir +
"/vertical/conform/")
move(
file, self.dest_dir + "/vertical/conform/" +
old_name)
else:
Utils.create_dir(self.dest_dir + "/")
move(file, self.dest_dir + "/" + old_name)
def main():
win = pygame.display.set_mode((Constants.WIDTH, Constants.WIDTH))
pygame.display.set_caption("Snake Game")
snake = Snake(Constants.SNAKE_COLOR)
cookie = Cookie(snake)
flag = True
clock = pygame.time.Clock() # create an object to help track time
while flag:
# pause the program for an amount of time
pygame.time.delay(90)
# clock.tick(FPS) # game will be capped at FPS fps
snake.move()
Utils.check_crossing(snake, cookie)
Utils.redraw_window(win, snake, cookie)
def insert_layer(self, z, color=None):
if not color:
if z == 0:
color = self.get_layer_by_z(z).color
else:
if z == self.numMasks:
color = self.get_layer_by_z(z-1).color
else:
color = Utils.average_hex_colors(self.get_layer_by_z(z-1).color, self.get_layer_by_z(z).color)
w, h = self.imgSize
cv_mask_bgr = np.zeros((h, w, 3), dtype=np.uint8)
cv_mask = cv.cvtColor(cv_mask_bgr, cv.COLOR_BGR2GRAY)
uid = self._generate_layer_uid()
json_layer = {"name": "New Layer",
"color": color,
"visible": True,
"locked": False}
self.layers[str(uid)] = Layer(json_layer, cv_mask)
if z < self.numMasks:
self.layerKeys.insert(z, str(uid))
else:
self.layerKeys.append(str(uid))
self.numMasks = len(self.layerKeys)
if self.activeMask >= z:
self.set_active_layer(self.activeMask + 1)
def download(self, url, total):
""" execute the get request and write the file """
self.sema.acquire()
headers = {
"Authorization": "Bearer " + self.api_key,
"Dropbox-API-Arg": json.dumps({"path": url[0]})
}
resp = requests.post("https://content.dropboxapi.com/2/files/download",
headers=headers)
self.write_to_file(url[1], resp.content)
self.total_downloaded += 1
Utils.print_progress_bar(self.total_downloaded,
total,
prefix="Downloading Files")
self.sema.release()
def __init__(self, img_path, p_max, p_min, fake_class_prob_to_get):
self.img = Utils.prepare_img(img_path)
self.p_max = p_max
self.p_min = p_min
self.fake_class_prob_to_get = fake_class_prob_to_get
self.keras = Keras("src/imagenet_classes.csv", self.img)
def __init__(self, data):
print("Initializing FlexiBO class")
self.utils = Utils()
self.sample = Sample()
self.df = data
cfg = ConfigReal()
(self.E, self.O, self.measurement) = cfg.set_design_space()
self.NUM_ITER = 200
self.NUM_OBJ = 2
self.O1_IND = 0
self.O2_IND = 1
self.m1 = "inference_time"
self.m2 = "power_consumption"
self.SM = SurrogateModel()
(self.X, self.Y1, self.Y2) = self.prepare_training_data()
self.fit_rf()
self.perform_bo_loop()
def _run_default(self):
now = str(round(time.time() * 1000))
self.OUTPUT_FILE += "-" + now
csv_filename = self.OUTPUT_FILE + "-" + now + ".csv"
csvfile = open(csv_filename, "w+")
try:
Utils.rm_dir(self.OUTPUT_FILE)
Utils.create_dir(self.OUTPUT_FILE)
path = os.getcwd() + self.directory + FILE_DEL + "**"
files_to_change = list(
filter(lambda x: not os.path.isdir(x),
glob(path, recursive=True)))
csv_writer = csv.writer(csvfile,
delimiter=',',
quotechar='|',
quoting=csv.QUOTE_MINIMAL)
csv_writer.writerow([
"Original Name", "Archive Name", "Original Directory",
"Archive Directory"
])
if not files_to_change:
os.remove(csv_filename)
ArchiveUtility.rm_dir(self.OUTPUT_FILE)
self.clean(csvfile,
"No files to archive found in: " + self.directory)
for index, f_name in enumerate(files_to_change):
fsplit = f_name.split(FILE_DEL)
old_name = fsplit[-1]
old_dirs_set = fsplit[self.get_depth_to_base(fsplit[1:]):-1]
old_dirs = FILE_DEL.join(old_dirs_set)
new_name = self.rename_file(old_name)
new_location = self.OUTPUT_FILE \
+ FILE_DEL + (self.dp if self.dp else "") \
+ old_dirs + FILE_DEL + new_name
if old_dirs not in self.created_dirs:
self.create_directories(old_dirs_set)
copyfile(f_name, new_location)
old_path = FILE_DEL.join(fsplit[4:-1])
csv_writer.writerow(
[old_name, new_name, old_path, new_location])
self.asset_number += 1
Utils.print_progress_bar(index + 1, len(files_to_change))
self.clean(
csvfile, "Finished. Your archived files can be found in " +
self.OUTPUT_FILE + "-" + now + "/ and your csv in " +
csv_filename + " || 00111100 00110011")
except Exception as e:
os.remove(csv_filename)
Utils.rm_dir(self.OUTPUT_FILE)
self.clean(csvfile, "Error: " + str(e))
def list_events(events_json):
events_list = []
for event in events_json:
start_time = event['start'].get('dateTime', event['start'].get('date'))
end_time = event['end'].get('dateTime', event['start'].get('date'))
print(event)
event_json = {
'title': event['summary'],
'image_url': url_for('static', filename=f'assets/img/calendar/goo.png', _external=True),
'subtitle': 'Start: ' + Utils.parse_date(start_time) + '\nEnd: ' + Utils.parse_date(end_time),
"buttons": [
{
'type': 'web_url',
'url': event['htmlLink'],
'title': 'Go to the event'
}
]
}
events_list.append(event_json)
return events_list
def handle_file_organizer():
organization_types = ["Dimension"]
print("How would you like to organize your files?")
for i, o_type in enumerate(organization_types):
print(str(i + 1) + ":" + o_type)
num = Utils.get_number_input(len(organization_types), "Enter Number: ")
selected_type = organization_types[num - 1]
src = Utils.get_string_input("Enter the source directory for all of your files you want to organize: ")
dest = Utils.get_string_input("Enter the destination directory for all of the files you want to organize: ")
run_forever = Utils.get_string_input("Would you like to run this continuously? (Yes or no): ", ["YES", "NO"])
if run_forever.upper() == "YES":
run_forever = True
else:
run_forever = False
input("Hit enter when you are ready to run.")
FileOrganizer(src, dest, selected_type).organize(run_forever)
class Admin:
def __init__(self, config):
self.drops_dao = DropsDAO(config)
self.config = config
self.utils = Utils()
def add_role_to_role(self, email, role):
user_id = self.drops_dao.select_user_id_by_email(email)
if user_id is None:
return None
current_roles = self.drops_dao.select_user_role_by_id(user_id[0])
if current_roles is None:
return None
roles = str(current_roles[0]).split(",")
if role in roles:
print("user " + email + "actually have the role " + role + "\n")
return None
new_roles = str(current_roles[0]) + "," + role
update = self.drops_dao.update_user_role_by_id(user_id[0], new_roles)
def add_list_of_roles(self, jsonfile):
for entry in jsonfile['entities']:
self.add_role_to_role(entry['email'], 'employee')
self.add_role_to_role(entry['email'], entry['entity'])
def add_oauth_client(self, client_id, client_secret, redirect_uri):
result = self.drops_dao.add_oauth_client(client_id, client_secret,
redirect_uri)
print(result)
def parser(self, subparsers):
parser = subparsers.add_parser('admin', help='setup docker ')
parser.add_argument('-r',
'--role',
nargs=2,
metavar=('email, role'),
help="add role to user")
parser.add_argument('-l',
'--list',
nargs=1,
metavar=('list'),
help="add list of email role tuple")
parser.set_defaults(which="admin")
return parser
def execute(self, args, parser):
if args.role is not None:
output = self.add_role(args.role[0], args.role[1])
print(output)
elif args.list is not None:
json_file = self.utils.load_json_from_file(args.list[0])
self.add_list_of_roles(json_file)
else:
parser.print_help()
def deploy_lambda_fn(fn_name, run_time, handler, role_arn, src_folder):
folder_path = path.join(path.dirname(path.abspath(__file__)), src_folder)
zip_file = Utils.make_zip_filebytes(path=folder_path)
return lambda_client().create_function(FunctionName=fn_name,
Runtime=run_time,
Role=role_arn,
Handler=handler,
Code={'ZipFile': zip_file},
Timeout=LAMBDA_TIMEOUT,
MemorySize=LAMBDA_MEM,
Publish=False)
def main(self):
print("Portfolio Manager started")
parser = argparse.ArgumentParser()
parser.add_argument('-database',
action='store',
dest='database',
help='Either of Sqlite/AWS')
parser.add_argument('-mode',
action='store',
dest='mode',
help='Test Mode[test/real]')
parser.add_argument('-image',
action='store',
dest='image',
help='Generate Images[generate]')
parser.add_argument('-model',
action='store',
dest='model',
help='create machine learning model[create]')
parser.add_argument('-training_period',
action='store',
dest='training_period',
help='Training period in years')
self.args = parser.parse_args()
scraper = Scraper()
config = Utils.read_properties()
av = AlphaV.AlphaVantage()
ti = TechIndicators(key=os.environ["ALPHA_VANTAGE_KEY"],
output_format='pandas')
ts = TimeSeries(key=os.environ["ALPHA_VANTAGE_KEY"],
output_format='pandas')
if self.args.mode == 'test':
symbol_list = scraper.read_symbols()
else:
symbol_list = self.get_exchange_symbol_list(
Constants.SP500, scraper)
if self.args.database == 'sqllite':
self.dump_market_data(symbol_list, Constants.SP500, av, ti, ts,
config)
elif self.args.database == 'AWS':
for exchange in Constants.Exchanges:
symbol_list = self.get_exchange_symbol_list(exchange, scraper)
self.dump_market_data(symbol_list, exchange, av, ti, ts,
config)
if self.args.model == 'create':
recommender = Recommender(self.args, symbol_list, True)
else:
recommender = Recommender(self.args, symbol_list, False)
#recommender.generate_recommendation(Constants.LearningModel.IMAGE_BASED_CLASSIFICATION, symbol_list)
#recommender.generate_recommendation(Constants.LearningModel.DECISION_TREE_CLASSIFICATION, symbol_list)
recommender.generate_recommendation(
Constants.LearningModel.LSTM_CLASIFICATION, symbol_list)
def run():
modes = {"Archival": handle_archival, "File Organize": handle_file_organizer, "Download":
handle_dropbox_download, "Retrieve File Names": handle_dropbox_file_name_retrieval}
print("Welcome to the Media Utility Tool")
print("What would you like to do?")
for i, mode in enumerate(modes.keys()):
print(str(i + 1) + ":" + mode)
choice = Utils.get_number_input(len(modes), "Enter number: ")
print("You selected: " + str(choice))
mode = modes[list(modes.keys())[choice - 1]]
mode()
def handle_archival():
print("Would you like to use an existing CSV file in the archival of files?")
with_csv = Utils.get_string_input("Yes or no: ", ["YES", "NO"])
directory, csv_path, column_number, destination_number, force, asset_prefix, source_name, directory_prefix = [
None] * 8
if with_csv.upper() == "YES":
csv_path = Utils.get_string_input("Enter the path to your CSV (i.e 'yourdirectory/yourcsv.csv'): ")
column_number = Utils.get_string_input("Enter the column number for the file name's original name: ")
destination_number = Utils.get_string_input("Enter the column number for the file name's expected new name: ")
force = Utils.get_number_input(2,
"Would you like to change the existing files' names or copy them into a new zipped "
"directory? "
"\n1. Existing File Names\n2. Copy It\nEnter Number:")
if force == 1:
force = True
else:
force = False
elif with_csv.upper() == "NO":
asset_prefix = input("Enter the asset prefix to append to the each renamed file (press enter to have none): ")
source_name = input("Enter the source name (press enter to have none):")
directory_prefix = input("Enter the prefix for your altered directories (i.e __archive) (press enter to have "
"none): ")
directory = Utils.get_string_input(
"Enter the path to the directory containing all of the files you want to alter: ")
input("Hit enter when you are ready to run.")
ArchiveUtility(directory, asset_prefix, source_name, directory_prefix, csv_path, column_number,
destination_number, force).run()
def set_zoom(self, zoom):
old_zoom = self.zoom
self.zoom = Utils.clamp(zoom, self.min_zoom, self.max_zoom)
# manually derive world and camera, since old camera position should only be scaled during zoom
delta_zoom = 1.0 * self.zoom / old_zoom
self.world_w = self.mask_w * self.zoom
self.world_h = self.mask_h * self.zoom
self.ws_camera_x = int(math.floor(self.ws_camera_x * delta_zoom))
self.ws_camera_y = int(math.floor(self.ws_camera_y * delta_zoom))
self._clamp_camera()
self._derive_space_vars()
def save(self, output_dir):
Utils.save_chart(self._generate_chart(),
Utils.get_path("chart.png", basepath=output_dir))
summary = self._generate_summary()
summary_file_path = Utils.get_path("summary.txt", basepath=output_dir)
with open(summary_file_path, 'w') as outfile:
outfile.write(summary)
result_img_file_path = Utils.get_path("result_img.jpg",
basepath=output_dir)
Utils.save_img(img=self.best_img, img_path=result_img_file_path)
def backpropagation(self):
error = self.output - self.outputNN
deltaK = error * ut.sigmoidDerivative(self.outputNN)
error2 = np.dot(deltaK, self.weightsHL1.T)
deltaJ = error2 * ut.sigmoidDerivative(self.outputLayer1)
self.deltaW = self.alfa * self.deltaW + self.learningRate * self.input.T.dot(
deltaJ)
self.deltaWB = self.alfa * self.deltaWB + self.learningRate * np.sum(
deltaJ)
self.deltaW2 = self.alfa * self.deltaW2 + self.learningRate * self.outputLayer1.T.dot(
deltaK)
self.deltaWB2 = self.alfa * self.deltaWB2 + self.learningRate * np.sum(
deltaK)
self.weightsInput += self.deltaW - self.lamb * self.weightsInput
self.weightsHL1 += self.deltaW2 - self.lamb * self.weightsHL1
# non uso la regolarizzazione per i bias
self.biasInput += self.deltaWB
self.biasHL1 += self.deltaWB2
def run_analysis():
population_size = np.arange(start=25, stop=35, step=5)
pixels_percentage_to_change = np.arange(start=0.4, stop=0.5, step=0.1)
max_generation_count = [100]
population_percentage_to_keep = np.arange(start=0.1,stop=0.3, step=0.1)
mutation_prob = np.arange(start=0.4, stop=0.6, step=0.2)
crossover_prob = np.arange(start=0.7, stop=0.8, step=0.1)
all_params_combination = itertools.product(population_size,
pixels_percentage_to_change,
max_generation_count,
population_percentage_to_keep,
mutation_prob,
crossover_prob)
fake_img_generator = FakeImgGenerator(Utils.get_test_image_path(IMG_NAME),
P_MAX,
P_MIN,
FAKE_CLASS_PROB_TO_GET)
results = []
output_dir = Utils.get_new_output_dir()
for i, c in enumerate(all_params_combination):
print("Test generation set number: {}".format(i))
generation_result = fake_img_generator.run(population_size=c[0],
pixels_percentage_to_change=c[1],
max_generations_count=c[2],
population_percentage_to_keep=c[3],
mutation_prob=c[4],
crossover_prob=c[5])
results.append(generation_result)
result_path = Utils.create_child_dir(basepath=output_dir, child_dir="result_{}".format(str(i)))
print("Result probability: {}".format(generation_result.get_last_probability()))
generation_result.save(output_dir=result_path)
_save_results_comparison(results, output_dir)
def _prompt_layer_hex_color(self, layer):
answer = simpledialog.askstring(
"Choose a new layer color",
"Enter new hex color of layer.\nEnter 3 or 6 hexdigits.")
if answer:
if Utils.is_hex_color(answer):
if answer[0] != '#':
answer = "#{}".format(answer)
self.model.set_layer_color(layer, answer.lower())
else:
is_ok = messagebox.askyesno(
"Invalid entry",
"{} is not a valid hex color. Try again?".format(answer))
if is_ok:
self._prompt_layer_hex_color(layer)
def send_message(message, context):
ASSISTANT.set_http_config({'timeout': 100})
encoded_msg = Utils.encode_msg(message)
response = ASSISTANT.message(workspace_id=WORKSPACE_ID,
input={'text': encoded_msg},
context=context)
json_response = json.dumps(response, indent=2)
dict_response = json.loads(json_response)
# intents = dict_response['intents']
# entities = extract_entities(dict_response['entities'])
newContext = dict_response['context']
output = dict_response['output']['text']
return (newContext, output, message)
