def generate_name(filename, asset_name):
ext = os.path.splitext(filename)[1].replace('.', '')
f_type = file_type(filename)
if f_type is None:
debug.show_error("CANNOT RECOGNIZE FILE TYPE. EXTENSION UNKNOWN.")
return
prefix = ConfigReader.asset_prefix(f_type)
suffix = ''
if f_type == 'textures':
suffix = ConfigReader.texture_suffix(filename)
if not suffix:
debug.show_error(
"Can't recognize texture type from the name. Check texture naming rules!"
)
return
# complete name
name = prefix + '_' + asset_name
if suffix:
name += '_' + suffix
name += '.' + ext
return name
def instructions():
""" Format a text with app istructions
Dynamically created from config data.
This is useful to display informations on GUI.
:return: An HTML text representing instructions
"""
txt_naming = ConfigReader.texture_naming_dict()
text = "<b>Texture naming rules:</b><br>(put an underscore _ at the end of file name; " \
"you can enumerate textures using two digits after texture type without any other character" \
"<br> e.g. _normal01 or in general _normalXX)" \
"<br>"
for key, value in txt_naming.iteritems():
text += "<br>- {0}: {1}".format(key,
', '.join(a for a in value['text']))
text += "<br>"
text += "<br><b>File formats:</b>"
text += "<br>Meshes:"
text += ConfigReader.generate_file_filter()
text += "<br>Textures:"
text += ConfigReader.generate_texture_filter()
return text
class App(loggable.Loggable):
_instance = False
def __init__(self):
self.config = False
def set_config(self, config):
self.config = ConfigReader(config)
def start(self):
self.log("Starting app. Initializing storage")
storage_configs = self.config.get_storages()
storage_list = {}
for storage_name in storage_configs:
storage = storage_configs[storage_name]
storage_list[storage_name] = storages.builder.build(storage)
manager = Manager(self.config.dir, self.config.get_domains(), storage_list)
self.log("Starting manager service")
manager.start()
self.log("Starting API service")
api_service = api.service.Service(manager, self.config.port, self.config.ssl)
api_service.start()
@staticmethod
def i():
if not App._instance:
App._instance = App()
return App._instance
def config():
"""
:return: returns the configuration items in section 'mail' of configuration.ini
"""
configreader = ConfigReader()
params = configreader.get_config("mail")
return params
def __init__(self, config_file):
# Set up config and image services
self._config = ConfigReader(config_file)
self._images = Images(self._config)
# Create Flask app and define routes
self._app = Flask(__name__, static_url_path="/static")
self._app.add_url_rule("/", "index", self.show_index)
self._app.add_url_rule("/show_image", "show_image", self.show_image)
self._app.add_url_rule("/image_tags.csv", "download_tags", self.download_tags)
self._app.add_url_rule(
"/store_tags", "store_tags", self.store_tags, methods=["POST"]
)
def get_files(self):
""" Open a file dialog in order to choose files to upload """
f_dialog = QtGui.QFileDialog()
f_dialog.setFileMode(QtGui.QFileDialog.ExistingFiles)
f_dialog.setDirectory(os.getenv("USERPROFILE"))
filters = ConfigReader.generate_file_filter(
) + " " + ConfigReader.generate_texture_filter()
f_dialog.setNameFilter("Assets data ({})".format(filters))
f_dialog.selectNameFilter("{}".format(filters))
if f_dialog.exec_():
filenames = f_dialog.selectedFiles()
self.ui.listWidget.addItems(filenames)
def start_nfc_module_use_python2():
os.system('python3 ./dev/blink_led/OnOrOffRedAndGreenLEDs.py 0 0')
print("start_nfc_module_use_python2: process pid:" + str(os.getpid()))
dev_path = ConfigReader.get_dev_full_path()
script_path = os.path.join(os.path.join(dev_path, 'beam_py_2'), 'start.sh')
print(script_path)
os.system(script_path)
def execute_job_all_platfrom(threads):
# number of threads.
threads = int(threads)
## cmd line job options.
## slice from second option.
cmd_jobs = sys.argv[2:]
configs = ConfigReader.get_configurations(threads, cmd_jobs=cmd_jobs)
## for each item in the configuration.
## for multi also configuration objects are adjusted to be the same.
for config in configs:
responses = Responses(config)
thread_list = []
for i in range(config.threads):
th = Thread(target=execute_single, args=(i, config, responses))
thread_list.append(th)
th.start()
#wait all
for th in thread_list:
th.join()
# write the response in csv
responses.finished()
def store_measure(args):
r = StrictRedis(host=args.host, port=args.port, db=args.db_number)
if args.fake:
sg = FakeStatusGetter()
else:
if args.config:
cr = ConfigReader(args.config)
methods = cr.get_methods()
params = cr.get_params()
selects = cr.get_select()
else:
methods = ['cpu_percent']
params = [{'interval': 1, 'percpu': True}]
selects = None
sg = StatusGetter(method_names=methods, params_for_calls=params, select_for_calls=selects)
ss = StatusSender(r, args.hostname, sg, args.max)
ss.store()
def store_measure(args):
r = StrictRedis(host=args.host, port=args.port, db=args.db_number)
if args.fake:
sg = FakeStatusGetter()
else:
if args.config:
cr = ConfigReader(args.config)
methods = cr.get_methods()
params = cr.get_params()
selects = cr.get_select()
else:
methods = ['cpu_percent']
params = [{'interval': 1, 'percpu': True}]
selects = None
sg = StatusGetter(method_names=methods,
params_for_calls=params,
select_for_calls=selects)
ss = StatusSender(r, args.hostname, sg, args.max)
ss.store()
def connect():
global connection
if connection is not None:
return connection
# relative path from project root
db_rel_path = ConfigReader.db_path()
# we are inside root/core, so we have to get the absolute path from root
db_path = os.path.join(os.path.dirname(__file__), '..', db_rel_path)
connection = sqlite3.connect(db_path)
connection.row_factory = dict_factory
return connection
def execute_job(job_name):
config = ConfigReader.get_configuration(job_name)
responses = Responses(config)
thread_list = []
for i in range(config.threads):
th = Thread(target=execute_single, args=(i, config, responses))
thread_list.append(th)
th.start()
#wait all
for th in thread_list:
th.join()
# write the response in csv
responses.finished()
def run_server_process():
print("start run_server_process pid:" + str(os.getpid()))
# get config object
config = ConfigReader.get_config()
# init tcp server
tcp_server = TCPServer.TCPServer()
# init udp server
udp_server = UDPServer.UDPServer()
# start event loop
event_loop = asyncio.get_event_loop()
udp_server.start_server(event_loop)
tasks = [tcp_server.start_server()]
event_loop.run_until_complete(asyncio.wait(tasks))
try:
event_loop.run_forever()
except KeyboardInterrupt:
pass
finally:
event_loop.close()
def test(model_path, dataset_path):
log_path = os.path.join(model_path, 'test_log')
config = ConfigReader(os.path.join(model_path, 'config.json')).read()
dataset_extractor = DatasetExtractor(dataset_path, config)
dataset_test, test_size = dataset_extractor.extract()
test_iterator = dataset_test.make_one_shot_iterator()
dataset_handle = tf.placeholder(tf.string, shape=[])
feedable_iterator = tf.data.Iterator.from_string_handle(dataset_handle, dataset_test.output_types,
dataset_test.output_shapes, dataset_test.output_classes)
signal, label, signal_len, _ = feedable_iterator.get_next()
label = tf.cast(label, dtype=tf.int32)
model = ModelFactory.get(config.model_name, signal, config)
optimizer = OptimizerFactory.get(config.optimizer, model.logits, label, signal_len)
decoder = DecoderFactory.get(config.decoder, model.logits, signal_len)
distance_op = tf.reduce_mean(tf.edit_distance(tf.cast(decoder.decoded, dtype=tf.int32), label))
saver = tf.train.Saver()
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
saver.restore(sess, os.path.join(model_path, "model.ckpt"))
tf.saved_model.simple_save(sess,
os.path.join(model_path, 'saved_model'),
inputs={"signal": signal,
"lengths": signal_len},
outputs={"logits": model.logits})
def main_func(self):
if os.path.isfile('config.yaml'): #if config file is there but empty
self._key = ConfigReader().getKey()
if not self._key:
self.keyConfig
self.location = input('Enter location ').strip().lower()
else: #if there is no config file then create one
self.keyConfig
self.location = input('Enter location ').strip().lower()
self.location_encoded = urllib.parse.quote(self.location)
self.URLGenerator_obj = URLGenerator(user_api=self._key,
location=self.location_encoded)
self.RequestGenerator_obj = RequestGenerator(self.URLGenerator_obj)
self.DataFetcher_obj = DataFetcher(self.RequestGenerator_obj,
location=self.location)
self.DataFetcher_obj.getData()
print('Print data to a CSV file?(Y/N)')
if input() == 'Y':
self.DataFetcher_obj.exportData()
self._restartApp()
else:
self._restartApp()
def keyConfig(self):
"""Creates a new key config file"""
self._key = '06744d1eed7fa471e121919b4f946742 '.strip()
ConfigReader().putKey(self._key)
def keyConfig(self):
"""Creates a new key config file"""
self._key = input('Enter API Key ').strip()
ConfigReader().putKey(self._key)
# -*- coding: utf-8 -*-
from config import ConfigReader
from interactive import InteractivePassy
class PasswordUpdateError(Exception):
def __init__(self, message, errors):
super().__init__(message)
self.errors = errors
# Run IT!
if __name__ == '__main__':
config = ConfigReader()
InteractivePassy(config.artifactory_url(),
config.maven_settings_xml()).run()
from puzzle_script.puzzle_world_writer.level_set_writer import LevelSetWriter as PSLevelSetWriter
from tile_world.tile_world_writer.level_set_writer import LevelSetWriter as TWLeveSetWriter
from tile_world.tile_world_writer.tile_world_solution_writer import TileWorldSolutionWriter
from validation.solver import Solver
from generation.aesthetic_settings import AestheticSettings
from creation.creator import Creator
from log import Log
from config import ConfigReader
from ratings.ratings import Ratings
import csv
import numpy as np
import random
import subprocess
config = ConfigReader('config.json')
Log.verbose = True
ratings = Ratings(config.ratings_file)
level_count = config.level_count
if config.play_or_generate == "replay":
level_count = ratings.level_count()
if level_count == 0:
print("You must generate at least one level, level_count =", level_count)
exit()
aesthetic_settings = config.aesthetic
def train(config_path,
train_dataset_path,
val_dataset_path,
output_path,
early_stop=False):
keep_training = True
if not os.path.exists(output_path):
os.makedirs(output_path)
copyfile(config_path, os.path.join(output_path, 'config.json'))
log_path = os.path.join(output_path, 'log')
config = ConfigReader(config_path).read()
train_dataset_extractor = DatasetExtractor(train_dataset_path, config)
val_dataset_extractor = DatasetExtractor(val_dataset_path, config)
dataset_train, train_size = train_dataset_extractor.extract()
train_iterator = dataset_train.make_one_shot_iterator()
dataset_val, val_size = val_dataset_extractor.extract()
dataset_test = dataset_val.take(300)
dataset_val = dataset_val.take(75)
val_iterator = dataset_val.make_initializable_iterator()
test_iterator = dataset_test.make_one_shot_iterator()
dataset_handle = tf.placeholder(tf.string, shape=[])
feedable_iterator = tf.data.Iterator.from_string_handle(
dataset_handle, dataset_train.output_types,
dataset_train.output_shapes, dataset_train.output_classes)
signal, label, signal_len, _ = feedable_iterator.get_next()
label = tf.cast(label, dtype=tf.int32)
model = ModelFactory.get(config.model_name, signal, config)
optimizer = OptimizerFactory.get(config.optimizer, model.logits, label,
signal_len)
decoder = DecoderFactory.get(config.decoder, model.logits, signal_len)
distance_op = tf.reduce_mean(
tf.edit_distance(tf.cast(decoder.decoded, dtype=tf.int32), label))
saver = tf.train.Saver()
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
if config.debug:
sess = tf_debug.LocalCLIDebugWrapperSession(sess)
sess.run(tf.global_variables_initializer())
training_handle = sess.run(train_iterator.string_handle())
validation_handle = sess.run(val_iterator.string_handle())
test_handle = sess.run(test_iterator.string_handle())
epoch = 0
steps = 0
previous_print_length = 0
losses = []
prev_val_distance = None
prev_val_loss = None
while keep_training:
try:
loss_value, _ = sess.run(
[optimizer.loss, optimizer.optimizer],
feed_dict={dataset_handle: training_handle})
losses.append(loss_value)
steps += config.batch_size
if previous_print_length > 0:
print('\b' * previous_print_length, end='', flush=True)
message = f"Epoch: {epoch} Step: {steps} Step Loss:{loss_value} Epoch Loss: {np.mean(losses)}"
log_to_file(log_path, message)
previous_print_length = len(message)
print(message, end='', flush=True)
if steps >= train_size:
saver.save(sess, os.path.join(output_path, f"model.ckpt"))
if config.validate or early_stop:
distances = []
val_losses = []
sess.run(val_iterator.initializer)
while True:
try:
distance, val_loss = sess.run(
[distance_op, optimizer.loss],
feed_dict={dataset_handle: validation_handle})
distances.append(distance)
val_losses.append(val_loss)
except tf.errors.InvalidArgumentError as e:
log_to_file(log_path, e.message)
raise e
except tf.errors.OutOfRangeError:
break
mean_distance = np.mean(distances)
mean_val_loss = np.mean(val_losses)
if prev_val_distance is not None and prev_val_loss is not None:
if prev_val_distance < mean_distance and prev_val_loss < mean_val_loss:
keep_training = False
prev_val_loss = mean_val_loss
prev_val_distance = mean_distance
print(flush=True)
log_message = f"Epoch: {epoch} Validation Loss: {mean_val_loss} Edit Distance: {mean_distance}"
print(log_message, flush=True)
log_to_file(log_path, log_message)
epoch += 1
steps = 0
previous_print_length = 0
losses = []
except tf.errors.OutOfRangeError:
break # End of dataset
saver.save(sess, os.path.join(output_path, "model.ckpt"))
test_distances = []
test_losses = []
while True:
try:
test_distance, test_loss = sess.run(
[distance_op, optimizer.loss],
feed_dict={dataset_handle: test_handle})
test_distances.append(test_distance)
test_losses.append(test_loss)
except tf.errors.OutOfRangeError:
break
mean_test_distance = np.mean(test_distances)
mean_test_loss = np.mean(test_losses)
print(flush=True)
log_message = f"Test Loss: {mean_test_loss} Edit Distance: {mean_test_distance}"
print(log_message, flush=True)
log_to_file(log_path, log_message)
from config import ConfigReader
from iptables import IPTables
from api import run_main
if __name__ == '__main__':
cfg = ConfigReader()
ipt = IPTables(cfg)
# Used one time for the initial setup.
# ipt.setup_chain()
# ipt.get_chain()
# ipt.add_rule('192.168.1.6')
run_main(cfg)
import json
import pytest
from hamcrest import *
from config import ConfigReader
from utils.cli_utils import run_console_command
test_data = json.loads(
run_console_command(
'{0}/yii db-status/tables-timestamp --column=updated_at --json'.format(
ConfigReader('urls').get('analytics_home'))))
class TestFields:
@pytest.mark.parametrize("row", test_data)
def test_updated_at(self, row):
assert_that(
row['has_key'], equal_to('1'),
'table {} has no updated_at index'.format(row['table_name']))
if __name__ == '__main__':
pytest.main()
def open(self):
cnf = ConfigReader()
self._connect = MySQLdb.connect(
host=cnf.get('DB_HOST'), user=cnf.get('DB_USER'),
passwd=cnf.get('DB_PASSWORD'), db=cnf.get('DB_NAME'))
self._cursor = self._connect.cursor()
class TagServer:
"""
Flask server for tagging images.
Parameters
----------
config_file : str
Path the YAML configuration file.
Methods
-------
start()
Starts the Flask server.
"""
def __init__(self, config_file):
# Set up config and image services
self._config = ConfigReader(config_file)
self._images = Images(self._config)
# Create Flask app and define routes
self._app = Flask(__name__, static_url_path="/static")
self._app.add_url_rule("/", "index", self.show_index)
self._app.add_url_rule("/show_image", "show_image", self.show_image)
self._app.add_url_rule("/image_tags.csv", "download_tags", self.download_tags)
self._app.add_url_rule(
"/store_tags", "store_tags", self.store_tags, methods=["POST"]
)
def start(self):
"""Starts the Flask server with the configured host settings."""
# Get config settings
host = self._config.get("server/host", "127.0.0.1")
port = self._config.get("server/port", "8080")
debug = self._config.get("server/debug mode", False)
host_str = f"http://{host}:{port}/"
# The reloader has not yet run - open the browser
if not os.environ.get("WERKZEUG_RUN_MAIN"):
webbrowser.open_new(host_str)
# Run Flask app
self._app.run(host=host, port=port, debug=debug)
def show_index(self):
"""Routing: Show the main page."""
image_id = request.args.get("image_id")
if not image_id:
untagged = self._images.first_untagged_id()
return redirect(f"/?image_id={untagged}")
else:
content = self._render_image(image_id)
metadata = {
"max_image_width": self._config.get("interface/max_width", 600),
"max_image_height": self._config.get("interface/max_height", 700),
}
return render_template("index.html", content=content, meta=metadata)
def show_image(self):
"""Routing: Sends the image file to the webbrowser."""
image_id = request.args.get("image_id")
image = self._images.get(image_id)
return send_file(image["path"], mimetype="image/jpg")
def store_tags(self):
"""Routing: Stores the (updated) tag data for the image."""
# Get form data
sep = self._config.get("tagging/multi-separator", ", ")
data = {
"id": request.form.get("id"),
"tags": sep.join(request.form.getlist("tags")),
"remark": request.form.get("remark").strip(),
}
self._images.store(data)
next_image = self._images.next_id(data["id"])
target = "/"
if next_image:
target = f"/?image_id={next_image}"
return redirect(location=target)
def download_tags(self):
"""Routing: Download tag information as CSV file."""
tags = self._images.dump_data()
# Use StringIO to capture the to_csv() output
csv_string = io.StringIO()
tags.to_csv(csv_string, index=False)
return Response(csv_string.getvalue(), mimetype="text/csv")
def _render_image(self, image_id):
"""
Renders image template for the provided image ID.
Parameters
----------
image_id : str
MD5 hash for the image to render.
"""
# Load all available tags from cofniguration
shortcuts = self._config.get("tagging/tags")
if not shortcuts:
raise ValueError("Could not read any tags from the configuration file.")
# Load and preprocess the data
data = self._images.get(image_id)
if not data:
raise RuntimeError(f"Cannot render image with ID '{image_id}'.")
if not isinstance(data["remark"], str):
data["remark"] = ""
# Split multiple tags
if isinstance(data["tags"], str):
sep = self._config.get("tagging/multi-separator", ", ")
data["tags"] = data["tags"].split(sep)
else:
data["tags"] = []
# Get next and previous IDs
data["prev_image_id"] = self._images.prev_id(image_id)
data["next_image_id"] = self._images.next_id(image_id)
# Add meta data
data["all_tags"] = list(shortcuts)
data["shortcuts"] = shortcuts
data["tag_question"] = self._config.get("tagging/tag question", "Select tags:")
data["allow_remarks"] = self._config.get("tagging/allow remarks", False)
data["multi_select"] = self._config.get("tagging/multi-select", False)
return render_template("tag_image.html", **data)
def set_config(self, config):
self.config = ConfigReader(config)
def read_config(config_file):
reader = ConfigReader(config_file)
return reader.get_config()
def file_type(filename):
ext = os.path.splitext(filename)[1].replace('.', '')
return ConfigReader.file_type_from_ext(ext)
def generate_dir_path(self):
self.dir = os.path.join('//', ConfigReader.server_name(),
ConfigReader.share_name(),
ConfigReader.content_name(), self.project_name,
'assets', self.group_name, self.asset_name)
os.path.normcase(self.dir)