def setup_logger(args):
options = {'debug': False, 'silent': False}
if args.debug:
options['debug'] = True
if args.silent:
options['silent'] = True
logger_module.setup(options=options)
global logger
logger = logging.getLogger('main')
def run(sc):
ts = datetime.datetime.now().replace(microsecond=0).isoformat('_')
sc.sim_time_start = ts
logger.setup(sc)
store_scenario(sc)
INFO('Init (%s)' % ts)
INFO('Fitness (minimal): %.6f' % sc.sol_fitness_min)
INFO('Fitness (maximal): %.6f' % sc.sol_fitness_max)
INFO('Fitness (average): %.6f' % sc.sol_fitness_avg)
INFO('Creating %d agents' % sc.opt_m)
agents = dict()
for aid, search_space, initial_value in zip(sc.agent_ids,
sc.agent_search_spaces,
sc.agent_initial_values):
agents[aid] = Agent(aid, search_space, initial_value)
INFO('Connecting agents')
for a, neighbors in sc.network.items():
for n in neighbors:
# Consistency check
assert a != n, 'cannot add myself as neighbor!'
# Add neighbor
DEBUG('', 'Connecting', a, '->', n)
if n not in agents[a].neighbors:
agents[a].neighbors[n] = agents[n]
else:
WARNING(n, 'is already neighbor of', a)
INFO('Starting simulation')
mas = Mas(sc, agents)
logger.set_mas(mas)
stats = Stats(sc, agents)
stats.eval(mas.current_time)
AGENT(mas.aid, 'Notifying initial agent (%s)' % sc.sim_initial_agent)
kappa = Working_Memory(sc.objective, dict(),
Solution_Candidate(None, dict(), float('-inf')))
msg = Message(mas.aid, sc.sim_initial_agent, kappa)
mas.msg(msg)
while mas.is_active():
mas.step()
stats.eval(mas.current_time)
if not stats.is_converged():
ERROR('convergence not reached!')
ts = datetime.datetime.now().replace(microsecond=0).isoformat('_')
INFO('End (%s)' % ts)
# Store scenario again, this time with simulation result
store_scenario(sc, overwrite=True)
return stats
def run(sc):
ts = datetime.datetime.now().replace(microsecond=0).isoformat('_')
sc.sim_time_start = ts
logger.setup(sc)
store_scenario(sc)
INFO('Init (%s)' % ts)
INFO('Fitness (minimal): %.6f' % sc.sol_fitness_min)
INFO('Fitness (maximal): %.6f' % sc.sol_fitness_max)
INFO('Fitness (average): %.6f' % sc.sol_fitness_avg)
INFO('Creating %d agents' % sc.opt_m)
agents = dict()
for aid, search_space, initial_value in zip(
sc.agent_ids, sc.agent_search_spaces, sc.agent_initial_values):
agents[aid] = Agent(aid, search_space, initial_value)
INFO('Connecting agents')
for a, neighbors in sc.network.items():
for n in neighbors:
# Consistency check
assert a != n, 'cannot add myself as neighbor!'
# Add neighbor
DEBUG('', 'Connecting', a, '->', n)
if n not in agents[a].neighbors:
agents[a].neighbors[n] = agents[n]
else:
WARNING(n, 'is already neighbor of', a)
INFO('Starting simulation')
mas = Mas(sc, agents)
logger.set_mas(mas)
stats = Stats(sc, agents)
stats.eval(mas.current_time)
AGENT(mas.aid, 'Notifying initial agent (%s)' % sc.sim_initial_agent)
kappa = Working_Memory(sc.objective, dict(),
Solution_Candidate(None, dict(), float('-inf')))
msg = Message(mas.aid, sc.sim_initial_agent, kappa)
mas.msg(msg)
while mas.is_active():
mas.step()
stats.eval(mas.current_time)
if not stats.is_converged():
ERROR('convergence not reached!')
ts = datetime.datetime.now().replace(microsecond=0).isoformat('_')
INFO('End (%s)' % ts)
# Store scenario again, this time with simulation result
store_scenario(sc, overwrite=True)
return stats
def main(args):
template_loader = JinjaLoader(
os.path.join(os.path.dirname(__file__), 'templates/'))
application = BusinessIssues(get(),
template_loader=template_loader,
static_path=os.path.join(
os.path.dirname(__file__), "static"),
xsrf_cookies=False,
debug=args.debug)
server = HTTPServer(application, xheaders=True)
server.listen(args.port, '0.0.0.0')
logger.setup()
install_tornado_shutdown_handler(tornado.ioloop.IOLoop.instance(), server,
logging.getLogger())
logging.info('start service at ' + time.ctime() + '\n')
tornado.ioloop.IOLoop.current().start()
def main():
global requestSize
global serverWithoutFile
appPath = os.path.dirname(os.path.abspath(__file__))
logger.setup(appPath + "/Logs/log.txt")
#logger.logt("main")
# дескриптор файла (что пришел по сокету с сервера с запросом)
fd = int(sys.argv[1])
print("--------")
# запрос
requestData = str(os.read(fd, requestSize)) #os.read(fd, 2048).decode('utf-8') #str(os.read(fd, 2048))
# парсинг запроса
request = requester.getHeaders(requestData)
requester.parseParams(request)
#logger.logt("request url:"+request.getHead("URL"))
# ответl
response = zetypes.ZeResponse();
response.setAppPath(appPath) # root path (from main)
response.setRequest(request)
# content
respDeprecatedVar = responser.createResponseHtml(serverWithoutFile, response, request)
content = response.buildHttpResponse()
#with open(fd, 'w') as outf:
# outf.write(content)
try:
with open(fd, 'w') as outf:
outf.write(content)
except IOError as e:
if e.errno == errno.EPIPE:
print("pipe err")
#logger.logt("end ok")
#logger.log("-------------")
print("ok")
def __init__(self,
cluster=None,
ipaddress=None,
username=None,
password=None,
source_dir=None,
target_dir=None,
copyback_files=None,
copyback_dir=None,
command=None,
logfile=None,
start_clean=True,
cleanup=True,
timeout=None,
all=None,
source_files=None,
apikey=None):
self.cluster = cluster
if isinstance(cluster, str):
self.cluster = picluster.PiBoardTable(cluster, apikey)
self.ipaddress = ipaddress
self.username = username
self.password = password
self.source_dir = source_dir
self.source_files = source_files
self.target_dir = target_dir
self.copyback_files = copyback_files
self.copyback_dir = copyback_dir
self.command = command
self.start_clean = start_clean
self.cleanup = cleanup
self.logfile = logfile
self.timeout = timeout
self.all = all
self.machine = None
self.ssh = None
self.buffer = None
# global logger is hooked up to parent modules by module name and this
# logger can see all the remote command output from all commands, which
# will be formatted differently with "ThreadId: " prefix so user can
# make sense of the combined output when remote commands are running in
# parallel.
if self.logfile:
self.logger = logger.setup(self.logfile)
else:
self.logger = logger.get()
if not cluster and not ipaddress:
raise Exception("Error: required ipaddress or cluster or both")
# Sanity-check parameters
if self.target_dir and os.path.pathsep in self.target_dir:
raise Exception(
"Error: multilevel target directories not supported")
def main():
""" MAIN APP """
master_config = get_start_arguments()
logger.setup(master_config['log_file'], 'w')
config.init(master_config['config_file'])
config.set_section('GLOBAL')
_version = config.key['version']
_port = config.key['port']
logger.level(config.key['log_level'])
app = manager.App(camera_data=config.set_section('CAMERA'),
display_data=config.set_section('DISPLAY'),
led_data=config.set_section('LED'),
buzzer_data=config.set_section('BUZZER'),
random_data=config.set_section('STATS'),
version=_version,
port=get_mac_address(_port))
while app.run():
pass
app.close()
def __init__(self, address, interface, timeout=None,
encoding=None, logging=None):
if (not isinstance(interface, type) or
not issubclass(interface, JsonRpcIface)):
raise TypeError('Interface must be JsonRpcIface subclass')
self.interface = interface
self.timeout = timeout
self.encoding = encoding or 'utf-8'
logger.setup(logging)
try:
asyncore.dispatcher.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.bind(address)
self.listen(0)
except Exception:
logger.exception('Server run error')
raise
def main():
parser = argparse.ArgumentParser(
description="Converts ONNX model to ELL model\n"
"Example:\n"
"onnx_import.py model.onnx\n"
"This outputs 'model.ell' which can be compiled with ELL's 'wrap' tool\n"
)
parser.add_argument("input",
type=Text,
help="Input model file (onnx or protobuf)")
parser.add_argument("--zip_ell_model",
help="zips the output ELL model if set",
action="store_true")
parser.add_argument(
"--verbose",
help="print verbose output during the import. Helps to diagnose ",
action="store_true")
parser.add_argument(
'-o',
'--output_directory',
help='Path to output directory (default: input file directory)',
default=None)
model_options = parser.add_argument_group('model_options')
model_options.add_argument(
"--step_interval",
help="produce a steppable ELL model for a millisecond interval",
default=None)
model_options.add_argument(
"--lag_threshold",
help=
"number of step intervals to fall behind before notifying the caller.\n"
"used when step_interval is set\n",
default=None)
logger.add_logging_args(parser)
args = parser.parse_args()
logger.setup(args)
convert(args.input, args.output_directory, args.zip_ell_model,
args.step_interval, args.lag_threshold)
def __init__(self, *args, **kwargs):
self.args = list(args)
self.kwargs = dict(kwargs)
self.logger_kwargs = {
'directory': kwargs.get('logsdir', os.path.join('..', 'logs')),
'name': 'maioget.%s' % (kwargs.get('name', 'UNNAMED'),),
'daemon': kwargs.get('daemon', False),
'loglevel': kwargs.get('loglevel', 'INFO'),
}
try:
self.set_logger(logger.setup(**self.logger_kwargs))
except LogDirectoryException:
raise
def main(argv):
global log
parser = argparse.ArgumentParser()
parser.add_argument("src", help="The Bitcasa base64 path for file source")
parser.add_argument("dst", help="The final destination root dir or your files")
parser.add_argument("token", help="The access token from Bitcasa. To get one navigate to https://rose-llc.com/bitcasafilelist")
parser.add_argument("-t", "--temp", help="The temp dir to store downloaded files. (Should be a local folder)")
parser.add_argument("-l", "--log", help="Full path to log file")
parser.add_argument("--depth", type=int, help="Specify depth of folder traverse. 0 is same as --norecursion")
parser.add_argument("-m", "--threads", type=int, help="Specify the max number of threads to use for downloading. Default is 5")
parser.add_argument("--local", help="Only store file locally. Do not use temp dir", action="store_true")
parser.add_argument("--norecursion", help="Do not go below the src folder. (Same as --depth=0)", action="store_true")
parser.add_argument("--verbose", help="increase output verbosity", action="store_true")
args = parser.parse_args()
_log = ""
if (args.log == None or args.log == "") and not args.local:
_log = os.path.join(args.temp, "runlog.txt")
elif (args.log == None or args.log == "") and args.local:
_log = os.path.join(args.dst, "runlog.txt")
elif args.log != None and args.log != "":
_log = args.log
rec = not args.norecursion
if (args.temp == "" and not args.local) or args.dst == "" or args.src == "" or args.token == "":
sys.stderr.write("Please supply access token, temp, source, and destination locations. If this is a local copy, then specify -l or --local\n")
sys.exit(2)
elif args.temp != None and args.temp != "" and args.local:
sys.stdout.write("Local specified. Ignoring temp\n")
args.temp = args.dst
elif args.local:
args.temp = args.dst
#initialize temp dir
try:
if not os.path.isdir(args.temp):
os.makedirs(args.temp)
except:
sys.stderr.write("Error creating temp directory\n")
raise
#initialie logging
log = logger.setup(logfile=_log, debug=args.verbose)
if args.depth > 0 and args.norecursion:
log.info("Note: Non 0 depth and --no-recursion parameter present. Assuming recusion")
rec = True
log.debug("Initializing Bitcasa")
bitc = BitcasaDownload(args.depth, args.temp, args.src, args.dst, rec, args.local, args.token, args.threads)
bitc.process()
log.info("done")
import click
import logging
import logger
from deploy import run, js_code_snippet, preflight_checks, dashboard_url
logger = logger.setup()
@click.group()
@click.option('--debug', is_flag=True)
def cli(debug):
if debug:
logger.setLevel(logging.DEBUG)
@cli.command()
def preflight():
logger.info('running preflight checks')
preflight_checks()
@cli.command()
@click.option('--preflight/--no-preflight', default=True)
def deploy(preflight):
if preflight:
logger.info('running preflight checks')
if not preflight_checks():
return
logger.info('deploying')
run()
js_code_snippet()
def __init__(self, url, timeout=None, encoding=None, logging=None):
self.url = url
self.timeout = timeout
self.encoding = encoding or 'utf-8'
logger.setup(logging)
#!/usr/bin/env python
import os
import time
from PIL import Image
from ConfigParser import NoOptionError
import s3
import server
import settings
import db
import logger
l = logger.setup('hook')
cfg = settings.config()
sender = 0
def generate_key(fileName):
"""
Generate a unique filename based on the current timestamp.
"""
timestamp = str(time.time()).replace(".", "")
file_extension = fileName.split(".")[1]
return timestamp + "." + file_extension
def generate_thumb(src, dest):
"""
Create a thumbnail of the file using Pillow and return
the path to the created thumbnail.
"""
try:
quality = cfg.getint('thumbs', 'quality')
botwtools is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with botwtools. If not, see <https://www.gnu.org/licenses/>.
"""
import sys
if sys.version_info < (3, 0):
raise RuntimeError("This program requires Python 3.")
import logger
logger.setup('botwtools')
log = logger.logging.getLogger()
import argparse
from app import App
import codec
arg_parser = None # declare global
def _setupArgs():
global arg_parser
parser = argparse.ArgumentParser(
description="Breath of the Wild modding tools.")
arg_parser = parser
parser.add_argument('--extract',
# wrap it up and make a sendmany call
Args = namedtuple(
'args', 'dummy amounts minconf comment \
substractfeefrom replaceable conf_target estimate_mod output_assets')
params = Args("", addr2amt, 1, "wallet cleanup", pay_fee_addr, False, 6,
'UNSET', addr2asset)
if (input(
f"please confirm you want to do sendmany with the following arguments: \n{params}\n"
)) in ['y', 'yes']:
txid = r.call("sendmany", *params)
return txid
else:
logging.info("swipe aborted by user\n")
exit()
if __name__ == "__main__":
import logger
logger.setup(logger.Args('debug'))
# wipe all utxos to new addresses
txid = swipe_assets()
if not txid:
print("Something is wrong\n")
else:
print(txid)
# make a dump of the wallet.dat
# stop the elements service
# move wallet.dat to wallet.$m.$d
# restart elements service
from gevent import monkey; monkey.patch_all()
import env; env.setup()
import logger; logger.setup()
import logging
log = logging.getLogger()
# An example gevent/flask app
import flask
app = flask.Flask(__name__)
@app.route('/get', methods=['GET'])
def get():
# flask.request.args for GET
# flask.request.form for POST
return "Hello World %s" % flask.request.args
if __name__ == '__main__':
from gevent.wsgi import WSGIServer
http_server = WSGIServer(('127.0.0.1', 5000), app)
http_server.serve_forever()
def main(args):
arg_parser = argparse.ArgumentParser(
description="Converts CNTK model to ELL model\n"
"Example:\n"
" cntk_import.py model.cntk\n"
"This outputs 'model.ell' which can be compiled with ELL's 'wrap' tool\n")
arg_parser.add_argument(
"cntk_model_file",
help="path to a CNTK model file, or a zip archive of a CNTK model file")
arg_parser.add_argument(
"--zip_ell_model",
help="zips the output ELL model if set", action="store_true")
arg_parser.add_argument(
"--use_legacy_importer",
help="specifies whether to use the new importer engine or the legacy importer", action="store_true")
arg_parser.add_argument(
"--plot_model",
help="specifies whether to plot the model using SVG to cntk_model_file.svg", action="store_true")
arg_parser.add_argument(
"--verify_vision_model",
help="verifies the imported vision ELL model produces the same output as the original CNTK model",
action="store_true")
arg_parser.add_argument(
"--verify_audio_model",
help="verifies the imported audio ELL model produces the same output as the original CNTK model",
action="store_true")
model_options = arg_parser.add_argument_group('model_options')
model_options.add_argument(
"--step_interval",
help="produce a steppable ELL model for a millisecond interval",
type=float)
model_options.add_argument(
"--lag_threshold",
help="millisecond time lag before notifying the caller.\n"
"used when step_interval is set\n",
type=float)
logger.add_logging_args(arg_parser)
args = arg_parser.parse_args(args)
_logger = logger.setup(args)
args = vars(args)
step_interval = args['step_interval']
lag_threshold = args['lag_threshold']
if step_interval is not None and lag_threshold is None:
lag_threshold = step_interval * 2
plot_model = args["plot_model"]
verify_model = {"vision": args["verify_vision_model"],
"audio": args["verify_audio_model"]}
# extract the model if it's in an archive
unzip = ziptools.Extractor(args['cntk_model_file'])
success, filename = unzip.extract_file(".cntk")
if success:
_logger.info("Extracted: " + filename)
else:
# not a zip archive
filename = args['cntk_model_file']
if not args["use_legacy_importer"]:
_logger.info("-- Using new importer engine --")
ell_map = cntk_to_ell.map_from_cntk_model_using_new_engine(filename, step_interval, lag_threshold, plot_model,
verify_model)
else:
_logger.info("-- Using legacy importer --")
predictor = cntk_to_ell.predictor_from_cntk_model(filename)
ell_map = ell.neural.utilities.ell_map_from_predictor(predictor, step_interval, lag_threshold)
model_file_name = os.path.splitext(filename)[0] + ".ell"
_logger.info("\nSaving model file: '" + model_file_name + "'")
ell_map.Save(model_file_name)
if args["zip_ell_model"]:
_logger.info("Zipping model file: '" + model_file_name + ".zip'")
zipper = ziptools.Zipper()
zipper.zip_file(model_file_name, model_file_name + ".zip")
os.remove(model_file_name)
parser.add_argument(
'--granularity',
type=int,
default=60,
choices=[60, 300, 900, 3600, 21600, 86400],
help=
'granularity of candle data in seconds. choices: 1m, 5m, 15m, 6h, 24h')
args = parser.parse_args()
# use formatted datetime for filenames
curr_datetime_str = dt.today().strftime('%Y-%m-%d-%H-%M-%S')
# setup logging
log_name = 'exchanges.cb'
log_filename = 'logs/{:s}-{:s}'.format(args.market, curr_datetime_str)
log = logger.setup(log_name, log_filename, logger.levels[args.loglevel])
# setup pystore for storing time series data
ps_store = pystore.store('coinbase')
ps_collection = ps_store.collection('candles.minute')
ps_item = '{:s}'.format(args.market)
ps_item_exists = ps_item in ps_collection.list_items()
# track execution time to monitor avg request time
exec_time = time.time()
start_date = get_start_date(args.market)
dates = _gen_date_frames(start_date)
for count, (start, end) in enumerate(dates):
def parse_command_line(self, args=None):
arg_parser = _PassArgsParser(prog="wrap", description="""This tool wraps a given ELL model in a CMake buildable \
project that builds a language specific module that can call the ELL model on a given target platform.
The supported languages are:
python (default)
cpp
The supported target platforms are:
pi0 Raspberry Pi 0
pi3 Raspberry Pi 3
orangepi0 Orange Pi Zero
aarch64 arm64 Linux, works on Qualcomm DragonBoards
host (default) your host computer architecture""")
for arg in self.arguments.keys():
argdef = self.arguments[arg]
if "required" in argdef.keys():
arg_parser.add_argument("--" + arg, "-" + argdef["short"],
help=argdef["help"], required=True)
elif "choices" in argdef.keys():
arg_parser.add_argument("--" + arg, "-" + argdef["short"],
help=argdef["help"], default=argdef["default"],
choices=argdef["choices"])
elif type(argdef["default"]) is bool and not argdef["default"]:
arg_parser.add_argument("--" + arg, "-" + argdef["short"],
help=argdef["help"], action="store_true", default=False)
else:
arg_parser.add_argument("--" + arg, "-" + argdef["short"],
help=argdef["help"], default=argdef["default"])
compile_args = []
if '--' in args:
index = args.index('--')
compile_args = args[index + 1:]
args = args[:index]
logger.add_logging_args(arg_parser)
args = arg_parser.parse_args(args)
self.logger = logger.setup(args)
self.model_file = args.model_file
_, tail = os.path.split(self.model_file)
self.model_file_base = os.path.splitext(tail)[0]
self.model_name = args.module_name
if not self.model_name:
self.model_name = self.model_file_base.replace('-', '_')
self.language = args.language
self.target = args.target
self.objext = self.get_objext(self.target)
self.output_dir = args.outdir
if self.output_dir is None:
self.output_dir = self.target
if os.path.isfile(self.output_dir + ".py"):
raise Exception("You have a python module named '{}', which will conflict with the --outdir of '{}'. \
Please specify a different outdir.".format(self.output_dir + ".py", self.output_dir))
self.profile = args.profile
self.verbose = self.logger.getVerbose() or args.verbose
self.llvm_format = args.llvm_format
self.optimization_level = args.optimization_level
self.no_opt_tool = args.no_opt_tool or self.optimization_level in ['0', 'g']
self.no_llc_tool = args.no_llc_tool
self.optimize = not args.no_optimize
self.fuse_linear_ops = not args.no_fuse_linear_ops
self.optimize_reorder = not args.no_optimize_reorder
self.debug = args.debug
self.blas = self.str2bool(args.blas)
self.swig = self.language != "cpp"
self.cpp_header = self.language == "cpp"
self.compile_args = compile_args
"--auto_scale",
help=
"Whether to auto-scale audio input to range [-1, 1] (default false).",
action='store_true')
parser.add_argument(
"--output",
help="Name of text file to contain list of test results.",
default="results.json")
parser.add_argument("--prediction_algorithm",
"-pa",
help="Specify prediction algorithm (max or mean).",
default="max")
logger.add_logging_args(parser)
args = parser.parse_args()
log = logger.setup(args)
sample_rate = args.sample_rate
if args.list_file:
verify_file_exists("list_file", args.list_file)
elif args.dataset:
verify_file_exists("dataset", args.dataset)
else:
log.error("Expecting one of --list_file or --dataset")
sys.exit(1)
test = AudioModelTester(args.reset)
test.run_test(args.featurizer, args.classifier, args.list_file,
args.max_tests, args.dataset, args.categories, sample_rate,
args.auto_scale, args.output, args.prediction_algorithm,
def run(self):
model_id_string = "{} ({})".format(
self.profiler_data.model_name,
self.profiler_data.profile_options.to_long_string())
global_logger = logger.setup(
format="{} %(message)s".format(model_id_string))
global_logger.info("Compiling model {} ...".format(model_id_string))
profile_src_path = os.path.join(self.ell_build_root, "tools",
"utilities", "profile")
# Take all .cpp, .h, .cmd, .sh, and .cmake files from the build/tools/utilities/profile directory, and only hard-code the CMakeLists file to take
all_profile_files = os.listdir(profile_src_path)
profile_extensions_to_deploy = [".cpp", ".h", ".cmd", ".sh", ".cmake"]
profile_src_files = [
filename for filename in all_profile_files if True in
[filename.endswith(ext) for ext in profile_extensions_to_deploy]
]
cmakelists_file_to_deploy = "CMakeLists-device-parallel.txt.in"
profile_src_files.append(cmakelists_file_to_deploy)
profile_src_file_renames = {
cmakelists_file_to_deploy: "CMakeLists.txt"
}
base_wrap_args = [
"--model_file",
self.profiler_data.model_path,
"--module_name",
"ELL", # Profiler C++ code assumes the module name is ELL
"--target",
self.target,
"--language",
self.language,
"--outdir",
self.profiler_data.built_profilers_path,
]
base_wrap_args.extend(
self.profiler_data.profile_options.base_wrap_args())
opt_wrap_args = ["--llvm_format", "ir"]
no_opt_wrap_args = [
"--no_opt_tool", "--no_llc_tool", "--llvm_format", "obj"
]
profile_wrap_args = ["--profile"]
profile_options_additional_compile_args = self.profiler_data.profile_options.additional_compile_args(
)
if profile_options_additional_compile_args:
additional_compile_args = [
"--"
] + profile_options_additional_compile_args
else:
additional_compile_args = []
no_opt_noprofile_builder = wrap.ModuleBuilder()
no_opt_noprofile_wrap_args = base_wrap_args + no_opt_wrap_args + additional_compile_args
no_opt_noprofile_builder.parse_command_line(no_opt_noprofile_wrap_args)
no_opt_profile_builder = wrap.ModuleBuilder()
no_opt_profile_wrap_args = base_wrap_args + no_opt_wrap_args + profile_wrap_args + additional_compile_args
no_opt_profile_builder.parse_command_line(no_opt_profile_wrap_args)
opt_noprofile_builder = wrap.ModuleBuilder()
opt_noprofile_wrap_args = base_wrap_args + opt_wrap_args + additional_compile_args
opt_noprofile_builder.parse_command_line(opt_noprofile_wrap_args)
opt_profile_builder = wrap.ModuleBuilder()
opt_profile_wrap_args = base_wrap_args + opt_wrap_args + profile_wrap_args + additional_compile_args
opt_profile_builder.parse_command_line(opt_profile_wrap_args)
# Profiler and ExecuteModel binaries expect to find the following header and object files, built with different compile args:
# compiled_model.o
# compiled_model.h
# compiled_model_opt.o
# compiled_model_opt.h
# compiled_model_noprofile.o
# compiled_model_noprofile.h
# compiled_model_noprofile_opt.o
# compiled_model_noprofile_opt.h
built_name_prefix = "compiled_model"
noprofile_suffix = "_noprofile"
opt_suffix = "_opt"
obj_suffix = ".o"
header_suffix = ".h"
file_suffixes = [obj_suffix, header_suffix]
base_model_filename = os.path.basename(self.profiler_data.model_path)
base_model_name = os.path.splitext(base_model_filename)[0]
no_opt_noprofile_renames = {
(base_model_name + file_suffix):
(built_name_prefix + noprofile_suffix + file_suffix)
for file_suffix in file_suffixes
}
no_opt_profile_renames = {(base_model_name + file_suffix):
(built_name_prefix + file_suffix)
for file_suffix in file_suffixes}
opt_noprofile_renames = {
(base_model_name + file_suffix):
(built_name_prefix + noprofile_suffix + opt_suffix + file_suffix)
for file_suffix in file_suffixes
}
opt_profile_renames = {(base_model_name + file_suffix):
(built_name_prefix + opt_suffix + file_suffix)
for file_suffix in file_suffixes}
builders_and_rename_maps = [(no_opt_profile_builder,
no_opt_profile_renames),
(opt_profile_builder, opt_profile_renames)]
if self.include_exercise_models:
builders_and_rename_maps.extend([
(no_opt_noprofile_builder, no_opt_noprofile_renames),
(opt_noprofile_builder, opt_noprofile_renames)
])
target_files = []
for builder_and_rename_map in builders_and_rename_maps:
target_files.extend([
builder_and_rename_map[1][filename]
for filename in builder_and_rename_map[1]
])
existing_files = os.listdir(self.profiler_data.built_profilers_path)
need_to_build = False in [
filename in existing_files for filename in target_files
]
if need_to_build:
try:
for builder_and_rename_map in builders_and_rename_maps:
builder = builder_and_rename_map[0]
rename_map = builder_and_rename_map[1]
builder.run()
for filename in rename_map:
src_path = os.path.join(
self.profiler_data.built_profilers_path, filename)
dst_path = os.path.join(
self.profiler_data.built_profilers_path,
rename_map[filename])
shutil.copy(src_path, dst_path)
except:
errorType, value, traceback = sys.exc_info()
msg = "### WrapException: %s: %s" % (str(errorType),
str(value))
global_logger.error(msg)
sys.exit(1)
else:
global_logger.info(
"Target files already exist in path {}, skipping build".format(
self.profiler_data.built_profilers_path))
keep_files = target_files
# Copy profile source code into place
for filename in profile_src_files:
src_path = os.path.join(profile_src_path, filename)
dst_name = profile_src_file_renames[
filename] if filename in profile_src_file_renames else filename
dst_path = os.path.join(self.profiler_data.built_profilers_path,
dst_name)
keep_files.append(dst_name)
shutil.copy(src_path, dst_path)
all_built_files = os.listdir(self.profiler_data.built_profilers_path)
delete_files = [
filename for filename in all_built_files
if filename not in keep_files
]
for filename in delete_files:
full_path = os.path.join(self.profiler_data.built_profilers_path,
filename)
if os.path.isfile(full_path):
global_logger.info(
"Cleaning temporary build file {}".format(full_path))
os.remove(full_path)
global_logger.info("Done compiling model {}.".format(model_id_string))
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
fromfile_prefix_chars='@')
parser.add_argument('--temp', type=str, default='/Volumes/RAM Disk/',
help='temporary filesystem')
parser.add_argument('--no-s3', action='store_true', default=True,
help='do not modify S3 objects')
parser.add_argument('--no-fs', action='store_true', default=True,
help='do not modify source files')
parser.add_argument('--workers', type=int, default=2, metavar='#',
help='sorta like threads')
parser.add_argument('--debug', type=str, default='DEBUG', metavar='LEVEL',
help='debug level or module1=level,module2=level')
parser.add_argument('files', nargs='*', metavar='FILE',
help='filenames to process')
if 'xterm' in os.environ.get('TERM', ''):
print "\033]2;{}\007".format(title)
def debug_level(option):
if '=' in option:
return option.split('=')
if option.upper() in ('DEBUG', 'INFO', 'WARN', 'ERROR'):
return ('', option)
return (option, 'DEBUG')
cliargs = parser.parse_args()
debuglevels = dict([debug_level(level) for level in cliargs.debug.split(',')])
logger.setup(debuglevels.get('', 'WARN'), **debuglevels)
main(**cliargs.__dict__)
APP_PATH = construct_path(sys.argv[2])
# Install settings if not yet installed
if not os.path.exists(CONFIG_PATH + 'bot_settings.py'):
shutil.move(os.path.join(APP_PATH, 'settings.py'),
CONFIG_PATH + 'bot_settings.py')
print('Edit ' + CONFIG_PATH + 'bot_settings.py with you settings.')
exit(1)
sys.path.insert(0, CONFIG_PATH)
# Set up logger
LOG_PATH = os.path.join(CONFIG_PATH, 'logs/')
if not os.path.exists(LOG_PATH):
os.makedirs(LOG_PATH)
import logger
logger.setup(LOG_PATH)
# local imports
import bot_settings as settings
from db_manager import DbManager
# Create twitter instance
twitter = get_twython_instance(settings.API)
# Initialize the Database
following = twitter.get_friends_ids(
screen_name=settings.API['TWITTER_HANDLE'])['ids']
db = DbManager(following, CONFIG_PATH)
unfollow_users(twitter, db.delete_user_check())
while (1):
def main(argv):
requests.packages.urllib3.disable_warnings()
global loggerMigrator
loggerMigrator = logger.setup('migrator', 'myapp.log')
parser = argparse.ArgumentParser()
parser.add_argument("apikey", type=str, help="Mashery V2 API Key")
parser.add_argument("secret", type=str, help="Mashery V2 API Secret")
parser.add_argument("env", type=str, help="Is this a production area or a sandbox area")
parser.add_argument("area_name", type=str, help="Mashery Area Name")
parser.add_argument("config_file", type=str, help="Full path to config file containing json blobs")
parser.add_argument("file_with_service_ids", type=str, help="Full path to file containging list of service ids for which to create packages", nargs='?')
args = parser.parse_args()
apikey = args.apikey
secret = args.secret
env = args.env
area_name = args.area_name
config_file = args.config_file
print 'APIKey: ' + apikey
print 'Secret: ' + secret
print 'Production? ' + env
print "Area: " + area_name
service_ids = []
if args.file_with_service_ids != None: # adding capability to pass in a list of service ids for which packages should be created
service_ids = [line.rstrip() for line in open(args.file_with_service_ids)]
apis = fetch(env, area_name, apikey, secret, 'service_definitions', '*, service, service_definition_endpoints, service.service_classes, service.service_classes.developer_class', '')
api_configs = fetch_service_configs(config_file)
packages = fetch(env, area_name, apikey, secret, 'packages', '*, plans', '')
email_template_sets = fetch(env, area_name, apikey, secret, 'email_template_sets', '*', '')
for api in apis:
if args.file_with_service_ids != None: # checking to see if passed in service ids
if api['service_key'] not in service_ids: # if passed in, then check to see if the api is in the list, if not, skipe
print "Skipping " + api['service_key']
continue
print api['service_key']
api_config = []
if api['service_key'] in api_configs:
api_config = api_configs[api['service_key']]
else:
print ""
print "Need to create the package for this service manually: "
print api
print ""
continue
package = getPackageForService(api, packages)
if (package != None):
print 'Deleting Package...' + package['name']
delete(env, area_name, apikey, secret, 'package', package)
email_template_set = getEmailTemplateForService(api, email_template_sets)
if (email_template_set != None):
print 'Deleting Email Template Set..' + email_template_set['name']
delete(env, area_name, apikey, secret, 'email_template_set', email_template_set)
email_template_set = getEmailTemplate(api_config)
created_email_template_set = create(env, area_name, apikey, secret, 'email_template_set', email_template_set)
package = buildPackagedApi(api, api_config['notifications'])
print 'Creating Package...' + package['name']
created_package = create(env, area_name, apikey, secret, 'package', package)
created_plans = fetch(env, area_name, apikey, secret, 'plans', '*', 'REQUIRE RELATED package WITH id = ' + str(created_package['result']['id']))
for plan in created_plans:
if created_email_template_set != None:
email_template_set_t = {}
email_template_set_t['id'] = created_email_template_set['result']['id']
plan['email_template_set'] = email_template_set_t
update(env, area_name, apikey, secret, 'plan', plan)
plan_service = buildPlanService(api, plan)
created_plan_service = create(env, area_name, apikey, secret, 'plan_service', plan_service)
plan_endpoints = buildPlanEndpoints(api, plan)
created_plan_endpoints = create(env, area_name, apikey, secret, 'plan_endpoint', plan_endpoints)
return
#!/usr/bin/env python
import os
import time
from PIL import Image
from ConfigParser import NoOptionError
import s3
import server
import settings
import db
import logger
l = logger.setup('hook')
cfg = settings.config()
sender = 0
def generate_key(fileName):
"""
Generate a unique filename based on the current timestamp.
"""
timestamp = str(time.time()).replace(".", "")
file_extension = fileName.split(".")[1]
return timestamp + "." + file_extension
def generate_thumb(src, dest):
"""
Create a thumbnail of the file using Pillow and return
the path to the created thumbnail.
"""
####################################################################################################
import sys
import argparse
import os
import shutil
from dask import compute, delayed
import dask.multiprocessing
import optimizer_util
import remoterunner
import picluster
import logger
global_logger = logger.setup()
remote_working_directory = "/home/pi/profilers"
default_platform_regex = {
"pi3": "ARMv7 Processor rev 4 \(v7l\) \(4 cores\) BCM2835",
"pi0": "ARMv6-compatible processor rev 7 \(v6l\) \(1 cores\) BCM2835"
}
class RemoteProfileRunner:
def __init__(self, profiler_data, cluster, fallback_ip_address, username, password, output_path, platform_re=None, ignore_extensions=[".ll", ".bc"], logging_args=None):
self.profiler_data = profiler_data
self.cluster = cluster
self.fallback_ip_address = fallback_ip_address
self.username = username
self.password = password
#!/usr/bin/env python3
"""This is a much more complex server that needs to exist solely to do ZLIB
decompression because we can't do it client-side because lol.
"""
import logger
logger.setup("WebServer", logfile=None)
log = logger.logging.getLogger(__name__)
import io
import os
import os.path
import sys
import time
import json
import asyncio
import aiohttp
from aiohttp import web
import datetime
import subprocess
import zlib
import struct
from texture import ImageFormat, BITS_PER_PIXEL, decode_image, encode_image
class ZlibView(aiohttp.web.View):
"""A view that decompresses ZLB.
Given a request path like: /zlb/path?offset=x&length=y
attempts to decompress the ZLB data at offset `x` with length `y`
from the specified file.
If decompression fails, returns error 500.
def test_write_msg(self):
class target():
pass
err = Exception('Test error')
test_context = {"ctx-key1": 'val1', "ctx-key2": 'val2'}
test_data = {"key1": 'val1', "key2": 'val2'}
now = '2020-01-02T03:04:05Z'
got_messages = []
def mock_transport(message):
got_messages.append(message)
logger.setup(
target=target,
transport=mock_transport,
now_fn=lambda: now,
)
target.error('Hello Error',
context=test_context,
data=test_data,
err=err)
target.warn('Hello Warn',
context=test_context,
data=test_data,
err=err)
target.info('Hello Info',
context=test_context,
data=test_data,
err=err)
target.debug('Hello Debug',
context=test_context,
data=test_data,
err=err)
self.assertEqual(got_messages, [
logger.json_formatter(msg='Hello Error',
level='error',
now=now,
context=test_context,
data=test_data,
err=err),
logger.json_formatter(msg='Hello Warn',
level='warn',
now=now,
context=test_context,
data=test_data,
err=err),
logger.json_formatter(msg='Hello Info',
level='info',
now=now,
context=test_context,
data=test_data,
err=err),
logger.json_formatter(msg='Hello Debug',
level='debug',
now=now,
context=test_context,
data=test_data,
err=err),
])
import app
import logger
config = app.config.load()
logger.setup(transport=app.setup.logger_transport(config['log']))
server = app.create_server(config)
def startServer():
try:
server.start()
except KeyboardInterrupt as err:
server.stop()
except BaseException as err:
logger.error("Failed to start server\n", err=err)
if config["server"]["enabled"] == True:
startServer()
#!/usr/bin/env python
import json
import s3
import server
import logger
import db
l = logger.setup('background');
sender = 0
def upload_full_image(fileId):
"""
Upload a full-resolution image to Amazon S3.
"""
l.debug("Transferring full-resolution photo %s to Amazon S3.", fileId)
dest = str(sender) + '/full/' + fileId
src = 'full/' + fileId
saved = s3.save(src, dest, verbose=True)
if saved:
l.info("Full-resolution photo %s transferred to Amazon S3.", fileId)
return saved
else:
l.error("Failed to transfer full-resolution photo %s to Amazon S3.", fileId)
return False
def post_to_server(saved, fileId):
"""
Notify photostreamer-server that a new full-resolution photo is available
on Amazon S3.
"""
import os
import json
from glob import glob
from functools import partial # for rawincount function which counts lines in a file
import urllib.parse
from updateSources import update_all_sources
from timeit import default_timer as timer
import logging
import logger
# setup logger
logger.setup(__name__)
log = logging.getLogger(__name__)
# Project Settings
BASEDIR_PATH = os.path.dirname(os.path.realpath(__file__))
def get_default():
return {
"destinationpath": os.path.join(BASEDIR_PATH, "rulegroups"),
"sourcespath": os.path.join(BASEDIR_PATH, "sources"),
"sourcedatafilename": "update.json",
"sources": [],
}
def get_default_hostnames():
return {"localhost", "localhost.localdomain", "local", "broadcasthost", "localhost4",
from Map import Point, Segment, Graph, Stop, Route, Car, Map
from Compute import get_closest_segment_np
from matplotlib import pyplot as plt
import pandas as pd
from datetime import datetime
import numpy as np
import geojson
from dbhelper import DBhelper
import time
import logger
logging = logger.setup('main')
logging.info('Started Main.py')
db = DBhelper()
gps_log = db.get_all_gps()
# test 3
# gps_log = db.get_gps("0TU0007 (3)",
# datetime(2019, 11, 19, 3, 27, 0),
# datetime(2019, 11, 19, 4, 0, 0))
# test 1b
# gps_log = db.get_gps()
# test 4
# gps_log = db.get_gps("0TU0008 (4)",
# datetime(2019, 11, 19, 0, 3, 0),
# datetime(2019, 11, 19, 3, 0, 0))
# test 5 changed from 2
# gps_log = db.get_gps("0TU0021 (2)",
arg_parser.add_argument("--apikey",
help="API key for picluster server",
default=None)
arg_parser.add_argument("--username",
help="Username for logon to remote machine",
default=None)
arg_parser.add_argument("--password",
help="Password for logon to remote machine",
default=None)
arg_parser.add_argument("--command",
help="The command to run on the remote machine",
default=None)
arg_parser.add_argument(
"--timeout",
type=int,
default=300,
help="Timeout for the command in seconds (default 300 seconds)")
logger.add_logging_args(arg_parser)
args = arg_parser.parse_args()
logger.setup(args)
runner = RemoteRunner(ipaddress=args.ipaddress,
cluster=args.cluster,
username=args.username,
password=args.password,
command=args.command,
timeout=args.timeout,
apikey=args.apikey)
runner.run_command()
from dbhelper import DBhelper
from Compute import get_closest_segment_np
from Map import Point, Segment, Graph, Stop, Route, Car, Map
import pandas as pd
import numpy as np
import multiprocessing as mp
import time
import logger
logging = logger.setup('build-map')
logging.info('START build-map.py')
st = time.time()
db = DBhelper()
# create Graph and build
G = Graph()
G.load()
# create MAP
M = Map()
M.set_graph(G)
# build Route
route_no_list = ['1a', '1a_b', '1b', '2', '3', '4', '5']
for i in route_no_list:
route_df = pd.read_csv(f'data/route-{i}.csv')
route = Route(f'route-{i}')
# load route
for index, way in route_df.iterrows():
route.add_way(G, way['from'], way['to'], way['stop'], way['dist'],
way['change'])
from kafka import KafkaConsumer, KafkaProducer
import time
from Map import Map, Car
import pandas as pd
import re
import logger
logging = logger.setup('estimator')
logging.info('Started estimator.py')
# load Map
M = Map()
M.load()
import json
def init_car(cid):
match = re.match(r'.*TU\d{2}(\d{2}).*\((.{1,2})\).*', cid)
if not match:
# logging.error(f"wrong cid format {cid}")
return
rid = f'route-{match.group(2).lower()}'
r = M.R[rid]
car = Car(cid, r, match.group(1))
M.add_car(car)
logging.info(f'init new car {cid}')
def format_processed_data(car):
# [alias, route_id, lat, lon, direction, status]
import numpy as np
import logger as log
import logging
from enum import Enum
from pathlib import Path
import tkinter as tk
logger = log.setup(name=str(Path(__file__).stem))
logger.setLevel(logging.DEBUG)
class CubeVals(Enum):
"""
Each CubeVal contains it's RGB values
"""
RED = (255, 0, 0)
GREEN = (0, 255, 0)
ORANGE = (255, 153, 51)
BLUE = (0, 0, 255)
YELLOW = (255, 255, 0)
WHITE = (255, 255, 255)
class Face:
def __init__(self, ndarray: np.ndarray = None):
if ndarray is None:
self.values = np.zeros((3, 3))
else:
if not isinstance(ndarray[0][0], CubeVals) or ndarray.shape != (3, 3):
raise ValueError("Face class takes a 3x3 array of CubeVals")
else:
def main():
home_dir = os.path.expanduser("~")
parser = argparse.ArgumentParser()
parser.add_argument("--package_directory",
"-p",
default=os.path.join(home_dir, ".litepkg/packages"),
help="Set directory for package installer directory." +
" (default: ~/.litepkg/packages)")
parser.add_argument("--artifacts_directory",
"-a",
default=os.path.join(home_dir, ".litepkg/artifacts"),
help="Directory for any artifacts. (default: " +
"~/.litepkg/artifacts")
parser.add_argument("--binaries_directory",
"-b",
default=os.path.join(home_dir, "bin"),
help="Directory where executables should be " +
"symlinked. (default: ~/bin)")
parser.add_argument("--console_log_level",
"-l",
default="INFO",
help="Log level for console output." +
" (default: INFO)")
parser.add_argument("--file_log_level",
"-L",
default="DEBUG",
help="Log level for log file." + " (default: DEBUG)")
parser.add_argument("--log_file",
"-f",
default="/dev/null",
help="Log file path. (default: /dev/null)")
parser.add_argument("--verbose",
"-v",
default=False,
action="store_true",
help="Show verbose output on the console, implies" +
" --console_log_level DEBUG.")
parser.add_argument("verb", help="litepkg verb")
parser.add_argument("targets", nargs="*", help="verb targets")
config.args = parser.parse_args()
config.start_cwd = os.getcwd()
if config.args.verbose:
config.args.console_log_level = "DEBUG"
logger.setup(logPath=config.args.log_file,
fileLevel=config.args.file_log_level,
consoleLevel=config.args.console_log_level)
logging.debug("Application started.")
logger.prettyLog(config.args, msg="args")
config.args.package_directory = \
os.path.realpath(config.args.package_directory)
config.args.artifacts_directory= \
os.path.realpath(config.args.artifacts_directory)
config.args.binaries_directory= \
os.path.realpath(config.args.binaries_directory)
validate_dirs()
handle_verb()
#!/usr/bin/env python
import logger
logger.setup("python-glsl")
log = logger.logging.getLogger(__name__)
import gi
try:
gi.require_version('Gtk', '3.0')
except AttributeError:
raise RuntimeError("python-gobject is not installed")
from gi.repository import Gtk, GLib
from app import MainWindow
from MyRenderer import MyRenderer
win = MainWindow(MyRenderer())
def redraw():
win.queue_draw()
return True
GLib.timeout_add(100, redraw)
Gtk.main() # won't return until quit event (ie window is closed)
print("Bye!")
import ConfigParser
import logger
l = logger.setup(__name__)
def config():
config = ConfigParser.ConfigParser()
try:
config.readfp(open('config.cfg'))
return config
except IOError:
l.exception("Error reading config.cfg file.")
raise
sys.exit("Couldn't find credentials file auth.json")
action = " ".join(
without(
without(argv, "-p"), "--silent"
)
).lower()
# read authentication data
auth_data = json.load(open("auth.json", "r"))
ACCESS_ID = auth_data["access_id"]
SECRET_KEY = auth_data["secret_key"]
settings_filename = "settings.json"
logger.setup()
with open(settings_filename, "r") as f:
lines = f.readlines()
# remove comments in json
lines = map(lambda line: re.sub("/\*.*\*/", "", line), lines)
settings_file_contents = "".join(lines)
settings_raw = json.loads(settings_file_contents)
#settings_log_text = None
# for line in log:
# if line['Activity'] in ['Create', 'Update']:
# settings_log_text = line['Data']
# settings_log_raw = json.loads(settings_log_text) if settings_log_text else settings_raw
# settings_in_log = settings.parse(settings_log_raw)
settings_in_file = settings.parse(settings_raw)
# settings_modified = (action is not "show status" and settings_in_log is not settings_in_file)
# direct message to correct consumer
if module == 'SENSOR_TEMPERATURE':
cf.toggle_fan()
def start_services(q_in, q_out):
try:
Thread(target=ts.setup, args=(q_in, q_out)).start()
except Exception as ex:
logger.write_e(MODULE, "unable to start thread")
sys.exit(-1)
if __name__ == '__main__':
logger.setup()
q_in = []
q_out = []
start_services(q_in, q_out)
logger.write(MODULE, 'booted')
while True:
try:
msg = q_in.pop(0)
handle_reading(msg)
time.sleep(READ_TIMEOUT)
except:
# no readings
pass
parser.add_argument('config_file',
help='Path to darknet configuration file')
parser.add_argument('weights_file', help='Path to darknet weights file')
parser.add_argument(
'-o',
'--output_directory',
help='Path to output directory (default: input weights file directory)'
)
model_options = parser.add_argument_group('model_options')
model_options.add_argument(
"--step_interval",
help="produce a steppable ELL model for a millisecond interval",
type=float)
model_options.add_argument(
"--lag_threshold",
help=
"number of step intervals to fall behind before notifying the caller.\n"
"used when step_interval is set\n",
type=float)
logger.add_logging_args(parser)
args = parser.parse_args()
_logger = logger.setup(args)
parser_args = vars(args)
importer = DarknetImporter(parser_args)
importer.run()
