def __init__(self, conf_file=None, environment=None):
self.log = utils.setupLogging()
self.containers = {}
self.templates = {}
self.state = 'live'
if environment:
self.load(environment)
else:
# If we didn't get an absolute path to a file, look for it in the current directory.
if not conf_file.startswith('/'):
conf_file = os.path.join(os.path.dirname(sys.argv[0]), conf_file)
data = open(conf_file, 'r')
self.config = yaml.load(data)
# On load, order templates into the proper startup sequence
self.start_order = utils.order(self.config['templates'])
import shutil
import sys
import os
import utils
from decoderState import DecoderState
from x86.decoder import X86Decoder
from strategy.linearSweep import LinearSweepDecoder
from strategy.recursiveDescent import RecursiveDescent
# Testers...
#from test import fromUnit
#from test import fromOnline # not needed... blerg!
#from test import fromExample
utils.setupLogging()
def parseArgs():
parser = argparse.ArgumentParser()
#parser.add_argument("x", type=int, help="the base")
#parser.add_argument("y", type=int, help="the exponent")
parser.add_argument("-b",
"--binary",
nargs=1,
help="Disassemble the given binary file.")
parser.add_argument(
"-v",
'''
Created on 22 Nov 2017
@author: martinr
'''
import logging
from utils import setupLogging, versionInfo, getAllConf, getInputData, sparkEnv
setupLogging()
sparkEnv()
from pyspark.sql import SparkSession # @UnresolvedImport
from pyspark.sql.types import StructType, StructField, LongType, DateType, StringType, Row # @UnresolvedImport
spark = SparkSession.builder.appName("SparkTest").master(
"local[50]").getOrCreate()
logger = logging.getLogger(__name__)
logger.info("Starting....")
[logger.info(s) for s in versionInfo(spark).split('\n')]
[logger.info(s) for s in getAllConf(spark).split('\n')]
empSchema = StructType([
StructField("emp_no", LongType(), False),
StructField("birth_date", DateType(), False),
StructField("first_name", StringType(), True),
StructField("last_name", StringType(), True),
StructField("gender", StringType(), True),
StructField("hire_date", DateType(), False)
])
"""
This script collects the premine (for which the regtest keys are publicly
known) into the current wallet, so that sufficient balance is available
for all testing.
"""
import utils
from xayagametest import premine
import argparse
import sys
desc = "Collects the regtest Xaya premine"
parser = argparse.ArgumentParser(description=desc)
parser.add_argument("--xaya_rpc_url",
required=True,
help="JSON-RPC interface of regtest Xaya Core")
parser.add_argument("--address",
default="",
help="can be set to override the target address")
args = parser.parse_args()
logger = utils.setupLogging()
rpc = utils.connectRegtestRpc(args.xaya_rpc_url, logger)
addr = args.address
if addr == "":
addr = None
premine.collect(rpc, addr, logger)
json_txt = json.dumps(out_labels)
out_file = save_img_dir + "/real_output_labels.json"
with open(out_file, 'w') as f:
f.write(json_txt)
def generate(self):
"""
Intialize a generic detectnet data generator class. It finds the filenames for canvas and paste images, and
labels, and splits them into train and validation spilts.
"""
logging.info("Initializing image dataset generator ...")
logging.info("Generating training images.")
self.generateImages(self.canvas_img_files, self.paste_labels,
self.train_img_dir, self.train_label_dir)
logging.info("Finished generating images.")
if __name__ == "__main__":
np.random.seed(long(time.time()))
utils.setupLogging('real_image_gen')
args = processArgs()
gen = RealImageDataGen(args)
gen.initialize()
gen.generate()
"""
Intialize a generic detectnet data generator class. It finds the filenames for canvas and paste images, and
labels, and splits them into train and validation spilts.
"""
logging.info("Initializing image dataset generator ...")
logging.info("Generating training images.")
self.generateImages(self.canvas_train_img_files,
self.paste_train_img_files, self.train_img_dir,
self.train_label_dir)
self.all_paste_files_used = False
logging.info("Generating validation images.")
self.generateImages(self.canvas_val_img_files,
self.paste_val_img_files, self.val_img_dir,
self.val_label_dir)
logging.info("Finished generating images.")
if __name__ == "__main__":
utils.setupLogging('detectnet_data_gen')
args = processArgs()
gen = DetectNetDataGenerator(args)
gen.initialize()
gen.generate()
img[where[0], where[1]] = contour_img[where[0], where[1]]
# utils.showAndWait('streamers', img)
# detector = cv2.SimpleBlobDetector_create()
# keypoints = detector.detect(img2)
# im_with_keypoints = cv2.drawKeypoints(img2, keypoints, np.array([]), (0, 0, 255),
# cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS)
# utils.showAndWait('streamers', im_with_keypoints)
# return contour_img
return img
if __name__ == "__main__":
utils.setupLogging('analyze_shot')
args = processArgs()
if not os.path.exists(args.input_video):
raise RuntimeError("input video does not exist: {}".format(args.input_video))
# Create a VideoCapture object and read from input file
cap = cv2.VideoCapture(args.input_video)
# Check if camera opened successfully
if (cap.isOpened() == False):
raise RuntimeError("video file not opened.")
out_cap = cv2.VideoCapture(0)
out = cv2.VideoWriter(args.output_video, get_video_type(args.output_video), 25, get_dims(out_cap, res))
out_file = save_img_dir + "/sim_output_labels.json"
with open(out_file, 'w') as f:
f.write(json_txt)
def generate(self):
"""
Intialize a generic detectnet data generator class. It finds the filenames for canvas and paste images, and
labels, and splits them into train and validation spilts.
"""
logging.info("Initializing image dataset generator ...")
logging.info("Generating training images.")
self.generateImages(self.canvas_img_files, self.paste_img_files,
self.paste_label_dir, self.train_img_dir,
self.train_label_dir)
logging.info("Finished generating images.")
if __name__ == "__main__":
np.random.seed(long(time.time()))
utils.setupLogging('sim_image_gen')
args = processArgs()
gen = SimImageDataGen(args)
gen.initialize()
gen.generate()
5:(final_mask.shape[1] - 5)]
# utils.showAndWait('final_mask', final_mask)
# cv2.imwrite(self.save_dir + '/final_mask.png', final_mask)
# Save the new label image.
label_out = self.save_dir + '/' + base_name + '_label.png'
logging.info("saving label file {}".format(label_out))
cv2.imwrite(label_out, final_mask)
def generate(self):
for label_file in self.label_files:
# label_file = '/media/dcofer/Ubuntu_Data/drone_images/drones/30_a.png'
logging.info("processing {}".format(label_file))
orig_filename = os.path.basename(label_file)
orig_basename = os.path.splitext(orig_filename)[0]
self.generateLabelFile(label_file, orig_basename)
if __name__ == "__main__":
utils.setupLogging('segment_data_gen')
args = processArgs()
gen = SimImageDataGen(args)
gen.initialize()
gen.generate()
help='Data dir containing images.')
parser.add_argument('--label_dir',
type=str,
required=True,
help='dir where labels are stored.')
parser.add_argument('--output_file',
type=str,
required=True,
help='sloth json file that will be generated.')
args, unknown = parser.parse_known_args()
return args
if __name__ == "__main__":
utils.setupLogging('yolo_to_sloth')
args = processArgs()
img_files = utils.findFilesOfType(args.image_dir,
['jpg', 'jpeg']) # 'png',
json_labels = []
for img_file in img_files:
orig_filename = os.path.basename(img_file)
orig_basename = os.path.splitext(orig_filename)[0]
label_file = args.label_dir + '/' + orig_basename + '.txt'
labels = utils.loadYoloLabels(label_file)
def main():
"""
Command line stuff...
"""
try:
utils.setupLogging("koan")
except:
# most likely running RHEL3, where we don't need virt logging anyway
pass
p = opt_parse.OptionParser()
p.add_option("-k", "--kopts",
dest="kopts_override",
help="append additional kernel options")
p.add_option("-l", "--list",
dest="list_items",
help="lists remote items (EX: profiles, systems, or images)")
p.add_option("-v", "--virt",
dest="is_virt",
action="store_true",
help="install new virtual guest")
p.add_option("-V", "--virt-name",
dest="virt_name",
help="use this name for the virtual guest")
p.add_option("-r", "--replace-self",
dest="is_replace",
action="store_true",
help="reinstall this host at next reboot")
p.add_option("-D", "--display",
dest="is_display",
action="store_true",
help="display the configuration stored in cobbler for the given object")
p.add_option("-p", "--profile",
dest="profile",
help="use this cobbler profile")
p.add_option("-y", "--system",
dest="system",
help="use this cobbler system")
p.add_option("-i", "--image",
dest="image",
help="use this cobbler image")
p.add_option("-s", "--server",
dest="server",
help="attach to this cobbler server")
p.add_option("-t", "--port",
dest="port",
help="cobbler xmlrpc port (default 25151)")
p.add_option("-P", "--virt-path",
dest="virt_path",
help="override virt install location")
p.add_option("-T", "--virt-type",
dest="virt_type",
help="override virt install type")
p.add_option("-B", "--virt-bridge",
dest="virt_bridge",
help="override virt bridge")
p.add_option("-n", "--nogfx",
action="store_true",
dest="no_gfx",
help="disable Xen graphics (xenpv,xenfv)")
p.add_option("", "--add-reinstall-entry",
dest="add_reinstall_entry",
action="store_true",
help="when used with --replace-self, just add entry to grub, do not make it the default")
p.add_option("-C", "--livecd",
dest="live_cd",
action="store_true",
help="used by the custom livecd only, not for humans")
(options, args) = p.parse_args()
if not os.getuid() == 0:
print "koan requires root access"
return 3
try:
k = Koan()
k.list_items = options.list_items
k.server = options.server
k.is_virt = options.is_virt
k.is_replace = options.is_replace
k.is_display = options.is_display
k.profile = options.profile
k.system = options.system
k.image = options.image
k.live_cd = options.live_cd
k.virt_path = options.virt_path
k.virt_type = options.virt_type
k.virt_bridge = options.virt_bridge
k.no_gfx = options.no_gfx
k.add_reinstall_entry = options.add_reinstall_entry
k.kopts_override = options.kopts_override
if options.virt_name is not None:
k.virt_name = options.virt_name
if options.port is not None:
k.port = options.port
k.run()
except Exception, e:
(xa, xb, tb) = sys.exc_info()
try:
getattr(e,"from_koan")
print str(e)[1:-1] # nice exception, no traceback needed
except:
print xa
print xb
print string.join(traceback.format_list(traceback.extract_tb(tb)))
return 1
help='json file to use for training.')
parser.add_argument('--val_json',
type=str,
required=True,
help='json file to use for val.')
parser.add_argument('--image_dir',
type=str,
required=True,
help='directory where images are located.')
args, unknown = parser.parse_known_args()
return args
if __name__ == "__main__":
utils.setupLogging('split_json_train_val')
args = processArgs()
train_labels = []
val_labels = []
with open(args.root_json, "r") as read_file:
labels = json.load(read_file)
if len(labels) <= 0:
raise RuntimeError("Labels were empty.")
with open(args.train_list, "r") as read_file:
train_files = read_file.readlines()
scaled_base_file))
if min_file_area > 0:
min_areas.append(min_area)
logging.info("Finished val images.")
avg_min_areas = np.average(min_areas)
std_min_areas = np.std(min_areas)
logging.info(
"Average min area detected: {} +- {} from {} images.".format(
avg_min_areas, std_min_areas, len(min_areas)))
# json_txt = json.dumps(all_labels)
# out_file = self.save_dir + "/real_scaled_labels.json"
# with open(out_file, 'w') as f:
# f.write(json_txt)
if __name__ == "__main__":
np.random.seed(long(time.time()))
utils.setupLogging('analyze_scaled_imgs')
args = processArgs()
gen = AnalyzeScaledImages(args)
gen.initialize()
gen.generate()
def main():
# ====================================================================
# setup logging and gpu
# ====================================================================
utils.setupLogging()
utils.setupGPUenvironment()
if FLAGS.exp == 'celebA':
opts = configs.config_celebA
elif FLAGS.exp == 'celebA_small':
opts = configs.config_celebA_small
elif FLAGS.exp == 'mnist':
opts = configs.config_mnist
elif FLAGS.exp == 'mnist_small':
opts = configs.config_mnist_small
elif FLAGS.exp == 'dsprites':
opts = configs.config_dsprites
elif FLAGS.exp == 'grassli':
opts = configs.config_grassli
elif FLAGS.exp == 'grassli_small':
opts = configs.config_grassli_small
elif FLAGS.exp == 'mog':
opts = configs.config_mog
else:
assert False, 'Unknown experiment configuration'
opts['mode'] = FLAGS.mode
if opts['mode'] == 'test':
assert FLAGS.checkpoint is not None, 'Checkpoint must be provided'
opts['checkpoint'] = FLAGS.checkpoint
if FLAGS.zdim is not None:
opts['zdim'] = FLAGS.zdim
if FLAGS.pz is not None:
opts['pz'] = FLAGS.pz
if FLAGS.lr is not None:
opts['lr'] = FLAGS.lr
if FLAGS.w_aef is not None:
opts['w_aef'] = FLAGS.w_aef
if FLAGS.z_test is not None:
opts['z_test'] = FLAGS.z_test
if FLAGS.lambda_schedule is not None:
opts['lambda_schedule'] = FLAGS.lambda_schedule
if FLAGS.work_dir is not None:
opts['work_dir'] = FLAGS.work_dir
if FLAGS.wae_lambda is not None:
opts['lambda'] = FLAGS.wae_lambda
if FLAGS.enc_noise is not None:
opts['e_noise'] = FLAGS.enc_noise
if opts['verbose']:
logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(message)s')
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(message)s')
utils.create_dir(opts['work_dir'])
utils.create_dir(os.path.join(opts['work_dir'],
'checkpoints'))
if opts['e_noise'] == 'gaussian' and opts['pz'] != 'normal':
assert False, 'Gaussian encoders compatible only with Gaussian prior'
return
# Dumping all the configs to the text file
with utils.o_gfile((opts['work_dir'], 'params.txt'), 'w') as text:
text.write('Parameters:\n')
for key in opts:
text.write('%s : %s\n' % (key, opts[key]))
# Loading the dataset
data = DataHandler(opts)
assert data.num_points >= opts['batch_size'], 'Training set too small'
if opts['mode'] == 'train':
# Creating WAE model
wae = WAE(opts, data.num_points)
# Training WAE
wae.train(data)
elif opts['mode'] == 'test':
# Do something else
improved_wae.improved_sampling(opts)
lemma = word if (lemmas is None or len(lemmas) == 0) else lemmas[0]
res = DICT.query(lemma)
res = {} if res is None else res
res['lemma'] = lemma
res['query'] = word
return res
@app.route('/api/ecdict/status')
def status():
return json.dumps({'status': True }), 200, {'content-type':'application/json'}
@app.route('/api/ecdict/query', methods=['GET', 'POST'])
def tts():
if request.method == 'POST':
words = request.json['words']
else:
words = request.args['words'].split(',')
results = [query(word) for word in words]
res = {}
res['results'] = results
res['ok'] = True
return json.dumps(res), 200, {'content-type':'application/json'}
if __name__ == "__main__":
env = os.environ.get("ENV", default="dev")
setupLogging(logging.INFO)
logging.info(f'use env {env}')
app.run(host='0.0.0.0', debug = env != 'prod')
logging.info(f'start the server done!')
import csv
import random
from nk import sim, model
runConfigs = [
# [sim.SearchMethod.STEEPEST, 2, False],
# [sim.SearchMethod.STEEPEST, 2, False],
# [sim.SearchMethod.STEEPEST, 1, False],
# [sim.SearchMethod.STEEPEST, 1, True],
# [sim.SearchMethod.GREEDY, 2, True],
# [sim.SearchMethod.STEEPEST, 2, False],
# [sim.SearchMethod.GREEDY, 1, False],
[sim.SearchMethod.RANDOMTHENSTEEPEST, 2, True]
]
random.seed(utils.getDefaultSeedObject())
utils.setupLogging();
results = []
resFile = open(settings.RESULTS_CSV_FILE, 'a')
resWriter = csv.writer(resFile)
header = ["N", "K", "Fitness", "SearchMethod", "Distance", "CumulativeDistance", "NumberOfLandscapes", "StdDevFitness", "AttemptedFlips", "AcceptedFlips"]
resWriter.writerow(header)
logger = logging.getLogger(__name__)
landscapes = settings.landscapes
for nVal in settings.nList:
for kVal in settings.kList:
if kVal <= nVal:
if kVal == nVal:
realK = kVal-1
else:
realK = kVal
import shutil
import sys
import os
import utils
from decoderState import DecoderState
from x86.decoder import X86Decoder
from strategy.linearSweep import LinearSweepDecoder
from strategy.recursiveDescent import RecursiveDescent
# Testers...
from test import fromUnit
#from test import fromOnline # not needed... blerg!
from test import fromExample
utils.setupLogging()
def parseArgs():
parser = argparse.ArgumentParser()
#parser.add_argument("x", type=int, help="the base")
#parser.add_argument("y", type=int, help="the exponent")
parser.add_argument("-b", "--binary", nargs=1, help="Disassemble the given binary file.")
parser.add_argument("-v", "--verbosity", action="count", default=0, help="Show verbosity. Add more -v's to show more detail")
parser.add_argument("--recursive-descent", action="store_true", help="Use the recursive descent method. ")
parser.add_argument("--linear-sweep", action="store_true", help="Use the linear sweep method. ")
