def on_guardar_clicked(self, widget):
#print(self.txtNB.get_text())
config = Config()
config.save(self.txtNB.get_text(),self.txtGF.get_text())
delete = Deleter(self.txtNB.get_text().replace(" ","\ "),self.txtGF.get_text().replace(" ","\ "))
delete.tumbaTodo()
return
def __init__(self):
asyncore.dispatcher.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.set_reuse_addr()
self.bind((Config.get("WebServer","listen_address"), Config.getint("WebServer", "port")))
self.listen(5) # maximum number of queued connections
logging.info("Started WebServer on %s:%s" % ((Config.get("WebServer","listen_address"), Config.getint("WebServer", "port"))))
def __init__(self, main_instance):
AbstractRole.__init__(self, main_instance)
self.dialog = Dialog(self)
self.has_run = False
self.shares = {}
self.wm = None
Config.read()
def __init__(self):
logging.debug("Init incubator.")
config = Config()
day_tmp = config.get_day()
q = Queue()
self.q = q
self.pwm_controller = PWMController(q)
self.pwm_controller.start()
self.roller = Roller(q, config.get_roll_intervall())
self.ssr = SSRRegulator()
# self.io_handler = IoHandler()
self.ventilation = Ventilation(q)
self.ventilation.set_point(40)
self.ventilation.start()
self.config = Config()
self._running = True
if day_tmp != 1:
self.start_time = datetime.today() - timedelta(days=day_tmp - 1)
self.roll_time = datetime.today()
else:
self.start_time = datetime.today()
self.roll_time = datetime.today()
def run(self, options):
config = Config()
config.keep_results = options['keep_results']
config.create_diffs = options['create_diffs']
config.update_refs = options['update_refs']
t = Timer()
docs = options['tests']
docs_dir = options['docs_dir']
if len(docs) == 1:
if os.path.isdir(docs[0]):
if docs_dir is None:
docs_dir = docs[0]
if docs_dir == docs[0]:
docs = []
else:
if docs_dir is None:
docs_dir = os.path.dirname(docs[0])
else:
if docs_dir is None:
docs_dir = os.path.commonprefix(docs).rpartition(os.path.sep)[0]
tests = TestRun(docs_dir, options['refs_dir'], options['out_dir'])
status = tests.run_tests(docs)
tests.summary()
get_printer().printout_ln("Tests run in %s" % (t.elapsed_str()))
return status
def test_rfftStftConfig_01():
with make_scope() as session:
layer_name = "stft_layer"
fft_size = 400
frame_size = 400
frame_shift = 160
window = "hanning"
test_input = np.ones((1, 32000, 2), dtype=np.float32)
config = Config()
config.update({
"num_outputs": int(fft_size / 2) + 1 * test_input.shape[2],
"num_inputs": test_input.shape[2],
"network": {
layer_name: {
"class": "multichannel_stft_layer", "frame_shift": frame_shift, "frame_size": frame_size, "window": window, "fft_size": fft_size, "use_rfft": True, "nr_of_channels": 2, "is_output_layer": True}
}})
network = TFNetwork(config=config, train_flag=True)
network.construct_from_dict(config.typed_value("network"))
layer = network.layers[layer_name]
test_output = session.run(layer.output.placeholder, {network.get_extern_data('data').placeholder: test_input})
ref0 = _get_ref_output(test_input, fft_size, frame_size, frame_shift, window, 0, 0)
# np.fft.rfft and tensorflow.python.ops.rfft differ a little bit in their
# results, thus an error margin is allowed in the result
resultDiff = np.abs(test_output[0, 0, 0:(int(fft_size / 2) + 1)] - ref0)
assert np.mean(resultDiff) < 0.02
assert np.max(resultDiff) < 1
pass
def test_complexLinearProjectionLayer():
with make_scope() as session:
n_in, n_out = 514, 128
layer_name = "clp_layer"
config = Config()
config.update({
"num_outputs": n_out,
"num_inputs": n_in,
"network": {
layer_name: {
"class": "complex_linear_projection", "nr_of_filters": n_out, "n_out": n_out, "is_output_layer": True}
}})
network = TFNetwork(config=config, train_flag=True)
network.construct_from_dict(config.typed_value("network"))
layer = network.layers[layer_name]
assert isinstance(layer, ComplexLinearProjectionLayer)
i_r = np.ones((1, n_in // 2))
i_i = np.ones((1, n_in // 2)) * 0.5
test_input = np.expand_dims(np.reshape(np.transpose(
np.reshape(np.concatenate([i_r, i_i], axis=1), (1, 2, 257)), [0, 2, 1]), (1, 514)), 0)
test_clp_kernel = np.ones((2, n_in // 2, 128))
test_clp_output = session.run(
layer.output.placeholder,
feed_dict={network.get_extern_data('data').placeholder: test_input, layer._clp_kernel: test_clp_kernel})
assert test_clp_output[0, 0, 0] - 6.00722122 < 1e-5
def run(argv):
Options.stack.clear()
queue = argv[1:]
while len(queue) > 0:
literal = queue.pop(0)
option = get_option(literal)
if option.__class__ in Options.stack:
stderr.write("Option {} appears twice\n".format(option.name))
exit(2)
if option.requires_argument:
if len(queue) == 0:
stderr.write("Option {} requires an argument\n".format(option.name))
exit(2)
option.argument = queue.pop(0)
Options.stack[option.__class__] = option
if Options.Help in Options.stack:
print_usage()
for option in Options.stack.values():
if option.check() is False:
exit(2)
config = Config(Options.Config.get_config_path())
if not config.load():
exit(1)
UI(config).start()
def main():
init()
config = Config()
if not config.getConfig(sys.argv):
sys.exit()
dt = datetime.now()
fslog = ScanWriter('log' + os.path.sep + dt.strftime('%Y%m%d %H%M%S') + '.log',config.log)
tclient = TravianClient(config, fslog)
if config.ReLogin:
if not tclient.login():
print 'Invalid username or password'
sys.exit()
else:
#cookie check
strHtml = tclient.getKarteZHtml(320801)
if strHtml.find('login') > 0 and strHtml.find(u'用户名:') > 0 and strHtml.find(u'密码:') > 0 :
print 'Cookie time out, relogin needed. Please use -l option or try --help option.'
sys.exit()
fsVillage = ScanWriter('result' + os.path.sep + dt.strftime('%Y%m%d %H%M%S') + 'Village.csv',config.Output[0])
fsFarm = ScanWriter('result' + os.path.sep + dt.strftime('%Y%m%d %H%M%S') + 'Farm.csv',config.Output[1])
fsOasis = ScanWriter('result' + os.path.sep + dt.strftime('%Y%m%d %H%M%S') + 'Oasis.csv',config.Output[2])
fsVillage.write(u'村庄,玩家,居民,联盟,x,y\n')
fsFarm.write(u'伐木场,泥坑,铁矿场,农场,x,y\n')
fsOasis.write(u'老鼠,蜘蛛,野猪,蛇,蝙蝠,狼,熊,鳄鱼,老虎,大象,绿洲类型,x,y\n')
scaner = Scaner(config, tclient, [fsVillage,fsFarm,fsOasis])
scaner.scan()
def execute(self, parameter, chatid):
self.script_buffer=""
if Config.debug:
Config.debugPrint(self.name)
self.__parameter = parameter
awaitingOptions = {}
for script in self.scripts:
if Config.pause:
Config.debugPause()
if script.startswith('<<') or script.startswith('[['):
self.__doScript(chatid, script)
if self.__choices == 2:
if self.script_buffer!="":
question=self.script_buffer
self.script_buffer=""
else:
question="Auswahl："
awaitingOptions = self.__makeChoice(question, chatid)
self.nextName = "awaiting answer"
break
continue
if self.__if[-1]:
if self.script_buffer!="":
self.__delay(chatid)
sendmessage(self.script_buffer,chatid)
self.script_buffer=""
self.script_buffer=script
if self.__jumpNow:
break
if self.script_buffer!="":
self.__delay(chatid)
sendmessage(self.script_buffer,chatid)
return self.nextName, self.__parameter, awaitingOptions
def __init__(self, getStatisticForBeast=None):
"""
constructor --> initialising Variables
@param getStatisticForBeast List: insert a tupel of chars ('a','b')
"""
threading.Thread.__init__(self)
self.server = None
self.gui = None
self.beastObjectMap = {}
self.rankingList = []
self.worldMap = None
self.foodCounter = 0
self.enableUrwidVisualisation = Config.__getUseUrwidVisualisation__()
self.useNetworking = Config.__getUseNetworking__()
self.startTimeMillis = time.time() + Config.__getStartInSeconds__()
self.startTime = time.ctime(self.startTimeMillis)
self.roundCounter = 0
self.getStatisticForBeast = getStatisticForBeast
self.useBeastAnalytics = True if(getStatisticForBeast) else False
self.deadBeasts = 0
self.urwidRoundDelay = Config.__getUrwidRoundDelay__() / 1000.0
self.gameStarted = False
self.gameFinished = False
self.running = False
self.log = logging.getLogger('beast-arena-logging')
if self.enableUrwidVisualisation:
self.useBeastAnalytics = True
self.getStatisticForBeast = ''
self.beastAnalytics = BeastAnalytics() if(self.useBeastAnalytics) else False
def __init__(self):
asynchat.async_chat.__init__(self)
self.data = []
self.logonce = True
self.scheduler = (Config.get("Networking", "listen_address"), Config.getint("Networking", "port"))
self.doConnect()
self.set_terminator("#EOM#")
def onInit(self):
self.dataPath = self.config.getDataDir()
config = Config(self.config.data(), self.dataPath)
self.plots = config.getPlots()
self.graphs = config.getGraphs()
self.interval = self.determineInterval()
log.info('Service started with %s plots, %s graphs, interval %s' % (len(self.plots), len(self.graphs), self.interval))
def stop(self):
env = C.get('env');
host = C.get('rds_host_' + env)
db = C.get('rds_db_' + env)
rds = Redis(host = host, port = 6379, db = db)
tradeChannel = C.get('channel_trade')
rds.publish(tradeChannel, 'stop');
def main():
logging.getLogger().setLevel(logging.DEBUG)
logging.info("Starting...")
config = Config("/etc/pyvision/server.conf")
config.load()
dispatch.AsServer(port=8080)
logging.info("Exiting...")
def test_config1_basic(): config = Config() config.update(config1_dict) desc = LayerNetworkDescription.from_config(config) assert_is_instance(desc.hidden_info, list) assert_equal(len(desc.hidden_info), len(config1_dict["hidden_size"])) assert_equal(desc.num_inputs, config1_dict["num_inputs"])
class Interface:
def __init__(self, busnum=-1, addr=0x20, debug=False):
self.lcd = LCD(busnum, addr, debug)
self.config = Config()
self.lcd.backlight(int(self.config.get('Display', 'Colour')))
self.lcd.clear()
def scytheHome(self):
self.display('<> Scythe L/R\n^v Choose image')
def chooseImage(self, imageName):
output = 'Image chosen:\n' + imageName
self.display(output)
def display(self, text):
self.lcd.display()
self.lcd.backlight(int(self.config.get('Display', 'Colour')))
self.lcd.clear()
self.lcd.message(text)
def buttons(self):
return self.lcd.buttons()
def off(self):
self.lcd.backlight(self.lcd.OFF)
self.lcd.noDisplay()
def test_Updater_add_check_numerics_ops():
class _Layer(DummyLayer):
def _get_loss_value(self):
return tf.log(self.x)
from TFNetwork import TFNetwork, ExternData
from Config import Config
with make_scope() as session:
config = Config()
config.set("debug_add_check_numerics_ops", True)
network = TFNetwork(extern_data=ExternData(), train_flag=True)
network.add_layer(name="output", layer_class=_Layer, initial_value=1.0)
network.initialize_params(session=session)
updater = Updater(config=config, network=network)
updater.set_learning_rate(1.0, session=session)
updater.set_trainable_vars(network.get_trainable_params())
updater.init_optimizer_vars(session=session)
# Should succeed.
session.run(updater.get_optim_op())
# One gradient descent step from ln(x), x = 1.0: gradient is 1.0 / x, thus x - 1.0 = 0.0.
assert_almost_equal(session.run(network.get_default_output_layer().output.placeholder), 0.0)
try:
# Now, should fail.
session.run(updater.get_optim_op())
except tf.errors.InvalidArgumentError as exc:
print("Expected exception: %r" % exc)
else:
assert False, "should have raised an exception"
def create_first_epoch(config_filename): config = Config() config.load_file(config_filename) engine = Engine([]) engine.init_train_from_config(config=config, train_data=None) engine.epoch = 1 engine.save_model(engine.get_epoch_model_filename(), epoch=engine.epoch)
def normalizeall (self,input,output,rangeb,rangea):# Currently disabled
#VERIFICAR WHILE DEBIDO A QUE LOS QUIERIES SON < y no <=
config=Config()
INCREMENT=config.getCronTime()
start=rangeb
end=rangea
join = [[0 for i in range(7)] for j in range(((end-start)/300)-1)]
c1=c2=x=0
start =( start - start%INCREMENT) + INCREMENT # CHANGE TO INCREMENT
while start <end :
if c1<len(input) and input[c1][3]==start :
join[x][0]=input[c1][0]
join[x][2]=input[c1][1]
join[x][4]=input[c1][2]
c1=c1+1
else:
join[x][0]=0
join[x][2]=0
join[x][4]=0
if c2<len(output) and output[c2][3]==start :
join[x][1]=output[c2][0]
join[x][3]=output[c2][1]
join[x][5]=output[c2][2]
c2=c2+1
else:
join[x][1]=0
join[x][3]=0
join[x][5]=0
join[x][6]=start
start=start+INCREMENT
x=x+1
t=tuple(tuple(x) for x in join)
return t
def test_stftConfig_multi_res_02():
with make_scope() as session:
layer_name = "stft_layer"
fft_sizes = [400, 200, 800]
frame_sizes = [400, 200, 800]
frame_shift = 160
window = "hanning"
test_input = np.random.normal(0, 0.6, (1, 3200, 2))
num_outputs = int(np.sum([(int(fft_size / 2) + 1) * test_input.shape[2] for fft_size in fft_sizes]))
config = Config()
config.update({
"num_outputs": num_outputs,
"num_inputs": test_input.shape[2],
"network": {
layer_name: {
"class": "multichannel_multiresolution_stft_layer", "frame_shift": frame_shift, "frame_sizes": frame_sizes, "window": window, "fft_sizes": fft_sizes, "use_rfft": True, "nr_of_channels": 2, "is_output_layer": True}
}})
network = TFNetwork(config=config, train_flag=True)
network.construct_from_dict(config.typed_value("network"))
layer = network.layers[layer_name]
test_output = session.run(layer.output.placeholder, {network.get_extern_data('data').placeholder: test_input})
assert test_output.shape[2] == num_outputs
comparison_frame = 6
ref00 = _get_ref_output_single_res(test_input, fft_sizes[0], frame_sizes[0], frame_shift, window, comparison_frame, 0)
ref01 = _get_ref_output_single_res(test_input, fft_sizes[0], frame_sizes[0], frame_shift, window, comparison_frame, 1)
ref10 = _get_ref_output_single_res(test_input, fft_sizes[1], frame_sizes[1], frame_shift, window, comparison_frame, 0)
ref11 = _get_ref_output_single_res(test_input, fft_sizes[1], frame_sizes[1], frame_shift, window, comparison_frame, 1)
ref20 = _get_ref_output_single_res(test_input, fft_sizes[2], frame_sizes[2], frame_shift, window, comparison_frame, 0)
ref21 = _get_ref_output_single_res(test_input, fft_sizes[2], frame_sizes[2], frame_shift, window, comparison_frame, 1)
ref = np.concatenate([ref00, ref01, ref10, ref11, ref20, ref21], axis=0)
resultDiff = np.abs(test_output[0, comparison_frame, :] - ref)
assert np.mean(resultDiff) < 0.06
assert np.max(resultDiff) < 1
def getSensor(self, sensorID):
"""
Sends a sensor info request to the Live! API with the wanted sensor"s index.
Returns the sensor data received.
"""
from Config import Config
config = Config("configuration.json")
sensorinfo = config.getItem("sensorinfo", "")
for sensor in sensorinfo:
if sensor["id"] == sensorID:
return sensor
# Debug code, uses local database
if sensorID == "174963":
response = {u'ignored': 0, u'protocol': u'oregon', u'name': u'Kasvari - l\xe4mmin puoli', u'editable': 1, u'lastUpdated': 1365097299, u'timezoneoffset': 10800, u'sensorId': u'139', u'data': [{u'name': u'temp', u'value': u'2.3'}, {u'name': u'humidity', u'value': u'87'}], u'id': u'174963', u'clientName': u'Koti'}
return response
elif sensorID == "895892":
response = {u'ignored': 0, u'protocol': u'fineoffset', u'name': u'Kellari', u'editable': 1, u'lastUpdated': 1367497808, u'timezoneoffset': 10800, u'sensorId': u'73', u'data': [{u'name': u'temp', u'value': u'13.1'}], u'id': u'895892', u'clientName': u'Kasvari'}
return response
else:
return {"error": "Sensor doesn't exist"}
return {"error": "Sensor doesn't exist"}
return self.request("sensor/info", {"id": sensorID})
def test_NetworkDescription_to_json_config1():
config = Config()
config.update(config1_dict)
desc = LayerNetworkDescription.from_config(config)
desc_json_content = desc.to_json_content()
pprint(desc_json_content)
assert_in("hidden_0", desc_json_content)
assert_equal(desc_json_content["hidden_0"]["class"], "forward")
assert_in("hidden_1", desc_json_content)
assert_in("output", desc_json_content)
orig_network = LayerNetwork.from_description(desc)
assert_in("hidden_0", orig_network.hidden)
assert_in("hidden_1", orig_network.hidden)
assert_equal(len(orig_network.hidden), 2)
assert_is_instance(orig_network.hidden["hidden_0"], ForwardLayer)
assert_equal(orig_network.hidden["hidden_0"].layer_class, "hidden")
orig_json_content = orig_network.to_json_content()
pprint(orig_json_content)
assert_in("hidden_0", orig_json_content)
assert_equal(orig_json_content["hidden_0"]["class"], "hidden")
assert_in("hidden_1", orig_json_content)
assert_in("output", orig_json_content)
new_network = LayerNetwork.from_json(
desc_json_content,
config1_dict["num_inputs"],
{"classes": (config1_dict["num_outputs"], 1)})
new_json_content = new_network.to_json_content()
if orig_json_content != new_json_content:
print(dict_diff_str(orig_json_content, new_json_content))
assert_equal(orig_json_content, new_network.to_json_content())
def setup_backtest():
""""This function gets all the information from the command line and ini file """
from optparse import OptionParser
p = OptionParser(usage='usage: %prog [options] [START_DATE] [END_DATE]')
p.add_option('-c', '--config', dest='config', metavar='CONFIGFILE', help='Specifies input config file')
p.add_option('-s', '--silent', dest='silent', action='store_true', default=False, help="Specifies if logging to console should be disabled.")
opts,args = p.parse_args()
if opts.config is None:
print >>sys.stderr, "No config file specified. Use --help"
sys.exit(1)
config = Config(opts, args)
start_date = config.get_value('PORTFOLIO', 'startdate')
end_date = config.get_value('PORTFOLIO', 'enddate')
log_file_name = ".".join([str(start_date), str(end_date), log_file_ext])
global stat_file_name
stat_file_name = ".".join([str(start_date), str(end_date), stat_file_ext])
logging.basicConfig(filename=log_file_name, filemode='w') ## file will be overwritten each time
logger = logging.getLogger('backtester')
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler()
console_handler.setLevel(logging.WARNING if config.get_value('BACKTEST', 'silent') else logging.INFO)
logger.addHandler(console_handler)
return config, logger
def test_stftConfig_single_res_01():
with make_scope() as session:
layer_name = "stft_layer"
fft_sizes = [400]
frame_sizes = [400]
frame_shift = 160
window = "hanning"
test_input = np.ones((1, 32000, 2), dtype=np.float32)
num_outputs = (int(fft_sizes[0] / 2) + 1) * test_input.shape[2]
config = Config()
config.update({
"num_outputs": num_outputs,
"num_inputs": test_input.shape[2],
"network": {
layer_name: {
"class": "multichannel_multiresolution_stft_layer", "frame_shift": frame_shift, "frame_sizes": frame_sizes, "window": window, "fft_sizes": fft_sizes, "use_rfft": True, "nr_of_channels": 2, "is_output_layer": True}
}})
network = TFNetwork(config=config, train_flag=True)
network.construct_from_dict(config.typed_value("network"))
layer = network.layers[layer_name]
test_output = session.run(layer.output.placeholder, {network.get_extern_data('data').placeholder: test_input})
ref0 = _get_ref_output_single_res(test_input, fft_sizes[0], frame_sizes[0], frame_shift, window, 0, 0)
resultDiff = np.abs(test_output[0, 0, 0:(int(fft_sizes[0] / 2) + 1)] - ref0)
assert test_output.shape[2] == num_outputs
assert np.mean(resultDiff) < 0.02
assert np.max(resultDiff) < 1
def test_enc_dec1_init(): config = Config() config.load_file(StringIO(config_enc_dec1_json)) network_json = LayerNetwork.json_from_config(config) assert_true(network_json) network = LayerNetwork.from_json_and_config(network_json, config) assert_true(network)
def get_config(pytest_config):
global _config, _config_file_name
if _config is None:
_config_file_name = pytest_config.getoption('config')
print("\nconfig file is {}".format(_config_file_name))
_config = Config(_config_file_name)
_config.read_config()
return _config
class OsmWrapper():
def __init__(self, url=None, api=None):
self.cfg = Config()
if url:
self.url = url
else:
try:
self.url = self.cfg.get("DEFAULT", "api")
except NoOptionError:
self.url = "api.openstreetmap.org"
if not api:
self.api = OsmApi(api=self.url,
username=self.cfg.get("Authentication", "username"),
password=b64decode(self.cfg.get("Authentication", "password")),
appid="Piote/%s" % Piote.version)
else:
self.api = api
def Get(self, obj, id):
if obj == "node":
f = self.api.NodeGet
elif obj == "way":
f = self.api.WayGet
elif obj == "relation":
f = self.api.RelationGet
Piote.data = f(id)
self.data = Piote.data
return Piote.data["tag"]
def Put(self, msg, obj, model, data=None):
changeset = self.api.ChangesetCreate({u"comment": unicode(msg, "utf-8")})
tags = defaultdict(str)
for row in model:
tags[unicode(row[0], "utf-8")] = unicode(row[1], "utf-8")
if data:
data["tag"] = tags
else:
Piote.data["tag"] = tags
if obj == "node":
f = self.api.NodeUpdate
elif obj == "way":
f = self.api.WayUpdate
elif obj == "relation":
f = self.api.RelationUpdate
if data:
result = f(data)
data["version"] = result["version"]
else:
result = f(Piote.data)
Piote.data["version"] = result["version"]
#self.api.ChangesetUpload([{"type":obj, "action":"modify", "data":self.data}])
self.api.ChangesetClose()
def log(msg="", log_type=LOG_SYSTEM, log_level=ERROR, log_file=""):
config = Config()
if config.get("log_record") == "False":
return False
if msg == "":
#捕获异常信息
msg = traceback.format_exc().splitlines()
#写日志
l.write(msg, log_file, log_type, log_level)
def initConfig(self):
cfg = Config()
if not hasattr(cfg, "device"):
cfg.device = u""
if not hasattr(cfg, "datFile"):
cfg.datFile = u""
if not hasattr(cfg, "simulateEmulator"):
cfg.simulateEmulator = False
self.config = cfg
raise ValueError('查询结果和填充参数不匹配，无法转化！')
for i in range(len(result)):
fill_field_value = result[i]
if isinstance(
result[i],
decimal.Decimal) and decimal_type_convert_to == 1:
fill_field_value = float(result[i])
if isinstance(
result[i],
decimal.Decimal) and decimal_type_convert_to == 2:
fill_field_value = int(result[i])
if isinstance(result[i],
datetime.datetime): # 如果数据库查询返回的值为时间类型，则需要转换
fill_field_value = str(result[i])
json_dic.update({fill_field_list[i]: fill_field_value})
result_json_list.append(json_dic)
else:
raise TypeError('查询结果和填充参数必须为列表类型！')
return result_json_list
if __name__ == '__main__':
sql = '''select a.FID,a.FAccount,a.FCompanyID from TAccounts as a where a.FAccount='ttt11' and a.FCompanyID in (select a.FCompanyID from TAccounts as a where a.FAccount='csdqt')'''
data = DB_Util(Config().db_config_name).excute_sql(sql)
data = DB_Util.sql_result_to_json(data, ['id', 'name', 'cid'])
print(data)
# encoding: utf-8
from __future__ import unicode_literals, print_function
import datetime, sys, os
from workflow import Workflow, ICON_WEB, web
from Config import Config
log = None
if __name__ == u"__main__":
wf = Workflow()
log = wf.logger
log.debug("script started")
c = Config()
#routes = c.get_route()
# Get route name as selected from the user
route_name = wf.decode(os.getenv('env_route_name'))
log.debug("Got route name %s" % (route_name))
#Route name is now e.g. Kåk
#utf8_route_name = unicode(route_name, 'utf-8')
#log.debug("Created the utf8 route name %s" % (utf8_route_name))
#log.debug(route_name)
#log.debug(route_name.encode("utf-8"))
from Config import Config, ChromosomeConfig, FunctionParameters
from math import pi
from XmlFile import XmlFileReader
import os
global config
global window
if(os.path.exists('config.xml')):
reader = XmlFileReader('config.xml')
config = reader.getConfig()
else:
fc = FunctionParameters(20, 0.2, 2*pi)
cc = ChromosomeConfig()
config = Config(100, cc)
window = 1
def test_RecLayer_NativeLstm_Nan():
print("test_RecLayer_NativeLstm_Nan()")
print("GPU available:", is_gpu_available())
numpy.set_printoptions(precision=15)
num_inputs = 4
num_outputs = 3
config = Config()
config.update({
"num_inputs": num_inputs,
"num_outputs": {
"data": [num_inputs, 2],
"classes": [num_outputs, 2]
}, # dense output
"network": {
"output": {
"class": "rec",
"unit": "NativeLSTM",
"loss": "mse"
}
},
"adam": True,
"debug_grad_summaries": True,
"debug_save_updater_vars": True,
"debug_add_check_numerics_ops": True,
})
print("Reset default graph...")
tf.reset_default_graph()
print("Create network...")
network = TFNetwork(config=config, train_flag=True)
network.construct_from_dict(config.typed_dict["network"])
# Depending on the seed, I get nan earlier, later, or not at all.
# limit=5.0: seed=3 -> nan in step 4094. seed=1 -> nan in step 2463.
random = numpy.random.RandomState(seed=1)
limit = 10.0 # The higher, the more likely you get nan.
def make_feed_dict(seq_len=10):
return {
network.extern_data.data["data"].placeholder:
random.uniform(-limit, limit, (1, seq_len, num_inputs)),
network.extern_data.data["data"].size_placeholder[0]:
numpy.array([seq_len]),
network.extern_data.data["classes"].placeholder:
random.uniform(-limit, limit, (1, seq_len, num_outputs)),
network.extern_data.data["classes"].size_placeholder[0]:
numpy.array([seq_len]),
}
print("Creating session...")
with tf.Session() as session:
print("Init params...")
network.initialize_params(session=session)
print("Test run...")
output_data1 = session.run(
network.get_default_output_layer().output.placeholder,
feed_dict=make_feed_dict(5))
assert_equal(output_data1.shape,
(5, 1, num_outputs)) # (time, batch, dim)
layer = network.layers["output"]
loss_t = network.get_total_loss(
) * layer.get_loss_normalization_factor()
weights_t = layer.params["W"]
weights_grad_t, = tf.gradients(network.get_objective(), weights_t)
def find_op_by_type(type_name):
for op in session.graph.get_operations():
assert isinstance(op, tf.Operation)
if op.type == type_name:
return op
lstm_grad_op = find_op_by_type("GradOfLstmGenericBase")
assert lstm_grad_op is not None
lstm_grad_ins_t = list(lstm_grad_op.inputs)
lstm_grad_outs_t = list(lstm_grad_op.outputs)
lstm_grad_func = _lstm_grad_op(session=session)
demo_grad_t = lstm_grad_func(*_demo_lstm_grad_args())
demo_grad2_input_placeholders = [
tf.placeholder(v.dtype) for v in lstm_grad_ins_t
]
demo_grad2_t = lstm_grad_func(*demo_grad2_input_placeholders)[1]
print("Create updater...")
from TFUpdater import Updater
updater = Updater(config=config, network=network, tf_session=session)
updater.set_trainable_vars(network.get_trainable_params())
updater.set_learning_rate(0.1)
optim_op = updater.get_optim_op()
assert isinstance(updater.optimizer, tf.train.AdamOptimizer)
adam_weights_m_t = updater.optimizer.get_slot(var=weights_t, name="m")
adam_weights_v_t = updater.optimizer.get_slot(var=weights_t, name="v")
assert isinstance(adam_weights_m_t, tf.Variable)
assert isinstance(adam_weights_v_t, tf.Variable)
summaries_t = tf.summary.merge_all()
# https://github.com/tensorflow/tensorflow/blob/03beb65cecbc1e49ea477bca7f54543134b31d53/tensorflow/core/kernels/training_ops_gpu.cu.cc
adam_update_t = adam_weights_m_t / (tf.sqrt(adam_weights_v_t) + 1e-8)
import tempfile
tmp_tf_logdir = tempfile.mkdtemp("tmp-tf-log")
print("Write TF logs to:", tmp_tf_logdir)
writer = tf.summary.FileWriter(tmp_tf_logdir)
writer.add_graph(session.graph)
print("Training...")
recent_info = [] # type: list[dict[str]]
for i in range(10000):
feed_dict = make_feed_dict(5)
weights_grad, lstm_grad_ins, lstm_grad_outs = session.run(
[weights_grad_t, lstm_grad_ins_t, lstm_grad_outs_t],
feed_dict=feed_dict)
try:
if not numpy.all(numpy.isfinite(weights_grad)):
raise Exception("weights_grad has inf or nan.")
loss, _opt, summaries, weights, adam_update = session.run(
[loss_t, optim_op, summaries_t, weights_t, adam_update_t],
feed_dict=feed_dict)
except Exception as exc:
print("Exception in step %i." % i)
print(exc)
print("Most recent summaries:")
summary_proto = tf.Summary()
summary_proto.ParseFromString(recent_info[-1]["summaries"])
for val in summary_proto.value:
# Assuming all summaries are scalars.
print(" %s: %r" % (val.tag, val.simple_value))
print("Most recent weights:")
print(recent_info[-1]["weights"])
print("Current weights:")
print(session.run(weights_t))
print("Most recent Adam update:")
print(recent_info[-1]["adam_update"])
print("Current Adam update:")
print(session.run(adam_update_t))
print("Used weights grad:")
print(weights_grad)
print("GradOfLstmGenericBase inputs:")
for t, v in zip(lstm_grad_ins_t, lstm_grad_ins):
print("%r:" % t)
print(repr(v))
print("GradOfLstmGenericBase outputs:")
for t, v in zip(lstm_grad_outs_t, lstm_grad_outs):
print("%r:" % t)
print(repr(v))
print("Demo grad:")
print(session.run(demo_grad_t))
print("Demo grad2:")
print(
session.run(
demo_grad2_t,
feed_dict={
k: v
for (k, v) in zip(demo_grad2_input_placeholders,
lstm_grad_ins)
}))
print("Demo grad2 via eval:")
print(
session.run(
demo_grad2_t,
feed_dict={
k: eval(repr(v), vars(numpy))
for (k, v) in zip(demo_grad2_input_placeholders,
lstm_grad_ins)
}))
print("Demo grad2 via args:")
print(
session.run(
demo_grad2_t,
feed_dict={
k: v
for (k, v) in zip(demo_grad2_input_placeholders,
_demo_lstm_grad_args())
}))
raise Exception("Exception in step %i." % i)
writer.add_summary(summaries, global_step=i)
if len(recent_info) > 1000:
recent_info.pop(0)
recent_info.append({
"step": i,
"loss": loss,
"summaries": summaries,
"weights": weights,
"adam_update": adam_update
})
if not numpy.isfinite(loss) or i % 100 == 0:
print("step %i, loss: %r" % (i, loss))
assert numpy.isfinite(loss)
print("Done.")
import shutil
shutil.rmtree(tmp_tf_logdir)
def btc_analyzer_cli(start_date, last_date):
"""
Blockchain analyzer is a tool for parse a Bitcoin blocks and track a specific transaction.
"""
conf = Config()
# root folder path.
root_folder = path.dirname(path.dirname(__file__))
# path to log file
log_path = "{}/log".format(root_folder)
# create log folder if not exists.
if not path.exists(log_path):
makedirs(log_path)
# cofigure logger
conf_logger(log_path=log_path, file_name="blockchain_blocks_analyzer")
logger = logging.getLogger("blockchain_blocks_analyzer")
if not (start_date and last_date):
click.echo("start date and last date must be specify.")
return
logger.info("starting btc-analyzer.")
# swap dates if start date is bigger.
if start_date > last_date:
start_date, last_date = last_date, start_date
# get all date range in a list
logger.info("calculating dates.")
num_days = last_date - start_date
dates = [
start_date + timedelta(days=d) for d in tqdm(range(0, num_days.days))
]
# request all the blocks between the range date.
logger.info("fetching all blocks hash by given date.")
blocks_hash = []
for date in tqdm(dates):
try:
blocks = get_blocks_by_date(time_mils=to_millis(date))
except json.decoder.JSONDecodeError:
logger.error('json decode for date: {} failed.'.format(date))
continue
blocks_hash.extend(list(map(lambda b: b['hash'], blocks['blocks'])))
# parse transaction.
logger.info("fetching all transaction by given block.")
trxs = []
for block_hash in blocks_hash:
try:
trxs.extend(generate_docs(get_single_block(block_hash=block_hash)))
except json.decoder.JSONDecodeError:
logger.error('json decode for hash: {} failed.'.format(block_hash))
continue
if len(trxs) >= 50:
# upload date to elasticsearch
logger.info("uploading transactions so far to elasticsearch.")
add_index(es_url=conf.elasticsearch_url,
body=trxs,
index_name='btc_transactions',
doc_type='btc_trx')
trxs.clear()
# upload date to elasticsearch
logger.info("uploading all transaction to elasticsearch.")
add_index(es_url=conf.elasticsearch_url,
body=trxs,
index_name='btc_transactions',
doc_type='btc_trx')
# track algorithm, after specific transaction.
pass
import os
from selenium import webdriver
from Config import Config
#from selenium.webdriver.support.ui import WebDriverWait
import time
#PATHCHROMEDRIVER = "/usr/local/bin/chromedriver"
#URL = 'https://phptravels.com/demo/'
#URL = 'http://192.168.11.186/Migracion-Web/admin/login'
#**********************************************************************
#Config:
config = Config()
URL = config.getUrl()
PATHCHROMEDRIVER = config.getPathChromeDriver()
print(URL)
print(PATHCHROMEDRIVER)
chromedriver = PATHCHROMEDRIVER
os.environ["webdriver.chrome.driver"] = chromedriver
driver = webdriver.Chrome(chromedriver)
driver.get(URL)
#**********************************************************************
#Admin Login:
#Ingresa Datos en Formulario Login (username, password)
driver.find_element_by_id("username").send_keys("zweicom")
def __init__(self, machine_status: MachineStatus,
pulse_transmitter: PulseTransmitter):
super().__init__(machine_status, pulse_transmitter)
self.input_pin = Config.get_config_variable('extruder_input_pin')
self.pulse_transmitter.register_receiver(self.transmit_pulse)
from contributors import getContributorList
from logging.handlers import RotatingFileHandler
from Config import Config
from Constants import *
from flask import Flask
from flask import request
import json
import logging
import os
import requests
app = Flask(__name__)
config = Config('app.properties')
username = config.getProperty('DEFAULT', 'username')
password = config.getProperty('DEFAULT', 'password')
message = config.getProperty('comments', 'message')
newUsersOnly = config.getBoolean('DEFAULT', 'new-users')
postPREndPoint = config.getProperty('DEFAULT', 'end-point')
@app.route(postPREndPoint, methods=["POST"])
def postPRCommentToNewContributor():
pullRequestEvent = json.loads(request.data)
# Only comment on PRs that have just been opened
if (pullRequestEvent.has_key(ACTION)
and pullRequestEvent[ACTION] == OPENED):
pullRequest = pullRequestEvent[PULL_REQUEST]
def __init__(self, dry=False, debug=False):
self.phase = STEP_END
self.mc = MultiCommand(dry=dry, debug=debug)
self.branches = Config().getBranches()
def info():
logger = Log('info')
config = Config()
def dump_external_info(title, args):
logger.info('')
logger.info(f'*****{title}******')
p = subprocess.Popen(args, stdout=subprocess.PIPE)
out, err = p.communicate()
lines = filter(lambda x: x.strip(), out.decode("utf-8").splitlines())
for l in lines:
logger.info(l)
def dump_class(title, clazz):
logger.info('')
logger.info(f'*****{title}******')
for attr in clazz.__dict__:
if attr.startswith('_'):
logger.info('%s = %r' % (attr[1:], getattr(config, attr)))
def dump_dict(title, dic):
logger.info('')
logger.info(f'*****{title}******')
for x, y in dic.items():
logger.info('%s = %r' % (x, y))
logger.info('*****COMPONENT******')
name = os.environ.get('C_NAME')
version = os.environ.get('C_VERSION')
logger.info(f'Name: {name}')
logger.info(f'Version: {version}')
dump_class('*****CONFIG******', config)
dump_dict(
'*****SYSTEM******', {
'platform': platform.system(),
'release': platform.release(),
'type': platform.uname().system,
'arch': platform.uname().machine,
'cpus': json.dumps(os.cpu_count()),
'hostname': socket.gethostname()
})
dump_dict('*****ENVIROMENT VARS******', dict(os.environ))
dump_dict(
'*****PYTHON******', {
'python_version': platform.python_version(),
'python_build': platform.python_build(),
'python_revision': platform.python_revision(),
'python_compiler': platform.python_compiler(),
'python_branch': platform.python_branch(),
'python_implementation': platform.python_implementation()
})
dump_external_info('TERRAFORM VERSION', ['terraform', '--version'])
dump_external_info('AZURECLI VERSION', ['az', '--version'])
dump_external_info('ANSIBLE VERSION', ['ansible', '--version'])
dump_external_info('ANSIBLE CONFIG', ['ansible-config', 'dump'])
dump_external_info('ANSIBLE-VAULT VERSION', ['ansible-vault', '--version'])
return 0
import re
import os
import types
import csv
import time
import pickle
import numpy as np
import pandas as pd
import random
from Config import Config
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
np.random.seed(1337)
config = Config()
def getEmbedding(emb_file, char2id):
emb_dic = {}
with open(emb_file, 'rb') as f:
for line in f.readlines():
line = line.rstrip().decode('utf-8')
line_list = line.split('\t')
key = line_list[0]
line_list.pop(0)
for i in xrange(len(line_list)):
line_list[i] = float(line_list[i])
emb_dic[key] = line_list
embedding_matrix = np.zeros(
(len(char2id.keys()), config.model_para['input_dim']))
def __init__(self):
self.config = Config.Config()
self.log = Log.MyLog()
while not exists("project_ui_elements.png"):
wait(1)
if exists("add_first_person_dialog.png"):
click(find(Pattern("cancel_button.png").similar(0.67)))
wait(2)
app.switchingProject()
wait(2)
if exists("project_ui_elements.png"):
print "Switching between projects was successful"
else:
print "Something went wrong...Switching project was failed"
app.closeApp()
base_cases.BaseMyHeritageTestCase.cleanUserData()
Config.init()
suite = unittest.TestSuite()
suite.addTest(unittest.TestLoader().loadTestsFromTestCase(SwitchingProjectTestCase))
#suite.addTest(unittest.TestLoader().loadTestsFromTestCase(sikuli1.CreatingProjectTest))
outfile = open(Config.get_reports_path() + "/%s.html" % (project_name), "w")
runner = HTMLTestRunner.HTMLTestRunner(stream=outfile, title=' Report Title', description='desc..' )
runner.run(suite)
outfile.close()
def setUp(self):
self.conf = Config.get_instance()
def do_lpwd(self, line):
print self.fs.pwd()
def do_lls(self, line):
if len(line.strip()) > 0:
res = self.fs.ls_native2(line.split(" "))
else:
res = self.fs.ls_native2()
for l in res:
print l
if __name__ == "__main__":
rundir = os.path.dirname(sys.argv[0])
basedir = os.path.abspath(rundir) + "/"
config = Config.conf_to_dict(basedir + "/config.txt")
config["BASEDIR"] = basedir
# replace relative path'
for key in config:
if key.startswith("PATH_"):
config[key] = os.path.abspath(config["BASEDIR"] + config[key])
try:
dev_file_in_path = config["HID_RAW_DEV"]
dev_file_out_path = config["HID_RAW_DEV"]
HIDin_file = open(dev_file_in_path, "rb")
HIDout_file = open(dev_file_out_path, "wb")
# the linklayer starts several communication threads
class Indexer:
"""
Class to create, store and load an inverted index
"""
def __init__(self, new_config):
"""
Namespace config: Arguments passed on the command line
"""
self.root_dir = os.path.dirname(
os.path.dirname(os.path.abspath(__file__)))
stored_config = self.get_config(new_config)
stored_config.update(vars(new_config))
self.config = Config(**stored_config)
def get_config(self, params):
"""
Returns the configuration from the disk (if it exists), otherwise an empty dict
Namespace params: Configuration params from command line
"""
config = {}
try:
with open(
self.root_dir + '/' + params.index_dir + '/' +
params.config_file_name, 'r') as f:
# Load config from disk
config = json.load(f)
except:
# Return empty dictionary
config = {}
return config
def load_data(self):
"""
Loads a data file from the disk for any extension, currently only .json is supported
"""
data_file = self.root_dir + '/' + self.config.data_dir + \
'/' + self.config.data_file_name
with open(data_file, 'r') as f:
root, ext = os.path.splitext(data_file)
# Check if file extension is .json
if ext == '.json':
# Load json data from file and return
return json.load(f)
def create_inverted_index(self, compressed):
"""
Creates and returns an inverted index
bool compressed: Flag to choose between a compressed / uncompressed index
"""
inverted_index = InvertedIndex(self.config, compressed)
data = self.load_data()
doc_id = -1
for scene in data['corpus']:
doc_id += 1
scene_text = scene['text']
# Filter None removes empty strings from the list after the split on space
terms = list(filter(None, scene_text.split()))
doc_meta = {
'playId': scene['playId'],
'sceneId': scene['sceneId'],
'sceneNum': scene['sceneNum'],
'sceneLength': len(terms)
}
inverted_index.update_docs_meta(doc_id, doc_meta)
inverted_index.update_collection_stats(
doc_length=doc_meta['sceneLength'])
for position, term in enumerate(terms):
inverted_index.update_map(term, doc_id, position)
inverted_index.update_collection_stats(average_length=True)
inverted_index.load_vocabulary()
return inverted_index
def get_inverted_index(self, compressed):
"""
Loads an inverted index from file or calls the create method if it doesn't exist
bool compressed: Flag to choose between a compressed / uncompressed index
"""
inverted_index = None
try:
with open(
self.root_dir + '/' + self.config.index_dir + '/' +
self.config.collection_stats_file_name,
'rb') as collection_stats_file:
with open(
self.root_dir + '/' + self.config.index_dir + '/' +
self.config.docs_meta_file_name,
'rb') as docs_meta_file:
if not compressed:
with open(
self.root_dir + '/' + self.config.index_dir +
'/' + self.config.uncompressed_dir + '/' +
self.config.lookup_table_file_name,
'r') as lookup_table_file:
with open(
self.root_dir + '/' +
self.config.index_dir + '/' +
self.config.uncompressed_dir + '/' +
self.config.inverted_lists_file_name,
'rb') as inverted_lists_file:
# Load lookup table, docs meta info and inverted lists(if in_memory is True) from uncompressed version on disk
inverted_index = self.load_inverted_index_in_memory(
collection_stats_file, docs_meta_file,
lookup_table_file, inverted_lists_file,
False)
if compressed:
with open(
self.root_dir + '/' + self.config.index_dir +
'/' + self.config.compressed_dir + '/' +
self.config.lookup_table_file_name,
'r') as lookup_table_file:
with open(
self.root_dir + '/' +
self.config.index_dir + '/' +
self.config.compressed_dir + '/' +
self.config.inverted_lists_file_name,
'rb') as inverted_lists_file:
# Load lookup table, docs meta info and inverted lists(if in_memory is True) from compressed version on disk
inverted_index = self.load_inverted_index_in_memory(
collection_stats_file, docs_meta_file,
lookup_table_file, inverted_lists_file,
True)
except Exception as e:
# Create inverted index
inverted_index = self.create_inverted_index(compressed)
self.dump_inverted_index_to_disk(inverted_index)
if not self.config.in_memory:
self.remove_inverted_index_from_memory(inverted_index)
return inverted_index
def load_inverted_index_in_memory(self, collection_stats_file,
docs_meta_file, lookup_table_file,
inverted_lists_file, compressed):
"""
Loads an inverted index in memory, inverted lists are not loaded by default
buffer collection_stats_file: Buffer for the collection stats file
buffer docs_meta_file: Buffer for the docs meta file
buffer lookup_table_file: Buffer for the lookup table file
buffer inverted_lists_file: Buffer for the inverted lists file
"""
inverted_index = InvertedIndex(self.config, compressed)
# Load collection statistics
collection_stats = json.load(collection_stats_file)
inverted_index.load_collection_stats(collection_stats)
# Load meta info for documents
docs_meta = json.load(docs_meta_file)
inverted_index.load_docs_meta(docs_meta)
# Load lookup table
lookup_table = json.load(lookup_table_file)
inverted_index.load_lookup_table(lookup_table)
# Load vocabulary
inverted_index.load_vocabulary()
# Load inverted lists only if in_memory is True
if self.config.in_memory:
index_map = defaultdict(InvertedList)
for term, term_stats in lookup_table.items():
inverted_list = index_map[term]
inverted_list_binary = inverted_index.read_inverted_list_from_file(
inverted_lists_file, term_stats['posting_list_position'],
term_stats['posting_list_size'])
inverted_list.bytearray_to_postings(inverted_list_binary,
compressed,
term_stats['df'])
inverted_index.load_map(index_map)
return inverted_index
def create_document_vectors(self, inverted_index):
with open(
self.root_dir + '/' + self.config.index_dir + '/' +
self.config.document_vectors_file_name, 'wb') as file_buffer:
data = self.load_data()
doc_id = -1
vocabulary = inverted_index.get_vocabulary()
document_vectors = defaultdict(DocumentVector)
for scene in data['corpus']:
doc_id += 1
document_vector = document_vectors[doc_id]
document_vector.set_doc_id(doc_id)
scene_text = scene['text']
terms = list(filter(None, scene_text.split()))
# Get the unique terms from the text as we need to calculate score for each term
unique_terms = set(terms)
for term in unique_terms:
term_id = vocabulary.index(term)
# Refer Chapter - 7, page 242 for the calculation below
fik = terms.count(term)
N = inverted_index.get_total_docs()
nk = inverted_index.get_df(term)
term_value = 0
if fik:
term_value = (math.log(fik) + 1) * \
math.log((N + 1) / (nk + 0.5))
# Add an entry with this term_id, term_value pair to the doc vector for this doc
document_vector.add_doc_vector_entry(term_id, term_value)
numerator = np.array(
list(document_vector.get_doc_vector().values()))
denominator = np.linalg.norm(numerator)
normalized_vector = numerator / denominator
for idx, term_id in enumerate(
list(document_vector.get_doc_vector().keys())):
normalized_term_value = float(normalized_vector[idx])
document_vector.add_doc_vector_entry(
term_id, normalized_term_value)
doc_vector_binary, size_in_bytes = document_vector.vector_to_bytearray(
)
position_in_file = file_buffer.tell()
file_buffer.write(doc_vector_binary)
doc_meta = inverted_index.get_doc_meta(doc_id)
doc_meta['document_vector_position'] = position_in_file
doc_meta['document_vector_size'] = size_in_bytes
inverted_index.update_docs_meta(doc_id, doc_meta)
with open(
self.root_dir + '/' + self.config.index_dir + '/' +
self.config.docs_meta_file_name, 'w') as f:
json.dump(inverted_index.get_docs_meta(), f)
def get_document_vectors(self, inverted_index):
number_of_docs = inverted_index.get_total_docs()
document_vectors = defaultdict(DocumentVector)
with open(
self.root_dir + '/' + self.config.index_dir + '/' +
self.config.document_vectors_file_name,
'rb') as document_vectors_file:
for doc_id in range(number_of_docs):
doc_meta = inverted_index.get_doc_meta(doc_id)
position_in_file = doc_meta['document_vector_position']
size_in_bytes = doc_meta['document_vector_size']
document_vectors_file.seek(position_in_file)
document_vector = document_vectors[doc_id]
document_vector.set_doc_id(doc_id)
document_vector_binary = bytearray(
document_vectors_file.read(size_in_bytes))
document_vector.bytearray_to_vector(document_vector_binary,
size_in_bytes)
return document_vectors
def create_prior(self, inverted_index, prior_type):
with open(
self.root_dir + '/' + self.config.index_dir + '/' +
prior_type + '_priors', 'wb') as file_buffer:
total_docs = inverted_index.get_total_docs()
random.seed(0)
prior = 0
# Initialize an empty bytearray
prior_binary = bytearray()
size_in_bytes = 0
format_prior = '<d'
for doc_id in range(total_docs):
if prior_type == 'uniform':
prior = math.log(1 / total_docs)
else:
prior = math.log(random.random())
# Convert prior to binary using little-endian byte-order and float format (8 bytes)
prior_binary += struct.pack(format_prior, prior)
size_in_bytes += struct.calcsize(format_prior)
file_buffer.write(prior_binary)
def dump_inverted_lists_to_disk(self, file_buffer, inverted_index):
"""
Stores the inverted lists on disk
buffer file_buffer: Buffer for the inverted lists file
class inverted_index: Instance of the inverted index being used
"""
for term, inverted_list in inverted_index.get_map().items():
position_in_file = file_buffer.tell()
inverted_list_binary, size_in_bytes = inverted_list.postings_to_bytearray(
inverted_index.compressed)
file_buffer.write(inverted_list_binary)
inverted_index.update_lookup_table(term, position_in_file,
size_in_bytes)
def dump_inverted_index_to_disk(self, inverted_index):
"""
Stores the docs meta, configuration, lookup table and inverted lists on disk
class inverted_index: Instance of the inverted index being used
"""
# Create the index directory if it doesn't exist
if not os.path.exists(self.root_dir + '/' + self.config.index_dir):
os.mkdir(self.root_dir + '/' + self.config.index_dir)
if not self.config.compressed:
# Create uncompressed index directory if it doesn't exist
if not os.path.exists(self.root_dir + '/' + self.config.index_dir +
'/' + self.config.uncompressed_dir):
os.mkdir(self.root_dir + '/' + self.config.index_dir + '/' +
self.config.uncompressed_dir)
with open(
self.root_dir + '/' + self.config.index_dir + '/' +
self.config.uncompressed_dir + '/' +
self.config.inverted_lists_file_name, 'wb') as f:
self.dump_inverted_lists_to_disk(f, inverted_index)
with open(
self.root_dir + '/' + self.config.index_dir + '/' +
self.config.uncompressed_dir + '/' +
self.config.lookup_table_file_name, 'w') as f:
json.dump(inverted_index.get_lookup_table(), f)
if self.config.compressed:
# Create compressed index directory if it doesn't exist
if not os.path.exists(self.root_dir + '/' + self.config.index_dir +
'/' + self.config.compressed_dir):
os.mkdir(self.root_dir + '/' + self.config.index_dir + '/' +
self.config.compressed_dir)
with open(
self.root_dir + '/' + self.config.index_dir + '/' +
self.config.compressed_dir + '/' +
self.config.inverted_lists_file_name, 'wb') as f:
self.dump_inverted_lists_to_disk(f, inverted_index)
with open(
self.root_dir + '/' + self.config.index_dir + '/' +
self.config.compressed_dir + '/' +
self.config.lookup_table_file_name, 'w') as f:
json.dump(inverted_index.get_lookup_table(), f)
with open(
self.root_dir + '/' + self.config.index_dir + '/' +
self.config.collection_stats_file_name, 'w') as f:
json.dump(inverted_index.get_collection_stats(), f)
with open(
self.root_dir + '/' + self.config.index_dir + '/' +
self.config.docs_meta_file_name, 'w') as f:
json.dump(inverted_index.get_docs_meta(), f)
with open(
self.root_dir + '/' + self.config.index_dir + '/' +
self.config.config_file_name, 'w') as f:
json.dump(self.config.get_params(), f)
def remove_inverted_index_from_memory(self, inverted_index):
"""
Removes an inverted index from memory to free up memory
class inverted_index: Instance of the inverted index being used
"""
inverted_index.delete_map()
from Cecilio import *
from Config import Config
if __name__ == "__main__":
run(Config(meeting_distances_config, commonConfiguration, meet_object_v2))
def test_engine_search():
from GeneratingDataset import DummyDataset
seq_len = 5
n_data_dim = 2
n_classes_dim = 3
dataset = DummyDataset(input_dim=n_data_dim,
output_dim=n_classes_dim,
num_seqs=2,
seq_len=seq_len)
dataset.init_seq_order(epoch=1)
config = Config()
config.update({
"model": "/tmp/model",
"batch_size": 5000,
"num_outputs": n_classes_dim,
"num_inputs": n_data_dim,
"network": {
"output": {
"class": "rec",
"from": [],
"max_seq_len": 10,
"target": "classes",
"unit": {
"prob": {
"class": "softmax",
"from": ["prev:output"],
"loss": "ce",
"target": "classes"
},
"output": {
"class": "choice",
"beam_size": 4,
"from": ["prob"],
"target": "classes",
"initial_output": 0
},
"end": {
"class": "compare",
"from": ["output"],
"value": 0
}
}
},
"decision": {
"class": "decide",
"from": ["output"],
"loss": "edit_distance"
}
}
})
engine = Engine(config=config)
# Normally init_network can be used. We only do init_train here to randomly initialize the network.
engine.init_train_from_config(config=config,
train_data=dataset,
dev_data=None,
eval_data=None)
print("network:")
pprint(engine.network.layers)
assert "output" in engine.network.layers
assert "decision" in engine.network.layers
engine.search(dataset=dataset)
print("error keys:")
pprint(engine.network.error_by_layer)
assert engine.network.total_objective is not None
assert "decision" in engine.network.error_by_layer
engine.finalize()
def test_cudnn_save_restore():
import tempfile, shutil, os
from tensorflow.python.training.saver import BaseSaverBuilder
model_tmp_dir = tempfile.mkdtemp("tmp-checkpoint")
model_filename = model_tmp_dir + "/model"
try:
num_inputs = 4
input_data = numpy.array(
[[[1, -0.2, 0.3, -4], [2, -0.6, 0.7, -1.8], [1, 0.3, -0.1, -0.8],
[0.1, -0.2, 0.2, .8]]],
dtype="float32")
seq_lens = numpy.array([4], dtype="int32")
assert input_data.shape == (1, seq_lens[0], num_inputs)
num_outputs = 3
print("Storing network with cuDNN.")
tf.reset_default_graph()
with tf.Session() as session:
config1 = Config()
config1.update({
"num_outputs": num_outputs,
"num_inputs": num_inputs,
"network": {
"layer1": {
"class": "rec",
"n_out": 6,
"unit": "CudnnLSTM"
},
"layer2": {
"class": "rec",
"n_out": 6,
"unit": "CudnnLSTM",
"from": ["layer1"]
},
"output": {
"class": "linear",
"activation": None,
"n_out": num_outputs,
"from": ["layer2"]
}
}
})
network1 = TFNetwork(config=config1, train_flag=True)
network1.construct_from_dict(config1.typed_dict["network"])
network1.initialize_params(session=session)
params = {
} # type: dict[str,dict[str,numpy.ndarray]] # layer -> param -> numpy.ndarray
for layer_name, layer1 in sorted(network1.layers.items()):
print("layer: %r" % layer_name)
assert isinstance(layer1, LayerBase)
params[layer_name] = {}
for param_name, param1 in sorted(layer1.params.items()):
print(" param %r: %r" % (param_name, param1))
params[layer_name][param_name] = param1.eval(session)
if param1 in layer1.saveable_param_replace:
saveable_object = layer1.saveable_param_replace[param1]
print(" saveable object: %r" % saveable_object)
assert isinstance(saveable_object,
BaseSaverBuilder.SaveableObject)
print(" op: %r" % saveable_object.op)
print(" name: %r" % saveable_object.name)
for spec in saveable_object.specs:
print(" spec: %r" % spec)
assert isinstance(spec, BaseSaverBuilder.SaveSpec)
print(" name: %r" % spec.name)
print(" tensor: %r" % spec.tensor)
output_data1 = session.run(
network1.get_default_output_layer().output.placeholder,
feed_dict={
network1.extern_data.data["data"].placeholder: input_data,
network1.extern_data.data["data"].size_placeholder[0]:
seq_lens
})
assert_equal(output_data1.shape,
(seq_lens[0], 1, num_outputs)) # (time, batch, dim)
print("Saveable params:", network1.get_saveable_params_list())
network1.save_params_to_file(filename=model_filename,
session=session)
print()
# First test if we can load the same network as-is. This will involve the RNNParamsSaveable.
print("Testing restore of same network with cuDNN.")
tf.reset_default_graph()
with tf.Session() as session:
network1a = TFNetwork(config=config1, train_flag=True)
network1a.construct_from_dict(config1.typed_dict["network"])
network1a.load_params_from_file(filename=model_filename,
session=session)
for layer_name, layer1 in sorted(network1a.layers.items()):
print("layer: %r" % layer_name)
for param_name, param1 in sorted(layer1.params.items()):
print(" param %r: %r" % (param_name, param1))
param1old = params[layer_name][param_name]
param1new = param1.eval(session)
numpy.testing.assert_almost_equal(param1old, param1new)
output_data1a = session.run(
network1a.get_default_output_layer().output.placeholder,
feed_dict={
network1a.extern_data.data["data"].placeholder:
input_data,
network1a.extern_data.data["data"].size_placeholder[0]:
seq_lens
})
numpy.testing.assert_almost_equal(output_data1, output_data1a)
print()
print("Testing restore of network with LSTMBlockCell.")
tf.reset_default_graph()
with tf.Session() as session:
# Now, in CPU, we would automatically use LSTMBlockCell instead.
# Check if the import of the model works correctly in load_params_from_file().
config2 = Config()
config2.update({
"num_outputs": num_outputs,
"num_inputs": num_inputs,
"network": {
"layer1": {
"class": "rec",
"n_out": 6,
"unit": "LSTMBlockFused"
},
"layer2": {
"class": "rec",
"n_out": 6,
"unit": "LSTMBlockFused",
"from": ["layer1"]
},
"output": {
"class": "linear",
"activation": None,
"n_out": num_outputs,
"from": ["layer2"]
}
}
})
network2 = TFNetwork(config=config2, train_flag=True)
network2.construct_from_dict(config2.typed_dict["network"])
network2.load_params_from_file(filename=model_filename,
session=session)
output_data2 = session.run(
network2.get_default_output_layer().output.placeholder,
feed_dict={
network2.extern_data.data["data"].placeholder: input_data,
network2.extern_data.data["data"].size_placeholder[0]:
seq_lens
})
# Not sure if sth is incorrect... Only decimal=2 works.
numpy.testing.assert_almost_equal(output_data1,
output_data2,
decimal=2)
finally:
shutil.rmtree(model_tmp_dir)
def test_engine_search_attention():
from GeneratingDataset import DummyDataset
seq_len = 5
n_data_dim = 2
n_classes_dim = 3
dataset = DummyDataset(input_dim=n_data_dim,
output_dim=n_classes_dim,
num_seqs=2,
seq_len=seq_len)
dataset.init_seq_order(epoch=1)
print("Hello search!")
config = Config()
config.update({
"model": "/tmp/model",
"batch_size": 5000,
"max_seqs": 2,
"num_outputs": n_classes_dim,
"num_inputs": n_data_dim,
"network": {
"encoder": {
"class": "linear",
"activation": "tanh",
"n_out": 5
},
"output": {
"class": "rec",
"from": [],
"unit": {
'output': {
'class': 'choice',
'target': 'classes',
'beam_size': 4,
'from': ["output_prob"]
},
"end": {
"class": "compare",
"from": ["output"],
"value": 0
},
'orth_embed': {
'class': 'linear',
'activation': None,
'from': ['output'],
"n_out": 7
},
"s": {
"class": "rnn_cell",
"unit": "LSTMBlock",
"from": ["prev:c", "prev:orth_embed"],
"n_out": 7
},
"c_in": {
"class": "linear",
"activation": "tanh",
"from": ["s", "prev:orth_embed"],
"n_out": 5
},
"c": {
"class": "dot_attention",
"from": ["c_in"],
"base": "base:encoder",
"base_ctx": "base:encoder"
},
"output_prob": {
"class": "softmax",
"from": ["prev:s", "c"],
"target": "classes",
"loss": "ce"
}
},
"target": "classes",
"max_seq_len": 10
},
"decision": {
"class": "decide",
"from": ["output"],
"loss": "edit_distance"
}
}
})
engine = Engine(config=config)
print("Init network...")
engine.start_epoch = 1
engine.use_dynamic_train_flag = False
engine.use_search_flag = True
engine.init_network_from_config(config)
print("network:")
pprint(engine.network.layers)
assert "output" in engine.network.layers
assert "decision" in engine.network.layers
print("Search...")
engine.search(dataset=dataset)
print("error keys:")
pprint(engine.network.error_by_layer)
assert engine.network.total_objective is not None
assert "decision" in engine.network.error_by_layer
engine.finalize()
def __init__(self, config=None, blocking=True):
if not config:
config = Config()
config.update(dummyconfig_dict)
super(DummyDevice, self).__init__(device="cpu", config=config, blocking=blocking)
import os
import time
import tensorflow as tf
import datetime as dt
import gym
import policy_gradient
from time import gmtime, strftime
from Config import Config
conf = Config()
def generate_predictions(ticker):
global pg
# Initialize environment and make prediction
env = gym.make(conf.env_name)
env = env.unwrapped
prediction = pg.test_model(env)
if prediction == 0:
pred_str = 'BUY (0)'
elif prediction == 1:
pred_str = 'FLAT (1)'
elif prediction == 2:
pred_str = 'SELL (2)'
print('---------Predicted action is:')
print(pred_str)
def __init__(self):
'''GUI的初始化
'''
self.logger = logging.getLogger(__name__)
self.logger.info("开始记录日志")
try:
self.logger.info("开始检查网络连接")
# requests.get("https://www.baidu.com", verify=False) # 访问百度以验证网络是否通畅
# self.logger.info("网络连接成功")
self.logger.info("开始加载配置文件")
self.cf = Config()
self.APP_ID = self.cf.APP_ID
self.API_KEY = self.cf.API_KEY
self.SECRET_KEY = self.cf.SECRET_KEY
self.MK_APIKEY = self.cf.MK_APIKEY
self.logger.info("开始加载窗体")
self.window = tk.Tk()
self.window.title('截图翻译工具(V4.5)')
self.window.attributes("-alpha", 0.9)
self.window['background'] = 'PowderBlue'
self.filename = resource_path(
os.path.join("res", "bitbug_favicon.ico"))
self.window.iconbitmap(self.filename)
self.window.resizable(0, 0)
self.menubar = tk.Menu(self.window)
self.filemenu = tk.Menu(self.menubar, tearoff=0)
self.menubar.add_cascade(label='导入文件', menu=self.filemenu)
self.filemenu.add_command(label='从文本文件导入', command=self.getFromTXT)
self.filemenu.add_command(label='从图像文件导入',
command=self.getFromImage)
self.setmenu = tk.Menu(self.menubar, tearoff=0)
self.menubar.add_cascade(label='设置', menu=self.setmenu)
self.setmenu.add_command(label='查看APP_ID', command=self.getAppId)
self.setmenu.add_command(label='生成配置文件', command=self.useConfig)
self.helpmenu = tk.Menu(self.menubar, tearoff=0)
self.menubar.add_cascade(label='帮助', menu=self.helpmenu)
self.helpmenu.add_command(label='关于', command=self.aboutClick)
self.helpmenu.add_command(label='更新与反馈', command=self.upClick)
self.window.config(menu=self.menubar)
tk.Label(self.window, text="原文:",
background='PowderBlue').grid(row=0,
column=0,
sticky=tk.W)
self.comboxlist0 = ttk.Combobox(self.window)
self.comboxlist0["values"] = ("使用谷歌翻译", "C/C++代码美化", "发布临时文字")
self.comboxlist0.bind("<<ComboboxSelected>>", self.comboxlist0_msg)
self.comboxlist0["state"] = "readonly"
self.comboxlist0.current(0)
self.comboxlist0.grid(row=0,
column=0,
columnspan=2,
padx=38,
sticky=tk.W)
self.comboxlist = ttk.Combobox(self.window)
self.comboxlist["values"] = ("翻译为中文(简体)", "翻译为中文(繁体)", "翻译为英文",
"翻译为日文", "翻译为韩文")
self.comboxlist["state"] = "readonly"
self.comboxlist.current(0)
self.comboxlist.grid(row=0, column=1, sticky=tk.E)
self.inputText = tk.Text(self.window, height=12, width=50)
self.inputText.grid(row=1, column=0, columnspan=2, padx=8, pady=8)
self.inputTextsection = section()
self.inputTextsection.s = self.inputText
self.inputTextmenu = tk.Menu(self.window, tearoff=0)
self.inputTextmenu.add_command(
label="复制", command=self.inputTextsection.onCopy)
self.inputTextmenu.add_separator()
self.inputTextmenu.add_command(
label="粘贴", command=self.inputTextsection.onPaste)
self.inputTextmenu.add_separator()
self.inputTextmenu.add_command(label="剪切",
command=self.inputTextsection.onCut)
self.inputText.bind(
"<Button-3>", lambda event: self.inputTextmenu.post(
event.x_root, event.y_root))
tk.Label(self.window, text="译文:",
background='PowderBlue').grid(row=2,
column=0,
sticky=tk.W)
self.translateBtn = tk.Button(self.window,
text="翻译",
bg='Lavender',
command=self.translateBtnClick)
self.translateBtn.grid(row=2, column=0, padx=40, sticky=tk.W)
self.OCRLANGUAGE = 'eng'
self.comboxlist1 = ttk.Combobox(self.window)
self.comboxlist1["values"] = ("使用百度OCR", "使用OCRMAKER")
self.comboxlist1.bind("<<ComboboxSelected>>", self.comboxlist1_OCR)
self.comboxlist1["state"] = "readonly"
self.comboxlist1.current(0)
self.comboxlist1.grid(row=2,
column=0,
padx=80,
columnspan=2,
sticky=tk.W)
self.clearBtn = tk.Button(self.window,
text="清屏",
bg='Lavender',
command=self.clearBtnClick)
self.clearBtn.grid(row=2, column=1, padx=90, sticky=tk.E)
self.cb1var = tk.IntVar()
self.cb1var.set(1)
self.cb1 = tkinter.Checkbutton(self.window,
text='监听剪切板',
background='PowderBlue',
variable=self.cb1var)
self.cb1.grid(row=2, column=0, columnspan=2, sticky=tk.E)
self.outText = tk.Text(self.window, height=12, width=50)
self.outText.grid(row=3, column=0, columnspan=2, padx=8, pady=8)
self.outTextsection = section()
self.outTextsection.s = self.outText
self.outTextmenu = tk.Menu(self.window, tearoff=0)
self.outTextmenu.add_command(label="复制",
command=self.outTextsection.onCopy)
self.outTextmenu.add_separator()
self.outTextmenu.add_command(label="粘贴",
command=self.outTextsection.onPaste)
self.outTextmenu.add_separator()
self.outTextmenu.add_command(label="剪切",
command=self.outTextsection.onCut)
self.outText.bind(
"<Button-3>", lambda event: self.outTextmenu.post(
event.x_root, event.y_root))
# self.window.bind(sequence="<Key>", func=self.outTexeP)
self.window.state('normal')
self.window.wm_attributes('-topmost', 1)
self.window.protocol("WM_DELETE_WINDOW", self.on_closing)
self.menu_options = (("显示", None, self.showwin), )
systray = SysTrayIcon(self.filename,
"截图翻译工具",
self.menu_options,
on_quit=self.on_quit_callback)
systray.start()
self.logger.info("窗体加载成功")
self.logger.info("开始加载插件模块")
self.maindll = WinDLL(
resource_path(os.path.join("res", "pymain.dll")))
dlllist = [
i for i in glob.glob(r"*.dll") if i[0] == 'p' and i[1] == 'y'
]
self.logger.info("搜寻到如下插件{0}".format(str(dlllist)))
self.logger.info("开始载入dll")
self.dlllist = []
s = create_string_buffer(''.encode(), 8192) #in dll is 8192
for i in dlllist:
self.logger.info("正在载入{0}".format(i))
dll = WinDLL(os.path.join(os.getcwd(), i))
self.dlllist.append(dll)
self.logger.info("载入{0}成功".format(i))
self.logger.info("正在获取{0}的名字".format(i))
s = create_string_buffer(''.encode(), 9000) #in dll is 8192
dll.name(s)
s = string_at(s).decode('utf-8')
self.logger.info("{0}的名字为:{1}".format(i, s))
self.comboxlist0["values"] = self.comboxlist0["values"] + (s, )
self.logger.info("开始启动插件事件监听线程")
_thread.start_new_thread(self.apiloop, ("", )) # 用于插件的监听
self.logger.info("插件事件监听线程启动成功")
self.logger.info("开始启动剪切板事件监听线程")
_thread.start_new_thread(
self.thread_fun, (self.window, self.inputText, self.outText))
self.logger.info("剪切板事件监听线程启动成功")
self.logger.info("进入窗体消息循环")
self.window.mainloop()
except Exception as e:
self.logger.error(str(e))
tk.messagebox.showerror(
title="致命错误",
message=
"请将具体信息发送给开发者:\n网页:https://github.com/super1207/translate\n错误信息:"
+ str(e))
def getLimitValue(self, limit_name):
config = Config.getInstance()
limit_type = self._convertStringToEnum(ELimitType, limit_name)
return config.getLimitValue(limit_type)
"""
Wrap mosquito MQTT client and provide basic for receiving and sending data to MQTT broker
handle client and topics registrations
"""
from Config import Config
import paho.mqtt.client as mqtt
conf = Config.Config().get_config()
class _MessageHandler(object):
def __init__(self, broker_addres, broker_port=1883, keep_alive=30):
self.broker_address = broker_addres
self.broker_port = broker_port
self.keep_alive = keep_alive
self.client = None
self.is_connected = False
try:
self._connect()
except ConnectionRefusedError:
print(conf.get('msg', 'mqtt_error'))
def _connect(self):
self.client = mqtt.Client()
self.client.on_connect = self._on_connect_callback
self.client.on_disconnect = self._on_disconnect_callback
try:
self.client.connect(self.broker_address, self.broker_port,
def getGlobalState(self, state_name):
config = Config.getInstance()
state_type = self._convertStringToEnum(EGlobalStateType, state_name)
return config.getGlobalState(state_type)
def train_label_none_label_classification(label_folder,
non_label_folder,
model_file=None):
c = Config()
# Build or load model
if model_file is None:
# create model
img_input = Input(shape=(28, 28, 3))
# prediction = model_cnn_2_layer.nn_classify_label_non_label(img_input)
# prediction = model_cnn_3_layer.nn_classify_label_non_label(img_input)
prediction = nn_cnn_3_layer.nn_classify_label_non_label(img_input)
model = Model(inputs=img_input, outputs=prediction)
model.compile(loss='categorical_crossentropy',
optimizer=RMSprop(),
metrics=['accuracy'])
else:
model = load_model(model_file)
model.summary()
# Load and normalize data
x_train, y_train, x_test, y_test = load_train_validation_data(
label_folder, non_label_folder)
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train[:, :, :, 0] -= c.img_channel_mean[0]
x_train[:, :, :, 1] -= c.img_channel_mean[1]
x_train[:, :, :, 2] -= c.img_channel_mean[2]
x_test[:, :, :, 0] -= c.img_channel_mean[0]
x_test[:, :, :, 1] -= c.img_channel_mean[1]
x_test[:, :, :, 2] -= c.img_channel_mean[2]
x_train /= 255
x_test /= 255
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')
# x_train.reshape(x_train.shape[0], 28, 28, 3)
# x_test.reshape(x_test.shape[0], 28, 28, 3)
# convert class vectors to binary class matrices
y_train = keras.utils.to_categorical(y_train, 2)
y_test = keras.utils.to_categorical(y_test, 2)
# Checkpointing is to save the network weights only when there is an improvement in classification accuracy
# on the validation dataset (monitor=’val_acc’ and mode=’max’).
file_path = "weights-improvement-{epoch:04d}-{val_acc:.4f}.hdf5"
checkpoint = ModelCheckpoint(file_path,
monitor='val_acc',
verbose=1,
save_best_only=True,
mode='max')
callbacks_list = [checkpoint]
model.fit(x_train,
y_train,
batch_size=128,
epochs=100,
verbose=1,
callbacks=callbacks_list,
validation_data=(x_test, y_test))
score = model.evaluate(x_test, y_test, verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
model.save('final_model.h5')
def setGlobalState(self, state_name, state_value):
config = Config.getInstance()
state_type = self._convertStringToEnum(EGlobalStateType, state_name)
config.setGlobalState(state_type, state_value)
"""
from Config import Config
from GraphUtil import convNodeToProblems, convAllNodeToProblems
from MasterEncoder import MasterEncoder
import warnings
from Fingerprint import Fingerprint
from CompDatabase import CompDatabase
from OtherEncoders import ValueEncoder
from tqdm import tqdm
import numpy as np
warnings.filterwarnings('ignore')
CF = Config()
class PrepDataset:
def __init__(self, graphBasePath="graph/"):
self.mode = "Training"
self.graphList = []
self.parallel = True
#set compound encoder
def setCompoundEncoder(self, compFilename):
"""
compFilename: csv path for a file containing smiles
calc descriptors and save those info as self.CompDat
"""
def create_profile(self, profile):
config = Config()
expdate = profile.exp_date.split()
exp_month = expdate[0]
if '0' in exp_month[0]:
exp_month_value = exp_month[1]
else:
exp_month_value = exp_month
exp_year = f'20{expdate[2]}'
file = str([{
"name": profile.profile_name,
"uuid": f'{config.get_random_string_lowercase(8)}-{config.get_random_string_lowercase(4)}-{config.get_random_string_lowercase(4)}-{config.get_random_string_lowercase(4)}-{config.get_random_string_lowercase(12)}',
"email": profile.email,
"shipping": {
"first_name": profile.first_name,
"last_name": profile.last_name,
"address_line_1": profile.address,
"address_line_2": profile.apt,
"company": "",
"state": {
"label": profile.state, "value": config.us_state_abbreviations[profile.state]
}
,
"country": {
"label": profile.country, "value": config.country_abbrev[profile.country]
}
,
"phone": profile.phone,
"city": profile.city,
"zipcode": profile.zip_code
}
,
"billing": {
"first_name": profile.first_name,
"last_name": profile.last_name,
"address_line_1": profile.address,
"address_line_2": profile.apt,
"company": "",
"state": {
"label": profile.state, "value": config.us_state_abbreviations[profile.state]
}
,
"country": {
"label": profile.country, "value": config.country_abbrev[profile.country]
}
,
"phone": profile.phone,
"city": profile.city,
"zipcode": profile.zip_code
}
,
"card": {
"number": profile.card_number,
"expiry": {
"month": {
"label": exp_month, "value": exp_month_value
}
,
"year": {
"label": exp_year, "value": exp_year
}
}
,
"cvv": profile.cvv
}
,
"sameAddress": "true",
"id": config.get_random_string_uppercase(9)
}
])
file = file.replace("'", '"').replace(": ", ":").replace(", ", ",").replace('"true"', "true")
file = StringIO(file)
return file
