def load_gan(args):
outdir = args.outdir
modelDict = AttrDict({})
modelDict.discriminator = loadModel(outdir + "/discriminator")
modelDict.gen_disc = loadModel(outdir + "/gen_disc")
modelDict.generator = loadModel(outdir + "/generator")
return modelDict
def load_autoencoder(args):
outdir = args.outdir
modelDict = AttrDict({})
modelDict.ae = loadModel(outdir + "/ae")
modelDict.encoder = loadModel(outdir + "/encoder")
modelDict.generator = loadModel(outdir + "/generator")
return modelDict
def explain(self, info: AttrDict) -> str:
'Format the query results.'
info.arrDep = dict(A='到达', D='离开').get(info.arrDepId, '到达')
info.cdyName = dict(E='空车', L=info.cdyName).get(info.carLE, '')
if info.cdyName[-1].isdigit():
info.cdyName += '类货物'
if not info.carKind.endswith('车'):
info.carKind += '车'
if not info.wbID:
info.wbID = info.wbNbr
explaination = '''
截至 {eventDate} 时为止，您查询的{conName[由{}托运的]}
{carNo[ {} 号]}{carType[ {} 型]}{carKind}
%s已{arrDep}{eventProvince[位于{}{eventCity}的]}
{eventAdm}{eventStation}站
{dzlc[，距离终点站{destStation}站还有 {} km]}。
'''
explaination %= '''
已被编入由{cdyAdm}{cdyStation}站
开{destAdm[往{}]}{destStation[{}站]}
的 {trainId} 次列车机后第 {trainOrder} 位，
负责运送{wbID[编号为 {} 的]}{cdyName}。
该列车现
''' if int(info.trainOrder) else ''
return self.format(strip_lines(explaination), **info)
def combine_stations() -> Iterable[List[str]]:
'Combine the two railway station datasets by telecode.'
stations = AttrDict()
names = {}
stations_95306 = hyfw.dfs()
stations_12306 = kyfw.stations()
for s in stations_95306:
pinyin = s['PYM'].lower()
if len(pinyin) > 3:
pinyin = pinyin[:2] + pinyin[-1]
elif len(pinyin) < 3:
pinyin = ''
name, telecode = s['ZMHZ'], s['DBM']
stations[telecode] = [pinyin, name, s['TMIS'], s['SSJC']]
names[name] = telecode
for s in stations_12306:
old = stations.get(s.telecode)
new = [s.pinyin_code, s.name, '', '']
if s.name in names and s.telecode != names[s.name]:
conflict = '%s/%s -> %s/%s' % (names[s.name], new, s.telecode, old)
if s.telecode in stations:
conflict = 'Name conflict: %s -> ?' % conflict
else:
conflict = 'Solved conflict: %s' % conflict
old = stations[s.telecode] = stations.pop(names[s.name])
old[:2] = new[:2]
elif s.telecode in stations and s.name != stations[s.telecode][1]:
new[-2] = tmis(s.name).get(s.name, '')
conflict = '%s/(%s => %s)' % (s.telecode, old, new)
if new[-2] and old[-2] != new[-2]: # TMIS codes conflict
conflict = 'Ambiguous telecode: %s' % conflict
else:
conflict = 'Solved conflict: %s' % conflict
old[:2] = new[:2]
else:
if s.telecode not in stations:
stations[s.telecode] = [''] * 4
stations[s.telecode][:2] = new[:2]
continue
# resolve merge conflicts manually
shell(dict(vars(), s=stations), '\n%s' % conflict)
for k, v in stations.items():
# drop telecodes with spaces
# so those can be used as temporary names in conflict solving
v.insert(2, '' if ' ' in k else k)
yield v
def load_config(config):
keys = load_yaml(abs_path("default"))
keys['credentials'] = {}
if os.path.exists(abs_path("credentials")):
keys['credentials'] = load_yaml(abs_path("credentials"))
if config != 'default':
keys.update(load_yaml(abs_path(config)))
if "aws_access_key" in keys['credentials']:
keys['s3'] = boto3.resource(
's3',
region_name=keys['region'],
aws_access_key_id=keys['credentials']['aws_access_key'],
aws_secret_access_key=keys['credentials']['aws_access_secret'])
keys['s3_client'] = boto3.client(
's3',
region_name=keys['region'],
aws_access_key_id=keys['credentials']['aws_access_key'],
aws_secret_access_key=keys['credentials']['aws_access_secret'])
else:
keys['s3'] = boto3.resource('s3', region_name=keys['region'])
keys['s3_client'] = boto3.client('s3', region_name=keys['region'])
return AttrDict(keys)
def track(self, **kwargs) -> AttrDict:
'Send the tracking request and parse the response message.'
# insert the namespace prefix for each key
data = {'hwzz.' + k: v for k, v in kwargs.items()}
response = self.fetch('hwzz_uouii.action', data=data)
assert response.success, response.get('message', response.get('msg'))
return AttrDict(self.decipher(response.object)[0])
def fetch(self, path, params=None, method='POST', json=True, **kwargs):
'Initiate an API request.'
url = self.__doc__ + path
if not isinstance(params, dict):
params = self.params.get(params, {})
query = 'params' if method == 'GET' else 'data'
params.update(kwargs.get(query, {}))
kwargs[query] = params
response = self.session.request(method, url, **kwargs)
return AttrDict(response.json()) if json else response
def __init__(self,
sess,
ob_space,
ac_space,
n_env,
n_steps,
n_batch,
args,
reuse=tf.compat.v1.AUTO_REUSE,
**kwargs):
super(EmbeddingPolicy, self).__init__(sess,
ob_space,
ac_space,
n_env,
n_steps,
n_batch,
reuse=reuse,
scale=False)
self.args = args
self.step_counter = 0
graph_args = AttrDict({
'BATCH_SIZE': 1,
'NODE_COUNT': args.graph_node_count,
'UPDATE_ITERATION': args.graph_update_iteration,
'EMBEDDING_SIZE': args.graph_embedding_size,
'HIDDEN_LAYERS': args.graph_hidden_layers,
'HIDDEN_ACTIVATION': tf.nn.relu,
'END_ACTIVATION': tf.nn.relu,
'RESIDUAL': True,
'ROOT_EMBEDDING': False,
})
with tf.variable_scope("model", reuse=tf.compat.v1.AUTO_REUSE):
self.graph = Graph(graph_args)
pi_latent, vf_latent = graph_extractor(self.processed_obs,
self.graph, args)
self._value_fn = linear(vf_latent, 'vf', 1)
self._policy = pi_latent
self._proba_distribution = self.pdtype.proba_distribution_from_flat(
pi_latent)
# self.q_value = vf_latent
self.q_value = linear(vf_latent,
'q',
self.args.n_action_slots,
init_scale=0.01)
# self._proba_distribution, self._policy, self.q_value = \
# self.pdtype.proba_distribution_from_latent(pi_latent, vf_latent, init_scale=0.01)
self._setup_init()
def query(self, depart: str, arrive: str, date=today, student=False):
'List trains between two stations.'
response = self.fetch('otn/leftTicket/query',
method='GET',
params=AttrDict([
('leftTicketDTO.train_date', date),
('leftTicketDTO.from_station', depart),
('leftTicketDTO.to_station', arrive),
('purpose_codes',
'0x00' if student else 'ADULT')
]))
return [train.split('|') for train in response.data['result']]
def mergeParamsWithInis(args_param, ini_files_param="ini_file"):
args = AttrDict()
args_param_dict = vars(args_param)
for k in args_param_dict:
args[k] = heuristicCast(args_param_dict[k])
if args_param_dict[ini_files_param]:
for ini_file in args_param_dict[ini_files_param]:
args_ini_dict = paramsFromIni(ini_file)
for k in args_ini_dict:
args[k] = args_ini_dict[k]
return args
def mergeParamsWithInis(default_args_param, args_param, ini_files_param="ini_file"):
args = AttrDict()
args_param_dict = vars(args_param)
default_args_param_dict = vars(default_args_param)
for k in args_param_dict:
args[k] = heuristicCast(args_param_dict[k])
if args_param_dict[ini_files_param]:
for ini_file in args_param_dict[ini_files_param]:
args_ini_dict = paramsFromIni(ini_file)
for k in args_ini_dict:
if k in default_args_param_dict and default_args_param_dict[k] == args_param_dict[k]: # only update defaults, not those overwritten on the command line
args[k] = args_ini_dict[k]
return args
def paramsFromIni(ini_file):
params = AttrDict()
f = open(ini_file)
for l in f:
l = l.strip()
if l.startswith("#"):
continue
try:
k, v = l.split(None, 1)
except:
print("SKIPPING MALFORMED CONFIG LINE: " + l)
continue
try:
v = heuristicCast(v)
except ValueError:
assert False, "Malformed value for parameter {}: {}".format(k, v)
params[k] = v
return params
def __init__ (self, frame):
wx.Panel.__init__(self, frame, -1)
# Properties
self._network = Network(x=0, y=0) # Root network.
self._selected = None
self._chan_start_point = None
self._filename = None
self._chan_count = 0
# Events
self.Bind(wx.EVT_PAINT, self.on_paint)
self.Bind(wx.EVT_LEFT_DOWN, self.on_left_down)
self.Bind(wx.EVT_LEFT_UP, self.on_left_up)
self.Bind(wx.EVT_MOTION, self.on_motion)
self.Bind(wx.EVT_RIGHT_DOWN, self.on_right_down)
# Allow drop.
self.SetDropTarget(CanvasDropTarget(self))
self.style = AttrDict()
self.style.background = (255, 255, 255)
def explain(self, info: AttrDict) -> str:
'Convert the query result to a human-readable text message.'
# remove trailing whitespace and null values
for k, v in info.items():
info[k] = str(v).rstrip()
if info[k] in ['0', '-1', '发货人']:
info[k] = ''
# rename some fields to fit into the template
converters = {
'fz': 'cdyStation',
'dz': 'destStation',
'pm': 'cdyName',
'xh': 'carNo',
'xt': 'arrDepId',
'carType': 'carKind',
'tyrName': 'shpName',
'conName': 'shpName',
'wbID': 'wbNbr',
}
for k, v in converters.items():
info[k] = info.get(k, '')
info[v] = info.get(v) or info[k]
if info.conName == info.shpName:
info.conName = ''
elif info.conName:
info.conName = '，发往' + info.conName
if info.xh:
info.carKind = '集装箱'
info.carLE = 'L' if info.cdyName else 'E'
elif info.carType.startswith(info.carKind):
info.carKind = '车辆'
if not info.cdyName:
status = ''
elif info.carLE == 'L':
status = '负责运送{wbNbr[单号为 {} 的]}{cdyName}'
if info.cdyName[-1:].isdigit():
info.cdyName += '型集装箱'
else:
status = '当前状态为{cdyName}{wbNbr[，运单号为 {}]}'
if info.cdyName.endswith('空'):
info.cdyName += '车'
if info.get('trainId'):
info.train = ' %s 次列车' % info.trainId
info.arrDep = {
'A': '到达',
'D': '离开',
'在站': '到达',
'在途': '离开',
}.get(info.arrDepId, '到达')
explanation = '''
截至 {eventDate} 时为止，您查询的{shpName[由{}托运{conName}的]}
{carNo[ {} 号]}{carType[ {} 型]}{carKind}
{cdyStation[已从{cdyAdm}{}站发出，]}
{destStation[正前往{destAdm}{}站，]}%s{cdyName[。该车]}
{train[现被编入{}]}{trainOrder[机后第 {} 位]}{train[，]}
目前已{arrDep}{eventProvince[位于{}{eventCity}的]}
{eventAdm}{eventStation}站
{dzlc[，距离终点站{destStation}站还有 {} km]}。
''' % status
return self.format(strip_lines(explanation), **info)
def read_data(datafile, args):
# read data
data = {}
reader = util.json_readr(datafile)
for item in reader:
goal = tuple(item[1])
action = tuple(item[5])
if args.NORMALIZE_VARS:
goal = tuple(embed.normalize_vars(goal))
action = tuple(embed.normalize_vars(action))
reward = item[0]
if goal in data:
if action in data[goal]:
data[goal][action].append(reward)
else:
data[goal][action] = [reward]
else:
data[goal] = {action: [reward]}
# count the number of times a goal lead to success
success_dict = {}
for goal in data:
success = 0
for action in data[goal]:
success += np.sum(data[goal][action])
success_dict[goal] = success
# create data matrices
emb_goal = []
emb_action = []
conn_goal = []
conn_action = []
varconn_goal = []
varconn_action = []
rewards = []
goal_actions = []
for goal in data:
for action in data[goal]:
if args.SAME_LABEL_COLLAPSE == "max":
reward = np.max(data[goal][action])
elif args.SAME_LABEL_COLLAPSE == "mean":
reward = np.mean(data[goal][action])
elif args.SAME_LABEL_COLLAPSE == "prob":
if success_dict[goal] == 0:
reward = 0
else:
reward = np.sum(data[goal][action]) / success_dict[goal]
else:
assert False, "Not Yet Implemented " + args.SAME_LABEL_COLLAPSE
goal2 = list(goal)
action2 = embed.add_or_node(list(action))
[embeddingG, connectionG,
var_connectionG] = embed.clause2embedding(goal2,
args.EMBEDDING_SIZE,
args.NODE_COUNT,
collapse=args.COLLAPSE,
add_var=args.ADD_VAR)
[embeddingA, connectionA,
var_connectionA] = embed.clause2embedding(action2,
args.EMBEDDING_SIZE,
args.NODE_COUNT,
collapse=args.COLLAPSE,
add_var=args.ADD_VAR)
emb_goal.append(embeddingG)
emb_action.append(embeddingA)
conn_goal.append(connectionG)
conn_action.append(connectionA)
varconn_goal.append(var_connectionG)
varconn_action.append(var_connectionA)
rewards.append(reward)
goal_actions.append((goal2, action2))
emb_goal = np.array(emb_goal)
emb_action = np.array(emb_action)
conn_goal = np.array(conn_goal)
conn_action = np.array(conn_action)
varconn_goal = np.array(varconn_goal)
varconn_action = np.array(varconn_action)
rewards = np.array(rewards)
goal_actions = np.array(goal_actions)
result = AttrDict({
"emb_goal": emb_goal,
"emb_action": emb_action,
"conn_goal": conn_goal,
"conn_action": conn_action,
"varconn_goal": varconn_goal,
"varconn_action": varconn_action,
"rewards": rewards,
"goal_actions": goal_actions,
})
if args.FILTER is not None:
result = filter_data(result, args.FILTER)
return result
def callback(*args):
data = AttrDict()
for key, value in event_map.items():
data[key] = q_simplify(args[value])
self.fire(event, data)
def get_context_data(self, **kwargs):
from redis import Redis
from .log import get_redis_key, END
from util import AttrDict
data = \
super(SchedulerResultDetailView, self).get_context_data(**kwargs)
red = Redis()
task_id = self.kwargs['task_id']
try:
# all the objects should have the same content type, here...
object_list = [
scheduler.content_object
for scheduler in Scheduler.objects.filter(logger_name=task_id)
]
object_type_name = type(object_list[0]).__name__.lower()
data[object_type_name + '_list'] = object_list
if isinstance(object_list[0], Dataset):
data['source'] = data['dataset_list'][0].source
except IndexError:
dataset_pk = self.request.GET.get('dataset')
source_pk = self.request.GET.get('source')
if dataset_pk:
data['dataset_list'] = Dataset.objects.filter(pk=dataset_pk)
data['source'] = data['dataset_list'][0].source
elif source_pk:
data['source'] = Source.objects.get(pk=source_pk)
data['dataset_list'] = data['source'].datasets.all()
else:
pass
key = get_redis_key(task_id)
logs = []
start = max(int(self.request.GET.get('start', 0)), 0)
for idx, log in enumerate(red.lrange(key, start, -1)):
log_dict = AttrDict(json.loads(log))
log_dict.index = start + idx
log_dict.show_args = False
try:
if isinstance(log_dict.args, dict):
args = (log_dict.args, )
else:
args = tuple(log_dict.args)
log_dict.msg = log_dict.msg % args
except: # pylint: disable=W0702
log_dict.show_args = True
logs.append(log_dict)
data['logs'] = logs
data['task_id'] = task_id
if self.request.is_ajax():
data['parent'] = 'layout/ajax.html'
else:
data['parent'] = 'layout/site_base.html'
data['END'] = END
return data
class Menu(BaseInterface):
def __init__(self, disp, nested_options, has_parent=False):
"""
:disp: an instance of an Adafruit_SSD1306 display
:nested_options: top level must be:
- an OrderedDict (specifies a nested option list)
- a list (specifies plain text options)
child levels may also be None or Shim
# Note: currently this enforces uniqueness -- may want to change that
# in the future
:has_parent: was this instantiated by a parent menu?
"""
self.has_parent = has_parent
if type(nested_options) == list:
self.options = nested_options
self.nested_options = None
elif isinstance(nested_options, OrderedDict):
self.options = nested_options.keys() # top level options
self.nested_options = nested_options
else:
raise Exception("Options passed must be OrderedDict or list")
print('Show menu:', self.options)
super(Menu, self).__init__(disp)
def loop(self):
""" Main render loop """
#self.draw_legend()
self.redraw_menu()
return self.handle_buttons()
def redraw_menu(self):
# Draw menu
for i, opt in enumerate(self.options):
if self.is_line_entirely_in_display(i):
self.write_line(opt, line=i)
self.draw_line_pointer(self.current_index)
def handle_buttons(self):
# Handle button presses
if self.is_button_pressed(self.gpio.NEXT):
print("NEXT")
self.increment_select()
if self.is_button_first_pressed(self.gpio.SELECT):
print("SELECT")
self.selected_action()
if self.is_button_first_pressed(self.gpio.BACK) and self.has_parent:
print("BACK")
return True # quits outer render loop
# Update prev pressed values
for name in self.gpio.values():
self.was_button_pressed[name] = self.is_button_pressed(name)
def draw_legend(self):
"""Draw button legend & status"""
legend_width = 6
legend_x = self.display.width - legend_width - 1
legend_center = legend_x + legend_width / 2
# Button A - down arrow
self.draw.polygon(translate_coords([(0, 0), (6, 0), (3, 6)],
legend_center - 3, self.PADDING),
outline=255,
fill=int(self.is_button_pressed(self.gpio.NEXT)))
# Button B - circle
self.draw_circle(legend_center,
self.display.height / 2,
6,
outline=255,
fill=int(self.is_button_pressed(self.gpio.SELECT)))
# Button C - back arrow
self.draw.polygon(
translate_coords([(0, 3), (6, 0), (6, 6)], legend_center - 3,
self.display.height - 6 - self.PADDING),
outline=255,
fill=int(self.is_button_pressed(self.gpio.BACK)))
# Internal state management
current_index = 0 # Current selection index
scroll_frame = 0 # Top index of scroll frame
# Mapping from gpio functions to pins
gpio = AttrDict({
'NEXT': 'A',
'SELECT': 'B',
'BACK': 'C',
})
# Was button pressed on prev loop?
was_button_pressed = {name: False for name in gpio.values()}
def is_button_first_pressed(self, name):
"""Clean button press detection. Ignores continuous pressing."""
return self.is_button_pressed(
name) and not self.was_button_pressed[name]
def increment_select(self):
self.current_index = (self.current_index + 1) % len(self.options)
if not self.is_line_entirely_in_display(self.current_index):
self.scroll_frame += 1
if self.current_index == 0:
self.scroll_frame = 0
def selected_action(self):
"""Perform action for currently selected item"""
if not self.nested_options: # Flat list
return
child = self.nested_options.values()[self.current_index]
# No children, do nothing
if not child:
return
# Child is a Shim, perform action
elif isinstance(child, Shim):
if child.loading_message:
self.show_loading_screen(child.loading_message)
child.run()
if child.finished_message:
time.sleep(0.3)
self.show_loading_screen(child.finished_message)
time.sleep(0.3)
# Nested children, open a submenu
else:
submenu = Menu(self.display, child, has_parent=True)
# TODO: add 3rd button and then implement back functionality
# Line convenience helpers
def get_display_line_index(self, line):
"""Translate line index to the scroll frame"""
return line - self.scroll_frame
def is_line_entirely_in_display(self, line):
return ((self.get_menu_line_y(line + 1) < self.display.height)
and (self.get_menu_line_y(line) >= 0))
def is_line_out_of_frame(self, line):
return ((self.get_menu_line_y(line + 1) < 0)
or (self.get_menu_line_y(line) >= self.display.height))
# Constants
LINE_HEIGHT = 10
PADDING = 5
# Drawing helpers
def show_loading_screen(self, text):
"""Redraw screen with a loading message"""
self.clear()
# TODO: wrap text
self.write_line(text, line=1)
self.update_display()
def get_menu_line_y(self, line):
""":line: index is in absolute frame, not display frame"""
return self.PADDING + self.get_display_line_index(
line) * self.LINE_HEIGHT
def write_line(self, text, line=0):
"""Write text on line (check width; will overflow)"""
self.draw.text((2 * self.PADDING, self.get_menu_line_y(line)),
text,
font=self.font,
fill=1)
def draw_line_pointer(self, line, fill=0):
line = self.get_display_line_index(line)
self.draw.polygon(translate_coords([(2, 8), (6, 10.5), (2, 13)], 0,
line * self.LINE_HEIGHT),
outline=255,
fill=fill)
def draw_vertical_line(self, x, y0=0, y1=None, **kwargs):
if y1 is None:
y1 = self.display.height
self.draw.line([(x, y0), (x, y1)], **kwargs)
def draw_horizontal_line(self, y, x0=0, x1=None, **kwargs):
if x1 is None:
x1 = self.display.width
self.draw.line([(x0, y), (x1, y)], **kwargs)
def draw_circle(self, x, y, d, **kwargs):
"""Draw circle centered at (x, y) with diameter d"""
r = d * 0.5
self.draw.ellipse((x - r, y - r, x + r, y + r), **kwargs)
# graph.var_connectionA: var_connectionA,
})
prediction = prediction[0]
return prediction
DEFAULT_ARGS = AttrDict({
'BATCH_SIZE': 10,
'NODE_COUNT': 40,
'EPOCHS': 500,
'UPDATE_ITERATION': 4,
'EMBEDDING_SIZE': 100,
'HIDDEN_LAYERS': [100, 100],
'HIDDEN_ACTIVATION': tf.nn.relu,
'END_ACTIVATION': tf.nn.relu,
'LEARNING_RATE': 0.0001, # 0.002,
'ADD_VAR': True,
'COLLAPSE': False,
'RESIDUAL': True,
'ROOT_EMBEDDING': False,
'FILTER': None,
'NORMALIZE_VARS': True,
'SAME_LABEL_COLLAPSE': "mean", # "mean" / "max" / "prob"
'SAVE_FREQUENCY': 100,
})
def main():
# datafile = "trajectory.json"
# modelfile = "tmp/model_simple_pure"
# datafile = "robinson_random_longer1_trajectory.json"
def get_context_data(self, **kwargs):
from redis import Redis
from .log import get_redis_key, END
from util import AttrDict
data = super(SchedulerResultDetailView, self).get_context_data(**kwargs)
red = Redis()
task_id = self.kwargs["task_id"]
try:
# all the objects should have the same content type, here...
object_list = [scheduler.content_object for scheduler in Scheduler.objects.filter(logger_name=task_id)]
object_type_name = type(object_list[0]).__name__.lower()
data[object_type_name + "_list"] = object_list
if isinstance(object_list[0], Dataset):
data["source"] = data["dataset_list"][0].source
except IndexError:
dataset_pk = self.request.GET.get("dataset")
source_pk = self.request.GET.get("source")
if dataset_pk:
data["dataset_list"] = Dataset.objects.filter(pk=dataset_pk)
data["source"] = data["dataset_list"][0].source
elif source_pk:
data["source"] = Source.objects.get(pk=source_pk)
data["dataset_list"] = data["source"].datasets.all()
else:
pass
key = get_redis_key(task_id)
logs = []
start = max(int(self.request.GET.get("start", 0)), 0)
for idx, log in enumerate(red.lrange(key, start, -1)):
log_dict = AttrDict(json.loads(log))
log_dict.index = start + idx
log_dict.show_args = False
try:
if isinstance(log_dict.args, dict):
args = (log_dict.args,)
else:
args = tuple(log_dict.args)
log_dict.msg = log_dict.msg % args
except: # pylint: disable=W0702
log_dict.show_args = True
logs.append(log_dict)
data["logs"] = logs
data["task_id"] = task_id
if self.request.is_ajax():
data["parent"] = "layout/ajax.html"
else:
data["parent"] = "layout/site_base.html"
data["END"] = END
return data
def track_car(self, car_no: str) -> AttrDict:
'Track your rail shipment by car number.'
self.query['hwzz.carNo'] = car_no
response = self.fetch('hwzz_uouii.action', self.query)
assert response.success, response.get('message', response.get('msg'))
return AttrDict(response.object[0])
