class InfluxUplinkPlugin(AbstractPlugin, AggregateResultListener):
'''InfluxDB data uploader'''
SECTION = 'influx'
@staticmethod
def get_key():
return __file__
def __init__(self, core):
AbstractPlugin.__init__(self, core)
AggregateResultListener.__init__(self)
self.client = None
self.decoder = None
def get_available_options(self):
return ["address", "port", "tank_tag", "grafana_root", "grafana_dashboard"]
def start_test(self):
self.start_time = datetime.datetime.now()
def end_test(self, retcode):
self.end_time = datetime.datetime.now() + datetime.timedelta(minutes=1)
return retcode
def configure(self):
'''Read configuration'''
self.tank_tag = self.get_option("tank_tag", "none")
address = self.get_option("address", "localhost")
port = int(self.get_option("port", "8086"))
self.client = InfluxDBClient(
address, port, 'root', 'root', 'mydb')
grafana_root = self.get_option("grafana_root", "http://localhost/")
grafana_dashboard = self.get_option("grafana_dashboard", "tank-dashboard")
uuid=self.core.get_uuid()
LOG.info(
"Grafana link: {grafana_root}"
"dashboard/db/{grafana_dashboard}?var-uuid={uuid}&from=-5m&to=now".format(
grafana_root=grafana_root,
grafana_dashboard=grafana_dashboard,
uuid=uuid,
)
)
self.decoder = Decoder(self.tank_tag, uuid)
aggregator = self.core.get_plugin_of_type(AggregatorPlugin)
aggregator.add_result_listener(self)
def aggregate_second(self, data):
"""
@data: SecondAggregateData
"""
if self.client:
points = self.decoder.decode_aggregate(data)
self.client.write_points(points, 's')
def monitoring_data(self, data):
if self.client:
points = self.decoder.decode_monitoring(data)
self.client.write_points(points, 's')
def __init__(self):
self.decoded = Decoder()
self.decoded.decode()
self.calib = Calib()
self.minmax = self.decoded.minmax_img
self.pattern = self.decoded.decoded_indices
self.proj = (self.decoded.proj[1], self.decoded.proj[0])
self.out_cols = self.decoded.proj[1]
self.out_rows = self.decoded.proj[0]
self.pointcloud = np.zeros((self.out_rows, self.out_cols, 3))
self.cam_points = defaultdict(list)
self.proj_points = {}
self.threshold = THRESH
self.max_dist = MAX_DIST
def predict_train(self, data, print_stats=False):
golds, preds, graphs = [], [], []
fmaps = self.fx.fx_maps
scorer = self.clf.compute_scores
# initialize the decoder
arg_labels = ["ARG0", "ARG1", "ARG2", "ARG3", "ARG4", "ARG5",
# "ARG0-of", "ARG1-of", "ARG2-of", "ARG3-of", "ARG4-of", "ARG5-of",
]
arg_ids = [self.fx.fx_maps["ri_l2i"][l] for l in arg_labels]
self.decoder = Decoder(arg_elabs=arg_labels, arg_ids=arg_ids)
for idx, instance in enumerate(data):
goldG = instance["amrG"]
feats = instance["ri_data"]
aligned_nodes = filter(lambda x: goldG.node[x[0]]["span"][0] is not None, goldG.nodes(data=True))
G = nx.DiGraph(root=None, comments=instance["comments"])
if len(aligned_nodes) == 0:
snt = " ".join([T["word"] for T in instance["tokens"]])
print "No aligned nodes in AMR for snt (%s): %s" % (instance["comments"]["id"], snt)
# tokid = 0
# random_concept = instance["tokens"][tokid]["lemma"]
# self.ruleset.fill_one_node(G, tokid, random_concept, var=True)
# graphs.append(G)
# fixme: in JAMR, they exclude these graphs at all?
goldG.graph["comments"] = instance["comments"]
graphs.append(goldG)
continue
G.add_nodes_from(aligned_nodes)
# In Flanigan et al., these are edges from the CI stage to be preserved
# we disregard them, as there are hardly any
# E_0 = G.edges(data=False)
self.decoder.decode_one(feats, G, scorer, fmaps)
preds.append([(e[0], e[1], e[2]["label"]) for e in G.edges(data=True)])
golds.append([(e[0], e[1], e[2]["label"]) for e in goldG.edges(data=True)])
graphs.append(G)
# print "ND graphs decoding (+/-): %d/%d" %(self.decoder.success, self.decoder.failed)
self.decoder.success = self.decoder.failed = 0
return golds, preds, graphs
def get_decode(filename):
#dekey_dic = {'expback_2078pyc_dis.py': '28221b5847a673c1d138bec680e5aab1421a9eed705aca4c316159fd2291b910', 'expback_12pyc_dis.py': 'c3ff499db0ad225bc0cbd0f9cbd7910edaa8861a1f3ebe4cc7b168d1bdad3254', 'expback_189pyc_dis.py': 'a13b6776facce2ce24b9407fe76b7d9a2ac9f97fd11b4c03da49c5dc1bfdd4ed', 'expback_77pyc_dis.py': '345095a6a09c0643bcf41007fd1311cdf4889004e886b2bca8d4881fb27a7fca', 'expback_72pyc_dis.py': '8c97d8c12ebb049684db59720d39ad8b38b0081d8cc8d022bd7768ab0bc7c699', 'expback_24pyc_dis.py': 'a27d8237a5f282cdeae742d17cd4e2ca40a686f9debe7307d414394cf8eb469a', 'expback_88pyc_dis.py': 'e77e60afe46271e855a1aaf4738acb8d2712649d7a0e38ee6de5e3cb6e102b19', 'expback_2065pyc_dis.py': '345095a6a09c0643bcf41007fd1311cdf4889004e886b2bca8d4881fb27a7fca', 'exp4.py': '253f4221df8307dfb23c39726a022382162a520739738590c39520b032c15c30', 'expback_38pyc_dis.py': '8505495a868258d1a09f88cf12b87431531bedca34a3dafc03ab58d741c0bbd7', 'expback_1756pyc_dis.py': 'ab5d21d688e5789a47b617d47dc68e48a51243b1b93d3beaf737e411310753fe', 'exp104.py': '145f01b740b46451cc03bbae9d56fe31e385aa681381a9f4ce445f7997baeff5', 'expback_76pyc_dis.py': '0d98a92bbfd99bbfbcf9419d686661ac36d55d2a7ecc0c19768b0d7b0bd9191d', 'expback_18pyc_dis.py': '38ad0c291a56f74acaee1019f24f188a7ddbb6cc51d7e4c29fa993b568404fd1', 'expback_70pyc_dis.py': 'efb1fdfd9905e92bacd3a5367c4727dc7ae722ab7f214e1434b6e25041d34190', 'expback_8pyc_dis.py': 'a13b6776facce2ce24b9407fe76b7d9a2ac9f97fd11b4c03da49c5dc1bfdd4ed', 'expback_6pyc_dis.py': '1d2d57097a9f25403de685038570b272b68e61ec0ad821f5db4ce2e380ba4f4e', 'expback_33pyc_dis.py': '48a2a41fd1a72f6feb52c058c767b31726ef9480f5624b9733cf8088e26475b6', 'expback_26pyc_dis.py': 'a27d8237a5f282cdeae742d17cd4e2ca40a686f9debe7307d414394cf8eb469a', 'expback_28pyc_dis.py': 'c3ff499db0ad225bc0cbd0f9cbd7910edaa8861a1f3ebe4cc7b168d1bdad3254', 'expback_1994pyc_dis.py': 'ffcc396606397e831f857f22b90d87ca05dd77452e7c2760df0f39d9d3f664cb', 'expback_22pyc_dis.py': '3f9632f701953df91e7b13b428d18ab7a549520831cc2a46984c83e81b933673', 'expback_23pyc_dis.py': 'e0a46b005bc3e4b63bf33f9097023d87614810c0b71a355e0934a7bc8a862f32', 'expback_1766pyc_dis.py': 'ac03b075a298860de07a3b68886ffff1dd2cec245ee4a7ea7c3e677dd9cf9cb0', 'expback_1070pyc_dis.py': '24000815b2f04e2f070d02f649539c6c6330a1bc45ad798962466966b7a220a4', 'expback_83pyc_dis.py': 'dd9108e2cb4dce78981dc247e24bd0df5c4b004058ebf271788a8e792d85a026', 'expback_100pyc_dis.py': 'a13b6776facce2ce24b9407fe76b7d9a2ac9f97fd11b4c03da49c5dc1bfdd4ed', 'expback_2112pyc_dis.py': 'ac03b075a298860de07a3b68886ffff1dd2cec245ee4a7ea7c3e677dd9cf9cb0', 'expback_29pyc_dis.py': '24000815b2f04e2f070d02f649539c6c6330a1bc45ad798962466966b7a220a4', 'expback_89pyc_dis.py': '0d4af56f54b549460eae50cb9dc579022c7e046e050fbc72242da5f616e21867', 'expback_57pyc_dis.py': 'ef632082c7620cf54876da74a1660bfb9c06eb94549b5f3bca646474000d0c46', 'expback_63pyc_dis.py': 'a13b6776facce2ce24b9407fe76b7d9a2ac9f97fd11b4c03da49c5dc1bfdd4ed', 'expback_64pyc_dis.py': 'ef632082c7620cf54876da74a1660bfb9c06eb94549b5f3bca646474000d0c46', 'expback_811pyc_dis.py': 'f8feb1e2013b989686230a93b8a543f2db83f2cc6b4dbc40f8b30bdf5e0dfeb9', 'expback_54pyc_dis.py': '4903f9969575cdef55f7b2ed2a12f89b97664e7fb5eb0898e6fd4f6775f166f3', 'expback_34pyc_dis.py': 'd289da3e7b9c736756e3429c23db20228f8e3547d3a4b540da1f86aaf22ff02f', 'expback_2083pyc_dis.py': '345095a6a09c0643bcf41007fd1311cdf4889004e886b2bca8d4881fb27a7fca', 'expback_15pyc_dis.py': 'a13b6776facce2ce24b9407fe76b7d9a2ac9f97fd11b4c03da49c5dc1bfdd4ed', 'expback_1055pyc_dis.py': '7b650aed66397b6e1ba67dfdd39c9626c4c9fec89eebf397a76caa9d0ce45d26', 'expback_60pyc_dis.py': 'a13b6776facce2ce24b9407fe76b7d9a2ac9f97fd11b4c03da49c5dc1bfdd4ed', 'expback_65pyc_dis.py': '24000815b2f04e2f070d02f649539c6c6330a1bc45ad798962466966b7a220a4', 'expback_59pyc_dis.py': '6ff51d6ad8855be91270868f2f9d9b2e225c722941906d5c0728a9409ef23b50', 'expback_2067pyc_dis.py': 'ac03b075a298860de07a3b68886ffff1dd2cec245ee4a7ea7c3e677dd9cf9cb0', 'expback_75pyc_dis.py': '3f9632f701953df91e7b13b428d18ab7a549520831cc2a46984c83e81b933673', 'expback_62pyc_dis.py': 'de92d01ce1a391792c2d4f41996a0b77c6c47605f032bdca21b0b5fd36c0f0f6', 'expback_1786pyc_dis.py': '345095a6a09c0643bcf41007fd1311cdf4889004e886b2bca8d4881fb27a7fca', 'expback_1745pyc_dis.py': '39b9843a532fed97e232f0a471ffbfb4079d8163a5e7dd52681e57c1e6520e53', 'expback_61pyc_dis.py': 'e1244c9d55465d4083a2a832fc4732472b91426238d7935d0e39126138afdac1', 'expback_39pyc_dis.py': '9a176f89756545161a807d6b5803333756eccaaad7ea2daa4e5eeb6c37a09ec0', 'expback_58pyc_dis.py': '2ec6a4bd513d71efdaaccb3aaf5c27487aef537f6091d14fcce0617c29dbf424', 'expback_2110pyc_dis.py': '33d7e04bb6b1e472eddb674398e5a3fce8abed59863e43f727ad78498ed63c27', 'expback_13pyc_dis.py': '253f4221df8307dfb23c39726a022382162a520739738590c39520b032c15c30', 'expback_1995pyc_dis.py': 'ef632082c7620cf54876da74a1660bfb9c06eb94549b5f3bca646474000d0c46', 'expback_71pyc_dis.py': 'ef632082c7620cf54876da74a1660bfb9c06eb94549b5f3bca646474000d0c46', 'expback_37pyc_dis.py': 'bce1c8cb73e24b4a1518702a3080d00aad9583c2a25f4c59b4dffb06c009a25f', 'expback_101pyc_dis.py': 'a13b6776facce2ce24b9407fe76b7d9a2ac9f97fd11b4c03da49c5dc1bfdd4ed', 'expback_25pyc_dis.py': 'ac03b075a298860de07a3b68886ffff1dd2cec245ee4a7ea7c3e677dd9cf9cb0', 'expback_16pyc_dis.py': 'f5cebead3a728f681d272e67879a393bc37e3c857076d1b09926eeee1a34739b', 'expback_45pyc_dis.py': '6eaf26b1043248ae94ca258db5d5b068a610a213aa1d2af703532163d0bd1717', 'expback_30pyc_dis.py': 'e034570d4d73b2deeed98ff76911c89ff03ae6f0cef61a09f4091b55783c18b2', 'expback_102pyc_dis.py': '1d2d57097a9f25403de685038570b272b68e61ec0ad821f5db4ce2e380ba4f4e', 'expback_19pyc_dis.py': 'bce1c8cb73e24b4a1518702a3080d00aad9583c2a25f4c59b4dffb06c009a25f', 'expback_1062pyc_dis.py': '345095a6a09c0643bcf41007fd1311cdf4889004e886b2bca8d4881fb27a7fca', 'expback_17pyc_dis.py': 'b9e36259d273b00edcbf28048f0b716e08634efaab283f693ca067fe2162f575', 'expback_36pyc_dis.py': '145f01b740b46451cc03bbae9d56fe31e385aa681381a9f4ce445f7997baeff5', 'expback_74pyc_dis.py': 'a13b6776facce2ce24b9407fe76b7d9a2ac9f97fd11b4c03da49c5dc1bfdd4ed', 'expback_641pyc_dis.py': 'bb4d873dfab45ce19dbb43ea954ddf1248a2a41fd1a72f6feb52c058c767b317', 'exp622.py': '145f01b740b46451cc03bbae9d56fe31e385aa681381a9f4ce445f7997baeff5', 'expback_73pyc_dis.py': 'c3facf1cb2752b65516b130189a508e1469f38055b509392cba7b314ffa070fb', 'expback_1071pyc_dis.py': '0d4af56f54b549460eae50cb9dc579022c7e046e050fbc72242da5f616e21867', 'expback_2066pyc_dis.py': '345095a6a09c0643bcf41007fd1311cdf4889004e886b2bca8d4881fb27a7fca', 'expback_1056pyc_dis.py': 'd876bfdb7f8cc8e92678c67ed1db7e37a95dd5ef8aaeb304cb7f8a8e86dd9dbc'}
dekey_dic = {'expback_64pyc_dis.py': 'ef632082c7620cf54876da74a1660bfb9c06eb94549b5f3bca646474000d0c46', 'expback_2066pyc_dis.py': '345095a6a09c0643bcf41007fd1311cdf4889004e886b2bca8d4881fb27a7fca', 'expback_57pyc_dis.py': 'ef632082c7620cf54876da74a1660bfb9c06eb94549b5f3bca646474000d0c46', 'exp2366.py': '', 'exp2016.py': '', 'expback_100pyc_dis.py': 'a13b6776facce2ce24b9407fe76b7d9a2ac9f97fd11b4c03da49c5dc1bfdd4ed', 'expback_1055pyc_dis.py': '7b650aed66397b6e1ba67dfdd39c9626c4c9fec89eebf397a76caa9d0ce45d26', 'expback_45pyc_dis.py': '6eaf26b1043248ae94ca258db5d5b068a610a213aa1d2af703532163d0bd1717', 'expback_22pyc_dis.py': '3f9632f701953df91e7b13b428d18ab7a549520831cc2a46984c83e81b933673', 'expback_25pyc_dis.py': 'ac03b075a298860de07a3b68886ffff1dd2cec245ee4a7ea7c3e677dd9cf9cb0', 'expback_15pyc_dis.py': '', 'expback_28pyc_dis.py': 'c3ff499db0ad225bc0cbd0f9cbd7910edaa8861a1f3ebe4cc7b168d1bdad3254', 'expback_13pyc_dis.py': '253f4221df8307dfb23c39726a022382162a520739738590c39520b032c15c30', 'expback_61pyc_dis.py': 'e1244c9d55465d4083a2a832fc4732472b91426238d7935d0e39126138afdac1', 'expback_33pyc_dis.py': '48a2a41fd1a72f6feb52c058c767b31726ef9480f5624b9733cf8088e26475b6', 'expback_1071pyc_dis.py': '3d500608da701822cdb9d87c98f28f44eb85b9df06157f3719926283397b40f0', 'expback_101pyc_dis.py': 'a13b6776facce2ce24b9407fe76b7d9a2ac9f97fd11b4c03da49c5dc1bfdd4ed', 'expback_23pyc_dis.py': 'e0a46b005bc3e4b63bf33f9097023d87614810c0b71a355e0934a7bc8a862f32', 'expback_29pyc_dis.py': '24000815b2f04e2f070d02f649539c6c6330a1bc45ad798962466966b7a220a4', 'expback_39pyc_dis.py': '9a176f89756545161a807d6b5803333756eccaaad7ea2daa4e5eeb6c37a09ec0', 'expback_59pyc_dis.py': '6ff51d6ad8855be91270868f2f9d9b2e225c722941906d5c0728a9409ef23b50', 'expback_62pyc_dis.py': 'de92d01ce1a391792c2d4f41996a0b77c6c47605f032bdca21b0b5fd36c0f0f6', 'expback_2065pyc_dis.py': '345095a6a09c0643bcf41007fd1311cdf4889004e886b2bca8d4881fb27a7fca', 'expback_1070pyc_dis.py': '', 'expback_2112pyc_dis.py': 'ac03b075a298860de07a3b68886ffff1dd2cec245ee4a7ea7c3e677dd9cf9cb0', 'expback_98pyc_dis.py': '', 'exp1484.py': '', 'expback_82pyc_dis.py': 'a13b6776facce2ce24b9407fe76b7d9a2ac9f97fd11b4c03da49c5dc1bfdd4ed', 'expback_1056pyc_dis.py': 'd876bfdb7f8cc8e92678c67ed1db7e37a95dd5ef8aaeb304cb7f8a8e86dd9dbc', 'expback_1756pyc_dis.py': 'ab5d21d688e5789a47b617d47dc68e48a51243b1b93d3beaf737e411310753fe', 'expback_953pyc_dis.py': '', 'expback_63pyc_dis.py': 'a13b6776facce2ce24b9407fe76b7d9a2ac9f97fd11b4c03da49c5dc1bfdd4ed', 'expback_60pyc_dis.py': 'a13b6776facce2ce24b9407fe76b7d9a2ac9f97fd11b4c03da49c5dc1bfdd4ed', 'expback_83pyc_dis.py': 'dd9108e2cb4dce78981dc247e24bd0df5c4b004058ebf271788a8e792d85a026', 'expback_2083pyc_dis.py': '345095a6a09c0643bcf41007fd1311cdf4889004e886b2bca8d4881fb27a7fca', 'expback_641pyc_dis.py': 'bb4d873dfab45ce19dbb43ea954ddf1248a2a41fd1a72f6feb52c058c767b317', 'expback_1766pyc_dis.py': 'ac03b075a298860de07a3b68886ffff1dd2cec245ee4a7ea7c3e677dd9cf9cb0', 'expback_2110pyc_dis.py': '33d7e04bb6b1e472eddb674398e5a3fce8abed59863e43f727ad78498ed63c27', 'exp679.py': '', 'expback_19pyc_dis.py': '8505495a868258d1a09f88cf12b87431531bedca34a3dafc03ab58d741c0bbd7', 'expback_76pyc_dis.py': '0d98a92bbfd99bbfbcf9419d686661ac36d55d2a7ecc0c19768b0d7b0bd9191d', 'expback_70pyc_dis.py': 'efb1fdfd9905e92bacd3a5367c4727dc7ae722ab7f214e1434b6e25041d34190', 'exp2106.py': '', 'expback_72pyc_dis.py': '8c97d8c12ebb049684db59720d39ad8b38b0081d8cc8d022bd7768ab0bc7c699', 'expback_73pyc_dis.py': 'c3facf1cb2752b65516b130189a508e1469f38055b509392cba7b314ffa070fb', 'expback_89pyc_dis.py': '0d4af56f54b549460eae50cb9dc579022c7e046e050fbc72242da5f616e21867', 'expback_18pyc_dis.py': '54cf6db8b1c27e5c0c73a151a9a0ccdd7b3e2dbccc5c00c041c3479d205a62c4', 'expback_77pyc_dis.py': '345095a6a09c0643bcf41007fd1311cdf4889004e886b2bca8d4881fb27a7fca', 'expback_30pyc_dis.py': 'e034570d4d73b2deeed98ff76911c89ff03ae6f0cef61a09f4091b55783c18b2', 'exp2355.py': '', 'expback_65pyc_dis.py': '345095a6a09c0643bcf41007fd1311cdf4889004e886b2bca8d4881fb27a7fca', 'exp4.py': '', 'expback_2067pyc_dis.py': 'ac03b075a298860de07a3b68886ffff1dd2cec245ee4a7ea7c3e677dd9cf9cb0', 'expback_35pyc_dis.py': '', 'expback_6pyc_dis.py': '3f9632f701953df91e7b13b428d18ab7a549520831cc2a46984c83e81b933673', 'expback_1994pyc_dis.py': 'ffcc396606397e831f857f22b90d87ca05dd77452e7c2760df0f39d9d3f664cb', 'expback_1062pyc_dis.py': '345095a6a09c0643bcf41007fd1311cdf4889004e886b2bca8d4881fb27a7fca', 'expback_1786pyc_dis.py': '345095a6a09c0643bcf41007fd1311cdf4889004e886b2bca8d4881fb27a7fca', 'expback_58pyc_dis.py': '2ec6a4bd513d71efdaaccb3aaf5c27487aef537f6091d14fcce0617c29dbf424', 'expback_24pyc_dis.py': 'a27d8237a5f282cdeae742d17cd4e2ca40a686f9debe7307d414394cf8eb469a', 'exp2357.py': '', 'expback_34pyc_dis.py': 'd289da3e7b9c736756e3429c23db20228f8e3547d3a4b540da1f86aaf22ff02f', 'exp622.py': '', 'expback_17pyc_dis.py': 'b9e36259d273b00edcbf28048f0b716e08634efaab283f693ca067fe2162f575', 'expback_36pyc_dis.py': '145f01b740b46451cc03bbae9d56fe31e385aa681381a9f4ce445f7997baeff5', 'expback_54pyc_dis.py': '4903f9969575cdef55f7b2ed2a12f89b97664e7fb5eb0898e6fd4f6775f166f3', 'expback_12pyc_dis.py': 'e3bb69fcc78187a0039fccb03c46298456d5ffb095a1203945c91c59ca3e1993', 'expback_71pyc_dis.py': 'ef632082c7620cf54876da74a1660bfb9c06eb94549b5f3bca646474000d0c46', 'expback_8pyc_dis.py': 'f8302acee10371dc21ac9029b3a35f45bcdc1b3ecfefefb25771bac202ac32ec', 'expback_811pyc_dis.py': 'f8feb1e2013b989686230a93b8a543f2db83f2cc6b4dbc40f8b30bdf5e0dfeb9', 'expback_75pyc_dis.py': '3f9632f701953df91e7b13b428d18ab7a549520831cc2a46984c83e81b933673', 'exp1962.py': '', 'expback_1995pyc_dis.py': 'ef632082c7620cf54876da74a1660bfb9c06eb94549b5f3bca646474000d0c46', 'expback_1745pyc_dis.py': '39b9843a532fed97e232f0a471ffbfb4079d8163a5e7dd52681e57c1e6520e53', 'exp849.py': '', 'expback_16pyc_dis.py': 'f5cebead3a728f681d272e67879a393bc37e3c857076d1b09926eeee1a34739b', 'expback_88pyc_dis.py': 'e77e60afe46271e855a1aaf4738acb8d2712649d7a0e38ee6de5e3cb6e102b19', 'expback_2078pyc_dis.py': '28221b5847a673c1d138bec680e5aab1421a9eed705aca4c316159fd2291b910', 'expback_74pyc_dis.py': 'a13b6776facce2ce24b9407fe76b7d9a2ac9f97fd11b4c03da49c5dc1bfdd4ed', 'expback_26pyc_dis.py': 'bb4d873dfab45ce19dbb43ea954ddf1248a2a41fd1a72f6feb52c058c767b317', 'expback_189pyc_dis.py': 'a13b6776facce2ce24b9407fe76b7d9a2ac9f97fd11b4c03da49c5dc1bfdd4ed', 'expback_37pyc_dis.py': 'bce1c8cb73e24b4a1518702a3080d00aad9583c2a25f4c59b4dffb06c009a25f', 'expback_102pyc_dis.py': '1d2d57097a9f25403de685038570b272b68e61ec0ad821f5db4ce2e380ba4f4e', 'expback_38pyc_dis.py': '8505495a868258d1a09f88cf12b87431531bedca34a3dafc03ab58d741c0bbd7','expback_1054pyc_dis.py':'ac03b075a298860de07a3b68886ffff1dd2cec245ee4a7ea7c3e677dd9cf9cb0'}
try:
de_key = dekey_dic[filename]
except:
return ''
decode = Decoder(binascii.a2b_hex(de_key)).decode
return decode
def configure(self):
'''Read configuration'''
self.tank_tag = self.get_option("tank_tag", "none")
address = self.get_option("address", "localhost")
port = int(self.get_option("port", "8086"))
self.client = InfluxDBClient(
address, port, 'root', 'root', 'mydb')
grafana_root = self.get_option("grafana_root", "http://localhost/")
grafana_dashboard = self.get_option("grafana_dashboard", "tank-dashboard")
uuid=self.core.get_uuid()
LOG.info(
"Grafana link: {grafana_root}"
"dashboard/db/{grafana_dashboard}?var-uuid={uuid}&from=-5m&to=now".format(
grafana_root=grafana_root,
grafana_dashboard=grafana_dashboard,
uuid=uuid,
)
)
self.decoder = Decoder(self.tank_tag, uuid)
aggregator = self.core.get_plugin_of_type(AggregatorPlugin)
aggregator.add_result_listener(self)
def predict_test(self, data,evaluate=False):
graphs = []
fmaps = self.fx.fx_maps
scorer = self.clf.compute_scores
self.fx.fx_dataset_ri(data, train=False)
# initialize the decoder
arg_labels = ["ARG0", "ARG1", "ARG2", "ARG3", "ARG4", "ARG5",
# "ARG0-of", "ARG1-of", "ARG2-of", "ARG3-of", "ARG4-of", "ARG5-of",
]
arg_ids = [self.fx.fx_maps["ri_l2i"][l] for l in arg_labels]
self.decoder = Decoder(arg_elabs=arg_labels, arg_ids=arg_ids)
for idx, instance in enumerate(data):
feats = instance["ri_data"]
# we have a partially built G from the CI stage
# no nodes are unaligned
G = instance["amrG"]
if len(G.nodes()) == 0:
snt = " ".join([T["word"] for T in instance["tokens"]])
print "No aligned nodes! Sentence (%s): %s" % (instance["comments"]["id"], snt)
print "This should not happen!"
sys.exit()
tokid = 0
random_concept = instance["tokens"][tokid]["lemma"]
self.ruleset.fill_one_node(G, tokid, random_concept, var=True)
graphs.append(G)
continue
self.decoder.decode_one(feats, G, scorer, fmaps)
graphs.append(G)
return graphs
def __init__(self, username, password):
self.username = username
self.password = password
self.decoder = Decoder()
parser.add_argument('-alpha', '-a', default=0.3, help="complexity threshold")
parser.add_argument(
'-outfile',
'-o',
default="embedded_vessel.png",
help="desired vessel output name(default 'embedded_vessel.png')")
args = parser.parse_args()
#args
op = args.op
alpha = args.alpha
infile = args.infile
vessel = args.vessel
outfile = args.outfile
if (op == 'encode' or op == '-e'): #encode
encoder = Encoder(infile, outfile, vessel, alpha)
print("Verifying image capacity...")
encoder.verify()
print("Embedding data into vessel...")
encoder.embed()
print("Reconstructing vessel...")
encoder.reconstruct()
elif (op == 'decode' or op == '-d'): #decode
decoder = Decoder(outfile, alpha)
print("Retreiving data from vessel...")
decoder.bitPlaneArr()
print("Extracting files...")
decoder.extract()
else:
parser.print_help()
def run(self):
# update per-sentence grammars, if there's any
for g in self.grammars:
g.update(self.id)
self.flog = open('%s/%s_%s' % (FLAGS.run_dir, 'log', self.suffix), 'w')
if FLAGS.show_time:
self.flog.write('running on %s\n\n' % socket.gethostname())
self.flog.flush()
fwords = self.line.split()
if FLAGS.preprocess:
self.fidx2replacement = {}
j = 0
for i, token in enumerate(fwords):
if token in ('$number', '$date'):
self.fidx2replacement[i] = self.special[j][1]
j += 1
self.flog.write('[%s][%s words] %s\n' %
(self.id, len(fwords), self.line))
decoder = Decoder(fwords, self.grammars, self.features)
begin_time = time()
if FLAGS.decoding_method == 'agenda':
item = decoder.decode()
elif FLAGS.decoding_method == 'cyk':
item = decoder.decode_cyk()
elif FLAGS.decoding_method == 'earley':
item = decoder.decode_earley()
else:
assert False, '"%s" not valid decoding option' \
% FLAGS.decoding_method
self.time = time() - begin_time
if item is None:
self.out = '[decoder failed to build a goal item]'
else:
hg = Hypergraph(item)
hg.set_semiring(hypergraph.SHORTEST_PATH)
hg.set_functions(lambda x: x.cost, None, None)
hg.topo_sort()
self.kbest = hg.root.best_paths()
output_tokens = self.kbest[0].translation[:]
if FLAGS.preprocess:
for i in range(len(output_tokens)):
if output_tokens[i] in ('$number', '$date'):
fidx = self.kbest[0].composed_rule.we2f[i]
if fidx is not None:
output_tokens[i] = self.fidx2replacement[fidx]
self.out = ' '.join(output_tokens[FLAGS.lm_order - 1:1 -
FLAGS.lm_order])
self.hg = hg
if FLAGS.output_hypergraph:
self.write_hypergraph()
self.flog.write('%s\n' % self.out)
self.flog.write('\n')
if item is not None:
self.flog.write(self.kbest[0].tree_str())
self.flog.write('\n')
self.flog.write(hg.stats())
self.flog.write('\n')
self.flog.write(decoder.agenda_stats())
self.flog.write('\n')
self.flog.write(decoder.chart.stats())
self.flog.write('\n')
for dotchart in decoder.dotcharts:
self.flog.write(dotchart.stats())
self.flog.write('\n')
if FLAGS.show_time:
timeline = '{:<35}{:>15.2f}\n'.format('[time]:', self.time)
self.flog.write(timeline)
self.write_output_file()
if FLAGS.output_kbest:
self.write_kbest_to_file()
self.flog.close()
# -*- coding: utf-8 -*-
import binascii, re
import fcntl
from decode import Decoder
de_key = 'e3bb69fcc78187a0039fccb03c46298456d5ffb095a1203945c91c59ca3e1993'
decode = Decoder(binascii.a2b_hex(de_key)).decode
from dummy import *
from miniCurl import Curl
curl = Curl()
#Embedded file name: robots.py
import re
import urlparse
if 0:
i11iIiiIii
def assign(service, arg):
if service != '''www''':
return
else:
OO0o = urlparse.urlparse(arg)
return (True, '''%s://%s/robots.txt''' % (OO0o.scheme, OO0o.netloc))
if 0:
Iii1I1 + OO0O0O % iiiii % ii1I - ooO0OO000o
def audit(arg):
ii11i = arg
oOooOoO0Oo0O, iI1, i1I11i, OoOoOO00, I11i = curl.curl(ii11i)
if oOooOoO0Oo0O == 200:
def __init__(self, feature_opts={}, decoding='mst'):
self.feature_opts = feature_opts
self.arc_perceptron = ArcPerceptron(self.feature_opts)
self.decoder = Decoder(decoding)
self.arc_accuracy = None
def post(self):
if 'image' in request.files:
image_arr = np.fromstring(request.files['image'].read(), np.uint8)
decoder = Decoder(image_arr)
return decoder.decode()
return "Error finding file, please upload image with key 'image'"
class reconstruct():
def __init__(self):
self.decoded = Decoder()
self.decoded.decode()
self.calib = Calib()
self.minmax = self.decoded.minmax_img
self.pattern = self.decoded.decoded_indices
self.proj = (self.decoded.proj[1], self.decoded.proj[0])
self.out_cols = self.decoded.proj[1]
self.out_rows = self.decoded.proj[0]
self.pointcloud = np.zeros((self.out_rows, self.out_cols, 3))
self.cam_points = defaultdict(list)
self.proj_points = {}
self.threshold = THRESH
self.max_dist = MAX_DIST
def reconstruct(self):
def pattern_invalid(thresh, pattern, minmax):
check = pattern[0] < 0 \
or pattern[0] >= self.proj[0] \
or pattern[1] < 0 \
or pattern[1] >= self.proj[1] \
or minmax[1] - minmax[0] < thresh
return check
# create 1-to-many correspondnence between projector coordinates
# and camera coordinates
pattern = self.pattern
minmax = self.minmax
for i in np.ndindex(pattern.shape[:2]):
if pattern_invalid(self.threshold, pattern[i], minmax[i]):
continue
proj_point = pattern[i]
index = proj_point[1] * self.out_cols + proj_point[0]
self.proj_points[index] = proj_point
self.cam_points[index].append(i)
# find average of the many camera coordinates
# for each projector coordinate
for key in self.proj_points.keys():
proj_point = self.proj_points[key]
cam_points = self.cam_points[key]
count = len(cam_points)
if count == 0: continue
x = 0
y = 0
for point in cam_points:
y += point[0]
x += point[1]
y /= count
x /= count
cam = (x, y)
proj = proj_point
p, dist = self.triangulate_stereo(cam, proj)
if dist < self.max_dist:
self.pointcloud[(proj[1], proj[0])] = np.asarray(p).flatten()
np.save('cloud.npy', self.pointcloud)
print "done"
def triangulate_stereo(self, p1, p2):
K1 = self.calib.cam_K
K2 = self.calib.proj_K
Rt = np.mat(self.calib.Rt)
T = np.mat(self.calib.T)
p_1 = np.array(p1, dtype=float)
p_2 = np.array(p2, dtype=float)
#TODO apply radial and tangential undistortion
u1 = np.array(((p_1[0] - K1[0, 2]) / K1[0, 0],
(p_1[1] - K1[1, 2]) / K1[1, 1], 1.0),
dtype=float)
u2 = np.array(((p_2[0] - K2[0, 2]) / K2[0, 0],
(p_2[1] - K2[1, 2]) / K2[1, 1], 1.0),
dtype=float)
u1 = np.mat(u1).reshape(3, 1)
u2 = np.mat(u2).reshape(3, 1)
v1 = u1
v2 = Rt * (u2 - T)
w1 = v1
w2 = Rt * u2
p, dist = self.appox_ray_x(v1, w1, v2, w2)
return p, dist
def appox_ray_x(self, v1, q1, v2, q2):
v1tv1 = float(v1.transpose() * v1)
v1tv2 = float(v1.transpose() * v2)
v2tv1 = float(v2.transpose() * v1)
v2tv2 = float(v2.transpose() * v2)
detV = v1tv1 * v2tv2 - v1tv2 * v2tv2
q2_q1 = q2 - q1
Q1 = float(v1.transpose() * q2_q1)
Q2 = -1.0 * float(v2.transpose() * q2_q1)
lam1 = (v2tv2 * Q1 + v1tv2 * Q2) / detV
lam2 = (v2tv1 * Q1 + v1tv1 * Q2) / detV
p1 = lam1 * v1 + q1
p2 = lam2 * v2 + q2
p3d = 0.5 * (p1 + p2)
dist = np.linalg.norm(p2 - p1)
return p3d, dist
#!/usr/bin/python
# -*- coding: utf-8 -*-
# vim:fileencoding=utf-8
from pyfinite import ffield
from poly import Poly
from code import CodeCreator
from encode import Encoder
from decode import Decoder
import time
import sys
###########################
if (sys.argv[1] == "mode=1"):
n = int(sys.argv[2])
p = float(sys.argv[3])
Code = CodeCreator(n, p)
Code.write_code_to_file(sys.argv[4])
elif (sys.argv[1] == "mode=2"):
enc = Encoder(sys.argv[2])
encode_str = enc.encode_file(sys.argv[3])
enc.write_to_file_with_noise(encode_str, sys.argv[4])
enc.write_to_file(encode_str, sys.argv[5])
elif (sys.argv[1] == "mode=3"):
dec = Decoder(sys.argv[2])
decode_str = dec.decode_file(sys.argv[3], sys.argv[4])
class RI_Stage(Stage):
def train_setup(self):
from featlists.ri import feats as flist
assert "train_pkl" in self.params and "dev_pkl" in self.params
logger.info(" * TRAINING: %s stage * " % self.name)
vocab = util.load_pkl(self.params["vocab_fn"])
train_data = util.load_pkl(self.params["train_pkl"])
val_data = util.load_pkl(self.params["dev_pkl"])
test_data = util.load_pkl(self.params["test_pkl"]) if "test_pkl" in self.params else None
labels = ["<null>", "<unk>"] + vocab["edges"].keys()
self.fx = Fxtractor(fx_maps=collections.defaultdict(), labels=labels, flist=flist)
self.fx.fx_dataset_ri(train_data, train=True)
self.fx.fx_dataset_ri(val_data, train=False)
n_features = self.fx.fx_maps["ri_fdim"]
n_classes = len(self.fx.fx_maps["ri_l2i"])
self.clf = Perceptron.SparsePerceptron()
self.clf.init_w(n_features, n_classes)
self.amr_printer = Printer()
logger.debug("Data: %d (train), %d (dev). Num feats %d" % (len(train_data), len(val_data), n_features))
logger.debug("Used features: \n%s", "\n".join(self.fx.flist))
return vocab, train_data, val_data, test_data
def train(self):
vocab, train_data, val_data, test_data = self.train_setup()
for i in range(self.params["nepochs"]):
random.seed(i)
random.shuffle(train_data)
logger.info("Iteration %d/%d" % (i + 1, self.params["nepochs"]))
for idx, instance in enumerate(train_data):
# if not idx % 1000:
# logger.info("Processing instance %d" % idx)
self.train_one(instance)
# evaluation on dev set
self.evaluate(val_data, section="dev",save=False)
# average the parameters and evaluate on the dev set
self.clf.avg_weights()
self.evaluate(val_data, section="dev", save=True)
# evaluate on the test set, if there is one
if test_data is not None:
self.fx.fx_dataset_ri(test_data, train=False)
self.evaluate(test_data, section="test", save=True)
def get_dephead_token(self, nid, aG, toposort):
start, end = aG.node[nid]["span"]
tokid_range = end - start
if tokid_range == 1:
head_tokid = start
head_idx = toposort.index(head_tokid)
else:
tokids = range(start, end)
head_idx = max([toposort.index(i) for i in tokids])
head_tokid = toposort[head_idx]
return (head_idx, head_tokid)
def train_one(self, instance):
amrG = instance["amrG"]
tokens = instance["tokens"]
gold_nodes = filter(lambda x: amrG.node[x]["span"][0] is not None, amrG.nodes())
ri_data = instance["ri_data"]
# A = [" ".join(T["concepts"][1:]) for T in tokens if len(T["node_id"]) > 1 and "name" not in T["concepts"][1:]]
# if len(A) > 0:
# pprint.pprint(A)
for node_pair in itertools.combinations(gold_nodes, 2):
# score each candidate fx and find the max one
nid1, nid2 = node_pair
# first direction
x1, y1 = ri_data[nid1][nid2]
x_ind, x_data = x1.indices, x1.data
scores1 = self.clf.compute_scores(x_ind, x_data)
best_clas1 = np.argmax(scores1)
if best_clas1 != y1:
self.clf.update_weights(y1, best_clas1, x_ind, x_data)
# second direction
x2, y2 = ri_data[nid2][nid1]
x_ind, x_data = x2.indices, x2.data
scores2 = self.clf.compute_scores(x_ind, x_data)
best_clas2 = np.argmax(scores2)
if best_clas2 != y2:
self.clf.update_weights(y2, best_clas2, x_ind, x_data)
def predict_train(self, data, print_stats=False):
golds, preds, graphs = [], [], []
fmaps = self.fx.fx_maps
scorer = self.clf.compute_scores
# initialize the decoder
arg_labels = ["ARG0", "ARG1", "ARG2", "ARG3", "ARG4", "ARG5",
# "ARG0-of", "ARG1-of", "ARG2-of", "ARG3-of", "ARG4-of", "ARG5-of",
]
arg_ids = [self.fx.fx_maps["ri_l2i"][l] for l in arg_labels]
self.decoder = Decoder(arg_elabs=arg_labels, arg_ids=arg_ids)
for idx, instance in enumerate(data):
goldG = instance["amrG"]
feats = instance["ri_data"]
aligned_nodes = filter(lambda x: goldG.node[x[0]]["span"][0] is not None, goldG.nodes(data=True))
G = nx.DiGraph(root=None, comments=instance["comments"])
if len(aligned_nodes) == 0:
snt = " ".join([T["word"] for T in instance["tokens"]])
print "No aligned nodes in AMR for snt (%s): %s" % (instance["comments"]["id"], snt)
# tokid = 0
# random_concept = instance["tokens"][tokid]["lemma"]
# self.ruleset.fill_one_node(G, tokid, random_concept, var=True)
# graphs.append(G)
# fixme: in JAMR, they exclude these graphs at all?
goldG.graph["comments"] = instance["comments"]
graphs.append(goldG)
continue
G.add_nodes_from(aligned_nodes)
# In Flanigan et al., these are edges from the CI stage to be preserved
# we disregard them, as there are hardly any
# E_0 = G.edges(data=False)
self.decoder.decode_one(feats, G, scorer, fmaps)
preds.append([(e[0], e[1], e[2]["label"]) for e in G.edges(data=True)])
golds.append([(e[0], e[1], e[2]["label"]) for e in goldG.edges(data=True)])
graphs.append(G)
# print "ND graphs decoding (+/-): %d/%d" %(self.decoder.success, self.decoder.failed)
self.decoder.success = self.decoder.failed = 0
return golds, preds, graphs
def predict_test(self, data,evaluate=False):
graphs = []
fmaps = self.fx.fx_maps
scorer = self.clf.compute_scores
self.fx.fx_dataset_ri(data, train=False)
# initialize the decoder
arg_labels = ["ARG0", "ARG1", "ARG2", "ARG3", "ARG4", "ARG5",
# "ARG0-of", "ARG1-of", "ARG2-of", "ARG3-of", "ARG4-of", "ARG5-of",
]
arg_ids = [self.fx.fx_maps["ri_l2i"][l] for l in arg_labels]
self.decoder = Decoder(arg_elabs=arg_labels, arg_ids=arg_ids)
for idx, instance in enumerate(data):
feats = instance["ri_data"]
# we have a partially built G from the CI stage
# no nodes are unaligned
G = instance["amrG"]
if len(G.nodes()) == 0:
snt = " ".join([T["word"] for T in instance["tokens"]])
print "No aligned nodes! Sentence (%s): %s" % (instance["comments"]["id"], snt)
print "This should not happen!"
sys.exit()
tokid = 0
random_concept = instance["tokens"][tokid]["lemma"]
self.ruleset.fill_one_node(G, tokid, random_concept, var=True)
graphs.append(G)
continue
self.decoder.decode_one(feats, G, scorer, fmaps)
graphs.append(G)
return graphs
def save_results(self, f_score, section, graphs, save_graphs=False):
default_fn = os.path.join(self.params["model_dir"], "%s.%0.3f" % (section, f_score))
base_fn = util.get_fname(default_fn)
amr_fn = "%s.amr" % base_fn
edge_fn = "%s.edges" % base_fn
self.amr_printer.save_penman(graphs, amr_fn, edge_fn)
param_fn = "%s.params.json" % base_fn
with open(param_fn, "w") as param_out:
json.dump(self.params, param_out)
if save_graphs:
graph_fn = "%s.graphs.pkl" % base_fn
self.amr_printer.pkl_graphs(graphs, graph_fn)
model_fn = "%s.ri_weights.pkl" % base_fn
with open(model_fn, "wb") as file_out:
logger.info("Saving the model into -----> %s " % (model_fn))
pickle.dump((self.clf.w, self.fx.fx_maps, self.fx.flist), file_out)
errors_fn = "%s.errors.json" % base_fn
with open(errors_fn, "w") as errors_out:
json.dump(self.errors, errors_out)
def load_model(self, model_fname):
logger.info("Loading RI model")
weights, feature_maps, feature_list = pickle.load(open(model_fname, "rb"))
self.clf = Perceptron.SparsePerceptron()
self.clf.w = weights
self.fx = Fxtractor(fx_maps=feature_maps, flist=feature_list)
self.amr_printer = Printer()
def evaluate(self, data, section="dev", save=False):
golds, preds, graphs = self.predict_train(data)
P, R, F1 = compute_prf(golds, preds, stage=self.name, save=save, savedir=self.params["stat_dir"])
logger.info("RI result on %s (1): %0.4f (P), %0.4f (R), %0.4f (F1)" % (section, P, R, F1))
if save:
self.save_results(F1, section, graphs, save_graphs=False)
def decode_viterbi(self):
decoder = Decoder(self.bottom_betas, self.betas8, self.betas16,
self.top_betas)
return decoder.viterbi_decode()
class AutoLoginTool(object):
"""隆众网自动登录"""
def __init__(self, username, password):
self.username = username
self.password = password
self.decoder = Decoder()
def gen_logined_cookie(self, data, set_cookie):
cookie_items = set_cookie.split(';')
cookie_dict = {}
for item in cookie_items:
idx = item.find(',')
if (idx > 0):
item = item.split(',')[1]
pair = item.split('=')
valueset = (pair[0].strip(), pair[1].strip())
cookie_dict[valueset[0]] = valueset[1]
cookie = 'parentid=0; userid=' + str(data['userNo']) + '; '
cookie += ('password='******'password'] + '; ')
cookie += ('AccessToken=' + cookie_dict['AccessToken'] + ';')
cookie += ('username='******'username'] + ';')
cookie += ('sidid=' + cookie_dict['sidid'] + ';')
cookie += ('userNo=' + str(data['userNo']) + ';')
cookie += ('LZ_' + str(data['userNo']) + '_UN=' +
urllib.quote('安信证券股份有限公司') + '(axzq1010);')
cookie += ('lz_usermember=' + urllib.quote(data['userMember']) + ';')
cookie += 'lz_usermember=0%2C2; auto=0; refcheck=ok; refpay=0; refsite=http%3A%2F%2Fnews.oilchem.net%2Flogin.shtml; '
cookie += 'Hm_lvt_47f485baba18aaaa71d17def87b5f7ec=1546486194; Hm_lpvt_47f485baba18aaaa71d17def87b5f7ec=1546486194'
return cookie
def try_login(self, login_url, code_value, set_cookie):
form = dict()
form['username'] = self.username
form['password'] = self.password
form['code'] = str(code_value)
form['rp'] = ''
encoded_form_data = urllib.urlencode(form)
headers = {
'Accept': 'application/json, text/plain, */*',
'User-Agent':
'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.23 Mobile Safari/537.36',
'Origin': 'http://news.oilchem.net',
'Referer': 'http://news.oilchem.net/login.shtml',
'Content-Length': len(encoded_form_data),
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'Accept-Encoding': 'gzip, deflate',
'Accept-Language': 'en,zh-CN;q=0.9,zh;q=0.8',
'cookie': set_cookie,
'X-Requested-With': 'XMLHttpRequest'
}
req = urllib2.Request(login_url,
headers=headers,
data=encoded_form_data)
response = urllib2.urlopen(req)
data = response.read()
message = demjson.decode(data)
print '$' * 100
print message.get('message') or message
print '$' * 100
if (message['data'] == None or message['data'] == ''
or message['data']['login'] != '1'):
raise Exception('failed to login.')
set_cookie = response.headers['Set-Cookie']
logined_cookie = self.gen_logined_cookie(message['data'], set_cookie)
return logined_cookie
def _get_ver_code_value(self, cookie):
timestamp_millisecond = str(int(time.time() * 1000))
verf_code_url = 'https://passport.oilchem.net/member/login/getImgCode?timestamp=' + timestamp_millisecond
req = urllib2.Request(verf_code_url)
req.add_header('cookie', cookie)
response = urllib2.urlopen(req)
imgData = response.read()
# img_file = '/home/ren/work/git_repos/pe_scrapy/pe/pe/work/oilchem/' + timestamp_millisecond + '.jpg'
img_file = tempfile.NamedTemporaryFile(
suffix='_oc_verf', dir=SpiderConfig().get_temp_dir(), delete=True)
img_file.write(imgData)
# codeImgFile = open(img_file, 'wb')
# codeImgFile.write(imgData)
# codeImgFile.close()
code_value = self.decoder.read_img(img_file)
return code_value
def try_get_verf_code_value(self, cookie, retry_times=5):
while (retry_times > 0):
retry_times = retry_times - 1
code_value = self._get_ver_code_value(cookie)
if len(code_value) == 4:
return code_value
return None
def submit_login_form(self, cookie):
login_url = "http://news.oilchem.net/user/userLogin.do?ajax=1&chkbdwx=0&closewindow=&rnd=" + str(
random.random()) + str(
random.random())[2:6] + '&b=c6cd86f957430ab076158365dac8a2d4'
max_retries = 1
while (max_retries > 0):
max_retries = max_retries - 1
code_value = self.try_get_verf_code_value(cookie)
if code_value is None:
raise Exception('failed to parse verf code')
try:
logined_cookie = self.try_login(login_url, code_value, cookie)
return logined_cookie
except:
print traceback.format_exc()
pass
raise Exception('failed to login')
def run(self):
# update per-sentence grammars, if there's any
for g in self.grammars:
g.update(self.id)
self.flog = open('%s/%s_%s' % (FLAGS.run_dir,
'log',
self.suffix),
'w')
if FLAGS.show_time:
self.flog.write('running on %s\n\n' % socket.gethostname())
self.flog.flush()
fwords = self.line.strip().split()
# added by freesunshine, build the local grammar for oov words for each sentence
rules = []
if self.oov_idx is not None and len(self.oov_idx) > 0:
#oov_weight = 8.0
oov_weight = 0.0001
for idx in self.oov_idx:
fw = fwords[idx]
ew = "."
rule_str = "[A0-0] ||| %s ||| %s ||| %lf %lf %lf" %(fw, ew, oov_weight, oov_weight, oov_weight)
rr = Rule()
rr.fromstr(rule_str)
rules.append(rr)
if self.ner_items is not None and len(self.ner_items) > 0:
for item in self.ner_items:
concept_weight = 10.0
st = item[0][0]
ed = item[0][1]
fw = ' '.join(fwords[st:ed])
#concept_weight *= pow((ed-st), 2)
ew = item[1]
value = int(ew[2])
#Here is the feature for difference of nonterminal type
#concept_weight /= pow(1.4, value)
#Here is the feature for the favor of longer spans
#concept_weight *= pow(2, ed-st)
#Here is the feature for the number of edges
#concept_weight /= pow(2.0, get_num_edges(ew))
#print >>sys.stder, ew, concept_weight
#rule_str = "[A1-1] ||| %s ||| %s ||| " % (fw, ew)
rule_str = "%s ||| " % ew
#weight = 5
if fw == ';':
rule_str += "%lf %lf %lf" % (concept_weight, concept_weight, concept_weight)
else:
rule_str += "%lf %lf %lf" % (concept_weight, concept_weight, concept_weight)
rr = Rule()
#print rule_str
rr.fromstr(rule_str)
rules.append(rr)
#print '===== local_gr ====='
#for r in rules:
# print r
local_gr = None
if len(rules) > 0:
local_gr = Grammar(FLAGS.rule_bin_size)
local_gr.build(rules, self.grammars[0].features)
if FLAGS.preprocess:
self.fidx2replacement = {}
j = 0
for i, token in enumerate(fwords):
if token in ('$number', '$date'):
self.fidx2replacement[i] = self.special[j][1]
j += 1
self.flog.write('[%s][%s words] %s\n' %
(self.id, len(fwords), self.line))
decoder = Decoder(fwords,
self.grammars,
self.features,
local_gr)
begin_time = time()
if FLAGS.decoding_method == 'agenda':
item = decoder.decode()
elif FLAGS.decoding_method == 'cyk':
item = decoder.decode_cyk()
elif FLAGS.decoding_method == 'earley':
item = decoder.decode_earley()
else:
assert False, '"%s" not valid decoding option' \
% FLAGS.decoding_method
self.time = time() - begin_time
if item is None:
self.out = '[decoder failed to build a goal item]'
else:
ttt, succ = item
item = ttt
hg = Hypergraph(item)
hg.set_semiring(hypergraph.SHORTEST_PATH)
hg.set_functions(lambda x: x.cost, None, None)
hg.topo_sort()
self.kbest = hg.root.best_paths()
#output_tokens = self.kbest[0].translation[:]
#if FLAGS.preprocess:
# for i in range(len(output_tokens)):
# if output_tokens[i] in ('$number', '$date'):
# fidx = self.kbest[0].composed_rule.we2f[i]
# if fidx is not None:
# output_tokens[i] = self.fidx2replacement[fidx]
# @freesunshine target side string output
#self.out = ' '.join(output_tokens[FLAGS.lm_order-1:
# 1-FLAGS.lm_order])
self.flog.write('Decuction Tree:\n%s\n' % self.kbest[0].tree_str())
#self.out = str(self.kbest[0].translation)
#if succ:
self.out = self.kbest[0].translation.to_amr_format()[0]
#else:
# self.out = self.kbest[0].translation.toAMR()
lines = [x.strip() for x in self.out.split('\n')]
self.out = "".join(lines)
self.hg = hg
if FLAGS.output_hypergraph:
self.write_hypergraph()
self.flog.write('%s\n' % self.out)
self.flog.write('\n')
#if item is not None:
# self.flog.write(self.kbest[0].tree_str())
# self.flog.write('\n')
# self.flog.write(hg.stats())
# self.flog.write('\n')
self.flog.write(decoder.agenda_stats())
self.flog.write('\n')
self.flog.write(decoder.chart.stats())
self.flog.write('\n')
for dotchart in decoder.dotcharts:
self.flog.write(dotchart.stats())
self.flog.write('\n')
if FLAGS.show_time:
timeline = '{:<35}{:>15.2f}\n'.format('[time]:', self.time)
self.flog.write(timeline)
self.write_output_file()
if FLAGS.output_kbest:
self.write_kbest_to_file()
self.flog.close()
def decode(aegs, n_spk):
# load config
config = get_config(args.conf_path)
# logger
logger = Logger(args.log_name, 'decoder', 'dataset')
# training device
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
logger.decoder.info('device: %s' % device)
# trainind settings and model
net = Net(config.model, n_spk, 1, device)
net.to(device)
# resume
dic = torch.load(args.checkpoint)
net.load_state_dict(dic['model'])
criteria_before = dic['criteria']
iter_count = dic['iter_count']
logger.decoder.info(net)
logger.decoder.info('Criteria before: %f' % criteria_before)
# dataset
datasets = {
'test':
Dataset(args.test_dir,
args.stats_dir,
logger.dataset,
pad_len=2800,
batch_len=-1,
device=device)
}
data_loaders = {
'test': DataLoader(datasets['test'], batch_size=1, shuffle=True)
}
# logging about training data
logger.dataset.info('number of test samples: %d' % len(datasets['test']))
# decoder for validation
decoder = Decoder(args, datasets['test'].scaler, logger=logger.decoder)
# decode
logger.decoder.info('start decoding!')
for i, batch in enumerate(data_loaders['test']):
# inputs
inputs = {
'feat':
torch.cat(
(batch['uv'], batch['lcf0'], batch['codeap'], batch['mcep']),
dim=-1).to(device),
'cv_stats':
torch.cat((batch['uv'], batch['lcf0'], batch['codeap']),
dim=-1).to(device),
'src_code':
batch['src_code'].to(device),
'trg_code':
batch['trg_code'].to(device),
'src_spk':
batch['src_id'].to(device),
'trg_spk':
batch['trg_id'].to(device),
'src':
batch['src_spk'],
'trg':
batch['trg_spk'],
'flen':
batch['flen'],
'f0':
batch['f0'],
'codeap':
batch['codeap'],
'mcep':
batch['mcep'],
'cv_f0':
batch['cv_f0']
}
# forward propagation with target-pos output
outputs = net(inputs)
# decode
decoder.decode(inputs, outputs, iter_count, i)
def main(argv):
tf.set_random_seed(111) # a seed value for randomness
# Create a batcher object that will create minibatches of data
# TODO change to pass number
# --------------- building graph ---------------
hparam_gen = [
'mode',
'model_dir',
'adagrad_init_acc',
'steps_per_checkpoint',
'batch_size',
'beam_size',
'cov_loss_wt',
'coverage',
'emb_dim',
'rand_unif_init_mag',
'gen_vocab_file',
'gen_vocab_size',
'hidden_dim',
'gen_lr',
'gen_max_gradient',
'max_dec_steps',
'max_enc_steps',
'min_dec_steps',
'trunc_norm_init_std',
'single_pass',
'log_root',
'data_path',
]
hps_dict = {}
for key, val in FLAGS.__flags.iteritems(): # for each flag
if key in hparam_gen: # if it's in the list
hps_dict[key] = val # add it to the dict
hps_gen = namedtuple("HParams4Gen", hps_dict.keys())(**hps_dict)
print("Building vocabulary for generator ...")
gen_vocab = Vocab(join_path(hps_gen.data_path, hps_gen.gen_vocab_file),
hps_gen.gen_vocab_size)
hparam_dis = [
'mode',
'vocab_type',
'model_dir',
'dis_vocab_size',
'steps_per_checkpoint',
'learning_rate_decay_factor',
'dis_vocab_file',
'num_class',
'layer_size',
'conv_layers',
'max_steps',
'kernel_size',
'early_stop',
'pool_size',
'pool_layers',
'dis_max_gradient',
'batch_size',
'dis_lr',
'lr_decay_factor',
'cell_type',
'max_enc_steps',
'max_dec_steps',
'single_pass',
'data_path',
'num_models',
]
hps_dict = {}
for key, val in FLAGS.__flags.iteritems(): # for each flag
if key in hparam_dis: # if it's in the list
hps_dict[key] = val # add it to the dict
hps_dis = namedtuple("HParams4Dis", hps_dict.keys())(**hps_dict)
if hps_gen.gen_vocab_file == hps_dis.dis_vocab_file:
hps_dis = hps_dis._replace(vocab_type="word")
hps_dis = hps_dis._replace(layer_size=hps_gen.emb_dim)
hps_dis = hps_dis._replace(dis_vocab_size=hps_gen.gen_vocab_size)
else:
hps_dis = hps_dis._replace(max_enc_steps=hps_dis.max_enc_steps * 2)
hps_dis = hps_dis._replace(max_dec_steps=hps_dis.max_dec_steps * 2)
if FLAGS.mode == "train_gan":
hps_gen = hps_gen._replace(batch_size=hps_gen.batch_size *
hps_dis.num_models)
if FLAGS.mode != "pretrain_dis":
with tf.variable_scope("generator"):
generator = PointerGenerator(hps_gen, gen_vocab)
print("Building generator graph ...")
gen_decoder_scope = generator.build_graph()
if FLAGS.mode != "pretrain_gen":
print("Building vocabulary for discriminator ...")
dis_vocab = Vocab(join_path(hps_dis.data_path, hps_dis.dis_vocab_file),
hps_dis.dis_vocab_size)
if FLAGS.mode in ['train_gan', 'pretrain_dis']:
with tf.variable_scope("discriminator"), tf.device("/gpu:0"):
discriminator = Seq2ClassModel(hps_dis)
print("Building discriminator graph ...")
discriminator.build_graph()
hparam_gan = [
'mode',
'model_dir',
'gan_iter',
'gan_gen_iter',
'gan_dis_iter',
'gan_lr',
'rollout_num',
'sample_num',
]
hps_dict = {}
for key, val in FLAGS.__flags.iteritems(): # for each flag
if key in hparam_gan: # if it's in the list
hps_dict[key] = val # add it to the dict
hps_gan = namedtuple("HParams4GAN", hps_dict.keys())(**hps_dict)
hps_gan = hps_gan._replace(mode="train_gan")
if FLAGS.mode == 'train_gan':
with tf.device("/gpu:0"):
print("Creating rollout...")
rollout = Rollout(generator, 0.8, gen_decoder_scope)
# --------------- initializing variables ---------------
all_variables = tf.get_collection_ref(tf.GraphKeys.GLOBAL_VARIABLES) + \
tf.get_collection_ref(tf.GraphKeys.WEIGHTS) + \
tf.get_collection_ref(tf.GraphKeys.BIASES)
sess = tf.Session(config=utils.get_config())
sess.run(tf.variables_initializer(all_variables))
if FLAGS.mode == "pretrain_gen":
val_dir = ensure_exists(
join_path(FLAGS.model_dir, 'generator', FLAGS.val_dir))
model_dir = ensure_exists(join_path(FLAGS.model_dir, 'generator'))
print("Restoring the generator model from the latest checkpoint...")
gen_saver = tf.train.Saver(
max_to_keep=3,
var_list=[
v for v in all_variables
if "generator" in v.name and "GAN" not in v.name
])
gen_dir = ensure_exists(join_path(FLAGS.model_dir, "generator"))
# gen_dir = ensure_exists(FLAGS.model_dir)
# temp_saver = tf.train.Saver(
# var_list=[v for v in all_variables if "generator" in v.name and "Adagrad" not in v.name])
ckpt_path = utils.load_ckpt(gen_saver, sess, gen_dir)
print('going to restore embeddings from checkpoint')
if not ckpt_path:
emb_path = join_path(FLAGS.model_dir, "generator", "init_embed")
if emb_path:
generator.saver.restore(
sess,
tf.train.get_checkpoint_state(
emb_path).model_checkpoint_path)
print(
colored(
"successfully restored embeddings form %s" % emb_path,
'green'))
else:
print(
colored("failed to restore embeddings form %s" % emb_path,
'red'))
elif FLAGS.mode in ["decode", "train_gan"]:
print("Restoring the generator model from the best checkpoint...")
dec_saver = tf.train.Saver(
max_to_keep=3,
var_list=[v for v in all_variables if "generator" in v.name])
gan_dir = ensure_exists(
join_path(FLAGS.model_dir, 'generator', FLAGS.gan_dir))
gan_val_dir = ensure_exists(
join_path(FLAGS.model_dir, 'generator', FLAGS.gan_dir,
FLAGS.val_dir))
gan_saver = tf.train.Saver(
max_to_keep=3,
var_list=[v for v in all_variables if "generator" in v.name])
gan_val_saver = tf.train.Saver(
max_to_keep=3,
var_list=[v for v in all_variables if "generator" in v.name])
utils.load_ckpt(dec_saver, sess, val_dir,
(FLAGS.mode in ["train_gan", "decode"]))
if FLAGS.mode in ["pretrain_dis", "train_gan"]:
dis_saver = tf.train.Saver(
max_to_keep=3,
var_list=[v for v in all_variables if "discriminator" in v.name])
dis_dir = ensure_exists(join_path(FLAGS.model_dir, 'discriminator'))
ckpt = utils.load_ckpt(dis_saver, sess, dis_dir)
if not ckpt:
if hps_dis.vocab_type == "word":
discriminator.init_emb(
sess, join_path(FLAGS.model_dir, "generator",
"init_embed"))
else:
discriminator.init_emb(
sess,
join_path(FLAGS.model_dir, "discriminator", "init_embed"))
# --------------- train models ---------------
if FLAGS.mode != "pretrain_dis":
gen_batcher_train = GenBatcher("train",
gen_vocab,
hps_gen,
single_pass=hps_gen.single_pass)
decoder = Decoder(sess, generator, gen_vocab)
gen_batcher_val = GenBatcher("val",
gen_vocab,
hps_gen,
single_pass=True)
val_saver = tf.train.Saver(
max_to_keep=10,
var_list=[
v for v in all_variables
if "generator" in v.name and "GAN" not in v.name
])
if FLAGS.mode != "pretrain_gen":
dis_val_batch_size = hps_dis.batch_size * hps_dis.num_models \
if hps_dis.mode == "train_gan" else hps_dis.batch_size * hps_dis.num_models * 2
dis_batcher_val = DisBatcher(
hps_dis.data_path,
"eval",
gen_vocab,
dis_vocab,
dis_val_batch_size,
single_pass=True,
max_art_steps=hps_dis.max_enc_steps,
max_abs_steps=hps_dis.max_dec_steps,
)
if FLAGS.mode == "pretrain_gen":
# get reload the
print('Going to pretrain the generator')
try:
pretrain_generator(generator, gen_batcher_train, sess,
gen_batcher_val, gen_saver, model_dir,
val_saver, val_dir)
except KeyboardInterrupt:
tf.logging.info("Caught keyboard interrupt on worker....")
elif FLAGS.mode == "pretrain_dis":
print('Going to pretrain the discriminator')
dis_batcher = DisBatcher(
hps_dis.data_path,
"decode",
gen_vocab,
dis_vocab,
hps_dis.batch_size * hps_dis.num_models,
single_pass=hps_dis.single_pass,
max_art_steps=hps_dis.max_enc_steps,
max_abs_steps=hps_dis.max_dec_steps,
)
try:
pretrain_discriminator(sess, discriminator, dis_batcher_val,
dis_vocab, dis_batcher, dis_saver)
except KeyboardInterrupt:
tf.logging.info("Caught keyboard interrupt on worker....")
elif FLAGS.mode == "train_gan":
gen_best_loss = get_best_loss_from_chpt(val_dir)
gen_global_step = 0
print('Going to tune the two using Gan')
for i_gan in range(hps_gan.gan_iter):
# Train the generator for one step
g_losses = []
current_speed = []
for it in range(hps_gan.gan_gen_iter):
start_time = time.time()
batch = gen_batcher_train.next_batch()
# generate samples
enc_states, dec_in_state, n_samples, n_targets_padding_mask = decoder.mc_generate(
batch, include_start_token=True, s_num=hps_gan.sample_num)
# get rewards for the samples
n_rewards = rollout.get_reward(sess, gen_vocab, dis_vocab,
batch, enc_states, dec_in_state,
n_samples, hps_gan.rollout_num,
discriminator)
# fine tune the generator
n_sample_targets = [samples[:, 1:] for samples in n_samples]
n_targets_padding_mask = [
padding_mask[:, 1:]
for padding_mask in n_targets_padding_mask
]
n_samples = [samples[:, :-1] for samples in n_samples]
# sample_target_padding_mask = pad_sample(sample_target, gen_vocab, hps_gen)
n_samples = [
np.where(
np.less(samples, hps_gen.gen_vocab_size), samples,
np.array([[gen_vocab.word2id(data.UNKNOWN_TOKEN)] *
hps_gen.max_dec_steps] * hps_gen.batch_size))
for samples in n_samples
]
results = generator.run_gan_batch(sess, batch, n_samples,
n_sample_targets,
n_targets_padding_mask,
n_rewards)
gen_global_step = results["global_step"]
# for visualization
g_loss = results["loss"]
if not math.isnan(g_loss):
g_losses.append(g_loss)
else:
print(colored('a nan in gan loss', 'red'))
current_speed.append(time.time() - start_time)
# Test
# if FLAGS.gan_gen_iter and (i_gan % 100 == 0 or i_gan == hps_gan.gan_iter - 1):
if i_gan % 100 == 0 or i_gan == hps_gan.gan_iter - 1:
print('Going to test the generator.')
current_speed = sum(current_speed) / (len(current_speed) *
hps_gen.batch_size)
everage_g_loss = sum(g_losses) / len(g_losses)
# one more process hould be opened for the evaluation
eval_loss, gen_best_loss = save_ckpt(
sess, generator, gen_best_loss, gan_dir, gan_saver,
gen_batcher_val, gan_val_dir, gan_val_saver,
gen_global_step)
if eval_loss:
print("\nDashboard for " +
colored("GAN Generator", 'green') + " updated %s, "
"finished steps:\t%s\n"
"\tBatch size:\t%s\n"
"\tVocabulary size:\t%s\n"
"\tCurrent speed:\t%.4f seconds/article\n"
"\tAverage training loss:\t%.4f; "
"eval loss:\t%.4f" % (
datetime.datetime.now().strftime(
"on %m-%d at %H:%M"),
gen_global_step,
FLAGS.batch_size,
hps_gen.gen_vocab_size,
current_speed,
everage_g_loss.item(),
eval_loss.item(),
))
# Train the discriminator
print('Going to train the discriminator.')
dis_best_loss = 1000
dis_losses = []
dis_accuracies = []
for d_gan in range(hps_gan.gan_dis_iter):
batch = gen_batcher_train.next_batch()
enc_states, dec_in_state, k_samples_words, _ = decoder.mc_generate(
batch, s_num=hps_gan.sample_num)
# shuould first tanslate to words to avoid unk
articles_oovs = batch.art_oovs
for samples_words in k_samples_words:
dec_batch_words = batch.target_batch
conditions_words = batch.enc_batch_extend_vocab
if hps_dis.vocab_type == "char":
samples = gen_vocab2dis_vocab(samples_words, gen_vocab,
articles_oovs, dis_vocab,
hps_dis.max_dec_steps,
STOP_DECODING)
dec_batch = gen_vocab2dis_vocab(
dec_batch_words, gen_vocab, articles_oovs,
dis_vocab, hps_dis.max_dec_steps, STOP_DECODING)
conditions = gen_vocab2dis_vocab(
conditions_words, gen_vocab, articles_oovs,
dis_vocab, hps_dis.max_enc_steps, PAD_TOKEN)
else:
samples = samples_words
dec_batch = dec_batch_words
conditions = conditions_words
# the unknown in target
inputs = np.concatenate([samples, dec_batch], 0)
conditions = np.concatenate([conditions, conditions], 0)
targets = [[1, 0] for _ in samples] + [[0, 1]
for _ in dec_batch]
targets = np.array(targets)
# randomize the samples
assert len(inputs) == len(conditions) == len(
targets
), "lengthes of the inputs, conditions and targests should be the same."
indices = np.random.permutation(len(inputs))
inputs = np.split(inputs[indices], 2)
conditions = np.split(conditions[indices], 2)
targets = np.split(targets[indices], 2)
assert len(inputs) % 2 == 0, "the length should be mean"
results = discriminator.run_one_batch(
sess, inputs[0], conditions[0], targets[0])
dis_accuracies.append(results["accuracy"].item())
dis_losses.append(results["loss"].item())
results = discriminator.run_one_batch(
sess, inputs[1], conditions[1], targets[1])
dis_accuracies.append(results["accuracy"].item())
ave_dis_acc = sum(dis_accuracies) / len(dis_accuracies)
if d_gan == hps_gan.gan_dis_iter - 1:
if (sum(dis_losses) / len(dis_losses)) < dis_best_loss:
dis_best_loss = sum(dis_losses) / len(dis_losses)
checkpoint_path = ensure_exists(
join_path(hps_dis.model_dir,
"discriminator")) + "/model.ckpt"
dis_saver.save(sess,
checkpoint_path,
global_step=results["global_step"])
print_dashboard("GAN Discriminator",
results["global_step"].item(),
hps_dis.batch_size, hps_dis.dis_vocab_size,
results["loss"].item(), 0.00, 0.00, 0.00)
print("Average training accuracy: \t%.4f" % ave_dis_acc)
if ave_dis_acc > 0.9:
break
# --------------- decoding samples ---------------
elif FLAGS.mode == "decode":
print('Going to decode from the generator.')
decoder.bs_decode(gen_batcher_train)
print("Finished decoding..")
# decode for generating corpus for discriminator
sess.close()
def decode_dfs(self):
decoder = Decoder(self.bottom_betas, self.betas8, self.betas16,
self.top_betas)
return decoder.dfs_decode()
