def train(args): if args.use_morph: train_src_data_path = '%s/train.seg.%s' % (args.data_dir, args.src) else: train_src_data_path = '%s/train.%s' % (args.data_dir, args.src) train_tgt_data_path = '%s/train.%s' % (args.data_dir, args.tgt) names = ['src', 'tgt'] paths = [train_src_data_path, train_tgt_data_path] if args.replace_unk: names.append('almt') paths.append('%s/train.align' % (args.data_dir)) train_corpus = prepare_corpus(names, paths, buckets, max_size=args.max_size, vocabs=vocabs, replace_unk=args.replace_unk, use_morph=args.use_morph) dictionary = train_corpus.dictionary if args.use_morph: dev_src_data_path = '%s/dev.seg.%s' % (args.data_dir, args.src) else: dev_src_data_path = '%s/dev.%s' % (args.data_dir, args.src) dev_tgt_data_path = '%s/dev.%s' % (args.data_dir, args.tgt) if args.DEBUG: # use smaller dev set for debugging if not os.path.isfile(dev_src_data_path + '.debug'): subprocess.call('head -n 100 %s > %s' % (dev_src_data_path, dev_src_data_path + '.debug'), shell=True) subprocess.call('head -n 100 %s > %s' % (dev_tgt_data_path, dev_tgt_data_path + '.debug'), shell=True) dev_src_data_path += '.debug' dev_tgt_data_path += '.debug' dev_corpus = prepare_corpus( ['src', 'tgt'], [dev_src_data_path, dev_tgt_data_path], buckets, max_size=args.max_size, use_all= True, # use all data for dev, which means do NOT discard very long sentences vocabs=vocabs, method='src', use_morph=args.use_morph) train_set = DataStreamRandom(train_corpus, vocabs=vocabs, **vars(args)) # random batching dev_set = DataStream( dev_corpus, vocabs=vocabs, **vars(args)) # no random batching, iterated by length trainer = Trainer(model, **vars(args)) datastate_path = '%s/datastate.pkl' % args.train_dir if os.path.isfile(datastate_path): trainer.tracker.load(datastate_path) trainer.train(train_set, dev_set, tgt_vocab, dictionary=dictionary)
def parse_ds(qaid, meas, conn): #Build DataStream Object ds = DataStream(qaid, meas, conn) #Build and return Query Object #Total Returns Frequency = 'D' (Trading Daily) return QueryObject(qaid, meas, 'D', ds.data, 'DataStream')
def test(args): train_src_data_path = '%s/train.%s' % (args.data_dir, args.src) train_tgt_data_path = '%s/train.%s' % (args.data_dir, args.tgt) names = ['src', 'tgt'] paths = [train_src_data_path, train_tgt_data_path] if args.replace_unk: names.append('almt') paths.append('%s/train.align' % (args.data_dir)) if args.use_morph: names.append('morph') paths.append('%s/seg.%s' % (args.data_dir, args.src)) train_corpus = prepare_corpus(names, paths, buckets, max_size=args.max_size, vocabs=vocabs, replace_unk=args.replace_unk) dictionary = train_corpus.dictionary test_src_data_path = '%s/test.%s' % (args.data_dir, args.src) #test_tgt_data_path = '%s/test.%s' %(args.data_dir, args.tgt) if args.test_path is not None: test_src_data_path = '%s.%s' % (args.test_path, args.src) #test_tgt_data_path = '%s.%s' %(args.test_path, args.tgt) else: test_src_data_path = '%s/test.%s' % (args.data_dir, args.src) #test_tgt_data_path = '%s/test.%s' %(args.data_dir, args.tgt) test_corpus = prepare_corpus(['src'], [test_src_data_path], buckets, vocabs={'src': vocabs['src']}, use_all=True, method='src') test_set = DataStream(test_corpus, vocabs=vocabs, **vars(args)) model.eval() translations = dict() alignments = dict() start = time.time() for b, batch in enumerate(test_set, 1): print('\rbatch %d: %.4fs' % (b, time.time() - start), end='') sys.stdout.flush() preds, alignments, _ = model.search(batch) translate_(translations, preds, batch, vocabs['tgt'], src_tokens=batch.src_tokens, dictionary=dictionary, alignments=alignments) print() bleu_test(translations, test_set, args.bleu_path, gold_path=None) # NOTE no bleu evaluation for test
def __init__(self, host, port, config_path, image_x=400, image_y=300): self._host = host self._port = port logging.debug('selected host %s' % host) logging.debug('selected port %s' % port) self._config_path = config_path logging.debug('set config path as port %s' % config_path) logging.debug('selected port %s' % port) self._port_control = self._port + 2 self._port_stream = self._port + 1 #self._fps =config.fps self._socket_world = socket_util.pers_connect(self._host, self._port) logging.debug("Connected to Unreal Server World Socket") self._socket_stream = 0 self._socket_control = 0 self._image_x = image_x self._image_y = image_y self._data_stream = DataStream(image_x, image_y) logging.debug("Started Unreal Client")
def restart(self): logging.debug("Trying to close clients") self.close_conections() connected = False self._data_stream._running = False while not connected: try: logging.debug("Trying to connect to the world thread") self._socket_world = socket_util.connect( self._host, self._port) connected = True except: logging.debug("Couldn't connected ... retry in 10 seconds...") time.sleep(10) self._data_stream = DataStream(self._image_x, self._image_y) self.startAgent() self.requestNewEpisode() scene, positions = self.receiveSceneConfiguration() self.newEpisode(self._latest_start, self._latest_end) logging.debug("restarted the world connection") return scene, positions
def __init__(self, reader, head, bits): self.streams = [] pages = reader.pages_from_size(head.directory_size) # 0..n in page of directory pages. bits.min_capacity(head.directory_size) directory_root_pages = len(head.directory_root) pages_per_page = head.page_size / 4 pages_to_go = pages for i in range(0, directory_root_pages): pages_in_this_page = pages_per_page if pages_to_go <= pages_per_page: pages_in_this_page = pages_to_go reader.seek(head.directory_root[i], 0); bits.append(reader.reader, pages_in_this_page * 4) pages_to_go -= pages_in_this_page bits.position = 0 stream = DataStream(head.directory_size, bits, pages) bits.min_capacity(head.directory_size) stream.read(reader, bits) # 0..3 in directory pages count = bits.read_int32() # 4..n sizes = bits.read_int32(count) # n..m self.streams = [] for i in range(0, count): if sizes[i] <= 0: self.streams.append(DataStream()) else: self.streams.append(DataStream( sizes[i], bits, reader.pages_from_size(sizes[i]) ))
def test_load_file(): logging.debug('---------------Begin test_load_file()') # data_set_dir = 'data_sets' # filenames = [os.path.join(data_set_dir, f) for f in os.listdir(data_set_dir) if not f.startswith('.')] for file in filenames: logging.debug(f'Loading {file}') ds = DataStream(file) logging.debug(f'Data is {ds}') if not file.endswith('empty'): assert np.array_equal( ds, [[4, 8, 0], [5, 9, 1], [6, 10, 2], [7, 11, 3]]) if file.find('header') != -1 or file.find( 'structured') != -1 or file.find('dict') != -1: assert ds._keys == ('x', 'y', 'time') elif file.endswith('empty'): assert ds.array.size == 0 assert ds._keys == () else: assert ds._keys == ('x', 'y', 'z')
def __init__(self, reader, head, bits): self.streams = [] pages = reader.pages_from_size(head.directory_size) # 0..n in page of directory pages. bits.min_capacity(head.directory_size) directory_root_pages = len(head.directory_root) pages_per_page = head.page_size / 4 pages_to_go = pages for i in range(0, directory_root_pages): pages_in_this_page = pages_per_page if pages_to_go <= pages_per_page: pages_in_this_page = pages_to_go reader.seek(head.directory_root[i], 0) bits.append(reader.reader, pages_in_this_page * 4) pages_to_go -= pages_in_this_page bits.position = 0 stream = DataStream(head.directory_size, bits, pages) bits.min_capacity(head.directory_size) stream.read(reader, bits) # 0..3 in directory pages count = bits.read_int32() # 4..n sizes = bits.read_int32(count) # n..m self.streams = [] for i in range(0, count): if sizes[i] <= 0: self.streams.append(DataStream()) else: self.streams.append( DataStream(sizes[i], bits, reader.pages_from_size(sizes[i])))
def stop(self): self.close_conections() connected = False self._data_stream._running = False self._data_stream = DataStream(self._image_x, self._image_y)
class CarlaUnreal(object): """ Normal instanciation of the class, creating also the thread class responsible for receiving data """ def __init__(self, host, port, config_path, image_x=400, image_y=300): self._host = host self._port = port logging.debug('selected host %s' % host) logging.debug('selected port %s' % port) self._config_path = config_path logging.debug('set config path as port %s' % config_path) logging.debug('selected port %s' % port) self._port_control = self._port + 2 self._port_stream = self._port + 1 #self._fps =config.fps self._socket_world = socket_util.pers_connect(self._host, self._port) logging.debug("Connected to Unreal Server World Socket") self._socket_stream = 0 self._socket_control = 0 self._image_x = image_x self._image_y = image_y self._data_stream = DataStream(image_x, image_y) logging.debug("Started Unreal Client") def setIniFile(self, config_path): self._config_path = config_path logging.debug('set config path as port %s' % config_path) """ Starting the Agent. The image stream port and the control port """ def startAgent(self): logging.debug("Going to Connect Stream and start thread") self._socket_stream = socket_util.pers_connect(self._host, self._port_stream) self._socket_control = socket_util.pers_connect( self._host, self._port_control) logging.debug("Control Socket Connected") self._data_stream.start(self._socket_stream) logging.debug("Streaming Thread Started") def receiveSceneConfiguration(self): try: data = socket_util.get_message(self._socket_world) scene = Scene() scene.ParseFromString(data) logging.debug("Received Scene Configuration") # parsing positions positions = [] number_of_positions = len( scene.positions) / 8 # Every 8 bytes you have a position for i in range(0, number_of_positions * 2, 2): x = struct.unpack('f', scene.positions[i * 4:(i + 1) * 4])[0] y = struct.unpack('f', scene.positions[(i + 1) * 4:(i + 2) * 4])[0] positions.append((x, y)) return scene, positions except: logging.debug("Died When receiving configuration") return self.restart() def requestNewEpisode(self, ini_file=None): requestEpisode = RequestNewEpisode() if ini_file == None: # You can send a new ini file to be open, if not open the one defined on start ini_path = self._config_path else: ini_path = ini_file with open(ini_path, "r") as myfile: data = myfile.read() logging.debug("Set the Init File") requestEpisode.init_file = data try: socket_util.send_message(self._socket_world, requestEpisode) except: logging.debug("Died When requesting new episode") self.restart() logging.debug("Send the new episode Request") def newEpisode(self, start_index, end_index): self._latest_start = start_index self._latest_end = end_index scene_init = EpisodeStart() scene_init.start_index = start_index scene_init.end_index = end_index try: socket_util.send_message(self._socket_world, scene_init) except: logging.debug("Died When confirming new episode") self.restart() logging.debug("Send the new episode Message") episode_ready = EpisodeReady() episode_ready.ready = False self._data_stream.clean() try: while not episode_ready.ready: data = socket_util.get_message(self._socket_world) episode_ready.ParseFromString(data) except: logging.debug("Died when trying to receive episode reading") self.restart() logging.debug("Episode is Ready") ### **** PROTOCOL 3 **** def getReward(self): logging.debug("Got A new Reward") while True: try: reward_data = self._data_stream.get_the_latest_data() return reward_data except: logging.debug("Got an empty reward") self.restart() return reward_data """ Command contains: Steering: -1 to 1 Acc : -1 to 1 """ def sendCommand(self, control): logging.debug( "Send Control Comand : acc -> %f , steer %f, brake %f, hand_brake %d, gear %d" % (control.gas, control.steer, control.brake, control.hand_brake, control.reverse)) try: socket_util.send_message(self._socket_control, control) except: logging.debug("Problems on sending the commands... restarting") self.restart( ) # the mensage is not resend because it likely lost its relevance. def restart(self): logging.debug("Trying to close clients") self.close_conections() connected = False self._data_stream._running = False while not connected: try: logging.debug("Trying to connect to the world thread") self._socket_world = socket_util.connect( self._host, self._port) connected = True except: logging.debug("Couldn't connected ... retry in 10 seconds...") time.sleep(10) self._data_stream = DataStream(self._image_x, self._image_y) self.startAgent() self.requestNewEpisode() scene, positions = self.receiveSceneConfiguration() self.newEpisode(self._latest_start, self._latest_end) logging.debug("restarted the world connection") return scene, positions def stop(self): self.close_conections() connected = False self._data_stream._running = False self._data_stream = DataStream(self._image_x, self._image_y) def close_conections(self): try: self._socket_world.shutdown(socket.SHUT_RDWR) self._socket_world.close() logging.debug("Close world") except Exception as ex: print(ex.message) try: self._socket_stream.shutdown(socket.SHUT_RDWR) self._socket_stream.close() logging.debug("Close Stream") except Exception as ex: print(ex.message) try: self._socket_control.shutdown(socket.SHUT_RDWR) self._socket_control.close() logging.debug("Close Control") except Exception as ex: print(ex.message)
def test_data_type(): logging.debug('---------------Begin test_dataType()') assert data_type(DataStream()) == 'DictArray' assert data_type(DataStream([[], [], []])) == 'DictArray'
def test_construct_equal(): ds1 = DataStream() add_point_tester(ds1) ds2 = DataStream(ds1) check_datastreams_equal(ds1, ds2)
def test_base_constructor(): logging.debug('---------------Begin test_base_constructor()') ds = DataStream()
import numpy as np import os from datastream import DataStream time, x, y = [0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11] data_set_dir = os.path.join('tests', 'data_sets') filenames = [ os.path.join(data_set_dir, f) for f in os.listdir(data_set_dir) if not f.startswith('.') ] known = DataStream({'x': x, 'y': y, 'time': time}) for f in filenames: print(f) r = DataStream(f) if not f.endswith('empty'): try: assert np.array_equal(r, known) except AssertionError: print('############', f) print(known) print(r) if f.find('dict') != -1 or f.find('header') != -1 or f.find( 'save_struct') != -1: assert set(r.keys()) == set(known.keys()) else: assert set(r.keys()) == set(['x', 'y', 'z']) else: assert r.size == 0 print('COMPLETED')