def test_64bit_symbolsize(self): """ Asserts that SymbolSizeExceptions are raised for invalid symbol sizes on 64 bit systems A good symbolsize for 64 bit is in increments of 8 BYTES """ #save the original original = config._64BIT try: config._64BIT = True # Try a bad symbolsize. with self.assertRaises(SymbolSizeException): c = Chunker(10, 7) try: # Try a good symbol size (exactly 8 BYTES) c = Chunker(10, 8) # Try a multiple of 8BYTES c = Chunker(10, 24) except Exception, e: self.fail(e.tostring()) finally: # Restore original config._64BIT = original
def execute(self): self._pool = Pool(config.pool_processors) # We need to generate blank files before writing them # Python doesnt allow those at execution time..? for parser in instance_parsers + node_parsers: write.write_header_csv(parser.file_name, parser.csv_header) logging.info('Created blank csv') self._pool.starmap(_parse, zip( repeat(self._writer,), repeat(config.run_log), repeat('test'), Chunker.chunkify(config.run_log, config.file_chunk_size), repeat(instance_parsers), )) for node in self._context.nodes.values(): self._pool.starmap(_parse, zip( repeat(self._writer), repeat(node.get_log_file()), repeat(node.name), Chunker.chunkify(node.get_log_file(), config.file_chunk_size), repeat(node_parsers), )) self._pool.close() logging.info('Parsing done')
def execute(self): self._pool = Pool(config.pool_processors) for parser in host_parsers + node_parsers: self._writer.write_header_csv(parser.file_name, parser.csv_header) logging.info('Created all empty csv files') self._pool.starmap( _parse, zip( repeat(self._writer, ), repeat(config.run_log), repeat('simcoin'), Chunker.chunkify(config.run_log, config.file_chunk_size), repeat(host_parsers), )) for node in self._context.nodes.values(): self._pool.starmap( _parse, zip( repeat(self._writer), repeat(node.get_log_file()), repeat(node.name), Chunker.chunkify(node.get_log_file(), config.file_chunk_size), repeat(node_parsers), )) self._pool.close() logging.info('Finished parsing of run_log and all node logs')
def __init__(self, *args, **kwargs): self.drives = [] self.drive_index = 0 self.got_ack = False self.chunker = Chunker(server=True) self.ws = None super().__init__(*args, **kwargs)
def loadMovieLensParallel(self): self.data = {} #init objects pool = mp.Pool(self.cores) jobs = [] #create jobs fname = self.path + "u.data" for chunkStart, chunkSize in Chunker.chunkify(fname): jobs.append( pool.apply_async(self.process_data, (fname, chunkStart, chunkSize))) #wait for all jobs to finish for job in jobs: job.get() #clean up pool.close() #init objects pool = mp.Pool(self.cores) jobs = [] fname = self.path + "u.item" for chunkStart, chunkSize in Chunker.chunkify(fname): jobs.append( pool.apply_async(self.process_item, (fname, chunkStart, chunkSize))) #wait for all jobs to finish for job in jobs: job.get() #clean up pool.close() #init objects pool = mp.Pool(self.cores) jobs = [] fname = self.path + "u.user" for chunkStart, chunkSize in Chunker.chunkify(fname): jobs.append( pool.apply_async(self.process_users, (fname, chunkStart, chunkSize))) #wait for all jobs to finish for job in jobs: job.get() #clean up pool.close()
def test_block_ids(self): """ Tests the generation of block ids The first should be 0 The second should be 0 + 1 """ c = Chunker(DEFAULT_K, DEFAULT_SYMBOLSIZE) expected_block_id = 0 # Check that first blockid = 0 self.assertEqual(c.get_block_id(), expected_block_id) # Check that the next block id = 1 expected_block_id = 1 self.assertEqual(c.get_block_id(), expected_block_id)
def _parse(writer, log_file, name, chunk, parsers): parsed_objects = {} for line in Chunker.parse(Chunker.read(log_file, chunk)): for parser in parsers: try: parsed_object = parser.from_log_line(line, name) parsed_objects.setdefault(parsed_object.file_name, []).append(parsed_object) break except ParseException: pass for key in parsed_objects: writer.append_csv(key, parsed_objects[key]) logging.info('Parsing the type of tick {}' .format(len(parsed_objects)))
def test_block_ids(self): """ Tests the generation of block ids The first should be 0 The second should be 0 + 1 """ c = Chunker(DEFAULT_K, DEFAULT_SYMBOLSIZE) expected_block_id = 0 # Check that first blockid = 0 self.assertEqual(c.get_block_id(), expected_block_id) # Check that the next block id = 1 expected_block_id = 1 self.assertEqual(c.get_block_id(), expected_block_id)
def scores(self, viterbi_sequences, id_to_tag_dict, validation_labels, batch): # Get performance of model: precision, recall, f1-score found_viterbi = 0 correct_viterbi = 0 found_y = 0 # i.e. correct_y for seq in range(len(viterbi_sequences)): viterbi_tag_idxs = viterbi_sequences[seq] viterbi_tags = [ id_to_tag_dict[i] for i in np.asarray(viterbi_tag_idxs) ] chunker = Chunker() viterbi_entities = chunker.extract_named_entities(viterbi_tags) y_tag_idxs = validation_labels[batch][seq][:len(viterbi_tag_idxs)] y_tags = [id_to_tag_dict[j] for j in y_tag_idxs] y_entities = chunker.extract_named_entities(y_tags) found_viterbi += len(viterbi_entities) found_y += len(y_entities) for ner in viterbi_entities: if ner in y_entities: correct_viterbi += 1 if found_viterbi > 0: print("Found (correct) " + str(correct_viterbi)) print("Found (guessed) " + str(found_viterbi)) self.precision = 100.00 * (correct_viterbi / found_viterbi) print("Precision: " + str(self.precision)) else: print("Precision 0. No named entities guessed") if found_y > 0: print("Found (correct) " + str(correct_viterbi)) print("Actual correct " + str(found_y)) self.recall = 100.00 * (correct_viterbi / found_y) print("Recall: " + str(self.recall)) else: print("Recall 0. No named entities in gold standard.") if (self.precision > 0) and (self.recall > 0): self.f1_score = 2.0 * self.precision * self.recall / ( self.precision + self.recall) print("F1 score: " + str(self.f1_score)) else: print("F1 score 0. Precision and recall are 0.")
def test_small_chunk(self): new_lijst = [i for i in range(20)] new_lijst_chunker = Chunker(new_lijst) new_lijst50 = list(new_lijst_chunker(50)) self.assertEqual(len(new_lijst50), 1) self.assertEqual(new_lijst50[0][19], 19)
def test_even_chunk(self): lijst = [i for i in range(300)] lijst_chunker = Chunker(lijst) lijst50 = list(lijst_chunker(50)) self.assertEqual(len(lijst50[0]), 50) self.assertEqual(lijst50[1][0], 50) self.assertEqual(len(lijst50), 6) self.assertEqual(lijst50[5][49], 299)
def test_odd_chunk(self): lijst = [i for i in range(253)] lijst_chunker = Chunker(lijst) lijst47 = list(lijst_chunker(47)) self.assertEqual(len(lijst47[0]), 47) self.assertEqual(lijst47[1][0], 47) self.assertEqual(len(lijst47), 6) self.assertEqual(lijst47[5][17], 252)
def test_32bit_init(self): """ Tests the initialization of the chunker on 32 bit systems """ original = config._64BIT config._64BIT = False try: c = Chunker(DEFAULT_K, DEFAULT_SYMBOLSIZE) self.assertEqual(c.dtype, "uint32") self.assertEqual(c.bytes_per_int, 4) finally: config._64BIT = original
def readMovies(self, path='/ml-25m'): self.data = {} #init objects pool = mp.Pool(8) jobs = [] #create jobs fname = self.path + path + "/ratings.csv" for chunkStart, chunkSize in Chunker.chunkify(fname): jobs.append( pool.apply_async(self.process_data, (fname, chunkStart, chunkSize))) for job in jobs: job.get() #clean up pool.close()
def test_chunker_init(self): """ Tests the basic initialization of the chunker class """ expected_blocksize = DEFAULT_SYMBOLSIZE * DEFAULT_K expected_block_id = 0 c = Chunker(DEFAULT_K, DEFAULT_SYMBOLSIZE) # Check k self.assertEqual(c.k, DEFAULT_K) # Check Symbolsize self.assertEqual(c.symbolsize, DEFAULT_SYMBOLSIZE) # Check blocksize self.assertEqual(c.blocksize, expected_blocksize) # Check expected block id self.assertEqual(c.block_id, expected_block_id)
def simulator_evaluate_all(file, evaluation_list=None): if evaluation_list is None: evaluation_list = [] chunker = Chunker(file) for each_chunk in chunker: simulator_jevaluate(each_chunk, evaluation_list)
from chunker import Chunker ch = Chunker() kalimat = "Richard Winger , rekan di Boston Consulting Group , menambahkan : Belakangan ini , sangat populer jika menghias diri anda dengan bendera ." print(ch.tree_to_str(ch.chunk_me1(kalimat))) print(ch.tree_to_str(ch.chunk_me2(kalimat))) print(ch.tree_to_str(ch.chunk_me3(kalimat)))
def parse(): "parse POSTed Korean sentence" # grab sentence to parse sentence = request.form.get('sentence') if not sentence: return jsonify(result="FAIL", msg="Missing sentence") # build a string for the KHaiii phoneme analyzer if sentence.strip()[-1] not in ['.', '?', '!']: sentence += '.' # run Khaiii words = [] for w in khaiiiAPI.analyze(sentence): for m in w.morphs: words.append('{0}:{1}'.format(m.lex.strip(), m.tag)) posString = ';'.join(words) # map POS through synthetic tag mapper mappedPosList = TagMap.mapTags(posString) # perform chunk parsing chunkTree = Chunker.parse(mappedPosList) # apply any synthetic-tag-related node renamings TagMap.mapNodeNames(chunkTree) # build descriptive phrase list phrases = Chunker.phraseList(chunkTree) # recursively turn the chunk tree into a Python nested dict for the JSON response def asDict(chunk): while isinstance(chunk, nltk.Tree) and len(chunk) == 1: # flatten degenerate tree nodes chunk = chunk[0] if isinstance(chunk, nltk.Tree): return dict( type='tree', tag='Sentence' if chunk.label() == 'S' else chunk.label(), children=[asDict(t) for t in chunk]) else: return dict(type='pos', word=chunk[0].strip(), tag=chunk[1]) # parseTree = asDict(chunkTree) debugging = dict(posList=pformat(words), mappedPosList=pformat(mappedPosList), phrases=pformat(phrases), parseTree=pformat(parseTree)) return jsonify(result="OK", posList=words, mappedPosList=mappedPosList, phrases=phrases, parseTree=parseTree, debugging=debugging) if False: # synthetic tag patterns - # patterns of these word:POC strings are preprocessed to define new # synthetic word:POC tags used in the chunking grammar below # at present, these are applied in the order longest-to-shortest pattern, we should probably make this a listfor explicit ordering tagMappings = { r'들:(TM|XSN)': r'들:PLU', # pluralizer r'기:(ETN|NNG)': r'기:GNOM', # nominalizer r'(ㄴ|는|ㄹ):ETM;것:NNB': r'\1 것:GNOM', # nominalizer r'(은|는):JX': r'\1:JKS', # turn topic-marking partcile into subject-marker (I think this is right??) r'(ㄹ|을|를):ETM;거:NNB;이:VCP': r'\1 거 이:FUT', # ㄹ/를 거 이다 future-tense conjugator (hack!) r'전:NNG;에:JKB': r'전에:BEF', # before r'때문:NNB;에:JKB': r'때문에:BEC', # because r'및:MAG': r'및:ALS', # also connector (why is it an adverb??) r'또는:MAG': r'또는:ALT', # alternative connector (why is it an adverb??) r'에:JKB;(대하|관하):VV;([^:]+):EC': r'에 \1\2:PRP', # preposition "about } # prepositional phrase suffix-patterns (generate a <PRPP> pos-tag + metadata to label the parsing # tag-pattern replacement subtree name-mapping reference links # (r'전:NNG;에:JKB', r'전에:PRP', "PrepositionalPhrase:Before", ("ttmik:lessons/level-3-lesson-10", "htsk:unit1/unit-1-lessons-17-25-2/lesson-24/#242")) # before # generate a version of the parser's original word:POC string including synthetic tag mappings above tagString = ';' + posString + ';' for old, new in sorted(tagMappings.items(), key=lambda x: len(x[0]), reverse=True): tagString = re.sub(';' + old + ';', ';' + new + ';', tagString) mappedWords = [ tuple(pos.split(':')) for pos in tagString.strip(';').split(';') ] # Korean phrase NLTK chunking grammar grammar = r""" HadaVerb: {<NN.*><XSV>} AuxiliaryVerb: {<EC><VX|VV>} Adverb: {<MAG>} NominalizedVerb: {<VV|HadaVerb><EP|FUT>*<GNOM>} Adjective: {<Adverb>*<VA><ETM>} DescriptiveVerb: {<VA>} Verb: {<VV|VCN|HadaVerb|DescriptiveVerb>} VerbSuffix: {<EP|FUT>*<EF|EC>} Location: {<JKB>} Title: {<XSN>} Preposition: {<PRP>} Noun: {<NN.*|NR|SL>} Pronoun: {<NP>} Substantive: {<Noun><Noun>*} {<Pronoun>} {<NominalizedVerb>} NounPhrase: {<XPN>*<MAG>*<Adjective>*<Substantive><Title>*<Location>*<PLU>*<JX>*<Preposition>*} Possessive: {<NounPhrase><JKG><NounPhrase>} Component: {<NounPhrase|Possessive><JC|ALS|ALT>} Connection: {<Component><Component>*<NounPhrase|Possessive>} Constituent: {<NounPhrase|Possessive|Connection>} Complement: {<Constituent><JKC>} Object: {<Constituent><JKO>} Subject: {<Constituent><JKS>} Before: {<Constituent|Object|Subject>*<Constituent><BEF>} Because: {<Constituent|Object|Subject>*<Constituent><BEC>} Copula: {<Constituent><Adverb>*<VCP><AuxiliaryVerb>*<VerbSuffix>} Predicate: {<Adverb>*<Verb><AuxiliaryVerb>*<VerbSuffix>} """ # Component: { < NounPhrase > < JC | ALS > } # Connection: { < Component > < Component > * < NounPhrase > } # Possessive: { < NounPhrase | Connection > < JKG > < NounPhrase | Connection > } # Constituent: {<Subject|Object|Complement>} # Clause: {<Constituent>*<Predicate>} # Sentence: {<Clause><Clause>*} # gen chunk tree from the word-POS list under the above chunking grammar parser = nltk.RegexpParser(grammar, trace=1) print(parser._stages) chunkTree = parser.parse(mappedWords) # heuristic subtree simplifications # toss sentence end node if not isinstance(chunkTree[-1], nltk.Tree) and chunkTree[-1][1] == 'SF': chunkTree.remove(chunkTree[-1]) # flatten connection trees def flattenConnections(t): for st in t: if isinstance(st, nltk.Tree): if st.label() == 'Connection': # if Connection node, pull up component tuples into one long connection sequence for i, c in enumerate(list(st)[:-1]): st[2 * i] = c[0] st.insert(2 * i + 1, c[1]) else: flattenConnections(st) flattenConnections(chunkTree) # generate phrase-descriptors from top-level subtrees hiddenTags = { 'Substantive', 'Constituent', 'NounPhrase', 'Connection', } def flattenPhrase(t, phrase): for st in t: if isinstance(st, nltk.Tree): phrase = flattenPhrase(st, phrase) if st.label() not in hiddenTags: phrase.append({"type": 'label', "word": st.label()}) else: phrase.append({ "type": 'word', "word": st[0].strip(), "tag": st[1] }) # st[1][0] if st[1][0] in ('N', 'V') else st[0].strip() return phrase # phrases = [] for t in chunkTree: if isinstance(t, nltk.Tree): phrase = flattenPhrase(t, []) if t.label() not in hiddenTags: phrase.append({"type": 'label', "word": t.label()}) phrases.append(phrase) else: phrases.append(('word', t[0].strip())) for p in phrases: print(p) # recursively turn the chunk tree into a Python nested dict for the JSON response def asDict(chunk): while isinstance(chunk, nltk.Tree) and len(chunk) == 1: # flatten degenerate tree nodes chunk = chunk[0] if isinstance(chunk, nltk.Tree): return dict( type='tree', tag='Sentence' if chunk.label() == 'S' else chunk.label(), children=[asDict(t) for t in chunk]) else: return dict(type='pos', word=chunk[0].strip(), tag=chunk[1]) # parseTree = asDict(chunkTree) debugging = dict(posList=pformat(words), mappedPosList=pformat(mappedWords), phrases=pformat(phrases), parseTree=pformat(parseTree)) return jsonify(result="OK", posList=words, mappedPosList=mappedWords, phrases=phrases, parseTree=parseTree, debugging=debugging)
def test_not_list_passed(self): with self.assertRaises(TypeError): dict = {'test': 15, 'test2': 16} Chunker(dict)
def test_chunker(reference_chunks): with open('bootstrap.utf.txt', encoding='utf-8') as bootstrap_file: chunker = Chunker(bootstrap_file) for index, each_chunk in enumerate(chunker): reference_chunk = reference_chunks[index] assert each_chunk == reference_chunk
def put(user_id, fileName): # read data from input file with open(fileName, 'rb') as fin: bytes = fin.read(BYTE_SIZE) fileSize = len(bytes) logging.debug("Bytes read with length {0}.".format(fileSize)) # begin chunking chunkObj = Chunker() cuts = chunkObj.chunking(bytes) logging.debug("Num of Cuts: {0}".format(len(cuts))) # for index, chunk in enumerate(chunkObj.divide_chunk(bytes, cuts)): # logging.debug("Chunk {0}: {1}".format(index, chunk)) logging.info("Chunking completed.\n") # create meta data fid = fileName + str(user_id) cryptObj = Crypto(fid, 'client') iv = cryptObj.gen_iv(bytes) metaObj = Meta(user_id, fileName, fileSize, iv) logging.debug("FileName: {0}".format(metaObj.fileName)) logging.debug("IV: {0}\n".format(iv)) # gen keys try: keys = [] with con("localhost", 18862) as keyserver: if keyserver.root.isExist(fid): logging.error("File Already Exist.") exit(1) for i, chunk in enumerate(chunkObj.divide_chunk(bytes, cuts)): key = keyserver.root.gen_key(fid, chunk) if not key: logging.error("Key is empty.") exit(1) keys.append(key) logging.debug("Key {0}: {1}".format(i, key)) except: logging.error("Cannot generate key.") exit(1) logging.info("Key generation completed.\n") # encrypt chunks ciphers = [] count = 0 for chunk, key in zip(chunkObj.divide_chunk(bytes, cuts), keys): cipher = cryptObj.encode(chunk, key, iv) ciphers.append(cipher) # logging.debug("Cipher {0}: {1}".format(count, cipher)) count += 1 del bytes del keys logging.info("Encryption completed.\n") # send ciphers to server try: with con("localhost", 18861) as master: if master.root.put(metaObj): for cipher in ciphers: master.root.put(metaObj, cipher) logging.info("Chunk with size {0} uploaded.".format( len(cipher))) else: logging.error("File Already Exist.") exit(1) except: logging.error("Cannot upload file.") exit(1)
import os from flask import Flask, request, json, render_template, Response from flask_cors import CORS from chunker import Chunker ch = Chunker() app = Flask(__name__) CORS(app) @app.route('/') def hello_world(): return 'Hello World' @app.route('/chunk', methods = ['POST']) def chunk(): data_all = request.get_json() kalimat = data_all['kalimat'] return str(ch.tree_to_str(ch.chunk_me2(kalimat))) # _ists = iSTSEngine() # return _ists.singleTest() if __name__ == '__main__': app.run(host="0.0.0.0", port=5000)
'-n', '--n-workers', type=int, default=DEFAULT_WORKERS, help=f"number of workers to use (default depends on your system)") parser.add_argument( '--spacy-langs', nargs='+', default=['de', 'en'], help='spaCy model language names (in order) to use for lemmatization') parser.add_argument( '--outdir', default=None, help='directory to put outputs (if different original file locations)') parser.add_argument( '--logdir', default=None, help='directory to put logs (default will be current working dir)') args = parser.parse_args() args = vars(args) fps = args.pop('fps') workers = args.pop('n_workers') b_size = args.pop('batch_size') max_tasks = args.pop('max_tasks_per_child') outdir = args.pop('outdir') representer = ParallelSpacy(**args) representer.chunker = Chunker(fps, b_size) representer.process(fps, workers, max_tasks, outdir=outdir)
def parseInput(input, parser="RD", showAllLevels=False, getWordDefinitions=True): "parse input string into list of parsed contained sentence structures" # parser can be RD for recusrsive descent (currently the most-developed) or "NLTK" for the original NLTK chunking-grammar parser # clean & build a string for the KHaiii phoneme analyzer input = input.strip() if input[-1] not in ['.', '?', '!']: input += '.' input = re.sub( r'\s+([\.\?\;\,\:])', r'\1', input) # elide spaces preceding clause endings, throws Khaiii off # input = input.replace(',', ' , ').replace(';', ' ; ').replace(':', ' : ') - adding a space before punctuation seems to mess tagging in Khaiii log("* parse {0}".format(input)) # run Khaiii, grab the parts-of-speech list it generates (morphemes + POS tags) and extract original word-to-morpheme groupings sentences = [] # handle possible multiple sentences posList = [] morphemeGroups = [] for w in khaiiiAPI.analyze(input): morphemeGroups.append( [w.lex, [m.lex for m in w.morphs if m.tag != 'SF']]) for m in w.morphs: posList.append('{0}:{1}'.format(m.lex.strip(), m.tag)) if m.tag == 'SF': # sentence end, store extractions & reset for possible next sentence sentences.append( dict(posList=posList, morphemeGroups=morphemeGroups, posString=';'.join(posList))) posList = [] morphemeGroups = [] for s in sentences: # map POS through synthetic tag mapper & extract word groupings mappedPosList, morphemeGroups = TagMap.mapTags( s['posString'], s['morphemeGroups']) #, disableMapping=True) log(" {0}".format(s['posString'])) log(" mapped to {0}".format(mappedPosList)) if parser == "NLTK": # NLTK chunking parser # perform chunk parsing chunkTree = Chunker.parse(mappedPosList, trace=2) chunkTree.pprint() # apply any synthetic-tag-related node renamings TagMap.mapNodeNames(chunkTree) # extract popup wiki definitions & references links & notes for implicated nodes references = TagMap.getReferences(chunkTree) # build descriptive phrase list phrases = Chunker.phraseList(chunkTree) # parseTreeDict = buildParseTree(chunkTree, showAllLevels=showAllLevels) else: # recursive-descent parser from rd_grammar import KoreanParser parser = KoreanParser([":".join(p) for p in mappedPosList]) parseTree = parser.parse(verbose=1) if parseTree: # apply any synthetic-tag-related node renamings parseTree.mapNodeNames() # extract popup wiki definitions & references links & notes for implicated nodes references = parseTree.getReferences() # build descriptive phrase list phrases = parseTree.phraseList() # get noun & verb translations from Naver wordDefs = getWordDefs( mappedPosList) if getWordDefinitions else {} # build JSONable parse-tree dict parseTreeDict = parseTree.buildParseTree( wordDefs=wordDefs, showAllLevels=showAllLevels) log(" {0}".format(parseTree)) else: # parsing failed, return unrecognized token parseTree = references = parseTreeDict = phrases = None s.update( dict(error="Sorry, failed to parse sentence", lastToken=parser.lastTriedToken())) log(" ** failed. Unexpected token {0}".format( parser.lastTriedToken())) # format debugging daat debugging = dict(posList=pformat(s['posList']), mappedPosList=pformat(mappedPosList), phrases=pformat(phrases), morphemeGroups=pformat(morphemeGroups), parseTree=pformat(parseTreeDict), references=references) # add parsing results to response structure s.update( dict(mappedPosList=mappedPosList, morphemeGroups=morphemeGroups, parseTree=parseTreeDict, references=references, phrases=phrases, debugging=debugging)) # return sentences
class WSView(RedisView): def __init__(self, *args, **kwargs): self.drives = [] self.drive_index = 0 self.got_ack = False self.chunker = Chunker(server=True) self.ws = None super().__init__(*args, **kwargs) @property def latest_drive(self): if not self.drives: return None drive_id = self.drives[self.drive_index] self.drive_index += 1 if self.drive_index >= len(self.drives): self.drive_index = 0 return drive_id async def send(self, data): await self.ws.send_str(json.dumps(data)) async def get(self): ws = aiohttp.web.WebSocketResponse(heartbeat=2) print('Opening a socket.') await ws.prepare(self.request) print('Websocket connection ready') self.got_ack = True self.ws = ws await self.create_redis() async for message in self.ws: if message.type != aiohttp.WSMsgType.TEXT: continue print( f"{message.data}; chunk: {self.chunker.current_chunk}; drive: {self.drive_index}; ack: {self.got_ack}; {len(self.drives)} drives" ) await self.handle_message(message) print('Websocket connection closed') await self.cleanup_redis() return self.ws async def handle_message(self, message): data = json.loads(message.data) if data["cmd"] == 'close': await self.ws.close() elif data["cmd"] == "ack": self.got_ack = True elif data["cmd"] == "setchunk": self.chunker.set_chunk(data["i"]) elif data["cmd"] == "ping": if not self.drives: await self.send({"cmd": "getinfo"}) return if not self.got_ack: return self.got_ack = False await self.send({ "cmd": "getchunk", "chunk_id": self.chunker.current_chunk, "address": self.latest_drive }) self.chunker.next_chunk() elif data["cmd"] == "drives": self.drives = data["drives"]