def main(): sequenceFile = "./../RAW_DATA/seq/seq.tab" outputPath = "./../P_DATA/loc_files/from_python/" print("processing files ... ") start_time = time.time() numberOfProbes = 0 numberOfGstackProbes = 0 with open(sequenceFile, encoding="utf-8") as sequenceFile: for line in sequenceFile: if numberOfProbes > 0: p_id, pset_id, p_x, p_y, assembly, seqname, start, stop, strand, p_seq, tar_std, cat = line.split( ) pg_loc = FindNTstack(p_seq, 'G') if pg_loc > 0 and cat.find("main") != -1: numberOfGstackProbes += 1 probe = Probe(pset_id, p_x, p_y, cat) probe.WriteProbes(outputPath, pg_loc, True) numberOfProbes += 1 textToRatioFile = ""\ + "Total No. of Probes in Seq File : \t" + str(numberOfProbes) \ + "\nTotal No. of G-Stack Probes in Seq File : \t" + str(numberOfGstackProbes) \ + "\nPercentage of G-Stack Probes : \t" + str(numberOfGstackProbes/numberOfProbes*100) \ + "\nTotal no. of Probe set containing G-Stack Probes : \t" \ + str(ProbeSetWithGstack(outputPath + "combined.txt")) \ + "\n\nProgram took " + str(round(time.time() - start_time, 2)) + " seconds to execute." WriteToFile(outputPath + "ratio.txt", textToRatioFile) print("done processing ... ")
def __init__ (self, ram_size, reg_count, code, consolog): self._ram = Memory(ram_size) self._reg = Memory(reg_count) self.__ra = reg_count - 1 self.__sp = reg_count - 2 self._code = code self._probe = Probe(consolog)
def run(): f = open('../input.txt', 'r') paths = [line.strip().split(',') for line in f.readlines()] f.close() probe = Probe(paths) print("Closest: {} | Fewest: {}".format(probe.closest_intersection(), probe.fewest_steps()))
def __helper__(self, a_link, current_path, root_page_path): probe = Probe() probe.parse_string(current_path, ''' <html><head></head> <body> <a href="%s"></a> </body></html> ''' % a_link) return probe.get_internal_url(root_page_path)
def start(): plot = Plot() probe = Probe() channel = Channel() plot.add_channel(channel) while not plot.closed: samples = probe.get_samples(1024) # samples = integrate_samples(samples) channel.add_samples(samples) plot.update_plot()
def plot_separatrix_crossings(self, axes, sepmode='patch', **kw): if sepmode == 'patch': try: xsep = self._get_xsep(**kw).reshape((-1, 2)) for ax in axes: for xlim in xsep: vrect(ax, xlim, **kw) except: warn("Could not generate patches, falling back to lines") Probe.plot_separatrix_crossings(self, axes, **kw) elif sepmode == 'lines': Probe.plot_separatrix_crossings(self, axes, **kw) else: raise ValueError("Unknown sepmode: '%s'" % sepmode)
def __init__ (self, ram_size, reg_count, code, consolog): self._ram = Memory(ram_size) self._reg = Memory(reg_count) self.__ra = reg_count - 1 self.__sp = reg_count - 2 self._code = code self._ip = 0 # init instruction pointer self._max_ip = len(self._code) self._reg[self.__sp] = self._ram._size # init stack pointer self._reg[self.__ra] = self._max_ip # init return address self._probe = Probe(consolog)
class TestProbeCheckInternal(unittest.TestCase): def setUp(self): self.probe = Probe() self.probe.url = 'http://www.google.co.kr' @unittest.skip('fail skip') def test_fail(self): self.assertEqual(1, 2) def test_full_internal_url(self): root_page_url = 'http://www.google.co.kr' current_url = 'http://www.google.co.kr/foo' result = self.probe.check_internal_url(root_page_url, current_url) self.assertTrue(result) def test_different_netloc(self): root_page_url = 'http://www.google.co.kr' current_url = 'http://foogle.co.kr/foo' result = self.probe.check_internal_url(root_page_url, current_url) self.assertFalse(result) def test_relative(self): root_page_url = 'http://www.google.co.kr' current_url = 'sub' result = self.probe.check_internal_url(root_page_url, current_url) self.assertTrue(result) def test_relative2(self): root_page_url = 'http://www.google.co.kr/sub' current_url = 'sub' result = self.probe.check_internal_url(root_page_url, current_url) self.assertTrue(result) def test_absolute_fail(self): root_page_url = 'http://www.google.co.kr/sub' current_url = '/othersub' result = self.probe.check_internal_url(root_page_url, current_url) self.assertFalse(result)
def __init__(self, shn, shot=None, head=None, dig=None, eqi=None): if shot is None: shot = self.find_shot(shn) if head is None: head = shot.head if dig is None: dig = shot.dig self.shot = shot digitizer = DigitizerClasses[dig](shn) try: viewers = eqi.get_viewers(self) except AttributeError: viewers = () Probe.__init__(self, head, digitizer, R0=1.645, z0=-0.966, eqi=eqi, viewers=viewers)
def create_probe_grid(self): positions = [] delta = 20 margin = 20 for x in range(margin, self.width - margin, delta): for y in range(margin, self.height - margin, delta): positions.append((x, y)) for pos in positions: probe = Probe() probe.body.position = pos # Create the probe's sensors probe.range_scanner = RangeScanner("RangeScan:nonlandmarks", WALL_MASK|ROBOT_MASK|ANY_PUCK_MASK|ANY_LANDMARK_MASK, WALL_MASK|ROBOT_MASK|RED_PUCK_MASK) probe.landmark_scanner = RangeScanner("RangeScan:landmarks", WALL_MASK|ANY_LANDMARK_MASK, WALL_MASK|ANY_LANDMARK_MASK) self.probes.append(probe)
def gauges(da): from probe import Probe entries = [ [0, 0], [34, 10], [34, 34], [10, 34], [0, 15], ] prb = Probe(da, entries) return prb
def main(files, out, threshold): out_files = [] for file_path in list(files): if Path(file_path).is_dir(): out_files.extend(handle_folder(file_path)) else: out_files.append(file_path) analyse_clips = Analysis(out_files).summary() output_timeline = Timeline().create_timeline(settings=analyse_clips) speech_detector = SpeechDetection(batch_size=8, threshold=threshold, progress_hook=progress) speech_detector_cpu = SpeechDetection(batch_size=8, device="cpu", threshold=threshold, progress_hook=progress) output_file = open(out, 'w+') for i, file_path in enumerate(out_files): file_properties = Probe(file_path).run().extract_summary() temp_dir = tempfile.TemporaryDirectory() temp_audio_file = Encoder.wav_audio_temp(file_path, temp_dir.name) print(f'Processing File {i + 1} of {len(out_files)}: {file_path}') for _ in range(0, 3): # Try two more times if there's an error. try: speech = speech_detector.run(temp_audio_file).to_frames( file_properties['video']['frame_rate']) break except Exception as e: print(e) try: speech = speech_detector_cpu.run( temp_audio_file).to_frames( file_properties['video']['frame_rate']) print("Processed using CPU") break except Exception as e: pass output_timeline.add_file(file_path, speech) output_file.seek(0) output_file.truncate() output_file.write(output_timeline.export()) temp_dir.cleanup() output_file.close()
def run_test_resource(resource): """tests a service and provides run metrics""" result = ResourceResult(resource) result.start() probes = resource.probe_vars for probe in probes: result.add_result(Probe.run(resource, probe)) result.stop() return result
def create_probe_grid(self): positions = [] delta = 20 margin = 20 for x in range(margin, self.width - margin, delta): for y in range(margin, self.height - margin, delta): positions.append((x, y)) for pos in positions: probe = Probe() probe.body.position = pos # Create the probe's sensors probe.range_scanner = RangeScanner( "RangeScan:nonlandmarks", WALL_MASK | ROBOT_MASK | ANY_PUCK_MASK | ANY_LANDMARK_MASK, WALL_MASK | ROBOT_MASK | RED_PUCK_MASK) probe.landmark_scanner = RangeScanner( "RangeScan:landmarks", WALL_MASK | ANY_LANDMARK_MASK, WALL_MASK | ANY_LANDMARK_MASK) self.probes.append(probe)
def _gendata(): jdict = Sensor.main() jdict.update(Probe()) insert_str = f"""\ INSERT INTO pi_probe( plc_mac, scan_data ) VALUES( '58:52:8a:d6:69:a1', '{json.dumps(jdict)}' ); """ send_d = {"sql": {"db": "piscan", "query": insert_str, "commit": True}} return send_d
def calib(self): Probe.calib(self) s = slice(5000) self.norm_to_region(s) S = self.S S['Rs'] = S['R'].smooth(100, mode='gaussian') S['tip1+tip2'] = S['tip1'] + S['tip2'] S['tip1+tip2'].update(label='Mach tips sum') if not S['tip1+tip2'].V.is_swept: mask = (S['tip1+tip2'].V > -150.) | (S['tip1+tip2'] > 1.9) S['tip1+tip2'] = S['tip1+tip2'].masked(mask) S['tip1'] = S['tip1'].masked(mask) S['tip2'] = S['tip2'].masked(mask) tips = self.head.tips A = [0.5*tips[0].area, 0.5*tips[1].area, tips[2].area] A.append(A[0] + A[1]) for tip, area in zip(('tip1', 'tip2', 'tip3', 'tip1+tip2'), A): S[tip + 'j'] = S[tip] / area S[tip + 'j'].update(type='Current density', units='A / m**2')
def run_test_resource(resource): """tests a service and provides run metrics""" result = ResourceResult(resource) if not resource.active: result.message = 'Skipped' return result result.start() probes = resource.probe_vars for probe in probes: result.add_result(Probe.run(resource, probe)) result.stop() return result
def run_test_resource(resource): """tests a service and provides run metrics""" result = ResourceResult(resource) if not resource.active: result.message = 'Skipped' return result result.start() probes = resource.probe_vars for probe in probes: result.add_result(Probe.run(resource, probe)) result.stop() return result
def sample_space(self, iterations, output_filename): outfile = open(output_filename, 'w') outfile.write( 'Probe 5 trunc,Probe 3 trunc,Probe* 5 trunc,Probe* 3 trunc,Probe blunt end,Sink 5 trunc,Sink 3 trunc,Sink* 5 trunc,Sink* 3 trunc,beta,self.SNP activation,self.WT activation,Background activation\n' ) for i in range(iterations): print('Iteration: ' + str(i)) p = Probe.Probe(self.SNP, self.WT, self.minlength, self.concentrations, self.params, self.mutation_rate) p.display() for t in p.truncations: outfile.write(str(t) + ',') for b in p.beta: outfile.write(str(b) + ',') outfile.write('\n') outfile.close()
def getProbePoints(self): print 'This will take time....' with open(self.probefile, 'r') as probe: i = 0 readData = csv.reader(probe) MPoutputdata = open("Partition6467MatchedPoints.csv", 'w') MPoutputdata.close() SPoutputdata = open("SlopeOutput.csv", 'w') SPoutputdata.close() data = 'sampleID, dateTime, sourceCode, latitude, longitude, altitude, speed, heading, linkPVID, direction, distFromRef, distFromLink' MPoutputdata = open("Partition6467MatchedPoints.csv", 'a') MPoutputdata.write(data + "\n") for data in readData: pdata = Probe(int(data[0]),str(data[1]),int(data[2]),float(data[3]),float(data[4]),\ float(data[5]),float(data[6]),float(data[7])) if i == 0: self.probepoints.append(pdata) i = i + 1 elif self.probepoints[i - 1].sampleID == pdata.sampleID: self.probepoints.append(pdata) i = i + 1 else: self.probepoints = [] self.probepoints.append(pdata) i = 1 link = self.plotProbePoint(pdata) linkPVID = link.linkPVID distFromLink = link.perDistance(pdata) distFromRef = link.haversine(pdata) slope = self.slope(link, pdata) direction = link.directionOfTravel if direction == 'B': direction = 'X' data = "%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s" % ( pdata.sampleID, pdata.dateTime, pdata.sourceCode, pdata.latitude, pdata.longitude, pdata.altitude, pdata.speed, pdata.heading, linkPVID, direction, distFromRef, distFromLink) MPoutputdata.write(data + "\n") print str(pdata) + ' Link ID ' + str(linkPVID) + ',' + str( direction) + ',' + str(distFromRef) + ',' + str( distFromLink) + ',' + '\n' + slope + '\n' MPoutputdata.close()
def sample_space_random(self, iterations, output_filename): outfile = open(output_filename, 'w') outfile.write( 'Probe 5 trunc,Probe 3 trunc,Probe* 5 trunc,Probe* 3 trunc,Probe blunt end,Sink 5 trunc,Sink 3 trunc,Sink* 5 trunc,Sink* 3 trunc,beta,self.SNP activation,self.WT activation,Background activation, self.SNP Sequence, self.WT Sequence, self.SNP Index\n' ) for i in range(iterations): print('Iteration: ' + str(i)) sequences = self.generate_target() self.SNP = sequences[0] self.WT = sequences[1] self.SNP_index = sequences[2] p = Probe.Probe(self.SNP, self.WT, self.minlength, self.concentrations, self.params, self.mutation_rate) p.display() for t in p.truncations: outfile.write(str(t) + ',') for b in p.beta: outfile.write(str(b) + ',') outfile.write(self.SNP + ',') outfile.write(self.WT + ',') outfile.write(str(self.SNP_index)) outfile.write('\n') outfile.close()
def plot(self, *args, **kw): kw.setdefault('sepmode', 'patch') return Probe.plot(self, *args, **kw)
def mk_R_axis(self, axes, Rj=None, tlab=None, xlab='Probe R (cm)', **kw): if tlab is None: tlab = np.r_[np.arange(131, 135), np.arange(135, 160, 5)] if Rj is None: Rj = 0.01*tlab return Probe.mk_R_axis(self, axes, Rj, tlab=tlab, xlab=xlab, **kw)
class CPU: def __init__ (self, ram_size, reg_count, code, consolog): self._ram = Memory(ram_size) self._reg = Memory(reg_count) self.__ra = reg_count - 1 self.__sp = reg_count - 2 self._code = code self._probe = Probe(consolog) def value (self, val): if val[0] == 'imm': return val[1] if val[0] == 'reg': return self._reg[val[1]] if val[0] == 'mem': return self._ram[val[1]] if val[0] == 'ref': v = self.value(val[1]) if len(val) == 3: v += self.value(val[2]) return self._ram[v] def write (self, addr, val): if addr[0] == 'imm': raise WriteError if addr[0] == 'reg': self._reg[addr[1]] = val if addr[0] == 'mem': self._ram[addr[1]] = val if addr[0] == 'ref': a = self.value(addr[1]) if len(addr) == 3: a += self.value(addr[2]) self._ram[a] = val def cycle (self, ip): instr = self._code[ip][0] opcode = instr[0] # opcodes with no arguments if opcode == 'nop': pass elif opcode == 'ret': return self.value(('reg', self.__ra)) else: # opcode with 1 argument dst = instr[1] if opcode == 'cal': self.write(('reg', self.__ra), ip + 1) return self.value(dst) if opcode == 'jmp': return self.value(dst) if opcode == 'dbg': self._probe.dbg(dst) elif opcode == 'prn': print(self.value(dst), end='') elif opcode == 'prx': print(format(self.value(dst), '02x'), end='') elif opcode == 'prX': print(format(self.value(dst), '04x'), end='') elif opcode == 'prc': print(chr(self.value(dst)), end='') else: # opcodes with 2 arguments val1 = instr[2] if opcode == 'prs': ptr = self.value(dst) for i in range(0, self.value(val1)): print(chr(self._ram[ptr+i]), end='') print('') elif opcode == 'mov': self.write(dst, self.value(val1)) elif opcode == 'not': self.write(dst, 0xffff ^ self.value(val1)) else: # opcodes with 3 arguments val2 = instr[3] if opcode == 'beq': if self.value(val1) == self.value(val2): return self.value(dst) elif opcode == 'bne': if self.value(val1) != self.value(val2): return self.value(dst) elif opcode == 'and': self.write(dst, self.value(val1) & self.value(val2)) elif opcode == 'orr': self.write(dst, self.value(val1) | self.value(val2)) elif opcode == 'xor': self.write(dst, self.value(val1) ^ self.value(val2)) elif opcode == 'lsl': self.write(dst, self.value(val1) << self.value(val2)) elif opcode == 'lsr': self.write(dst, self.value(val1) >> self.value(val2)) elif opcode == 'min': self.write(dst, min(self.value(val1), self.value(val2))) elif opcode == 'max': self.write(dst, max(self.value(val1), self.value(val2))) elif opcode == 'add': self.write(dst, self.value(val1) + self.value(val2)) elif opcode == 'sub': self.write(dst, self.value(val1) - self.value(val2)) elif opcode == 'mul': self.write(dst, self.value(val1) * self.value(val2)) elif opcode == 'div': self.write(dst, self.value(val1) // self.value(val2)) elif opcode == 'mod': self.write(dst, self.value(val1) % self.value(val2)) elif opcode == 'cmp': v1 = self.value(val1) v2 = self.value(val2) if v1 < v2: self.write(dst, 1) elif v1 > v2: self.write(dst, -1) else: self.write(dst, 0) return None def run (self): try: max_ip = len(self._code) self._reg[self.__sp] = self._ram._size # init stack pointer self._reg[self.__ra] = max_ip # init return address self._ip = 0 # init instruction pointer while self._ip >= 0 and self._ip < max_ip: ip = self.cycle(self._ip) if ip is not None: self._ip = ip else: self._ip += 1 self._probe.read(self._ram.get_activity()) self._probe.read(self._reg.get_activity()) self._probe.output_activity() sys.stdout.flush() except AddrError as e: print('Invalid address ' + str(e.addr) + self.dbg(self._ip)) except ValError as e: print('Invalid value ' + str(e.val) + self.dbg(self._ip)) except WriteError: print('Invalid write ' + self.dbg(self._ip)) def dbg (self, ip): return (' on line ' + str(self._code[ip][1][1]) + ' of file ' + self._code[ip][1][0])
socketio = SocketIO(app) config = settings.Config() MAT_SERIAL_IDENTIFIER = "M" LIGHT_SERIAL_IDENTIFIER = "L" ZERO_CROSS_IDENTIFIER = "Z" ON = 1 OFF = 0 MAT_TEMPERATURE_APPROACH_DELTA_LIMIT = 0.12 AMBIENT_TEMPERATURE_APPROACH_DELTA_LIMIT = 0.2 display = Display() probe = Probe(config.get('probe', 'READ_DIRECTORY'), 'Probe') dht1_temp = DHT22(gpio.DHT_SENSOR1_PIN, 1, 'Sensor1 (temperature)') dht2_humidity = DHT22(gpio.DHT_SENSOR2_PIN, 2, 'Sensor2 (humidity)') mat = DutyCycle("Mat", MAT_SERIAL_IDENTIFIER) light = DutyCycle("Light", LIGHT_SERIAL_IDENTIFIER) serialConnection = serial.Serial('/dev/serial0', 9600) poll_sensors = True probe_temp = 0.0 ambient_temp = 0.0 sensor_values = [{'temp': 0.0, 'hum': 0.0}, {'temp': 0.0, 'hum': 0.0}] parser = argparse.ArgumentParser() parser.add_argument('--gpio_disabled', action="store_true")
def mainProgram(): createLogger() logger.info("Starting wine fermentation data collection...") secrets = loadJSONConfig('/'.join([dirName, "secrets.json"])) settings = loadJSONConfig('/'.join([dirName, "settings.json"])) losantHelper.init(secrets["deviceId"], secrets["key"], secrets["secret"]) s1 = Probe('/sys/bus/w1/devices/28-000004d109a8/w1_slave') s2 = Probe('/sys/bus/w1/devices/28-0416841e1fff/w1_slave') s3 = Probe('/sys/bus/w1/devices/28-031683b233ff/w1_slave') s4 = Probe('/sys/bus/w1/devices/28-0316839543ff/w1_slave') s5 = Probe('/sys/bus/w1/devices/28-000004d0bcce/w1_slave') s6 = Probe('/sys/bus/w1/devices/28-04168411d7ff/w1_slave') logger.info("Initialization done."); while True: # get a reading from the plant state = {"S1": s1.read_temp(), "S2": s2.read_temp(), "S3": s3.read_temp(), "S4": s4.read_temp(), "S5": s5.read_temp(), "S6": s6.read_temp()} logger.debug(state) losantHelper.sendMeasurement(state) # delay between readings time.sleep(5) #close endMeasurements()
def setUp(self): self.probe = Probe() self.probe.url = 'http://www.google.co.kr'
def __init__(self, shn, plunge=1): digitizer = DigitizerRCP(shn, plunge) Probe.__init__(self, digitizer)
def matchData(): target = open(matchedData, "w") recentID = None candidate = [] # Iterate Probe data to match it against links for cnt, line in enumerate(open(probeData).readlines()): probe = Probe(line) # Just for status if cnt % 500 == 0: print(cnt) # Small - current smallest distance # linkID - holds the smallest distance # dist_point_refnode - the distance from the point to ref node of the link latlong = GeoPoint(probe.latitude + "/" + probe.longitude) # Check if probe sampleID is already matched or not if probe.sampleID != recentID: recentID = probe.sampleID # Iterate through each link to match with current probe for key in links.keys(): for link in links[key]: # Calculate distance of probe lat-long from link distance = link.calculateDistance(latlong) if not probe.distFromRef or distance < probe.distFromRef: probe.distFromRef, probe.linkID = distance, link.id # Calculating distance of current probe from Link probe.distFromLink = links[ probe.linkID][0].calculateDistanceFromLink(latlong) # Determine direction probe.getDirection(float(probe.heading), link.radian) # saving candidate with reference point and its respective candidate = [link.ref_point, link.non_ref_point] else: for candidate_point in candidate: for link in points[candidate_point.ID]: distance = link.calculateDistance(latlong) if not probe.distFromRef or distance < probe.distFromRef: probe.distFromRef, probe.linkID = distance, link.id # Calculating distance of current probe from Link probe.distFromLink = links[ probe.linkID][0].calculateDistanceFromLink(latlong) # Determine direction probe.getDirection(float(probe.heading), link.radian) # distance from the reference node to the map-matched probe point location on the link in decimal meters. probe.distFromRef = math.sqrt(probe.distFromRef) * AVG_EARTH_RADIUS_MLT # perpendicular distance from the map-matched probe point location on the link to the probe point in decimal meters. probe.distFromLink = probe.distFromLink * AVG_EARTH_RADIUS_MLT # Writing comma seperated string into file target.write(probe.toString()) target.close()
def predict(model_prefix, probes_dir, preds_dir, data_dir, data_file, layers, batch_size, hidden_dim, max_seq_length, device): # Extract examples tokenizer = AutoTokenizer.from_pretrained(model_prefix) processor = SquadV2Processor() dev_examples = processor.get_dev_examples(data_dir=data_dir, filename=data_file) # Extract dev features print("Loading dev features") dev_features, dev_dataset = squad_convert_examples_to_features( examples=dev_examples, tokenizer=tokenizer, max_seq_length=max_seq_length, doc_stride=128, max_query_length=64, is_training=False, return_dataset="pt", threads=1) # Initialize config and model config = AutoConfig.from_pretrained(model_prefix, output_hidden_states=True) model = AutoModelForQuestionAnswering.from_pretrained(model_prefix, config=config) # multi-gpu evaluate model = torch.nn.DataParallel(model) # Load probe for each layer print("Loading probes") probes = [] for i in range(layers): p = Probe(hidden_dim) p.load(probes_dir, i + 1, device) probes.append(p) # Extract IDs print("Extracting dev IDs") n = len(dev_examples) q_ids = [] for i in range(n): q_ids.append(dev_examples[i].qas_id) # Initialize dev data loader eval_sampler = SequentialSampler(dev_dataset) eval_dataloader = DataLoader(dev_dataset, sampler=eval_sampler, batch_size=batch_size) # Initialize predictions predictions = [] for i in range(layers): pred = pd.DataFrame() pred['Id'] = q_ids pred['Predicted'] = [""] * len(dev_examples) pred['Question'] = [""] * len(dev_examples) pred['Score'] = [0] * len(dev_examples) predictions.append(pred) # List to keep track of how many unique questions we've seen in each df, questions with # contexts longer than max seq len get split into multiple features based on doc_stride # a good alternative we may implement later is recording for all features, then simplifying with groupby and max # e.g. something like df.sort_values('Score', ascending=False).drop_duplicates(['Question']) question_ids = [0] * layers # Evaluation batches print("Predicting on dev set") for batch in tqdm(eval_dataloader, desc="Evaluating"): model.eval() batch = tuple(t.to(device) for t in batch) with torch.no_grad(): inputs = { "input_ids": batch[0], "attention_mask": batch[1], "token_type_ids": batch[2], } # Distil does not use token type ids if "distil" in model_dir: inputs.pop('token_type_ids') # ALBERT/BERT/Distilibert forward pass idx = batch[3] outputs = model(**inputs) attention_hidden_states = outputs[2][1:] # Compute prediction on eval indices for j, index in enumerate(idx): index = int(index.item()) # Extract tokens for the current batch tokens = tokenizer.convert_ids_to_tokens(batch[0][j]) # Find where context starts and ends, since we want to predict in context context_start = int(max_seq_length - torch.argmax( torch.flip(batch[2][j], [0])).item()) - 1 context_end = int(torch.argmax(batch[2][j]).item()) # Find the question, starting right after [CLS] and subtracting 1 to chop off the [SEP] token question_start = 1 question_end = context_start question = tokenizer.convert_tokens_to_string( tokens[question_start:question_end - 1]) # For each layer ... for i, p in enumerate(probes): # Extract predicted indicies score, start_idx, end_idx = p.predict( attention_hidden_states[i][j].unsqueeze(0), device, threshold=0, context_start=context_start, context_end=context_end) start_idx = int(start_idx[0]) end_idx = int(end_idx[0]) # Extract predicted answer, converting start tokens to empty strings (no answer) answer = tokenizer.convert_tokens_to_string( tokens[start_idx:end_idx + 1]) if answer == '[CLS]': answer = '' # Check if the question is the same as the last one, if it is go back to the last question id and keep the higher score. # If the question is not already in the dataframe, then assign it to the dataframe. # Note we first handle the case where there are no prior questions by storing since we know there are no duplicates if question_ids[i] == 0: predictions[i].loc[question_ids[i], 'Question'] = question predictions[i].loc[question_ids[i], 'Predicted'] = answer predictions[i].loc[question_ids[i], 'Score'] = score elif (predictions[i].loc[int(question_ids[i] - 1), 'Question'] == question): question_ids[i] -= 1 old_score = predictions[i].loc[question_ids[i], 'Score'] if score > old_score: predictions[i].loc[question_ids[i], 'Predicted'] = answer predictions[i].loc[question_ids[i], 'Score'] = score else: predictions[i].loc[question_ids[i], 'Question'] = question predictions[i].loc[question_ids[i], 'Predicted'] = answer predictions[i].loc[question_ids[i], 'Score'] = score # Increment to new question id (note, for duplicate answers this gets us back to where we were) question_ids[i] += 1 # Save predictions for each layer print("Saving predictions") if not os.path.exists(preds_dir): os.mkdir(preds_dir) for i, pred in enumerate(predictions): pred[['Id', 'Predicted']].to_csv(preds_dir + "/layer_" + str(i + 1) + ".csv", index=False)
def __init__(self, data_limits_for_women): self.data_limits_for_women = data_limits_for_women self.dates = [] self.probes = { "RBC": Probe("RBC", "[mln/µl]", [4.5, 5.9], [4.2, 5.4], u"Ilość czerwonych krwinek (erytrocytów)"), "HGB": Probe("HGB", "[g/dl]", [13.5, 18.5], [12, 16], u"Ilość hemoglobiny w ogólnej masie erytrocytów"), "HCT": Probe("HCT", "[%]", [40, 54], [40, 51], u"Hematokryt - procentowa objętość erytrocytów we krwi"), "MCV": Probe("MCV", "[fl]", [80, 98], [76, 102.9], u"Średnia objętość erytrocytu"), "MCH": Probe("MCH", "[pg]", [27, 34], [25.7, 35.7], u"Stężenie hemoglobiny w erytrocycie"), "MCHC": Probe("MCHC", "[g/dl]", [31, 37], [29.5, 38.8], u"Stężenie hemoglobiny w ogólnej masie erytrocytów"), "RDW": Probe("RDW", "[%]", [11.4, 14.5], [10.3, 15.9], u"Anizocytoza erytrocytów - rozpiętość rozkładu objętości erytrocytów"), "PLT": Probe("PLT", "[tys/µl]", [150, 450], [135, 495], u"Ilość płytek krwi (trombocytów)"), "PDW": Probe("PDW", "[%]", [11, 18], [11, 18], u"Anizocytoza trombocytów - rozpiętość rozkładu objętości trombocytów"), "MPV": Probe("MPV", "[fl]", [6, 11], [6, 11], u"Średnia objętość trombocytu"), "WBC": Probe("WBC", "[tys/µl]", [4, 10], [3.8, 10.5], u"Ilość białych krwinek (leukocytów)"), "PCT": Probe("PCT", "[%]", [0.2, 0.5], [0.2, 0.5], u"Poziom prokalcytoniny (białka tarczycy) w osoczu krwi"), "LYM%": Probe("LYM%", "[%]", [20, 40], [18, 44], u"Procentowy udział limfocytów w leukocytach"), "MON%": Probe("MON%", "[%]", [4, 8], [2.8, 10.4], u"Procentowy udział monocytów w leukocytach"), "NEU%": Probe("NEU%", "[%]", [55, 70], [49.5, 77], u"Procentowy udział neutrofili w leukocytach"), "BAS%": Probe("BAS%", "[%]", [0, 2], [0.0, 3.0], u"Procentowy udział bazofili w leukocytach"), "EOS%": Probe("EOS%", "[%]", [1, 5], [0.7, 6.5], u"Procentowy udział eozynofili w leukocytach"), "LYM": Probe("LYM", "[tys/µl]", [1, 4], [0.9, 4.4], u"Ilościowy udział limfocytów w leukocytach"), "MON": Probe("MON", "[tys/µl]", [0.2, 1], [0.1, 1.3], u"Ilościowy udział monocytów w leukocytach"), "NEU": Probe("NEU", "[tys/µl]", [2, 7.5], [1.8, 8.25], u"Ilościowy udział neutrofili w leukocytach"), "BAS": Probe("BAS", "[tys/µl]", [0, 0.2], [0, 0.3], u"Ilościowy udział bazofili w leukocytach"), "EOS": Probe("EOS", "[tys/µl]", [0, 0.5], [0, 0.65], u"Ilościowy udział eozynofili w leukocytach"), "ALY%": Probe("ALY%", "[%]", [0, 2.0], [0, 2.0], u"Procent limfocytów atypowych (reaktywnych)"), "ALY": Probe("ALY", "[tys/µl]", [0, 0.25], [0, 0.25], u"Ilość limfocytów atypowych (reaktywnych)"), "LIC": Probe("LIC", "[tys/µl]", [0, 0.3], [0, 0.3], u"Ilość dużych niedojrzałych komórek (limfoblastów)"), }
def probing(self, qs=2): pb = Probe(self.root, self.name) for frame_id, opt in pb.diff_next(qs=qs): self.elog.info("Qs: {}, FrameID: {}, time: {}". format(qs, frame_id, opt)) self.save("dn/size_{}".format(qs), pb.pdf)
def add_file(self, file_path, cuts): metadata = Probe(file_path).run() summary = metadata.extract_summary() timeline_rate = { "ntsc": str(is_ntsc(summary['video']['frame_rate'])).upper(), "timebase": str(summary['video']['frame_rate']) } masterclip_id = "masterclip-" + str(self._masterclip) clip = otio.schema.ExternalReference( target_url=file_path, available_range=otio.opentime.TimeRange( start_time=otio.opentime.RationalTime( value=0, rate=float(summary['video']['frame_rate'])), duration=otio.opentime.RationalTime( value=int(metadata.extract_video_data('nb_frames')), rate=float(summary['video']['frame_rate']))), metadata={ "fcp_xml": { "masterclipid": masterclip_id, "media": { "audio": { "channelcount": str(summary['audio']['channels']), "samplecharacteristics": { "depth": "16", "samplerate": str(summary['audio']['sample_rate']) } }, "video": { "samplecharacteristics": { "anamorphic": "FALSE", "fielddominance": "none", "width": str(summary['video']['width']), "height": str(summary['video']['height']), "pixelaspectratio": "square", "rate": timeline_rate } } }, "rate": timeline_rate, "timecode": { "displayformat": "DF" if is_ntsc(summary['video']['frame_rate']) else "NDF", "rate": timeline_rate } } }) for cut in cuts: self.video_track_1.append( otio.schema.Clip( name=Path(file_path).stem, media_reference=clip, source_range=otio.opentime.TimeRange( start_time=otio.opentime.RationalTime( value=cut['start'], rate=float(summary['video']['frame_rate'])), duration=otio.opentime.RationalTime( value=cut['end'] - cut['start'], rate=float(summary['video']['frame_rate']))), metadata={ "fcp_xml": { "masterclipid": masterclip_id, "pixelaspectratio": "square", } })) self.audio_track_1.append( otio.schema.Clip( name=Path(file_path).stem, media_reference=clip, source_range=otio.opentime.TimeRange( start_time=otio.opentime.RationalTime( value=cut['start'], rate=float(summary['video']['frame_rate'])), duration=otio.opentime.RationalTime( value=cut['end'] - cut['start'], rate=float(summary['video']['frame_rate']))), metadata={ "fcp_xml": { "@premiereChannelType": summary['audio']['channel_layout'].capitalize(), "masterclipid": masterclip_id, "sourcetrack": { "mediatype": "audio", "trackindex": "1" } } })) self._masterclip += 1 return self
aux_bc_pml(pml,pml_type,xi,xf,yi,yf,nx,ny) #from matplotlib import pylab #for i in range(8): # pylab.figure() # pylab.contourf(pml[i,:,:].copy()) # pylab.colorbar() #pylab.show() from probe import Probe entries=[[ 0, 0], [34,10], [34,34], [10,34], [0,15], ] prb = Probe(da, entries) def write(Q1,Q2,Q3,filename): io = PETSc.Viewer().createBinary(filename,mode="w") Q1.view(io) Q2.view(io) Q3.view(io) io.destroy() draw = PETSc.Viewer.DRAW() for t in range(1,100): if t == 1: qinit(Q1,Q2,Q3,da) qbc(Q1,Q2,Q3,da)
heading, linkPVID, direction, distFromRef, distFromLink, slope''' header = header.replace(" ", "") header = header.replace("\n", "") header = header.replace("\t", "") result_data.write(header + "\n") prev_probe = None print "Calculating the slope at each map matched point" print "This will take some time.." for line in probe_data: probe = Probe(line) if not prev_probe: probe.slope = 'X' elif probe.linkPVID != prev_probe.linkPVID: probe.slope = 'X' else: #################################################################### # LOGIC TO CALCULATE SLOPE OF LINK # #################################################################### opposite = float(probe.altitude) - float(prev_probe.altitude) start = map(float, [probe.longitude, probe.latitude]) end = map(float, [prev_probe.longitude, prev_probe.latitude]) #import pdb;pdb.set_trace() hypotenuse = haversine.haversine(start[0], start[1], end[0], end[1]) / 1000 probe.slope = math.atan(opposite / hypotenuse)
def get_sub_url(self, base_url, url): probe = Probe() probe.open(url) return probe.get_internal_url(base_url)
def probing(self, qs=2): pb = Probe(self.root, self.name) for frame_id, opt in pb.diff_next(qs=qs): self.elog.info("Qs: {}, FrameID: {}, time: {}".format( qs, frame_id, opt)) self.save("dn/size_{}".format(qs), pb.pdf)
def train(model_prefix, model_dir, data_dir, data_file, epochs, layers, batch_size, hidden_dim, max_seq_length, device): # Extract examples tokenizer = AutoTokenizer.from_pretrained(model_prefix) processor = SquadV2Processor() train_examples = processor.get_train_examples(data_dir=data_dir, filename=data_file) # Extract train features print("Loading train features") train_features, train_dataset = squad_convert_examples_to_features( examples=train_examples, tokenizer=tokenizer, max_seq_length=max_seq_length, doc_stride=128, max_query_length=64, is_training=True, return_dataset="pt", threads=1, ) # Initialize model config = AutoConfig.from_pretrained(model_prefix, output_hidden_states=True) model = AutoModelForQuestionAnswering.from_pretrained(model_prefix, config=config) # multi-gpu evaluate model = torch.nn.DataParallel(model) # Initialize probes print("Initializing probes") probes = [] for i in range(layers): p = Probe(hidden_dim) probes.append(p) # Training epochs for epoch in range(epochs): print("Training epoch: {}".format(epoch + 1)) # Initialize train data loader train_sampler = RandomSampler(train_dataset) train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=batch_size) # Training batches for batch in tqdm(train_dataloader, desc="Iteration"): # Get batch on the right device and prepare input dict batch = tuple(t.to(device) for t in batch) inputs = { "input_ids": batch[0], "attention_mask": batch[1], "token_type_ids": batch[2], "start_positions": batch[3], "end_positions": batch[4], } # Distil does not use token type ids if "distil" in model_dir: inputs.pop('token_type_ids') # ALBERT/BERT/Distilibert forward pass model.eval() with torch.no_grad(): outputs = model(**inputs) # Extract hiddent states all_layer_hidden_states = outputs[3][ 1:] # (layers, batch_size, max_seq_len, hidden_size) # Get labels, and update probes for batch start = batch[3] # (batch_size) end = batch[4] # (batch_size) for i, p in enumerate(probes): hiddens = all_layer_hidden_states[ i] # (batch_size, max_seq_len, hidden_size) p.train(hiddens, start, end, device) # Save probes after each epoch print("Epoch complete, saving probes") epoch_dir = model_dir + "/epoch_" + str(epoch + 1) if not os.path.exists(epoch_dir): os.mkdir(epoch_dir) probes_dir = epoch_dir + "/probes" if not os.path.exists(probes_dir): os.mkdir(probes_dir) # Save probes for each layer, both start and end index for i, p in enumerate(probes): p.save(probes_dir, i + 1)