def test(root, outputdir, invis='fusion20', inir='fusion', predict='vis', model=None): if torch.cuda.is_available(): torch.set_default_tensor_type('torch.cuda.FloatTensor') net = torch.load(model)[0] net.cuda() dirvis = root.replace('fusion', invis) + '/VIS' dirir = root.replace('fusion', inir) + '/IR' from progressbar import progressbar as pb for vis, ir in pb( zip(sorted(os.listdir(dirvis)), sorted(os.listdir(dirir)))): f = vis vis = f'{dirvis}/{vis}' ir = f'{dirir}/{ir}' imgvis = data.read(vis) imgir = data.read(ir) input = torch.cat([imgvis, imgir], dim=0) if predict != 'vis': input = input.roll(shifts=1, dims=0) input = input[:net.inchannels, ...] input = input.unsqueeze(0).to(device) with torch.no_grad(): out = net(input) write_tensor(f'{outputdir}/{f}.tif', out * 255)
def getAdministratorInfo(): new = False #boolean to allow for using new/load old data socketChat = False #boolean to allow for socket convo adminIn = input( "Would you like to train the agent on new data/create a new agent? (y/n) " ) #adminIn = 'y' if adminIn == 'y': new = True if new: data.read() config.createModel() model.create(config.training, config.output) print("Model created") else: data.load() config.createModel() model.load() print("Model loaded") socket = input( "Would you like to watch a conversation with another chatbot? (y/n) ") if socket == 'y': socketChat = True if socketChat: config.socketchat = True else: config.socketchat = False
def CheckPayment(details, email): details = ast.literal_eval(details) query = 'select Route from Reservation where Id = "{}"'.format(details[0]) route = data.read(query) route = route[0][0] query = 'select Cost from Routes where Name = "{}"'.format(route) price = data.read(query) price = price[0][0] query = 'select Coupons from Users where Email = "{}"'.format(email) coupons = data.read(query) coupons = coupons[0][0] if coupons > price: balance = coupons - price query = 'update Reservation set Seat = "{}" where Id = "{}"'.format( details[1], details[0]) info = data.data_write(query) query = 'update Reservation set Payment = "{}" where Id = "{}"'.format( price, details[0]) info = data.data_write(query) query = 'update Users set Coupons = "{}" where Email = "{}"'.format( balance, email) info = data.data_write(query) return "Payment successful" else: return "cant afford"
def read_data(args): mirna = data.read(args.mirna) mrna = data.read(args.mrna) cnv = data.read(args.cnv) methylation = data.read(args.methylation) mrna, mirna, cnv, methylation = data.harmonize(mrna, mirna, cnv, methylation) print(mrna.shape, mirna.shape, cnv.shape, methylation.shape) return mrna, mirna, cnv, methylation
def error_analysis(): """ Print information about linguistic variables that help for certain counties. """ data = twdata.read(path + 'counties.norm=user.liwc.perma.pkl') targets, target_alpha = twexpt.read_targets('/data/twcounty/targets.tsv', data.features.keys()) states = twexpt.read_states('/data/twcounty/states.tsv', data.features.keys()) state_cv = twexpt.make_state_cv(data.features, states, 5) county_indices = [item for f in state_cv for item in f[1]] county_ids = np.array(sorted(data.features))[county_indices] control_indices = [target_alpha.index(c) for c in control_labels] for lab in labels: # ['nosocialemotionalsupport', 'limitedaccesstohealthyfoods', 'ambulatorycaresensitiveconditions', 'unemployed']: emot_both = pickle.load(open(path + 'ridge_control_and_feats/counties.norm=user.liwc.perma.pkl.out/' + lab + '/test_folds.pkl', 'rb')) emot_ctl = pickle.load(open(path + 'ridge_control_only/counties.norm=none.unigrams.pkl.out/' + lab + '/test_folds.pkl', 'rb')) y = np.array([item for t in emot_ctl for item in t[0]]) ctl_pred = np.array([item for t in emot_ctl for item in t[1]]) both_pred = np.array([item for t in emot_both for item in t[1]]) maxi = np.argmax((np.abs(ctl_pred - y) - np.abs(both_pred - y)) * (y != 0)) #maxi = np.argmax(np.abs(ctl_pred - y) / y - np.abs(both_pred - y) / y) county_id = county_ids[maxi] print '\n', lab, '\nmost improved county is', county_id, '=', y[maxi], 'ctl=', ctl_pred[maxi], 'both=', both_pred[maxi] print 'this county controls=', np.array(targets[county_id])[control_indices] most_sim = [county_ids[np.argsort(np.abs(ctl_pred - ctl_pred[maxi]) - np.abs(y - y[maxi]) * (y != 0))[1]]] #most_sim = find_similar(county_id, targets, [target_alpha.index(i) for i in control_labels]) most_sim_idx = np.where(county_ids == most_sim[0]) print 'most sim preds=', most_sim[0], '=', y[most_sim_idx], 'ctl=', ctl_pred[most_sim_idx], 'both=', both_pred[most_sim_idx] print 'most sim controls=', np.array(targets[most_sim[0]])[control_indices] words = read_tsv(path + 'ridge_control_and_feats/counties.norm=user.liwc.perma.pkl.out/' + lab + '/words.txt', ['var', 'coef', 'f', 'alpha']) print 'top vector diffs=', vec_diff(data, county_id, most_sim[0], words)[:10] plt.plot(y, ctl_pred, 'yo') plt.plot(y, both_pred, 'bo') plt.title(lab) plt.xlabel('truth') plt.ylabel('predicted - truth')
def showDiag(self): home_dir = str(Path.cwd())+'/data' fname = QFileDialog.getOpenFileName(self, 'Open file', home_dir,'Text Files ( *.txt)') if fname[0]: self.input.setText(fname[0]) print(fname[0]) if '/data/ta20_5_1.txt' in fname[0]: nbLignes = 5 elif '/data/ta20_20_1.txt' in fname[0]: nbLignes = 20 elif '/data/ta20_20_2.txt' in fname[0]: nbLignes = 20 elif '/data/ta100_10_1.txt' in fname[0]: nbLignes = 10 else: nbLignes = 10 #Load the file and check if valid matrice = dataReader.read(fname[0], nbLignes) matrice = np.array(matrice) machine_count, job_count = matrice.shape #Display infos about job count ... self.label.setText('{} machines et {} jobs'.format(machine_count, job_count)) #Set the data as class var self.data=matrice #enable the execution button self.exec.setEnabled(True)
def main(): ap = argparse.ArgumentParser() ap.add_argument('--controls', default=None, help='comma separated list of column names from --targets to use as control variables ...') ap.add_argument('--controls-only', action='store_true', default=False, help='restrict features to control variables') ap.add_argument('--coords', default='/data/twcounty/stats2/counties/counties.top100.bounding.txt', help='tsv file with counties, lat/long, and bounding boxes') ap.add_argument('--input', default='/data/twcounty/counties.pkl', help='pickled county feature Data object') ap.add_argument('--model', default='ridge', help='name of function to build model [linreg, ridge, lasso]') ap.add_argument('--states', default='/data/twcounty/states.tsv', help='file like <state> <county_id>') ap.add_argument('--targets', default='/data/twcounty/targets.tsv', help='targets per county') args = ap.parse_args() coords = read_coords(args.coords) logger.info('args=%s' % str(args)) logger.info('reading data from %s' % args.input) county_data = data.read(args.input) counties = county_data.features data_alpha = sorted(county_data.alpha.keys(), key=lambda k: county_data.alpha[k]) logger.info('reading targets from %s' % args.targets) targets, target_alpha = read_targets(args.targets, counties.keys()) logger.info('reading states from %s' % args.states) states = read_states(args.states, counties.keys()) logger.info('targets for %s are %s' % (targets.keys()[0], str(targets[targets.keys()[0]]))) logger.info('running experiment on %d counties' % len(counties)) if args.controls: add_controls(args.controls, counties, data_alpha, targets, target_alpha, args.controls_only) model_f = getattr(sys.modules[__name__], args.model) run_expt(model_f, counties, states, targets, data_alpha, target_alpha, args, coords)
def main(): try: dataset = request.form["dataset"] if(dataset == "static/diabetes.csv"): target = "diabetes" feature1 = request.form["feature1"] feature2 = request.form["feature2"] classifier = request.form["classifier"] except KeyError: error = "Warning! Missing selections. Please select one dataset, two features from the dataset, and one classifier!" return render_template('select.html', error=error) df = read(dataset) X_train, X_test, y_train, y_test = split_proportional(df, target) y_train = pd.get_dummies(y_train) y_test = pd.get_dummies(y_test) clf = fit(X_train, y_train, classifier, feature1, feature2) data_train = select_features(X_train, (feature1, feature2)) data_test = select_features(X_test, (feature1, feature2)) accuracy_train = np.mean(cross_val_score(clf, data_train, y_train, cv=5)) accuracy_test = np.mean(cross_val_score(clf, data_test, y_test, cv=5)) plot_data = build_plot(data_test, y_test, clf) return render_template('plot.html', accuracy_train=accuracy_train, accuracy_test=accuracy_test, plot_url=plot_data)
def control(address, data_queue, connecting, exit): run = True #connect to device device = connect(address, exit, connecting) #infinitely loop to read from raspberry pi while (run): try: #wait for notifications from node then read data and send to queue device.waitForNotifications(6000) data = read(device) data_queue.put(data) #insert data into local sql database sql_data = [data] insert_db(sql_data) #if keyboard interrupt is pressed exit program except KeyboardInterrupt: exit.put(1) run = False quit() #log errors except Exception as err: method = 'control()' log_error(method,(address +' '+ str(err))) print 'error at control():', address, ' ',err #if disconnection occurs reconnect to peripheral if str(err) == "Device disconnected": device = connect(address, exit, connecting) else: run = False
def inputs(tfrecords_file): ''' create inputs ''' print tfrecords_file filename_queue = tf.train.string_input_producer( [tfrecords_file]) # ここで指定したepoch数はtrainableになるので注意 read_input = data.read(filename_queue) reshaped_image = tf.cast(read_input.image, tf.float32) height = CROP_SIZE width = CROP_SIZE resized_image = tf.image.resize_image_with_crop_or_pad( reshaped_image, width, height) float_image = tf.image.per_image_whitening(resized_image) min_fraction_of_examples_in_queue = 0.4 #min_fraction_of_examples_in_queue = 1 min_queue_examples = int(NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN * min_fraction_of_examples_in_queue) print( 'filling queue with %d train images before starting to train. This will take a few minutes.' % min_queue_examples) return _generate_image_and_label_batch(float_image, read_input.label, min_queue_examples)
def main(): """ Main interface """ parser = _create_parser() args = parser.parse_args() if len(sys.argv) == 1: parser.print_usage() parser.exit(status=1) elif args.countries: print("\n".join(map(str, data.countries()))) sys.exit() elif args.states: print("\n".join(map(str, data.states()))) sys.exit() region = data.read(args.region) if not region: print(f"Region '{args.region}' not found.") sys.exit(1) title = f"COVID-19 Infections ({region})" with tempfile.NamedTemporaryFile(mode="w") as datfile: for stats in region.daily_stats(): datfile.write(f"{stats.date}\t{stats.cases}\t{stats.avg}\n") datfile.flush() gnuplot = Gnuplot("./covid.gp", args.terminal, args.output) gnuplot.set_var("datfile", datfile.name) gnuplot.set_var("title", title) gnuplot.run()
def Check_email(email): try: query = 'select Email from Users where Email = "{}";'.format(email) returned = data.read(query) return returned[0][0] except Exception as e: return ""
def FreeSeats(reservation_id): a = ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11"] # a=[1,2,3,4,5,6,7,8,9,10,11] t = [] query = 'select Route, Date, Time from Reservation where Id ="{}"'.format( reservation_id) info = data.read(query) query = 'select Seat from Reservation where Route ="{}" and Date ="{}" and Time ="{}"'.format( info[0][0], info[0][1], info[0][2]) taken_seats = data.read(query) #taken_seats = ast.literal_eval(taken_seats) for x in taken_seats: t.append(x[0]) b = [x for x in a if x not in t] freeSeats = b return freeSeats
def distorted_inputs(tfrecords_file): ''' create inputs with real time augumentation. ''' print tfrecords_file filename_queue = tf.train.string_input_producer([tfrecords_file], shuffle=True) # ここで指定したepoch数はtrainableになるので注意 read_input = data.read(filename_queue) reshaped_image = tf.cast(read_input.image, tf.float32) height = CROP_SIZE width = CROP_SIZE # crop distorted_image = tf.image.random_crop(reshaped_image, [height, width]) # flip distorted_image = tf.image.random_flip_left_right(distorted_image) # you can add random brightness contrast distorted_image = tf.image.random_brightness(distorted_image, max_delta=63) distorted_image = tf.image.random_contrast(distorted_image, lower=0.2, upper=1.8) # whitening float_image = tf.image.per_image_whitening(distorted_image) min_fraction_of_examples_in_queue = 0.4 #min_fraction_of_examples_in_queue = 1 min_queue_examples = int(NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN * min_fraction_of_examples_in_queue) print ('filling queue with %d train images before starting to train. This will take a few minutes.' % min_queue_examples) return _generate_image_and_label_batch(float_image, read_input.label, min_queue_examples)
def Table(name): error = "" value = "" path = "/{}".format(name) if name == "Member Information": query = "select * from Member_Basic_info" if request.method == "POST": value = request.form['searched'] query = "select * from Member_Basic_info where Name like '%{}%'".format( value) everyone = data.read(query) columns = 6 titles = [ "Member Number", "National Id", "Member Name", "Member Station", "Phone number" ] if not everyone: error = "{} Does not exist".format(value.title()) return render_template("/table.html", columns=columns, rows=everyone, titles=titles, error=error, path=path) elif name == "Milk Collection": return redirect("/{}".format(name)) elif name == "Admin Panel": return render_template("/admin.html", tiles=tiles) else: return ("Nothing") return "asdfghj"
def plot_time_series_data_from_file(files, names, transform=None, ylabel=None, country=None, best_fit_degree=0, scatter=False, size=10, usa=False): plt.figure() if transform is None: transform = lambda ys: ys sp = smse = 0 for filename, name in zip(files, names): ys = transform(read(filename, country=country, usa=usa)) p, mse = plot_time_series_data(name, ys, scatter=scatter, size=size, best_fit_degree=best_fit_degree) if p is not None: sp += p smse += mse plt.xlabel("Days after January 22, 2020") c = (country if country != None else "Earth") if usa: c = "United States" if ylabel is None: plt.ylabel("Cases for " + c) else: plt.ylabel(ylabel + " for " + c) plt.legend() return sp, smse
def recebe(port): '''(str)->int, int''' #recepcao de dados dados = data.read(port, ".", 300) #conferencia de validade valid = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] if len(dados) != 7 or not (dados[0] in valid) or not ( dados[1] in valid) or not (dados[2] in valid) or dados[3] != "," or not ( dados[4] in valid) or not (dados[5] in valid) or not ( dados[6] in valid): print("invalid data received") if len(dados) >= 3 and dados[-1] in valid and dados[ -2] in valid and dados[-3] in valid: angulo = -1 distancia = int(leitura[-3] + leitura[-2] + leitura[-1]) // 10 return angulo, distancia return -2, 0 #criacao dos valores de angulo e distancia angulo = int(leitura[0] + leitura[1] + leitura[2]) distancia = 15 - int(leitura[4] + leitura[5] + leitura[6]) // 10 #conversao do angulo para radianos angulo = conversor.angle_degree(angulo)[0] #retorno return angulo, distancia
def distribution_test(df: pd.DataFrame = pd.DataFrame()): if df.empty: df = filter(read()) distribution.test_exp(df) distribution.test_gam(df) distribution.test_wei(df) distribution.test_logn(df) distribution.test_pare(df)
def error_analysis(): """ Print information about linguistic variables that help for certain counties. """ data = twdata.read(path + 'counties.norm=user.liwc.perma.pkl') targets, target_alpha = twexpt.read_targets('/data/twcounty/targets.tsv', data.features.keys()) states = twexpt.read_states('/data/twcounty/states.tsv', data.features.keys()) state_cv = twexpt.make_state_cv(data.features, states, 5) county_indices = [item for f in state_cv for item in f[1]] county_ids = np.array(sorted(data.features))[county_indices] control_indices = [target_alpha.index(c) for c in control_labels] for lab in labels: # ['nosocialemotionalsupport', 'limitedaccesstohealthyfoods', 'ambulatorycaresensitiveconditions', 'unemployed']: emot_both = pickle.load( open( path + 'ridge_control_and_feats/counties.norm=user.liwc.perma.pkl.out/' + lab + '/test_folds.pkl', 'rb')) emot_ctl = pickle.load( open( path + 'ridge_control_only/counties.norm=none.unigrams.pkl.out/' + lab + '/test_folds.pkl', 'rb')) y = np.array([item for t in emot_ctl for item in t[0]]) ctl_pred = np.array([item for t in emot_ctl for item in t[1]]) both_pred = np.array([item for t in emot_both for item in t[1]]) maxi = np.argmax( (np.abs(ctl_pred - y) - np.abs(both_pred - y)) * (y != 0)) #maxi = np.argmax(np.abs(ctl_pred - y) / y - np.abs(both_pred - y) / y) county_id = county_ids[maxi] print '\n', lab, '\nmost improved county is', county_id, '=', y[ maxi], 'ctl=', ctl_pred[maxi], 'both=', both_pred[maxi] print 'this county controls=', np.array( targets[county_id])[control_indices] most_sim = [ county_ids[np.argsort( np.abs(ctl_pred - ctl_pred[maxi]) - np.abs(y - y[maxi]) * (y != 0))[1]] ] #most_sim = find_similar(county_id, targets, [target_alpha.index(i) for i in control_labels]) most_sim_idx = np.where(county_ids == most_sim[0]) print 'most sim preds=', most_sim[0], '=', y[ most_sim_idx], 'ctl=', ctl_pred[most_sim_idx], 'both=', both_pred[ most_sim_idx] print 'most sim controls=', np.array( targets[most_sim[0]])[control_indices] words = read_tsv( path + 'ridge_control_and_feats/counties.norm=user.liwc.perma.pkl.out/' + lab + '/words.txt', ['var', 'coef', 'f', 'alpha']) print 'top vector diffs=', vec_diff(data, county_id, most_sim[0], words)[:10] plt.plot(y, ctl_pred, 'yo') plt.plot(y, both_pred, 'bo') plt.title(lab) plt.xlabel('truth') plt.ylabel('predicted - truth')
def main(): """ Reports the spatial autocorrelation for each target variable. """ ap = argparse.ArgumentParser() ap.add_argument( '--coords', default='/data/twcounty/stats2/counties/counties.top100.bounding.txt', help='tsv file with counties, lat/long, and bounding boxes') ap.add_argument( '--dbf', default= '/data/twcounty/stats2/counties/census_polygons/gz_2010_us_050_00_500k.dbf', help='county dbf files') ap.add_argument( '--input', default='/data/twcounty/features/counties.norm=none.liwc.pkl', help='pickled county feature Data object') ap.add_argument('--json', default='/data/twcounty/json', help='directory of jsons per county') ap.add_argument( '--shapes', default= '/data/twcounty/stats2/counties/census_polygons/gz_2010_us_050_00_500k.shp', help='county shape files') ap.add_argument('--targets', default='/data/twcounty/targets.tsv', help='targets per county') ap.add_argument('--tweets', default='/data/twcounty/tweets.tsv', help='tweets') args = ap.parse_args() coords = read_coords(args.coords) overlaps = get_overlaps(coords) print_tweets_in_overlaps(overlaps, args.json, args.tweets) shapes = pysal.open(args.shapes) print 'read', len(shapes), 'counties from shape file' dbf = pysal.open(args.dbf) counties = data.read(args.input).features targets, target_alpha = read_targets(args.targets, counties) for t in targets: didx, d = [(i, dd) for i, dd in enumerate(dbf) if dd[1] + dd[2] == t][0] targets[t]['dbf'] = d targets[t]['dbfi'] = didx shape = shapes[didx] targets[t]['shape'] = shape fips = sorted(targets) print '\t'.join(['mi.I ', 'mi.EI ', 'p_norm ', 'outcome']) weights = pysal.Kernel( np.array([targets[f]['shape'].centroid for f in fips])) for outcome in target_alpha: y = np.array([float(targets[t][outcome]) for t in fips]) y = y / np.sum(y) mi = pysal.Moran(y, weights) print '%.5f\t%.5f\t%.5f\t%s' % (mi.I, mi.EI, mi.p_norm, outcome)
def return_status(update, context): context.user_data['username'] = update.message.text markup = ReplyKeyboardMarkup(config.main_keyboard, one_time_keyboard=False) context.bot.send_message(chat_id=update.effective_chat.id, text=correct( data.read(context.user_data['username'])), reply_markup=markup) return ConversationHandler.END
def psnr(root, outputdir, predict='vis'): criterion = nn.MSELoss() d = 'VIS' if predict == 'ir': d = 'IR' gtdir = root + '/' + d + '/' filt = lambda l: sorted(filter(lambda x: not x.endswith('.psnr'), l)) from progressbar import progressbar as pb for gt, pred in pb( zip(filt(os.listdir(gtdir)), filt(os.listdir(outputdir)))): pred = f'{outputdir}/{pred}' groundtruth = data.read(f'{gtdir}/{gt}') predicted = data.read(pred) mse = criterion(groundtruth, predicted).item() psnr = mse_to_psnr(mse) open(pred + '.psnr', 'w').write(f'{psnr:.3f}')
def inputs(self): ''' create inputs mini batch ''' filename_queue = tf.train.string_input_producer(["data/train.tfrecords"], num_epochs=2) result = data.read(filename_queue) images, sparse_labels = tf.train.shuffle_batch( [result.image, result.label], batch_size=2, num_threads=2, capacity=3, # Ensures a minimum amount of shuffling of examples. min_after_dequeue=1) return images, sparse_labels
def load(job): try: with OracleConnection.from_properties(job.key_source) as source: with S3Bucket.from_properties(job.key_destination) as destination: df = read(source, job.read_query) full_path = f'{csv_dir}/{job.name}.csv' df.to_csv(full_path, index=False) destination.upload(full_file_path=full_path) except Exception as e: print(e) return job.name
def read_subscriptions(): try: json_s = data.read() data_s = json.loads(json_s) for f in data_s['feeds']: feed = feeds.get(f['name']) for channel in f['channels']: id = int(channel['id']) channel = bot.get_channel(id) if channel: feed.channels.append(channel) except KeyError: pass
def fig_scatter(): """ Print scatter plots for a few select outcomes. """ outcomes = ['uninsured', 'teenbirthrate', 'obese'] data = twdata.read(path + 'counties.norm=user.liwc.perma.pkl') targets, target_alpha = twexpt.read_targets('/data/twcounty/targets.tsv', data.features.keys()) states = twexpt.read_states('/data/twcounty/states.tsv', data.features.keys()) state_cv = twexpt.make_state_cv(data.features, states, 5) county_indices = [item for f in state_cv for item in f[1]] county_ids = np.array(sorted(data.features))[county_indices] for outcome in outcomes: folds = pickle.load( open( path + 'ridge/counties.norm=user.liwc.perma.pkl.out/' + outcome + '/test_folds.pkl', 'rb')) y = [item for t in folds for item in t[0]] pred = [item for t in folds for item in t[1]] if outcome == 'teenbirthrate': # this is in thousands, not hundredths y = np.array(y) / 10. pred = np.array(pred) / 10. fit = np.polyfit(y, pred, 1) print 'fit=', fit fit_fn = np.poly1d(fit) # plt.scatter(y, pred) plt.plot(y, pred, 'bo', y, fit_fn(y), 'k') plt.xlabel('truth (%)', size='16') plt.ylabel('predicted (%)', size='16') plt.title(label_map[outcome], size='20') worst = np.argmax(abs(np.array(y) - np.array(pred))) county_id_worst = county_ids[worst] best = np.argmin(abs(np.array(y) - np.array(pred))) county_id_best = county_ids[best] max_pred = np.argmax(pred) county_id_max = county_ids[max_pred] #i = np.argmax(abs((np.array(y) * fit[0] + fit[1]) - np.array(pred))) print 'worst: cty=', county_id_worst, 'tr=', y[worst], 'pr=', pred[ worst] print 'best: cty=', county_id_best, 'tr=', y[best], 'pr=', pred[best] print 'max: cty=', county_id_max, 'tr=', y[max_pred], 'pr=', pred[ max_pred] words = read_tsv( path + 'ridge/counties.norm=user.liwc.perma.pkl.out/' + outcome + '/words.txt', ['var', 'coef', 'f', 'alpha']) print 'top vector diffs=', vec_diff(data, county_id_worst, county_id_best, words)[:10] #plt.show() plt.savefig('paper/figs/scatter_' + outcome + '.pdf', bbox_inches='tight') plt.close()
def main(): ap = argparse.ArgumentParser() ap.add_argument( '--controls', default=None, help= 'comma separated list of column names from --targets to use as control variables ...' ) ap.add_argument('--controls-only', action='store_true', default=False, help='restrict features to control variables') ap.add_argument( '--coords', default='/data/twcounty/stats2/counties/counties.top100.bounding.txt', help='tsv file with counties, lat/long, and bounding boxes') ap.add_argument('--input', default='/data/twcounty/counties.pkl', help='pickled county feature Data object') ap.add_argument( '--model', default='ridge', help='name of function to build model [linreg, ridge, lasso]') ap.add_argument('--states', default='/data/twcounty/states.tsv', help='file like <state> <county_id>') ap.add_argument('--targets', default='/data/twcounty/targets.tsv', help='targets per county') args = ap.parse_args() coords = read_coords(args.coords) logger.info('args=%s' % str(args)) logger.info('reading data from %s' % args.input) county_data = data.read(args.input) counties = county_data.features data_alpha = sorted(county_data.alpha.keys(), key=lambda k: county_data.alpha[k]) logger.info('reading targets from %s' % args.targets) targets, target_alpha = read_targets(args.targets, counties.keys()) logger.info('reading states from %s' % args.states) states = read_states(args.states, counties.keys()) logger.info('targets for %s are %s' % (targets.keys()[0], str(targets[targets.keys()[0]]))) logger.info('running experiment on %d counties' % len(counties)) if args.controls: add_controls(args.controls, counties, data_alpha, targets, target_alpha, args.controls_only) model_f = getattr(sys.modules[__name__], args.model) run_expt(model_f, counties, states, targets, data_alpha, target_alpha, args, coords)
def read_data_and_targets(): """ Read variables and labeles from pickled data file and targets file. Return X, Y, data alphabet (minus control variables) and target_alphabet. """ county_data = twdata.read(path + 'counties.norm=user.liwc.perma.pkl') counties = county_data.features data_alpha = sorted(county_data.alpha.keys(), key=lambda k: county_data.alpha[k]) nonctl_alpha = copy.copy(data_alpha) control_names = '< 18,65 and over,Female,Afro-hispanic,med_income' targets, target_alpha = twexpt.read_targets('/data/twcounty/targets.tsv', counties.keys()) twexpt.add_controls(control_names, counties, data_alpha, targets, target_alpha, False) control_names = [unicode(c) for c in control_names.split(',')] X, Y = twexpt.to_matrices(counties, targets) return X, Y, nonctl_alpha, target_alpha
def inputs(self): ''' create inputs mini batch ''' filename_queue = tf.train.string_input_producer( ["data/train.tfrecords"], num_epochs=2) result = data.read(filename_queue) images, sparse_labels = tf.train.shuffle_batch( [result.image, result.label], batch_size=2, num_threads=2, capacity=3, # Ensures a minimum amount of shuffling of examples. min_after_dequeue=1) return images, sparse_labels
def predict(params): """ From a set of parameters, loads a network (model and weights), builds a prediction vector, which is returned together with the number of tendency errors found """ raw = data.read(params, params['pred_dataset']) normalized = data.normalize(raw, params) adjusted = parameters.adjust(normalized, params) # prepare test data _, _, X_test, Y_test = data.prepare(adjusted, params) # Perform the prediction. model1 = model.prediction_setup(params) print('Feeding X_test (shape=', X_test.shape, ')') (yhat, rmse, num_errors) = range_predict(model1, X_test, Y_test, params) return (params, model1, Y_test, yhat, rmse, num_errors)
def task(): taskApp = data.read() task = taskapp_pb2.Task() newTask = toBuffer(request.data, task, gettingJson(request)) matchingTasks = filterById(taskApp.tasks, newTask.id) if len(matchingTasks) > 0: data.update(matchingTasks[0], newTask) data.write(taskApp) else: newTask.time_created.GetCurrentTime() newTask.last_updated.GetCurrentTime() newTask.id = str(uuid.uuid4()) taskApp.tasks.extend([newTask]) data.write(taskApp) return fromBuffer(newTask, gettingJson(request))
def Authenticate(attempted_email, attempted_password): query = 'select * from Users where Email = "{}";'.format(attempted_email) info = data.read(query) try: password = info[0][3] user = { "Username": info[0][1], "Email": info[0][2], "Category": info[0][4], "State": info[0][5], } except: return "" Similer = sha256_crypt.verify(str(attempted_password), password) print(Similer) if Similer: return user else: return ""
def initialize(filename='params.yaml'): """ Reads a YAML file within the CWD of the current notebook to read all the params from there. """ home_path = str(Path.home()) project_path = 'Documents/SideProjects/sailboatsfactory' work_path = 'src/nn-core' params_path = join(home_path, join(project_path, work_path)) yaml_file = join(params_path, filename) print("Reading parameters from:", filename) with open(yaml_file, 'r') as f: my_params = load(f) my_params['x_scaler'] = MinMaxScaler(feature_range=(-1, 1)) my_params['y_scaler'] = MinMaxScaler(feature_range=(-1, 1)) raw = data.read(my_params) adjusted = adjust(raw, my_params) return adjusted, my_params
def run(**args): data = read(file_name) rows_number, columns_number, min_ingredient_number, max_total_ingredient_number, data = data data = np.array(data) shape = (1, max_total_ingredient_number) (x, y) = data.shape results = [] for row in range(x): for column in range(y): xmin = row xmax = xmin + 1 ymin = column ymax = ymin + max_total_ingredient_number - 1 if (is_slice_okay(xmin, xmax, ymin, ymax, data, min_ingredient_number, max_total_ingredient_number)): data = update_slice(xmin, xmax, ymin, ymax, data) results.append([xmin, ymin, xmax - 1, ymax - 1]) write(results)
def inputs(tfrecords_file): ''' create inputs ''' print tfrecords_file filename_queue = tf.train.string_input_producer([tfrecords_file]) # ここで指定したepoch数はtrainableになるので注意 read_input = data.read(filename_queue) reshaped_image = tf.cast(read_input.image, tf.float32) height = CROP_SIZE width = CROP_SIZE resized_image = tf.image.resize_image_with_crop_or_pad(reshaped_image, width, height) float_image = tf.image.per_image_whitening(resized_image) min_fraction_of_examples_in_queue = 0.4 #min_fraction_of_examples_in_queue = 1 min_queue_examples = int(NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN * min_fraction_of_examples_in_queue) print ('filling queue with %d train images before starting to train. This will take a few minutes.' % min_queue_examples) return _generate_image_and_label_batch(float_image, read_input.label, min_queue_examples)
def fig_scatter(): """ Print scatter plots for a few select outcomes. """ outcomes = ['uninsured', 'teenbirthrate', 'obese'] data = twdata.read(path + 'counties.norm=user.liwc.perma.pkl') targets, target_alpha = twexpt.read_targets('/data/twcounty/targets.tsv', data.features.keys()) states = twexpt.read_states('/data/twcounty/states.tsv', data.features.keys()) state_cv = twexpt.make_state_cv(data.features, states, 5) county_indices = [item for f in state_cv for item in f[1]] county_ids = np.array(sorted(data.features))[county_indices] for outcome in outcomes: folds = pickle.load(open(path + 'ridge/counties.norm=user.liwc.perma.pkl.out/' + outcome + '/test_folds.pkl', 'rb')) y = [item for t in folds for item in t[0]] pred = [item for t in folds for item in t[1]] if outcome == 'teenbirthrate': # this is in thousands, not hundredths y = np.array(y) / 10. pred = np.array(pred) / 10. fit = np.polyfit(y, pred, 1) print 'fit=', fit fit_fn = np.poly1d(fit) # plt.scatter(y, pred) plt.plot(y, pred, 'bo', y, fit_fn(y), 'k') plt.xlabel('truth (%)', size='16') plt.ylabel('predicted (%)', size='16') plt.title(label_map[outcome], size='20') worst = np.argmax(abs(np.array(y) - np.array(pred))) county_id_worst = county_ids[worst] best = np.argmin(abs(np.array(y) - np.array(pred))) county_id_best = county_ids[best] max_pred = np.argmax(pred) county_id_max = county_ids[max_pred] #i = np.argmax(abs((np.array(y) * fit[0] + fit[1]) - np.array(pred))) print 'worst: cty=', county_id_worst, 'tr=', y[worst], 'pr=', pred[worst] print 'best: cty=', county_id_best, 'tr=', y[best], 'pr=', pred[best] print 'max: cty=', county_id_max, 'tr=', y[max_pred], 'pr=', pred[max_pred] words = read_tsv(path + 'ridge/counties.norm=user.liwc.perma.pkl.out/' + outcome + '/words.txt', ['var', 'coef', 'f', 'alpha']) print 'top vector diffs=', vec_diff(data, county_id_worst, county_id_best, words)[:10] #plt.show() plt.savefig('paper/figs/scatter_' + outcome + '.pdf', bbox_inches='tight') plt.close()
def distorted_inputs(tfrecords_file): ''' create inputs with real time augumentation. ''' print tfrecords_file filename_queue = tf.train.string_input_producer( [tfrecords_file], shuffle=True) # ここで指定したepoch数はtrainableになるので注意 read_input = data.read(filename_queue) reshaped_image = tf.cast(read_input.image, tf.float32) height = CROP_SIZE width = CROP_SIZE # crop distorted_image = tf.image.random_crop(reshaped_image, [height, width]) # flip distorted_image = tf.image.random_flip_left_right(distorted_image) # you can add random brightness contrast distorted_image = tf.image.random_brightness(distorted_image, max_delta=63) distorted_image = tf.image.random_contrast(distorted_image, lower=0.2, upper=1.8) # whitening float_image = tf.image.per_image_whitening(distorted_image) min_fraction_of_examples_in_queue = 0.4 #min_fraction_of_examples_in_queue = 1 min_queue_examples = int(NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN * min_fraction_of_examples_in_queue) print( 'filling queue with %d train images before starting to train. This will take a few minutes.' % min_queue_examples) return _generate_image_and_label_batch(float_image, read_input.label, min_queue_examples)
import pickle import itertools import numpy as np import data import seq2seq from seq2seq.models import SimpleSeq2seq MAXLEN = 40 tweets, text, chars = data.read('tweets75k.pickle', minlen=20, maxlen=MAXLEN, padding=" ", trim_hash=True, shorten=True) chars_number = len(chars) tweets_number = len(tweets) model = SimpleSeq2seq(input_dim=1, hidden_dim=12, output_length=8, output_dim=1, depth=1) model.compile(loss='mse', optimizer='rmsprop') X = np.zeros((tweets_number, MAXLEN, chars_number), dtype=np.float32) model.fit(X, X, batch_size=32, nb_epoch=1000, verbose=1) y = model.predict(X) # for i in range(256): # print(X[i,:,0], y[i,:,0] > 0.5)
from keras.models import Sequential, model_from_json from keras.layers import Dense, Activation, Dropout from keras.layers import LSTM from keras.utils.data_utils import get_file import h5py import numpy as np import data seqlen = 40 seqstep = 3 # The tweet length should be above the learned sequence length. tweets, text, chars = data.read('tweets78k.txt', minlen=seqlen) print('Number of characters:', len(chars)) char_indices = dict((c, i) for i, c in enumerate(chars)) indices_char = dict((i, c) for i, c in enumerate(chars)) # Cut the text in semi-redundant sequences of seqlen characters. sentences = [] next_chars = [] for i in range(0, len(text) - seqlen, seqstep): sentences.append(text[i: i + seqlen]) next_chars.append(text[i + seqlen]) print('Number of sequences:', len(sentences))
else: continue if 'longitude' in item: lon = float(item['longitude']) else: continue if 'altitude' in item: alt = float(item['altitude']) else: alt = 10.0 if 'radius' in item: rad = int(item['radius'],10) else: rad = 100 self.ap_list.append(ap.AP(lat, lon, alt, rad)) self.ap_list[-1].get_cover() self.net = net.Network(self.ap_list) def to_xml(self): self.net.get_intersecting() self.net.get_cover() for g in self.net.coords: self.osm.polygon(g, tags=[]) self.osm.to_xml(self.output_filename) if __name__ == '__main__': ap_data = data.read('example.csv') map_obj = Map(ap_data, 'example', '0', 'user', '*****@*****.**') map_obj.get_network() map_obj.to_xml()
model_output_dir = sys.argv[2] problem_type = sys.argv[3] use_pretrained_embeddings = True if sys.argv[4].lower() == 'true' else False print >> sys.stderr, fn, model_output_dir, problem_type, use_pretrained_embeddings assert problem_type in ('regression', 'classification'), "Problem type should be either regression or classification" maxlen = 25 # maximum length for each sentence. max_features = 25000 # length of the vocabulary. batch_size = 32 nb_epoch = 4 additional_num_words = 2 # "UNK" and "PADDING" (X_train, y_train), (_, _), word_idx = data.read(fn, 0.0, maxlen, max_features, problem_type) print >> sys.stderr, 'X_train shape:', X_train.shape max_features = min(max_features, len(word_idx) + additional_num_words) if problem_type == 'regression': model = create_regression_model(maxlen, max_features, word_idx, use_pretrained_embeddings) else: model = create_logistic_model(maxlen, max_features, word_idx, use_pretrained_embeddings) train_model(model, X_train, y_train, batch_size, nb_epoch) model.save_weights(os.path.join(model_output_dir, 'weights.h5'), overwrite=True) print >> sys.stderr, "Weights are written" cPickle.dump(word_idx, codecs.open(os.path.join(model_output_dir, 'word_idx.json'), 'w', encoding='utf8')) print >> sys.stderr, "word_idx dict is written"
pattern.append(i.item) if i.count < min_support: min_support = i.count if min_support >= threshold: patterns.append(Pattern(pattern, min_support)) return patterns def print_tree(node, depth): s = "" for i in range(depth): s += " " print((s + "{0}:{1}").format(node.item, node.count)) for child in node.children: print_tree(child, depth + 1) def print_patterns(patterns): sorted_p = sorted(patterns, key=lambda p: (len(p.itemset), p.support)) for p in sorted_p: print("pattern: {0}, support:{1}".format(p.itemset, p.support)) if __name__ == "__main__": transactions = data.read("test.txt") frequent_items = _frequent_items(transactions, 3) fp_tree, header_table = _build_tree(transactions, frequent_items) p = Pattern([], sys.maxint) patterns = FP_growth(fp_tree, header_table, p, 3) print_patterns(patterns)
def execute():#执行一个clock周期中每一阶段的语句 global ret_pc global jmp_pc global call_pc global pred_pc global w_icode global w_valE global w_valM global w_dstE global w_dstM global m_icode global m_valE global m_valM global m_dstE global m_dstM global e_icode global e_bch global e_valE global e_valM global e_dstE global e_dstM global d_icode global d_ifunc global d_valC global d_valA global d_valB global d_dstE global d_dstM global d_srcA global d_srcB global f_icode global f_ifunc global f_ra global f_rb global f_valC global f_valP global total_cycle global valid_cycle while(data.isend()==False):#如果程序的结束状态为假,继续执行 #data.show() #global clocknum #clocknum+=1 #print("CLOCK_NO:%d" %(clocknum)) #PipeClock.PipeClock(); ret_pc="" jmp_pc="" call_pc="" pred_pc="" valid_cycle=data.intread("valid_cycle") total_cycle=data.intread("total_cycle") total_cycle+=1 #print("CLOCK-------------------------------") #WriteBack--------------------------------- #print("writeback:") w_icode=data.read("W_icode")#从外部寄存器读入信号 w_valE=data.read("W_valE") w_valM=data.read("W_valM") w_dstE=data.read("W_dstE") w_dstM=data.read("W_dstM") if(w_icode=="0"):#nop nothing() elif(w_icode=="1"):#halt 如果halt阶段执行完writeback阶段,程序就结束了,并且强行清空之前的4个阶段 data.write("M_index","X") data.write("M_icode","0") data.write("E_index","X") data.write("E_icode","0") data.write("D_index","X") data.write("D_icode","0") data.write("pc","X") data.write("F_icode","0") data.setend() elif(w_icode=="2"):#rrmovl data.RegWrite(w_dstE,w_valE) elif(w_icode=="3"):#irmovl data.RegWrite(w_dstE,w_valE) elif(w_icode=="4"):#rmmovl nothing() elif(w_icode=="5"):#mrmovl data.RegWrite(w_dstE,w_valM) elif(w_icode=="6"):#op data.RegWrite(w_dstE,w_valE) elif(w_icode=="7"):#jmp nothing() elif(w_icode=="8"):#call data.RegWrite("4",w_valE) elif(w_icode=="9"):#ret data.RegWrite("4",w_valE) elif(w_icode=="a"):#push data.RegWrite("4",w_valE) elif(w_icode=="b"):#popl data.RegWrite("4",w_valE) data.RegWrite(w_dstE,w_valM) elif(w_icode=="c"):#iaddl data.RegWrite(w_dstE,w_valE) elif(w_icode=="d"):#leave data.RegWrite("4",w_valE) data.RegWrite("5",w_valM) if(w_icode=="0"): nothing() else: valid_cycle+=1 #Memory-------------------------------------- #print("Memory:") m_icode="" m_valE="" m_valM="" m_dstE="" m_dstM="" m_icode=data.read("M_icode")#从外部读入信号 m_valE=data.read("M_valE") m_valM=data.read("M_valM") m_dstE=data.read("M_dstE") m_dstM=data.read("M_dstM") if(m_icode=="0"):#nop m_valE="" m_valM="" m_dstE="" m_dstM="" elif(m_icode=="1"):#halt 一旦halt出现就要强行结束之前的3个阶段 data.write("E_index","X") data.write("E_icode","0") data.write("D_index","X") data.write("D_icode","0") data.write("pc","X") data.write("F_icode","0") elif(m_icode=="2"):#rrmovl nothing() elif(m_icode=="3"):#irmovl nothing() elif(m_icode=="4"):#rmmovl data.MemoryWrite(m_dstM,m_valM) elif(m_icode=="5"):#mrmovl m_valM=data.MemoryRead(m_dstM) elif(m_icode=="6"):#op nothing() elif(m_icode=="7"):#jmp nothing() elif(m_icode=="8"):#call data.MemoryWrite(m_valE,m_valM) elif(m_icode=="9"):#ret ret阶段也要清空之前的所有阶段,直到下一步的指令明确 m_valM=data.MemoryRead(m_dstM) ret_pc=arc(m_valM)# 从内存中读取出来的pc,用于更新pred_pc data.write("E_index","X") data.write("E_icode","0") data.write("D_index","X") data.write("D_icode","0") data.write("pc","X") data.write("F_icode","0") elif(m_icode=="a"):#push data.MemoryWrite(m_valE,m_valM) elif(m_icode=="b"):#pop m_valM=data.MemoryRead(m_dstM) elif(m_icode=="c"):#iaddl nothing() elif(m_icode=="d"):#leave m_valM=data.MemoryRead(m_dstM) data.write("W_index",data.read("M_index")) data.write("W_icode",m_icode)#将信号写入下一阶段 data.write("W_valE",m_valE) data.write("W_valM",m_valM) data.write("W_dstE",m_dstE) data.write("W_dstM",m_dstM) #Execute---------------------------------------------- #print("Execute:") e_icode="" e_bch="" e_valE="" e_valM="" e_dstE="" e_dstM="" e_icode=data.read("E_icode")#从外部读入信号 e_ifunc=data.read("E_ifunc") e_valC=data.read("E_valC") e_valA=data.read("E_valA") e_valB=data.read("E_valB") e_dstE=data.read("E_dstE") e_dstM=data.read("E_dstM") srcA=data.read("srcA") srcB=data.read("srcB") if(e_icode=="0"):#nop nothing() e_bch="" e_valE="" e_valM="" e_dstE="" e_dstM="" elif(e_icode=="1"):#halt 清空之前所有阶段 data.write("D_index","X") data.write("D_icode","0") data.write("pc","X") data.write("F_icode","0") elif(e_icode=="2"):#rrmovl e_valE=e_valA elif(e_icode=="3"):#irmovl e_valE=e_valC elif(e_icode=="4"):#rmmovl aluA=e_valB aluB=e_valC e_dstM=data.alu("0",aluA,aluB) e_valM=e_valA elif(e_icode=="5"):#mrmovl aluA=e_valB aluB=e_valC e_dstM=data.alu("0",aluA,aluB) e_dstE=srcA elif(e_icode=="6"):#op aluA=e_valA aluB=e_valB e_valE=data.alu(e_ifunc,aluA,aluB) e_dstE=srcB elif(e_icode=="7"):#jmp if(data.check(e_ifunc)):#根据CF,SF,OF,ZF判断是否跳转 data.write("D_index","X") data.write("D_icode","0") data.write("pc","X") data.write("F_icode","0") jmp_pc=arc(e_valC)#如果要改变当前流程的话,记录对应的pc值 e_bch=jmp_pc elif(e_icode=="8"):#call e_valM=e_valB#将返回地址存入栈中 aluA=e_valA aluB="04000000" e_valE=data.alu("1",aluB,aluA) data.write("D_index","X") data.write("D_icode","0") data.write("pc","X") data.write("F_icode","0") call_pc=arc(e_valC) e_bch=call_pc elif(e_icode=="9"):#ret aluA=e_valA aluB="04000000" e_valE=data.alu("0",aluA,aluB) data.write("D_index","X") data.write("D_icode","0") data.write("pc","X") data.write("F_icode","0") e_dstM=e_valA elif(e_icode=="a"):#push aluA=e_valB aluB="04000000" e_valE=data.alu("1",aluB,aluA) e_valM=e_valA elif(e_icode=="b"):#popl aluA=e_valB aluB="04000000" e_valE=data.alu("0",aluA,aluB) e_dstM=e_valB e_dstE=srcA elif(e_icode=="c"):#iaddl aluA=e_valA aluB=e_valC e_valE=data.alu("0",aluA,aluB) e_dstE=srcA elif(e_icode=="d"):#leave aluA=e_valA aluB="04000000" e_valE=data.alu("0",aluB,aluA) e_dstM=e_valA e_dstE="4" data.write("M_index",data.read("E_index")) data.write("M_icode",e_icode)#把信号写入到下一阶段 data.write("bch",e_bch) data.write("M_valE",e_valE) data.write("M_valM",e_valM) data.write("M_dstE",e_dstE) data.write("M_dstM",e_dstM) #Decode---------------------------------- #print("Decode:") d_icode="" d_ifunc="" d_valC="" d_valA="" d_valB="" d_dstE="" d_dstM="" d_srcA="" d_srcB="" d_icode=data.read("D_icode")#从外部读入信号 d_ifunc=data.read("D_ifunc") ra=data.read("rA") rb=data.read("rB") d_valC=data.read("D_valC") d_valP=data.read("D_valP") if(d_icode=="0"):#nop nothing() d_ifunc="" d_valC="" d_valA="" d_valB="" d_dstE="" d_dstM="" d_srcA="" d_srcB="" elif(d_icode=="1"):#halt nothing() elif(d_icode=="2"):#rrmovl Temp=forwarding(ra)#判断是否存在加载使用冒险 if(Temp=="X"): data.setstall()#存在就暂停 else: d_valA=Temp#否则取得forwarding的值 d_dstE=rb d_srcA=ra d_srcB=rb elif(d_icode=="3"):#irmovl d_valA=d_valC d_dstE=rb elif(d_icode=="4"):#rmmovl------------------------------------------------------------- Temp=forwarding(ra)#判断要读取的寄存器a是否存在冒险 if(Temp=="X"): data.setstall() else: d_valA=Temp Temp=forwarding(rb)#判断要读取的寄存器b是否存在冒险 if(Temp=="X"): data.setstall() else: d_valB=Temp d_srcA=ra d_srcB=rb elif(d_icode=="5"):#mrmovl------------------------------------------------------- Temp=forwarding(rb)#判断寄存器b是否存在冒险 if(Temp=="X"): data.setstall() else: d_valB=Temp d_dstE=ra d_srcA=ra d_srcB=rb elif(d_icode=="6"):#op------------------------------------------------------------------------ Temp=forwarding(ra)#判断寄存器a是否存在冒险 if(Temp=="X"): data.setstall() else: d_valA=Temp Temp=forwarding(rb)#判断寄存器b是否存在冒险 if(Temp=="X"): data.setstall() else: d_valB=Temp d_srcA=ra d_srcB=rb d_dstE=rb elif(d_icode=="7"):#jmp---------------------- nothing() elif(d_icode=="8"):#call--------------------- Temp=forwarding("4")#特别要注意的是call隐式调用了esp,需要判断是否冒险 if(Temp=="X"): data.setstall() else: d_valA=Temp d_valB=d_valP d_srcA="4" d_dstE="4" elif(d_icode=="9"):#ret-------------------------------------- Temp=forwarding("4")#ret 调用了esp,需要判断冒险 if(Temp=="X"): data.setstall() else: d_valA=Temp d_dstE="4" d_srcA="4" elif(d_icode=="a"):#push------------------------------------- Temp=forwarding(ra)#push 同时调用了esp ebp需要同时判断冒险 if(Temp=="X"): data.setstall() else: d_valA=Temp Temp=forwarding("4") if(Temp=="X"): data.setstall() else: d_valB=Temp d_srcA=ra d_srcB="4" d_dstE="4" elif(d_icode=="b"):#popl--------------------------------------------------------------- Temp=forwarding("4")#pop 调用了esp,需要判断冒险 if(Temp=="X"): data.setstall() else: d_valB=Temp d_dstE=ra d_srcA=ra d_srcB="4" elif(d_icode=="c"):#iaddl------------------------------------------------------------- Temp=forwarding(ra)#判断iaddl的寄存器冒险 if(Temp=="X"): data.setstall() else: d_valA=Temp d_valB=d_valC d_dstE=ra d_srcA=ra elif(d_icode=="d"):#leave------------------------------------------------------------ Temp=forwarding("5")#leave要读取ebp,需要判断冒险 if(Temp=="X"): data.setstall() else: d_valA=Temp d_dstE="4" d_dstM="5" d_srcA="5" if(data.isstall()):#如果说cycle中的decode和fetch处于暂停状态的话,在下一个Execute阶段需要插入一个bubble data.write("E_icode","0") data.write("E_index","X") else:#其他情况下,正常复制信号到外部寄存器 data.write("E_index",data.read("D_index")) data.write("E_icode",d_icode) data.write("E_ifunc",d_ifunc) data.write("E_valC",d_valC) data.write("E_valA",d_valA) data.write("E_valB",d_valB) data.write("E_dstE",d_dstE) data.write("E_dstM",d_dstM) data.write("srcA",d_srcA) data.write("srcB",d_srcB) #Fetch--------------------------------------------- #print("Fetch:") f_icode="" f_ifunc="" f_ra="" f_rb="" f_valC="" f_valP="" f_icode=data.read("F_icode")#从外部寄存器读入信号 pc=data.read("pc") if(pc=="X"):#如果Fetch阶段因为stall不能取指,那么Fetch阶段本次不执行,但是需要将stall的一回合锁定取消,以保证下次能取指 data.write("D_index","X") data.write("D_icode","0") data.write("F_icode","1") f_ifunc="" f_ra="" f_rb="" f_valC="" f_valP="" data.write("D_ifunc",f_ifunc) data.write("rA",f_ra) data.write("rB",f_rb) data.write("D_valC",f_valC) data.write("D_valP",f_valP) elif(f_icode=="0"):#如果Fetch阶段是因为ret和jXX被清零,那么同样恢复合法状态,保证下次能取指 data.write("D_index","X") data.write("D_icode","0") data.write("F_icode","1") f_ifunc="" f_ra="" f_rb="" f_valC="" f_valP="" data.write("D_ifunc",f_ifunc) data.write("rA",f_ra) data.write("rB",f_rb) data.write("D_valC",f_valC) data.write("D_valP",f_valP) else:#如果就是合法,那么就根据内容正常取指 pred_pc=data.next(pc) s=data.MemoryGet(pc) if(s[0]=="0"):#nop nothing() if(s[0]=="1"):#halt nothing() if(s[0]=="2"):#rrmovl f_ra=s[2] f_rb=s[3] if(s[0]=="3"):#irmovl f_rb=s[3] f_valC=s[4:12] if(s[0]=="4"):#rmmovl f_ra=s[2] f_rb=s[3] f_valC=s[4:12] if(s[0]=="5"):#mrmovl f_ra=s[2] f_rb=s[3] f_valC=s[4:12] if(s[0]=="6"):#op f_ra=s[2] f_rb=s[3] if(s[0]=="7"):#jmp f_valC=s[2:10] if(s[0]=="8"):#call f_valC=s[2:10] f_ra="4" if(s[0]=="9"):#ret f_ra="4" nothing() if(s[0]=="a"):#push f_ra=s[2] f_rb="4" if(s[0]=="b"):#pop f_ra=s[2] f_rb="4" if(s[0]=="c"):#iaddl f_ra=s[3] f_valC=s[4:12] if(s[0]=="d"):#leave f_ra="5" f_icode=s[0] f_ifunc=s[1] if(data.isstall()): nothing() else:#不暂停的合法状态下,将信号写入下一个Decode阶段 data.write("D_index",pc) data.write("D_icode",f_icode) data.write("D_ifunc",f_ifunc) data.write("rA",f_ra) data.write("rB",f_rb) data.write("D_valC",f_valC) if(pred_pc[0]!="X"): data.write("D_valP",arc(pred_pc)) else: data.write("D_valP","X") #PC------------------------------------ data.intwrite("total_cycle",total_cycle)#将总周期数写入外部 data.intwrite("valid_cycle",valid_cycle)#将实际执行的周期数写入外部 #print("PC:") if(data.isstall()): data.start() return 0 if(ret_pc!=""):#判断下一个pc应该取什么值 data.write("pc",ret_pc)#有ret先取ret_pc elif(call_pc!=""): data.write("pc",call_pc)#有call,取call_pc elif(jmp_pc!=""): data.write("pc",jmp_pc)#有jXX,取jmp_pc elif(pred_pc!=""): data.write("pc",pred_pc)#都没有的情况下取得下一个pc return 0
import sys import itertools import numpy as np from seq2seq.models import SimpleSeq2seq import data MAXLEN = 40 tweets, text, chars = data.read('tweets78k.txt', minlen=20, maxlen=MAXLEN, trim_hash=True, shorten=True) num_chars = len(chars) num_tweets = len(tweets) print('Number of characters:', len(chars)) print('Number of sequences:', len(tweets)) model = SimpleSeq2seq(input_dim=num_chars, hidden_dim=256, output_length=MAXLEN, output_dim=num_chars, depth=1) model.compile(loss='mse', optimizer='rmsprop') char_indices = dict((c, i) for i, c in enumerate(chars)) indices_char = dict((i, c) for i, c in enumerate(chars)) print('Vectorization...') X = np.zeros((num_tweets, MAXLEN, num_chars), dtype=np.float32) for i, tweet in enumerate(tweets): sentence = list(itertools.chain(*itertools.repeat(tweet.replace('\4', '').replace('\2', '').replace('\3', ''), times=2)))[:MAXLEN] for t, char in enumerate(sentence):
def getY86(): #global register register={} #global memory register={} register["CF"]=data.read("CF") register["OF"]=data.read("OF") register["ZF"]=data.read("ZF") register["SF"]=data.read("SF") register["stall"]=data.read("stall") register["end"]=data.read("end") register["pc"]=data.read("pc") register["F_icode"]=data.read("F_icode") register["D_index"]=data.read("D_index") register["D_icode"]=data.read("D_icode") register["D_ifunc"]=data.read("D_ifunc") register["rA"]=data.read("rA") register["rB"]=data.read("rB") register["D_valC"]=data.read("D_valC") register["D_valP"]=data.read("D_valP") register["E_index"]=data.read("E_index") register["E_icode"]=data.read("E_icode") register["E_ifunc"]=data.read("E_ifunc") register["E_valC"]=data.read("E_valC") register["E_valA"]=data.read("E_valA") register["E_valB"]=data.read("E_valB") register["E_dstE"]=data.read("E_dstE") register["E_dstM"]=data.read("E_dstM") register["srcA"]=data.read("srcA") register["srcB"]=data.read("srcB") register["M_index"]=data.read("M_index") register["M_icode"]=data.read("M_icode") register["bch"]=data.read("bch") register["M_valE"]=data.read("M_valE") register["M_valM"]=data.read("M_valM") register["M_dstE"]=data.read("M_dstE") register["M_dstM"]=data.read("M_dstM") register["W_index"]=data.read("W_index") register["W_icode"]=data.read("W_icode") register["W_valE"]=data.read("W_valE") register["W_valM"]=data.read("W_valM") register["W_dstE"]=data.read("W_dstE") register["W_dstM"]=data.read("W_dstM") register["eax"]=data.RegRead("0") register["ecx"]=data.RegRead("1") register["edx"]=data.RegRead("2") register["ebx"]=data.RegRead("3") register["esp"]=data.RegRead("4") register["ebp"]=data.RegRead("5") register["esi"]=data.RegRead("6") register["edi"]=data.RegRead("7") register["total_cycle"]=data.intread("total_cycle") register["valid_cycle"]=data.intread("valid_cycle") memory=data.Memory return register,memory
def execute(): global ret_pc global jmp_pc global call_pc global pred_pc global w_icode global w_valE global w_valM global w_dstE global w_dstM global m_icode global m_valE global m_valM global m_dstE global m_dstM global e_icode global e_bch global e_valE global e_valM global e_dstE global e_dstM global d_icode global d_ifunc global d_valC global d_valA global d_valB global d_dstE global d_dstM global d_srcA global d_srcB global f_icode global f_ifunc global f_ra global f_rb global f_valC global f_valP while data.isend() == False: data.show() global clocknum clocknum += 1 print("CLOCK_NO:%d" % (clocknum)) PipeClock.PipeClock() ret_pc = "" jmp_pc = "" call_pc = "" pred_pc = "" # print("CLOCK-------------------------------") # WriteBack--------------------------------- # print("writeback:") w_icode = data.read("W_icode") w_valE = data.read("W_valE") w_valM = data.read("W_valM") w_dstE = data.read("W_dstE") w_dstM = data.read("W_dstM") if w_icode == "0": # nop nothing() elif w_icode == "1": # halt data.write("M_index", "X") data.write("M_icode", "0") data.write("E_index", "X") data.write("E_icode", "0") data.write("D_index", "X") data.write("D_icode", "0") data.write("pc", "X") data.write("F_icode", "0") data.setend() elif w_icode == "2": # rrmovl data.RegWrite(w_dstE, w_valE) elif w_icode == "3": # irmovl data.RegWrite(w_dstE, w_valE) elif w_icode == "4": # rmmovl nothing() elif w_icode == "5": # mrmovl data.RegWrite(w_dstE, w_valM) elif w_icode == "6": # op data.RegWrite(w_dstE, w_valE) elif w_icode == "7": # jmp nothing() elif w_icode == "8": # call data.RegWrite("4", w_valE) elif w_icode == "9": # ret data.RegWrite("4", w_valE) elif w_icode == "a": # push data.RegWrite("4", w_valE) elif w_icode == "b": # popl data.RegWrite("4", w_valE) data.RegWrite(w_dstE, w_valM) # Memory-------------------------------------- # print("Memory:") m_icode = "" m_valE = "" m_valM = "" m_dstE = "" m_dstM = "" m_icode = data.read("M_icode") m_valE = data.read("M_valE") m_valM = data.read("M_valM") m_dstE = data.read("M_dstE") m_dstM = data.read("M_dstM") if m_icode == "0": # nop nothing() m_valE = "" m_valM = "" m_dstE = "" m_dstM = "" elif m_icode == "1": # halt data.write("E_index", "X") data.write("E_icode", "0") data.write("D_index", "X") data.write("D_icode", "0") data.write("pc", "X") data.write("F_icode", "0") elif m_icode == "2": # rrmovl nothing() elif m_icode == "3": # irmovl nothing() elif m_icode == "4": # rmmovl data.MemoryWrite(m_dstM, m_valM) elif m_icode == "5": # mrmovl m_valM = data.MemoryRead(m_dstM) elif m_icode == "6": # op nothing() elif m_icode == "7": # jmp nothing() elif m_icode == "8": # call data.MemoryWrite(m_valE, m_valM) elif m_icode == "9": # ret m_valM = data.MemoryRead(m_dstM) ret_pc = arc(m_valM) data.write("E_index", "X") data.write("E_icode", "0") data.write("D_index", "X") data.write("D_icode", "0") data.write("pc", "X") data.write("F_icode", "0") elif m_icode == "a": # push data.MemoryWrite(m_valE, m_valM) elif m_icode == "b": # pop m_valM = data.MemoryRead(m_dstM) data.write("W_index", data.read("M_index")) data.write("W_icode", m_icode) data.write("W_valE", m_valE) data.write("W_valM", m_valM) data.write("W_dstE", m_dstE) data.write("W_dstM", m_dstM) # Execute---------------------------------------------- # print("Execute:") e_icode = "" e_bch = "" e_valE = "" e_valM = "" e_dstE = "" e_dstM = "" e_icode = data.read("E_icode") e_ifunc = data.read("E_ifunc") e_valC = data.read("E_valC") e_valA = data.read("E_valA") e_valB = data.read("E_valB") e_dstE = data.read("E_dstE") e_dstM = data.read("E_dstM") srcA = data.read("srcA") srcB = data.read("srcB") if e_icode == "0": # nop nothing() e_bch = "" e_valE = "" e_valM = "" e_dstE = "" e_dstM = "" elif e_icode == "1": # halt data.write("D_index", "X") data.write("D_icode", "0") data.write("pc", "X") data.write("F_icode", "0") elif e_icode == "2": # rrmovl e_valE = e_valA elif e_icode == "3": # irmovl e_valE = e_valC elif e_icode == "4": # rmmovl aluA = e_valB aluB = e_valC e_dstM = data.alu("0", aluA, aluB) e_valM = e_valA elif e_icode == "5": # mrmovl aluA = e_valB aluB = e_valC e_dstM = data.alu("0", aluA, aluB) e_dstE = srcA elif e_icode == "6": # op aluA = e_valA aluB = e_valB e_valE = data.alu(e_ifunc, aluA, aluB) e_dstE = srcB elif e_icode == "7": # jmp if data.check(e_ifunc): data.write("D_index", "X") data.write("D_icode", "0") data.write("pc", "X") data.write("F_icode", "0") jmp_pc = arc(e_valC) e_bch = jmp_pc elif e_icode == "8": # call e_valM = e_valB aluA = e_valA aluB = "04000000" e_valE = data.alu("1", aluB, aluA) data.write("D_index", "X") data.write("D_icode", "0") data.write("pc", "X") data.write("F_icode", "0") call_pc = arc(e_valC) e_bch = call_pc elif e_icode == "9": # ret aluA = e_valA aluB = "04000000" e_valE = data.alu("0", aluA, aluB) data.write("D_index", "X") data.write("D_icode", "0") data.write("pc", "X") data.write("F_icode", "0") e_dstM = e_valA elif e_icode == "a": # push aluA = e_valB aluB = "04000000" e_valE = data.alu("1", aluB, aluA) e_valM = e_valA elif e_icode == "b": # popl aluA = e_valB aluB = "04000000" e_valE = data.alu("0", aluA, aluB) e_dstM = e_valB e_dstE = srcA data.write("M_index", data.read("E_index")) data.write("M_icode", e_icode) data.write("bch", e_bch) data.write("M_valE", e_valE) data.write("M_valM", e_valM) data.write("M_dstE", e_dstE) data.write("M_dstM", e_dstM) # Decode---------------------------------- # print("Decode:") d_icode = "" d_ifunc = "" d_valC = "" d_valA = "" d_valB = "" d_dstE = "" d_dstM = "" d_srcA = "" d_srcB = "" d_icode = data.read("D_icode") d_ifunc = data.read("D_ifunc") ra = data.read("rA") rb = data.read("rB") d_valC = data.read("D_valC") d_valP = data.read("D_valP") if d_icode == "0": # nop nothing() d_ifunc = "" d_valC = "" d_valA = "" d_valB = "" d_dstE = "" d_dstM = "" d_srcA = "" d_srcB = "" elif d_icode == "1": # halt nothing() elif d_icode == "2": # rrmovl Temp = forwarding(ra) if Temp == "X": data.setstall() else: d_valA = Temp d_dstE = rb d_srcA = ra d_srcB = rb elif d_icode == "3": # irmovl d_valA = d_valC d_dstE = rb elif d_icode == "4": # rmmovl------------------------------------------------------------- Temp = forwarding(ra) if Temp == "X": data.setstall() else: d_valA = Temp Temp = forwarding(rb) if Temp == "X": data.setstall() else: d_valB = Temp d_srcA = ra d_srcB = rb elif d_icode == "5": # mrmovl------------------------------------------------------- Temp = forwarding(rb) if Temp == "X": data.setstall() else: d_valB = Temp d_dstE = ra d_srcA = ra d_srcB = rb elif d_icode == "6": # op------------------------------------------------------------------------ Temp = forwarding(ra) if Temp == "X": data.setstall() else: d_valA = Temp Temp = forwarding(rb) if Temp == "X": data.setstall() else: d_valB = Temp d_srcA = ra d_srcB = rb d_dstE = rb elif d_icode == "7": # jmp---------------------- nothing() elif d_icode == "8": # call--------------------- Temp = forwarding("4") if Temp == "X": data.setstall() else: d_valA = Temp d_valB = d_valP d_srcA = "4" d_dstE = "4" elif d_icode == "9": # ret-------------------------------------- Temp = forwarding("4") if Temp == "X": data.setstall() else: d_valA = Temp d_dstE = "4" d_srcA = "4" elif d_icode == "a": # push------------------------------------- Temp = forwarding(ra) if Temp == "X": data.setstall() else: d_valA = Temp Temp = forwarding("4") if Temp == "X": data.setstall() else: d_valB = Temp d_srcA = ra d_srcB = "4" d_dstE = "4" elif d_icode == "b": # popl--------------------------------------------------------------- Temp = forwarding("4") if Temp == "4": data.setstall() else: d_valB = Temp d_dstE = ra d_srcA = ra d_srcB = "4" if data.isstall(): data.write("E_icode", "0") data.write("E_index", "X") else: data.write("E_index", data.read("D_index")) data.write("E_icode", d_icode) data.write("E_ifunc", d_ifunc) data.write("E_valC", d_valC) data.write("E_valA", d_valA) data.write("E_valB", d_valB) data.write("E_dstE", d_dstE) data.write("E_dstM", d_dstM) data.write("srcA", d_srcA) data.write("srcB", d_srcB) # Fetch--------------------------------------------- # print("Fetch:") f_icode = "" f_ifunc = "" f_ra = "" f_rb = "" f_valC = "" f_valP = "" f_icode = data.read("F_icode") pc = data.read("pc") if pc == "X": data.write("D_index", "X") data.write("D_icode", "0") data.write("F_icode", "1") f_ifunc = "" f_ra = "" f_rb = "" f_valC = "" f_valP = "" data.write("D_ifunc", f_ifunc) data.write("rA", f_ra) data.write("rB", f_rb) data.write("D_valC", f_valC) data.write("D_valP", f_valP) elif f_icode == "0": data.write("D_index", "X") data.write("D_icode", "0") data.write("F_icode", "1") f_ifunc = "" f_ra = "" f_rb = "" f_valC = "" f_valP = "" data.write("D_ifunc", f_ifunc) data.write("rA", f_ra) data.write("rB", f_rb) data.write("D_valC", f_valC) data.write("D_valP", f_valP) elif data.isstall(): nothing() else: pred_pc = data.next(pc) s = data.MemoryGet(pc) if s[0] == "0": # nop nothing() if s[0] == "1": # halt nothing() if s[0] == "2": # rrmovl f_ra = s[2] f_rb = s[3] if s[0] == "3": # irmovl f_rb = s[3] f_valC = s[4:12] if s[0] == "4": # rmmovl f_ra = s[2] f_rb = s[3] f_valC = s[4:12] if s[0] == "5": # mrmovl f_ra = s[2] f_rb = s[3] f_valC = s[4:12] if s[0] == "6": # op f_ra = s[2] f_rb = s[3] if s[0] == "7": # jmp f_valC = s[2:10] if s[0] == "8": # call f_valC = s[2:10] f_ra = "4" if s[0] == "9": # ret f_ra = "4" nothing() if s[0] == "a": # push f_ra = s[2] f_rb = "4" if s[0] == "b": # pop f_ra = s[2] f_rb = "4" f_icode = s[0] f_ifunc = s[1] if data.isstall(): nothing() else: data.write("D_index", pc) data.write("D_icode", f_icode) data.write("D_ifunc", f_ifunc) data.write("rA", f_ra) data.write("rB", f_rb) data.write("D_valC", f_valC) if pred_pc[0] != "X": data.write("D_valP", arc(pred_pc)) else: data.write("D_valP", "X") # PC------------------------------------ # print("PC:") if data.isstall(): data.start() return 0 if ret_pc != "": data.write("pc", ret_pc) elif call_pc != "": data.write("pc", call_pc) elif jmp_pc != "": data.write("pc", jmp_pc) elif pred_pc != "": data.write("pc", pred_pc) return 0