def predict(): gender = mp.transform_gender(request.args.get('gender')) married = mp.transform_married(request.args.get('married')) dependents = int(request.args.get('dependents')) education = mp.transform_education(request.args.get('education')) selfemployed = mp.transform_self_emp(request.args.get('selfemployed')) applicant_income = int(request.args.get('app_income')) co_appllicant_income = int(request.args.get('co_income')) loan_amount = int(request.args.get('loan_amount')) loan_amt_term = int(request.args.get('loan_amt_term')) credit_hist = int(request.args.get('credit_hist')) prop_area = mp.transform_prop_area(request.args.get('prop_area')) trained_model = pickle.load(open('finalModel', 'rb')) result = trained_model.predict_proba([[ gender, married, dependents, education, selfemployed, applicant_income, co_appllicant_income, loan_amount, loan_amt_term, credit_hist, prop_area ]]) status_false = result[0][0] status_true = result[0][1] if status_false > status_true: return "The Result of The Prediction: <b>Loan Status for this user is False with Probablity " + str( status_false) + "</b>." else: return "The Result of The Prediction: <b>Loan Status for this user is True with Probablity " + str( status_true) + "</b>."
def init_mapping(representation): ### load the mapping mapping = {} shared_mapping_obj = Mapping.get_mapping_instance( FLAGS.shared_mapping_class) for lang in representation.keys(): if representation[lang] in ['phonetic', 'onehot_and_phonetic']: mapping[lang] = shared_mapping_obj elif representation[lang] == 'onehot_shared': mapping[lang] = shared_mapping_obj elif representation[lang] == 'onehot': mapping[lang] = Mapping.CharacterMapping() with open(FLAGS.mapping_dir + '/' + 'mapping_' + lang + '.json', 'r') as mapping_file: mapping[lang].load_mapping(mapping_file) ## Print Representation and Mappings print 'Mapping' print mapping print 'Vocabulary Statitics' for lang in representation.keys(): print '{}: {}'.format(lang, mapping[lang].get_vocab_size()) return mapping
def createTower(liste, degats, element, vitesseA, currentTower, colorint, MAP, colonne, ligne, price, money, sprite): if currentTower != 0: cursor = pygame.Rect(pygame.mouse.get_pos()[0], pygame.mouse.get_pos()[1], 1, 1) test = cursor.collidelist(Mapping.GetCaseRectList(colonne, ligne, MAP)) if test != -1: if test < colonne: currCase = Mapping.GetCase(test, 0, MAP) else: currCase = Mapping.GetCase( test - colonne * int(test / colonne), int(test / colonne), MAP) if currCase.IsPath == False and currCase.HasTower == False: sprite = pygame.transform.scale( sprite, (int(currCase.width), int(currCase.length))) newTower = tower(degats, element, vitesseA, colorint, sprite) newTower.setPos(currCase.innerRect) liste.append(newTower) currentTower = 0 currCase.Lock() MAP[int(test / colonne)][test - colonne * int(test / colonne)] = currCase money = money - price return currentTower, liste, MAP, money return liste
def pred_filter(infile1, infile2): mapped_pred = Mapping.mapper_pred(infile1) mapped_bench = Mapping.mapper_bench(infile2) outfile_name = infile1 + '_pred_set.txt' outfile_handle = open(outfile_name, 'w') bench = defaultdict() b_file = open(mapped_bench,'r') for lines in b_file: fields = lines.strip().split() bench[fields[0]] = 1 b_file.close() prot_ann = defaultdict() print "Filtering prediction data....\n" prediction_file = open(mapped_pred,'r') for data in prediction_file: fields = data.strip().split() if bench.has_key(fields[0]): print >> outfile_handle, fields[0] + '\t' + fields[1] + '\t' + fields[2] prot_ann[fields[0]] = 1 prediction_file.close() prot_ann.clear() bench.clear() return outfile_name, mapped_bench
def _command_line_mappings(self, mappings): if not mappings: return #try: Mapping.create_mapping('command-line') for m in mappings: (model, device) = m.split(':') Mapping.bind(model, device)
def read_mapping(self, t="ASTAR", fname=None): if t == "ASTAR": self.mappings.append(Mapping(self)) self.mappings[-1].read_astar_mapping(fname) elif t == "KEGG": self.mappings.append(Mapping(self)) self.mappings[-1].read_kegg_mapping(fname) elif t == "ARITA": self.mappings.append(Mapping(self)) self.mappings[-1].read_arita_mapping(fname)
def get_seq2seq_loss(lang_pairs, parallel_data, seq_loss_op, batch_sequences, batch_sequence_masks, batch_sequence_lengths, batch_sequences_2, batch_sequence_masks_2, batch_sequence_lengths_2, dropout_keep_prob, prefix_tgtlang, prefix_srclang, mapping, max_sequence_length): """ convenience function to compute the translation loss (cross entropy) lang_pairs: list of language pair tuples. parallel_data: Dictionary of ParallelDataReader object for various language pairs seq_loss_op: dictionary of sequence loss operation for every language pair batch_sequences, batch_sequence_masks, batch_sequence_lengths, batch_sequences_2, batch_sequence_masks_2, batch_sequence_lengths_2, dropout_keep_prob: placeholder variables prefix_srclang: see commandline flags prefix_tgtlang: see commandline flags mapping: Dictionary of mapping objects for each language """ ### start computation validation_loss = 0.0 for src_lang, tgt_lang in lang_pairs: lang_pair = (src_lang, tgt_lang) sequences,sequence_masks,sequence_lengths,sequences_2,sequence_masks_2,sequence_lengths_2 = \ parallel_data[lang_pair].get_data() if prefix_tgtlang: sequences, sequence_masks, sequence_lengths = Mapping.prefix_sequence_with_token( sequences, sequence_masks, sequence_lengths, tgt_lang, mapping[tgt_lang]) if prefix_srclang: sequences, sequence_masks, sequence_lengths = Mapping.prefix_sequence_with_token( sequences, sequence_masks, sequence_lengths, src_lang, mapping[src_lang]) validation_loss += sess.run(seq_loss_op[lang_pair], feed_dict={ batch_sequences: sequences, batch_sequence_masks: sequence_masks, batch_sequence_lengths: sequence_lengths, batch_sequences_2: sequences_2, batch_sequence_masks_2: sequence_masks_2, batch_sequence_lengths_2: sequence_lengths_2, dropout_keep_prob: 1.0 }) return validation_loss
def kijiji(city, pages): listings = Mapping.listings(city, pages) condodata = [] #Headers for condodata array condodata.append([ 'URL', 'Address', 'UnitType', 'AgreementType', 'Price', 'Date', 'MoveInDate', 'SquareFeet', 'Bedrooms', 'Bathrooms', 'Furnished', 'PostalCode', 'PostalTop', 'PostalBottom', 'Hydro', 'Heat', 'Water', 'Cable', 'Internet', 'LaundryIn', 'LaundryOut', 'Parking', 'Landline', 'AirConditioning', 'PetFriendly', 'Smoking', 'Yard', 'Balcony' ]) # Pulls "market data" from each listing for post in listings: kijijii = requests.get(post) soup = BeautifulSoup(kijijii.text, 'html.parser') info = Retrieve.datapoints(soup) address = Retrieve.location(soup) print(address) postalcode = Retrieve.postal(address) price = Retrieve.price(soup) date = Retrieve.dateposted(soup) condodata.append([ post, address, info.setdefault(condodata[0][2], 'NA'), info.setdefault(condodata[0][3], 'NA'), price, date, info.setdefault(condodata[0][6], 'NA'), info.setdefault(condodata[0][7], 'NA'), info.setdefault(condodata[0][8], 'NA'), info.setdefault(condodata[0][9], 'NA'), info.setdefault(condodata[0][10], 'NA'), postalcode, postalcode[:3], postalcode[-3:], info.setdefault(condodata[0][14], 'NA'), info.setdefault(condodata[0][15], 'NA'), info.setdefault(condodata[0][16], 'NA'), info.setdefault(condodata[0][17], 'NA'), info.setdefault(condodata[0][18], 'NA'), info.setdefault(condodata[0][19], 'NA'), info.setdefault(condodata[0][20], 'NA'), info.setdefault(condodata[0][21], 'NA'), info.setdefault(condodata[0][22], 'NA'), info.setdefault(condodata[0][23], 'NA'), info.setdefault(condodata[0][24], 'NA'), info.setdefault(condodata[0][25], 'NA'), info.setdefault(condodata[0][26], 'NA'), info.setdefault(condodata[0][27], 'NA') ]) return condodata
def _combine_hosts(self): ct = ConfiguredTest() ct.resources = Resources.resources.resources() ct.hosts = {} ct.end_policy = Schedule.get_schedule().test_end_policy() ct.setup_phase_delay = Schedule.get_schedule().setup_phase_delay() ct.triggers = Schedule.get_schedule().triggers() for h in Model.get_model().hosts(): host = ConfiguredHost() host.model = h host.device = h.bound() host.schedule = Schedule.get_schedule().host_schedule(h['name']) resources = set(h.needed_resources()) for event in host.schedule: resources.update(event.command().needed_resources()) def resolve_resource(r): if isinstance(r, str): return Utils.resolve_resource_name(r) return r host.resources = set(map(resolve_resource, resources)) ct.hosts[h['name']] = host ct.sanity_check() ct.model = Model.get_model() ct.laboratory = Laboratory.get_laboratory() ct.schedule = Schedule.get_schedule() ct.mapping = Mapping.get_mapping() self._configured_test = ct
def predict(): df= pd.read_csv("Job titles and industries.csv") # Features and Labels df['label'] = df['industry'].map({'IT': 0, 'Marketing': 1,'Education':2,'Accountancy':3}) x = df['job title'] y = df['label'] # Extract Feature With CountVectorizer tfidf = TfidfVectorizer() x = tfidf.fit_transform(df["job title"]).toarray() # Fit the Data from sklearn.model_selection import train_test_split x_train,x_test,y_train,y_test = train_test_split(x,y,test_size=0.25) model=RandomForestClassifier(n_estimators=100) model.fit(x_train,y_train) jop_title=request.args.get('jop_title') data=mp.run(jop_title) vect=tfidf.transform([data]).toarray() result = model.predict(vect) if result[0]==0: return "The Result of The Prediction: <b> IT" elif result[0]==1: return "The Result of The Prediction: <b> Marketing" elif result[0]==2: return "The Result of The Prediction: <b> Education" elif result[0]==3: return "The Result of The Prediction: <b> Accountancy"
def predict(): type = mp.transform_type(request.args.get('type')) amount = float(request.args.get('amount')) old_bal_org = float(request.args.get('old_bal_org')) new_bal_org = float(request.args.get('new_bal_org')) name_dest = mp.transform_nameDest(request.args.get('name_dest')) old_bal_det = float(request.args.get('old_bal_det')) new_bal_det = float(request.args.get('new_bal_det')) trained_model = pickle.load(open('finalModel', 'rb')) result = trained_model.predict([[ type, amount, old_bal_org, new_bal_org, name_dest, old_bal_det, new_bal_det ]]) if result == 1: return "The Result of The Prediction: <b> Take Care !! This Transaction is Fraud </b>." else: return "The Result of The Prediction: <b>Good !! This Transaction is not Fraud </b>."
def _sanity_check(self): if not Model.get_model(): raise Exceptions.SanityError("No model defined. You need to create a model. Did you forget to use 'create_model(name)' in your configuration?") if not Laboratory.get_laboratory(): raise Exceptions.SanityError("No laboratory defined. You need to create a laboratory. Did you forget to use 'create_laboratory(name)' in your configuration?") if not Mapping.get_mapping(): raise Exceptions.SanityError("No mapping defined. You need to create a mapping. Did you forget to use 'create_mapping(name)' in your configuration?") if not Schedule.get_schedule(): raise Exceptions.SanityError("No schedule defined. You need to create a schedule. Did you forget to use 'create_schedule(name)' in your configuration?")
##### B. Read in the structure struc = Structure(options["-f"].value, strict=options["-strict"].value) ##### C. Set the mapping dictionary # Convert force field tags to lower case # Default is backmapping from MARTINI to GROMOS53A6 # If to_ff == martini, default from_ff = gromos to_ff = options["-to"] and options["-to"].value.lower() or "gromos" if to_ff == "martini" and not options["-from"]: from_ff = "gromos" else: from_ff = options["-from"] and options["-from"].value.lower() or "martini" mapping = Mapping.Mapping(options["-mapdir"].value).get(source=from_ff, target=to_ff) backmapping = levels[from_ff] > levels[to_ff] reslist = mapping.keys() ##### D. Iterate over atoms to write out, based on residue names # Copy the residue list from the target topology # This gives a list we can pop from, while keeping # the original. # The solvent residues are skipped. topresidues = None if top: topresidues = [i for i in top.residues] print(options["-atomlist"], options["-atomlist"].value) if options["-atomlist"]: atm = open(options["-atomlist"].value, "w")
def run_print_vars(args): """ Debugging: print the names of the variables in the model """ ## check for required parameters if args.lang is None: print 'ERROR: --lang has to be set' sys.exit(1) if args.model_fname is None: print 'ERROR: --model_fname has to be set' sys.exit(1) if args.mapping_fname is None: print 'ERROR: --mapping_fname has to be set' sys.exit(1) ####################################### # Reading data and creating mappings # ####################################### # Creating mapping object to store char-id mappings mapping = Mapping.get_mapping_instance(args.mapping_class) with open(args.mapping_fname, 'r') as mapping_json_file: mapping.load_mapping(mapping_json_file) print 'Mapping' print mapping print 'Vocabulary Statitics' print '{}: {}'.format(args.lang, mapping.get_vocab_size()) print 'Finished Reading Data' ################################################################### # Interacting with model and creating computation graph # ################################################################### # Creating Model object model = LanguageModel(args.lang, mapping, args.representation, args.max_seq_length, args.embedding_size, args.rnn_size) for v in tf.all_variables(): print v.name ## Creating placeholder for sequences, masks and lengths and dropout keep probability pl_batch_sequences = tf.placeholder(shape=[None, args.max_seq_length], dtype=tf.int32) pl_batch_sequence_lengths = tf.placeholder(shape=[None], dtype=tf.float32) loss_op = model.average_loss(pl_batch_sequences, pl_batch_sequence_lengths, 1.0) #Saving model saver = tf.train.Saver(max_to_keep=0) #Start Session config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) #sess.run(tf.initialize_all_variables()) saver.restore(sess, args.model_fname) print 'after loading' for v in tf.all_variables(): print v.name
def run_test(args): """ Test a trained language mode """ ## check for required parameters if args.lang is None: print 'ERROR: --lang has to be set' sys.exit(1) if args.model_fname is None: print 'ERROR: --model_fname has to be set' sys.exit(1) if args.mapping_fname is None: print 'ERROR: --mapping_fname has to be set' sys.exit(1) if args.in_fname is None: print 'ERROR: --in_fname has to be set' sys.exit(1) ####################################### # Reading data and creating mappings # ####################################### # Creating mapping object to store char-id mappings mapping = Mapping.get_mapping_instance(args.mapping_class) with open(args.mapping_fname, 'r') as mapping_json_file: mapping.load_mapping(mapping_json_file) print 'Mapping' print mapping print 'Vocabulary Statitics' print '{}: {}'.format(args.lang, mapping.get_vocab_size()) ## Reading test data test_data = MonoDataReader.MonoDataReader(args.lang, args.in_fname, mapping, args.max_seq_length) print 'Finished Reading Data' ################################################################### # Interacting with model and creating computation graph # ################################################################### # Creating Model object model = LanguageModel(args.lang, mapping, args.representation, args.max_seq_length, args.embedding_size, args.rnn_size) ## Creating placeholder for sequences, masks and lengths and dropout keep probability pl_batch_sequences = tf.placeholder(shape=[None, args.max_seq_length], dtype=tf.int32) pl_batch_sequence_lengths = tf.placeholder(shape=[None], dtype=tf.float32) loss_op = model.average_loss(pl_batch_sequences, pl_batch_sequence_lengths, 1.0) #Saving model saver = tf.train.Saver(max_to_keep=0) #Start Session config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) #sess.run(tf.initialize_all_variables()) saver.restore(sess, args.model_fname) print "Session started" ## TEST LOSS test_start_time = time.time() test_loss = get_average_loss(test_data, loss_op, pl_batch_sequences, pl_batch_sequence_lengths, sess) test_end_time = time.time() epoch_test_time = (test_end_time - test_start_time) print "Test Perplexity: {}".format(test_loss) print "Test Time (hh:mm:ss)::: {}".format( utilities.formatted_timeinterval(epoch_test_time))
def convert_address_to_latlng(address_string): return Mapping.address_to_geo(address_string)
##### B. Read in the structure struc = Structure(options["-f"].value, strict=options["-strict"].value) ##### C. Set the mapping dictionary # Convert force field tags to lower case # Default is backmapping from MARTINI to GROMOS53A6 # If to_ff == martini, default from_ff = gromos to_ff = options["-to"] and options["-to"].value.lower() or "gromos" if to_ff == "martini" and not options["-from"]: from_ff = "gromos" else: from_ff = options["-from"] and options["-from"].value.lower() or "martini" mapping = Mapping.get(source=from_ff, target=to_ff) backmapping = levels[from_ff] > levels[to_ff] reslist = mapping.keys() ##### D. Iterate over atoms to write out, based on residue names # Copy the residue list from the target topology # This gives a list we can pop from, while keeping # the original. # The solvent residues are skipped. topresidues = None if top: topresidues = [i for i in top.residues] if options["-atomlist"]: atm = open(options["-atomlist"].value, "w") topatm = [j for i in topresidues for j in i]
### parse representation argument if args.representation in ['onehot','onehot_shared','phonetic','onehot_and_phonetic']: representation = {} for lang in lang_pair: representation[lang]=args.representation else: representation = dict([ x.split(':') for x in args.representation.split(',') ]) ## Print Representation and Mappings print 'Representation' print representation ### load the mapping mapping={} shared_mapping_obj = Mapping.get_mapping_instance(shared_mapping_class) for lang in representation.keys(): if representation[lang] in ['phonetic','onehot_and_phonetic']: mapping[lang]=shared_mapping_obj elif representation[lang]=='onehot_shared': mapping[lang]=shared_mapping_obj elif representation[lang]=='onehot': mapping[lang]=Mapping.CharacterMapping() with open(mapping_dir+'/'+'mapping_'+lang+'.json','r') as mapping_file: mapping[lang].load_mapping(mapping_file) ## Print Representation and Mappings print 'Mapping' print mapping
def FitTOD(tod, ra, dec, obs, clon, clat, cpang, prefix='', destripe=False, mode='mode1', justOffsets=True, doPlots=True): # Beam solid angle aken from James' report on wiki Optics nubeam = np.array([26., 33., 40.]) srbeam = np.array([2.1842e-6, 1.6771e-6, 1.4828e-6]) pmdl = interp1d(nubeam, srbeam) # straight interpolation nHorns = tod.shape[0] nSidebands = tod.shape[1] nChans = tod.shape[2] nSamps = tod.shape[3] if mode == 'mode1': nParams = 5 elif mode == 'mode2': nParams = 6 else: nParams = 0 print('WARNING: No fitting method selected') # Define the pixel grid # Pixel coordinates on sky wcs, xr, yr = Mapping.DefineWCS(naxis, cdelt, crval) r = np.sqrt((xr)**2 + (yr)**2) # Pixel coordinates in image xpix, ypix = np.meshgrid(np.arange(xr.shape[0]), np.arange(yr.shape[1]), indexing='ij') backgroundPixels = np.sqrt((xpix - xr.shape[0] / 2.)**2 + (ypix - xr.shape[1] / 2.)**2) > xr.shape[0] / 3 # Calculate RMS from adjacent pairs rms = CalcRMS(tod) # Set up data containers crossings = np.zeros(nHorns) # Crossing points time = np.arange(nSamps) # Useful for interpolation if justOffsets: #P1 = np.zeros((nHorns, nParams)) #P1e = np.zeros((nHorns, nParams)) #chis = np.zeros((nHorns)) P1 = np.zeros((nHorns, nSidebands, nChans, nParams - 2)) errors = np.zeros((nHorns, nSidebands, nChans, nParams - 2)) Pestout = np.zeros((nHorns, nParams)) Pstdout = np.zeros((nHorns, nParams)) else: P1 = np.zeros((nHorns, nSidebands, nChans, nParams - 2)) errors = np.zeros((nHorns, nSidebands, nChans, nParams - 2)) Pestout = np.zeros((nHorns, nParams)) Pstdout = np.zeros((nHorns, nParams)) #fig = pyplot.figure(figsize=(16,16)) for i in range(nHorns): # Rotate the RA/DEC to the source centre x, y = Pointing.Rotate(ra[i, :], dec[i, :], clon, clat, -cpang[i, :]) r = np.sqrt((x)**2 + (y)**2) close = (r < 12.5 / 60.) if np.sum((r < 6. / 60.)) < 10: print('Source not observed') continue # Average the data into a single timestream print(tod.shape) if tod.shape[2] == 1: todTemp = tod[0, 0, 0, :] else: todTemp = np.mean( np.mean(tod[i, :, :], axis=0), axis=0) # average all data to get location of peak in data: removeNaN(todTemp) rmsTemp = CalcRMS(todTemp) try: todBackground = RemoveBackground(todTemp, rmsTemp, close, sampleRate=50, cutoff=1.) todTemp -= todBackground except (ValueError, IndexError): todTemp -= np.median(todTemp) offsets = 0 print(crval) m, hits = Mapping.MakeMapSimple(todTemp, x, y, wcs) resid = todTemp[:todTemp.size // 2 * 2:2] - todTemp[1:todTemp.size // 2 * 2:2] residmap, rh = Mapping.MakeMapSimple(resid, x[:(todTemp.size // 2) * 2:2], y[:(todTemp.size // 2) * 2:2], wcs) m = m / hits residmap = residmap / rh mapNoise = np.nanstd(residmap) / np.sqrt(2) m -= np.nanmedian(m) m[np.isnan(m)] = 0. ipix = Mapping.ang2pixWCS(wcs, x, y).astype('int') gd = (np.isnan(ipix) == False) & (ipix >= 0) & (ipix < m.size) # Get an estimate of the peak location x0, y0, xpix0, ypix0 = ImagePeaks(m, xr, yr, mapNoise) if isinstance(x0, type(None)): print('No peak found') continue # Just select the near data and updated peak location r = np.sqrt((x - x0)**2 + (y - y0)**2) close = (r < 12.5 / 60.) near = (r < 25. / 60.) & (r > 15 / 60.) far = (r > 30. / 60.) fitselect = (r < 10. / 60.) & (np.isnan(todTemp) == False) plotselect = (r < 45. / 60.) if np.sum(fitselect) < 20: continue fitdata = todTemp[fitselect] fitra = x[fitselect] fitdec = y[fitselect] if mode == 'mode2': P0 = [ np.max(fitdata) - np.median(fitdata), 4. / 60. / 2.355, 4. / 60. / 2.355, x0, y0, np.median(fitdata) ] print(P0, lnprior(P0)) fout = leastsq(ErrorLstSq, P0, args=(fitra, fitdec, fitdata, 0, 0), full_output=True) #P0 = fout[0] ndim, nwalkers = len(P0), 100 #pos = np.zeros((nwalkers, ndim)) #pos[:,0] = np.abs(P0[0])*1e-4*np.random.randn(nwalkers) #pos[:,1:3] = np.abs(P0[1:3])[np.newaxis,:]*1e-4*np.random.randn((nwalkers,2)) ###pos[:,3:5] = np.abs(P0[3:5])[np.newaxis,:]+0.1*np.random.randn((nwalkers,2)) #pos[:,5] = np.abs(P0[5])*1e-4*np.random.randn(nwalkers) #pos = pos.T pos = [ np.array(P0) + 1e-4 * np.random.randn(ndim) for iwalker in range(nwalkers) ] sampler = emcee.EnsembleSampler(nwalkers, ndim, lnprob, args=(fitra, fitdec, fitdata, rmsTemp)) sampler.run_mcmc(pos, 1200) samples = sampler.chain[:, 500:sampler.chain.shape[1]:3, :].reshape( (-1, ndim)) print(samples.shape) Pest = np.mean(samples, axis=0) Pstd = np.std(samples, axis=0) #Pest = fout[0] chi2 = np.sum((fitdata - Gauss2d2FWHM(Pest, fitra, fitdec, 0, 0))**2 / rmsTemp**2) / (fitdata.size - len(Pest)) print(np.std(samples, axis=0)) if doPlots: pyplot.plot(fitdata, label='data') pyplot.plot(Gauss2d2FWHM(Pest, fitra, fitdec, 0, 0), label='fit') pyplot.legend(loc='upper right') pyplot.ylabel('T (K)') pyplot.xlabel('Sample') pyplot.text(0.05, 0.9, r'$\chi^2$=' + '{:.2f}'.format(chi2), transform=pyplot.gca().transAxes) pyplot.title('Horn {:d} obs {}'.format(i + 1, prefix)) pyplot.savefig('PeakFitPlots/PeakFits_Horn{:d}_{}.png'.format( i + 1, prefix), bbox_inches='tight') pyplot.clf() fig = corner.corner(samples) pyplot.title('Horn {:d} obs {}'.format(i + 1, prefix)) pyplot.savefig('PeakFitPlots/Corner_Horn{:d}_{}.png'.format( i + 1, prefix), bbox_inches='tight') pyplot.clf() del fig # pyplot.figure() #pyplot.plot(samples[:,0]) #pyplot.show() if justOffsets: #P1[i,:] = Pest #P1e[i,:] = np.std(samples, axis=0) Pestout[i, :] = Pest Pstdout[i, :] = Pstd #chis[i] = chi2 continue else: Pestout[i, :] = Pest Pstdout[i, :] = Pstd print(x0, y0) siga, sigb = Pest[1:3] x0, y0 = Pest[3:5] print(x0, y0) for j in range(nSidebands): for k in range(nChans): try: todBackground = RemoveBackground(tod[i, j, k, :], rms[i, j, k], close, sampleRate=50, cutoff=0.1) except (IndexError, ValueError): todBackground = np.median(tod[i, j, k, :]) tod[i, j, k, :] -= todBackground fitdata = tod[i, j, k, fitselect] fitra = x[fitselect] fitdec = y[fitselect] amax = np.argmax(fitdata) if mode == 'mode1': P0 = [ np.max(fitdata) - np.median(fitdata), 4. / 60. / 2.355, x0, y0, np.median(fitdata) ] fout = leastsq(ErrorLstSq, P0, args=(fitra, fitdec, fitdata, 0, 0), full_output=True) fbootmean, fbootstd = Bootstrap(P0, fitdata, fitra, fitdec, ErrorLstSq) fitModel = Gauss2d elif mode == 'mode2': P0 = [ np.max(fitdata) - np.median(fitdata), Pest[1], Pest[2], np.median(fitdata) ] fout = leastsq(ErrorLstSq2FWHM, P0, args=(fitra, fitdec, fitdata, x0, y0), full_output=True) fbootmean, fbootstd = Bootstrap(P0, fitdata, fitra, fitdec, ErrorLstSq2FWHM) fitModel = Gauss2d2FWHMFixed else: print('Warning: No fitting method selected') if isinstance(fout[1], type(None)): continue #fout = np.mean(samples,axis=0), P1[i, j, k, :] = fout[0] errors[i, j, k, :] = fbootstd #pyplot.plot(fitdata-fitModel(fout[0], fitra, fitdec, x0,y0), label='data') #pyplot.legend(loc='upper right') #pyplot.ylabel('T (K)') #pyplot.xlabel('Sample') #pyplot.text(0.05,0.9, r'$\chi^2$='+'{:.2f}'.format(chi2), transform=pyplot.gca().transAxes) #pyplot.title('Horn {:d} obs {}'.format(i+1, prefix)) #pyplot.show() #pyplot.errorbar(np.arange(P1.shape[2]), P1[i,j,:,1]*60, yerr=errors[i,j,:,1]*60) #pyplot.show() cross = np.argmax( fitModel(np.median(P1[i, 0, :, :], axis=0), x, y, x0, y0)) print(cross, nHorns) crossings[i] = cross if justOffsets: return P1, errors, Pestout, Pstdout, crossings # P1, P1e, chis else: return P1, errors, Pestout, Pstdout, crossings
initTemp = pygame.time.get_ticks() seconde = int(initTemp / 1000) return seconde couleurFond = (176, 224, 230) X = pygame.display.Info().current_w Y = pygame.display.Info().current_h events = pygame.event.get() fenetre = pygame.display.set_mode((0, 0), pygame.FULLSCREEN) plyManager = Player() Mapping.MAP = Mapping.InitMap(fenetre, Mapping.MAP) seconde = 0 Jeu = True while Jeu: events = pygame.event.get() fenetre.fill(couleurFond) seconde = Time(seconde) if plyManager.IsPlaying == False: #DimissPopup menu.activesPop = menu.DimissPopup(menu.activesPop, events) #drawMap Mapping.GRID, Mapping.caseWid, Mapping.caseLen = Mapping.drawMap( X, Y, Mapping.nbreColonne, Mapping.nbreLigne, fenetre)
"(but not in a two-color way)") parser.add_option('-m', dest='map', default=False, help="Input the map to estimate positions") parser.add_option('-o', dest='other', default='655') (opts, args) = parser.parse_args() flagdict = defaultdict((lambda : 'ERR'), {'1': '655', '2':'585'}) outfile = None framesets = [] markernums = defaultdict(int) print glob(args[0] + '*') filenum = 0 if opts.map is not False: import Mapping mapping = Mapping.loadmapping(opts.map) for fname in glob(args[0] + '*'): XMLTree = minidom.parse(fname) # Read out frame numbers from the file name if 'frame' in fname: frames = fname.split('frame')[1].rsplit('.xml')[0] startframe, endframe = map(int, frames.split('-')) framesets.append((startframe, endframe)) else: framenums = map(int, getAllData(XMLTree, 'MarkerZ')) startframe = min(framenums) endframe = max(framenums) framesets = getFrameSets(XMLTree, opts.color)
def workflow(args : list): """Main workflow of backward.py Parameters ---------- args : List(Str) Commandline arguments """ global options options = [ # option type number default description ("-f", Option(str, 1, None, "Input GRO/PDB structure")), ("-o", Option(str, 1, None, "Output GRO/PDB structure")), ("-raw", Option(str, 1, None, "Projected structure before geometric modifications")), # ("-c", Option(str, 1, None, "Output GRO/PDB structure of expanded CG beads for position restraints")), ("-n", Option(str, 1, None, "Output NDX index file with default groups")), ("-p", Option(str, 1, None, "Atomistic target topology")), ("-po", Option(str, 1, None, "Output target topology with matching molecules list")), ("-pp", Option(str, 1, None, "Processed target topology, with resolved #includes")), ("-atomlist", Option(str, 1, None, "Atomlist according to target topology")), ("-fc", Option(float, 1, 200, "Position restraint force constant")), ("-to", Option(str, 1, None, "Output force field")), ("-from", Option(str, 1, None, "Input force field")), ("-strict", Option(bool, 0, None, "Use strict format for PDB files")), ("-nt", Option(bool, 0, None, "Use neutral termini for proteins")), ("-sol", Option(bool, 0, None, "Write water")), ("-solname", Option(str, 1, "SOL", "Residue name for solvent molecules")), ("-kick", Option(float, 1, 0, "Random kick added to output atom positions")), ("-nopbc", Option(bool, 0, None, "Don't try to unbreak residues (like when having large residues in a small box)")), ("-mapdir", Option(str, 1, None, "Directory where to look for the mapping files")), ] if '-h' in args or '--help' in args: print("\n",__file__) print(desc or "\nSomeone ought to write a description for this script...\n") for thing in options: print(type(thing) != str and "%10s %s"%(thing[0],thing[1].description) or thing) print() sys.exit() # Convert the option list to a dictionary, discarding all comments options = dict([i for i in options if not type(i) == str]) # Process the command line - list the options that were given opts = [] while args: opts.append(args.pop(0)) options[opts[-1]].setvalue([args.pop(0) for i in range(options[opts[-1]].num)]) ## DONE PARSING ARGUMENTS ## ################################################################################ ## MAPPING ## ##### A. If a target topology was provided, read it in top = options["-p"] and Topology(options["-p"].value,out=options["-pp"].value) ##### B. Read in the structure struc = Structure(options["-f"].value,strict=options["-strict"].value) ##### C. Set the mapping dictionary # Convert force field tags to lower case # Default is backmapping from MARTINI to GROMOS53A6 # If to_ff == martini, default from_ff = gromos to_ff = options["-to"] and options["-to"].value.lower() or "gromos" if to_ff == "martini" and not options["-from"]: from_ff = "gromos" else: from_ff = options["-from"] and options["-from"].value.lower() or "martini" mapping = Mapping.Mapping(options["-mapdir"].value).get(source=from_ff,target=to_ff) backmapping = levels[from_ff] > levels[to_ff] reslist = mapping.keys() ##### D. Iterate over atoms to write out, based on residue names # Copy the residue list from the target topology # This gives a list we can pop from, while keeping # the original. # The solvent residues are skipped. topresidues = None if top: topresidues = [i for i in top.residues] if options["-atomlist"]: atm = open(options["-atomlist"].value,"w") topatm = [j for i in topresidues for j in i] atm.writelines("".join(["%6d %5s %5s\n"%(u,v[0],v[1]) for u,v in zip(range(1,len(topatm)+1),topatm)])) # Iterate over residues # If we are backmapping, we store the BB bead # positions, to generate a spline, which we use # afterwards to place the backbone atoms. # To set the positions, we use some bookkeeping # tricks for the atoms to place on, or relative # to, the spline. # The backbone list will end up being equal in # length to the number of (amino acid) residues. # It is processed afterwards to be three times # the length. Indices are used to indicate which # entry from the resulting interpolated spline # list need to be taken for the position, and an # offset (tuple) is added to control the placement # of hydrogens and oxygens to N/C. counter = 0 out = [] cg = [] raw = [] sol = [] ions = [] msgs = [] for residue,bb,nterm,cterm in zip(struc.residues,struc.backbone,struc.nterm,struc.cterm): counter += 1 # Ignore solvent molecules from the topology while topresidues and topresidues[0][0][1] in solvent_stuff: topresidues.pop(0) # Unpack first atom first, resn, resi, chain, x, y, z = residue[0] # Extract residue name and atom list resn = resn.strip() atoms = [i[0].strip() for i in residue] # Just read one residue from the CG structure # If we have a topology, we need to check whether # the residue we just read matches the next in the # topology. Several cases are possible: # # - The residuename is equal in both cases: # This is too easy! Just proceed and thank your deity. # # - The residues do not match, but the CG residuename # matches the AA moleculetype name: # This may happen with lipids, if the atomistic structure # is split in residues like in the De Vries model. # In this case, all residues corresponding to the molecule # need to be read from the topology, based on the chain # identifier. # # - The residuename does not match, but the residue does: # The residues should match, or at least the first # characters. # # - The residue does not match with either the residue or # moleculetype from the atomistic topology: # If the residue is solvent, then we leave the topology # untouched, and the atoms are generated based on the # mapping. # Check for solvent if resn in solvent.keys(): cx, cy, cz = residue[0][4:7] for atom, x, y, z in solvent[resn]: # Should add random rotation if atom in ion_stuff: atom = atoms[0] ions.append((atom,resn,counter,chain,cx+x,cy+y,cz+z)) # They are added at the end, which is safe, as the # ion position is taken from the CG bead position # anyway. else: # If we do not want solvent written then this is # a good time to break: the first atom of a stretch # of solvent. Note that this ensures that we write # ions if we have those. if not options["-sol"]: break resn = options["-solname"].value # Increase the counter if we have an oxygen. # A little hack to keep track of water molecules if atom[0] == "O": counter += 1 sol.append((atom,resn,counter,chain,cx+x,cy+y,cz+z)) # Go to next residue continue # Read a residue from the target topology if we have one # Read several if the mapping so requires # Make an atom list from the residues read if topresidues: # Check whether the CG residue corresponds to the next AA residue # or to the next moleculetype topres = topresidues.pop(0) if resn != topres[0][1] and resn == topres[0][7]: # Add residues based on chain id while topresidues and topresidues[0][0][3] == topres[0][3]: topres.extend(topresidues.pop(0)) topres = [i for j in range(mapnum.get(resn,1)) for i in topres] # Set the residue name to the moleculetype name topres[0][3] = topres[0][7] target = list(zip(*topres))[0] # Check for duplicate atom names if not len(target) == len(set(target)): print("The target list for residue %s contains duplicate names. Relying on mapping file."%resn) target = None else: target = None # Except for solvent, the residue name from a topology # takes precedence over the one from the structure. if top and topres[0][1] in mapping.keys(): resn = topres[0][1] # Check if the residue is in the list # or whether we have an ambiguity. # In that case the first part of the # residue proper is equal to what we have # and the atom lists should be equal if not resn in reslist: oldname = resn p = set(atoms) for i in reslist: if i.startswith(resn): if p == set([k for j in mapping[i].map.values() for k in j]): msg="Residue %s not found. Seems to match %s."%(resn,i) if not msg in msgs: print(msg) msgs.append(msg) resn = i break if resn == oldname: # Last resort ... Checking for partially matching atom lists for i in reslist: if i.startswith(resn): keys = mapping[i].map.values()+[mapping[i].prekeys] if p.issubset(set([k for j in keys for k in j])): msg="Residue %s not found. Seems to match %s."%(resn,i) if not msg in msgs: print(msg) msgs.append(msg) resn = i break if not resn in mapping.keys(): # If the residue is still not in the mapping list # then there is no other choice that to bail out raise ValueError("Residue not found in mapping dictionary: %s\n"%resn) o, r = mapping[resn].do(residue,target,bb,nterm,cterm,options["-nt"]) out.extend(o) raw.extend(r) ## Write out # Combine things out.extend(sol) out.extend(ions) raw.extend(sol) raw.extend(ions) # Write out # Title if backmapping: title = "Backmapped structure from MARTINI to %s\n"%options["-to"].value else: title = "Mapped structure from %s to MARTINI\n"%options["-from"].value write_gro(options["-o"].value, title, out, struc.groBoxString(), options["-kick"].value) if options["-raw"]: write_gro(options["-raw"].value, "Projected structure before modifications\n", raw, struc.groBoxString(), 0) ## Write the output topology if options["-p"] and options["-po"]: write_topology(options["-po"].value, options["-p"].value, sol, ions) ## Write an index file if options["-n"]: write_ndx(options["-n"].value, out, protein_stuff, solvent_stuff)
minRa = np.min(ra) maxRa = np.max(ra) minDec = np.min(dec) maxDec = np.max(dec) if (crval[0] < minRa) | (crval[0] > maxRa) | ( crval[1] < minDec) | (crval[1] > maxDec): print( 'WARNING: MAP CENTRE DOES NOT MATCH TELESCOPE POINTING CENTRE. CHECK COORDINATES' ) print('MEAN RA: {:.2f}, MEAN DEC: {:.2f}'.format( np.mean(ra), np.mean(dec))) #pyplot.plot(ra[0,:], dec[0,:]) #pyplot.show() wcs, _, _ = Mapping.DefineWCS(naxis, cdelt, crval) pix = Mapping.ang2pixWCS(wcs, dec[0, :], ra[0, :]).astype('int') print(tod[0, 0, 0, :].shape, pix.shape) nhorns = tod.shape[0] nsidebands = tod.shape[1] nchans = tod.shape[2] m = np.zeros((nhorns, nsidebands, nchans, naxis[0] * naxis[1])) # for i in range(nhorns): # pix = Mapping.ang2pixWCS(wcs, dec[i,:], ra[i,:]).astype('int')# # for j in range(nsidebands): # for k in range(nchans): # print(tod.shape, pix.shape, obs.shape) # m[i,j,k,:],a0 = Mapping.Destripe(tod[i,j,k,:], pix, obs, int(5/0.02), int(naxis[0]*naxis[1]))
def FitTOD(tod, ra, dec, obs, clon, clat, prefix='', destripe=False): # Beam solid angle aken from James' report on wiki Optics nubeam = np.array([26., 33., 40.]) srbeam = np.array([2.1842e-6, 1.6771e-6, 1.4828e-6]) pmdl = interp1d(nubeam, srbeam) # straight interpolation nHorns = tod.shape[0] nSidebands = tod.shape[1] nChans = tod.shape[2] nSamps = tod.shape[3] nParams = 5 + 0# 2 wcs, xr, yr = Mapping.DefineWCS(naxis, cdelt, crval) # Crossing indices crossings = np.zeros(nHorns) # Calculate RMS from adjacent pairs splitTOD = (tod[:,:,:,:(nSamps//2) * 2:2] - tod[:,:,:,1:(nSamps//2)*2:2]) rms = np.std(splitTOD,axis=3)/np.sqrt(2) t = np.arange(nSamps) P1 = np.zeros((nHorns, nSidebands, nChans, nParams+1)) errors = np.zeros((nHorns, nSidebands, nChans, nParams)) fig = pyplot.figure(figsize=(16,16)) # Rotate the ra/dec width = 4. pixwidth = 2./60. nbins = int(width/pixwidth) xygrid = [np.linspace(-width/2, width/2,nbins+1), np.linspace(-width/2, width/2.,nbins+1)] xgrid, ygrid = np.meshgrid(np.linspace(-width/2, width/2,nbins)+pixwidth/2., np.linspace(-width/2, width/2.,nbins) + pixwidth/2.) for i in range( nHorns): x, y = Pointing.Rotate(ra[i,:], dec[i,:], clon, clat, 0) # Bin the map up, apply a filter to remove spikes, to give first guess at pixel centre. hmap = np.histogram2d(y,x, xygrid)[0] todTemp = np.median(np.median(tod[i,:,:,:],axis=0),axis=0) rmsTemp = np.std(todTemp[1:todTemp.size//2 *2:2] - todTemp[:todTemp.size//2 * 2:2])/np.sqrt(2) smap = np.histogram2d(y,x, xygrid, weights = todTemp)[0] xpix = (x+width/2)//pixwidth ypix = (y+width/2)//pixwidth #pyplot.plot(xpix, ypix, 'o') #pyplot.figure() #pyplot.plot(x, y, 'o') #pyplot.show() if destripe: pixels = (ypix + nbins*xpix).astype(int) offset = 300 gd = (pixels > 0) & (pixels < nbins*nbins-1) m, offsets = Mapping.Destripe(todTemp[gd], pixels[gd], obs[gd], offset, nbins*nbins) m = np.reshape(m, (nbins, nbins)).T #m = smap/hmap else: #h, wx, wy = np.histogram2d(x,y, (xygrid[0], xygrid[1])) #s, wx, wy = np.histogram2d(x,y, (xygrid[0], xygrid[1]), weights=todTemp) #m = s/h offsets =0 m, hits = Mapping.MakeMapSimple(todTemp, x, y, wcs) m = m/hits m -= np.nanmedian(m) #m /= rmsTemp m[np.isnan(m)] = 0. #pyplot.plot(ra[i,:], todTemp-offsets,'.') #pyplot.figure() #pyplot.plot(dec[i,:], todTemp-offsets,'.') #pyplot.show() r = np.sqrt((xgrid)**2 + (ygrid)**2) d1 = ImagePeaks(m, r, 4) # if len(coords1) > 0: # ximax, yimax = coords1[0] # else: # m = median_filter(m, size=(2,2)) # Remove background? xgrid2,ygrid2 = np.meshgrid(np.linspace(-d1.shape[0]/2, d1.shape[0]/2, d1.shape[0]), np.linspace(-d1.shape[1]/2, d1.shape[1]/2, d1.shape[1])) background = np.sqrt((xgrid2 - 0)**2 + (ygrid2 - 0)**2) > 25 print(d1.shape, xgrid.shape) d1[background] = 0 ximax, yimax = np.unravel_index(np.argmax(d1), m.shape) print(ximax, yimax, np.max(d1)) #pyplot.imshow(m) #pyplot.plot(yimax, ximax,'ow') #pyplot.figure() #pyplot.imshow(d1) #pyplot.plot(yimax, ximax,'ow') #pyplot.show() # +/- 180 x[x > 180] -= 360 #pyplot.plot(x,y) #pyplot.figure() #pyplot.plot(az[i,:], el[i,:]) #pyplot.show() #xbins = np.linspace(-4, 4, 60*8+1) #ybins = np.linspace(-4, 4, 60*8+1) # Just select the near data y0,x0 = -xr[ximax,yimax], yr[ximax,yimax] print(x0, y0) #print(x0, y0) #pyplot.figure(figsize=(6,6)) #pyplot.plot(yr.flatten(),m.flatten(),'-') #pyplot.axvline(y0,color='r') #pyplot.show() background = np.sqrt((x - x0)**2 + (y - y0)**2) > 30./60. #x0, y0 = 0, 0 #pyplot.plot(x,tod[0,0,0,:]) #pyplot.axvline(x0) #pyplot.axvline(x[np.argmax(tod[0,0,0,:])],color='r') #pyplot.figure() #pyplot.plot(tod[0,0,0,:]) #pyplot.show() r = np.sqrt((x-x0)**2 + (y-y0)**2) close = (r < 12.5/60.) near = (r < 25./60.) & (r > 15/60.) far = (r > 15./60.) fitselect = (r < 20./60.) time = np.arange(tod.shape[-1]) #pyplot.plot(time[:], tod[0,0,0,:]) #pyplot.plot(time[fitselect], tod[0,0,0,fitselect]) #pyplot.show() #extent = [-naxis[0]/2. * cdelt[0], naxis[0]/2. * cdelt[0], # -naxis[1]/2. * cdelt[1], naxis[1]/2. * cdelt[1]] #pyplot.figure(figsize=(6,6)) #m2 = m*0 #r2 = np.sqrt((xr-x0)**2 + (yr-y0)**2) #m2 = (r2 < 10./60.) #pyplot.imshow(m.T,extent=extent,origin='lower') #pyplot.scatter(x0,y0, marker='o',color='r') #pyplot.figure() #pyplot.imshow(m2.T,extent=extent,origin='lower') #pyplot.scatter(x0,y0, marker='o',color='r') #pyplot.show() #pyplot.plot(time[:],todTemp[:],'.') #pyplot.plot(time[fitselect],todTemp[fitselect],'.') #pyplot.show() plotselect = (r < 120./60.) for j in range(nSidebands): for k in range(nChans): rmdl = np.poly1d(np.polyfit(time[background], tod[i,j,k,background]-offsets,11)) todBackground =RemoveBackground(tod[i,j,k,:], rms[i,j,k], close, sampleRate=50, cutoff=1.) if destripe: m, offsets = Mapping.Destripe(tod[i,j,k,gd], pixels[gd], obs[gd], offset, nbins*nbins) m = np.reshape(m, (nbins,nbins)).T else: m, hits = Mapping.MakeMapSimple(tod[i,j,k,:], x, y, wcs) m = m/hits #tod[i,j,k,:] = tod[i,j,k,:] -offsets #-(rmdl(ra[i,:])+dmdl(dec[i,:])) tod[i,j,k,:] -= todBackground#np.median(tod[i,j,k,:]) fitdata = tod[i,j,k,fitselect] fitra = x[fitselect] fitdec= y[fitselect] amax = np.argmax(fitdata) P0 = [np.max(fitdata) -np.median(fitdata) , 4./60./2.355, 4./60./2.355, np.median(fitdata), x0, y0, 0., 0., 0.]#,0.,0., 0., 0.] P0 = [np.max(fitdata) -np.median(fitdata) , 4./60./2.355, x0, y0, np.median(fitdata), 0., 0.]#,0.,0., 0., 0.] P0 = [np.max(fitdata) -np.median(fitdata) , 4./60./2.355, x0, y0, np.median(fitdata)]#,0.,0., 0., 0.] #P1[i,j,k,:nParams], cov_x, info, mesg, s = leastsq(Error, P0, args=(fitra, fitdec, fitdata, 0,0), full_output=True) #fout = fmin(ErrorFmin, P0, maxfun=3000, args=(fitra, fitdec, fitdata, 0,0), full_output=True) #fout = leastsq(ErrorLstSq, P0, args=(fitra, fitdec, fitdata, 0,0), full_output=True) ndim, nwalkers = len(P0), 100 pos = [np.array(P0) + 1e-4*np.random.randn(ndim) for iwalker in range(nwalkers)] sampler = emcee.EnsembleSampler(nwalkers, ndim, lnprobNoGrad, args=(fitra, fitdec, fitdata, rms[i,j,k])) sampler.run_mcmc(pos, 1200) samples = sampler.chain[:, 500:sampler.chain.shape[1]:3, :].reshape((-1, ndim)) fout = np.mean(samples,axis=0), P1[i,j,k,:nParams] = fout[0] print(fout[0]) print (P0) print(x0, y0) #import corner #pyplot.clf() #fig = corner.corner(samples) #pyplot.show() P2 = P1[i,j,k,:nParams]*1. P2[0] = 0. ntod = Gauss2dNoGrad(P1[i,j,k,:nParams], x, y, 0,0) nulltod = Gauss2dNoGrad(P2, x, y, 0.,0.) otod = tod[i,j,k,:] omaps, hits = Mapping.MakeMapSimple(otod, x, y, wcs) nmaps, hits = Mapping.MakeMapSimple(ntod, x, y, wcs) nulls, hits = Mapping.MakeMapSimple(nulltod, x, y, wcs) xgrid, ygrid = np.meshgrid(np.arange(naxis[0]), np.arange(naxis[1])) xgrid = (xgrid-naxis[0]/2.) ygrid = (ygrid-naxis[1]/2.) rgrid = np.sqrt(xgrid**2 + ygrid**2) getchi = (rgrid < 5) nearChi = (rgrid > 7.5) & (rgrid < 15) maprms = np.nanstd((nmaps[nearChi]-omaps[nearChi])/hits[nearChi]) rmap = (omaps[getchi]-nmaps[getchi])/hits[getchi] mapChi = np.nansum((rmap-np.mean(rmap))**2/maprms**2)/(nParams-1.) mapOrig = np.nansum(((omaps[getchi]-nulls[getchi])/hits[getchi])**2/maprms**2)/(nParams-1.) origChi = np.sum((tod[i,j,k,plotselect] - Gauss2dNoGrad(P2, x[plotselect], y[plotselect], 0.,0.))**2/rms[i,j,k]**2)/(nParams-1.) reducedChi = np.sum((tod[i,j,k,plotselect] - Gauss2dNoGrad(P1[i,j,k,:nParams], x[plotselect], y[plotselect], 0.,0.))**2/rms[i,j,k]**2)/(nParams-1.) #print('CHI2', origChi, reducedChi, mapChi, mapOrig, reducedChi/origChi) P1[i,j,k,nParams] = rms[i,j,k] if (k >= 0): #r2 = np.sqrt((x-P1[i,j,k,2])**2 + (y-P1[i,j,k,3])**2) #plotselect = (r2 < 15./60.) ax = fig.add_subplot(3,1,1) todPlot = tod[i,j,k,plotselect] todPlot -= np.median(todPlot) pyplot.plot(todPlot-Gauss2dNoGrad(P1[i,j,k,:nParams], x[plotselect], y[plotselect], 0,0)) pyplot.title('Horn {}, Sideband {}, Avg. Channel {} \n {}'.format(i,j,k, prefix)) pyplot.xlabel('Sample') pyplot.ylabel('Detector Units') pyplot.text(0.9,0.9,r'$rms$ = '+'{:.3f}'.format(rms[i,j,k]), ha='right', transform=ax.transAxes) ax = fig.add_subplot(3,1,2) pyplot.plot(todPlot) P3 = P1[i,j,k,:nParams]*1. P3[3] = 0 P3[7:8] = 0 p3Model = Gauss2dNoGrad(P3, x[plotselect], y[plotselect], 0,0) pyplot.plot(Gauss2dNoGrad(P1[i,j,k,:nParams], x[plotselect], y[plotselect], 0,0))#p3Model - np.nanmedian(p3Model)) pyplot.plot(Gauss2dNoGrad(P2, x[plotselect], y[plotselect], 0,0)) pyplot.xlabel('Sample') pyplot.ylabel('Detector Units') pyplot.text(0.9,0.9,r'$\chi^2$ = '+'{:.3f}'.format(mapChi), ha='right', transform=ax.transAxes) extent = [-naxis[0]/2. * cdelt[0], naxis[0]/2. * cdelt[0], -naxis[1]/2. * cdelt[1], naxis[1]/2. * cdelt[1]] ax = fig.add_subplot(3,3,7) ax.imshow(m, aspect='auto', extent=extent) ax.scatter(P1[i,j,k,2], P1[i,j,k,3], marker='o',color='r') ax = fig.add_subplot(3,3,8) ax.imshow(nmaps/hits, extent=extent) ax.scatter(P1[i,j,k,2], P1[i,j,k,3], marker='o',color='r') ax = fig.add_subplot(3,3,9) ax.imshow(nmaps/hits-m, extent=extent) ax.scatter(P1[i,j,k,2], P1[i,j,k,3], marker='o',color='r') #pyplot.tight_layout(True) #pyplot.show() pyplot.savefig('TODResidPlots/TODResidual_{}_H{}_S{}_C{}.png'.format(prefix, i,j,k), bbox_inches='tight') pyplot.clf() cross = np.argmax(Gauss2dNoGrad(np.median(P1[i,0,:,:nParams],axis=0), x, y, 0,0)) print( cross, nHorns) crossings[i] = cross return P1, errors, crossings
import Mapping, time #Define Resolution m = 1.0 dm = 0.1 cm = 0.01 mm = 0.001 if __name__ == '__main__': # time.sleep(10) # Mapping.collect_map(repeat=400) #save .npy map Mapping.convert_map( resulution=cm) #convert all .npy to .png resolution = 1cm
'onehot', 'onehot_shared', 'phonetic', 'onehot_and_phonetic' ]: representation = {} for lang in lang_pair: representation[lang] = args.representation else: representation = dict( [x.split(':') for x in args.representation.split(',')]) ## Print Representation and Mappings print 'Representation' print representation ### load the mapping mapping = {} shared_mapping_obj = Mapping.get_mapping_instance(shared_mapping_class) for lang in representation.keys(): if representation[lang] in ['phonetic', 'onehot_and_phonetic']: mapping[lang] = shared_mapping_obj elif representation[lang] == 'onehot_shared': mapping[lang] = shared_mapping_obj elif representation[lang] == 'onehot': mapping[lang] = Mapping.CharacterMapping() with open(mapping_dir + '/' + 'mapping_' + lang + '.json', 'r') as mapping_file: mapping[lang].load_mapping(mapping_file) ## Print Representation and Mappings print 'Mapping'
struc = Structure(options["-f"].value,strict=options["-strict"].value) ##### C. Set the mapping dictionary # Convert force field tags to lower case # Default is backmapping from MARTINI to GROMOS53A6 # If to_ff == martini, default from_ff = gromos to_ff = options["-to"] and options["-to"].value.lower() or "gromos" if to_ff == "martini" and not options["-from"]: from_ff = "gromos" else: from_ff = options["-from"] and options["-from"].value.lower() or "martini" mapping = Mapping.get(source=from_ff,target=to_ff) backmapping = levels[from_ff] > levels[to_ff] reslist = mapping.keys() ##### D. Iterate over atoms to write out, based on residue names # Copy the residue list from the target topology # This gives a list we can pop from, while keeping # the original. # The solvent residues are skipped. topresidues = None if top: topresidues = [i for i in top.residues] if options["-atomlist"]:
from PointCloud import * def signal_handler(signal, frame): print('Ctrl-C pressed') sys.exit(0) signal.signal(signal.SIGINT, signal_handler) print('Ctrl-C to close program') # initializations model = Camera() # camera model ds_cam = DatasetCamera() tr = Tracking() # tracking module mp = Mapping() # mapping module pc = PointCloud() # point cloud rospy.init_node('cloud_stream', anonymous=True) # video capture object # cap = cv2.VideoCapture(0) for im_id in xrange(108): im = cv2.imread('./dataset/image_00/data/' + str(im_id).zfill(10) + '.png', 0) if im_id == 0: tr.kp_old = detector.detect(im) tr.kp_old = np.array([x.pt for x in tr.kp_old], dtype=np.float32) tr.addView(im)
def convert_address_to_latlng(address_string): return Mapping.address_to_geo(address_string)
def raw_data_to_keys(extracted_data): return ''.join(map(Mapping.Mapping().raw_to_key, extracted_data))
def closeGripper(self): servoAngle = Mapping.getGripAngle(s.closeGripper) Servo.gripperServo(s.gripperPin, int(servoAngle)) time.sleep(1) s.gripAngle = s.closeGripper
def run_train(args): """ Training script """ ## check for required parameters if args.lang is None: print 'ERROR: --lang has to be set' sys.exit(1) if args.data_dir is None and args.mode == 'train': print 'ERROR: --data_dir has to be set' sys.exit(1) if args.data_dir is None and args.mode == 'test': print 'ERROR: --output_dir has to be set' sys.exit(1) # Create output folders if required models_dir = args.output_dir + '/models/' mappings_dir = args.output_dir + '/mappings/' log_dir = args.output_dir + '/log/' for folder in [models_dir, mappings_dir]: if not os.path.exists(folder): os.makedirs(folder) ####################################### # Reading data and creating mappings # ####################################### # Creating mapping object to store char-id mappings mapping = {} mapping = Mapping.get_mapping_instance(args.mapping_class) if args.use_mapping is not None: print 'Using existing vocabulary' with open(args.use_mapping, 'r') as mapping_json_file: mapping.load_mapping(mapping_json_file) print 'Start Reading Data' train_data = MonoDataReader.MonoDataReader( args.lang, args.data_dir + '/train.' + args.lang, mapping, args.max_seq_length) ## complete vocabulary creation if args.use_mapping is None: mapping.finalize_vocab() with open(mappings_dir + '/mapping_{}.json'.format(args.lang), 'w') as mapping_json_file: mapping.save_mapping(mapping_json_file) ## Print Representation and Mappings print 'Mapping' print mapping print 'Vocabulary Statitics' print '{}: {}'.format(args.lang, mapping.get_vocab_size()) sys.stdout.flush() # Reading Validation data valid_data = MonoDataReader.MonoDataReader( args.lang, args.data_dir + '/tun.' + args.lang, mapping, args.max_seq_length) ## Reading test data test_data = MonoDataReader.MonoDataReader( args.lang, args.data_dir + '/test.' + args.lang, mapping, args.max_seq_length) print 'Finished Reading Data' ################################################################### # Interacting with model and creating computation graph # ################################################################### # Creating Model object model = LanguageModel(args.lang, mapping, args.representation, args.max_seq_length, args.embedding_size, args.rnn_size) ## Creating placeholder for sequences, masks and lengths and dropout keep probability pl_batch_sequences = tf.placeholder(shape=[None, args.max_seq_length], dtype=tf.int32) pl_batch_sequence_lengths = tf.placeholder(shape=[None], dtype=tf.float32) optimizer_op = model.get_optimizer(pl_batch_sequences, pl_batch_sequence_lengths, args.learning_rate, args.dropout_keep_prob) tf.get_variable_scope().reuse_variables() loss_op = model.average_loss(pl_batch_sequences, pl_batch_sequence_lengths, 1.0) print "Done with creating graph. Starting session" sys.stdout.flush() #Saving model saver = tf.train.Saver(max_to_keep=0) final_saver = tf.train.Saver() #Start Session config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) sess.run(tf.initialize_all_variables()) if (args.start_from is not None): saver.restore( sess, '{}/models/model-{}'.format(args.output_dir, args.start_from)) completed_epochs = args.start_from tf.train.SummaryWriter(log_dir, sess.graph) print "Session started" # Fractional epoch: stores what fraction of each dataset is used till now after last completed epoch. fractional_epochs = 0.0 completed_epochs = 0 steps = 0 prev_steps = 0 validation_losses = [] epoch_train_time = 0.0 epoch_train_loss = 0.0 start_time = time.time() # Whether to continue or now cont = True print 'Starting training ...' while cont: # Selected the dataset whose least fraction is used for training in current epoch ### TRAIN update_start_time = time.time() sequences,sequence_masks,sequence_lengths, = \ train_data.get_next_batch(args.batch_size) _, step_loss = sess.run(optimizer_op, feed_dict={ pl_batch_sequences: sequences, pl_batch_sequence_lengths: sequence_lengths }) fractional_epochs += float(len(sequences)) / train_data.num_words epoch_train_loss += step_loss update_end_time = time.time() epoch_train_time += (update_end_time - update_start_time) # One more batch is processed steps += 1 # If all datasets are used for training epoch is complete if (fractional_epochs >= 1.0): # update epoch number completed_epochs += 1 ### VALIDATION LOSS # Find validation loss valid_start_time = time.time() validation_loss = get_average_loss(valid_data, loss_op, pl_batch_sequences, pl_batch_sequence_lengths, sess) validation_losses.append(validation_loss) valid_end_time = time.time() epoch_valid_time = (valid_end_time - valid_start_time) ## TEST LOSS test_start_time = time.time() test_loss = get_average_loss(test_data, loss_op, pl_batch_sequences, pl_batch_sequence_lengths, sess) test_end_time = time.time() epoch_test_time = (test_end_time - test_start_time) print "Epochs Completed : " + str(completed_epochs).zfill( 3) + "\t Training loss: " + str(epoch_train_loss / (steps - prev_steps)) print "Epochs Completed : " + str(completed_epochs).zfill( 3) + "\t Validation loss: " + str(validation_loss) print "Epochs Completed : " + str(completed_epochs).zfill( 3) + "\t Test loss: " + str(test_loss) # #### Comment it to avoid early stopping # ## If validation loss is increasing since last 3 epochs, take the last 4th model and stop training process # #if(completed_epochs>=4 and len(validation_losses)>=4 and all([i>j for (i,j) in zip(validation_losses[-3:],validation_losses[-4:-1])])): # # completed_epochs -= 3 # # saver.restore(sess,models_dir+'my_model-'+str(completed_epochs)) # # cont = False # If max_epochs are done if (completed_epochs >= args.max_epochs): cont = False saver.save(sess, models_dir + 'model', global_step=completed_epochs) print "Epochs Completed : "+str(completed_epochs).zfill(3)+ \ "\t Time (hh:mm:ss)::: train> {} valid> {} test> {}".format( utilities.formatted_timeinterval(epoch_train_time), utilities.formatted_timeinterval(epoch_valid_time), utilities.formatted_timeinterval(epoch_test_time), ) ## update epoch variables prev_steps = steps fractional_epochs = 0.0 epoch_train_time = 0.0 epoch_train_loss = 0.0 print "Epochs Completed : "+str(completed_epochs).zfill(3)+ \ "\t Number of training steps: {}".format(steps) print "Epochs Completed : "+str(completed_epochs).zfill(3)+ \ "\t Time of completion: {}".format(time.asctime()) sys.stdout.flush() # save final model final_saver.save( sess, args.output_dir + '/final_model_epochs_' + str(completed_epochs)) print 'End training' print 'Final number of training steps: {}'.format(steps) #### Print total training time end_time = time.time() print 'Total Time for Training (hh:mm:ss) : {}'.format( utilities.formatted_timeinterval(end_time - start_time))
def FitTOD(tod, ra, dec, clon, clat, cpang, prefix='', normalize=True, plotDir=None): """ args: tod - ra - dec - clon - clat - cpang - kwargs: prefix - destripe - normalize - justOffsets - """ # Define the pixel grid # Pixel coordinates on sky wcs, xr, yr = Mapping.DefineWCS(naxis, cdelt, crval) r = np.sqrt((xr)**2 + (yr)**2) # Pixel coordinates in image xpix, ypix = np.meshgrid(np.arange(xr.shape[0]), np.arange(yr.shape[1]), indexing='ij') # Calculate RMS from adjacent pairs rms = CalcRMS(tod) # Rotate the RA/DEC to the source centre x, y = Pointing.Rotate(ra, dec, clon, clat, -cpang) r = np.sqrt((x)**2 + (y)**2) close = (r < 3.) # Check if source is even with 3 degrees of field centre if np.sum((r < 6. / 60.)) < 10: print('Source not observed') return badval # Filter background or at least subtract a mean level try: todBackground = RemoveBackground(tod, rms, x, y, sampleRate=50, cutoff=1.) tod -= todBackground except (ValueError, IndexError): tod -= np.nanmedian(tod) # Create map of data centred on 0,0 ms, hits = Mapping.MakeMapSimple(tod, x, y, wcs) m = ms / hits # Calculate the pair subtracted TOD to creat a residual map residTod = tod[:tod.size // 2 * 2:2] - tod[1:tod.size // 2 * 2:2] residmap, rh = Mapping.MakeMapSimple(residTod, x[:(tod.size // 2) * 2:2], y[:(tod.size // 2) * 2:2], wcs) residmap = residmap / rh mapNoise = np.nanstd(residmap) / np.sqrt(2) m -= np.nanmedian(m) m[np.isnan(m)] = 0. # Get an estimate of the peak location x0, y0, xpix0, ypix0 = ImagePeaks(m, xr, yr, mapNoise) if isinstance(x0, type(None)): print('No peak found') return badval # Just select the near data and updated peak location # Probably should add some way of not having these be hardcoded... r = np.sqrt((x - x0)**2 + (y - y0)**2) close = (r < 12.5 / 60.) near = (r < 25. / 60.) & (r > 15 / 60.) far = (r > 30. / 60.) fitselect = (r < 10. / 60.) & (np.isnan(tod) == False) plotselect = (r < 45. / 60.) if np.sum(fitselect) < 20: return badval fitdata = tod[fitselect] fitra = x[fitselect] fitdec = y[fitselect] # Initial guesses for fit P0 = [ np.max(fitdata) - np.median(fitdata), 4. / 60. / 2.355, 4. / 60. / 2.355, x0, y0, np.median(fitdata) ] # Run mcmc fit: ndim, nwalkers = len(P0), 100 pos = [ np.array(P0) + 1e-4 * np.random.randn(ndim) for iwalker in range(nwalkers) ] sampler = emcee.EnsembleSampler(nwalkers, ndim, lnprob, args=(fitra, fitdec, fitdata, rms)) sampler.run_mcmc(pos, 1200) samples = sampler.chain[:, 500:sampler.chain.shape[1]:3, :].reshape( (-1, ndim)) Pest = np.mean(samples, axis=0) Pstd = np.std(samples, axis=0) chi2 = np.sum((fitdata - Gauss2d2FWHM(Pest, fitra, fitdec, 0, 0))**2 / rms**2) / (fitdata.size - len(Pest)) if not isinstance(plotDir, type(None)): pyplot.plot(fitdata, label='data') pyplot.plot(Gauss2d2FWHM(Pest, fitra, fitdec, 0, 0), label='fit') pyplot.legend(loc='upper right') pyplot.ylabel('T (K)') pyplot.xlabel('Sample') pyplot.text(0.05, 0.9, r'$\chi^2$=' + '{:.2f}'.format(chi2), transform=pyplot.gca().transAxes) pyplot.title(' {}'.format(prefix)) pyplot.savefig('{}/PeakFits_{}.png'.format(plotDir, prefix), bbox_inches='tight') pyplot.clf() #fig = corner.corner(samples) #pyplot.title('{}'.format(prefix)) #pyplot.savefig('{}/Corner_{}.png'.format(plotDir, prefix), bbox_inches='tight') #pyplot.clf() #del fig # Normalise by rms if normalize: ms /= Pest[0] # Output fits + sample of peak crossing cross = np.argmax(Gauss2d2FWHM(Pest, x, y, 0, 0)) return Pest, Pstd, cross, ms, hits, Gauss2d2FWHM( [1., Pest[1], Pest[2], Pest[3], Pest[4], 0], xr, yr, 0, 0) * outweight
import Android import Mapping import Manipulator import time xStart=6 yStart=7 xB=0 yB=0 myOrientation=-90 while True: #Wifi [xB,yB,C]=Android.getWifiData(xB,yB) #Mapping myOrientation=Mapping.drive(xStart,yStart,yB,xB,myOrientation) time.sleep(30) #Manipulator #Manipulator.grab(C) #2-Yelow Ball,1-Blue Ball #Mapping myOrientation=Mapping.drive(yB,xB,xStart,yStart,myOrientation)
crval = [0, 0] else: crval = [r0, d0] cdelt = np.array(json.loads(Parameters.get('Mapping','cdelt')))/60. minRa = np.min(ra) maxRa = np.max(ra) minDec = np.min(dec) maxDec = np.max(dec) if (crval[0] < minRa) | (crval[0] > maxRa) | (crval[1] < minDec) | (crval[1] > maxDec): print('WARNING: MAP CENTRE DOES NOT MATCH TELESCOPE POINTING CENTRE. CHECK COORDINATES') print('MEAN RA: {:.2f}, MEAN DEC: {:.2f}'.format(np.mean(ra), np.mean(dec))) wcs,_,_ = Mapping.DefineWCS(naxis, cdelt, crval) maps, hits = Mapping.MakeMaps(tod, ra, dec, wcs) dataout['hits'] = hits dataout['maps'] = maps dataout['naxis'] = np.array(naxis) dataout['cdelt'] = np.array(cdelt) dataout['crval'] = np.array(crval) sbStr = ''.join(str(e) for e in sidebands) hoStr = ''.join(str(e) for e in pixels) FileTools.WriteH5Py('{}/{}_{}_Horns{}_Sidebands{}.h5'.format(Parameters.get('Inputs', 'outputDir'), Parameters.get('Inputs', 'outputname'), prefix, hoStr, sbStr), dataout)
import Mapping # Define Start and Finish and the Starting Orientation xA=6 # x Column and Y Raw yA=7 xB=1 yB=7 myOrientation=-90 Mapping.drive(xA,yA,xB,yB,myOrientation)