def plot_table(path_dict, save_path): pd.set_option('display.precision', 4) pd.set_option('display.width', 40) pd.set_option('display.float_format', '{:,.3f}'.format) for i, act in enumerate(path_dict.keys()): row_labels, values_train, values_test = [], [], [] for path in path_dict[act]: with open(path + '/results.json', 'r') as f: results = json.load(f) if (plot_it is not None) and (results['combinator'] not in plot_it): continue if i == 0: col_labels = utils.fill_col_labels(results) temp_train, temp_test = utils.fill_row_values( results, path, act) values_train.append(temp_train) values_test.append(temp_test) if 'test_acc_hr_0.0' in results: temp_train, temp_test = utils.fill_row_values(results, path, act, hr=0.0) values_train.append(temp_train) values_test.append(temp_test) # create table table_train = utils.create_table(values_train, col_labels, act, 'train') table_test = utils.create_table(values_test, col_labels, act, 'test') # save table utils.save_table(table_train, table_test, save_path, act)
def create_database(args): file_struct.DEBUG = getattr(args,file_struct.debug_long) #Create tables in the database for i in range(0,len(file_struct.tables)): utils.create_table(file_struct.tables[i], file_struct.PKs[i],file_struct.foreign_key_relations[i]) #Add fields to each table in the database for j in range(0,len(file_struct.tables)): for i in range(0,(len(file_struct.table_fields[j]))): utils.add_field(file_struct.tables[j], file_struct.table_fields[j][i][0],file_struct.table_fields[j][i][1])
def render_cases(choice): if choice == 1: try: utils.create_database() utils.create_table() except Exception as e: print(e) print("UNABLE TO CREATE TABLE !!") finally: utils.input_and_save_user() print("Your membership has been created !! ") print("Have a nice day !") utils.display_main_menu() render_cases(int(input())) elif choice == 2: member_uuid = utils.login_member() if member_uuid: (book_name, fine) = utils.check_for_pending_books(member_uuid) if fine is not None: print("You have a pending book ") print("Book Name-> ", book_name) print("Due -> ", fine) utils.display_book_lending_options() handle_book_lend_cases(int(input()), member_uuid) else: print("User not found ! Please try again ") elif choice == 3: system('cls') print("Enter Username: "******"Enter Password: "******"admin" and admin_password == "admin") or utils.is_user_admin( admin_username, admin_password): system('cls') utils.display_admin_options() handle_admin_cases(int(input())) else: print("Invalid Credentials. Try again.") utils.display_main_menu() render_cases(int(input())) elif choice == 4: exit(0) else: print("Invalid Input !!")
def init_storage(self): #change: create the "storage" dir first, then do the rest utils.create_doc_dir(self.crawl_method, self.doc_paths) #create file storage dir self.url_file = open('storage/urls_{}.txt'.format(self.crawl_method), 'a') #create/connect to db file os.chdir('storage') self.conn = utils.create_conn(self.db_path) os.chdir('..') utils.create_table(self.conn, self.table_name) #create url table
def pretty_print(df): """ Display the sample in a appealing manner, given a DataFrame. \ This is only displayed prettily here, the `view` module only \ shows tables. """ return create_table(df)
def main(old_meo_obj, target_meo_obj): meo_features = [ 'temperature', 'dew_point', 'humidity', 'pressure', 'wind_speed', 'wind_bearing', 'cloud_cover', 'visibility' ] max_time = session.query(func.max( old_meo_obj.timestamp)).scalar().strftime('%Y-%m-%d %H:%M:%S') min_time = session.query(func.min( old_meo_obj.timestamp)).scalar().strftime('%Y-%m-%d %H:%M:%S') tz = pytz.timezone('America/Los_Angeles') time_df = pd.date_range(start=min_time, end=max_time, freq='1H') time_list = sorted(list(set([tz.localize(x) for x in time_df]))) print(len(time_list)) """ !!! Be careful, create table would overwrite the original table """ create_table(target_meo_obj) interpolate_time(old_meo_obj, target_meo_obj, time_list, meo_features)
def __create_table(self, num_players, dealer_index): chip_counts = [] for i in range(num_players): chip_counts.append(Currency(1000)) tuple = create_table(chip_counts, dealer_index) self.limit = tuple[0] self.table = tuple[1] self.players = tuple[2]
def classify_write(bucket_name, prefix, selected_pdf_folder, prediction_client, storage_client, bq_client, bq_dataset, bq_table, score_threshold, service_account, input_path, model_full_id): bucket = storage_client.bucket(bucket_name) params = {} lines = [] schema = [ bigquery.SchemaField('file', 'STRING', mode='REQUIRED'), bigquery.SchemaField('class', 'STRING', mode='REQUIRED'), bigquery.SchemaField('class_confidence', 'STRING', mode='REQUIRED'), ] table = utils.create_table(bq_client, bq_dataset, bq_table, schema) if score_threshold: params = {"score_threshold": str(score_threshold)} for blob in bucket.list_blobs(prefix=str(prefix + "/")): if blob.name.endswith(".png"): logger.info(os.path.basename(blob.name)) content = utils.sample_handler(storage_client, bucket_name, blob.name) payload = {"image": {"image_bytes": content}} response = prediction_client.predict(model_full_id, payload, params) for result in response.payload: logger.info("File location: {}".format( os.path.join('gs://', bucket_name, blob.name))) logger.info("Predicted class name: {}".format( result.display_name)) logger.info("Predicted class score: {}\n".format( result.classification.score)) if result.display_name == "datasheets": pass else: # Copy from the pdf folder to the selected_pdf_folder filename = os.path.basename(blob.name).replace( '.png', '.pdf') input_pdf_path = os.path.join(input_path, filename) selected_pdf_path = os.path.join(selected_pdf_folder, filename) bucket_input, blob_input = utils.get_bucket_blob( input_pdf_path) bucket_output, blob_output = utils.get_bucket_blob( selected_pdf_path) utils.copy_blob(bucket_input, blob_input, bucket_output, blob_output, service_account) rows_to_insert = [ (str(blob.name).replace(".png", ".pdf").replace( prefix, "").replace("/", ""), result.display_name, result.classification.score), ] load_job = bq_client.insert_rows(table, rows_to_insert)
def test_simple_serialize(self): tuple = create_table([1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000], 0) table = tuple[1] state = TableState(table) str = dumps(state) new_state = loads(str) self.assertEquals(state.name, new_state.name) self.assertEquals(10, len(new_state.seats))
def plot_table_attention(path_dict, save_path): for i, act in enumerate(path_dict.keys()): row_labels, values_train, values_test = [], [], [] if act not in COMBINED_ACT: continue for path in path_dict[act]: # print(act) with open(f'{path}/results.json', 'r') as f: results = json.load(f) if results['combinator'] not in ATT_LIST or results[ 'combinator'] not in plot_it: continue if i == 0: col_labels = utils.fill_col_labels(results, att=1) temp_train, temp_test = utils.fill_row_values(results, path, act, att=1) values_train.append(temp_train) values_test.append(temp_test) if 'test_acc_hr_0.0' in results: temp_train, temp_test = utils.fill_row_values(results, path, act, att=1, hr=0.0) values_train.append(temp_train) values_test.append(temp_test) # create table table_train = utils.create_table(values_train, col_labels, act, 'train', att=1) table_test = utils.create_table(values_test, col_labels, act, 'test', att=1) # save table utils.save_table(table_train, table_test, save_path + 'ATT_', act)
def plot_table_max(path_dict, save_path, limit): res_json = ['results.json', 'results_hr.json'] row_labels, values_train, values_test = [], [], [] for i, act in enumerate(path_dict.keys()): for path in path_dict[act]: for res in res_json: try: with open(f'{path}/{res}', 'r') as f: results = json.load(f) # att = 2 if res == 'results_hr.json' else 0 except Exception as e: continue if i == 0: col_labels = utils.fill_col_labels(results, max_=True, att=2) temp_train, temp_test = utils.fill_row_values(results, path, act, max_=True, att=2) if True not in np.where(temp_test[8] >= limit, True, False): continue values_train.append(temp_train) values_test.append(temp_test) # create table table_train = utils.create_table(values_train, col_labels, '', 'train', max_=True) table_test = utils.create_table(values_test, col_labels, '', 'test', max_=True) # save table utils.save_table(table_train, table_test, save_path, 'best')
def plot_table(path_dict, save_path): pd.set_option('display.precision', 4) pd.set_option('display.width', 40) for i, act in enumerate(path_dict.keys()): row_labels, values_train, values_test = [], [], [] for path in path_dict[act]: with open(path + '/results.json', 'r') as f: results = json.load(f) if i == 0: col_labels = utils.fill_col_labels(results) temp_train, temp_test = utils.fill_row_values( results, path, act) values_train.append(temp_train) values_test.append(temp_test) # create table table_train = utils.create_table(values_train, col_labels, act, 'train') table_test = utils.create_table(values_test, col_labels, act, 'test') # save table utils.save_table(table_train, table_test, save_path, act)
def create_database(args): if args.lite: if not os.path.exists(fs.SQLite_DB_path): os.mkdir(fs.SQLite_DB_path) if os.path.exists(fs.SQLite_DB_path + "/" + fs.DB_name): print("{0} already exists in {1} , exiting".format( fs.DB_name, fs.SQLite_DB_path)) exit() print("Creating {0} now".format(fs.DB_name)) fs.DEBUG = getattr(args, fs.debug_long) #Create tables in the database for i in range(0, len(fs.tables)): utils.create_table(fs.tables[i], fs.PKs[i], fs.foreign_key_relations[i], args) #Add fields to each table in the database for j in range(0, len(fs.tables)): for i in range(0, (len(fs.table_fields[j]))): utils.add_field(fs.tables[j], fs.table_fields[j][i][0], fs.table_fields[j][i][1], args)
def worker(*args, **kwargs): try: func(*args, **kwargs) except OperationalError as e: if "no such table" in str(e): logger.warning( "Tables are not initialized. Running SQL queries...") query = """ CREATE TABLE IF NOT EXISTS articles ( id INTEGER NOT NULL PRIMARY KEY, name TEXT NOT NULL, content TEXT NOT NULL, image TEXT NOT NULL, source TEXT NOT NULL, date TEXT NOT NULL ); """ create_table(query) logger.warning('Restarting script...') func(*args, **kwargs)
def plot_table_attention(path_dict, save_path): res_json = ['results.json', 'results_hr.json'] for i, act in enumerate(path_dict.keys()): row_labels, values_train, values_test = [], [], [] if act not in COMBINED_ACT: continue for path in path_dict[act]: # print(act) for res in res_json: try: with open(f'{path}/{res}', 'r') as f: results = json.load(f) # att = 1 if res == 'results_hr.json' else 0 except Exception as e: continue if results['combinator'] not in ATT_LIST: continue if i == 0: col_labels = utils.fill_col_labels(results, att=1) temp_train, temp_test = utils.fill_row_values(results, path, act, att=1) values_train.append(temp_train) values_test.append(temp_test) # create table table_train = utils.create_table(values_train, col_labels, act, 'train', att=1) table_test = utils.create_table(values_test, col_labels, act, 'test', att=1) # save table utils.save_table(table_train, table_test, save_path + 'ATT_', act)
def plot_table_max(path_dict, save_path, limit): row_labels, values_train, values_test = [], [], [] for i, act in enumerate(path_dict.keys()): for path in path_dict[act]: with open(f'{path}/results.json', 'r') as f: results = json.load(f) if (plot_it is not None) and (results['combinator'] not in plot_it): continue if i == 0: col_labels = utils.fill_col_labels(results, max_=True, att=2) temp_train, temp_test = utils.fill_row_values(results, path, act, max_=True, att=2) # print(temp_test[9]) if True not in np.where(temp_test[12] >= limit, True, False): continue values_train.append(temp_train) values_test.append(temp_test) # create table table_train = utils.create_table(values_train, col_labels, '', 'train', max_=True) table_test = utils.create_table(values_test, col_labels, '', 'test', max_=True) # save table utils.save_table(table_train, table_test, save_path, 'best')
def render_table(dataset_key): """ Create a display for the chosen dataset. Args: dataset_key (str): Value from the dropdown. It is the Redis \ key for the dataset. Returns: list: A list of dash components. """ if dataset_key is None: return [html.H4("Nothing selected.")] df = dill.loads(redis_conn.get(dataset_key)) return [ html.Br(), create_table(df), ]
def crawl_save_upload(): '''调用函数实现抓取、保存和上传数据文件''' print('-----数据抓取开始-----') wb = Workbook() engine, Base, session = get_mysql_connection() Order, Website = create_table(engine, Base) add_default_data(session, Website) codemart_crawler.main(wb, session, Order, Website) oschina_crawler.main(wb, session, Order, Website) rrkf_crawler.main(wb, session, Order, Website) shixian_crawler.main(wb, session, Order, Website) wywaibao_crawler.main(wb, session, Order, Website) yuanjisong_crawler.main(wb, session, Order, Website) print('-----数据抓取结束-----') print('-----文件保存开始-----') delete_data() now = datetime.now() file = r'data/%s.xlsx' % now.strftime("%Y-%m-%d %H-%M-%S") wb.save(file) time.sleep(3) print('-----文件保存结束-----') print('-----文件上传开始-----') media_id = get_media_id(file) if isinstance(media_id, str): upload_result = send_file(media_id) if upload_result == True: print('文件上传成功:%s' % file) else: message = '文件上传失败:%s' % upload_result[1] print(message) send_message(message) else: message = '获取media_id失败:%s' % media_id[1] print(message) send_message(message) print('-----文件上传结束-----')
r = sim_pdf.fitTo(dataset, RF.Save(True), RF.Minimizer("Minuit2", "Migrad"), RF.NumCPU(1), #RF.PrintLevel(3), #RF.Offset(True) ) for i, j in product(range(nBins_Y), range(nBins_PT)): Sig_Yields[i][j] = w.var("Nsig"+sample[i][j]).getVal() Sig_Yields_err[i][j] = w.var("Nsig"+sample[i][j]).getError() Bgr_Yields[i][j] = w.var("Ncombi"+sample[i][j]).getVal() Bgr_Yields_err[i][j] = w.var("Ncombi"+sample[i][j]).getError() utils.create_table(Sig_Yields, Sig_Yields_err, "Signal_Yields.txt",nBins_Y, nBins_PT) f_res = open("Fit_Result.txt", "w") for k in range(len(r.floatParsFinal())-1): f_res.write(str(r.floatParsFinal()[k])+"\n") f_res.write("\n") f_res.write("Fit status: "+str(r.covQual())+"\n") f_res.close() components = {"Signal": (RF.kSolid, R.kRed), "Background": (RF.kDashed, R.kOrange), } for i, j in product(range(nBins_Y), range(nBins_PT)): components = {"Signal"+sample[i][j]: (RF.kSolid, R.kRed),
def main(argv): print("starting db init") conn = utils.create_connection(db_file) utils.create_table(conn) get_messages(conn)
def test_world(): # Test World # ---------- env = UnityEnvironment(file_name=ENV) brain_name = env.brain_names[0] brain = env.brains[brain_name] env_info = env.reset(train_mode=True)[brain_name] num_agents = len(env_info.agents) print('\nNumber of Agents: ', num_agents) def take_actions(env_info, brain): num_agents = len(env_info.agents) action_size = brain.vector_action_space_size actions = np.random.randn(num_agents, action_size) actions = np.clip(actions, -1, 1) env_info = env.step(actions)[brain_name] next_states = env_info.vector_observations return env_info, actions, next_states action_size = brain.vector_action_space_size states = env_info.vector_observations state_vector_names = [ 'racket x pos', 'racket y pos', 'racket x velocity', 'racket y velocity', 'ball x pos', 'ball y pos', 'ball x velocity', 'ball y velocity' ] print('\nA state vector for one of the agent looks like:') state = states[0].reshape(3, 8) table = create_table(state, state_vector_names) print(table) # But only the last row provides new information to each state, so we could simply get those values print('\nKeeping only the last row would be: ') state0 = states[0].reshape(3, 8)[-1] table0 = create_table(state0, state_vector_names) print(table0) # If we take a step in the environment env_info, _, next_states = take_actions(env_info, brain) print('\n\nTaking 1 action, the state vector would look like::') state1 = next_states[0].reshape(3, 8) table1 = create_table(state1, state_vector_names) print('t = 0') print(table) print('t = 1') print(table1) print( 'Only the last row is providing new information - It have just shifted \n\n' ) print( 'Although, we can think than given the three provides temporal relationships \n\n' ) # If we take another step in the environment env_info, _, next_states = take_actions(env_info, brain) print('\n\nTaking another action to be sure') state2 = next_states[0].reshape(3, 8) table2 = create_table(state2, state_vector_names) print('t = 0') print(table) print('t = 1') print(table1) print('t = 2') print(table2) print('Yes, only the last row is providing new information !') env.close() # Test Agent # ---------- state_size, action_size = brain.vector_observation_space_size, brain.vector_action_space_size agent = Agent(num_agents=num_agents, state_size=state_size, action_size=action_size) print('\n\nCapacity of the Actor (# of parameters): ', count_parameters(agent.actor_local)) print('Capacity of the Critic (# of parameters): ', count_parameters(agent.critic_local)) return
def main(): """Runs queries to create training and prediction tables from clean data.""" # Load config shared by all steps of feature creation. config_path = utils.parse_arguments(sys.argv).config_path config = utils.read_config(config_path) # Project-wide config. global_config = config['global'] # Path to SQL files. queries_path = config['file_paths']['queries'] # SQL files for different pipeline steps. query_files = config['query_files'] # Parameters unique to individual pipeline steps. query_params = config['query_params'] # Create the dataset to hold data for the pipeline run. utils.create_dataset( destination_project=global_config['destination_project_id'], destination_dataset=global_config['destination_dataset']) # Query to remove nulls from the target column (company_response_to_consumer) # and from complaint_narrative column. remove_nulls_params = utils.merge_dicts(global_config, query_params['remove_nulls']) utils.create_table( query_path=os.path.join(queries_path, query_files['remove_nulls']), query_params=remove_nulls_params, destination_project=global_config['destination_project_id'], destination_dataset=global_config['destination_dataset'], destination_table=global_config['nulls_removed_table'], partition_field=None) # Query to cleanup the categories of issue, subissue, product, subproduct. utils.create_table( query_path=os.path.join(queries_path, query_files['clean_categories']), query_params=global_config, destination_project=global_config['destination_project_id'], destination_dataset=global_config['destination_dataset'], destination_table=global_config['cleaned_features_table'], partition_field=None) # Query to merge the cleaned features and the table with nulls removed. utils.create_table( query_path=os.path.join(queries_path, query_files['combine_tables']), query_params=global_config, destination_project=global_config['destination_project_id'], destination_dataset=global_config['destination_dataset'], destination_table=global_config['clean_table'], partition_field=None) # Query to split the clean dataset into training and prediction datasets. # The training dataset will be fed to the AutoML Tables for training and # the prediction dataset will be used for batch prediction. features_split_params = utils.merge_dicts( global_config, query_params['train_predict_split']) utils.create_table( query_path=os.path.join(queries_path, query_files['train_predict_split']), query_params=features_split_params, destination_project=global_config['destination_project_id'], destination_dataset=global_config['destination_dataset'], destination_table=global_config['train_predict_split'], partition_field=None) # Query to create the prediction table. features_split_params = utils.merge_dicts( global_config, query_params['train_predict_split']) utils.create_table( query_path=os.path.join(queries_path, query_files['prediction_features']), query_params=features_split_params, destination_project=global_config['destination_project_id'], destination_dataset=global_config['destination_dataset'], destination_table=global_config['features_predict_table'], partition_field=None) # Query to create the training table along with the manual split into train, # validation and test rows for the AutoML tables. features_split_params = utils.merge_dicts( global_config, query_params['train_predict_split']) utils.create_table( query_path=os.path.join(queries_path, query_files['training_features']), query_params=features_split_params, destination_project=global_config['destination_project_id'], destination_dataset=global_config['destination_dataset'], destination_table=global_config['features_train_table'], partition_field=None)
category='', pub_time=pub_time, is_valid=is_valid, is_delete=False if is_valid else True) order.website = website session.add(order) if is_valid == True: sheet.append( [count, desc, link, pub_time, contact, '']) count += 1 else: message = '人人开发详情爬取第%d行出错:%s' % (details[0], details[1]) print(message) send_message(message) session.commit() elif isinstance(info_list, tuple): message = '人人开发爬取第%d行出错:%s' % (info_list[0], info_list[1]) print(message) send_message(message) print('结束爬取人人开发订单') if __name__ == '__main__': wb = Workbook() engine, Base, session = get_mysql_connection() Order, Website = create_table(engine, Base) add_default_data(session, Website) main(wb, session, Order, Website) now = datetime.now() wb.save(r'data/%s.xlsx' % now.strftime("%Y-%m-%d %H-%M-%S"))
w_import = getattr(w, "import") w.factory("x_M[%f,%f]" % (x_M_LL, x_M_UL)) x_M = w.var("x_M") w.factory("x_Y[%f,%f]" % (x_Y_LL, x_Y_UL)) x_Y = w.var("x_Y") w.factory("x_PT[%f,%f]" % (x_PT_LL, x_PT_UL)) x_PT = w.var("x_PT") f_input = R.TFile(local_dir + "/JPsi_mc_sub.root") sel_tree = f_input.subTree mc_tree = f_input.MCTree sel_cut = "" bin_cut = utils.define_cuts(nBins_Y, nBins_PT, x_Y, x_PT) sample = utils.define_samples(nBins_Y, nBins_PT) efficiency = [[0 for x in xrange(nBins_Y)] for x in xrange(nBins_PT)] efficiency_err = [[0 for x in xrange(nBins_Y)] for x in xrange(nBins_PT)] for i, j in product(range(nBins_Y), range(nBins_PT)): cut = sel_cut + bin_cut[i][j] N_sel = sel_tree.GetEntries(cut) N_init = mc_tree.GetEntries(bin_cut[i][j]) efficiency[i][j] = float(N_sel) / float(N_init) efficiency_err[i][j] = efficiency[i][j] * R.TMath.sqrt(1. / N_sel + 1. / N_init) utils.create_table(efficiency, efficiency_err, "Efficiency.txt", nBins_Y, nBins_PT)
yields = utils.parse_table("../fitting/Signal_Yields.txt",nBins_Y, nBins_PT)[0] yields_err = utils.parse_table("../fitting/Signal_Yields.txt",nBins_Y, nBins_PT)[1] yield_distr = R.TH2F("yield_distr", "Yield distribution", nBins_Y, x_Y_LL , x_Y_UL , nBins_PT, x_PT_LL , x_PT_UL ) yield_distr.GetXaxis().SetTitle("#eta") yield_distr.GetYaxis().SetTitle("P^{T} [GeV/c]") yield_distr_table = [[0 for x in xrange(nBins_Y)] for x in xrange(nBins_PT)] yield_distr_table_err = [[0 for x in xrange(nBins_Y)] for x in xrange(nBins_PT)] for i, j in product(range(nBins_Y), range(nBins_PT)): yield_distr_table[i][j]=float(yields[i][j])/float(efficiencies[i][j]) yield_distr_table_err[i][j] = float(yields[i][j])/float(efficiencies[i][j])*R.TMath.sqrt((float(efficiencies_err[i][j])/float(efficiencies[i][j]))**2+(float(yields_err[i][j])/float(yields[i][j]))**2) yield_distr.SetBinContent(i+1, j+1, yield_distr_table[i][j]) yield_distr.SetBinError(i+1, j+1, yield_distr_table_err[i][j]) txt = [[0 for x in xrange(nBins_Y)] for x in xrange(nBins_PT)] c = R.TCanvas() R.gStyle.SetOptStat(0) yield_distr.Draw("COLZ") for i, j in product(range(nBins_Y), range(nBins_PT)): txt[i][j] = R.TText(yield_distr.GetXaxis().GetBinLowEdge(i+1), yield_distr.GetYaxis().GetBinCenter(j+1), "%d +/- %d"%(yield_distr.GetBinContent(i+1,j+1), yield_distr.GetBinError(i+1,j+1))) txt[i][j].Draw("SAME") c.Update() c.SaveAs("Yield_distr.pdf") utils.create_table(yield_distr_table, yield_distr_table_err, "Final_Yield_Distr.txt",nBins_Y, nBins_PT)
def init(self): utils.create_table(self.sql, self.table_name)
def render_table(api_choice, user_id): """ Create a display for the chosen dataset. Args: api_choice (str): Value from the dropdown. user_id (str): Session/user id. Returns: list: A list of dash components. """ if api_choice is None: return [html.H4("Nothing selected.")] if api_choice == "twitter_api": api = pickle.loads(r.get(f"{user_id}_{api_choice}_handle")) return pretty_print_tweets(api, 5) elif api_choice == "reddit_api": # No need to get the api here # TODO: But maybe this change? return [ html.H4("Write the name of a subreddit:"), dcc.Input( id="subreddit_choice", type="text", value="", ), html.Button("Gimme dem reddits", id="reddit_submit"), html.Br(), html.Br(), html.Div(id="subreddit_posts"), ] elif api_choice == "spotify_api": spotify = pickle.loads(r.get(f"{user_id}_{api_choice}_handle")) top_playlists = spotify.category_playlists( "toplists")["playlists"]["items"] posts = [ html.Div([ dbc.Card([ dbc.CardHeader([ html.H4(playlist["name"]), html.A("listen on Spotify", href=playlist["external_urls"]["spotify"]), ]), dbc.CardBody([ dbc.CardTitle( f"Owner: {playlist['owner']['display_name']}"), dbc.CardText( f"{playlist['tracks']['total']} total tracks"), ]), ]), html.Br(), ]) for playlist in top_playlists ] return posts elif api_choice == "quandl_api": df = get_data(api_choice, user_id) else: df = get_data(api_choice, user_id) if df is None: return [html.H4("Nothing to display")] df = df[df.columns[:10]] return [ html.Br(), create_table(df), ]
# bulk stress profile stress_bulk_parser = stress_sub_parser.add_parser( 'bulk', help='Bulk profile of stress') # incremental stress profile stress_incr_parser = stress_sub_parser.add_parser( 'increment', help='Incremental profile of stress') args = parser.parse_args() print args if args.cmd_name == 'create': logging.debug('Create command') if args.obj_type == 'table': utils.create_table(table_path_prefix=args.prefix, start_idx=args.startidx, num_tables=args.numtables) elif args.obj_type == 'volume': utils.create_volume(volume_path_prefix=args.prefix, start_idx=args.startidx, num_volumes=args.numvolumes) else: logging.error('Unrecognized object. Cannot create.') sys.exit(-1) elif args.cmd_name == 'delete': logging.debug('Delete command') if args.obj_type == 'table': utils.delete_table(table_path_prefix=args.prefix, start_idx=args.startidx, num_tables=args.numtables) elif args.obj_type == 'volume':
w_import = getattr(w, "import") w.factory("x_M[%f,%f]"%(x_M_LL, x_M_UL)) x_M = w.var("x_M") w.factory("x_Y[%f,%f]"%(x_Y_LL, x_Y_UL)) x_Y = w.var("x_Y") w.factory("x_PT[%f,%f]"%(x_PT_LL, x_PT_UL)) x_PT = w.var("x_PT") f_input = R.TFile(local_dir + "/JPsi_mc_sub.root") sel_tree = f_input.subTree mc_tree = f_input.MCTree sel_cut = "" bin_cut = utils.define_cuts(nBins_Y, nBins_PT, x_Y, x_PT) sample = utils.define_samples(nBins_Y, nBins_PT) efficiency = [[0 for x in xrange(nBins_Y)] for x in xrange(nBins_PT)] efficiency_err = [[0 for x in xrange(nBins_Y)] for x in xrange(nBins_PT)] for i, j in product(range(nBins_Y), range(nBins_PT)): cut = sel_cut + bin_cut[i][j] N_sel = sel_tree.GetEntries(cut) N_init = mc_tree.GetEntries(bin_cut[i][j]) efficiency[i][j] = float(N_sel)/float(N_init) efficiency_err[i][j] = efficiency[i][j]*R.TMath.sqrt(1./N_sel+1./N_init) utils.create_table(efficiency, efficiency_err, "Efficiency.txt",nBins_Y, nBins_PT)
from utils import get_report, create_table, clean, create_cli_table, display_cli_menu def help_me(): print('usage: \n python reportdroid.py -c [for CLI] \n python reportdroid.py -v [for html report]') sys.exit(0) if len(sys.argv) < 2: help_me() if sys.argv[1] == '-v': # visual version clean() password = getpass.getpass("Type Jenkins Password: "******"Type Jenkins Password: "******"please run: \"pip install PyInquirer\" first if haven't done already") report_map = create_cli_table(get_report(password)) display_cli_menu(report_map) else: help_me()
import utils as ut ### read data ii = ut.read_graph("ii") ui = ut.read_graph("ui") ### Hypergeom Test ii_lou = map(lambda x: ut.hypergeom_test(ii, x), filter(ut.check_length_mod, ut.louvain(ii))) ii_mcl = map(lambda x: ut.hypergeom_test(ii, x), filter(ut.check_length_mod, ut.mcl(ii))) ui_lou = map(lambda x: ut.hypergeom_test(ui, x), filter(ut.check_length_mod, ut.louvain(ui))) ui_mcl = map(lambda x: ut.hypergeom_test(ui, x), filter(ut.check_length_mod, ut.mcl(ui))) # Create tables ii_mod = ut.create_table("ii_mod", list(ii_lou), list(ii_mcl)) ui_mod = ut.create_table("ui_mod", list(ui_lou), list(ui_mcl)) # Visualize clusters ut.louvain(ii, 'ii', viz=True) ut.louvain(ui, 'ui', viz=True) ut.mcl(ii, viz=True) ut.mcl(ui, viz=True)
def detect_object(gcs_image_folder, gcs_cropped_image_folder, main_project_id, model_id, bq_dataset_output, bq_table_output, prediction_client, storage_client, bq_client): match = re.match(r'gs://([^/]+)/(.+)', gcs_image_folder) bucket_name = match.group(1) prefix = match.group(2) dataset_ref = bq_client.dataset(bq_dataset_output) table_ref = dataset_ref.table(bq_table_output) bucket = storage_client.bucket(bucket_name) params = {"timeout": "60.0s"} lines = [] schema = [ bigquery.SchemaField('file', 'STRING', mode='REQUIRED'), bigquery.SchemaField('object', 'STRING', mode='REQUIRED'), bigquery.SchemaField('confidence', 'STRING', mode='REQUIRED'), bigquery.SchemaField('x_min', 'STRING', mode='REQUIRED'), bigquery.SchemaField('x_max', 'STRING', mode='REQUIRED'), bigquery.SchemaField('y_min', 'STRING', mode='REQUIRED'), bigquery.SchemaField('y_max', 'STRING', mode='REQUIRED'), ] table = utils.create_table(bq_client, bq_dataset_output, bq_table_output, schema) for blob in bucket.list_blobs(prefix=str(prefix + "/")): if blob.name.endswith(".png"): logger.info("File location: {}".format( os.path.join('gs://', bucket_name, blob.name))) content = utils.sample_handler(storage_client, bucket_name, blob.name) name = 'projects/{}/locations/us-central1/models/{}'.format( main_project_id, model_id) payload = {'image': {'image_bytes': content}} params = {} request = prediction_client.predict(name, payload, params) for result in request.payload: logger.info("Figure detected in file.") rows_to_insert = [ (str(blob.name).replace(".png", ".pdf").replace(prefix,"").replace("/",""), \ result.display_name, \ result.image_object_detection.score, \ result.image_object_detection.bounding_box.normalized_vertices[0].x, result.image_object_detection.bounding_box.normalized_vertices[1].x, \ result.image_object_detection.bounding_box.normalized_vertices[0].y, result.image_object_detection.bounding_box.normalized_vertices[1].y), ] load_job = bq_client.insert_rows(table, rows_to_insert) # As below, crop the object and save the cropped part as a separated image file file_name = blob.name _, temp_local_filename = tempfile.mkstemp() blob.download_to_filename(temp_local_filename) im = Image.open(temp_local_filename) width, height = im.size r_xmin = width * result.image_object_detection.bounding_box.normalized_vertices[ 0].x r_ymin = height * result.image_object_detection.bounding_box.normalized_vertices[ 0].y r_xmax = width * result.image_object_detection.bounding_box.normalized_vertices[ 1].x r_ymax = height * result.image_object_detection.bounding_box.normalized_vertices[ 1].y box = (r_xmin, r_ymin, r_xmax, r_ymax) im = Image.open(temp_local_filename) im2 = im.crop(box) im2.save(temp_local_filename.replace('.png', '-crop.png'), 'png') # Upload cropped image to gcs bucket new_file_name = os.path.join( gcs_cropped_image_folder, os.path.basename(blob.name).replace('.png', '-crop.png')) new_file_bucket, new_file_name = utils.get_bucket_blob( new_file_name) new_blob = blob.bucket.blob(new_file_name) new_blob.upload_from_filename(temp_local_filename) os.remove(temp_local_filename) else: pass
# robot elif args.run: print('<<START>>') loop = asyncio.get_event_loop() loop.run_until_complete(processor()) try: loop.run_forever() except KeyboardInterrupt: pass finally: print('<<END>>') loop.close() if __name__ == '__main__': create_table() parser = argparse.ArgumentParser(description='Robot Navigation') parser.add_argument( '--loaddata', help= 'Loading data is easy: just call \'manage.py loaddata <filename>\' where' ' <filename> is the name of the data file you\'ve created') parser.add_argument( '--landmark', nargs='+', help='Creating landmark is easy: just call \'manage.py landmark <coords>' '<landmarkname>\' where <coords> is the coordinates of landmark and' '<landmarkname> is the name of the landmark you\'ve crated') parser.add_argument('--run', help='Start to work with your routes')
#-*- coding: utf-8 -*- import logging import utils import config from sqlhelper import SqlHelper if __name__ == '__main__': logging.basicConfig(filename='log/exporttosql.log', format='%(levelname)s %(asctime)s: %(message)s', level=logging.DEBUG) sql = SqlHelper() utils.create_table(sql, config.assetstore_table_name) utils.export_to_sql(sql, 'Plugins/all')