def shell(args): print("Opening a shell and importing all models...") from utils.db import Session, get_all_models_classes # Import all models models_names = [] for cls in get_all_models_classes(): model_name = cls.__name__ models_names.append(model_name) globals()[model_name] = getattr( __import__('models', fromlist=[model_name]), model_name) # Create a new session sex = Session() # Print some info print('') print("The following models have been loaded:\n{}".format(models_names)) print("A session has been open with name `sex`.") print("You can run a query like:\n sex.query({}).all()".format(models_names[0])) print('') # Start a debug session # Try import bpdb (which requires Bpython), if not import plain pdb try: import bpdb as debug # This is a trick to manually import the models because the automatic import of all # models works in pdb, but not in bpdb. from models import Provider, BearerToken except: import pdb as debug debug.set_trace() print("Done.")
def set_tag_many_to_many_on_nodes(page_size=10000): print 'Starting {}...'.format(sys._getframe().f_code.co_name) node_count = 0 m2m_count = 0 start = datetime.now() total = MODMNode.find(build_query(m2m_tag_fields, MODMNode)).count() print '{} Nodes'.format(total) while node_count < total: with transaction.atomic(): for modm_node in MODMNode.find( build_query(m2m_tag_fields, MODMNode)).sort( '-date_modified')[node_count:page_size + node_count]: django_node = Node.objects.get( pk=modm_to_django[modm_node._id]) for m2m_tag_field in m2m_tag_fields: try: attr = getattr(django_node, m2m_tag_field) except AttributeError as ex: # node field doesn't exist on node pass else: # node field exists, do the stuff django_pks = [] for modm_m2m_value in getattr(modm_node, m2m_tag_field, []): suffix = 'system' if m2m_tag_field == 'system_tags' else 'not_system' if isinstance(modm_m2m_value, MODMTag): django_pks.append( modm_to_django['{}:{}'.format( modm_m2m_value._id, suffix)]) elif isinstance(modm_m2m_value, basestring): django_pks.append( modm_to_django['{}:{}'.format( modm_m2m_value, suffix)]) elif modm_m2m_value is None: print 'Tag of None found on Node {}'.format( modm_node._id) else: # wth print '\a' # bells! print '\a' print '\a' print '\a' print '\a' print '\a' print '\a' import bpdb bpdb.set_trace() if len(django_pks) > 0: attr.add(*django_pks) m2m_count += len(django_pks) node_count += 1 if node_count % page_size == 0 or node_count == total: print 'Through {} nodes and {} m2m'.format( node_count, m2m_count) print 'Done with {} in {} seconds...'.format( sys._getframe().f_code.co_name, (datetime.now() - start).total_seconds())
def analyze(args): collector = WST_ArangoTreeCollector( args.repo_url, workers=args.workers, database_conn=args.db, commit_sha=args.target_commit, ) collector.setup() log.debug(f"Set up collector: {collector}") if args.interactive_debug: log.warn("Starting debugging:") bpdb.set_trace() try: collector.collect_all(overwrite_incomplete=args.overwrite_incomplete) except RepoExistsError as e: if args.skip_exists: log.warn( f"Skipping collection since repo document already present for commit {collector._current_commit_hash}" ) return else: raise except Exception as e: log.crit(f"{collector} run failed.") raise e
def set_tag_many_to_many_on_nodes(page_size=10000): print 'Starting {}...'.format(sys._getframe().f_code.co_name) node_count = 0 m2m_count = 0 start = datetime.now() total = MODMNode.find(build_query(m2m_tag_fields, MODMNode)).count() print '{} Nodes'.format(total) while node_count < total: with transaction.atomic(): for modm_node in MODMNode.find(build_query( m2m_tag_fields, MODMNode)).sort('-date_modified')[ node_count:page_size + node_count]: django_node = Node.objects.get( pk=modm_to_django[modm_node._id]) for m2m_tag_field in m2m_tag_fields: try: attr = getattr(django_node, m2m_tag_field) except AttributeError as ex: # node field doesn't exist on node pass else: # node field exists, do the stuff django_pks = [] for modm_m2m_value in getattr(modm_node, m2m_tag_field, []): suffix = 'system' if m2m_tag_field == 'system_tags' else 'not_system' if isinstance(modm_m2m_value, MODMTag): django_pks.append(modm_to_django[ '{}:{}'.format(modm_m2m_value._id, suffix)]) elif isinstance(modm_m2m_value, basestring): django_pks.append(modm_to_django[ '{}:{}'.format(modm_m2m_value, suffix)]) elif modm_m2m_value is None: print 'Tag of None found on Node {}'.format( modm_node._id) else: # wth print '\a' # bells! print '\a' print '\a' print '\a' print '\a' print '\a' print '\a' import bpdb bpdb.set_trace() if len(django_pks) > 0: attr.add(*django_pks) m2m_count += len(django_pks) node_count += 1 if node_count % page_size == 0 or node_count == total: print 'Through {} nodes and {} m2m'.format(node_count, m2m_count) print 'Done with {} in {} seconds...'.format( sys._getframe().f_code.co_name, (datetime.now() - start).total_seconds())
def plot_test(env, agent): df_state = agent.estimator.get_state_log_df() df_meas = agent.estimator.get_residuals_log_df() if env.centralized: df_state_Z = env.agent_centralized.estimator.get_state_log_df() df_meas_Z = env.agent_centralized.estimator.get_residuals_log_df() set_trace()
def set_node_many_to_many_on_users(page_size=5000): print 'Starting {}...'.format(sys._getframe().f_code.co_name) user_count = 0 m2m_count = 0 start = datetime.now() total = MODMUser.find(build_query(m2m_node_fields, MODMUser)).count() print '{} Users'.format(total) while user_count < total: with transaction.atomic(): for modm_user in MODMUser.find(build_query( m2m_node_fields, MODMUser)).sort('-date_registered')[ user_count:page_size + user_count]: django_user = User.objects.get( pk=modm_to_django[modm_user._id]) for m2m_node_field in m2m_node_fields: try: attr = getattr(django_user, m2m_node_field) except AttributeError as ex: # node field doesn't exist on user pass else: # node field exists, do the stuff django_pks = [] for modm_m2m_value in getattr(modm_user, m2m_node_field, []): if isinstance(modm_m2m_value, MODMNode): django_pks.append(modm_to_django[ modm_m2m_value._id]) elif isinstance(modm_m2m_value, basestring): django_pks.append(modm_to_django[ modm_m2m_value]) elif isinstance(modm_m2m_value, Pointer): django_pks.append(modm_to_django[ modm_m2m_value.node._id]) else: # wth print '\a' # bells! print '\a' print '\a' print '\a' print '\a' print '\a' print '\a' import bpdb bpdb.set_trace() if len(django_pks) > 0: attr.add(*django_pks) m2m_count += len(django_pks) user_count += 1 if user_count % page_size == 0 or user_count == total: print 'Through {} users and {} m2m'.format(user_count, m2m_count) print 'Done with {} in {} seconds...'.format( sys._getframe().f_code.co_name, (datetime.now() - start).total_seconds())
def set_node_many_to_many_on_users(page_size=5000): print 'Starting {}...'.format(sys._getframe().f_code.co_name) user_count = 0 m2m_count = 0 start = datetime.now() total = MODMUser.find(build_query(m2m_node_fields, MODMUser)).count() print '{} Users'.format(total) while user_count < total: with transaction.atomic(): for modm_user in MODMUser.find( build_query(m2m_node_fields, MODMUser)).sort( '-date_registered')[user_count:page_size + user_count]: django_user = User.objects.get( pk=modm_to_django[modm_user._id]) for m2m_node_field in m2m_node_fields: try: attr = getattr(django_user, m2m_node_field) except AttributeError as ex: # node field doesn't exist on user pass else: # node field exists, do the stuff django_pks = [] for modm_m2m_value in getattr(modm_user, m2m_node_field, []): if isinstance(modm_m2m_value, MODMNode): django_pks.append( modm_to_django[modm_m2m_value._id]) elif isinstance(modm_m2m_value, basestring): django_pks.append( modm_to_django[modm_m2m_value]) elif isinstance(modm_m2m_value, Pointer): django_pks.append( modm_to_django[modm_m2m_value.node._id]) else: # wth print '\a' # bells! print '\a' print '\a' print '\a' print '\a' print '\a' print '\a' import bpdb bpdb.set_trace() if len(django_pks) > 0: attr.add(*django_pks) m2m_count += len(django_pks) user_count += 1 if user_count % page_size == 0 or user_count == total: print 'Through {} users and {} m2m'.format( user_count, m2m_count) print 'Done with {} in {} seconds...'.format( sys._getframe().f_code.co_name, (datetime.now() - start).total_seconds())
def main(): if disper.prompt("Load map?") == 8: crt.loader() if disper.prompt("Zero in?") == 8: print "Zeroin" xx.zeroin() yy.zeroin() elif debug: bpdb.set_trace() if xx.zeroed and yy.zeroed: autokick()
def test_get_document(): access_token = config('TESTUSER_ID_TOKEN') #access_token = get_access_token()['access_token'] headers = { 'Content-Type': 'application/json', 'authorization': 'Bearer ' + access_token } result = requests.get(post_url, headers=headers) print(result) from bpdb import set_trace set_trace()
def main(): data_folder = "sample_data/" news_data_df = pd.read_csv(os.path.join(data_folder, "news_sample.csv")) market_data_df = pd.read_csv( os.path.join(data_folder, "marketdata_sample.csv")) news_data_df["assetCodes"] = convert_asset_codes( news_data_df["assetCodes"]) asset_code_map = create_asset_code_map(news_data_df["assetCodes"]) bpdb.set_trace( ) # ------------------------------ Breakpoint ------------------------------ # pass
def set_node_many_to_many_on_nodes(page_size=5000): print 'Starting {}...'.format(sys._getframe().f_code.co_name) node_count = 0 m2m_count = 0 start = datetime.now() total = MODMNode.find( build_query(m2m_node_fields, MODMNode), allow_institution=True).count() print '{} Nodes'.format(total) while node_count < total: with transaction.atomic(): for modm_node in MODMNode.find( build_query(m2m_node_fields, MODMNode), allow_institution=True).sort('-date_modified')[ node_count:page_size + node_count]: django_node = Node.objects.get( pk=modm_to_django[modm_node._id]) for m2m_node_field in m2m_node_fields: attr = getattr(django_node, m2m_node_field) django_pks = [] for modm_m2m_value in getattr(modm_node, m2m_node_field, []): if isinstance(modm_m2m_value, MODMNode): django_pks.append(modm_to_django[ modm_m2m_value._id]) elif isinstance(modm_m2m_value, basestring): django_pks.append(modm_to_django[modm_m2m_value]) elif isinstance(modm_m2m_value, Pointer): django_pks.append(modm_to_django[ modm_m2m_value.node._id]) else: # wth print '\a' print '\a' print '\a' print '\a' print '\a' print '\a' print '\a' import bpdb bpdb.set_trace() if len(django_pks) > 0: attr.add(*django_pks) m2m_count += len(django_pks) node_count += 1 if node_count % page_size == 0 or node_count == total: print 'Through {} nodes and {} m2m'.format(node_count, m2m_count) print 'Done with {} in {} seconds...'.format( sys._getframe().f_code.co_name, (datetime.now() - start).total_seconds())
def set_node_many_to_many_on_nodes(page_size=5000): print 'Starting {}...'.format(sys._getframe().f_code.co_name) node_count = 0 m2m_count = 0 start = datetime.now() total = MODMNode.find(build_query(m2m_node_fields, MODMNode), allow_institution=True).count() print '{} Nodes'.format(total) while node_count < total: with transaction.atomic(): for modm_node in MODMNode.find( build_query(m2m_node_fields, MODMNode), allow_institution=True).sort( '-date_modified')[node_count:page_size + node_count]: django_node = Node.objects.get( pk=modm_to_django[modm_node._id]) for m2m_node_field in m2m_node_fields: attr = getattr(django_node, m2m_node_field) django_pks = [] for modm_m2m_value in getattr(modm_node, m2m_node_field, []): if isinstance(modm_m2m_value, MODMNode): django_pks.append( modm_to_django[modm_m2m_value._id]) elif isinstance(modm_m2m_value, basestring): django_pks.append(modm_to_django[modm_m2m_value]) elif isinstance(modm_m2m_value, Pointer): django_pks.append( modm_to_django[modm_m2m_value.node._id]) else: # wth print '\a' print '\a' print '\a' print '\a' print '\a' print '\a' print '\a' import bpdb bpdb.set_trace() if len(django_pks) > 0: attr.add(*django_pks) m2m_count += len(django_pks) node_count += 1 if node_count % page_size == 0 or node_count == total: print 'Through {} nodes and {} m2m'.format( node_count, m2m_count) print 'Done with {} in {} seconds...'.format( sys._getframe().f_code.co_name, (datetime.now() - start).total_seconds())
def test_get_document(): access_token = config('TESTUSER_ID_TOKEN') #access_token = get_access_token()['access_token'] headers = { 'Content-Type': 'application/json', 'authorization': 'Bearer ' + access_token } contents = { #"会社_2__会社名": "アイ·エム·アイ株式会社" "company__company_name": "imi" } result = requests.get(post_url, params=contents, headers=headers) print(result) from bpdb import set_trace set_trace()
def create_rel_dict(df, column): mod_to_class_dict = {} class_to_mod_dict = {} for i, row in tqdm(df.iterrows(), total=len(df)): mod_to_class_dict[i] = row[column] if row[column] not in class_to_mod_dict: try: class_to_mod_dict[row[column]] = [i] except: import bpdb bpdb.set_trace() else: class_to_mod_dict[row[column]].append(i) return mod_to_class_dict, class_to_mod_dict
def run_sim(): N = 5 epsilon_x = 0 epsilon_z = 0 for i in range(N): succeeded = False while not succeeded: env = setup_env(config_file) try: env.run(until=env.MAX_TIME) for agent in env.agents: agent.cleanup() eps_x, eps_z = check_NEES_NIS(env) epsilon_x += eps_x epsilon_z += eps_z set_trace() succeeded = True except np.linalg.LinAlgError: for agent in env.agents: agent.cleanup() continue epsilon_x_bar = epsilon_x / N epsilon_z_bar = epsilon_z / N J_NEES = abs(log(epsilon_x_bar)) J_NIS = abs(log(epsilon_z_bar)) print(f"{J_NEES=}") print(f"{J_NIS=}") R = -(J_NEES + J_NIS) return R
def do_doc2vec(df): df_train, df_test, df_val = df_train_test_val_split(df[KEEP_COLS]) X_train, y_train, X_test, y_test, X_val, y_val = train_test_val_split( df[KEEP_COLS], 'score') train_dict, test_dict, val_dict = train_test_val_dicts(df, 'url', 'score') # use grid search to optimize stage 1 & 2 model parameters #grid_search_naive_bayes_with_tfidf(df_train['text'].values, df_train['stars'].values) #grid_search_stage_two(X_train, y_train) # fit and print classification results for the two stage tfidf model #two_stage_model(X_train, y_train, X_test, y_test) # fit and print classification results for the Doc2Vec model model = doc2vec_model(df, train_dict, test_dict, k=25) export_vec(model) import bpdb bpdb.set_trace() # use the Doc2Vec model doc2vec_examples(model)
def test_create_document(): access_token = config('ADMINUSER_ID_TOKEN') #access_token = get_access_token()['access_token'] headers = { 'Content-Type': 'application/json', 'authorization': 'Bearer ' + access_token } contents = { 'name': 'frompython', 'tenant': 'testtenant2', 'document_type': 'invoice', 'document_content': {} } with open(json_path, 'r') as f: content = json.load(f) contents['document_content'] = json.dumps(content) #contents['document_content'] = content result = requests.post(post_url, json.dumps(contents), headers=headers) print(result) from bpdb import set_trace set_trace()
def test_files(pkl_paths, csv_paths, commit_hash, version_number): for pkl_path, csv_path in zip(pkl_paths, csv_paths): csv = pd.read_csv(csv_path) df = pd.read_pickle(pkl_path) assert_equal(commit_hash, df._metadata['commit_hash']) assert_equal(version_number, df._metadata['version_number']) assert_equal(len(df), len(csv)) csv_ids = set(csv[csv.columns[0]]) df_ids = set(df.index) assert csv_ids == df_ids, get_csv_error_message( pkl_path, csv_path, csv_ids, df_ids) columns_to_check = [ 'narration_id', 'participant_id', 'video_id', 'start_timestamp', 'stop_timestamp', 'start_frame', 'stop_frame', 'narration', 'verb_class', 'noun_class', 'verb', 'noun' ] if 'narration_id' in csv.columns: csv = csv.set_index('narration_id') for column in columns_to_check: if column in df.columns: try: assert (csv[column] == df[column]).all() except: import bpdb bpdb.set_trace() raise Exception( 'Column {} doesn\'t match between {} and {}'.format( column, csv_path, pkl_path)) if 'narration_timestamp' in df.columns: train_missing = pd.read_csv( './EPIC_100_train_missing_timestamp_narrations.csv') val_missing = pd.read_csv( './EPIC_100_validation_missing_timestamp_narrations.csv') missing = set(pd.concat([train_missing, val_missing]).narration_id) diff = set(df[ csv['narration_timestamp'] != df['narration_timestamp']].index) assert len(diff) == 0 or diff.issubset(missing)
def set_node_foreign_keys_on_nodes(page_size=10000): print 'Starting {}...'.format(sys._getframe().f_code.co_name) node_count = 0 fk_count = 0 cache_hits = 0 cache_misses = 0 start = datetime.now() total = MODMNode.find( build_query(fk_node_fields, MODMNode), allow_institution=True).count() while node_count < total: with transaction.atomic(): for modm_node in MODMNode.find( build_query(fk_node_fields, MODMNode), allow_institution=True).sort('-date_modified')[ node_count:node_count + page_size]: django_node = Node.objects.get(_guid__guid=modm_node._id) for fk_node_field in fk_node_fields: value = getattr(modm_node, fk_node_field, None) if value is not None: if isinstance(value, basestring): # value is a guid, try the cache table for the pk if value in modm_to_django: setattr(django_node, '{}_id'.format(fk_node_field), modm_to_django[value]) cache_hits += 1 else: # it's not in the cache, do the query node_id = Node.objects.get( _guid__guid=value).pk setattr(django_node, '{}_id'.format(fk_node_field), node_id) # save for later modm_to_django[value] = node_id cache_misses += 1 elif isinstance(value, MODMNode): # value is a node object, try the cache table for the pk if value._id in modm_to_django: setattr(django_node, '{}_id'.format(fk_node_field), modm_to_django[value._id]) cache_hits += 1 else: # it's not in the cache, do the query node_id = Node.objects.get( _guid__guid=value._id).pk setattr(django_node, '{}_id'.format(fk_node_field), node_id) # save for later modm_to_django[value._id] = node_id cache_misses += 1 else: # whu happened? print '\a' print '\a' print '\a' print '\a' print '\a' print '\a' import bpdb bpdb.set_trace() fk_count += 1 django_node.save() node_count += 1 if node_count % page_size == 0 or node_count == total: then = datetime.now() print 'Through {} nodes and {} foreign keys'.format( node_count, fk_count) print 'Cache: Hits {} Misses {}'.format(cache_hits, cache_misses) print 'Done with {} in {} seconds...'.format( sys._getframe().f_code.co_name, (datetime.now() - start).total_seconds())
def set_user_foreign_keys_on_users(page_size=10000): print 'Starting {}...'.format(sys._getframe().f_code.co_name) user_count = 0 fk_count = 0 cache_hits = 0 cache_misses = 0 start = datetime.now() total = MODMUser.find(build_query(fk_user_fields, MODMUser)).count() while user_count < total: with transaction.atomic(): for modm_user in MODMUser.find(build_query( fk_user_fields, MODMUser)).sort('-date_registered')[ user_count:user_count + page_size]: django_user = User.objects.get(_guid__guid=modm_user._id) for fk_user_field in fk_user_fields: value = getattr(modm_user, fk_user_field, None) if value is not None: if isinstance(value, basestring): # value is a guid, try the cache table for the pk if value in modm_to_django: setattr(django_user, '{}_id'.format(fk_user_field), modm_to_django[value]) cache_hits += 1 else: # it's not in the cache, do the query user_id = User.objects.get( _guid__guid=value).pk setattr(django_user, '{}_id'.format(fk_user_field), user_id) # save for later modm_to_django[value] = user_id cache_misses += 1 elif isinstance(value, MODMUser): # value is a user object, try the cache table for the pk if value._id in modm_to_django: setattr(django_user, '{}_id'.format(fk_user_field), modm_to_django[value._id]) cache_hits += 1 else: # it's not in the cache, do the query user_id = User.objects.get( _guid__guid=value._id).pk setattr(django_user, '{}_id'.format(fk_user_field), user_id) # save for later modm_to_django[value._id] = user_id cache_misses += 1 else: # that's odd. print '\a' print '\a' print '\a' print '\a' print '\a' print '\a' import bpdb bpdb.set_trace() fk_count += 1 django_user.save() user_count += 1 if user_count % page_size == 0 or user_count == total: print 'Through {} users and {} foreign keys'.format( user_count, fk_count) print 'Cache: Hits {} Misses {}'.format(cache_hits, cache_misses) print 'Done with {} in {} seconds...'.format( sys._getframe().f_code.co_name, (datetime.now() - start).total_seconds())
def set_user_many_to_many_on_nodes(page_size=5000): print 'Starting {}...'.format(sys._getframe().f_code.co_name) node_count = 0 m2m_count = 0 start = datetime.now() total = MODMNode.find(build_query(m2m_user_fields, MODMNode), allow_institution=True).count() print '{} Nodes'.format(total) while node_count < total: with transaction.atomic(): for modm_node in MODMNode.find( build_query(m2m_user_fields, MODMNode), allow_institution=True).sort( '-date_modified')[node_count:page_size + node_count]: django_node = Node.objects.get( pk=modm_to_django[modm_node._id]) for m2m_user_field in m2m_user_fields: if m2m_user_field in ['permissions', 'recently_added']: continue attr = getattr(django_node, m2m_user_field) django_pks = [] for modm_m2m_value in getattr(modm_node, m2m_user_field, []): if isinstance(modm_m2m_value, MODMUser): if m2m_user_field == 'contributors': visible = modm_m2m_value._id in modm_node.visible_contributor_ids admin = 'admin' in modm_node.permissions[ modm_m2m_value._id] read = 'read' in modm_node.permissions[ modm_m2m_value._id] write = 'write' in modm_node.permissions[ modm_m2m_value._id] Contributor.objects.get_or_create( user_id=modm_to_django[modm_m2m_value._id], node=django_node, visible=visible, admin=admin, read=read, write=write) m2m_count += 1 else: django_pks.append( modm_to_django[modm_m2m_value._id]) elif isinstance(modm_m2m_value, basestring): if m2m_user_field == 'contributors': visible = modm_m2m_value in modm_node.visible_contributor_ids admin = 'admin' in modm_node.permissions[ modm_m2m_value] read = 'read' in modm_node.permissions[ modm_m2m_value] write = 'write' in modm_node.permissions[ modm_m2m_value] Contributor.objects.get_or_create( user_id=modm_to_django[modm_m2m_value], node=django_node, visible=visible, admin=admin, read=read, write=write) m2m_count += 1 else: django_pks.append( modm_to_django[modm_m2m_value]) else: # wth print '\a' # bells print '\a' print '\a' print '\a' print '\a' print '\a' print '\a' import bpdb bpdb.set_trace() if len(django_pks) > 0: attr.add(*django_pks) m2m_count += len(django_pks) node_count += 1 if node_count % page_size == 0 or node_count == total: print 'Through {} nodes and {} m2m'.format( node_count, m2m_count) print 'Done with {} in {} seconds...'.format( sys._getframe().f_code.co_name, (datetime.now() - start).total_seconds())
def set_user_foreign_keys_on_users(page_size=10000): print 'Starting {}...'.format(sys._getframe().f_code.co_name) user_count = 0 fk_count = 0 cache_hits = 0 cache_misses = 0 start = datetime.now() total = MODMUser.find(build_query(fk_user_fields, MODMUser)).count() while user_count < total: with transaction.atomic(): for modm_user in MODMUser.find( build_query(fk_user_fields, MODMUser)).sort( '-date_registered')[user_count:user_count + page_size]: django_user = User.objects.get(_guid__guid=modm_user._id) for fk_user_field in fk_user_fields: value = getattr(modm_user, fk_user_field, None) if value is not None: if isinstance(value, basestring): # value is a guid, try the cache table for the pk if value in modm_to_django: setattr(django_user, '{}_id'.format(fk_user_field), modm_to_django[value]) cache_hits += 1 else: # it's not in the cache, do the query user_id = User.objects.get( _guid__guid=value).pk setattr(django_user, '{}_id'.format(fk_user_field), user_id) # save for later modm_to_django[value] = user_id cache_misses += 1 elif isinstance(value, MODMUser): # value is a user object, try the cache table for the pk if value._id in modm_to_django: setattr(django_user, '{}_id'.format(fk_user_field), modm_to_django[value._id]) cache_hits += 1 else: # it's not in the cache, do the query user_id = User.objects.get( _guid__guid=value._id).pk setattr(django_user, '{}_id'.format(fk_user_field), user_id) # save for later modm_to_django[value._id] = user_id cache_misses += 1 else: # that's odd. print '\a' print '\a' print '\a' print '\a' print '\a' print '\a' import bpdb bpdb.set_trace() fk_count += 1 django_user.save() user_count += 1 if user_count % page_size == 0 or user_count == total: print 'Through {} users and {} foreign keys'.format( user_count, fk_count) print 'Cache: Hits {} Misses {}'.format( cache_hits, cache_misses) print 'Done with {} in {} seconds...'.format( sys._getframe().f_code.co_name, (datetime.now() - start).total_seconds())
_germs_lite = {upgrade_circuits('germs_lite')} _fiducials = {upgrade_circuits('fiducials')} _prepStrs = {upgrade_circuits('prepStrs')} _effectStrs = {upgrade_circuits('effectStrs')} clifford_compilation = {cc_src} global_fidPairs = {name_or_none('global_fidPairs')} pergerm_fidPairsDict = {name_or_none('pergerm_fidPairsDict')} global_fidPairs_lite = {name_or_none('global_fidPairs_lite')} pergerm_fidPairsDict_lite = {name_or_none('pergerm_fidPairsDict_lite')} @property def _target_model(self): return {target_model_src} import sys sys.modules[__name__] = _Module() """ import bpdb; bpdb.set_trace() # Dump it to stdout print(template_str)
def set_user_many_to_many_on_nodes(page_size=5000): print 'Starting {}...'.format(sys._getframe().f_code.co_name) node_count = 0 m2m_count = 0 start = datetime.now() total = MODMNode.find( build_query(m2m_user_fields, MODMNode), allow_institution=True).count() print '{} Nodes'.format(total) while node_count < total: with transaction.atomic(): for modm_node in MODMNode.find( build_query(m2m_user_fields, MODMNode), allow_institution=True).sort('-date_modified')[ node_count:page_size + node_count]: django_node = Node.objects.get( pk=modm_to_django[modm_node._id]) for m2m_user_field in m2m_user_fields: if m2m_user_field in ['permissions', 'recently_added']: continue attr = getattr(django_node, m2m_user_field) django_pks = [] for modm_m2m_value in getattr(modm_node, m2m_user_field, []): if isinstance(modm_m2m_value, MODMUser): if m2m_user_field == 'contributors': visible = modm_m2m_value._id in modm_node.visible_contributor_ids admin = 'admin' in modm_node.permissions[ modm_m2m_value._id] read = 'read' in modm_node.permissions[ modm_m2m_value._id] write = 'write' in modm_node.permissions[ modm_m2m_value._id] Contributor.objects.get_or_create( user_id=modm_to_django[modm_m2m_value._id], node=django_node, visible=visible, admin=admin, read=read, write=write) m2m_count += 1 else: django_pks.append(modm_to_django[ modm_m2m_value._id]) elif isinstance(modm_m2m_value, basestring): if m2m_user_field == 'contributors': visible = modm_m2m_value in modm_node.visible_contributor_ids admin = 'admin' in modm_node.permissions[ modm_m2m_value] read = 'read' in modm_node.permissions[ modm_m2m_value] write = 'write' in modm_node.permissions[ modm_m2m_value] Contributor.objects.get_or_create( user_id=modm_to_django[modm_m2m_value], node=django_node, visible=visible, admin=admin, read=read, write=write) m2m_count += 1 else: django_pks.append(modm_to_django[ modm_m2m_value]) else: # wth print '\a' # bells print '\a' print '\a' print '\a' print '\a' print '\a' print '\a' import bpdb bpdb.set_trace() if len(django_pks) > 0: attr.add(*django_pks) m2m_count += len(django_pks) node_count += 1 if node_count % page_size == 0 or node_count == total: print 'Through {} nodes and {} m2m'.format(node_count, m2m_count) print 'Done with {} in {} seconds...'.format( sys._getframe().f_code.co_name, (datetime.now() - start).total_seconds())
J_NEES = abs(log(epsilon_x_bar)) J_NIS = abs(log(epsilon_z_bar)) print(f"{J_NEES=}") print(f"{J_NIS=}") R = -(J_NEES + J_NIS) return R # pbounds = {"q": (-2, 2), "t_window": (-3, 0)} # optimizer = BayesianOptimization( # f=lambda q, t_window: run_sim(str(10 ** q), str(10 ** q), str(10 ** t_window)), # pbounds=pbounds, # random_state=1, # ) # optimizer.maximize( # init_points=5, # n_iter=45, # ) # print(optimizer.max) R = run_sim() set_trace()
def set_node_foreign_keys_on_nodes(page_size=10000): print 'Starting {}...'.format(sys._getframe().f_code.co_name) node_count = 0 fk_count = 0 cache_hits = 0 cache_misses = 0 start = datetime.now() total = MODMNode.find(build_query(fk_node_fields, MODMNode), allow_institution=True).count() while node_count < total: with transaction.atomic(): for modm_node in MODMNode.find( build_query(fk_node_fields, MODMNode), allow_institution=True).sort( '-date_modified')[node_count:node_count + page_size]: django_node = Node.objects.get(_guid__guid=modm_node._id) for fk_node_field in fk_node_fields: value = getattr(modm_node, fk_node_field, None) if value is not None: if isinstance(value, basestring): # value is a guid, try the cache table for the pk if value in modm_to_django: setattr(django_node, '{}_id'.format(fk_node_field), modm_to_django[value]) cache_hits += 1 else: # it's not in the cache, do the query node_id = Node.objects.get( _guid__guid=value).pk setattr(django_node, '{}_id'.format(fk_node_field), node_id) # save for later modm_to_django[value] = node_id cache_misses += 1 elif isinstance(value, MODMNode): # value is a node object, try the cache table for the pk if value._id in modm_to_django: setattr(django_node, '{}_id'.format(fk_node_field), modm_to_django[value._id]) cache_hits += 1 else: # it's not in the cache, do the query node_id = Node.objects.get( _guid__guid=value._id).pk setattr(django_node, '{}_id'.format(fk_node_field), node_id) # save for later modm_to_django[value._id] = node_id cache_misses += 1 else: # whu happened? print '\a' print '\a' print '\a' print '\a' print '\a' print '\a' import bpdb bpdb.set_trace() fk_count += 1 django_node.save() node_count += 1 if node_count % page_size == 0 or node_count == total: then = datetime.now() print 'Through {} nodes and {} foreign keys'.format( node_count, fk_count) print 'Cache: Hits {} Misses {}'.format( cache_hits, cache_misses) print 'Done with {} in {} seconds...'.format( sys._getframe().f_code.co_name, (datetime.now() - start).total_seconds())
_germs_lite = {upgrade_circuits('germs_lite')} _fiducials = {upgrade_circuits('fiducials')} _prepStrs = {upgrade_circuits('prepStrs')} _effectStrs = {upgrade_circuits('effectStrs')} clifford_compilation = {cc_src} global_fidPairs = {name_or_none('global_fidPairs')} pergerm_fidPairsDict = {name_or_none('pergerm_fidPairsDict')} global_fidPairs_lite = {name_or_none('global_fidPairs_lite')} pergerm_fidPairsDict_lite = {name_or_none('pergerm_fidPairsDict_lite')} @property def _target_model(self): return {target_model_src} import sys sys.modules[__name__] = _Module() """ import bpdb bpdb.set_trace() # Dump it to stdout print(template_str)
def autokick(): acura = 20000 watched = False watchcnt = 0 axi = False if debug: bpdb.set_trace() while True: disper.upda(xx.pos, yy.pos) distx = 0 disty = 0 slop = (xx.acu + yy.acu) / 1000 if disper.prox > 7: watched = True watchcnt = 10 #lower this! else: if watched: watchcnt -= 1 if watchcnt < 0: watched = False xid = xx.think() yid = yy.think() xx.direct() yy.direct() dirs = crt.nocks(xx.pos, yy.pos) dirx = dirs[0] diry = dirs[1] #flip flop axi = randint(0, 13) % 2 if dirx == diry: if axi: #axi = False if dirx == 0: dirx = randir() diry = 0 else: #axi = True if diry == 0: diry = randir() dirx = 0 sizer = (1500 - crt.sumer() * 4) + 100 #times 2... mag = randint(100, 2 * sizer) style = 2 if watched: fly(dirx * mag, diry * mag, sizer) cross(sizer, style) #poke(style) # time.sleep(0.5) # poke(0) if debug: print " I am %d sloppy and %d watched. Thinking of %d %d this big %d" % ( slop, watched, xid, yid, sizer) print "pos %d %d ORDIR %d %d TARG %d %d" % ( xx.pos, yy.pos, dirs[0], dirs[1], dirx, diry)