def shell(args):
print("Opening a shell and importing all models...")
from utils.db import Session, get_all_models_classes
# Import all models
models_names = []
for cls in get_all_models_classes():
model_name = cls.__name__
models_names.append(model_name)
globals()[model_name] = getattr(
__import__('models', fromlist=[model_name]), model_name)
# Create a new session
sex = Session()
# Print some info
print('')
print("The following models have been loaded:\n{}".format(models_names))
print("A session has been open with name `sex`.")
print("You can run a query like:\n sex.query({}).all()".format(models_names[0]))
print('')
# Start a debug session
# Try import bpdb (which requires Bpython), if not import plain pdb
try:
import bpdb as debug
# This is a trick to manually import the models because the automatic import of all
# models works in pdb, but not in bpdb.
from models import Provider, BearerToken
except:
import pdb as debug
debug.set_trace()
print("Done.")
def set_tag_many_to_many_on_nodes(page_size=10000):
print 'Starting {}...'.format(sys._getframe().f_code.co_name)
node_count = 0
m2m_count = 0
start = datetime.now()
total = MODMNode.find(build_query(m2m_tag_fields, MODMNode)).count()
print '{} Nodes'.format(total)
while node_count < total:
with transaction.atomic():
for modm_node in MODMNode.find(
build_query(m2m_tag_fields, MODMNode)).sort(
'-date_modified')[node_count:page_size + node_count]:
django_node = Node.objects.get(
pk=modm_to_django[modm_node._id])
for m2m_tag_field in m2m_tag_fields:
try:
attr = getattr(django_node, m2m_tag_field)
except AttributeError as ex:
# node field doesn't exist on node
pass
else:
# node field exists, do the stuff
django_pks = []
for modm_m2m_value in getattr(modm_node, m2m_tag_field,
[]):
suffix = 'system' if m2m_tag_field == 'system_tags' else 'not_system'
if isinstance(modm_m2m_value, MODMTag):
django_pks.append(
modm_to_django['{}:{}'.format(
modm_m2m_value._id, suffix)])
elif isinstance(modm_m2m_value, basestring):
django_pks.append(
modm_to_django['{}:{}'.format(
modm_m2m_value, suffix)])
elif modm_m2m_value is None:
print 'Tag of None found on Node {}'.format(
modm_node._id)
else:
# wth
print '\a' # bells!
print '\a'
print '\a'
print '\a'
print '\a'
print '\a'
print '\a'
import bpdb
bpdb.set_trace()
if len(django_pks) > 0:
attr.add(*django_pks)
m2m_count += len(django_pks)
node_count += 1
if node_count % page_size == 0 or node_count == total:
print 'Through {} nodes and {} m2m'.format(
node_count, m2m_count)
print 'Done with {} in {} seconds...'.format(
sys._getframe().f_code.co_name,
(datetime.now() - start).total_seconds())
def analyze(args):
collector = WST_ArangoTreeCollector(
args.repo_url,
workers=args.workers,
database_conn=args.db,
commit_sha=args.target_commit,
)
collector.setup()
log.debug(f"Set up collector: {collector}")
if args.interactive_debug:
log.warn("Starting debugging:")
bpdb.set_trace()
try:
collector.collect_all(overwrite_incomplete=args.overwrite_incomplete)
except RepoExistsError as e:
if args.skip_exists:
log.warn(
f"Skipping collection since repo document already present for commit {collector._current_commit_hash}"
)
return
else:
raise
except Exception as e:
log.crit(f"{collector} run failed.")
raise e
def set_tag_many_to_many_on_nodes(page_size=10000):
print 'Starting {}...'.format(sys._getframe().f_code.co_name)
node_count = 0
m2m_count = 0
start = datetime.now()
total = MODMNode.find(build_query(m2m_tag_fields, MODMNode)).count()
print '{} Nodes'.format(total)
while node_count < total:
with transaction.atomic():
for modm_node in MODMNode.find(build_query(
m2m_tag_fields, MODMNode)).sort('-date_modified')[
node_count:page_size + node_count]:
django_node = Node.objects.get(
pk=modm_to_django[modm_node._id])
for m2m_tag_field in m2m_tag_fields:
try:
attr = getattr(django_node, m2m_tag_field)
except AttributeError as ex:
# node field doesn't exist on node
pass
else:
# node field exists, do the stuff
django_pks = []
for modm_m2m_value in getattr(modm_node, m2m_tag_field,
[]):
suffix = 'system' if m2m_tag_field == 'system_tags' else 'not_system'
if isinstance(modm_m2m_value, MODMTag):
django_pks.append(modm_to_django[
'{}:{}'.format(modm_m2m_value._id,
suffix)])
elif isinstance(modm_m2m_value, basestring):
django_pks.append(modm_to_django[
'{}:{}'.format(modm_m2m_value, suffix)])
elif modm_m2m_value is None:
print 'Tag of None found on Node {}'.format(
modm_node._id)
else:
# wth
print '\a' # bells!
print '\a'
print '\a'
print '\a'
print '\a'
print '\a'
print '\a'
import bpdb
bpdb.set_trace()
if len(django_pks) > 0:
attr.add(*django_pks)
m2m_count += len(django_pks)
node_count += 1
if node_count % page_size == 0 or node_count == total:
print 'Through {} nodes and {} m2m'.format(node_count,
m2m_count)
print 'Done with {} in {} seconds...'.format(
sys._getframe().f_code.co_name,
(datetime.now() - start).total_seconds())
def plot_test(env, agent):
df_state = agent.estimator.get_state_log_df()
df_meas = agent.estimator.get_residuals_log_df()
if env.centralized:
df_state_Z = env.agent_centralized.estimator.get_state_log_df()
df_meas_Z = env.agent_centralized.estimator.get_residuals_log_df()
set_trace()
def set_node_many_to_many_on_users(page_size=5000):
print 'Starting {}...'.format(sys._getframe().f_code.co_name)
user_count = 0
m2m_count = 0
start = datetime.now()
total = MODMUser.find(build_query(m2m_node_fields, MODMUser)).count()
print '{} Users'.format(total)
while user_count < total:
with transaction.atomic():
for modm_user in MODMUser.find(build_query(
m2m_node_fields, MODMUser)).sort('-date_registered')[
user_count:page_size + user_count]:
django_user = User.objects.get(
pk=modm_to_django[modm_user._id])
for m2m_node_field in m2m_node_fields:
try:
attr = getattr(django_user, m2m_node_field)
except AttributeError as ex:
# node field doesn't exist on user
pass
else:
# node field exists, do the stuff
django_pks = []
for modm_m2m_value in getattr(modm_user,
m2m_node_field, []):
if isinstance(modm_m2m_value, MODMNode):
django_pks.append(modm_to_django[
modm_m2m_value._id])
elif isinstance(modm_m2m_value, basestring):
django_pks.append(modm_to_django[
modm_m2m_value])
elif isinstance(modm_m2m_value, Pointer):
django_pks.append(modm_to_django[
modm_m2m_value.node._id])
else:
# wth
print '\a' # bells!
print '\a'
print '\a'
print '\a'
print '\a'
print '\a'
print '\a'
import bpdb
bpdb.set_trace()
if len(django_pks) > 0:
attr.add(*django_pks)
m2m_count += len(django_pks)
user_count += 1
if user_count % page_size == 0 or user_count == total:
print 'Through {} users and {} m2m'.format(user_count,
m2m_count)
print 'Done with {} in {} seconds...'.format(
sys._getframe().f_code.co_name,
(datetime.now() - start).total_seconds())
def set_node_many_to_many_on_users(page_size=5000):
print 'Starting {}...'.format(sys._getframe().f_code.co_name)
user_count = 0
m2m_count = 0
start = datetime.now()
total = MODMUser.find(build_query(m2m_node_fields, MODMUser)).count()
print '{} Users'.format(total)
while user_count < total:
with transaction.atomic():
for modm_user in MODMUser.find(
build_query(m2m_node_fields, MODMUser)).sort(
'-date_registered')[user_count:page_size + user_count]:
django_user = User.objects.get(
pk=modm_to_django[modm_user._id])
for m2m_node_field in m2m_node_fields:
try:
attr = getattr(django_user, m2m_node_field)
except AttributeError as ex:
# node field doesn't exist on user
pass
else:
# node field exists, do the stuff
django_pks = []
for modm_m2m_value in getattr(modm_user,
m2m_node_field, []):
if isinstance(modm_m2m_value, MODMNode):
django_pks.append(
modm_to_django[modm_m2m_value._id])
elif isinstance(modm_m2m_value, basestring):
django_pks.append(
modm_to_django[modm_m2m_value])
elif isinstance(modm_m2m_value, Pointer):
django_pks.append(
modm_to_django[modm_m2m_value.node._id])
else:
# wth
print '\a' # bells!
print '\a'
print '\a'
print '\a'
print '\a'
print '\a'
print '\a'
import bpdb
bpdb.set_trace()
if len(django_pks) > 0:
attr.add(*django_pks)
m2m_count += len(django_pks)
user_count += 1
if user_count % page_size == 0 or user_count == total:
print 'Through {} users and {} m2m'.format(
user_count, m2m_count)
print 'Done with {} in {} seconds...'.format(
sys._getframe().f_code.co_name,
(datetime.now() - start).total_seconds())
def main():
if disper.prompt("Load map?") == 8:
crt.loader()
if disper.prompt("Zero in?") == 8:
print "Zeroin"
xx.zeroin()
yy.zeroin()
elif debug:
bpdb.set_trace()
if xx.zeroed and yy.zeroed:
autokick()
def test_get_document():
access_token = config('TESTUSER_ID_TOKEN')
#access_token = get_access_token()['access_token']
headers = {
'Content-Type': 'application/json',
'authorization': 'Bearer ' + access_token
}
result = requests.get(post_url, headers=headers)
print(result)
from bpdb import set_trace
set_trace()
def main():
data_folder = "sample_data/"
news_data_df = pd.read_csv(os.path.join(data_folder, "news_sample.csv"))
market_data_df = pd.read_csv(
os.path.join(data_folder, "marketdata_sample.csv"))
news_data_df["assetCodes"] = convert_asset_codes(
news_data_df["assetCodes"])
asset_code_map = create_asset_code_map(news_data_df["assetCodes"])
bpdb.set_trace(
) # ------------------------------ Breakpoint ------------------------------ #
pass
def set_node_many_to_many_on_nodes(page_size=5000):
print 'Starting {}...'.format(sys._getframe().f_code.co_name)
node_count = 0
m2m_count = 0
start = datetime.now()
total = MODMNode.find(
build_query(m2m_node_fields, MODMNode),
allow_institution=True).count()
print '{} Nodes'.format(total)
while node_count < total:
with transaction.atomic():
for modm_node in MODMNode.find(
build_query(m2m_node_fields, MODMNode),
allow_institution=True).sort('-date_modified')[
node_count:page_size + node_count]:
django_node = Node.objects.get(
pk=modm_to_django[modm_node._id])
for m2m_node_field in m2m_node_fields:
attr = getattr(django_node, m2m_node_field)
django_pks = []
for modm_m2m_value in getattr(modm_node, m2m_node_field,
[]):
if isinstance(modm_m2m_value, MODMNode):
django_pks.append(modm_to_django[
modm_m2m_value._id])
elif isinstance(modm_m2m_value, basestring):
django_pks.append(modm_to_django[modm_m2m_value])
elif isinstance(modm_m2m_value, Pointer):
django_pks.append(modm_to_django[
modm_m2m_value.node._id])
else:
# wth
print '\a'
print '\a'
print '\a'
print '\a'
print '\a'
print '\a'
print '\a'
import bpdb
bpdb.set_trace()
if len(django_pks) > 0:
attr.add(*django_pks)
m2m_count += len(django_pks)
node_count += 1
if node_count % page_size == 0 or node_count == total:
print 'Through {} nodes and {} m2m'.format(node_count,
m2m_count)
print 'Done with {} in {} seconds...'.format(
sys._getframe().f_code.co_name,
(datetime.now() - start).total_seconds())
def set_node_many_to_many_on_nodes(page_size=5000):
print 'Starting {}...'.format(sys._getframe().f_code.co_name)
node_count = 0
m2m_count = 0
start = datetime.now()
total = MODMNode.find(build_query(m2m_node_fields, MODMNode),
allow_institution=True).count()
print '{} Nodes'.format(total)
while node_count < total:
with transaction.atomic():
for modm_node in MODMNode.find(
build_query(m2m_node_fields, MODMNode),
allow_institution=True).sort(
'-date_modified')[node_count:page_size + node_count]:
django_node = Node.objects.get(
pk=modm_to_django[modm_node._id])
for m2m_node_field in m2m_node_fields:
attr = getattr(django_node, m2m_node_field)
django_pks = []
for modm_m2m_value in getattr(modm_node, m2m_node_field,
[]):
if isinstance(modm_m2m_value, MODMNode):
django_pks.append(
modm_to_django[modm_m2m_value._id])
elif isinstance(modm_m2m_value, basestring):
django_pks.append(modm_to_django[modm_m2m_value])
elif isinstance(modm_m2m_value, Pointer):
django_pks.append(
modm_to_django[modm_m2m_value.node._id])
else:
# wth
print '\a'
print '\a'
print '\a'
print '\a'
print '\a'
print '\a'
print '\a'
import bpdb
bpdb.set_trace()
if len(django_pks) > 0:
attr.add(*django_pks)
m2m_count += len(django_pks)
node_count += 1
if node_count % page_size == 0 or node_count == total:
print 'Through {} nodes and {} m2m'.format(
node_count, m2m_count)
print 'Done with {} in {} seconds...'.format(
sys._getframe().f_code.co_name,
(datetime.now() - start).total_seconds())
def test_get_document():
access_token = config('TESTUSER_ID_TOKEN')
#access_token = get_access_token()['access_token']
headers = {
'Content-Type': 'application/json',
'authorization': 'Bearer ' + access_token
}
contents = {
#"会社_2__会社名": "アイ·エム·アイ株式会社"
"company__company_name": "imi"
}
result = requests.get(post_url, params=contents, headers=headers)
print(result)
from bpdb import set_trace
set_trace()
def create_rel_dict(df, column):
mod_to_class_dict = {}
class_to_mod_dict = {}
for i, row in tqdm(df.iterrows(), total=len(df)):
mod_to_class_dict[i] = row[column]
if row[column] not in class_to_mod_dict:
try:
class_to_mod_dict[row[column]] = [i]
except:
import bpdb
bpdb.set_trace()
else:
class_to_mod_dict[row[column]].append(i)
return mod_to_class_dict, class_to_mod_dict
def run_sim():
N = 5
epsilon_x = 0
epsilon_z = 0
for i in range(N):
succeeded = False
while not succeeded:
env = setup_env(config_file)
try:
env.run(until=env.MAX_TIME)
for agent in env.agents:
agent.cleanup()
eps_x, eps_z = check_NEES_NIS(env)
epsilon_x += eps_x
epsilon_z += eps_z
set_trace()
succeeded = True
except np.linalg.LinAlgError:
for agent in env.agents:
agent.cleanup()
continue
epsilon_x_bar = epsilon_x / N
epsilon_z_bar = epsilon_z / N
J_NEES = abs(log(epsilon_x_bar))
J_NIS = abs(log(epsilon_z_bar))
print(f"{J_NEES=}")
print(f"{J_NIS=}")
R = -(J_NEES + J_NIS)
return R
def do_doc2vec(df):
df_train, df_test, df_val = df_train_test_val_split(df[KEEP_COLS])
X_train, y_train, X_test, y_test, X_val, y_val = train_test_val_split(
df[KEEP_COLS], 'score')
train_dict, test_dict, val_dict = train_test_val_dicts(df, 'url', 'score')
# use grid search to optimize stage 1 & 2 model parameters
#grid_search_naive_bayes_with_tfidf(df_train['text'].values, df_train['stars'].values)
#grid_search_stage_two(X_train, y_train)
# fit and print classification results for the two stage tfidf model
#two_stage_model(X_train, y_train, X_test, y_test)
# fit and print classification results for the Doc2Vec model
model = doc2vec_model(df, train_dict, test_dict, k=25)
export_vec(model)
import bpdb
bpdb.set_trace()
# use the Doc2Vec model
doc2vec_examples(model)
def test_create_document():
access_token = config('ADMINUSER_ID_TOKEN')
#access_token = get_access_token()['access_token']
headers = {
'Content-Type': 'application/json',
'authorization': 'Bearer ' + access_token
}
contents = {
'name': 'frompython',
'tenant': 'testtenant2',
'document_type': 'invoice',
'document_content': {}
}
with open(json_path, 'r') as f:
content = json.load(f)
contents['document_content'] = json.dumps(content)
#contents['document_content'] = content
result = requests.post(post_url, json.dumps(contents), headers=headers)
print(result)
from bpdb import set_trace
set_trace()
def test_files(pkl_paths, csv_paths, commit_hash, version_number):
for pkl_path, csv_path in zip(pkl_paths, csv_paths):
csv = pd.read_csv(csv_path)
df = pd.read_pickle(pkl_path)
assert_equal(commit_hash, df._metadata['commit_hash'])
assert_equal(version_number, df._metadata['version_number'])
assert_equal(len(df), len(csv))
csv_ids = set(csv[csv.columns[0]])
df_ids = set(df.index)
assert csv_ids == df_ids, get_csv_error_message(
pkl_path, csv_path, csv_ids, df_ids)
columns_to_check = [
'narration_id', 'participant_id', 'video_id', 'start_timestamp',
'stop_timestamp', 'start_frame', 'stop_frame', 'narration',
'verb_class', 'noun_class', 'verb', 'noun'
]
if 'narration_id' in csv.columns:
csv = csv.set_index('narration_id')
for column in columns_to_check:
if column in df.columns:
try:
assert (csv[column] == df[column]).all()
except:
import bpdb
bpdb.set_trace()
raise Exception(
'Column {} doesn\'t match between {} and {}'.format(
column, csv_path, pkl_path))
if 'narration_timestamp' in df.columns:
train_missing = pd.read_csv(
'./EPIC_100_train_missing_timestamp_narrations.csv')
val_missing = pd.read_csv(
'./EPIC_100_validation_missing_timestamp_narrations.csv')
missing = set(pd.concat([train_missing, val_missing]).narration_id)
diff = set(df[
csv['narration_timestamp'] != df['narration_timestamp']].index)
assert len(diff) == 0 or diff.issubset(missing)
def set_node_foreign_keys_on_nodes(page_size=10000):
print 'Starting {}...'.format(sys._getframe().f_code.co_name)
node_count = 0
fk_count = 0
cache_hits = 0
cache_misses = 0
start = datetime.now()
total = MODMNode.find(
build_query(fk_node_fields, MODMNode),
allow_institution=True).count()
while node_count < total:
with transaction.atomic():
for modm_node in MODMNode.find(
build_query(fk_node_fields, MODMNode),
allow_institution=True).sort('-date_modified')[
node_count:node_count + page_size]:
django_node = Node.objects.get(_guid__guid=modm_node._id)
for fk_node_field in fk_node_fields:
value = getattr(modm_node, fk_node_field, None)
if value is not None:
if isinstance(value, basestring):
# value is a guid, try the cache table for the pk
if value in modm_to_django:
setattr(django_node,
'{}_id'.format(fk_node_field),
modm_to_django[value])
cache_hits += 1
else:
# it's not in the cache, do the query
node_id = Node.objects.get(
_guid__guid=value).pk
setattr(django_node,
'{}_id'.format(fk_node_field), node_id)
# save for later
modm_to_django[value] = node_id
cache_misses += 1
elif isinstance(value, MODMNode):
# value is a node object, try the cache table for the pk
if value._id in modm_to_django:
setattr(django_node,
'{}_id'.format(fk_node_field),
modm_to_django[value._id])
cache_hits += 1
else:
# it's not in the cache, do the query
node_id = Node.objects.get(
_guid__guid=value._id).pk
setattr(django_node,
'{}_id'.format(fk_node_field), node_id)
# save for later
modm_to_django[value._id] = node_id
cache_misses += 1
else:
# whu happened?
print '\a'
print '\a'
print '\a'
print '\a'
print '\a'
print '\a'
import bpdb
bpdb.set_trace()
fk_count += 1
django_node.save()
node_count += 1
if node_count % page_size == 0 or node_count == total:
then = datetime.now()
print 'Through {} nodes and {} foreign keys'.format(
node_count, fk_count)
print 'Cache: Hits {} Misses {}'.format(cache_hits,
cache_misses)
print 'Done with {} in {} seconds...'.format(
sys._getframe().f_code.co_name,
(datetime.now() - start).total_seconds())
def set_user_foreign_keys_on_users(page_size=10000):
print 'Starting {}...'.format(sys._getframe().f_code.co_name)
user_count = 0
fk_count = 0
cache_hits = 0
cache_misses = 0
start = datetime.now()
total = MODMUser.find(build_query(fk_user_fields, MODMUser)).count()
while user_count < total:
with transaction.atomic():
for modm_user in MODMUser.find(build_query(
fk_user_fields, MODMUser)).sort('-date_registered')[
user_count:user_count + page_size]:
django_user = User.objects.get(_guid__guid=modm_user._id)
for fk_user_field in fk_user_fields:
value = getattr(modm_user, fk_user_field, None)
if value is not None:
if isinstance(value, basestring):
# value is a guid, try the cache table for the pk
if value in modm_to_django:
setattr(django_user,
'{}_id'.format(fk_user_field),
modm_to_django[value])
cache_hits += 1
else:
# it's not in the cache, do the query
user_id = User.objects.get(
_guid__guid=value).pk
setattr(django_user,
'{}_id'.format(fk_user_field), user_id)
# save for later
modm_to_django[value] = user_id
cache_misses += 1
elif isinstance(value, MODMUser):
# value is a user object, try the cache table for the pk
if value._id in modm_to_django:
setattr(django_user,
'{}_id'.format(fk_user_field),
modm_to_django[value._id])
cache_hits += 1
else:
# it's not in the cache, do the query
user_id = User.objects.get(
_guid__guid=value._id).pk
setattr(django_user,
'{}_id'.format(fk_user_field), user_id)
# save for later
modm_to_django[value._id] = user_id
cache_misses += 1
else:
# that's odd.
print '\a'
print '\a'
print '\a'
print '\a'
print '\a'
print '\a'
import bpdb
bpdb.set_trace()
fk_count += 1
django_user.save()
user_count += 1
if user_count % page_size == 0 or user_count == total:
print 'Through {} users and {} foreign keys'.format(
user_count, fk_count)
print 'Cache: Hits {} Misses {}'.format(cache_hits,
cache_misses)
print 'Done with {} in {} seconds...'.format(
sys._getframe().f_code.co_name,
(datetime.now() - start).total_seconds())
def set_user_many_to_many_on_nodes(page_size=5000):
print 'Starting {}...'.format(sys._getframe().f_code.co_name)
node_count = 0
m2m_count = 0
start = datetime.now()
total = MODMNode.find(build_query(m2m_user_fields, MODMNode),
allow_institution=True).count()
print '{} Nodes'.format(total)
while node_count < total:
with transaction.atomic():
for modm_node in MODMNode.find(
build_query(m2m_user_fields, MODMNode),
allow_institution=True).sort(
'-date_modified')[node_count:page_size + node_count]:
django_node = Node.objects.get(
pk=modm_to_django[modm_node._id])
for m2m_user_field in m2m_user_fields:
if m2m_user_field in ['permissions', 'recently_added']:
continue
attr = getattr(django_node, m2m_user_field)
django_pks = []
for modm_m2m_value in getattr(modm_node, m2m_user_field,
[]):
if isinstance(modm_m2m_value, MODMUser):
if m2m_user_field == 'contributors':
visible = modm_m2m_value._id in modm_node.visible_contributor_ids
admin = 'admin' in modm_node.permissions[
modm_m2m_value._id]
read = 'read' in modm_node.permissions[
modm_m2m_value._id]
write = 'write' in modm_node.permissions[
modm_m2m_value._id]
Contributor.objects.get_or_create(
user_id=modm_to_django[modm_m2m_value._id],
node=django_node,
visible=visible,
admin=admin,
read=read,
write=write)
m2m_count += 1
else:
django_pks.append(
modm_to_django[modm_m2m_value._id])
elif isinstance(modm_m2m_value, basestring):
if m2m_user_field == 'contributors':
visible = modm_m2m_value in modm_node.visible_contributor_ids
admin = 'admin' in modm_node.permissions[
modm_m2m_value]
read = 'read' in modm_node.permissions[
modm_m2m_value]
write = 'write' in modm_node.permissions[
modm_m2m_value]
Contributor.objects.get_or_create(
user_id=modm_to_django[modm_m2m_value],
node=django_node,
visible=visible,
admin=admin,
read=read,
write=write)
m2m_count += 1
else:
django_pks.append(
modm_to_django[modm_m2m_value])
else:
# wth
print '\a' # bells
print '\a'
print '\a'
print '\a'
print '\a'
print '\a'
print '\a'
import bpdb
bpdb.set_trace()
if len(django_pks) > 0:
attr.add(*django_pks)
m2m_count += len(django_pks)
node_count += 1
if node_count % page_size == 0 or node_count == total:
print 'Through {} nodes and {} m2m'.format(
node_count, m2m_count)
print 'Done with {} in {} seconds...'.format(
sys._getframe().f_code.co_name,
(datetime.now() - start).total_seconds())
def set_user_foreign_keys_on_users(page_size=10000):
print 'Starting {}...'.format(sys._getframe().f_code.co_name)
user_count = 0
fk_count = 0
cache_hits = 0
cache_misses = 0
start = datetime.now()
total = MODMUser.find(build_query(fk_user_fields, MODMUser)).count()
while user_count < total:
with transaction.atomic():
for modm_user in MODMUser.find(
build_query(fk_user_fields, MODMUser)).sort(
'-date_registered')[user_count:user_count + page_size]:
django_user = User.objects.get(_guid__guid=modm_user._id)
for fk_user_field in fk_user_fields:
value = getattr(modm_user, fk_user_field, None)
if value is not None:
if isinstance(value, basestring):
# value is a guid, try the cache table for the pk
if value in modm_to_django:
setattr(django_user,
'{}_id'.format(fk_user_field),
modm_to_django[value])
cache_hits += 1
else:
# it's not in the cache, do the query
user_id = User.objects.get(
_guid__guid=value).pk
setattr(django_user,
'{}_id'.format(fk_user_field), user_id)
# save for later
modm_to_django[value] = user_id
cache_misses += 1
elif isinstance(value, MODMUser):
# value is a user object, try the cache table for the pk
if value._id in modm_to_django:
setattr(django_user,
'{}_id'.format(fk_user_field),
modm_to_django[value._id])
cache_hits += 1
else:
# it's not in the cache, do the query
user_id = User.objects.get(
_guid__guid=value._id).pk
setattr(django_user,
'{}_id'.format(fk_user_field), user_id)
# save for later
modm_to_django[value._id] = user_id
cache_misses += 1
else:
# that's odd.
print '\a'
print '\a'
print '\a'
print '\a'
print '\a'
print '\a'
import bpdb
bpdb.set_trace()
fk_count += 1
django_user.save()
user_count += 1
if user_count % page_size == 0 or user_count == total:
print 'Through {} users and {} foreign keys'.format(
user_count, fk_count)
print 'Cache: Hits {} Misses {}'.format(
cache_hits, cache_misses)
print 'Done with {} in {} seconds...'.format(
sys._getframe().f_code.co_name,
(datetime.now() - start).total_seconds())
_germs_lite = {upgrade_circuits('germs_lite')}
_fiducials = {upgrade_circuits('fiducials')}
_prepStrs = {upgrade_circuits('prepStrs')}
_effectStrs = {upgrade_circuits('effectStrs')}
clifford_compilation = {cc_src}
global_fidPairs = {name_or_none('global_fidPairs')}
pergerm_fidPairsDict = {name_or_none('pergerm_fidPairsDict')}
global_fidPairs_lite = {name_or_none('global_fidPairs_lite')}
pergerm_fidPairsDict_lite = {name_or_none('pergerm_fidPairsDict_lite')}
@property
def _target_model(self):
return {target_model_src}
import sys
sys.modules[__name__] = _Module()
"""
import bpdb; bpdb.set_trace()
# Dump it to stdout
print(template_str)
def set_user_many_to_many_on_nodes(page_size=5000):
print 'Starting {}...'.format(sys._getframe().f_code.co_name)
node_count = 0
m2m_count = 0
start = datetime.now()
total = MODMNode.find(
build_query(m2m_user_fields, MODMNode),
allow_institution=True).count()
print '{} Nodes'.format(total)
while node_count < total:
with transaction.atomic():
for modm_node in MODMNode.find(
build_query(m2m_user_fields, MODMNode),
allow_institution=True).sort('-date_modified')[
node_count:page_size + node_count]:
django_node = Node.objects.get(
pk=modm_to_django[modm_node._id])
for m2m_user_field in m2m_user_fields:
if m2m_user_field in ['permissions', 'recently_added']:
continue
attr = getattr(django_node, m2m_user_field)
django_pks = []
for modm_m2m_value in getattr(modm_node, m2m_user_field,
[]):
if isinstance(modm_m2m_value, MODMUser):
if m2m_user_field == 'contributors':
visible = modm_m2m_value._id in modm_node.visible_contributor_ids
admin = 'admin' in modm_node.permissions[
modm_m2m_value._id]
read = 'read' in modm_node.permissions[
modm_m2m_value._id]
write = 'write' in modm_node.permissions[
modm_m2m_value._id]
Contributor.objects.get_or_create(
user_id=modm_to_django[modm_m2m_value._id],
node=django_node,
visible=visible,
admin=admin,
read=read,
write=write)
m2m_count += 1
else:
django_pks.append(modm_to_django[
modm_m2m_value._id])
elif isinstance(modm_m2m_value, basestring):
if m2m_user_field == 'contributors':
visible = modm_m2m_value in modm_node.visible_contributor_ids
admin = 'admin' in modm_node.permissions[
modm_m2m_value]
read = 'read' in modm_node.permissions[
modm_m2m_value]
write = 'write' in modm_node.permissions[
modm_m2m_value]
Contributor.objects.get_or_create(
user_id=modm_to_django[modm_m2m_value],
node=django_node,
visible=visible,
admin=admin,
read=read,
write=write)
m2m_count += 1
else:
django_pks.append(modm_to_django[
modm_m2m_value])
else:
# wth
print '\a' # bells
print '\a'
print '\a'
print '\a'
print '\a'
print '\a'
print '\a'
import bpdb
bpdb.set_trace()
if len(django_pks) > 0:
attr.add(*django_pks)
m2m_count += len(django_pks)
node_count += 1
if node_count % page_size == 0 or node_count == total:
print 'Through {} nodes and {} m2m'.format(node_count,
m2m_count)
print 'Done with {} in {} seconds...'.format(
sys._getframe().f_code.co_name,
(datetime.now() - start).total_seconds())
J_NEES = abs(log(epsilon_x_bar))
J_NIS = abs(log(epsilon_z_bar))
print(f"{J_NEES=}")
print(f"{J_NIS=}")
R = -(J_NEES + J_NIS)
return R
# pbounds = {"q": (-2, 2), "t_window": (-3, 0)}
# optimizer = BayesianOptimization(
# f=lambda q, t_window: run_sim(str(10 ** q), str(10 ** q), str(10 ** t_window)),
# pbounds=pbounds,
# random_state=1,
# )
# optimizer.maximize(
# init_points=5,
# n_iter=45,
# )
# print(optimizer.max)
R = run_sim()
set_trace()
def set_node_foreign_keys_on_nodes(page_size=10000):
print 'Starting {}...'.format(sys._getframe().f_code.co_name)
node_count = 0
fk_count = 0
cache_hits = 0
cache_misses = 0
start = datetime.now()
total = MODMNode.find(build_query(fk_node_fields, MODMNode),
allow_institution=True).count()
while node_count < total:
with transaction.atomic():
for modm_node in MODMNode.find(
build_query(fk_node_fields, MODMNode),
allow_institution=True).sort(
'-date_modified')[node_count:node_count + page_size]:
django_node = Node.objects.get(_guid__guid=modm_node._id)
for fk_node_field in fk_node_fields:
value = getattr(modm_node, fk_node_field, None)
if value is not None:
if isinstance(value, basestring):
# value is a guid, try the cache table for the pk
if value in modm_to_django:
setattr(django_node,
'{}_id'.format(fk_node_field),
modm_to_django[value])
cache_hits += 1
else:
# it's not in the cache, do the query
node_id = Node.objects.get(
_guid__guid=value).pk
setattr(django_node,
'{}_id'.format(fk_node_field), node_id)
# save for later
modm_to_django[value] = node_id
cache_misses += 1
elif isinstance(value, MODMNode):
# value is a node object, try the cache table for the pk
if value._id in modm_to_django:
setattr(django_node,
'{}_id'.format(fk_node_field),
modm_to_django[value._id])
cache_hits += 1
else:
# it's not in the cache, do the query
node_id = Node.objects.get(
_guid__guid=value._id).pk
setattr(django_node,
'{}_id'.format(fk_node_field), node_id)
# save for later
modm_to_django[value._id] = node_id
cache_misses += 1
else:
# whu happened?
print '\a'
print '\a'
print '\a'
print '\a'
print '\a'
print '\a'
import bpdb
bpdb.set_trace()
fk_count += 1
django_node.save()
node_count += 1
if node_count % page_size == 0 or node_count == total:
then = datetime.now()
print 'Through {} nodes and {} foreign keys'.format(
node_count, fk_count)
print 'Cache: Hits {} Misses {}'.format(
cache_hits, cache_misses)
print 'Done with {} in {} seconds...'.format(
sys._getframe().f_code.co_name,
(datetime.now() - start).total_seconds())
_germs_lite = {upgrade_circuits('germs_lite')}
_fiducials = {upgrade_circuits('fiducials')}
_prepStrs = {upgrade_circuits('prepStrs')}
_effectStrs = {upgrade_circuits('effectStrs')}
clifford_compilation = {cc_src}
global_fidPairs = {name_or_none('global_fidPairs')}
pergerm_fidPairsDict = {name_or_none('pergerm_fidPairsDict')}
global_fidPairs_lite = {name_or_none('global_fidPairs_lite')}
pergerm_fidPairsDict_lite = {name_or_none('pergerm_fidPairsDict_lite')}
@property
def _target_model(self):
return {target_model_src}
import sys
sys.modules[__name__] = _Module()
"""
import bpdb
bpdb.set_trace()
# Dump it to stdout
print(template_str)
def autokick():
acura = 20000
watched = False
watchcnt = 0
axi = False
if debug:
bpdb.set_trace()
while True:
disper.upda(xx.pos, yy.pos)
distx = 0
disty = 0
slop = (xx.acu + yy.acu) / 1000
if disper.prox > 7:
watched = True
watchcnt = 10 #lower this!
else:
if watched:
watchcnt -= 1
if watchcnt < 0:
watched = False
xid = xx.think()
yid = yy.think()
xx.direct()
yy.direct()
dirs = crt.nocks(xx.pos, yy.pos)
dirx = dirs[0]
diry = dirs[1]
#flip flop
axi = randint(0, 13) % 2
if dirx == diry:
if axi:
#axi = False
if dirx == 0:
dirx = randir()
diry = 0
else:
#axi = True
if diry == 0:
diry = randir()
dirx = 0
sizer = (1500 - crt.sumer() * 4) + 100 #times 2...
mag = randint(100, 2 * sizer)
style = 2
if watched:
fly(dirx * mag, diry * mag, sizer)
cross(sizer, style)
#poke(style)
# time.sleep(0.5)
# poke(0)
if debug:
print " I am %d sloppy and %d watched. Thinking of %d %d this big %d" % (
slop, watched, xid, yid, sizer)
print "pos %d %d ORDIR %d %d TARG %d %d" % (
xx.pos, yy.pos, dirs[0], dirs[1], dirx, diry)
