def process_and_save(self, news): news_object = build_news_object(news, SOURCE_NAME) if is_valid(news_object): news_object.link = BASE_URL+str(news_object.link) # print news_object self.do_save(news_object)
def __init__(self, from_addr, outputs, privKey, fee=None,
change_addr=None):
if not change_addr:
change_addr = from_addr
if fee is None:
fee = 100
for address, value in outputs:
assert is_valid(address)
amount = sum(map(lambda x: x[1], outputs))
inputs = self.getInputs(from_addr)
total = sum(map(lambda x: x[1], inputs))
change_amount = total - (amount + fee)
print 'Change:', change_amount
print 'Total:', total
print 'Amount:', amount
if change_amount < 0:
raise TransactionError('Not enought funds, or transaction using'
' all inputs need to be confirmed')
if amount < 1000000 and fee < 50000:
raise TransactionError("Not enought fee :"
" a fee of 0.0005 is required for"
" small transactions")
outputs.append((change_addr, change_amount))
self.tx = self.signed_tx(inputs, outputs, privKey)
if not self.pushTx():
raise TransactionError("An error occur while sending transaction")
else:
raise TransactionSubmitted("Transaction successfully transmitted")
def validate():
""" Uploads a file to a draft area then tests if it is valid.
Expects:
index_view_slug: unique name of tif file to lookup(string)
file: in request.files for upload
"""
index_view_slug = request.form.get('index_view_slug')
if not index_view_slug or 'file' not in request.files:
abort(400)
if index_view_slug.endswith('_raster'):
index_view_slug = index_view_slug[:-7]
draft_file = os.path.join(conf.DRAFT_DIR,
'{}.tif'.format(index_view_slug))
request.files['file'].save(draft_file)
result = {'errors': []}
try:
valid = is_valid(
index_view_slug, base_path=conf.DRAFT_DIR
)
result = {
'md5': md5sum(draft_file),
'valid': valid
}
os.remove(draft_file)
except NotImplementedError as e:
result['errors'].append(e.message)
return jsonify(result)
def publish():
""" Uploads a file to a draft area then moves it to data directory if valid.
Expects:
index_view_slug: unique name of tif file to lookup(string)
file: in request.files for upload
"""
index_view_slug = request.form.get('index_view_slug')
if not index_view_slug or 'file' not in request.files:
abort(400)
if index_view_slug.endswith('_raster'):
index_view_slug = index_view_slug[:-7]
draft_file = os.path.join(conf.DRAFT_DIR,
'{}.tif'.format(index_view_slug))
request.files['file'].save(draft_file)
result = {'errors': []}
try:
result['valid'] = is_valid(index_view_slug, base_path=conf.DRAFT_DIR)
if result['valid']:
live_file = make_live(draft_file)
if os.path.exists(live_file):
result = {
'md5': md5sum(live_file),
}
#except Exception as e:
except NotImplementedError as e:
result['errors'].append(e.message)
return jsonify(result)
def bands_files_are_valid(img, bands, api, directory):
"""
Check if all bands images files are valid.
"""
filenames = ['{}_band_{}.tif'.format(img['filename'], b) for b in bands]
paths = [os.path.join(directory, f) for f in filenames]
return all(utils.is_valid(p) for p in paths)
def log_prob_to_change(log_prob_1_given_0,
log_prob_0_given_0,
multiple,
hard_cutoff=None):
'''NLL to change scaled from 0 to 1'''
#Clamp rare -infs to min non inf val
#print(f'Min self probs: {log_prob_0_given_0.min()}')
log_prob_1_given_0 = clamp_infs(log_prob_1_given_0)
log_prob_0_given_0 = clamp_infs(log_prob_0_given_0)
if hard_cutoff == None:
# Get statistics of 0 given 0 for comparison
base_mean = log_prob_0_given_0.mean(dim=-1).unsqueeze(-1)
base_std = log_prob_0_given_0.std(dim=-1).unsqueeze(-1)
#print(f'Mean {base_mean.item()} std: {base_std.item()}')
# Minimum change criterion (all values smaller than base_mean by more than multiple*base_std)
changed_mask = log_prob_1_given_0 < base_mean - multiple * base_std
else:
changed_mask = log_prob_1_given_0 < hard_cutoff
max_change = log_prob_1_given_0.max(dim=-1)[0].unsqueeze(-1)
min_change = log_prob_1_given_0.min(dim=-1)[0].unsqueeze(-1)
log_prob_1_given_0 = 1 - (log_prob_1_given_0 - min_change) / (max_change -
min_change)
log_prob_1_given_0[~changed_mask] = 0.0
assert is_valid(log_prob_1_given_0)
return log_prob_1_given_0
def bands_files_are_valid(img, bands, search_api, directory):
"""
Check if all bands images files are valid.
"""
name = filename_from_metadata_dict(img, search_api)
filenames = ['{}_band_{}.tif'.format(name, b) for b in bands]
paths = [os.path.join(directory, f) for f in filenames]
return all(utils.is_valid(p) for p in paths)
async def oauth_redirect(request: Request, path: str, state: str = ""):
target_domain, _, _ = state.partition("|")
path = f"/{path}"
if not is_valid(target_domain, path):
log.info("unauthorized domain=%s path=%s", target_domain, path)
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST)
full_url = f"https://{target_domain}{path}"
return responses.RedirectResponse(full_url, status_code=302)
def main():
'''
Launch method
'''
parse_ssv()
while True:
number = get_input('Please enter a credit card number:')
number = (regex_check_for_errors(number))
if utils.is_valid(number):
print_credit_card_results(number)
def fetch(args):
print 'Welcome to Twitter Spell Checking : Fetching !'
CONFIG = ConfigParser.ConfigParser()
CONFIG.read(args.config)
settings = items_to_dict(CONFIG.items('twitter'))
config = items_to_dict(CONFIG.items('namespace:%s' % args.namespace))
api = twitter.Api(consumer_key=settings['consumer_key'], consumer_secret=settings['consumer_secret'], access_token_key=settings['access_token'], access_token_secret=settings['access_token_secret'])
accounts = [account.replace(' ', '') for account in config['accounts'].split(',')]
max_tweets_file = os.path.join(os.path.dirname(config['files']), 'max_tweets_%s.txt' % args.namespace)
def save_max_tweets():
open(max_tweets_file, 'w').write(json.dumps(max_tweets))
if os.path.exists(max_tweets_file):
max_tweets = json.loads(open(max_tweets_file).read())
else:
max_tweets = dict()
print max_tweets_file
f = open(os.path.join(config['files'], 'tweets_%s.txt' % args.namespace), 'a')
for account in accounts:
if account in max_tweets and max_tweets[account] > 0:
retrieving = "new"
else:
retrieving = "old"
page = 0
while True:
if retrieving == "new":
print 'process %s since id %s' % (account, max_tweets[account])
try:
tweets = api.GetUserTimeline(account, count=200, include_rts=False, since_id=max_tweets[account])
except twitter.TwitterError, e:
print 'error : %s' % str(e)
tweets = []
else:
print 'process %s from zero, page %s' % (account, page)
try:
tweets = api.GetUserTimeline(account, count=200, include_rts=False, page=page)
except twitter.TwitterError, e:
print 'error : %s' % str(e)
tweets = []
if tweets:
for s in tweets:
if is_valid(s, account):
f.write(clean(s.text).lower().encode('UTF-8') + '\n')
if account not in max_tweets or s.id > max_tweets[account]:
max_tweets[account] = s.id
if retrieving == "old":
page += 1
save_max_tweets()
else:
print 'no more tweets for %s' % account
break
def change_program_properties(self):
def is_valid(mystring, mybeg, myend):
if not utils.is_int(mystring): return False
myint = int(mystring)
while not (myint >= mybeg and myint <= myend):
return False
return True
# -----------------------------------------
print("Entering procedure change_program_properties")
voice_properties = [
"change rate", "change volume", "change voice", "quit"
]
for count, elem in enumerate(voice_properties):
print("{}) {}".format(count + 1, elem))
user_input = input("> ").lower().strip()
while not utils.is_valid(user_input, voice_properties):
user_input = input("> ").lower().strip()
if user_input == "quit": sys.exit()
print("You chose: {}".format(user_input))
user_input = int(user_input)
user_choice = voice_properties[user_input - 1]
print(user_choice)
# -----------------------------------------
if user_choice == "change rate":
print(
"The defaut for 'rate' is 200. What would you like to change it to? (1 to 400)"
)
user_input = input("> ").lower().strip()
while not is_valid(user_input, mybeg=1, myend=400):
user_input = input("> ").lower().strip()
self.engine.setProperty('rate', user_input)
self.speak()
elif user_choice == "change volume":
print(
"The defaut for 'volume' is 1. What would you like to change it to? (0 to 3)"
)
user_input = input("> ").lower().strip()
while not is_valid(user_input, 0, 3):
user_input = input("> ").lower().strip()
self.engine.setProperty('volume', user_input)
self.speak()
elif user_choice == "change voice":
print("You are about to change the voice.")
# print("Would you like more information about the available voices? (y/n)")
# user_input = ""
# while not user_input in ["y", "yes", "n", "no"]:
# user_input = input("> ").lower().strip()
# if user_input in ["y", "yes"]:
self.select_voice()
elif user_choice == "quit":
raise NotImplemented
else:
raise ValueError("Error!")
def test_is_valid(self):
self.assertTrue(utils.is_valid('5558397375275489'))
self.assertTrue(utils.is_valid('4573055613536303098'))
self.assertFalse(utils.is_valid('abc'))
self.assertFalse(utils.is_valid(''))
self.assertFalse(utils.is_valid('!'))
self.assertFalse(utils.is_valid('abc123'))
def new_url():
"""
Create short url.
"""
if request.method == 'POST':
original_url = request.data.get('url')
if not original_url:
return Response('No url to shorten was provided', 401)
if not is_valid(original_url):
return Response('No valid url to was provided', 401)
short_url = shortify(original_url)
db.store(short_url, original_url)
base = request.url
return {'shortened_url': base + short_url}, status.HTTP_201_CREATED
def do_save(self, news_object, tags=None, category=None):
if tags != None:
news_object.tags = tags
if category != None:
news_object.category = category
try:
if is_valid(news_object):
news_object.save()
return True
else:
logging.warn('Already exists!')
return False
except Exception, e:
# import pdb;pdb.set_trace()
logging.error("Error Occured! %s" % e)
print "Error Occured! %s" % e
pass
def main(FLAGS):
print("#" * 50)
print("# Parameters:", FLAGS)
print("#" * 50)
# read a set of configuration parameters
config_params = get_config(FLAGS.config_file)
print(config_params)
assert is_valid(FLAGS, config_params), \
"Please check out parameters again."
if FLAGS.mode == "single":
print("Single-Field Mode")
single(FLAGS, config_params)
elif FLAGS.mode == "multi":
print("Multi-Field Mode")
multi(FLAGS, config_params)
def __init__(self,
from_addr,
outputs,
privKey,
fee=None,
change_addr=None):
if not change_addr:
change_addr = from_addr
if fee is None:
fee = 100
for address, value in outputs:
assert is_valid(address)
amount = sum(map(lambda x: x[1], outputs))
inputs = self.getInputs(from_addr)
total = sum(map(lambda x: x[1], inputs))
change_amount = total - (amount + fee)
print 'Change:', change_amount
print 'Total:', total
print 'Amount:', amount
if change_amount < 0:
raise TransactionError('Not enought funds, or transaction using'
' all inputs need to be confirmed')
if amount < 1000000 and fee < 10000:
raise TransactionError("Not enought fee :"
" a fee of 0.0001 is required for"
" small transactions")
if change_amount != 0:
outputs.append((change_addr, change_amount))
self.tx = self.signed_tx(inputs, outputs, privKey)
if not self.pushTx():
raise TransactionError("An error occur while sending transaction")
else:
raise TransactionSubmitted("Transaction successfully transmitted")
def simulated_annealing(graph,
num_buses,
size_bus,
constraints,
test_mode=False):
all_names = graph.nodes()
x = all_names[0]
print(graph[x])
print(graph[x]['weight'])
CD = constraint_dictionary(all_names, constraints)
#assignment = random_assignment(all_names, num_buses, size_bus)
assignment = greedy(graph, num_buses, size_bus, constraints)
groups = assignment_to_groups(assignment)
assert is_valid(graph, num_buses, size_bus, assignment)
T0 = 1.0
iterations = 5000
student_status = calculate_student_status(graph, constraints, assignment)
starting_score = evaluate_assignment(graph, constraints, assignment)
current_score = starting_score
for i in range(iterations):
T = T0 * (1 - i / iterations)
swap = propose_swap(assignment, num_buses, size_bus, groups=groups)
if swap is None:
continue
val, student_status_swap = swap_value(graph, constraints, CD,
student_status, assignment, swap)
if val >= 0 or random.random() < np.exp(val / T):
group_A, A, group_B, B = swap
if A is not None:
assignment[A] = group_B
groups[group_B].append(A)
groups[group_A].remove(A)
assignment[B] = group_A
groups[group_A].append(B)
groups[group_B].remove(B)
student_status = student_status_swap
if (test_mode):
new_score = evaluate_assignment(graph, constraints, assignment)
student_status_check = calculate_student_status(
graph, constraints, assignment)
for s in student_status_check:
assert student_status[s] == student_status_check[s]
if new_score != current_score + val:
print(new_score, current_score, val)
print(swap)
print(student_status)
print(assignment_to_groups(assignment))
print(constraints)
nodes = []
for x in assignment:
if assignment[x] == group_A or assignment[x] == group_B:
nodes.append(x)
import matplotlib.pyplot as plt
subgraph = nx.subgraph(graph, nodes)
nx.draw(subgraph, with_labels=True, font_weight='bold')
plt.show()
assert False
current_score = new_score
assert is_valid(graph, num_buses, size_bus, assignment)
#print('Score:', evaluate_assignment(graph, constraints, assignment))
return assignment
def ilp(graph, num_buses, size_bus, constraints):
edges = graph.edges()
buses = range(0, num_buses)
num_students = graph.nodes()
cxt = Context.make_default_context()
cxt.timeout = 1
with Model(name="solver", context=cxt) as mdl:
X_dict = {}
es = []
for s in graph.nodes():
name = "x" + str(s)
xslist = mdl.binary_var_list(keys=num_buses, name=name)
X_dict[s] = xslist
for s in graph.nodes():
cons = find_student_in_constraints(s, constraints)
xslist = X_dict[s]
constraining_values = []
for b in buses:
for C in cons:
relevant_dvs_names = ["x" + s + "_" + str(b) for s in C]
relevant_dvs = [
mdl.get_var_by_name(name)
for name in relevant_dvs_names
]
constraining_values.append(len(C) - mdl.sum(relevant_dvs))
rs = mdl.integer_var(name="r" + s)
expr = (rs <= mdl.min(constraining_values))
mdl.add_constraint(expr)
for e in edges:
name = "e" + e[0] + e[1]
edge_indic = mdl.binary_var_list(keys=num_buses, name=name)
es.extend(edge_indic)
obj = mdl.sum(es)
mdl.set_objective("max", obj)
for e in edges:
for b in buses:
u = e[0]
v = e[1]
name = "e" + u + v + "_" + str(b)
uexists = "x" + u + "_" + str(b)
vexists = "x" + v + "_" + str(b)
rsnameu = "r" + u
rsnamev = "r" + v
euvi = mdl.get_var_by_name(name)
xui = mdl.get_var_by_name(uexists)
xvi = mdl.get_var_by_name(vexists)
rsu = mdl.get_var_by_name(rsnameu)
rsv = mdl.get_var_by_name(rsnamev)
mdl.add_constraint((euvi <= xui))
mdl.add_constraint((euvi <= xvi))
mdl.add_constraint((euvi <= rsu))
mdl.add_constraint((euvi <= rsv))
for xslist in X_dict.values():
expr = (mdl.sum(xslist) == 1)
mdl.add_constraint(expr)
for b in buses:
relevant_list = [xslist[b] for xslist in X_dict.values()]
expr = (mdl.sum(relevant_list) <= size_bus)
expr2 = (mdl.sum(relevant_list) >= 1)
mdl.add_constraint(expr)
mdl.add_constraint(expr2)
mdl.set_time_limit(90)
solution = mdl.solve(log_output=True)
solved = []
for i in range(0, num_buses):
solved.append([])
iterv = solution.iter_var_values()
v = next(iterv)
i = 0
while i < (len(graph.nodes())):
name = str(v[0])
index = name.find('_')
student = name[1:index]
bus = int(name[index + 1:])
solved[bus].append(student)
i += 1
v = next(iterv)
print(solved)
assignment = utils.groups_to_assignment(solved)
if not (utils.is_valid(graph, num_buses, size_bus, assignment)):
print('\n')
print('NEET')
print('\n')
return None
print('yeet')
return assignment
def _validate(index_view_slug):
return is_valid(
index_view_slug, base_path=conf.DRAFT_DIR
)
def update_product(id):
"""
Update a Product
This endpoint will update a Product based on the data that is posted
---
tags:
- Products
consumes:
- application/json
produces:
- application/json
parameters:
- name: id
in: path
description: ID of product to retrieve
type: integer
required: true
- in: body
name: body
schema:
id: data
required:
- type
- quantity
properties:
type:
type: string
description: type for the Product
quantity:
type: integer
description: the quantity type you specified
responses:
200:
description: Product returned
schema:
id: Product
properties:
product_id:
type: integer
description: unique id assigned internally by service
location_id:
type: integer
description: unique location id assigned internally by service
restock_level:
type: integer
description: max space allocated for the product
new:
type: integer
description: quantity of new products
used:
type: integer
description: quantity of used products
open_box:
type: integer
description: quantity of open_box products
400:
description: Bad Request (the posted data was not valid)
404:
description: Product not found
"""
data = inventory.get_product(id)
info = request.get_json()
if data is None:
return make_response("Product not found", HTTP_404_NOT_FOUND)
if not utils.is_valid(info):
return make_response("Product data is not valid", HTTP_400_BAD_REQUEST)
total = int(data[USED]) + int(data[NEW]) + int(data[OPEN_BOX])
prod_type = info[TYPE]
if total - int(data[prod_type]) + int(info[QUANTITY]) > int(
data[RESTOCK_LEVEL]):
return make_response("Product amount exceed restock level",
HTTP_400_BAD_REQUEST)
if int(info[QUANTITY]) < 0:
return make_response("Product amount below zero", HTTP_400_BAD_REQUEST)
data[prod_type] = int(info[QUANTITY])
inventory.put_product(id, data)
return make_response(jsonify(data), HTTP_200_OK)
def view_index(self,
index,
multiple=3.,
gen_std=0.6,
hard_cutoff=None,
point_size=5):
sample_distrib = models.Normal(torch.zeros(1),
torch.ones(1) * gen_std,
shape=(2000,
config['latent_dim'])).to(device)
dataset_out = self.dataset[index]
dataset_out = [x.to(device).unsqueeze(0) for x in dataset_out]
voxel_0_large, voxel_1_small, extra_context, voxel_1_large_self, voxel_1_small_self, voxel_opposite_small, voxel_opposite_large, voxel_0_small_self, voxel_0_large_self, voxel_0_small_original, voxel_1_small_original = dataset_out
if not self.config['using_extra_context']:
extra_context = None
batch_1_0 = [voxel_0_large, voxel_1_small, extra_context]
batch_0_1 = [voxel_opposite_large, voxel_opposite_small, extra_context]
batch_0_0 = [voxel_0_large_self, voxel_0_small_self, extra_context]
batch_1_1 = [voxel_1_large_self, voxel_1_small_self, extra_context]
loss, log_prob_1_0, nats = inner_loop(batch_1_0, self.model_dict,
config)
_, log_prob_0_0, _ = inner_loop(batch_0_0, self.model_dict, config)
assert is_valid(log_prob_1_0)
change_1_0 = log_prob_to_change(log_prob_1_0,
log_prob_0_0,
multiple=multiple,
hard_cutoff=hard_cutoff)
assert is_valid(change_1_0)
is_valid(loss)
loss, log_prob_0_1, nats = inner_loop(batch_0_1, self.model_dict,
config)
_, log_prob_1_1, _ = inner_loop(batch_1_1, self.model_dict, config)
change_0_1 = log_prob_to_change(log_prob_0_1,
log_prob_1_1,
multiple=multiple,
hard_cutoff=hard_cutoff)
assert is_valid(log_prob_0_1)
sample_points_given_0 = make_sample(
n_points=4000,
extract_0=voxel_0_large[0].unsqueeze(0),
models_dict=self.model_dict,
config=config,
sample_distrib=sample_distrib,
extra_context=extra_context)
cond_nump = voxel_0_large[0].cpu().numpy()
cond_nump[:, 3:6] = np.clip(cond_nump[:, 3:6] * 255, 0, 255)
sample_points_given_0 = sample_points_given_0.cpu().numpy().squeeze()
sample_points_given_0[:, 3:6] = np.clip(
sample_points_given_0[:, 3:6] * 255, 0, 255)
fig_gen_given_0 = view_cloud_plotly(sample_points_given_0[:, :3],
sample_points_given_0[:, 3:],
show=False,
point_size=point_size)
fig_0 = view_cloud_plotly(voxel_0_small_original[0][:, :3],
voxel_0_small_original[0][:, 3:],
show=False,
point_size=point_size)
fig_1 = view_cloud_plotly(voxel_1_small_original[0][:, :3],
voxel_1_small_original[0][:, 3:],
show=False,
point_size=point_size)
sample_points_given_1 = make_sample(
n_points=4000,
extract_0=voxel_opposite_large[0].unsqueeze(0),
models_dict=self.model_dict,
config=config,
sample_distrib=sample_distrib,
extra_context=extra_context)
sample_points_given_1 = sample_points_given_1.cpu().numpy().squeeze()
sample_points_given_1[:, 3:6] = np.clip(
sample_points_given_1[:, 3:6] * 255, 0, 255)
fig_gen_given_1 = view_cloud_plotly(sample_points_given_1[:, :3],
sample_points_given_1[:, 3:],
show=False,
point_size=point_size)
combined_points = torch.cat((voxel_0_small_original[0][:, :3],
voxel_1_small_original[0][:, :3]),
dim=0)
change_0_1[change_0_1 > 0] = 1.0
changes_0_1_count = change_0_1.sum()
change_1_0[change_1_0 > 0] = 1.0
changes_1_0_count = change_1_0.sum()
combined_change = torch.cat((change_0_1, change_1_0), dim=-1).squeeze()
if changes_0_1_count > 0:
fig_0_given_1 = view_cloud_plotly(voxel_0_small_original[0][:, :3],
change_0_1.squeeze(),
show=False,
colorscale='Bluered',
point_size=point_size)
else:
fig_0_given_1 = view_cloud_plotly(
voxel_0_small_original[0][:, :3],
torch.zeros_like(voxel_0_small_original[0][:, :3]) +
torch.tensor([0, 0, 1]).to(device),
show=False,
point_size=point_size)
if changes_1_0_count > 0:
fig_1_given_0 = view_cloud_plotly(voxel_1_small_original[0][:, :3],
change_1_0.squeeze(),
show=False,
colorscale='Bluered',
point_size=point_size)
else:
fig_1_given_0 = view_cloud_plotly(
voxel_1_small_original[0][:, :3],
torch.zeros_like(voxel_1_small_original[0][:, :3]) +
torch.tensor([0, 0, 1]).to(device),
show=False,
point_size=point_size)
changed_percentage = (combined_change.sum() /
combined_change.numel()).item()
print(f'Changed percentage: {changed_percentage:.2f}')
combined_fig = view_cloud_plotly(combined_points,
combined_change,
show=False,
colorscale='Bluered',
point_size=point_size)
return fig_0, fig_1, fig_gen_given_0, fig_gen_given_1, combined_fig, fig_0_given_1, fig_1_given_0, changed_percentage
def testValidCardNumber(self):
card_number = 30536723505217
self.assertTrue(utils.is_valid(card_number))
def evaluate_on_test(model_dict,
config,
batch_size=None,
generate_samples=False):
with torch.no_grad():
device = 'cuda'
batch_size = config['batch_size'] if batch_size == None else batch_size
dataset = AmsVoxelLoader(
config['directory_path_train'],
config['directory_path_test'],
out_path='save/processed_dataset',
preload=True,
n_samples=config['sample_size'],
final_voxel_size=config['final_voxel_size'],
device=device,
n_samples_context=config['n_samples_context'],
context_voxel_size=config['context_voxel_size'],
mode='test',
include_all=True)
dataloader = DataLoader(dataset,
batch_size=batch_size,
num_workers=config['num_workers'],
pin_memory=True,
prefetch_factor=2,
drop_last=True,
shuffle=False)
change_mean_list = []
print(f'Evaluating on test')
nats_avg = 0
for batch_ind, batch in enumerate(tqdm(dataloader)):
voxel_0_large, voxel_1_small, extra_context, voxel_1_large_self, voxel_1_small_self, voxel_opposite_small, voxel_opposite_large, voxel_0_small_self, voxel_0_large_self, voxel_0_small_original, voxel_1_small_original = [
x.to(device) for x in batch
]
if not config['using_extra_context']:
extra_context = None
batch_1_0 = [voxel_0_large, voxel_1_small, extra_context]
batch_0_1 = [
voxel_opposite_large, voxel_opposite_small, extra_context
]
batch_0_0 = [voxel_0_large_self, voxel_0_small_self, extra_context]
batch_1_1 = [voxel_1_large_self, voxel_1_small_self, extra_context]
loss, log_prob_1_0, nats = inner_loop(batch_1_0, model_dict,
config)
_, log_prob_0_0, _ = inner_loop(batch_0_0, model_dict, config)
assert is_valid(log_prob_1_0)
change_1_0 = log_prob_to_change(log_prob_1_0,
log_prob_0_0,
multiple=5.4)
assert is_valid(change_1_0)
is_valid(loss)
change_means = (change_1_0 > 0).float().mean(dim=-1).tolist()
change_mean_list.extend(change_means)
if generate_samples:
change_val = change_means[0]
loss, log_prob_0_1, nats = inner_loop(batch_0_1, model_dict,
config)
_, log_prob_1_1, _ = inner_loop(batch_1_1, model_dict, config)
change_0_1 = log_prob_to_change(log_prob_0_1,
log_prob_1_1,
multiple=5.4)
assert is_valid(log_prob_0_1)
sample_points_given_0 = make_sample(
n_points=4000,
extract_0=voxel_0_large[0].unsqueeze(0),
models_dict=model_dict,
config=config,
sample_distrib=None,
extra_context=extra_context[0].unsqueeze(0))
cond_nump = voxel_0_large[0].cpu().numpy()
cond_nump[:, 3:6] = np.clip(cond_nump[:, 3:6] * 255, 0, 255)
sample_points_given_0 = sample_points_given_0.cpu().numpy(
).squeeze()
sample_points_given_0[:, 3:6] = np.clip(
sample_points_given_0[:, 3:6] * 255, 0, 255)
fig_gen_given_0 = view_cloud_plotly(
sample_points_given_0[:, :3],
sample_points_given_0[:, 3:],
show=False)
fig_gen_given_0.write_html(
f'save/examples/test_set_changes/{change_val:.2f}_{batch_ind}_gen_given_0.html'
)
fig_0 = view_cloud_plotly(voxel_0_small_original[0][:, :3],
voxel_0_small_original[0][:, 3:],
show=False)
fig_0.write_html(
f'save/examples/test_set_changes/{change_val:.2f}_{batch_ind}_0_small.html'
)
fig_1 = view_cloud_plotly(voxel_1_small_original[0][:, :3],
voxel_1_small_original[0][:, 3:],
show=False)
fig_1.write_html(
f'save/examples/test_set_changes/{change_val:.2f}_{batch_ind}_1_small.html'
)
sample_points_given_1 = make_sample(
n_points=4000,
extract_0=voxel_opposite_large[0].unsqueeze(0),
models_dict=model_dict,
config=config,
sample_distrib=None,
extra_context=extra_context[0].unsqueeze(0))
sample_points_given_1 = sample_points_given_1.cpu().numpy(
).squeeze()
sample_points_given_1[:, 3:6] = np.clip(
sample_points_given_1[:, 3:6] * 255, 0, 255)
fig_gen_given_1 = view_cloud_plotly(
sample_points_given_1[:, :3],
sample_points_given_1[:, 3:],
show=False)
fig_gen_given_1.write_html(
f'save/examples/test_set_changes/{change_val:.2f}_{batch_ind}_gen_given_1.html'
)
combined_points = torch.cat((voxel_0_small_original[0][:, :3],
voxel_1_small_original[0][:, :3]),
dim=0)
change_0_1[log_prob_0_1[0] < 0] = 1.0
change_1_0 = torch.zeros(voxel_1_small_original.shape[1])
change_1_0[log_prob_1_0[0] < 0] = 1.0
combined_change = torch.cat((change_0_1, change_1_0), dim=-1)
combined_change[combined_change > 0] = 1.0
combined_fig = view_cloud_plotly(combined_points,
combined_change,
show=False,
colorscale='Bluered')
combined_fig.write_html(
f'save/examples/test_set_changes/{change_val:.2f}_{batch_ind}_change.html'
)
nats = nats.item()
nats_avg = (nats_avg * (batch_ind) + nats) / (batch_ind + 1)
print(f'Nats: {nats_avg}')
return nats_avg, change_mean_list
def testInvalidCardNumber(self):
card_number = '305367235--05217'
self.assertFalse(utils.is_valid(card_number))
def train(config_path):
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(f'Using device: {device}')
wandb.init(project="flow_change", config=config_path)
config = wandb.config
models_dict = initialize_flow(config, device, mode='train')
if config['data_loader'] == 'AmsVoxelLoader':
dataset = AmsVoxelLoader(
config['directory_path_train'],
config['directory_path_test'],
out_path='save/processed_dataset',
preload=config['preload'],
n_samples=config['sample_size'],
final_voxel_size=config['final_voxel_size'],
device=device,
n_samples_context=config['n_samples_context'],
context_voxel_size=config['context_voxel_size'],
mode='train',
self_pairs_train=config['self_pairs_train']
if 'self_pairs_train' in config else None)
else:
raise Exception('Invalid dataset type!')
dataloader = DataLoader(dataset,
shuffle=True,
batch_size=config['batch_size'],
num_workers=config["num_workers"],
collate_fn=None,
pin_memory=True,
prefetch_factor=2,
drop_last=True)
if config["optimizer_type"] == 'Adam':
optimizer = torch.optim.Adam(models_dict['parameters'],
lr=config["lr"],
weight_decay=config["weight_decay"])
elif config["optimizer_type"] == 'Adamax':
optimizer = torch.optim.Adamax(models_dict['parameters'],
lr=config["lr"],
weight_decay=config["weight_decay"],
polyak=0.999)
elif config["optimizer_type"] == 'AdamW':
optimizer = torch.optim.AdamW(models_dict['parameters'],
lr=config["lr"],
weight_decay=config["weight_decay"])
elif config['optimizer_type'] == 'SGD':
optimizer = torch.optim.SGD(models_dict['parameters'],
lr=config["lr"],
momentum=0,
dampening=0,
weight_decay=config["weight_decay"],
nesterov=False)
else:
raise Exception('Invalid optimizer type!')
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
optimizer,
patience=config['patience'],
factor=config['lr_factor'],
threshold=config['threshold_scheduler'],
min_lr=config["min_lr"],
verbose=True)
save_model_path = r'save/conditional_flow_compare'
loss_running_avg = 0
# Load checkpoint params if specified path
if config['load_checkpoint']:
print(f"Loading from checkpoint: {config['load_checkpoint']}")
checkpoint_dict = torch.load(
config['load_checkpoint'], map_location='cpu'
) #Map to cpu to avoid weird pytorch extra gpu mem usage
models_dict = load_flow(checkpoint_dict, models_dict)
models_dict['flow'].train()
#optimizer.load_state_dict(checkpoint_dict['optimizer'])
scheduler.load_state_dict(checkpoint_dict['scheduler'])
for g in optimizer.param_groups:
g['lr'] = checkpoint_dict['optimizer']['param_groups'][0]['lr']
else:
print("Starting training from scratch!")
# Watch models:
detect_anomaly = False
if detect_anomaly:
print('DETECT ANOMALY ON')
torch.autograd.set_detect_anomaly(True)
torch.backends.cudnn.deterministic = False
torch.backends.cudnn.benchmark = True
scaler = torch.cuda.amp.GradScaler(enabled=config['amp'])
best_so_far = math.inf
last_save_path = None
for epoch in range(config["n_epochs"]):
print(f"Starting epoch: {epoch}")
for batch_ind, batch in enumerate(tqdm(dataloader)):
with torch.cuda.amp.autocast(enabled=config['amp']):
if config['time_stats']:
torch.cuda.synchronize()
t0 = perf_counter()
batch = [x.to(device) for x in batch]
# Set to None if not using
if not config['using_extra_context']:
batch[-1] = None
extract_0 = batch[0]
extra_context = batch[-1]
loss, _, nats = inner_loop(batch, models_dict, config)
is_valid(loss)
scaler.scale(loss).backward()
torch.nn.utils.clip_grad_norm_(models_dict['parameters'],
max_norm=config['grad_clip_val'])
scaler.step(optimizer)
scaler.update()
scheduler.step(loss)
optimizer.zero_grad(set_to_none=True)
current_lr = optimizer.param_groups[0]['lr']
if config['time_stats']:
torch.cuda.synchronize()
time_batch = perf_counter() - t0
else:
time_batch = np.NaN
loss_item = loss.item()
loss_running_avg = (loss_running_avg *
(batch_ind) + loss_item) / (batch_ind + 1)
if (batch_ind % config['batches_per_save']) == 0 and batch_ind > 0:
if loss_running_avg < best_so_far:
if last_save_path != None:
os.remove(last_save_path)
print(f'Saving!')
savepath = os.path.join(
save_model_path,
f"{wandb.run.name}_e{epoch}_b{batch_ind}_model_dict.pt"
)
print(f'Loss epoch: {loss_running_avg}')
save_flow(models_dict, config, optimizer, scheduler,
savepath)
last_save_path = savepath
best_so_far = min(loss_running_avg, best_so_far)
loss_running_avg = 0
if ((batch_ind + 1) % config['batches_per_sample']
== 0) and config['make_samples']:
with torch.no_grad():
cond_nump = extract_0[0].cpu().numpy()
if config['using_extra_context']:
sample_extra_context = extra_context[0].unsqueeze(0)
else:
sample_extra_context = None
sample_points = make_sample(
n_points=4000,
extract_0=extract_0[0].unsqueeze(0),
models_dict=models_dict,
config=config,
sample_distrib=None,
extra_context=sample_extra_context)
cond_nump[:, 3:6] = np.clip(cond_nump[:, 3:6] * 255, 0,
255)
sample_points = sample_points.cpu().numpy().squeeze()
sample_points[:,
3:6] = np.clip(sample_points[:, 3:6] * 255,
0, 255)
wandb.log({
"Cond_cloud": wandb.Object3D(cond_nump[:, :6]),
"Gen_cloud": wandb.Object3D(sample_points[:, :6]),
'loss': loss_item,
'nats': nats.item(),
'lr': current_lr,
'time_batch': time_batch
})
else:
pass
wandb.log({
'loss': loss_item,
'nats': nats.item(),
'lr': current_lr,
'time_batch': time_batch
})
wandb.log({'epoch': epoch, "loss_epoch": loss_running_avg})
def testValidSeparatedNumber(self):
card_number = '1234-1234-1234-1234'
self.assertTrue(utils.is_valid(card_number))
def greedy(graph, num_buses, size_bus, constraints):
G = graph.copy()
groups = [[] for i in range(num_buses)]
constraint_status = {
i: (-1, len(constraints[i]))
for i in range(len(constraints))
}
CD = constraint_dictionary(G.nodes(), constraints)
def greedy_value(student, group, group_num, num_neighbors):
rowdy = set()
group_set = set(group + [student])
for ci in CD[student]:
(g, p) = constraint_status[ci]
if g != group_num or p > 1:
continue
for s in constraints[ci]:
rowdy.add(s)
if len(rowdy) > 0:
return -len(rowdy)
return num_neighbors[student]
fill_the_rest = False
partial_assignment = {n: None for n in G.nodes()}
for i in range(num_buses):
# Pick the person with the most friends
names = G.nodes()
x = max(names, key=lambda n: G.degree(n))
group = []
num_neighbors = {n: 0 for n in names}
for j in range(size_bus):
if j == 0 and (num_buses - i == len(num_neighbors)) or (
num_buses - i - 1 == len(num_neighbors)):
groups[i] = group
fill_remainder(groups, num_buses, list(num_neighbors.keys()))
fill_the_rest = True
break
group_set = set(group)
def value(student):
return greedy_value(student, group, i, num_neighbors)
# Add people to a bus until we can't add any more people
y = max(num_neighbors, key=value)
for n in G[y]:
if n not in group:
num_neighbors[n] += 1
group.append(y)
partial_assignment[y] = i
neutralized_constraints = []
for ci in CD[y]:
(g, p) = constraint_status[ci]
if g == -1:
constraint_status[ci] = (i, p - 1)
elif g != i:
neutralized_constraints.append(ci)
else:
constraint_status[ci] = (g, p - 1)
for ci in neutralized_constraints:
for s in constraints[ci]:
CD[s].remove(ci)
del num_neighbors[y]
if len(num_neighbors) == 0:
break
if fill_the_rest:
break
groups[i] = group
for n in group:
G.remove_node(n)
assignment = utils.groups_to_assignment(groups)
assert utils.is_valid(graph, num_buses, size_bus, assignment)
return assignment
