def main():
# Setup logger
logging.basicConfig(filename='log.txt',
level=logging.INFO,
format='%(asctime)s %(levelname)s : %(message)s',
datefmt='%m/%d/%Y %H:%M:%S')
logging.info('Running file : {}'.format(__file__))
# parse command line arguments
parser = argparse.ArgumentParser()
parser.add_argument('--seed', type=int, default=None, help='random seed')
parser.add_argument('--save-to',
type=str,
C > default=None,
help='name of a file to save')
args = parser.parse_args()
# get dataset
dataset, test_dataset, emb_weights = get_dataset(min_freq=5, test_set=True)
dataset.fields['review'].include_lengths = True
# the above line adds a tensor with lengths of reviews in a batch,
# so that we can pass batches to embeddingbag
ds_train, ds_val = dataset.split(split_ratio=[0.9, 0.1])
if args.seed is not None:
random.seed(args.randomseed)
bs = 64
logging.info('Initialising the model with the embedding layer frozen.')
model = Baseline_model(emb_weights.clone()).to(device)
loss_fn = nn.CrossEntropyLoss().to(device)
optimiser = AdamW(model.parameters(), weight_decay=1e-2)
steps_per_cycle = (len(ds_train) // bs + 1)
scheduler = CyclicLRDecay(optimiser,
1e-4,
1e-2,
steps_per_cycle,
0.25,
gamma_factor=0.80)
learner = Learner(model, loss_fn, optimiser, scheduler, ds_train, ds_val,
device)
logging.info('Training')
learner.train(epochs=6, bs=bs, grad_clip=(0.2, 0.4))
logging.info('Unfreezing the embedding layer')
model.embedding.weight.requires_grad_(True)
learner.train(epochs=5, bs=bs, grad_clip=(0.2, 0.4))
logging.info('--- Final statistics ---')
logging.info('Training loss : {:.4f}, accuracy {:.4f}'.format(
*validate(ds_train, loss_fn, model, bs)))
logging.info('Test loss : {:.4f}, accuracy {:.4f}'.format(
*validate(test_dataset, loss_fn, model, bs)))
if args.save_to is not None:
if not os.path.exists('models'):
os.makedirs('models')
logging.info('Model saved to: models/{}'.format(args['--save-to']))
torch.save(model.state_dict(), 'models/{}'.format(args['--save-to']))
def get_pvlist(self):
'''get the PVs from the XML file'''
pvlist_file = self.configuration['PVLIST_FILE']
if not os.path.exists(pvlist_file):
utils.logMessage('could not find file: ' + pvlist_file)
return
try:
tree = etree.parse(pvlist_file)
except:
msg = 'could not parse file: ' + pvlist_file
utils.logMessage(msg)
raise CouldNotParseXml(msg)
utils.validate(tree, XML_SCHEMA_FILE)
msg = 'validated file: ' + pvlist_file
utils.logMessage(msg)
for key in tree.findall(".//EPICS_PV"):
if key.get("_ignore_", "false").lower() == "false":
mne = key.get("mne")
pv = key.get("PV")
desc = key.get("description")
fmt = key.get("display_format", "%s") # default format
as_string = key.get("as_string", False) # default format
# :see: http://cars9.uchicago.edu/software/python/pyepics3/pv.html?highlight=as_string#pv.get
try:
self.add_pv(mne, pv, desc, fmt, as_string)
except:
msg = "%s: problem connecting: %s" % (pvlist_file,
etree.tostring(key))
utils.logException(msg)
utils.logMessage('all PVs added')
def save(video,format='txt'):
for case in switch(format):
if case('xls'):
import xlwt
if len(video['comments']) > 0:
wbk = xlwt.Workbook()
sheet = wbk.add_sheet(tech.validate(video['title']))
bigs = wbk.add_sheet('Bigrams')
tris = wbk.add_sheet('Trigrams')
context = wbk.add_sheet('Context')
for_nlp = tech.flatten(video['comments'][0])
for idx,comment in enumerate(video['comments'][0]):
sheet.write(idx,0,' '.join(comment))
for idx,bigram in enumerate(tech.bigrams(for_nlp,self.term)):
bigs.write(idx,0,' '.join(bigram))
for idx,trigram in enumerate(tech.trigrams(for_nlp,self.term)):
tris.write(idx,0,' '.join(trigram))
for idx,con in enumerate(tech.context(for_nlp,self.term)):
context.write(idx,0,' '.join(con))
wbk.save(tech.validate(video['title'])+'.xls')
print 'Videos, trigrams, bigrams, and contexts saved to XLS files.'
#indexing is zero-based, row then column
break
if case('txt'):
if len(video['comments']) > 0:
with open(path.join(dir_path, tech.validate(video['title'])),'a') as f:
f.write(video['comments'])
print 'Saved %s as text' % video['title']
break
async def resolve_generate(
parent: Optional[Any],
args: Dict[str, Any],
ctx: Dict[str, Any],
info: "ResolveInfo",
) -> Dict[str, Any]:
"""
Resolver for generating both Kurodoko and Ohno puzzles.
:param parent: initial value filled in to the engine `execute` method
:param args: computed arguments related to the field
:param ctx: context filled in at engine initialization
:param info: information related to the execution and field resolution
:type parent: Optional[Any]
:type args: Dict[str, Any]
:type ctx: Dict[str, Any]
:type info: ResolveInfo
:return: the solution to the given puzzle
:rtype: Dict[str, Any]
"""
u.validate(rows=args.get('rows', None),
cols=args.get('cols', None),
puzzle_type=args.get('type', None),
difficulty=args.get('difficulty', None))
gen = k.generate if args['type'] == u.KURODOKO else o.generate
return gen(rows=args['rows'],
cols=args['cols'],
difficulty=args.get('difficulty', None),
seed=args.get('seed', None))
def post(self):
data = request.json
ALLOWED_KEYS = ['sentences', 'n_lines', 'temperature']
if data is None:
return {'error': 'You should send me json data!'}, 400
sentences_val_errors = validate(data, 'sentences', list)
n_lines_val_errors = validate(data, 'n_lines', int)
if sentences_val_errors: return {'error': sentences_val_errors}, 400
if n_lines_val_errors: return {'error': n_lines_val_errors}, 400
for sentence in data['sentences']:
if not type(sentence) is str: return {'error': '`sentence` must be a string!'}, 400
if not type(data['n_lines']) is int: return {'error': '`n_lines` must be an integer!'}, 400
if not 0 < data['n_lines'] <= 100: return {'error': '`n_lines` must be between 1 and 100!'}, 400
if not all(k in ALLOWED_KEYS for k in data): return {'error': 'some keys are not known'}, 400
try:
return {'result': generate_dialog(
data['sentences'],
data['n_lines'],
temperature=data.get('temperature'),
max_len=data.get('max_len')
)}
except Exception:
traceback.print_exc()
torch.cuda.empty_cache()
return {'error': 'Something went wrong'}, 500
def assertion():
prop = "x or not y"
litts = validate(prop)
prop2 = "x and ( not y or z )"
litts2 = validate(prop2)
print(recursive_tree(litts, prop))
print(fifo(litts2, prop2))
print(recursive_tree(litts2, prop2))
cnf = "(x or y) and ( z or w) and (not x or w)"
CNF = CNF_clauses(cnf)
set_variabs = CNF_variables(CNF)
print(cnf)
print(CNF)
print(set_variabs)
print("recursive_tree")
t = time.time()
recursive_tree(list(set_variabs), cnf)
print(time.time() - t)
print("fifo")
t = time.time()
fifo(list(set_variabs), cnf)
print(time.time() - t)
print("DPLL")
t = time.time()
print(DPLL(CNF, set_variabs))
print(time.time() - t)
def validate_response(response):
utils.validate(response, conf.RESPONSE_SCHEMA)
max_size_in_bytes = 262144
response_size = len(serialise_response(response))
ensure(
response_size <= max_size_in_bytes,
"response size (%s) is too large: %s" %
(response_size, max_size_in_bytes), ValidationError)
def run(self):
"""
Main function to run program
:return:
"""
while True:
# load data to memory
if not self.all_data:
tmp_data = []
with open(self.filename, 'r') as f:
for line in f:
data = line.strip().split(',')
if validate(data[1]):
tmp_data.append([data[0], validate(data[1]), 0])
self.all_data = sort_data(tmp_data)
print('''Greetings user! Let's make an addressbook!
Would you like to:
1.) Add a New Contact
2.) List All Contacts
3.) Search Contacts
4.) Delete A Contact
5.) Make Call
6.) List top 5 Call
7.) Quit Program\n''')
choice = input("Select an option: ")
if choice == "1":
try:
self.create_contact()
except NotAPhoneNumberException:
print("The phone number entered is invalid, creation aborted!")
elif choice == "2":
self.list_all_contact()
elif choice == "3":
self.search_contact()
elif choice == "4":
self.delete_contact()
elif choice == "5":
self.make_call()
elif choice == "6":
self.list_top_contact()
elif choice == "7":
print("Ending Contact Book.\nHave a nice day!")
break
else:
print("Invalid Input! Try again.")
def test_validate():
for s, g in ((101, 2), (103, 4), (113, 14), (360653, 96),
(18361375334787046697, 1550),
(9691983639208775401081992556968666567067, 2982)):
assert utils.validate(s, g), (s, g)
assert utils.validate(str(s), g), (s, g)
if g > 2:
assert not utils.validate(s, g - 2)
assert not utils.validate(s, g + 2)
def read_xml(xml_file):
'''
return the configuration details as a dictionary
:param return: dictionary
the dictionary WILL contain these definitions for use by :meth:`pvwatch.PvWatch`:
======================== =============== =================================================
dictionary key example (type) description
======================== =============== =================================================
PVLIST_FILE pvlist.xml PVs to be monitored
LOCAL_WWW_LIVEDATA_DIR ./localwww absolute path to local directory with "web site"
LOG_INTERVAL_S 300 (float) writing messages to log file
REPORT_INTERVAL_S 10 (float) updates to HTML pages
SLEEP_INTERVAL_S 0.1 (float) sleeps at end of main loop
MAINLOOP_COUNTER_TRIGGER 10000 (int) another logging message interval
PATTERNS \*.html upload all files that match these patterns
======================== =============== =================================================
'''
if not os.path.exists(xml_file):
raise IOError(xml_file + ' file not found')
tree = etree.parse(xml_file)
root = tree.getroot()
schema_version = root.attrib['version']
utils.validate(tree, XML_SCHEMA_FILES[schema_version])
root = tree.getroot()
if root.tag != ROOT_TAG:
msg = 'XML root tag must be ' + ROOT_TAG
msg += ', found: ' + root.tag
raise ValueError(msg)
pattern_handlers = {
'1.0': patterns_1_0,
'1.0.1': patterns_1_0_1,
}
conf = dict(PATTERNS=pattern_handlers[schema_version](tree),
SCHEMA_VERSION=schema_version)
for node in tree.findall(".//var"):
key = node.get('name')
value = node.get('value')
data_type = node.get('type', 'string').lower()
if data_type in ('float', 'int'):
# represent number types as directed
typeconversion = dict(float=float, int=int)[data_type]
value = typeconversion(value)
conf[key] = value
return conf
def send_response(outgoing, response):
# `response` here is the result of `mkresponse` below
try:
utils.validate(response, conf.RESPONSE_SCHEMA)
channel = outgoing.write
except ValidationError as err:
# response doesn't validate. this probably means
# we had an error decoding request and have no id or token
# because the message will not validate, we will not be sending it back
response['validation-error-msg'] = str(err)
channel = outgoing.error
channel(utils.json_dumps(response))
return response
def create_contact(self):
"""
Function to create a contact
:return: none
"""
name = input("Enter name: ")
phone = input("Enter phone number: ")
if not validate(str(phone)):
raise NotAPhoneNumberException
self.all_data.append([name, validate(phone), 0])
self.all_data = sort_data(self.all_data)
def train_fixmatch(model,
ema_model,
train_dl_l,
train_dl_ul,
val_dl,
optimizer,
sched,
params,
conf_thresh=0.95,
w_u=1.,
start_epoch=0,
eval_interval=1):
num_epochs = params['epochs']
for epoch in range(start_epoch, num_epochs):
bar_label_prefix = f'Epoch {epoch+1}/{num_epochs}: '
# train
epoch_loss, train_acc = train_epoch_fixmatch(
epoch,
model,
ema_model,
train_dl_l,
train_dl_ul,
optimizer,
sched,
conf_thresh=conf_thresh,
w_u=w_u,
bar_label_prefix=bar_label_prefix)
if (epoch + 1) % eval_interval == 0:
# validate
val_loss, val_acc = validate(model,
nn.CrossEntropyLoss(),
val_dl,
bar_label_prefix=bar_label_prefix)
lr = optimizer.param_groups[0]['lr']
print(f'\nepoch: {epoch+1:3d}, lr: {lr:0.6f}, '
f'epoch_loss: {epoch_loss:4.4f}, val_loss: {val_loss:4.4f}, '
f'train_acc: {train_acc:0.4f}, val_acc: {val_acc:0.4f}')
# validate
val_loss, val_acc = validate(ema_model,
nn.CrossEntropyLoss(),
val_dl,
bar_label_prefix=bar_label_prefix)
lr = optimizer.param_groups[0]['lr']
print(f'\nepoch: {epoch+1:3d}, lr: {lr:0.6f}, '
f'epoch_loss: {epoch_loss:4.4f}, val_loss: {val_loss:4.4f}, '
f'train_acc: {train_acc:0.4f}, val_acc: {val_acc:0.4f}')
def make_call(self):
"""
Funtion to make call and increment counter
:return: msg
"""
number = input("Please enter phone number: ")
if validate(number):
try:
self.all_data[[i for i, x in enumerate(self.all_data) if x[1] == validate(number)][0]][2] += 1
print('Successful call')
except IndexError:
print('The number you are trying to call does not exist in the Phonebook')
else:
print('The number is not valid')
def profile():
emailid = request.form['emailid'].strip()
validated = validate(emailid)
if not validated:
return render_template("sorry.html")
else:
try:
redis_server = redis.Redis(connection_pool=POOL)
if not redis_server.exists(emailid):
payload = {
'key': API_KEY,
'person_email': emailid,
}
response = requests.get(API, params=payload, verify=False)
if response.status_code == 200:
data = json.loads(response.text)
if data.get('profile').get('status').get(
'has_person_data'):
cleaned_data = cleanup(data)
redis_server.set(emailid, json.dumps(cleaned_data))
redis_server.bgsave()
else:
return render_template("sorry.html")
try:
data = json.loads(redis_server.get(emailid))
except ValueError:
data = literal_eval(redis_server.get(emailid))
if data.get('profile').get('status').get('has_person_data', False):
return render_template("data.html",
user=data['profile']['person_data'])
else:
return render_template("sorry.html")
except (requests.exceptions.ConnectionError, TypeError):
return render_template("sorry.html")
def project(htmlname):
if session.get('author', 'nologin') == 'nologin':
return redirect('/login')
headers['authorization'] = session['author']
validate_result = json.loads(
utils.validate(session['author'], app.config['passport_key']))
if int(validate_result['code']) == 0:
# data['method'] = 'userprojects.getlist'
data['method'] = 'project.getlist' # 只在申请列表中通过jinja2渲染
data['params'] = {}
r = requests.post(get_url(), headers=headers, json=data)
result = json.loads(r.text)
result = json.loads(result['result'])
if int(result['code']) == 0:
return render_template(htmlname + '.html',
info=session,
user=session['user'],
result=result['result'])
else:
return render_template(htmlname + '.html',
info=session,
result=result['errmsg'])
else:
return render_template(htmlname + '.html',
errmsg=validate_result['errmsg'])
def train_ensamble():
""" train func of weighted ensamble
"""
X_train, y_train, X_test, y_test = load_train_data(
preds_list=p_list, mode='train')
train_dataset = MyDataset(X_train, y_train)
train_loader = torch.utils.data.DataLoader(
train_dataset,
batch_size=batch_size,
shuffle=True,
num_workers=4,
pin_memory=True,
sampler=None)
val_dataset = MyDataset(X_test, y_test)
val_loader = torch.utils.data.DataLoader(
val_dataset,
batch_size=batch_size,
shuffle=False,
num_workers=4,
pin_memory=True)
print(f'[+] trainning with {len(train_dataset)} samples, '
f' validation with {len(val_dataset)} samples')
model = WeightedEnsambleModel(num_classes, len(p_list))
criterion = torch.nn.CrossEntropyLoss().cuda()
EPOCHS = 100
min_loss = float("inf")
lr = 0.001
patience = 0
for epoch in range(EPOCHS):
print(f'[+] epoch {epoch}')
if patience == 3:
patience = 0
model.load_state_dict(torch.load(best_checkpoint_file))
lr /= 3
print(f'[+] set lr={lr}')
optimizer = torch.optim.Adam(
filter(lambda p: p.requires_grad, model.parameters()), lr=lr)
# train for one epoch
utils.train_one_epoch(train_loader, model, criterion, optimizer, epoch)
# evaluate on validation set after one epoch
log_loss = utils.validate(val_loader, model, criterion)
if log_loss < min_loss:
torch.save(model.state_dict(), best_checkpoint_file)
print(f'[+] lr = {lr}, val loss improved from {min_loss:.5f} to {log_loss:.5f}. Saved!')
min_loss = log_loss
patience = 0
else:
patience += 1
print(f'[*] trainning done with {EPOCHS} epochs')
def preprocess_and_save(df, val_mask, text_field, label_field, preprocessing_function=None,
additional_fields_and_preps={}, save_dir="./", postfix="", verbose=False, print_items=[]):
val = df[val_mask]
train = df[~val_mask]
if preprocessing_function is not None:
print_items.append("preprocessing descriptions")
train[text_field] = train[text_field].map(lambda s: preprocessing_function(str(s)))
train = train[train[text_field] != ""]
val[text_field] = val[text_field].map(lambda s: preprocessing_function(str(s)))
if verbose:
[print(i) for i in print_items]
train["label"] = train[label_field].apply(lambda x: "__label__{}".format(x))
val["label"] = val[label_field].apply(lambda x: "__label__{}".format(x))
use_cols = ["label", text_field] + list(additional_fields_and_preps.keys())
train_data = [" ".join(i) for i in train[use_cols].values]
val_data = [" ".join(i) for i in val[use_cols].values]
save_dir = validate(save_dir)
save_path_train = join(save_dir, "train{}.txt".format(postfix))
save_path_val = join(save_dir, "val{}.txt".format(postfix))
with open(save_path_train, "w+") as outfile:
outfile.write("\n".join(train_data))
with open(save_path_val, "w+") as outfile:
outfile.write("\n".join(val_data))
return save_path_train, save_path_val
def train(model, training_data, validation_data, optimizer, args, device):
"""Training."""
logger.info("Start training!")
best_mrr = 0.
config = getattr(configs, 'config')()
def save_model(model, path):
if not os.path.exists(path):
os.mkdir(path)
torch.save(model.state_dict(), path)
for epoch in range(args.Epoch):
info = '[ Epoch ' + str(epoch) + ' ]'
logger.info(info)
train_loss = train_epoch(model, training_data, validation_data,
optimizer, config, args, device)
logger.info("The loss of epoch {} is: {}".format(
epoch, np.mean(train_loss)))
logger.info("Validating.")
re, acc, mrr, _, _ = validate(validation_data, model, 1,
config['sim_measure'])
valid_mrr = mrr
if best_mrr < valid_mrr:
best_mrr = valid_mrr
print("The current best mrr score is: ", best_mrr)
path = args.model_path + 'joint_embed_model.h5'
save_model(model, path)
def api_enumeration_write(request):
if request.method == 'POST':
unauthenticated_response = get_unauthenticated_response(request)
if unauthenticated_response:
return unauthenticated_response
validation_errors = validate(request.body)
if validation_errors:
provider_write_response = {
"code": 400,
"status": "ERROR",
"message": "Enumeration create/update failed.",
"errors": validation_errors
}
return HttpResponse(json.dumps(provider_write_response, indent=4),
content_type="application/json")
else:
save_response = save_api_enumeration(request)
return HttpResponse(json.dumps(save_response, indent=4),
content_type="application/json")
#this is a GET
provider_write_response = {
"code":
200,
"message":
"POST Provider JSON to this URL to use the API. See https://github.com/HHSIDEAlab/pjson for details",
}
return HttpResponse(json.dumps(provider_write_response, indent=4),
content_type="application/json")
def create(event, context):
name_module = "client"
data = json.loads(event['body'])
required_values = {'firstName', 'lastName', 'birthday'}
for value in required_values:
if value not in data:
response = utils.validate(value)
return response
timestamp = int(time.time())
table = dynamodb.Table(os.environ['DYNAMODB_TABLE'])
response = table.scan()
filter_result = response['Items']
item = {
'id': str(uuid.uuid1()),
'firstName': data['firstName'],
'lastName': data['lastName'],
'birthday': data['birthday'],
'create_at': timestamp,
'updated_at': timestamp,
'status': 1,
}
# write the todo to the database
table.put_item(Item=item)
# create a response
response = utils.response(name_module, item, 'create')
return response
def train_model(i):
###################### training ######################
print('Begin training...')
print('model = %s' % models_name[i])
best_accuracy = 0.0
accuracies = [[], []]
model = models[i]().to(DEVICE)
criterion = nn.CrossEntropyLoss()
optimizer = SGD(model.parameters(), lr=0.1)
for epoch in range(EPOCH):
print('Epoch: %d' % epoch)
train_accuracy = train(model, criterion, optimizer, train_loader,
DEVICE, print_iteration)
validate_accuracy = validate(model, criterion, test_loader, DEVICE)
accuracies[0].append(train_accuracy)
accuracies[1].append(validate_accuracy)
if validate_accuracy > best_accuracy:
best_accuracy = validate_accuracy
torch.save(model.state_dict(), 'results/%s.pt' % models_name[i])
plot(accuracies, 'results/result_%s.png' % models_name[i])
print('Finish training with best accuracy %.3f.' % best_accuracy)
def duplicate(paste, lang=None):
if not utils.validate(paste):
return flask.redirect('/')
data = get_paste(paste, lang)
if not data:
flask.redirect('/')
return flask.render_template('new.html', dup=data)
def test_already_published_response_means_aok(self):
"bot-lax coerces already-published error responses to successful 'published' responses"
lax_resp = {
'status': conf.ERROR,
'message': 'mock',
'token': 'a',
'id': 'b',
'code': 'already-published'
}
# coercion happens
resp = adapt.mkresponse(**lax_resp)
self.assertEqual(resp['status'], conf.PUBLISHED)
# coercion doesn't result in an invalid response
utils.validate(resp, conf.RESPONSE_SCHEMA)
def main():
classifiers = []
trainset = utils.fetchData()
devset = utils.loadDevData()
testset = utils.loadTestData()
# train OvA classifiers
for i in range(utils.numOfClasses):
binData = utils.transformToBinaryClasses(trainset, positiveClass=[i])
model = utils.SVM(utils.inputDim, utils.eta, 1, 50)
model.train(binData)
classifiers.append(model)
print("finished with #{} model".format(i))
# Validation - Evaluate Test Data by Hamming Distance
utils.validate(devset,
utils.HammingDistance,
ecocMat,
'test.onevall.ham.pred',
classifiers,
distanceMetric="Hamming")
# Validation - Evaluate Test Data by Loss Base Decoding
utils.validate(devset,
utils.lossBaseDecoding,
ecocMat,
'test.onevall.ham.pred',
classifiers,
distanceMetric="LBD")
# Test - Evaluate test data by Hamming Distance
utils.evaluate(testset,
utils.HammingDistance,
ecocMat,
'test.onevall.ham.pred',
classifiers,
distanceMetric="Hamming")
# Test - Evaluate test data by Loss Base Decoding
utils.evaluate(testset,
utils.lossBaseDecoding,
ecocMat,
'test.onevall.loss.pred',
classifiers,
distanceMetric="LBD")
def arithmetic_arranger(problems, showresults=False):
if len(problems) > 5:
return "Error: Too many problems."
# split of every problem with the length of longest digit
oplist = [(x[0], x[1], x[2], getmaxlen(x[0], x[2]))
for x in (x.split() for x in problems)]
error = validate(oplist)
if error:
return error
result = ''
separator = ' '
# adding 1st numbers of every operation to result
i = 0
for op in oplist:
result += '{}{}'.format(getspaces(op[3] - len(op[0]) + 2), op[0])
if i < len(oplist) - 1:
result += separator
else:
result += '\n'
i += 1
# adding symbol and 2nd numbers of every operation to result
i = 0
for op in oplist:
result += '{} {}{}'.format(op[1], getspaces(op[3] - len(op[2])), op[2])
if i < len(oplist) - 1:
result += separator
else:
result += '\n'
i += 1
# adding dashes to result
i = 0
for op in oplist:
for x in range(op[3] + 2):
result += '-'
if i < len(oplist) - 1:
result += separator
i += 1
if showresults:
# adding results to result
result += '\n'
i = 0
for op in oplist:
opresult = str(eval(problems[i]))
result += '{}{}'.format(getspaces(op[3] - len(opresult) + 2),
opresult)
if i < len(oplist) - 1:
result += separator
i += 1
return result
def api(htmlname):
if session.get('author','nologin') == 'nologin':
return redirect('/login')
headers['authorization'] = session['author']
validate_result = json.loads(utils.validate(session['author'], app.config['passport_key']))
if int(validate_result['code']) == 0:
return render_template(htmlname+'.html',info=session,user=session['user'])
else:
return render_template(htmlname+'.html',errmsg=validate_result['errmsg'])
def train_cl(model,
train_dl,
val_dl,
optimizer,
sched,
params,
w_scale=1,
rampup_epochs=80,
rampdown_epochs=50,
eval_interval=1):
num_epochs = params['epochs']
def _get_w(step):
N = len(train_dl)
rampdown_start = (num_epochs - rampdown_epochs) * N
rampup_steps = rampup_epochs * N
rampdown_steps = rampdown_epochs * N
if step <= rampup_steps:
# ramp up
w = w_scale * np.exp(-5 * (1 - (step / rampup_steps))**2)
elif step >= rampdown_start:
# ramp down
w = w_scale * np.exp(-((
(step - rampdown_start) * 0.5)**2) / rampdown_steps)
else:
w = w_scale
return w
for epoch in range(num_epochs):
bar_label_prefix = f'Epoch {epoch+1}/{num_epochs}: '
epoch_loss, train_acc = train_cl_epoch(
epoch,
model,
train_dl,
nn.CrossEntropyLoss(),
optimizer,
_get_w,
bar_label_prefix=bar_label_prefix)
if (epoch + 1) % eval_interval == 0:
# validate
val_loss, val_acc = validate(
model,
nn.CrossEntropyLoss(),
val_dl,
bar_label_prefix=bar_label_prefix)
lr = optimizer.param_groups[0]['lr']
print(f'\nepoch: {epoch+1:3d}, lr: {lr:0.6f}, '
f'epoch_loss: {epoch_loss:4.4f}, val_loss: {val_loss:4.4f}, '
f'train_acc: {train_acc:0.4f}, val_acc: {val_acc:0.4f}')
sched.step()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--dataset_loc', default='./cv_simplified')
parser.add_argument('--checkpoint_loading_path', default='./saved_model')
parser.add_argument('--checkpoint_saving_path', default='./saved_model')
parser.add_argument('--on_cluster', action='store_true', default=False)
parser.add_argument('--resume', action='store_true', default=False)
parser.add_argument('--line_search', action='store_true', default=False)
parser.add_argument('--wandb_logging', action='store_true', default=False)
parser.add_argument('--wandb_exp_name')
parser.add_argument('--step_size', type=float, default=1e-06)
parser.add_argument('--batch_size', type=int, default=500)
parser.add_argument('--regularization_param', type=float, default=1e-03)
parser.add_argument('--max_pass', type=int, default=10)
parser.add_argument('--opt_method', default='adam')
args = parser.parse_args()
if args.on_cluster:
device = torch.device('cuda')
else:
device = torch.device('cpu')
if args.wandb_logging:
wandb.init(project='quick_draw_crf', name=args.wandb_exp_name)
wandb.config.update({"step_size": args.step_size, "l2_regularize": args.regularization_param,
"opt_method":'sag'})
torch.manual_seed(1)
num_features = 851968
num_cats = 113
num_val_data_per_cat = 500
data_fh = h5py.File(args.dataset_loc, 'r')
num_tr_data = len(data_fh['tr_data'])
tr_dataset = Image_dataset(data=data_fh['tr_data'], label=data_fh['tr_label'], data_size=num_tr_data, num_cats=num_cats, device=device)
val_dataset = Validation_dataset(data=data_fh['val_data'], label=data_fh['val_label'], num_cats=num_cats, num_data_per_cat=num_val_data_per_cat, num_features=num_features, device=device)
train_loader = data.DataLoader(tr_dataset, batch_size=args.batch_size, shuffle=True)
validate_loader = data.DataLoader(val_dataset, batch_size=args.batch_size)
validate_data_true_label = parse_validation_data_labels(data_fh['val_label'][:]) # Assumed no zero entries here!
crf = CRF(num_cats, num_features, num_tr_data, args.step_size, args.regularization_param, args.line_search, device)
iter = 0
for num_pass in range(args.max_pass):
for image_batch, label_vecs, labels, indicies in train_loader:
num_data_in_batch = image_batch.shape[0]
for i in range(num_data_in_batch):
crf.update(image_batch[i], label_vecs[i], labels[i], indicies[i])
iter += 1
if iter % 1000 == 0:
val_err = validate(crf, validate_data_true_label, validate_loader, val_dataset.data_size, device)
if args.wandb_logging:
grad = torch.abs(crf.full_grad/crf.num_sample_visited)
step_size = 1/crf.line_search_lr if args.line_search else crf.lr
wandb.log({'val_err': val_err, 'grad_l1': torch.sum(torch.sum(grad)), 'grad_l_inf':torch.max(grad), 'step_size':step_size})
def pick_lamda_size(model, paramtune_loader, alpha, kreg, randomized, allow_zero_sets):
# Calculate lamda_star
best_size = iter(paramtune_loader).__next__()[0][1].shape[0] # number of classes
# Use the paramtune data to pick lamda. Does not violate exchangeability.
for temp_lam in [0.001, 0.01, 0.1, 0.2, 0.5]: # predefined grid, change if more precision desired.
conformal_model = ConformalModelLogits(model, paramtune_loader, alpha=alpha, kreg=kreg, lamda=temp_lam, randomized=randomized, allow_zero_sets=allow_zero_sets, naive=False)
top1_avg, top5_avg, cvg_avg, sz_avg = validate(paramtune_loader, conformal_model, print_bool=False)
if sz_avg < best_size:
best_size = sz_avg
lamda_star = temp_lam
return lamda_star
def wrapper(*arg, **kwargs):
try:
authorization = request.headers.get('authorization', 'None')
res = utils.validate(authorization, app.config['passport_key'])
res = json.loads(res)
if int(res['code']) == 1:
utils.write_log('api').warning("Request forbiden:%s" % res['errmsg'])
return json.dumps({'code': 1, 'errmsg': '%s' % res['errmsg']})
except:
utils.write_log('api').warning("Validate error: %s" % traceback.format_exc())
return json.dumps({'code': 1, 'errmsg': '验证异常'})
return func(res, *arg, **kwargs)
def mkreq(path, **overrides):
ensure(not path.startswith('http://'), "no insecure requests, please")
# TODO: consider removing these two lines
if path.lstrip('/').startswith('article-xml/articles/'):
path = os.path.abspath(path)
path = 'file://' + path if not path.startswith('https://') else path
msid, ver = utils.version_from_path(path)
request = {
'action': conf.INGEST,
'location': path,
'id': msid,
'version': ver,
'force': True, # TODO: shouldn't this be False?
'validate-only': False,
'token': 'pants-party'
}
request.update(overrides)
# don't ever generate an invalid request
utils.validate(request, conf.REQUEST_SCHEMA)
return request
def artist(name=""):
if 'un' not in session or session['un']==0:
return redirect(url_for("home"))
#return render_template("home.html",error=error)
d = utils.returnd()
d2 = utils.returnartists()
if request.method=="GET":
return render_template("artist.html",dic=d2,stagename="artists",un=session['un'])
else:
person = request.form["button"]
if utils.validate(person):
return render_template("artist.html",dic=d,stagename=person)
else:
return render_template("artist.html",dic=d2,stagename="artists")
def post(self):
'''This function handles the signup form and vlidates user input.'''
username = self.request.get('username')
password = self.request.get('password')
verify = self.request.get('verify')
email = self.request.get('email')
# check the input and return dict of errors
errors = validate(username, password, verify, email)
if len(errors) != 0:
self.render_response(self.tempate, params=None, **errors)
else:
u = register(username, password, email)
u.put()
self.login(u)
self.redirect_to('home')
def register():
if request.method == "GET":
users = db.users.find()
count = 0
for u in users:
if u['logged_in'] == True:
count += 1
return render_template("register.html", logged_in = count)
else:
if request.form["b"] == "Start Poopin'":
message = validate(request.form, db)
if message == 'Valid':
register_user(request.form, db)
flash('Account created')
return redirect("login")
else:
return render_template("register.html", message = message)
elif request.form["b"] == "Log In":
return redirect("login")
elif request.form["b"] == "About":
return redirect("about")
else:
return render_template("register.html")
def handler(json_request, outgoing):
response = partial(send_response, outgoing)
try:
request = utils.validate(json_request, conf.REQUEST_SCHEMA)
except ValueError:
# bad data. who knows what it was. die
return response(mkresponse(ERROR, "request could not be parsed: %s" % json_request))
except ValidationError as err:
# data is readable, but it's in an unknown/invalid format. die
return response(mkresponse(ERROR, "request was incorrectly formed: %s" % str(err)))
except Exception as err:
# die
msg = "unhandled error attempting to handle request: %s" % str(err)
return response(mkresponse(ERROR, msg))
# we have a valid request :)
LOG.info("valid request")
params = subdict(request, ['action', 'id', 'token', 'version', 'force', 'validate-only'])
params = renkeys(params, [('validate-only', 'dry_run'), ('id', 'msid')])
# if we're to ingest/publish, then we expect a location to download article data
if params['action'] in [INGEST, INGEST_PUBLISH]:
try:
article_xml = download(request['location'])
if not article_xml:
raise ValueError("no article content available")
except AssertionError as err:
msg = "refusing to download article xml: %s" % str(err)
return response(mkresponse(ERROR, msg, request))
except Exception as err:
msg = "failed to download article xml from %r: %s" % (request['location'], str(err))
return response(mkresponse(ERROR, msg, request))
LOG.info("got xml")
try:
article_data = scraper.render_single(article_xml,
version=params['version'],
location=request['location'])
LOG.info("rendered article data ")
except Exception as err:
error = str(err) if hasattr(err, 'message') else err
msg = "failed to render article-json from article-xml: %s" % error
LOG.exception(msg, extra=params)
return response(mkresponse(ERROR, msg, request))
LOG.info("successful scrape")
try:
article_json = utils.json_dumps(article_data)
except ValueError as err:
msg = "failed to serialize article data to article-json: %s" % str(err)
return response(mkresponse(ERROR, msg, request))
LOG.info("successfully serialized article-data to article-json")
# phew! gauntlet ran, we're now confident of passing this article-json to lax
# lax may still reject the data as invalid, but we'll proxy that back if necessary
params['article_json'] = article_json
try:
LOG.info("calling lax")
lax_response = call_lax(**params)
LOG.info("lax response: %r", lax_response)
return response(mkresponse(**lax_response))
except Exception as err:
# lax didn't understand us or broke
msg = "lax failed attempting to handle our request: %s" % str(err)
response(mkresponse(ERROR, msg, request))
# when lax fails, we fail
raise
def main():
# If arguemnts given but not enough throw and error
if len(argv) != 1 and len(argv) != 4:
print '****************************************'
print "Please enter authentication and update target as arguments or at the prompt"
print '****************************************'
sys.exit()
else:
# if no arguments, prompt for user, password, and in date
if len(argv) == 1:
# validation credentials and ETL date target
username = raw_input('Enter your MongoDB username:'******'Enter your MongoDB password:'******'Enter the date you want to update (mm/yyyy):')
# if validation args given, pass
elif len(argv) == 4:
username = argv[1]
password = argv[2]
in_date = argv[3]
# check to see if all three fields have input
if len(username) < 1 or len(password) < 1 or len(in_date) < 1:
print '****************************************'
print "You must enter a username, password and date"
print '****************************************'
sys.exit()
else:
# validation check and str format
update_date = validate(in_date)
# grab target data from morph.io API
update = getNewData(update_date)
# if data returned, proceed
if len(update) == 0:
print '****************************************'
print 'No data returned from API. Check for API status or whether DPKO has published new numbers.'
print '****************************************'
sys.exit()
else:
# normalize data
norm_data = normalize(update, update_date)
status_check = dateCheck(norm_data[1]['cont_date'],username,password)
if status_check == 1:
# Creat csv files for R script and for archive
csvCreator(norm_data, update_date)
# Run reporting script
r = robjects.r
r.source("../r/reporting.R")
# load data into mongodb
mongoLoad(norm_data, username, password)
# Load generated files to s3 bucket
s3LoadFiles()
elif status_check == 0:
print '****************************************'
print "Data already up to date."
print '****************************************'
sys.exit()
def validate_response(response):
utils.validate(response, conf.RESPONSE_SCHEMA)
max_size_in_bytes = 262144
response_size = len(serialise_response(response))
ensure(response_size <= max_size_in_bytes, "response size (%s) is too large: %s" % (response_size, max_size_in_bytes), ValidationError)
def test_can_accept_valid_range(self):
val = u.validate('2AW7', 1, self.ranges, self.known)
self.assertTrue(val)
