def list_identifiers(status=[], first=0, max_size=0, order_by=None):
db = get_db_instance()
models = Models(db)
model_identifiers = models.getIdentifiers()
items = model_identifiers.all(status=status,
skip=first,
limit=max_size,
order_by=('updated', 1))
i = 0
def show(x, alt, f=lambda a: a):
return f(x) if x is not None else alt
def isodate(t):
return datetime.datetime.fromtimestamp(t).isoformat()
for item in items:
ean = show(item.ean, '-')
asin = show(item.asin, '-')
status = show(item.status, '-', str)
updated = show(item.updated, '-', isodate)
print(
'#{i}\tasin:{asin}\tean:{ean}\tstatus:{status}\tupdated:{updated}'.
format(i=i, asin=asin, ean=ean, status=status, updated=updated))
i += 1
def test_create_model(make_stubber, error_code):
lookoutvision_client = boto3.client('lookoutvision')
lookoutvision_stubber = make_stubber(lookoutvision_client)
project_name = 'test-project_name'
model_version = 'test-model'
model_arn = 'test-arn'
out_buck = 'doc-example-bucket'
out_folder = 'test-results'
training_results = f's3://{out_buck}/{out_folder}'
status = 'TRAINED'
lookoutvision_stubber.stub_create_model(project_name, out_buck, out_folder,
model_arn, model_version)
lookoutvision_stubber.stub_describe_model(project_name,
model_version,
model_arn,
status,
error_code=error_code)
if error_code is None:
got_status, got_version = Models.create_model(lookoutvision_client,
project_name,
training_results)
assert got_status == status
assert got_version == model_version
else:
with pytest.raises(ClientError) as exc_info:
Models.create_model(lookoutvision_client, project_name,
training_results)
assert exc_info.value.response['Error']['Code'] == error_code
def outside_images(self):
models = Models()
number_of_files = 0
print "Retrieving all online avisos. Please, wait..."
array_avisos_online = models.get_all_avisos_online()
print '[Ok]'
print "Retrieving all histogramed avisos. Please, wait..."
array_avisos_with_hist = models.get_all_avisos_with_histogram()
print '[Ok]'
for aviso_online_id in array_avisos_online:
is_on_hist_table = False
for aviso_online_with_hist_id in array_avisos_with_hist:
if aviso_online_id == aviso_online_with_hist_id:
is_on_hist_table = True
break
if not is_on_hist_table:
aviso_json = {"id_aviso":aviso_online_id}
models.save_id_aviso_not_histogram_table(aviso_json)
if number_of_files%100==0:
print number_of_files
number_of_files +=1
def initialise(self, param):
self.parameters = param
self.intrinsic_motivation = IntrinsicMotivation(param)
self.models = Models(param)
self.exp_iteration = param.get('exp_iteration')
self.iteration = 0
self.pos = []
self.cmd = []
self.img = []
self.goal_code = []
self.current_goal_x = -1
self.current_goal_y = -1
self.current_goal_idx = -1
self.prev_goal_idx = -1
self.goal_image = np.zeros(
(1, param.get('image_size'), param.get('image_size'),
param.get('image_channels')), np.float32)
np.random.seed() # change the seed
self.prev_pos = self.get_starting_pos()
def process(self):
try:
self.progress.emit('Проверка имени пользователя и пароля')
if not self.checkAuth():
self.progress.emit(
'<font color=red>Неверное имя пользователя и/или пароль</font>'
)
self.finished.emit()
return
self.progress.emit(
'Синхронизация локального хранилища с сервером...')
for client in Options.get().local_clients:
Models.get().clients.saveItem(client)
for application in Options.get().local_applications:
Models.get().applications.saveItem(application)
Options.get().local_clients = []
Options.get().local_applications = []
self.progress.emit('Загрузка типов заявок')
Models.get().application_types.loadData()
self.progress.emit('Загрузка заказчиков')
Models.get().clients.loadData()
self.progress.emit('Загрузка специалистов')
Models.get().specialists.loadData()
self.progress.emit('')
self.result.emit(True)
except requests.exceptions.ConnectionError:
self.result.emit(False)
self.finished.emit()
def train(self, token_lists, model_file):
self.model_file = "./trained_model/random3/randomhole.tfl"
self.prepare_data(token_lists)
xs, ys = self.getTrainData(token_lists)
with tf.Graph().as_default():
self.model = Models().create_network(
self.in_max_int,
self.out_max_int,
model_name="bidirectional_attention_rnn",
in_seq_len=self.in_seq_len,
out_seq_len=self.out_seq_len,
num_layers=2,
memory_size=64,
embedding_size=128,
num_heads=6,
scope="randomhole")
self.model.load(self.model_file)
self.model.fit(xs,
ys,
n_epoch=1,
batch_size=256,
shuffle=True,
show_metric=False,
run_id="Random Hole Completion")
self.model.save(self.model_file)
def create_images_histogram_collection(self):
models = Models()
number_of_files = 0
for dir_name, dir_names, file_names in os.walk(Constants.LOCAL_DIR_SAVE_PHOTO):
#walking inside all ads
for subdir_name in dir_names:
aviso_json = {"id_aviso":subdir_name, "photos":[]}
for dir_name_, dir_names_, file_names_ in os.walk(os.path.join(dir_name, subdir_name)):
#walking inside all photos of the specific ad
for file_name_ in file_names_:
try:
#generating the histogram and adding it to the json to be added to mongo
hist = self.get_histogram(os.path.join(dir_name, subdir_name, file_name_))
hist_json = {"photo_path":subdir_name + "/" + file_name_, "histogram":json.dumps(hist.tolist())}
aviso_json["photos"].append(hist_json)
except:
pass
if number_of_files%1000==0:
print number_of_files
number_of_files +=1
#calling model to add item to mongo
models.add_image_histogram(aviso_json)
print "[OK] Created histograms for " + str(number_of_files) + " photos."
def train_model(lookoutvision_client, bucket, project_name):
"""
Trains a model.
:param lookoutvision_client: A Boto3 Lookout for Vision client.
:param bucket: The bucket where the training output is stored.
:param project_name: The project that you want to train.
"""
print("Training model...")
training_results = f"{bucket}/{project_name}/output/"
status, version = Models.create_model(lookoutvision_client, project_name,
training_results)
Models.describe_model(lookoutvision_client, project_name, version)
if status == "TRAINED":
print(
"\nCheck the performance metrics and decide if you need to improve "
"the model performance.")
print(
"\nMore information: "
"https://docs.aws.amazon.com/lookout-for-vision/latest/developer-guide/improve.html"
)
print("If you are satisfied with your model, you can start it.")
start_model(lookoutvision_client, project_name, version)
else:
print("Model training failed.")
def save_models():
all_auto = model_auto()
for auto in all_auto:
for brand in Brands.select().where(Brands.brand == auto['brand']):
# print(brand.brand)
for model in auto['models']:
Models.create(brand=brand, models=model)
def delete(num):
db = Models()
if not db.view():
return render_template('success.html', not_delete=True)
else:
if request.method == 'GET':
db.delete(str(num))
return render_template('success.html', delete=True)
def __init__(self, json=None): self.fields = ['buyoutPrice', 'reservePrice', 'imgUrl', 'status', 'user', 'categorie', 'lat', 'long'] self.unique = [] # Unique fields are also mandatory self.mandatory = ['title', 'description', 'dateStart', 'dateLength', 'startPrice'] # Champs obligatoires // status : encherissable, en envoi, envoyé ? self.editable_fields = ['title', 'description', 'imgUrl', 'lat', 'long'] # Champs éditables par l'utilisateur self.belongs_to = ['user', 'categorie'] # Est rattaché à un utilisateur, crée un champ 'user_id' self.has_many = ['bids'] Models.__init__(self, json)
def train(self, token_lists, model_file):
self.total = 0
self.correct = 0
self.incorrect = 0
self.top2 = 0
self.top3 = 0
self.prefix_model_file = "./trained_model/prefix_final/prefix.tfl"
self.suffix_model_file = "./trained_model/suffix_final/suffix.tfl"
self.prepare_data(token_lists)
# xs = xs[:40960]
# ys = ys[:40960]
(pxs, pys, sxs, sys) = self.getTrainData(token_lists)
with tf.Graph().as_default():
self.prefix_model = Models().create_network(
len(self.string_to_number),
len(self.string_to_number),
model_name="bidirectional_attention_rnn",
in_seq_len=self.in_seq_len,
out_seq_len=self.out_seq_len,
num_layers=2,
memory_size=64,
embedding_size=128,
num_heads=8,
scope="prefix")
self.prefix_model.load(self.prefix_model_file)
self.prefix_model.fit(pxs,
pys,
n_epoch=1,
batch_size=512,
shuffle=True,
show_metric=False,
run_id="Prefix Completion")
self.prefix_model.save(self.prefix_model_file)
with tf.Graph().as_default():
self.suffix_model = Models().create_network(
len(self.string_to_number),
len(self.string_to_number),
model_name="bidirectional_attention_rnn",
in_seq_len=self.in_seq_len,
out_seq_len=self.out_seq_len,
num_layers=2,
memory_size=64,
embedding_size=128,
num_heads=8,
scope="suffix")
self.suffix_model.load(self.suffix_model_file)
self.suffix_model.fit(sxs,
sys,
n_epoch=1,
batch_size=512,
shuffle=True,
show_metric=False,
run_id="Suffix Completion")
self.suffix_model.save(self.suffix_model_file)
def cross_test(self, rfclf, feature_models, evalTag = True):
#load data
[tt_idx, x_te, y_te] = np.load(self.data_file_test)
if self.model_dict['lrTModel']:
[x_te_title, y_te] = np.load(self.data_file_title_test)
if self.model_dict['scoreModel']:
[x_te_score] = np.load(self.data_file_score_test)
if self.model_dict['scoreTModel']:
[x_te_score_title] = np.load(self.data_file_score_title_test)
if self.model_dict['dictModel']:
dic_result = np.array(np.load(self.dict_result)[0])[tt_idx]
# dic_result[dic_result > 0] = 1
# dic_result[dic_result < 0] = -1
#get features
n_features = []
for key in self.model_dict.keys():
if self.model_dict[key]:
x_test = x_te
if key == 'dictModel':
n_features.append(dic_result)
continue
elif key == 'scoreModel':
n_features.append(Models().select_demo(key, 0, 0).predict(x_te_score))
continue
elif key == 'scoreTModel':
n_features.append(Models().select_demo(key, 0, 0).predict(x_te_score_title))
continue
elif key == 'lrTModel':
x_test = x_te_title
if key == 'lrModel' or key == 'lrTModel':
n_features.append(feature_models[key].predict_proba(x_test))
else:
n_features.append(feature_models[key].predict(x_test))
#start the second RF model
x_second_te = np.column_stack(tuple(n_features))
rf_result = rfclf.predict(x_second_te)
rf_result_proba = rfclf.predict_proba(x_second_te)
#evaluate the precision of model
rf_rp = np.copy(rf_result)
rf_rp[rf_result_proba.max(axis=1) < 0.5] = 0
if evalTag:
self.evaluate(rf_result, y_te)
# self.DT.write_data('../data/negative.xls', self.origin_data_file,
# tt_idx[rf_result == -1], y_te[rf_result == -1])
# self.DT.write_data('../data/positive.xls', self.origin_data_file,
# tt_idx[rf_result == 1], y_te[rf_result == 1])
# self.DT.write_data('../data/zeros.xls', self.origin_data_file,
# tt_idx[rf_result == 0], y_te[rf_result == 0])
return rf_result, y_te, rf_rp
def __init__(self, user_name = None, user_password=None, session = None):
'''
This constructor has at least a session object to start petitions agains
'''
if session != None:
self.session = session
elif user_name != None and user_password != None:
self.session = Models.get_session(user_name, user_password, sqlite=False)
else:
self.session = Models.get_session()
def _init_model(self):
M = Models()
model = M.FPN(img_ch=3, output_ch=1)
# model = U_Net(img_ch=3, output_ch=1)
if torch.cuda.device_count() > 1 and self.args.mgpu:
print("Let's use", torch.cuda.device_count(), "GPUs!")
# dim = 0 [30, xxx] -> [10, ...], [10, ...], [10, ...] on 3 GPUs
model = nn.DataParallel(model)
self.model = model.to(self.device)
def creating_dataframe(self, dictionary):
final_words = []
final_words1 = []
documents = []
l = []
z = []
docs = {}
keys = dictionary.keys()
for key in keys:
kk = str(key)
k = re.findall(r'\d{8}', kk)
l.append(k)
for i in l:
for j in i:
z.append(j)
for key in z:
# if key == '19234329':
print(
"###################### Generating topic labels for {} ############################"
.format(key))
df = pd.DataFrame(dictionary[key])
df.columns = ['Text']
df_ = df['Text'].apply(lambda x: ''.join(x))
df_ = df_.str.lower()
df_ = df_.apply(self.tokenize)
df_ = df_.apply(self.replace)
df_ = df_.apply(self.split)
df_ = df_.apply(self.terms_only)
df_ = df_.apply(lambda x: ' '.join(x))
df_ = df_.apply(lambda x: re.sub(r' +', ' ', x))
[final_words.append("".join(i).strip().split()) for i in df_]
[final_words1.append(i) for i in final_words if len(i) >= 5]
[
documents.append(re.sub(r' +', " ", (' '.join(i))))
for i in final_words1
]
if key in docs:
docs[key].append(documents)
else:
docs[key] = documents
mm = Models(5, 10, **docs)
terms_to_wiki = mm.calling_methods('LDA')
ll = Labels(terms_to_wiki)
wiki_titles = ll.get_titles_wiki()
equal_length = ll.remove_all_null_dicts_returned_from_wiki(
**wiki_titles)
frq = ll.calculating_word_frequency(**equal_length)
results = ll.predicting_label(**frq)
print(key, results)
print('########### FINAL FILE EXECUTED ##################')
def search(query):
assert isinstance(query, str)
# Model
db = get_db_instance()
models = Models(db)
model_caches = models.getCaches()
# Yahoo shopping API
appid = get_yahoo_appid()
assert appid is not None
# Run
y = YahooShopping(appid=appid, cache=model_caches)
content = y.query(query=query, ignoreCache=False)
print(content)
def load(self, token_lists, model_file):
self.model_file = "./trained_model/random4/randomhole.tfl"
self.prepare_data(token_lists)
with tf.Graph().as_default():
self.model = Models().create_network(self.in_max_int,
self.out_max_int,
model_name="bidirectional_attention_rnn",
in_seq_len=self.in_seq_len, out_seq_len=self.out_seq_len,
num_layers=2, memory_size=128,
embedding_size=128, num_heads=8, scope="randomhole")
self.model.load(self.model_file)
def start(self):
bag_of_words, words = TermFrequency(self.trained).create_vocabulary()
v = Vectorizer(self.trained, self.classify, words, bag_of_words)
tfidf_trained = v.tfidf_for_tweets_trained
evaluations = v.evaluations
tfidf_to_classify = v.tfidf_for_tweets_to_classify
models = Models(tfidf_trained, evaluations, tfidf_to_classify)
prediction = models.svm_linear()
return prediction
def start(self):
bag_of_words, words = TermFrequency(self.trained).create_vocabulary()
v = Vectorizer(self.trained, self.classify, words, bag_of_words)
tfidf_trained = v.tfidf_for_tweets_trained
evaluations = v.evaluations
tfidf_to_classify = v.tfidf_for_tweets_to_classify
models = Models(tfidf_trained, evaluations, tfidf_to_classify)
prediction = models.svm_linear()
return prediction
def saveModels():
'''
This function opens the tarfile, preprocess the data, train models on it and it
saves the model in the Models directory.
'''
tar = tarfile.open('Data/babi_tasks_1-20_v1-2.tar.gz')
challenges = {
# QA1 with 10,000 samples
'single_supporting_fact_10k': 'tasks_1-20_v1-2/en-10k/qa1_single-supporting-fact_{}.txt',
# QA2 with 10,000 samples
'two_supporting_facts_10k': 'tasks_1-20_v1-2/en-10k/qa2_two-supporting-facts_{}.txt',
}
## Single Supporting Fact Challenge
ss_train_stories, ss_test_stories, \
ss_stories_train, ss_questions_train, ss_answers_train, \
ss_stories_test, ss_questions_test, ss_answers_test, \
ss_story_maxlen, ss_story_maxsents, ss_question_maxlen, \
ss_vocab, ss_vocab_size, ss_word2idx = \
Preprocess.getData(challenges['single_supporting_fact_10k'], tar)
ss_idx2word = {value : key for key, value in ss_word2idx.items()}
single_model, single_debug_model = \
Models.singleModel(ss_story_maxlen, ss_story_maxsents, ss_question_maxlen, ss_vocab_size, \
ss_stories_train, ss_questions_train, ss_answers_train, \
ss_stories_test, ss_questions_test, ss_answers_test, \
EMBEDDING_DIM, NUM_EPOCHS, BATCH_SIZE)
Utilities.saveModel(single_model, 'single_model')
Utilities.saveModel(single_debug_model, 'single_debug_model')
## Two Supporting Fact challenge
ts_train_stories, ts_test_stories, \
ts_stories_train, ts_questions_train, ts_answers_train, \
ts_stories_test, ts_questions_test, ts_answers_test, \
ts_story_maxlen, ts_story_maxsents, ts_question_maxlen, \
ts_vocab, ts_vocab_size, ts_word2idx = \
Preprocess.getData(challenges['two_supporting_facts_10k'], tar)
ts_idx2word = {value : key for key, value in ts_word2idx.items()}
double_model, double_debug_model = \
Models.doubleModel(ts_story_maxlen, ts_story_maxsents, ts_question_maxlen, ts_vocab_size, \
ts_stories_train, ts_questions_train, ts_answers_train, \
ts_stories_test, ts_questions_test, ts_answers_test, \
EMBEDDING_DIM, NUM_EPOCHS_2, BATCH_SIZE)
Utilities.saveModel(double_model, 'double_model')
Utilities.saveModel(double_debug_model, 'double_debug_model')
def train():
embeddings = np.load('text_embedding.npy', allow_pickle=True)
sentiments = np.load('sentiments.npy', allow_pickle=True)
texts = np.load('texts.npy', allow_pickle=True)
all_texts = np.load('text_cache.npy', allow_pickle=True)
categorical_sentiments = to_categorical(sentiments, num_classes=5)
tokenizer = Tokenizer(num_words=300000, oov_token=None)
tokenizer.fit_on_texts(all_texts)
X_train, X_test, Y_train, Y_test = train_test_split(texts,
categorical_sentiments,
test_size=0.2)
np.save("text_train.npy", X_train)
np.save("sentiment_train.npy", Y_train)
models = Models()
logdir = "logs/scalars/" + datetime.now().strftime("%Y%m%d-%H%M%S")
tensorboard_callback = TensorBoard(log_dir=logdir)
models = []
bgruModel = "ensemble_bgru.h5"
models.buil_pre_model(embeddings)
model = models.model
if os.path.isfile(filepath):
model = load_model(filepath)
checkpoint = ModelCheckpoint(filepath,
monitor='loss',
verbose=1,
save_best_only=True,
mode='min')
callbacks_list = [checkpoint, tensorboard_callback]
model.fit(pad_sequences(tokenizer.texts_to_sequences(X_train[:500000]),
maxlen=75),
Y_train[:500000],
batch_size=512,
epochs=50,
validation_data=(pad_sequences(tokenizer.texts_to_sequences(
X_test[:5000]),
maxlen=75), Y_test[:5000]),
callbacks=callbacks_list,
shuffle=True)
result = model.predict_on_batch(
pad_sequences(tokenizer.texts_to_sequences([
" What happened 2 ur vegan food options?! At least say on ur site so i know I won't be able 2 eat anything for next 6 hrs #fail",
" I sleep hungry and It gets harder everyday",
"everything is great, i have lost some weight",
"awesome, really cool", "should I play cards",
"I am full and inshape", "is it okay to be that hungry at night?"
]),
maxlen=75))
print("result: ", np.argmax(result, axis=-1), "\n")
def update(num):
form = NoteUpdate(request.form) # instantiating form to use the forms defined from the Form class in form.py
if request.method == 'GET':
return render_template('update.html', update=True, form=form, num=num)
else:
if not form.validate_on_submit(): # making sure that the form is validated before submission
return render_template('update.html', update=True, not_validate=True, form=form, num=num)
else:
db = Models()
name = request.form['note_name']
subject = request.form['note_subject']
content = request.form['note_content']
db.update(str(num), name, subject, content)
return render_template('success.html', update=True, form=form)
def __init__(self, logger=None):
if logger is None:
self.logger = logging.getLogger()
else:
self.logger = logger
# Model
db = get_db_instance()
models = Models(db)
model_caches = models.getCaches()
# Proxy
proxies = get_amazon_scraping_proxy()
# Run
self.amazon_scraper = AmazonScraper(cache=model_caches, proxies=proxies,
logger=self.logger)
def __init__(self, json=None):
self.fields = ["name"] # Default fields, can be blank in the request.
self.unique = [
"email",
"username",
"rate",
"rateNumber",
] # Unique fields are also mandatory // Rate = note de l'utilisateur & rate_number = nombre de vote (pour le calcul coté client)
self.mandatory = ["password"] # Mandatory fields.
self.intern_fields = ["token"]
self.editable_fields = ["name", "password", "email"]
self.has_many = ["products", "bids", "addresses", "payments"]
self.__password = None # Special field with a callback
Models.__init__(self, json)
def predict(self, pred_file = None):
#load models
rfclf = joblib.load(self.rfmodel_file)
feature_models = dict()
disable_models = ['dictModel', 'scoreModel', 'scoreTModel']
for key in self.model_dict.keys():
if self.model_dict[key] and (key not in disable_models):
clf = joblib.load(self.model_dir + '/' + key)
feature_models.setdefault(key, clf)
if pred_file != None:
test_data = self.pre_data_treate(pred_file)
[x_te, x_te_title, x_te_score, x_te_score_title] = test_data
#get features
n_features = []
for key in self.model_dict.keys():
if self.model_dict[key]:
x_test = x_te
if key == 'scoreModel':
n_features.append(Models().select_demo(key, 0, 0).predict(x_te_score))
continue
elif key == 'scoreTModel':
n_features.append(Models().select_demo(key, 0, 0).predict(x_te_score_title))
continue
elif key == 'lrTModel':
x_test = x_te_title
if key == 'lrModel' or key == 'lrTModel':
n_features.append(feature_models[key].predict_proba(x_test))
else:
n_features.append(feature_models[key].predict(x_test))
#start the second RF model
x_second_te = np.column_stack(tuple(n_features))
rf_result = rfclf.predict(x_second_te)
rf_result_proba = rfclf.predict_proba(x_second_te)
#evaluate the precision of model
rf_rp = np.copy(rf_result)
rf_rp[rf_result_proba.max(axis=1) < 0.5] = 0
self.DT.write_data('../data/negative.xls', pred_file,
range(rf_result.shape[0]), rf_result)
self.DT.write_data('../data/negative1.xls', pred_file,
range(rf_result.shape[0]), rf_rp)
else:
return self.cross_test(rfclf, feature_models)
def _init_model(self):
criterion = nn.BCELoss()
self.criterion = criterion.to(self.device)
M = Models()
model = M.PSP(img_ch=3, output_ch=1)
self.model = model.to(self.device)
# init_weights(self.model, 'kaiming', gain=1)
# summary(self.model, input_size=(4, 448, 448))
self.model_optimizer = optim.Adamax(model.parameters(),
lr=1e-3,
weight_decay=0.01)
self.scheduler = optim.lr_scheduler.CosineAnnealingLR(
self.model_optimizer, T_max=len(self.train_queue))
def fromDict(cls, data):
obj = cls()
obj.id = data['id']
obj.date = QDate().fromString(data['date'], 'yyyy-MM-dd')
obj.type = Models.get().application_types.getItemById(data['type'])
obj.client = Models.get().clients.getItemById(data['client'])
for item in data['entries']:
obj.entries.append(ApplicationEntry.fromDict(item))
if data['contract']:
obj.contract = Contract.fromDict(data['contract'])
if data['account']:
obj.account = Account.fromDict(data['account'])
if data['order']:
obj.order = Order.fromDict(data['order'])
return obj
def initialise(self, param):
self.parameters = param
if not param.get('goal_selection_mode') == 'som':
print('wrong goal selection mode, exit!')
sys.exit(1)
self.intrinsic_motivation = IntrinsicMotivation(param)
if (self.parameters.get('train_cae_offline')) or (
self.parameters.get('train_som_offline')):
self.models = Models(param, train_images=self.train_images)
else:
self.models = Models(param)
plot_encoded = self.models.encoder.predict(
np.asarray([self.test_images[0:5]
]).reshape(5, self.parameters.get('image_size'),
self.parameters.get('image_size'),
self.parameters.get('image_channels')))
plots.plots_cae_decoded(
self.models.decoder,
plot_encoded,
self.test_images[0:5],
image_size=self.parameters.get('image_size'),
directory=self.parameters.get('directory_pretrained_models'))
self.experiment_id = param.get('experiment_id')
self.iteration = 0
self.pos = []
self.cmd = []
self.img = []
self.goal_code = []
self.current_goal_x = -1
self.current_goal_y = -1
self.current_goal_idx = -1
self.prev_goal_idx = -1
self.goal_image = np.zeros(
(1, param.get('image_size'), param.get('image_size'),
param.get('image_channels')), np.float32)
np.random.seed() # change the seed
self.prev_pos = self.get_starting_pos()
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"-i",
"--eval-dir",
required=True,
help="List of problem instances, i.e., [filename].txt files.")
parser.add_argument(
"-o",
"--output-dir",
required=True,
help="List of truth files, one for each txt file in our --eval-dir.")
args = parser.parse_args()
if os.path.exists(args.output_dir):
logging.warn(
'[%s] already exists! Will possibly override files in it!' %
args.output_dir)
else:
logging.debug('Creating [%s].' % args.output_dir)
os.makedirs(args.output_dir)
texts = load_texts(args.eval_dir)
scalers = load_scalers()
models = Models()
splitted_texts = split_texts(texts)
# NOTE using standard scaler
splitted_texts_vectors = to_vectors(splitted_texts,
scalers['standard'].transform)
logging.debug('Making predictions...')
for i, chunks_vectors_tuple in enumerate(splitted_texts_vectors, 1):
chunks_vectors_pairs = itertools.combinations(chunks_vectors_tuple, 2)
confidences = [0, 0] # (for false, for true)
for a, b in chunks_vectors_pairs:
pred_for_false, pred_for_true = models.classify_proba(a, b)
# TODO if very confident about style change - break or something?
confidences[0] += pred_for_false
confidences[1] += pred_for_true
y_pred = bool(np.argmax(confidences))
with open(os.path.join(args.output_dir, 'problem-%d.truth' % i),
'w') as f:
json.dump({'changes': y_pred}, f)
logging.debug('Done.')
def post(self, request, *args, **kwargs):
"""
Form submit.
"""
if not request.is_ajax():
return HttpResponseBadRequest
model_name = request.POST.get('model_name', '')
model = Models.get_model(model_name)
if model is None:
return HttpResponseBadRequest()
form_class = DynamicModelForm(model_name).get_form()
form = form_class(request.POST)
if form.is_valid():
form.save()
response = {
'result': 'success'
}
else:
errors = {}
for k in form.errors:
errors[k] = form.errors[k][0]
response = {
'result': 'error',
'errors': errors
}
return HttpResponse(json.dumps(response, cls=DateTimeEncoder), mimetype='application/json')
def post(self, request, *args, **kwargs):
"""
Changing the model field via table.
"""
if 'model_name' not in request.POST or 'field_index' not in request.POST or 'model_pk' not in request.POST or 'value' not in request.POST:
return HttpResponseBadRequest()
else:
try:
model = Models.get_model(request.POST.get('model_name'))
obj = model.objects.get(pk=request.POST.get('model_pk'))
field = [f.name for f in model._meta.fields][int(request.POST.get('field_index'))]
try:
setattr(obj, field, cgi.escape(request.POST.get('value')))
obj.full_clean()
obj.save()
response = {
'result': 'success'
}
except ValidationError:
response = {
'result': 'error',
'error': u"Поле заполнено неправильно"
}
return HttpResponse(json.dumps(response, cls=DateTimeEncoder), mimetype='application/json')
except model.DoesNotExist:
raise Http404
def evaluate_model(trainX, trainy, testX, testy):
epochs, batch_size = 15, 64
verbose, n_steps, n_length = 0, 4, 32
n_timesteps, n_features, n_outputs = trainX.shape[1], trainX.shape[
2], trainy.shape[1]
model = 'lstm'
if model == 'cnnlstm':
trainX = trainX.reshape(
(trainX.shape[0], n_steps, n_length, n_features))
testX = testX.reshape((testX.shape[0], n_steps, n_length, n_features))
elif model == 'convlstm':
trainX = trainX.reshape(
(trainX.shape[0], n_steps, 1, n_length, n_features))
testX = testX.reshape(
(testX.shape[0], n_steps, 1, n_length, n_features))
model = Models(model, n_timesteps, n_features, n_outputs, n_steps,
n_length)
model.model.fit(trainX,
trainy,
epochs=epochs,
batch_size=batch_size,
verbose=verbose)
_, accuracy = model.model.evaluate(testX,
testy,
batch_size=batch_size,
verbose=verbose)
print('Accuracy: {:.4f}'.format(accuracy))
def __init__(self, model):
model = Models.get_model(model)
meta = type('Meta', (), {
"model": model,
})
self.form_object = type('DynamicModelForm', (forms.ModelForm, ),
{"Meta": meta})
def main():
if len(sys.argv) < 2:
print(OutputError.no_arguments)
print("Using default sample.json")
input_file = "sample.json"
else:
input_file = sys.argv[1]
print("Processing : " + input_file)
with open(input_file) as json_file:
json_data = json.load(json_file)
data_out = {
"meshes":[],
"textures": json_data["textures"],
"materials": json_data["materials"],
"nodes": json_data["nodes"]
}
# Processing meshes, that will be split into submeshes and all the data
# will be merged into one single file
with open(input_file.replace(".json", ".data"), "wb") as file_stream:
file_size = 0
for mesh_element in json_data[Models.json_tag]:
json_element, file_size = Models.process_meshes((mesh_element), file_stream, file_size)
data_out["meshes"].append(json_element)
# Writing metadata file out
with open(input_file.replace(".json", ".metadata"), "w") as file_stream:
json.dump(data_out, file_stream, indent=4, separators=(',', ': '))
def setupUi(self):
self.ui.edtDate.setDate(QDate().currentDate())
self.ui.tblElevatorsData.horizontalHeader().setVisible(True)
self.ui.cmbApplicationType.setModel(Models.get().application_types)
self.ui.tblElevatorsData.setModel(DataTableModel())
self.ui.tblElevatorsData.setItemDelegate(DataTableDelegate())
def make_model(train_image, train_label, test_image, test_label, name):
try:
m = Models(shape=train_image.shape[1:], name=name)
model = exec('m.' + bp.USE_MODEL + '(train_image, train_label, test_image, test_label)')
print('info: input_shape:{}'.format(train_image.shape[1:]))
except Exception as e:
print(e)
exit()
def insert_test_clients(self):
Models.insert_test(self.session)
def validate_grouped_equals(self): models = Models() models.validate_grouped_equals()
def __init__(self, json=None):
self.belongs_to = ["user"]
self.fields = ["number", "security", "date", "name"]
self.editable_fields = ["number", "security", "date", "name"]
Models.__init__(self, json)
def __init__(self, json=None): self.belongs_to = ['user'] self.fields = ['street1', 'street2', 'country', 'zipcode', 'city'] self.editable_fields = ['street1', 'street2', 'country', 'zipcode', 'city'] Models.__init__(self, json)
def build_model(self, save_tag = True):
#load data
tr_data = np.load(self.data_file_train)
[tt_idx, x_tr, y_tr] = tr_data
if self.model_dict['lrTModel']:
[x_tr_title, y_tr] = np.load(self.data_file_title_train)
if self.model_dict['dictModel']:
dicResult = np.array(np.load(self.dict_result)[0], dtype=float)[tt_idx]
# dicResult[dicResult > 0] = 1
# dicResult[dicResult < 0] = -1
if self.model_dict['scoreModel']:
[x_tr_score] = np.load(self.data_file_score_train)
if self.model_dict['scoreTModel']:
[x_tr_score_title] = np.load(self.data_file_score_title_train)
mid = int(x_tr.shape[0]/2)
#train model
models = Models()
#get the features
n_features = []
for key in self.model_dict.keys():
#remove the models unused
if not self.model_dict[key]:
continue
x_train = x_tr
if key == 'dictModel':
n_features.append(dicResult)
continue
elif key == 'lrTModel':
x_train = x_tr_title
elif key == 'scoreModel':
x_train = x_tr_score
elif key == 'scoreTModel':
x_train = x_tr_score_title
clf = models.select_demo(key, x_train[:mid, :], y_tr[:mid])
if key == 'lrModel' or key == 'lrTModel':
mid_to_end = list(clf.predict_proba(x_train[mid:,:]))
else:
mid_to_end = list(clf.predict(x_train[mid:,:]))
clf = models.select_demo(key, x_train[mid:, :], y_tr[mid:])
if key == 'lrModel' or key == 'lrTModel':
top_to_mid = list(clf.predict_proba(x_train[:mid, :]))
else:
top_to_mid = list(clf.predict(x_train[:mid, :]))
n_features.append(np.array(top_to_mid + mid_to_end))
#get the true model
clf = models.select_demo(key, x_train, y_tr)
if save_tag and key != 'scoreModel' and key != 'scoreTModel':
joblib.dump(clf, self.model_dir + '/' + key)
if len(n_features) > 1:
x_tr_second = np.column_stack(tuple(n_features))
else:
print 'Error: less models'
sys.exit(1)
self.GBDT.fit(x_tr_second, y_tr)
print self.GBDT.feature_importances_
#train the second model
rfclf = models.rfdemo(x_tr_second, y_tr)
if save_tag:
joblib.dump(rfclf, self.rfmodel_file)
def create_images_histogram_from_images_backup_iw(self):
models = Models()
self.number_of_files = 0
print "Retrieving all online avisos. Please, wait..."
self.array_avisos_online = models.get_all_avisos_online()
print "[Ok]"
def step((ext, self), dir_name, files):
download_100x75 = True
aviso_id = dir_name[dir_name.rfind("Constants.LOCAL_DIR_SAVE_PHOTO")+len(Constants.LOCAL_DIR_SAVE_PHOTO)+2:dir_name.rfind("/")]
aviso_id = aviso_id.translate(None, "/")
if "100x75" in dir_name or "1200x1200" in dir_name:
dir100x75 = dir_name[:dir_name.rfind("/")] + "/100x75"
dir1200x1200 = dir_name[:dir_name.rfind("/")] + "/1200x1200"
try:
aviso_id_int = int(aviso_id)
except ValueError:
aviso_id_int = 0
if aviso_id_int in self.array_avisos_online:
if "1200x1200" in dir_name:
if os.path.isdir(dir100x75):
download_100x75 = False
if download_100x75:
aviso_json = {"id_aviso":aviso_id, "photos":[]}
for file_name in files:
if file_name.lower().endswith(ext):
try:
#generating the histogram and adding it to the json to be added to mongo
hist = self.get_histogram(os.path.join(dir_name, file_name))
hist_json = {"photo_path":dir_name + "/" + file_name, "histogram":json.dumps(hist.tolist())}
aviso_json["photos"].append(hist_json)
except:
pass
models.add_image_histogram(aviso_json)
if self.number_of_files%100==0:
print self.number_of_files
self.number_of_files +=1
else:
if self.number_of_files%100==0:
print self.number_of_files
self.number_of_files +=1
print aviso_id
def save_compressed_histogram_online(self): models = Models() models.save_compressed_histogram_online()
def create_tuples_equals_avisos_collection(self): models = Models() models.create_tuples_equals_avisos_collection()
def create_raw_equal_avisos(self): models = Models() models.create_raw_equal_avisos()
def validate_arr(self): models = Models() models.validate_arr()
def create_duplicateds_group_collection_new(self): models = Models() models.create_duplicateds_group_collection_new()
def __init__(self, json=None): self.fields = ['name'] self.has_many = ['products'] self.editable_fields = ['name'] Models.__init__(self, json)
def create_images_histogram_from_online_ads(self):
models = Models()
number_of_files = 0
print "Retrieving all online avisos. Please, wait..."
self.array_avisos_online = models.get_all_avisos_online()
print "[Ok]"
for aviso_id in self.array_avisos_online:
if number_of_files%100==0:
print number_of_files
number_of_files +=1
aviso_json = {"id_aviso":aviso_id, "photos":[]}
# print os.walk(Constants.LOCAL_DIR_SAVE_PHOTO + complete_folder)
try:
if len(str(aviso_id))<10:
aviso_id = format(int(aviso_id), "010")
aviso_id_splitted = re.findall(r'.{1,2}',str(aviso_id),re.DOTALL)
complete_folder = ""
for folder_name in aviso_id_splitted:
complete_folder += folder_name + "/"
for root, dirs, files in os.walk(Constants.LOCAL_DIR_SAVE_PHOTO + complete_folder):
folder_to_download = ""
for folder in dirs:
if folder == "100x75":
folder_to_download = "100x75"
break
elif folder == "1200x1200":
folder_to_download = "1200x1200"
break
folder_name = Constants.LOCAL_DIR_SAVE_PHOTO + complete_folder + folder_to_download
for file in os.listdir(folder_name):
if file.endswith(".jpg"):
hist = self.get_histogram(os.path.join(folder_name, file))
hist_json = {"photo_path":folder_name + "/" + file, "histogram":json.dumps(hist.tolist())}
aviso_json["photos"].append(hist_json)
if len(os.listdir(folder_name))>0:
models.add_image_histogram(aviso_json)
break
except:
pass
def create_similar_photos_collection(self): models = Models() models.create_similar_photos_collection()
def __init__(self, json=None): self.fields = ['price', 'date'] self.belongs_to = ['user', 'product'] # Est rattaché à un utilisateur et un produit self.editable_fields = ['price', 'date'] Models.__init__(self, json)
