def __init__(self):
self.imgframe = tc.load_sframe('model/final/final.sframe')
self.model = tc.load_model('model/final/final_model')
self.sample = tc.Image()
self.results = SFrame()
self.rows = SArray()
self.pathlist = []
self.distance_list = []
def recode_step2_page():
try:
data_id = request.args.get('data_id')
target = request.args.get('target')
name = request.args.get('name')
my_data = UserData.query.filter_by(id=data_id).first()
my_model = TrainedModel()
form = TrainModelForm(request.form, obj=my_model)
data_frame = tc.load_sframe(my_data.sname)
names=data_frame.column_names()
types=data_frame.column_types()
orig_data = data_frame[str(target)]
norig_data = orig_data.to_numpy()
target_data = data_frame[str(target)].unique()
ntarget_data = target_data.to_numpy()
if request.method == 'POST':
mapped_values = []
data_frame = safely_add_col(str(target) + '_uncoded', data_frame[str(target)], data_frame)
for x in range(0, ntarget_data.__len__()):
mapped_values.append(str(request.form['new_value' + str(x)]))
cross_ref = []
for x in range(0, names.__len__()):
if (str(types[x].__name__) == "str"):
cross_ref.append(str(names[x]))
new_data = []
for field in norig_data:
for y in range(0, ntarget_data.__len__()):
if str(ntarget_data[y]) == str(field):
new_data.append(int(mapped_values[y]))
sa = SArray(new_data)
data_frame[str(target)] = sa
fwd_id = save_data(my_data, name, data_frame)
flash('Successfully re-coded ' + target + '!', 'success')
return redirect(url_for('data.data_details_page', data_id=fwd_id))
return render_template('pages/data/transforms/code_field_step2.html',
my_data=my_data,
form=form,
data_frame=data_frame,
names=names,
name=name,
types=types,
ntarget_data=ntarget_data,
target=target)
except Exception as e:
flash('Opps! Something unexpected happened. On the brightside, we logged the error and will absolutely look at it and work to correct it, ASAP.', 'error')
error = ErrorLog()
error.user_id = current_user.id
error.error = str(e.__class__)
error.parameters = request.args
db.session.add(error)
db.session.commit()
return redirect(request.referrer)
def convert_magic_page():
try:
data_id = request.args.get('data_id')
my_data = UserData.query.filter_by(id=data_id).first()
my_model = TrainedModel()
form = TrainModelForm(request.form, obj=my_model)
data_frame = tc.load_sframe(my_data.sname)
names=data_frame.column_names()
types=data_frame.column_types()
if request.method == 'POST':
magic = str(request.form['magic'])
name = str(request.form['name'])
for feature in request.form.getlist('features'):
orig_data = data_frame[str(feature)]
norig_data = orig_data.to_numpy()
new_data = []
for item in norig_data:
if str(item) == magic:
new_data.append(None)
else:
new_data.append(item)
sa = SArray(new_data)
data_frame[str(feature)] = sa
fwd_id = save_data(my_data, name, data_frame)
flash('Successfully cleared magic values!', 'success')
return redirect(url_for('data.data_details_page', data_id=fwd_id))
return render_template('pages/data/transforms/convert_magic.html',
my_data=my_data,
data_frame=data_frame,
names=names,
types=types,
form=form)
except Exception as e:
flash('Opps! Something unexpected happened. On the brightside, we logged the error and will absolutely look at it and work to correct it, ASAP.', 'error')
error = ErrorLog()
error.user_id = current_user.id
error.error = str(e.__class__)
error.parameters = request.args
db.session.add(error)
db.session.commit()
return redirect(request.referrer)
def custom_transform_page():
try:
data_id = request.args.get('data_id')
my_data = UserData.query.filter_by(id=data_id).first()
form = UserDataForm(request.form, obj=None)
data_frame = tc.load_sframe(my_data.sname)
cols = data_frame.column_names()
types = data_frame.column_types()
if request.method == 'POST':
try:
transform_code = request.form['transform_code']
target = request.form['target']
name = request.form['name']
transformed_data = []
local_space = {}
for val in data_frame[str(target)]:
context = {"in_var": 1, "scipy_stats": scipy_stats, "np": np}
exec(transform_code) in context
transformed_data.append(context['out_var'])
sa = SArray(transformed_data)
data_frame[str(name)] = sa
fwd_id = save_data(my_data, request.form['name'], data_frame)
flash('Data transform is sucessful!', 'success')
return redirect(url_for('data.data_details_page', data_id=fwd_id))
except Exception as inst:
flash('Failed to run Python code! ' + str(inst), 'error')
return render_template('pages/data/transforms/custom_transform.html',
my_data=my_data,
names=cols,
types=types,
form=form)
except Exception as e:
flash('Opps! Something unexpected happened. On the brightside, we logged the error and will absolutely look at it and work to correct it, ASAP.', 'error')
error = ErrorLog()
error.user_id = current_user.id
error.error = str(e.__class__)
error.parameters = request.args
db.session.add(error)
db.session.commit()
return redirect(request.referrer)
def safely_add_col(col_name, data_to_add, data_frame):
cols = data_frame.column_names()
if col_name in cols:
data_frame.remove_column(col_name)
sa = SArray(data=data_to_add)
return data_frame.add_column(sa, col_name)
return json_data
csv_file = sys.argv[1]
csv = pd.read_csv(csv_file,
quotechar='\'',
doublequote=True,
converters={'annotations': JSONParser})
for i, row in csv.iterrows():
image_folder = str(os.path.split(row['path'])[0])
break
data = tc.image_analysis.load_images(image_folder, with_path=True)
annotations = []
for j, item in enumerate(data):
has_annotation = False
for i, row in csv.iterrows():
if str(row['path']) == item['path']:
annotations.append(row['annotations'])
has_annotation = True
break
if not has_annotation:
annotations.append([])
data['annotations'] = SArray(data=annotations, dtype=list)
data.save('annotations.sframe')
print(data)
def smote_page():
# try:
data_id = request.args.get('data_id')
name = request.args.get('name')
target = request.args.get('target')
seed = request.args.get('seed')
algorithm = request.args.get('algorithm')
neighbors = request.args.get('neighbors')
my_data = UserData.query.filter_by(id=data_id).first()
my_model = TrainedModel()
form = TrainModelForm(request.form, obj=my_model)
data_frame = tc.load_sframe(my_data.sname)
data_frame_cleaned = data_frame
orig_data = data_frame[str(target)]
norig_data = orig_data.to_numpy()
target_data = data_frame[str(target)].unique()
ntarget_data = target_data.to_numpy()
if request.method == 'POST':
features_utf = request.form.getlist('features')
features_utf = request.form.getlist('features')
features_str = []
variables = []
for feat in features_utf:
features_str.append(str(feat))
variables.append(str(feat))
data_frame_cleaned = data_frame_cleaned.dropna(str(feat), how="any")
variables.append(str(request.form['target']))
df = shuffle(data_frame_cleaned.to_dataframe())
strategy = {}
for x in range(0, ntarget_data.__len__()):
strategy[int(ntarget_data[x])] = int(request.form['new_value' + str(x)])
sm = None
y_res = None
x_res = None
try:
if algorithm == "SMOTE":
sm = SMOTE(random_state=int(seed), sampling_strategy = strategy, k_neighbors=int(neighbors)-1)
else:
sm = ADASYN(random_state=int(seed), sampling_strategy = strategy, n_neighbors=int(neighbors)-1)
x_res, y_res = sm.fit_sample(df[variables], df[str(request.form['target'])])
except Exception as e:
flash(str(e), 'error')
return redirect(request.referrer)
my_dict = {}
np_y_res = np.array(y_res)
res = np_y_res.astype(str)
my_dict[str(request.form['target'])] = res
x=0
sf = tc.SFrame({ str(request.form['target']): y_res })
df = pd.DataFrame(x_res)
for feat in features_str:
dat = []
for val in df.ix[:, x]:
dat.append(val)
sa = SArray(data=dat)
sf = sf.add_column(sa, feat)
x = x + 1
sa = SArray(data=y_res, dtype=str)
sf = sf.add_column(sa, str(request.form['target']) + "_uncoded")
fwd_id = save_data(my_data, request.form['name'], sf)
flash('Data transform is sucessful!', 'success')
return redirect(url_for('data.data_details_page', data_id=fwd_id))
return render_template('pages/data/transforms/smote.html',
my_data=my_data,
form=form,
data_frame=data_frame,
num_rows=data_frame.num_rows(),
name=name,
target=target,
ntarget_data=ntarget_data,
names=data_frame.column_names(),
types=data_frame.column_types())
props['coordinates'] = {
'height': height,
'width': width,
'x': csv.iat[i, 3] + math.floor(width / 2),
'y': csv.iat[i, 5] + math.floor(height / 2)
}
labels.append(str(row['label']))
annotations.append([props])
found = True
break
if found == False:
annotations.append([])
labels.append('')
# make an array from labels and annotations, matching the data order
data['annotations'] = SArray(data=annotations, dtype=list)
data['label'] = SArray(data=labels)
# Explore interactively
# data['image_with_ground_truth'] = tc.object_detector.util.draw_bounding_boxes(data["image"], data["annotations"])
# data.explore()
# Make a train-test split
train_data, test_data = data.random_split(0.9)
# Create a model
model = tc.object_detector.create(train_data,
feature='image',
annotations='annotations',
max_iterations=2000)