def test_model_acc(self):
pred_par_path = Path(config.OUTPUT_MODEL_PAR_PATH, 'preds')
pred_path = Path(pred_par_path, 'df_test_pred.pkl')
df_test = pd.read_pickle(pred_path)
pd.options.display.max_columns = 10
pd.options.display.max_colwidth = 30
pd.options.display.max_rows = 999
df_scored = df_test[['video_name_stem', 'real_or_fake_digit', 'y']]
# df_scored = df_scored.sort_values(by=['video_name_stem'])
# logger.info(f'Head: {df_scored.head(100)}')
# df_grouped = df_scored.groupby('video_name_stem')['y'].mean()
df_grouped = df_scored.groupby('video_name_stem')
total = len(df_grouped)
matched = 0
for name, group in df_grouped:
# logger.info(group)
rof = group.iloc[0, group.columns.get_loc('real_or_fake_digit')]
score = group['y'].mean()
pred = 1 if score > .5 else 0
if rof == pred:
matched += 1
logger.info(f'matched: { group["y"].shape[0]}: {rof == pred}')
logger.info(f"%Acc: {matched/total}")
# df_test.sort_by(col)
logger.info(f'Head: {df_grouped.head(100)}')
def segmenter_data_bunch(config):
split_df = pd.read_csv(config.split_csv)
if config.debug_run: split_df = split_df.loc[:200]
train_data_paths, valid_data_paths, train_label_paths, valid_label_paths = [], [], [], []
if config.oversample:
split_df = oversample_train(split_df, random_seed=config.random_seed)
for i in range(len(split_df)):
data_path = './data/train_images/' + split_df.loc[i, 'ImageId_ClassId']
label_path = './data/train_masks/' + split_df.loc[i, 'ImageId_ClassId'].replace('.jpg', '.png')
if split_df.loc[i, 'is_valid']:
if config.load_valid_crops:
for i in range(7):
valid_data_paths.append(Path(data_path.replace('.jpg', '_c{}.jpg'.format(i))))
valid_label_paths.append(Path(label_path.replace('.png', '_c{}.png'.format(i))))
else:
valid_data_paths.append(Path(data_path))
valid_label_paths.append(Path(label_path))
else:
if config.load_valid_crops:
for i in range(config.train_duplicate): # So we don't spend a lot of time validating
if config.load_train_crops:
train_data_paths.append(Path(data_path.replace('.jpg', '_c{}.jpg'.format(i))))
train_label_paths.append(Path(label_path.replace('.png', '_c{}.png'.format(i))))
else:
train_data_paths.append(Path(data_path))
train_label_paths.append(Path(label_path))
else:
train_data_paths.append(Path(data_path))
train_label_paths.append(Path(label_path))
item_list = SegmentationItemListOneHot if config.one_hot_labels else SegmentationItemList
train = item_list(train_data_paths)
valid = item_list(valid_data_paths)
# train_label = SegmentationLabelList(train_label_paths)
# valid_label = SegmentationLabelList(valid_label_paths)
src = (item_list.from_folder('.')
.split_by_list(train=train, valid=valid)
.label_from_lists(train_labels=train_label_paths, valid_labels=valid_label_paths, classes=[1, 2, 3, 4]))
data = (src.transform(get_steel_transforms(config=config), size=config.imsize, tfm_y=True)
.databunch(bs=config.batch_size)
.normalize(imagenet_stats))
return data
async def infer():
path = Path('data/')
learn = load_learner(path)
data = await request.get_data()
img = open_image(BytesIO(data))
pred_class, pred_idx, outputs = learn.predict(img)
return str(pred_class)
def from_folder(cls, path, recurse=True, **kwargs):
"""This function makes a H5pyList from all the datasets in all the files of a given folder"""
items = []
path = Path(path)
fns = get_files(path, recurse=recurse, extensions='.h5py')
cls._append_h5py_items(items, fns)
return cls(items, path=path, **kwargs)
def delete_uploadfiles(upload_path):
for file in Path(upload_path).glob('*'):
try:
file.unlink()
except OSError as e:
print("Error: %s : %s" % (f, e.strerror))
return True
def test(self):
path = Path(
'C:\\Kaggle Downloads\\deepfake-detection-challenge\\output\\decorate_df\\dataframes\\df.pkl'
)
df = pd.read_pickle(path)
df['path'] = df['path'].apply(lambda x: str(x)).astype(str)
df['original_path'] = df['original_path'].apply(
lambda x: str(x)).astype(str)
logger.info(f'DF types: {df.dtypes}')
output_path = Path(
'C:\\Kaggle Downloads\\deepfake-detection-challenge\\output\\decorate_df\\dataframes\\df_cp.pkl'
)
df.to_pickle(output_path)
logger.info(f"Cols: {df.columns}")
def from_file(cls, path, **kwargs):
"""This function makes an H5pyList from all of the datasets in a given file"""
items = []
path = Path(path)
with h5py.File(path, 'r') as f:
for name in f:
items.append((path, name))
return cls(items, path=path.parent, **kwargs)
def get_scans(data_path, patients: List[str] = None):
scans = []
if patients is None:
patients = get_patients(data_path)
for patient in patients:
bc_dir = Path(data_path / patient / BC_NAME)
scans += [str(p) for p in bc_dir.ls()]
return sorted(scans)
def __getitem__(self, idx):
img_path = self.img[idx]
img = Image.open(self.path/img_path)
img = img.convert('L')
label = Path(self.path/img_path).name[:-4]
label_oh = []
for i in label:
label_oh += encode(i)
if self.transform is not None:
img = self.transform(img)
return img, np.array(label_oh), label
def main():
path_data = Path("../input/data/data")
vocab = ['alarm', 'lock', 'movie', 'rain', 'weather']
dim = 224
X = np.array([])
for cat in tqdm(vocab):
path_tmp = path_data / cat
imgs = path_tmp.ls()
cat_imgs = []
cat_imgs_ten = []
for im in imgs:
seq = []
for i in im.ls():
img = np.array(PIL.Image.open(i).resize((dim, dim))) / 255
seq.append(img)
cat_imgs.append(np.array(seq))
cat_imgs = np.array(cat_imgs)
X = np.append(X, cat_imgs)
X_pca = []
pca = PCA(224)
max_idx = 0
X_pca_tmp = []
for seq in tqdm(X):
seq_t = []
for im in seq:
pca.fit(im.reshape(224, 224 * 3))
tmp_idx = np.where(
np.cumsum(pca.explained_variance_ratio_) > 0.98)[0][0]
if max_idx < tmp_idx:
max_idx = tmp_idx
seq_t.append(np.array(pca.singular_values_))
X_pca_tmp.append(np.array(seq_t))
min_seq = min([i.shape[0] for i in X])
pca = PCA(max_idx if max_idx < min_seq else min_seq)
for seq in tqdm(X_pca_tmp):
pca.fit(seq)
X_pca.append(np.array(pca.singular_values_))
X_pca = np.array(X_pca)
Y = np.array([
np.argmax(np.array(pd.get_dummies(vocab).iloc[u // 10]))
for u in range(50)
])
X_train, X_test, y_train, y_test = train_test_split(X_pca,
Y,
test_size=0.30,
shuffle=True)
return X_train, X_test, y_train, y_test
def from_detailed_df(cls, df, path, fn_col, dsn_col,
folder=None, suffix='', **kwargs):
"""This function is called by from_df if dsn_col is passed into it."""
suffix = suffix or ''
path = Path(path)
if not (isinstance(fn_col, int) and isinstance(dsn_col, int)):
raise "Filenames and dataset names must each be in a single column"
fn_inputs = df.iloc[:,fn_col]
ds_inputs = df.iloc[:, dsn_col]
assert fn_inputs.isna().sum().sum() + ds_inputs.isna().sum().sum() == 0, f"You have NaN values in column(s) {[fn_col, dsn_col]} of your dataframe, please fix it."
files = cls.add_prefix_suffix(fn_inputs.values.flatten(), path, folder, suffix)
items = [[f, ds] for f,ds in zip(files, ds_inputs.values.flatten())]
return cls(items, path=path, inner_df=df, **kwargs)
def test_ImageList(self):
# Arrange
# C:/Kaggle Downloads/deepfake-detection-challenge/output/decorate_df/dataframes/df.pkl
image_path = Path(
'C:\\Kaggle Downloads\\deepfake-detection-challenge\\output\\merged\\images'
)
path = Path(
'C:\\Kaggle Downloads\\deepfake-detection-challenge\\output\\decorate_df\\dataframes\\df.pkl'
)
df = pd.read_pickle(path)
logger.info(f'DF: {df.head()}')
df_train = df[df['test_train_split'] == 'train']
df_val = df[df['test_train_split'] == 'validation']
df_test = df[df['test_train_split'] == 'test']
num_fake = df_train[df_train["gross_label"] == "fake"].shape[0]
num_real = df_train[df_train["gross_label"] == "real"].shape[0]
logger.info(f'rat: {num_fake}: {num_real}: ')
return
def from_df(cls, df, path, cols,
dsn_col=None, folder=None, suffix='', **kwargs):
"""Use dsn_col if your df contains an entry for the name of every dataset paired with its corresponding
filename. The filenames must be in a single column. This function does not allow you to use
label_from_df unless you include a dsn_col."""
if dsn_col is not None:
return cls.from_detailed_df(df, path, cols, dsn_col, folder, suffix, **kwargs)
suffix = suffix or ''
path = Path(path)
inputs = df.iloc[:,df_names_to_idx(cols,df)]
assert inputs.isna().sum().sum() == 0, f"You have NaN values in column(s) {cols} of your dataframe, please fix it."
files = cls.add_prefix_suffix(inputs.values.flatten(), path, folder, suffix)
items = []
cls._append_h5py_items(items, files)
return cls(items, path=path, **kwargs)
def upload_file():
if request.method == 'POST':
image = request.files['file']
filename = secure_filename(image.filename)
#saving file in upload path
image.save(Path(app.config["IMAGE_UPLOADS"]+"/"+ filename))
my_dict = {}
#loading images from upload path
img_list_loader = ImageList.from_folder(upload_path)
#Checking if valid images are uploaded
if len(img_list_loader.items)>0:
#loading model
load_model = load_learner(model,
test=img_list_loader)
#running inference
preds,y = load_model.get_preds(ds_type=DatasetType.Test)
index =0
#Processing results for UI
for preds,img_src in zip(preds,img_list_loader.items):
top3_return_msg,top_pred = print_top_3_pred(preds)
if(np.round(preds[top_pred].numpy()*100,2)<threshold):
custom_msg = "NA"
Prediction_percent = "NA"
else:
custom_msg= str(get_label(int(top_pred)))
Prediction_percent = str("{:.2f}%".format(np.round(preds[top_pred].numpy()*100,2)))
temp_val=[]
temp_val.append(img_src)
temp_val.append(custom_msg)
temp_val.append(Prediction_percent)
temp_val.append(top3_return_msg)
my_dict[index]=temp_val
index+=1
return render_template('result.html', mydict=my_dict)
elif len(img_list_loader.items)== 0:
return "ERROR: Invalid image. Go back to upload new image"
class BearClassifier(Singleton):
path = Path('model')
classes = {
'teddys': 'Teddy Bear',
'black': 'Black Bear',
'grizzly': 'Grizzly Bear'
}
def predict(self, bytes):
defaults.device = torch.device('cpu')
learn = load_learner(self.path)
img = open_image(bytes)
pred_class = learn.predict(img)
return self.classes[str(pred_class[0])]
def main(epochs):
Task.init(project_name="examples",
task_name="fastai with tensorboard callback")
path = untar_data(URLs.MNIST_SAMPLE)
data = ImageDataBunch.from_folder(path,
ds_tfms=(rand_pad(2, 28), []),
bs=64,
num_workers=0)
data.normalize(imagenet_stats)
learn = cnn_learner(data, models.resnet18, metrics=accuracy)
tboard_path = Path("data/tensorboard/project1")
learn.callback_fns.append(
partial(LearnerTensorboardWriter, base_dir=tboard_path, name="run0"))
accuracy(*learn.get_preds())
learn.fit_one_cycle(epochs, 0.01)
def resize_json_dir(file_data,
src_path,
dest_path,
number_files='all',
height=800):
"""
Resize src_path directory of JSON files and store in dest_path directory
There needs to be resized jpg files in the dest_path directory.
An example is 226260 - 1|0.436047|221|.jpg where |0.436047|221| marks the factor and offset factors to apply
:param src_path: Source path where json files are
:param dest_path: Destination path to store resized json files
:param number_files: Number of json files to process, leave empty for all files in directory
:return: number of file processed
"""
df = pandas.read_csv(file_data, index_col=0)
__number_files = df.shape[0]
print(
f'Number of JSON files: {__number_files}, Number to resize: {number_files}'
)
Path(dest_path).mkdir(parents=True, exist_ok=True)
if isinstance(number_files, int):
__number_files = number_files
for i in range(__number_files):
f_stem = df.loc[i, 'Name'].split('.')[0]
scale = float(height) / df.loc[i, 'Height']
offset = 0
data = resize_json_file(f'{src_path}/{f_stem}.json',
scale=float(scale),
offset=int(offset))
data['imagePath'] = df.loc[i, 'Name']
with open(f'{dest_path}/{f_stem}.json', 'w') as outfile:
json.dump(data, outfile, ensure_ascii=False, indent=4)
progress_bar(i + 1, 50)
print('')
print(i + 1, ' json files processed')
return i + 1
def __init__(self,
type_pretrained='BERT',
text_cols="comment_text",
list_files=["train.csv", "test.csv"],
label_cols=[
"toxic", "severe_toxic", "obscene", "threat", "insult",
"identity_hate"
],
data_root=Path("..") / "api/app/dataset/jigsaw",
model_dir='model',
batch_size=12):
self.data_root = data_root
self.model_dir = model_dir
self.batch_size = batch_size
self.label_cols = label_cols
self.text_cols = text_cols
self.list_files = list_files
self.type_pretrained = type_pretrained
gc.collect()
log.debug('type_pretrained: ' + type_pretrained)
if self.type_pretrained == 'BERT':
self.tokenizer_pretrained_coruscant = BertTokenizer.from_pretrained(
"bert-base-uncased")
def csv_to_json_dir(src_path, dest_path, number_files='all'):
""" Convert an entire directory of Techion CSV files to JSON files. Store in the dest directory
CSV format is x,y,species_number
1383,1571,2
1687,1822,1
2036,1327,1
"""
print('Converting an entire directory of Techion CSV files to JSON files')
fnames_csv = sorted(get_files(src_path, extensions=['.csv']))
fnames_jpg = sorted(get_files(src_path, extensions=['.jpg']))
if isinstance(number_files, int):
fnames_csv = fnames_csv[:number_files]
fnames_jpg = fnames_jpg[:number_files]
print('Number of csv & jpg files to convert', len(fnames_csv),
len(fnames_jpg))
Path(dest_path).mkdir(parents=True, exist_ok=True)
print(f"src_path {src_path}")
print(f"dest_path {dest_path}")
for i, fn in enumerate(fnames_jpg):
parent = Path(fn).parents[0]
name = Path(fn).name.split('.')[0]
csv_fn = f"{parent}/{name}.csv"
jpg_fn = f"{parent}/{name}.jpg"
json_fn = f"{parent}/{name}.json"
if Path(jpg_fn).is_file():
img = PIL.Image.open(jpg_fn)
data = create_json_from_CSV(Path(csv_fn).name,
Path(jpg_fn).name,
height=img.size[1],
width=img.size[0])
with open(json_fn, 'w') as outfile:
# json.dump(data, outfile, indent=2)
# print(f'Saving File {json_fn}')
json.dump(data, outfile, ensure_ascii=False, indent=4)
progress_bar(i + 1, 50)
# Web
import uvicorn
from starlette.applications import Starlette
from starlette.middleware.cors import CORSMiddleware
from starlette.responses import HTMLResponse, JSONResponse
from starlette.staticfiles import StaticFiles
export_file_url = 'https://github.com/cooperhammond/your-technocrats/releases/download/1.0/technocrats-resnet50-v1.pkl'
export_file_name = 'technocrats-resnet50-v1.pkl'
classes = [
'bill-gates', 'elon-musk', 'jack-ma', 'jeff-bezos', 'larry-ellison',
'larry-page', 'mark-zuckerberg'
]
path = Path(__file__).parent
app = Starlette()
app.add_middleware(CORSMiddleware,
allow_origins=['*'],
allow_headers=['X-Requested-With', 'Content-Type'])
app.mount('/static', StaticFiles(directory='app/static'))
@app.route("/")
async def homepage(request):
html_file = path / 'view' / 'index.html'
return HTMLResponse(html_file.open().read())
if __name__ == '__main__':
def upload():
target = os.path.join(APP_ROOT, 'images/')
print(target)
if not os.path.isdir(target):
os.mkdir(target)
else:
print("Couldn't create upload directory: {}".format(target))
print(request.files.getlist("file"))
for upload in request.files.getlist("file"):
print(upload)
print("{} is the file name".format(upload.filename))
filename = upload.filename
destination = "/".join([target, filename])
print("Accept incoming file:", filename)
print("Save it to:", destination)
upload.save(destination)
segmentaion_model_path = './Segmentation_model'
learn = load_learner(segmentaion_model_path)
image_path = Path(f'./images/{filename}')
img2, img = cv2.imread(destination), open_image(destination)
mask = get_pred(learn, img)
contours = measure.find_contours(mask[:, :, 0], 0.002)
contour1 = []
for i in range(len(contours)):
ele = []
for j in range(len(contours[i])):
y, x = contours[i][j]
ele.append([x, y])
contour1.append(np.array(ele))
poly1 = []
for i in contour1:
c = np.expand_dims(i.astype(np.float32), 1)
# Convert it to UMat obj
c = cv2.UMat(c)
area = cv2.contourArea(c)
if area > 50:
#poly.append(Polygon(i))
poly1.append(i)
df_contour = pd.DataFrame()
_id = []
img_names = []
contour_data = []
for i in range(len(poly1)):
x, y, w, h = cv2.boundingRect(np.int32([poly1[i]]))
ROI = img2[y:y + h + 50, x:x + w + 50]
_id.append(i)
img_names.append(f'IMG_025_0{i}.png')
contour_data.append(poly1[i])
cv2.imwrite(f"./crop_file/IMG_025_0{i}.png", ROI)
df_contour['id'] = _id
df_contour['image_names'] = img_names
df_contour['contour'] = contour_data
classification_model_path = './classification_model'
learn_classi = load_learner(classification_model_path)
tfms = get_transforms(flip_vert=True,
max_rotate=0.2,
max_warp=0.,
max_zoom=1.1,
max_lighting=0.4)
test_path = Path(f'./crop_file')
test_fns = [o for o in sorted(test_path.iterdir()) if '.png' in o.name]
preds = []
pred_classes = []
image_names = []
for fn in tqdm_notebook(test_fns):
try:
img = open_image(fn)
img = img.apply_tfms(tfms[1],
resize_method=ResizeMethod.SQUISH,
padding_mode='zeros')
pred_class, pred_idx, outputs = learn_classi.predict(img)
image_names.append(fn.name)
preds.append(list(to_np(outputs)))
pred_classes.append(str(pred_class))
except Exception as exc:
print(f'{exc}')
preds.append([-1, -1, -1, -1])
pred_classes.append('error')
df_pred_img = pd.DataFrame()
df_pred_img['id'] = range(len(image_names))
df_pred_img['image_names'] = image_names
df_pred_img['predict_classes'] = pred_classes
fig, ax = plt.subplots(figsize=(10, 10))
for i, j in enumerate(image_names):
x = df_pred_img[df_pred_img['image_names'] ==
j]['predict_classes'].values[0]
y = df_contour[df_contour['image_names'] == j]['contour'].values[0]
if x == 'Complete':
final_plot = cv2.drawContours(img2, [y.astype(int)], 0,
(0, 255, 0), 3)
elif x == 'Incomplete':
final_plot = cv2.drawContours(img2, [y.astype(int)], 0,
(255, 0, 0), 3)
elif x == 'Foundation':
final_plot = cv2.drawContours(img2, [y.astype(int)], 0,
(0, 0, 255), 3)
else:
final_plot = cv2.drawContours(img2, [y.astype(int)], 0,
(255, 255, 255), 3)
cv2.imwrite('./images/final_plot.png', final_plot)
return render_template("complete.html", image_name=filename)
def _append_h5py_items(item_list, fns):
for file in fns:
with h5py.File(file, 'r') as f:
for name in f:
item_list.append([Path(file),name])
return item_list
def test_from_mp4(self):
# Arrange
df_batch = batch_data_loader_service.get_all_metadata('c')
image_par_path = config.OUTPUT_VIRGIN_TEST_IMAGES_PAR_PATH
# vid_path = df_batch.iloc[0, :][COL_VID_PATH]
# vid_path = Path('C:\\Kaggle Downloads\\deepfake-detection-challenge\\train\\dfdc_train_part_40\\dkkqtmyitk.mp4')
data_list = []
for ndx, row in df_batch.iterrows():
vid_path = row['vid_path']
results = video_service.process_specific_video_frames(
video_file_path=vid_path,
fnProcess=self.get_random_swatch,
frames=list(range(0, 25)))
vid_stem = vid_path.stem
for r in results:
image_cropped, height_new, width_new, frame_index = r
output_path = Path(image_par_path,
f'{vid_stem}_{frame_index}.png')
if output_path.exists():
output_path.unlink()
image_converted = cv2.cvtColor(image_cropped,
cv2.COLOR_BGR2RGB)
cv2.imwrite(str(output_path), image_converted)
vid_name = vid_path.name
real_or_fake = df_batch[df_batch['candidate_filename'] ==
vid_name]['label'].tolist()[0].lower()
real_or_fake_digit = 1 if real_or_fake == 'fake' else 0
row = {
'vid_path': str(vid_path),
'filename': output_path.name,
'path': str(output_path),
'gross_label': real_or_fake,
'real_or_fake_digit': real_or_fake_digit,
'video_name_stem': vid_path.stem
}
data_list.append(row)
df = pd.DataFrame(data=data_list,
columns=[
'vid_path', 'filename', 'path',
'video_name_stem', 'gross_label',
'real_or_fake_digit'
])
pd.options.display.max_columns = 10
pd.options.display.max_colwidth = 30
pd.options.display.max_rows = 999
logger.info(df.head())
df_f = df[df['gross_label'] == 'fake']
df_r = df[df['gross_label'] == 'real']
num_fakes = df_f.shape[0]
num_reals = df_r.shape[0]
logger.info(f'fakes: {num_fakes}; reals: {num_reals}')
max_num = num_fakes if num_reals > num_fakes else num_reals
df_bal = pd.concat([df_f.iloc[:max_num, :], df_r.iloc[:max_num, :]])
logger.info(f'Total: {df_bal.shape[0]}')
df_path = Path(config.OUTPUT_VIRGIN_TEST_DF_PAR_PATH,
'df_virgin_test.pkl')
df_bal.to_pickle(df_path)
'Pleural Effusion', 'Enlarged Cardiomediastinum', 'Lung Opacity',
'Lung Lesion', 'Pneumonia', 'Pneumothorax', 'Pleural Other', 'Fracture',
'Support Devices'
]
EVAL_LBLS = [
'Cardiomegaly',
'Edema',
'Consolidation',
'Atelectasis',
'Pleural Effusion',
]
chexpert_folder = 'home1/amey/CheXpert-v1.0-downsampled'
folder_path = '/'
model_path = Path('/home/amey/LTTS/fastai/models/')
model_names = {
0: 'fastai-densenet-320-u0-stage-2',
1: 'fastai-densenet-320-u1-stage-1',
2: 'fastai-resnet-320-u0-stage-1',
3: 'fastai-resnet-320-u1-stage-2',
4: 'fastai-resnext-320-u1-stage-1',
5: 'fastai-vgg-320-u0-stage-2',
6: 'fastai-vgg-320-u1-stage-1',
7: 'fastai-densenet-CT-phase2-u1-stage-2'
}
def ensemble_method(outputs, mode='avg'):
for idx in range(1, len(outputs)):
def from_file(cls, fpath, key=None, idxs=None, **kwargs):
fpath = Path(fpath)
file = h5py.File(fpath.name, 'r')
key = ifnone(key, list(file.keys())[0])
items = ifnone(idxs, list(range(len(file[key]))))
return cls(items, path=fpath.parent, file=file, key=key, **kwargs)
def custom_tta(learn:Learner, ds_type:DatasetType=DatasetType.Valid):
dl = learn.dl(ds_type)
ds = dl.dataset
old_open_image = fastai.vision.data.open_image
try:
maxNumberOfCrops = 20
for i in range(maxNumberOfCrops):
#print("starting")
setupNewCrop(i)
yield get_preds(learn.model, dl, activ=_loss_func2activ(learn.loss_func))[0]
finally:
fastai.vision.data.open_image = old_open_image
DATA = Path('/home/josh/git/AudioTagging/data')
WORK = Path('/home/josh/git/AudioTagging/work')
CSV_TRN_MERGED = DATA/'train_merged.csv'
CSV_SUBMISSION = DATA/'sample_submission.csv'
TRN_CURATED = DATA/'train_curated'
TRN_NOISY = DATA/'train_noisy'
IMG_TRN_CURATED = WORK/'image/trn_curated'
IMG_TRN_NOISY = WORK/'image/trn_noisy'
IMG_TEST = WORK/'image/test'
TEST = DATA/'test'
print("Reading training data")
def __init__(self, app, model_filename='seasons.pkl'):
self.path_to_imgs = Path(app.config['UPLOADED_PATH'])
self.path_to_models = Path(app.config['MODELS_PATH'])
self.model_filename = model_filename
self.result = []
from fastai.vision import Path, ImageDataBunch, cnn_learner, get_transforms, imagenet_stats, models, error_rate
import numpy as np
path = Path('/usr/local/airflow/data/')
np.random.seed(42)
data = ImageDataBunch.from_folder(path,
train=".",
valid_pct=0.2,
ds_tfms=get_transforms(),
size=224,
num_workers=0).normalize(imagenet_stats)
learn = cnn_learner(data, models.resnet34, metrics=error_rate)
learn.fit_one_cycle(4)
learn.save('stage-1')
learn.export()
import sys
import pyrealsense2 as rs
import numpy as np
import cv2
import argparse
import os.path
from fastai.vision import Path
if not os.path.exists("data"):
os.makedirs("data")
path_main = Path("./bag_data")
for path_i in path_main.ls():
print("Checking:", path_i)
found = False
while not found:
try:
path_i = path_i.ls()
bags_c = []
count = 0
for b in path_i:
if os.path.splitext(str(b))[1] == ".bag":
bags_c.append(str(b))
count += 1
if count != 0:
found = True
except:
print("No bags files found")
break
print("{} bag files found.".format(count))
import pandas as pd
from fastai.vision import Path, ClassificationInterpretation, models
from fastai.metrics import accuracy_thresh, fbeta
from utils import my_cl_int_plot_top_losses, get_data_augmentation_transforms, get_frequency_batch_transforms, create_cnn
from functools import partial
from audio_databunch import AudioItemList
ClassificationInterpretation.plot_audio_top_losses = my_cl_int_plot_top_losses
DATA = Path('data')
CSV_TRN_CURATED = DATA / 'train_curated.csv'
CSV_TRN_NOISY = DATA / 'train_noisy.csv'
CSV_SUBMISSION = DATA / 'sample_submission.csv'
TRN_CURATED = DATA / 'train_curated'
TRN_NOISY = DATA / 'train_noisy'
TEST = DATA / 'test'
WORK = Path('work')
IMG_TRN_CURATED = WORK / 'image/trn_curated'
IMG_TRN_NOISY = WORK / 'image/trn_curated'
IMG_TEST = WORK / 'image/test'
for folder in [WORK, IMG_TRN_CURATED, IMG_TRN_NOISY, IMG_TEST]:
Path(folder).mkdir(exist_ok=True, parents=True)
df = pd.read_csv(CSV_TRN_CURATED)
df_n = pd.read_csv(CSV_TRN_NOISY)
test_df = pd.read_csv(CSV_SUBMISSION)
n_fft = 512 # output of fft will have shape [513 x n_frames]
n_hop = 94 # width of Spectogram = max_seconds * sample rate / n_hop
n_mels = 128 # Height of spectogram