from __future__ import print_function, division, absolute_import
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.utils.model_zoo as model_zoo
import os
import sys
from util.logs import get_logger
from . import MODEL_REGISTRY
__all__ = ['InceptionV4', 'inceptionv4']
logger = get_logger('inception')
pretrained_settings = {
'inceptionv4': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/inceptionv4-8e4777a0.pth',
'input_space': 'RGB',
'input_size': [3, 299, 299],
'input_range': [0, 1],
'mean': [0.5, 0.5, 0.5],
'std': [0.5, 0.5, 0.5],
'num_classes': 1000
},
'imagenet+background': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/inceptionv4-8e4777a0.pth',
'input_space': 'RGB',
'input_size': [3, 299, 299],
'input_range': [0, 1],
'mean': [0.5, 0.5, 0.5],
'std': [0.5, 0.5, 0.5],
import torch
from torch import nn
from util.logs import get_logger
import numpy as np
from . import MODEL_REGISTRY
logger = get_logger('dark loss')
def conv_batch(in_num, out_num, kernel_size=3, padding=1, stride=1):
return nn.Sequential(
nn.Conv2d(in_num,
out_num,
kernel_size=kernel_size,
stride=stride,
padding=padding,
bias=False), nn.BatchNorm2d(out_num), nn.LeakyReLU())
# Residual block
class DarkResidualBlock(nn.Module):
def __init__(self, in_channels):
super(DarkResidualBlock, self).__init__()
reduced_channels = int(in_channels / 2)
self.layer1 = conv_batch(in_channels,
reduced_channels,
kernel_size=1,
padding=0)
self.layer2 = conv_batch(reduced_channels, in_channels)
def forward(self, x):
import numpy as np
from itertools import product
import math
from util.logs import get_logger
logger = get_logger('safas')
class ImageSlider:
@staticmethod
def calculate_start_p(stride, size):
num_step = math.ceil(size / stride)
stride = size / num_step
for i in range(num_step + 1):
yield math.floor(stride * i)
def __init__(self, image_shape=None, width=512, overlapping=0.2):
self.image_shape = image_shape[:2]
self.width = width
self.overlapping = overlapping
self.top_left_points = []
self.stride = (1 - self.overlapping) * self.width
def calculate_start_p(stride, size):
num_step = math.ceil(size / stride)
stride = size / num_step
for i in range(num_step + 1):
yield math.floor(stride * i)
import torch.nn as nn
import torch.nn.functional
import matplotlib.pyplot as plt
from torch.optim import SGD
from torch.utils.data import Dataset, DataLoader
from tqdm import tqdm
from util.logs import get_logger
from voc2012 import VOC
from tensorboardX import SummaryWriter
from util.npdraw import draw_bounding_box
from functools import reduce
import random
import matplotlib.pyplot as plt
logger = get_logger('f**k me')
class VGG(nn.Module):
def __init__(self, init_weights=True):
super(VGG, self).__init__()
self.features = self.make_layers(
[
64,
64,
'M', # 3, 5, 6
128,
128,
'M', # 10, 14 16
256,
256,
import yaml
from util.logs import get_logger
from abc import ABC, abstractmethod
from io import StringIO
from collections.abc import Mapping
from abc import ABC, abstractmethod
import argparse
import json
try:
import yaml
except ModuleNotFoundError as e:
yaml = None
logger = get_logger('config')
class Field(ABC):
def _get_readable_value(self):
raise Exception('not implemented')
def _get_bin_value(self):
raise Exception('not implemented')
def _set_value(self, value):
raise Exception('not implemented')
def _to_dict(self):
return self._get_readable_value()
from PIL import Image
from torch.utils.data import Dataset, DataLoader
from torch.optim import SGD
from tqdm import tqdm
from tensorboardX import SummaryWriter
from torchvision.utils import make_grid
from augment import FundusAOICrop, CompostImageAndLabel
from model import mean_iou, Mrcnn
from util.files import assert_exist, check_exist
from util.logs import get_logger
from util.npdraw import draw_bounding_box
from util.segmentation2bbox import segmentation2bbox
from model import restore_box_reg
import matplotlib.pyplot as plt
logger = get_logger('ma detection')
logWriter = SummaryWriter(logdir=f'log/f**k')
debug = os.getenv('DEBUG')
class VGG(nn.Module):
def __init__(self, init_weights=True):
super(VGG, self).__init__()
self.features = self.make_layers(
[64, 64, 'M', # 3, 5, 6
128, 128, 'M', # 10, 14 16
256, 256, 512, 'M', # 24, 32, 40, 44
512, 512, 512, 'M', # 60, 76, 92, 100
# 512, 512, 512, 'M',
], # 132, 164, 196, 212
import random
from collections.abc import Iterable
from typing import Callable
import traceback
import base64
from os.path import join as pjoin
import pickle
import io
try:
import pymongo
import gridfs
except ImportError as e:
pymongo = None
logger = get_logger('Processing')
class TaskFunc():
def init_env(self):
pass
def __call__(self, *arg, **kwargs):
pass
class ProcessPool(object):
class DelayedResult:
def __init__(self, task_id, parent):
self.task_id = task_id
self.parent = parent
data[k] = v
except Exception as e:
logger.info(e, exc_info=True)
logger.info(f'passing image succ!')
elif type(v) in [dict, list, tuple]:
process_data(v)
# config = DCLCONFIG.build()
# config.from_yaml('config_files/yolo_test.yaml')
# config.parse_args()
# pp = Predictor(config)
# pp.init_env()
app = Flask(__name__)
logger = get_logger('server logger')
# redis_host = 'localhost'
# app.config.from_object(FlaskConfig())
# logger.info('f**k')
@app.route("/")
def f**k():
return 'f**k you !'
@app.route("/api/grade", methods=['POST'])
def grade_one():
data = request.get_data(as_text=True)
data = json.loads(data)
logger.info(data)
import torch.nn as nn
import torch
from torch.nn import Parameter
import torch.nn.init as init
import math
from util.logs import get_logger
from .inceptionv4 import InceptionV4
from .resnet import ResNet18, ResNet101
from torchvision.models.utils import load_state_dict_from_url
from . import MODEL_REGISTRY
logger = get_logger('experiment')
@MODEL_REGISTRY.register()
class ExperimentModel(nn.Module):
def __init__(self, num_classes=10):
super(ExperimentModel, self).__init__()
# self.features = nn.Sequential(
# nn.Conv2d(1, 20, kernel_size=5, stride=1),
# nn.ReLU(inplace=True),
# nn.MaxPool2d(kernel_size=2, stride=2),
# nn.Conv2d(20, 50, kernel_size=5),
# nn.ReLU(inplace=True),
# nn.MaxPool2d(kernel_size=2, stride=2),
# )
####################################
# self.features = InceptionV4()
# state_dict = load_state_dict_from_url(
import datetime
import psycopg2
import math
import base64
from hashlib import md5
import uuid
from peewee import fn, DoesNotExist
import time
from elasticsearch import Elasticsearch
from playhouse.shortcuts import model_to_dict
from playhouse.postgres_ext import Match
import six
import math
import zerorpc
logger = get_logger('search project learning')
app = Flask(__name__, static_folder='statics', static_url_path='/static')
api = Api(app)
app.config['MAX_CONTENT_LENGTH'] = 256 * 1024 * 1024
es = Elasticsearch(['es01:9200'])
psql_db = DatabaseProxy()
def json_encoder_default(obj):
datetime_format = "%Y/%m/%d %H:%M:%S"
date_format = "%Y/%m/%d"
time_format = "%H:%M:%S"
# if isinstance(obj, Decimal):
# return str(obj)
if isinstance(obj, datetime.datetime):
import click
from main import ImageStorage, psql_db
import playhouse.db_url
import os
from util.logs import get_logger
logger = get_logger('dataset log')
psql_db.initialize(
playhouse.db_url.connect(
'postgresql://*****:*****@localhost:25068/fuckdb'))
@click.command()
@click.option('--path', help='path of images')
@click.option('--session_name', help='session name')
def add_image(path, session_name):
for p, _, f in os.walk(path):
for fname in f:
fname = os.path.join(p, fname)
ImageStorage.add_file(fname, session_name)
logger.info(fname)
if __name__ == "__main__":
add_image()
import PIL
import pandas as pd
import cv2
import numpy as np
import pickle
from torch.utils.data import Dataset
from main import ImageAnnotation, psql_db, ImageStorage
import playhouse.db_url
from util.logs import get_logger
import torch
import datetime
import random
logger = get_logger('f**k sql db')
class SqlDB(Dataset):
def __init__(self,
split='train',
db_url='postgresql://*****:*****@localhost:25068/fuckdb',
table_name='imageannotation',
max_box=2):
self.db_url = db_url
self.max_box = max_box
def _init_env(self):
if not hasattr(self, 'images'):
if psql_db.is_closed():
psql_db.initialize(playhouse.db_url.connect(self.db_url))
result = ImageAnnotation.select().where(
(ImageAnnotation.session_name=='disk_set') and
class Session(BaseModel):
session_name = TextField(default=lambda: uuid.uuid4().hex, unique=True)
timestamp = DateTimeField(default=datetime.datetime.now)
try:
psql_db.initialize(
playhouse.db_url.connect('postgresql://*****:*****@db:5432/fuckdb'))
psql_db.connect()
psql_db.create_tables([ImageStorage, ImageAnnotation, Session])
except Exception as e:
print(e)
logger = get_logger('interactive learning')
def image2base64(f):
# check if there are any image, parse image to base64
def wrapper(*args, **kwargs):
result = f(*args, **kwargs)
if type(result) is not dict:
if type(result) is np.ndarray:
result = {'data': result}
else:
logger.warn(f'cannot parse {type(result)} type.')
for k, v in result.items():
if type(v) is np.ndarray:
if np.issubdtype(v.dtype, np.floating):
v = (v * 255).astype(np.uint8)
# from skimage.filters import gaussian
import numpy as np
import random
import cv2
from util.logs import get_logger
from typing import Callable
from util.process_pool import run_once
import traceback
import inspect
import pickle
import os.path
import zlib
# import torchvision
logger = get_logger('augment.logger')
class Transform(ABC):
counter = 0
def __init__(self):
self.id = Transform.counter
Transform.counter += 1
def __call__(self, *img):
"""
this is the function that calls the actual transformation function.
I added this layer for easy managing, such as caching, logging etc.
DO NOT override this function.
:param img: input image
from util.logs import get_logger
from torch.utils.data import DataLoader
from dataset.lesion_seg_mask_rcnn import LesionSegMask
from os import cpu_count
from torch.utils.data import Dataset
from util.augment import Compose, ToFloat, ToTensor
from model.maskrcnn import MaskRCNN
from torch.optim import Adam, SGD
from sqlitedict import SqliteDict
from io import BytesIO
from tqdm import tqdm
import torch
import os
from tensorboardX import SummaryWriter
logger = get_logger('main')
num_processor = 2
device = torch.device('cuda')
summery_writer = SummaryWriter(logdir=f'log/maskrcnn/log')
def wtire_summary(loss_map, tag='train', step=None):
for k, v in loss_map.items():
summery_writer.add_scalar(f'{tag}/{k}_loss',
v.detach().cpu().numpy(),
global_step=step)
class TrainEvalDataset(Dataset):
def __init__(self, data_reader, split='train'):
super().__init__()
from util.augment import augment_map, Compose
from util.logs import get_logger
from scipy.special import softmax
import sys
from dataset import datasets as avaliable_datasets
import argparse
from sklearn.metrics import average_precision_score, roc_auc_score
import util.bconfig
import model
from model.deeplab_v3 import CrossEntropy2d
import pickle
from itertools import chain
from model.experiment import ExperimentLoss
from os import cpu_count
logger = get_logger('fff')
device = torch.device('cuda')
num_processor = cpu_count()
class NetModel(nn.Module):
def __init__(self, config, with_feature=False):
super().__init__()
if config.net_parameters is None:
config.net_parameters = {}
self.base_net = config.basenet(**config.net_parameters)
self.config = config
self.with_feature = with_feature
for i in self.config.outputs:
logger.info(f'{i.layer_parameters}')
layer = i.model(**i.layer_parameters)
import unittest
import numpy as np
from util.process_pool import CachedFunction
from util.image_process import ImageSlider
from util.logs import get_logger
import matplotlib.pyplot as plt
import hashlib
logger = get_logger('f**k')
class TestCachedFunc(unittest.TestCase):
def test_int_single_key(self):
@CachedFunction(cache_dir='/tmp/test/cache')
def sqrt(x):
return x**2
for i in range(100):
self.assertEqual(sqrt(i), i**2)
for i in range(100):
self.assertEqual(sqrt(i), i**2)
self.assertEqual('f**k', 'f**k')
def test_cached_func_file_input(self):
np.random.seed(5153424)
bytes_arr = [np.random.bytes(1024 * 325) for i in range(100)]
md5_arr = []
for i in bytes_arr:
md5 = hashlib.md5()
md5.update(i)
from torch.utils.data import DataLoader, Dataset
import pandas as pd
import numpy as np
from PIL import Image
from torch.utils.data import Dataset
# import nvidia.dali.ops as ops
# import nvidia.dali.types as types
# from nvidia.dali.pipeline import Pipeline
from torch.utils.data._utils.collate import default_collate
from util.logs import get_logger
import torch
# from nvidia.dali.plugin.pytorch import DALIGenericIterator
from tqdm import tqdm
logger = get_logger('annotate')
# class ExternalInputIterator(object):
# def __init__(
# self, batch_size, split='train', test_split=0,
# needed_labels=[]):
# self.split = 'train' if split is None else split
# self.root = '../data/annotation4'
# files = pd.read_csv(f'{self.root}/dataset.csv')
# if self.split == 'train':
# self.image_list = files[files.split != test_split]
# else:
# self.image_list = files[files.split == test_split]
# self.batch_size = batch_size