def read_dataset(data_dir):
pickle_filename = "celebA.pickle"
pickle_filepath = os.path.join(data_dir, pickle_filename)
if not os.path.exists(pickle_filepath):
utils.maybe_download_and_extract(data_dir, DATA_URL, is_zipfile=True)
celebA_folder = os.path.splitext(DATA_URL.split("/")[-1])[0]
dir_path = os.path.join(data_dir, celebA_folder)
if not os.path.exists(dir_path):
print("CelebA dataset needs to be downloaded and unzipped manually")
print("Download from: %s" % DATA_URL)
raise ValueError("Dataset not found")
result = create_image_lists(dir_path)
print ("Training set: %d" % len(result['train']))
print ("Test set: %d" % len(result['test']))
print ("Validation set: %d" % len(result['validation']))
print ("Pickling ...")
with open(pickle_filepath, 'wb') as f:
pickle.dump(result, f, pickle.HIGHEST_PROTOCOL)
else:
print ("Found pickle file!")
with open(pickle_filepath, 'rb') as f:
result = pickle.load(f)
celebA = CelebA_Dataset(result)
del result
return celebA
def prepare(data_dir):
maybe_download_and_extract(dest_directory=DATA_DIR, data_url=DATA_URL)
directory = os.path.join(DATA_DIR, 'cifar-10-batches-py')
files = ['batches.meta',
'data_batch_1', 'data_batch_2', 'data_batch_3', 'data_batch_4',
'data_batch_5', 'test_batch']
files = [os.path.abspath(os.path.join(directory, f))
for f in files]
for f in files:
if f.endswith("meta"):
unpickled = _unpickle(f)
# unpickled['num_vis'] = 3 * 32**2
meta['num_cases_per_batch'] = unpickled['num_cases_per_batch']
meta['label_names'] = unpickled['label_names']
meta['labelname2index'] = {}
for index, labelname in enumerate(meta['label_names']):
meta['labelname2index'][labelname] = index
elif f.endswith("test_batch"):
unpickled = _unpickle(f)
tu = unpickled['data'], unpickled['labels'], unpickled['filenames']
for img, label, filename in zip(tu[0], tu[1], tu[2]):
img = img.reshape((3, 32, 32)).transpose((1, 2, 0)).reshape(-1)
test_data.append({'img': img,
'label': label,
'filename': filename})
else:
# ['data', 'labels', 'batch_label', 'filenames']
unpickled = _unpickle(f)
tu = unpickled['data'], unpickled['labels'], unpickled['filenames']
for img, label, filename in zip(tu[0], tu[1], tu[2]):
img = img.reshape((3, 32, 32)).transpose((1, 2, 0)).reshape(-1)
train_data.append({'img': img,
'label': label,
'filename': filename})
meta['examples_per_epoch'] = len(train_data)
def inputs(eval_data):
"""Construct input for ShapeOverlap evaluation using the Reader ops.
Args:
eval_data: bool, indicating if one should use the train or eval data set.
Returns:
images: Images. 4D tensor of [batch_size, IMAGE_SIZE, IMAGE_SIZE, 6] size.
labels: Labels. 1D tensor of [batch_size] size.
Raises:
ValueError: If no data_dir
"""
maybe_download_and_extract(FLAGS.data_dir, FLAGS.DATA_URL)
with tf.variable_scope('READ'):
if not FLAGS.data_dir:
raise ValueError('Please supply a data_dir')
locks, keys, labels = overlap_input.inputs(eval_data=eval_data,
data_dir=FLAGS.data_dir,
batch_size=FLAGS.batch_size)
if FLAGS.use_fp16:
locks = tf.cast(locks, tf.float16)
keys = tf.cast(keys, tf.float16)
labels = tf.cast(labels, tf.float16)
return locks, keys, labels
def read_dataset(data_dir):
pickle_filename = "MITSceneParsing.pickle"
pickle_filepath = os.path.join(data_dir, pickle_filename)
if not os.path.exists(pickle_filepath):
utils.maybe_download_and_extract(data_dir, DATA_URL, is_zipfile=True)
SceneParsing_folder = os.path.splitext(DATA_URL.split("/")[-1])[0]
result = create_image_lists(os.path.join(data_dir,
SceneParsing_folder))
print("Pickling ...")
with open(pickle_filepath, 'wb') as f:
pickle.dump(result, f, pickle.HIGHEST_PROTOCOL)
else:
print("Found pickle file!")
with open(pickle_filepath, 'rb') as f:
result = pickle.load(f)
training_records = result['training']
validation_records = result['validation']
del result
return training_records, validation_records
'--limit',
help=
'The sum of the images in the original folder and the augmentations.',
type=int,
default=8000)
parser.add_argument('--gpus', help='Which GPUs to use', default='0,1,2,3')
parser.add_argument('--threads',
help='How many threads to use pr GPU',
default=1,
type=int)
parser.add_argument('--model_dir',
help='Path to Inception files',
default='/tmp/imagenet')
args = parser.parse_args()
maybe_download_and_extract(args.model_dir)
q = mp.Queue()
processes = 0
gpus = args.gpus.split(',')
for gpu in gpus:
for _ in range(args.threads):
if processes == len(args.source):
break
mp.Process(target=augment_images,
args=(q, gpu, args.target, args.limit,
.9 / args.threads, args.model_dir)).start()
processes += 1
for h5 in args.source:
parser.add_argument(
'--gpus',
help='Which GPUs to use',
default='0,1,2,3')
parser.add_argument(
'--threads',
help='How many threads to use pr GPU',
default=1,
type=int)
parser.add_argument(
'--model_dir',
help='Path to Inception files',
default='/tmp/imagenet')
args = parser.parse_args()
maybe_download_and_extract(args.model_dir)
q = mp.Queue()
processes = 0
gpus = args.gpus.split(',')
for gpu in gpus:
for _ in range(args.threads):
if processes == len(args.source):
break
mp.Process(target=augment_images, args=(q, gpu, args.target, args.limit,
.9/args.threads, args.model_dir)).start()
processes += 1
for h5 in args.source:
q.put(h5)
run_bot = True
PATH = "/home/jpeel/PycharmProjects/DreamBot/commented.txt"
regexes = "!dreambot(?:1[0-8]|[1-9])?(?:x[2-3])?", "!dbhowto"
combined = re.compile('|'.join('(?:{0})'.format(x) for x in regexes))
model_fn = "tensorflow_inception_graph.pb"
inception_download_url = "http://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip"
save_folder = "./renderedImages/"
graph = tf.Graph()
sess = tf.InteractiveSession(graph=graph)
with tf.gfile.FastGFile(model_fn, 'rb') as f:
###
utils.maybe_download_and_extract(inception_download_url, ".")
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
t_input = tf.placeholder(np.float32, name='input')
#default 117.0
imagenet_mean = 117.0
t_preprocessed = tf.expand_dims(t_input - imagenet_mean, 0)
tf.import_graph_def(graph_def, {'input': t_preprocessed})
##Not neeeded, but useful for getting available layer names#
layers = [
op.name for op in graph.get_operations()
if op.type == 'Conv2D' and 'import/' in op.name
]
print(layers)
def main(_):
ps_hosts = FLAGS.ps_hosts.split(",")
worker_hosts = FLAGS.worker_hosts.split(",")
# Create a cluster from the parameter server and worker hosts.
cluster = tf.train.ClusterSpec({"ps": ps_hosts, "worker": worker_hosts})
if FLAGS.job_name == "ps":
ps_config = tf.ConfigProto(
gpu_options=tf.GPUOptions(
per_process_gpu_memory_fraction=0.00001
))
# Create and start a server for the local task.
server = tf.train.Server(cluster,
# protocol = "grpc_rdma",
job_name=FLAGS.job_name,
task_index=FLAGS.task_index,
config = ps_config)
server.join()
elif FLAGS.job_name == "worker":
maybe_download_and_extract(FLAGS.data_dir, FLAGS.data_url)
cifar.modify_flags(FLAGS)
print (FLAGS.data_dir)
# Create and start a server for the local task.
server = tf.train.Server(cluster,
# protocol = "grpc_rdma",
job_name=FLAGS.job_name,
task_index=FLAGS.task_index)
local_worker_device = "/job:worker/task:%d" % FLAGS.task_index
with tf.device(tf.train.replica_device_setter(
ps_device='/job:ps/cpu:0',
worker_device=local_worker_device,
cluster=cluster)):
if FLAGS.network == 'fc':
from models.fullyconnect import KitModel
elif FLAGS.network == 'cifar':
from models.cifar import KitModel
elif FLAGS.network == 'alexnet':
from models.alexnet import KitModel
elif FLAGS.network == 'vgg19' or FLAGS.network == 'vgg_e':
from models.vgg19 import KitModel
elif FLAGS.network == 'inception_v3' :
from models.inception_v3 import KitModel
elif FLAGS.network == 'resnet':
from models.resnet import KitModel
else:
sys.exit("Invalid network [%s]" % FLAGS.network)
this_model = KitModel(FLAGS)
images, labels = cifar.distorted_inputs(FLAGS)
logits = this_model.inference(images)
loss = this_model.loss(labels)
train_op = this_model.train()
train_dir = tempfile.mkdtemp()
sess_config = tf.ConfigProto(
allow_soft_placement=True,
log_device_placement=False,
device_filters=["/job:ps", "/job:worker/task:%d" % FLAGS.task_index],
graph_options=tf.GraphOptions(
optimizer_options=tf.OptimizerOptions(opt_level=tf.OptimizerOptions.L1)
),
gpu_options=tf.GPUOptions(
visible_device_list=""
)
)
if FLAGS.infer_shapes == True:
sess_config.graph_options.infer_shapes = FLAGS.infer_shapes
sv = tf.train.Supervisor(
is_chief = (FLAGS.task_index == 0),
logdir = train_dir,
init_op = tf.global_variables_initializer(),
global_step = this_model.global_step,
summary_writer = None,
saver = None)
if FLAGS.task_index == 0:
print("Worker %d: Initializing session..." % FLAGS.task_index)
else:
print("Worker %d: Waiting for session to be initialized..." % FLAGS.task_index)
sess = sv.prepare_or_wait_for_session(server.target, config = sess_config, start_standard_services = True)
print_model()
print ("Start warmup for %d mini-batch." % FLAGS.warmup)
for _ in range(FLAGS.warmup):
sess.run(this_model.train_op)
current_step = 0
current_epoch = 1
duration = 0
FLAGS.epoch = FLAGS.epoch * NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN / FLAGS.batch_size
print ("Start Training for %d mini-batch." % FLAGS.epoch)
while current_step < FLAGS.epoch:
current_step += 1
start_time = time.time()
_, step_loss = sess.run([this_model.train_op, this_model.cost])
end_time = time.time()
# print("Finish step %d, loss = %f, speed = %f sampes/s, duration = %f seconds" % (current_step, step_loss, FLAGS.batch_size / (end_time - start_time), end_time - start_time))
duration += end_time - start_time
print("Time: %f seconds, step_loss: %f" % (duration, step_loss))
if current_step * FLAGS.batch_size > current_epoch * NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN:
print ("Finish epoch %d" % (current_epoch))
current_epoch += 1
print ("Total Time = %f s." % duration)
#writer.close()
else:
sys.exit("Invalid job role name [%s]!" % FLAGS.job_name)