def load_data(data_path, category, split):
mkdir(data_path, clean=False)
data_path = os.path.join(data_path, '{0}-{1}.pkl'.format(category, split))
if not os.path.exists(data_path):
local_path = os.path.join('/tmp/', '{0}.npz'.format(category))
remote_path = os.path.join(
'https://storage.googleapis.com/quickdraw_dataset/sketchrnn/',
'{0}.full.npz'.format(category))
os.system('wget {0} -O {1}'.format(remote_path, local_path))
# load
data = np.load(local_path, encoding='latin1')[split]
data = truncate_data(data)
data = rescale_data(data)
# save
pickle.dump(data, open(data_path, 'wb'))
return pickle.load(open(data_path, 'rb'))
#!/usr/bin/python
'''Script de pruebas
Este Script sirve para comprobar que las librarias funcionan bien.
'''
import os.path, sys
PATH = os.path.realpath('metaconf')
if PATH not in sys.path:
sys.path.append(PATH)
from utils import debug, shell
debug.echo_debug('Empieza el script')
shell.mkdir('/tmp/etc')
shell.mkdir('/tmp/etc/X11')
from hardware import kmodules, disks, xconf
for i in kmodules.get_pci_modules():
print i,
print '\n'
X = xconf.Xconf()
print 'X Driver: %s' % X.get_card()
kmodules.make_modules('/tmp')
disks.make_fstab('/tmp')
for i in disks.get_cdroms():
def analyze_reprs(max_dims=16, threshold=.5, bound=8., step=.2):
reprs_path = os.path.join('exp', args.exp, 'reprs')
mkdir(reprs_path, clean=True)
images_path = os.path.join(reprs_path, 'images')
mkdir(images_path, clean=True)
# statistics
x, ym, yv = [], [], []
for k in range(means.shape[1]):
x.extend([k, k])
ym.extend([np.min(means[:, k]), np.max(means[:, k])])
yv.extend([np.min(log_vars[:, k]), np.max(log_vars[:, k])])
plt.figure()
plt.bar(x, ym, .5, color='b')
plt.xlabel('dimension')
plt.ylabel('mean')
plt.savefig(os.path.join(images_path, 'means.png'), bbox_inches='tight')
plt.figure()
plt.bar(x, yv, .5, color='b')
plt.xlabel('dimension')
plt.ylabel('log(var)')
plt.savefig(os.path.join(images_path, 'vars.png'), bbox_inches='tight')
# dimensions
values = np.arange(-bound, bound + step, step)
magnitudes = np.max(np.abs(means), axis=0)
indices = np.argsort(-magnitudes)
dimensions = [k for k in indices if magnitudes[k] > threshold][:max_dims]
print('==> dominated dimensions = {0}'.format(dimensions))
for split in ['train', 'test']:
inputs, targets = iter(loaders[split]).next()
inputs, targets = to_var(inputs, volatile=True), to_var(targets,
volatile=True)
outputs, z = model.forward(inputs, returns='z')
for dim in tqdm(dimensions):
repr = to_np(z).copy()
samples = []
for val in tqdm(values, leave=False):
repr[:, dim] = val
sample = model.forward(inputs, z=to_var(repr, volatile=True))
samples.append(visualize(inputs, sample))
for k in range(args.batch):
images = [sample[k] for sample in samples]
image_path = os.path.join(
images_path, '{0}-{1}-{2}.gif'.format(split, k, dim))
save_images(images,
image_path,
duration=.1,
channel_first=True)
inputs = visualize(inputs)
for k in range(args.batch):
image_path = os.path.join(images_path,
'{0}-{1}.png'.format(split, k))
save_image(inputs[k], image_path, channel_first=True)
# visualization
with open(os.path.join(reprs_path, 'index.html'), 'w') as fp:
print('<h3>statistics</h3>', file=fp)
print('<img src="{0}">'.format(os.path.join('images', 'means.png')),
file=fp)
print('<img src="{0}">'.format(os.path.join('images', 'vars.png')),
file=fp)
print('<h3>inputs</h3>', file=fp)
print('<table border="1" style="table-layout: fixed;">', file=fp)
for split in ['train', 'test']:
print('<tr>', file=fp)
for k in range(args.batch):
image_path = os.path.join('images',
'{0}-{1}.png'.format(split, k))
print(
'<td halign="center" style="word-wrap: break-word;" valign="top">',
file=fp)
print(
'<img src="{0}" style="width:128px;">'.format(image_path),
file=fp)
print('</td>', file=fp)
print('</tr>', file=fp)
print('</table>', file=fp)
for dim in dimensions:
print('<h3>dimension [{0}]</h3>'.format(dim), file=fp)
print('<table border="1" style="table-layout: fixed;">', file=fp)
for split in ['train', 'test']:
print('<tr>', file=fp)
for k in range(args.batch):
image_path = os.path.join(
'images', '{0}-{1}-{2}.gif'.format(split, k, dim))
print(
'<td halign="center" style="word-wrap: break-word;" valign="top">',
file=fp)
print('<img src="{0}" style="width:128px;">'.format(
image_path),
file=fp)
print('</td>', file=fp)
print('</tr>', file=fp)
print('</table>', file=fp)
def analyze_fmaps(size=256):
fmaps_path = os.path.join('exp', args.exp, 'fmaps')
mkdir(fmaps_path, clean=True)
images_path = os.path.join(fmaps_path, 'images')
mkdir(images_path, clean=True)
# feature maps
for split in ['train', 'test']:
inputs, targets = iter(loaders[split]).next()
inputs, targets = to_var(inputs, volatile=True), to_var(targets,
volatile=True)
outputs, features = model.forward(inputs, returns='features')
num_scales, num_channels = len(features), features[0].size(1)
for s in trange(num_scales):
input, feature = inputs[0][-1], features[s]
for b in trange(args.batch, leave=False):
image = resize_image(to_np(input[b]),
size=size,
channel_first=True)
for c in trange(num_channels, leave=False):
fmap = resize_image(to_np(feature[b, c]),
size=size,
channel_first=True)
if np.min(fmap) < np.max(fmap):
fmap = (fmap - np.min(fmap)) / (np.max(fmap) -
np.min(fmap))
image_path = os.path.join(
images_path,
'{0}-{1}-{2}-{3}.gif'.format(split, s, c, b))
save_images([image, fmap], image_path, channel_first=True)
# visualization
with open(os.path.join(fmaps_path, 'index.html'), 'w') as fp:
for s in range(num_scales):
for c in range(num_channels):
print('<h3>scale [{0}] - channel [{1}]</h3>'.format(
s + 1, c + 1),
file=fp)
print('<table border="1" style="table-layout: fixed;">',
file=fp)
for split in ['train', 'test']:
print('<tr>', file=fp)
for b in range(args.batch):
image_path = os.path.join(
'images',
'{0}-{1}-{2}-{3}.gif'.format(split, s, c, b))
print(
'<td halign="center" style="word-wrap: break-word;" valign="top">',
file=fp)
print('<img src="{0}" style="width:128px;">'.format(
image_path),
file=fp)
print('</td>', file=fp)
print('</tr>', file=fp)
print('</table>', file=fp)
loaders[split] = DataLoader(data[split],
batch_size=args.batch,
shuffle=True,
num_workers=args.workers)
print('==> dataset loaded')
print('[size] = {0} + {1}'.format(len(data['train']), len(data['test'])))
# model
model = VDNet().cuda()
# optimizer
optimizer = torch.optim.Adam(model.parameters(), lr=args.learning_rate)
# experiment path
exp_path = os.path.join('exp', args.exp)
mkdir(exp_path, clean=False)
# logger
logger = Logger(exp_path)
# load snapshot
if args.resume is not None:
epoch, args.beta = load_snapshot(args.resume,
model=model,
optimizer=optimizer,
returns=['epoch', 'beta'])
print('==> snapshot "{0}" loaded (with beta = {1})'.format(
args.resume, args.beta))
else:
epoch = 0
# arguments
args = parser.parse_args()
print('==> arguments parsed')
for key in vars(args):
print('[{0}] = {1}'.format(key, getattr(args, key)))
# cuda devices
set_cuda_devices(args.gpu)
# model
model = SketchVAE().cuda()
# demo path
demo_path = os.path.join('exp', args.exp, 'demo')
mkdir(demo_path, clean=True)
# load snapshot
if os.path.isfile(args.resume):
epoch = load_snapshot(args.resume, model=model, returns='epoch')
print('==> snapshot "{0}" loaded (epoch {1})'.format(
args.resume, epoch))
else:
raise FileNotFoundError('no snapshot found at "{0}"'.format(
args.resume))
model.train(False)
for k in trange(args.samples // args.batch):
outputs = model.forward(batch_size=args.batch)
outputs = outputs.data.cpu().numpy()
def make_xf86config(root = '/'):
'''make_xf86config(root = '/')
Crea el archivo /etc/X11/XF86Config-4 a partir de la clase
Xconf.
Admite el parámetro "root" que indica el directorio
padre del sistema donde se quieren configurar las X.
'''
template = '''
Section "Files"
FontPath "unix/:7100" # local font server
# if the local font server has problems, we can fall back on these
FontPath "/usr/lib/X11/fonts/Type1"
FontPath "/usr/lib/X11/fonts/CID"
FontPath "/usr/lib/X11/fonts/Speedo"
FontPath "/usr/lib/X11/fonts/misc"
FontPath "/usr/lib/X11/fonts/cyrillic"
FontPath "/usr/lib/X11/fonts/100dpi"
FontPath "/usr/lib/X11/fonts/75dpi"
EndSection
Section "Module"
Load "GLcore"
Load "bitmap"
Load "dbe"
Load "ddc"
Load "dri"
Load "extmod"
Load "glx"
Load "int10"
Load "record"
Load "speedo"
Load "type1"
Load "vbe"
Load "xtt"
EndSection
Section "InputDevice"
Identifier "Generic Keyboard"
Driver "keyboard"
Option "CoreKeyboard"
Option "XkbRules" "xfree86"
Option "XkbModel" "pc105"
Option "XkbLayout" "es"
EndSection
Section "ServerFlags"
Option "AllowMouseOpenFail" "true"
EndSection
Section "InputDevice"
Identifier "Generic Mouse Serial"
Driver "mouse"
Option "CorePointer"
Option "Device" "/dev/ttyS0"
Option "Protocol" "Microsoft"
Option "Emulate3Buttons" "true"
Option "ZAxisMapping" "4 5"
EndSection
Section "InputDevice"
Identifier "Generic Mouse PS/2"
Driver "mouse"
Option "Protocol" "ImPS/2"
Option "Device" "/dev/psaux"
Option "Emulate3Buttons" "true"
Option "Emulate3Timeout" "70"
Option "ZAxisMapping" "4 5"
Option "SendCoreEvents" "true"
EndSection
Section "InputDevice"
Identifier "Generic Mouse USB"
Driver "mouse"
Option "SendCoreEvents" "true"
Option "Device" "/dev/input/mice"
Option "Protocol" "ImPS/2"
Option "Emulate3Buttons" "true"
Option "ZAxisMapping" "4 5"
EndSection
Section "Device"
Identifier "Generic Video Card"
Driver "@@DRIVER@@"
BusID "@@BUSID@@"
EndSection
Section "Monitor"
Identifier "Generic Monitor"
HorizSync @@HSYNC@@
VertRefresh @@VSYNC@@
EndSection
Section "Screen"
Identifier "Default Screen"
Device "Generic Video Card"
Monitor "Generic Monitor"
DefaultDepth 16
SubSection "Display"
Depth 1
Modes "1024x768" "800x600" "640x480"
EndSubSection
SubSection "Display"
Depth 4
Modes "1024x768" "800x600" "640x480"
EndSubSection
SubSection "Display"
Depth 8
Modes "1024x768" "800x600" "640x480"
EndSubSection
SubSection "Display"
Depth 15
Modes "1024x768" "800x600" "640x480"
EndSubSection
SubSection "Display"
Depth 16
Modes "1024x768" "800x600" "640x480"
EndSubSection
SubSection "Display"
Depth 24
Modes "1024x768" "800x600" "640x480"
EndSubSection
EndSection
Section "ServerLayout"
Identifier "Default Layout"
Screen "Default Screen"
InputDevice "Generic Keyboard"
InputDevice "Generic Mouse Serial"
InputDevice "Generic Mouse PS/2"
InputDevice "Generic Mouse USB"
EndSection
Section "DRI"
Mode 0666
EndSection
'''
from utils.debug import echo_debug
from utils.shell import joinpath, mkdir
# Se crea un objeto Xconf
X = Xconf()
# Se sustituyen los valores
newx = template.replace('@@DRIVER@@',X.get_card())
newx = newx.replace('@@BUSID@@',X.get_busid())
newx = newx.replace('@@HSYNC@@',X.get_hsync())
newx = newx.replace('@@VSYNC@@',X.get_vsync())
# Se crea el archivo con los nuevos valores
try:
dirname = joinpath(root, 'etc/X11/')
mkdir(dirname)
filename = dirname + 'XF86Config-4'
xf86config = open(filename, 'w')
except IOError:
echo_debug('''No se ha podido crear el archivo XF86Config-4 en
%s/etc/X11/, compruebe que existen los directorios.''' % root \
, 'ERROR')
import sys
sys.exit(1)
xf86config.write(newx)
xf86config.close()