def train_from_scratch(self, config):
logger.info('Training from scratch....')
logger.info(self)
logger.info('config=\n' + json.dumps(config, indent=2))
n_steps_per_epoch = int(
np.ceil(config['n_examples_for_train'] // config['batch_size']))
n_steps_for_train = config['n_epochs_for_train'] * n_steps_per_epoch
# since we're training from scratch,
# so always remove the existing directory first.
utils.delete_if_exists(config['train_dir'])
if not os.path.exists(config['ckpt_path']):
logger.info(
'Checkpoint file not exists yet, extracting from %s...' %
config['ckpt_tar_path'])
utils.extract_to(config['ckpt_tar_path'], tempfile.gettempdir())
train_op, summary_op = self._build_train_op()
last_loss = slim.learning.train(
train_op,
logdir=config['train_dir'],
master=config['master'],
summary_op=summary_op,
init_fn=self._get_init_fn(config),
log_every_n_steps=config['log_every'],
save_summaries_secs=config['save_summaries_secs'],
number_of_steps=n_steps_for_train)
logger.info('Last loss: %3f' % last_loss)
def train_from_scratch(self, config):
logger.info('Training from scratch...')
logger.info(self)
logger.info('config=\n' + json.dumps(config, indent=2))
n_steps_per_epoch = int(
math.ceil(config['n_examples_for_train'] // config['batch_size']))
n_steps_for_train = config['n_epochs_for_train'] * n_steps_per_epoch
h, w = config['input_size']
stride = config['stride']
label_h, label_w = h // stride, w // stride
n_epochs_per_decay = config['n_epochs_per_decay']
decay_steps = n_epochs_per_decay * n_steps_per_epoch
lr = tf.train.exponential_decay(
self.lr,
global_step=tf.train.get_or_create_global_step(),
decay_rate=config['lr_decay_rate'],
decay_steps=decay_steps,
staircase=True)
train_op, summary_op = self._build_train(config['batch_size'],
(label_h, label_w), lr)
utils.delete_if_exists(config['train_dir'])
last_loss = slim.learning.train(
train_op,
logdir=config['train_dir'],
summary_op=summary_op,
log_every_n_steps=config['log_every'],
save_summaries_secs=config['save_summaries_secs'],
number_of_steps=n_steps_for_train)
logger.info('Last loss: %.3f' % last_loss)
def _reap_old_torrent_files(torrent_cache_path, torrent_max_last_accessed):
"""Delete any torrent files that haven't been accessed recently."""
if not torrent_max_last_accessed:
logging.debug("Reaping old torrent files disabled, skipping...")
return
logging.debug("Preparing to reap old torrent files,"
" torrent_max_last_accessed=%d" % torrent_max_last_accessed)
for fname in os.listdir(torrent_cache_path):
torrent_path = os.path.join(torrent_cache_path, fname)
last_accessed = time.time() - os.path.getatime(torrent_path)
if last_accessed > torrent_max_last_accessed:
logging.debug("Reaping '%s', last_accessed=%d" %
(torrent_path, last_accessed))
utils.delete_if_exists(torrent_path)
def _reap_old_torrent_files(torrent_cache_path, torrent_max_last_accessed):
"""Delete any torrent files that haven't been accessed recently."""
if not torrent_max_last_accessed:
logging.debug("Reaping old torrent files disabled, skipping...")
return
logging.debug("Preparing to reap old torrent files,"
" torrent_max_last_accessed=%d" % torrent_max_last_accessed)
for fname in os.listdir(torrent_cache_path):
torrent_path = os.path.join(torrent_cache_path, fname)
last_accessed = time.time() - os.path.getatime(torrent_path)
if last_accessed > torrent_max_last_accessed:
logging.debug("Reaping '%s', last_accessed=%d" % (
torrent_path, last_accessed))
utils.delete_if_exists(torrent_path)
def _reap_finished_seeds(seed_cache_path):
"""Delete any cached seeds where the seeder process has died."""
logging.debug("Preparing to reap finished seeds")
missing = {}
for fname in os.listdir(seed_cache_path):
seed_path = os.path.join(seed_cache_path, fname)
missing[seed_path] = None
for cmdline in _active_seeder_processes():
for seed_path in missing.keys():
seed_filename = os.path.basename(seed_path)
if seed_filename in cmdline:
del missing[seed_path]
for seed_path in missing:
logging.debug("Reaping cached seed '%s'" % seed_path)
utils.delete_if_exists(seed_path)
def _reap_finished_seeds(seed_cache_path):
"""Delete any cached seeds where the seeder process has died."""
logging.debug("Preparing to reap finished seeds")
missing = {}
for fname in os.listdir(seed_cache_path):
seed_path = os.path.join(seed_cache_path, fname)
missing[seed_path] = None
for cmdline in _active_seeder_processes():
for seed_path in missing.keys():
seed_filename = os.path.basename(seed_path)
if seed_filename in cmdline:
del missing[seed_path]
for seed_path in missing:
logging.debug("Reaping cached seed '%s'" % seed_path)
utils.delete_if_exists(seed_path)
def optimize_image(dest, img_format, ext, broken_file):
if img_format == 'JPEG':
process = subprocess.Popen(
['/usr/bin/jpegoptim', '-p', '--strip-all', dest],
stdout=subprocess.PIPE)
if process.wait() != 0:
create_broken_image(broken_file + ext, dest)
LOGGER.error('JPEG optimisation failed: %s',
process.communicate()[0])
return 4
elif img_format == 'PNG':
process = subprocess.Popen([
'/usr/bin/pngcrush', '-rem', 'gAMA', '-rem', 'alla', '-rem',
'text', dest, dest + '.tmp'
],
stdout=subprocess.PIPE)
if process.wait() != 0:
delete_if_exists(dest + '.tmp')
create_broken_image(broken_file + ext, dest)
LOGGER.error('PNG optimisation (pngcrush) failed: %s',
process.communicate()[0])
return 4
delete_if_exists(dest)
process = subprocess.Popen([
'/usr/bin/optipng', '-o9', '-preserve', '--force', '-out', dest,
dest + '.tmp'
],
stdout=subprocess.PIPE)
if process.wait() != 0:
delete_if_exists(dest + '.tmp')
create_broken_image(broken_file + ext, dest)
LOGGER.error('PNG optimisation (optipng) failed: %s',
process.communicate()[0])
return 4
delete_if_exists(dest + '.tmp')
process = subprocess.Popen(
['/usr/bin/advpng', '--recompress', '--shrink-insane', dest],
stdout=subprocess.PIPE)
if process.wait() != 0:
create_broken_image(broken_file + ext, dest)
LOGGER.error('PNG optimisation (advpng) failed: %s',
process.communicate()[0])
return 4
elif img_format == 'GIF':
process = subprocess.Popen(['/usr/bin/gifsicle', '-O2', '-b', dest],
stdout=subprocess.PIPE)
if process.wait() != 0:
create_broken_image(broken_file + ext, dest)
LOGGER.error('GIF optimisation failed: %s',
process.communicate()[0])
return 4
else:
LOGGER.error('Unexpected error while cropping')
return 5
return 0
def _seed_if_needed(seed_cache_path, tarball_path, torrent_path,
torrent_seed_duration, torrent_seed_chance,
torrent_listen_port_start, torrent_listen_port_end,
torrent_max_seeder_processes_per_host):
seed_filename = os.path.basename(tarball_path)
seed_path = os.path.join(seed_cache_path, seed_filename)
if _should_seed(seed_path, torrent_seed_duration, torrent_seed_chance,
torrent_max_seeder_processes_per_host):
logging.debug("Preparing to seed '%s' for %d secs" %
(seed_path, torrent_seed_duration))
utils._rename(tarball_path, seed_path)
# Daemonize and seed the image
_seed(torrent_path, seed_cache_path, torrent_seed_duration,
torrent_listen_port_start, torrent_listen_port_end)
else:
utils.delete_if_exists(tarball_path)
def _seed_if_needed(seed_cache_path, tarball_path, torrent_path,
torrent_seed_duration, torrent_seed_chance,
torrent_listen_port_start, torrent_listen_port_end,
torrent_max_seeder_processes_per_host):
seed_filename = os.path.basename(tarball_path)
seed_path = os.path.join(seed_cache_path, seed_filename)
if _should_seed(seed_path, torrent_seed_duration, torrent_seed_chance,
torrent_max_seeder_processes_per_host):
logging.debug("Preparing to seed '%s' for %d secs" % (
seed_path, torrent_seed_duration))
utils._rename(tarball_path, seed_path)
# Daemonize and seed the image
_seed(torrent_path, seed_cache_path, torrent_seed_duration,
torrent_listen_port_start, torrent_listen_port_end)
else:
utils.delete_if_exists(tarball_path)
def main(argv=None):
if argv is None:
argv = sys.argv
gearman_workload = sys.stdin.read()
params = json.loads(gearman_workload)
file_hash = params['file_hash']
file_format = params['format']
# Validate inputs
if not is_hex(file_hash):
LOGGER.error('file_hash is not a hexadecimal value')
return 1
if file_format != 'jpg' and file_format != 'png' and file_format != 'gif':
LOGGER.error('file_format is not recognized')
return 1
filename = "%s.%s" % (file_hash, file_format)
delete_if_exists(settings.UPLOADED_FILES_ROOT + filename)
delete_if_exists(settings.USER_FILES_ROOT + filename)
return 0
def main(argv=None):
if argv is None:
argv = sys.argv
gearman_workload = sys.stdin.read()
params = json.loads(gearman_workload)
file_hash = params['file_hash']
file_format = params['format']
# Validate inputs
if not is_hex(file_hash):
logger.error('file_hash is not a hexadecimal value')
return 1
if file_format != 'jpg' and file_format != 'png' and file_format != 'gif':
logger.error('file_format is not recognized')
return 1
filename = "%s.%s" % (file_hash, file_format)
delete_if_exists(settings.UPLOADED_FILES_ROOT + filename)
delete_if_exists(settings.USER_FILES_ROOT + filename)
return 0
def main(argv=None):
if argv is None:
argv = sys.argv
gearman_workload = sys.stdin.read()
params = json.loads(gearman_workload)
photo_hash = params['photo_hash']
photo_format = params['photo_format']
md5_hash = params['md5_hash']
sha256_hash = params['sha256_hash']
# Validate inputs
if photo_hash and not is_hex(photo_hash):
LOGGER.error('photo_hash is not a hexadecimal value')
return 1
if photo_format and photo_format != 'jpg' and photo_format != 'png' and photo_format != 'gif':
LOGGER.error('photo_format is not recognized')
return 1
if md5_hash and not is_hex(md5_hash):
LOGGER.error('md5_hash is not a hexadecimal value')
return 1
if not is_hex(sha256_hash): # mandatory
LOGGER.error('sha256_hash is not a hexadecimal value')
return 1
# Remove old image
if md5_hash:
delete_if_exists(settings.AVATAR_ROOT + md5_hash)
delete_if_exists(settings.AVATAR_ROOT + sha256_hash)
# Delete all resized images
for size in xrange(settings.AVATAR_MIN_SIZE, settings.AVATAR_MAX_SIZE):
size_dir = settings.AVATAR_ROOT + '%s/' % size
if md5_hash:
delete_if_exists(size_dir + md5_hash)
delete_if_exists(size_dir + sha256_hash)
if not photo_hash:
return 0
source_filename = settings.USER_FILES_ROOT + photo_hash + '.' + photo_format
return create_links(source_filename, md5_hash, sha256_hash)
def main(argv=None):
if argv is None:
argv = sys.argv
gearman_workload = sys.stdin.read()
params = json.loads(gearman_workload)
photo_hash = params['photo_hash']
photo_format = params['photo_format']
md5_hash = params['md5_hash']
sha256_hash = params['sha256_hash']
# Validate inputs
if photo_hash and not is_hex(photo_hash):
logger.error('photo_hash is not a hexadecimal value')
return 1
if photo_format and photo_format != 'jpg' and photo_format != 'png' and photo_format != 'gif':
logger.error('photo_format is not recognized')
return 1
if md5_hash and not is_hex(md5_hash):
logger.error('md5_hash is not a hexadecimal value')
return 1
if not is_hex(sha256_hash): # mandatory
logger.error('sha256_hash is not a hexadecimal value')
return 1
# Remove old image
if md5_hash:
delete_if_exists(settings.AVATAR_ROOT + md5_hash)
delete_if_exists(settings.AVATAR_ROOT + sha256_hash)
# Delete all resized images
for size in xrange(settings.AVATAR_MIN_SIZE, settings.AVATAR_MAX_SIZE):
size_dir = settings.AVATAR_ROOT + '%s/' % size
if md5_hash:
delete_if_exists(size_dir + md5_hash)
delete_if_exists(size_dir + sha256_hash)
if not photo_hash:
return 0
source_filename = settings.USER_FILES_ROOT + photo_hash + '.' + photo_format
return create_links(source_filename, md5_hash, sha256_hash)
def optimize_image(dest, img_format, ext, broken_file):
if img_format == 'JPEG':
process = subprocess.Popen(['/usr/bin/jpegoptim', '-p', '--strip-all', dest], stdout=subprocess.PIPE)
if process.wait() != 0:
create_broken_image(broken_file + ext, dest)
LOGGER.error('JPEG optimisation failed: %s', process.communicate()[0])
return 4
elif img_format == 'PNG':
process = subprocess.Popen(['/usr/bin/exiftran', '-ai', dest], stdout=subprocess.PIPE)
if process.wait() != 0:
create_broken_image(broken_file + ext, dest)
LOGGER.error('PNG optimisation (exiftran) failed: %s', process.communicate()[0])
return 4
process = subprocess.Popen(['/usr/bin/pngcrush', '-rem', 'gAMA', '-rem', 'alla', '-rem', 'text', dest, dest + '.tmp'], stdout=subprocess.PIPE)
if process.wait() != 0:
delete_if_exists(dest + '.tmp')
create_broken_image(broken_file + ext, dest)
LOGGER.error('PNG optimisation (pngcrush) failed: %s', process.communicate()[0])
return 4
delete_if_exists(dest)
process = subprocess.Popen(['/usr/bin/optipng', '-o9', '-preserve', '--force', '-out', dest, dest + '.tmp'], stdout=subprocess.PIPE)
if process.wait() != 0:
delete_if_exists(dest + '.tmp')
create_broken_image(broken_file + ext, dest)
LOGGER.error('PNG optimisation (optipng) failed: %s', process.communicate()[0])
return 4
delete_if_exists(dest + '.tmp')
process = subprocess.Popen(['/usr/bin/advpng', '--recompress', '--shrink-insane', dest], stdout=subprocess.PIPE)
if process.wait() != 0:
create_broken_image(broken_file + ext, dest)
LOGGER.error('PNG optimisation (advpng) failed: %s', process.communicate()[0])
return 4
elif img_format == 'GIF':
process = subprocess.Popen(['/usr/bin/gifsicle', '-O2', '-b', dest], stdout=subprocess.PIPE)
if process.wait() != 0:
create_broken_image(broken_file + ext, dest)
LOGGER.error('GIF optimisation failed: %s', process.communicate()[0])
return 4
else:
LOGGER.error('Unexpected error while cropping')
return 5
return 0
def optimize_image(dest, img_format, ext, broken_file):
if 'JPEG' == img_format:
process = subprocess.Popen(['jpegoptim', '-p', '--strip-all', dest], stdout=subprocess.PIPE)
if process.wait() != 0:
create_broken_image(broken_file + ext, dest)
logger.error('JPEG optimisation failed: %s' % process.communicate()[0])
return 4
elif 'PNG' == img_format:
process = subprocess.Popen(['pngcrush', '-rem', 'gAMA', '-rem', 'alla', '-rem', 'text', dest, dest + '.tmp'], stdout=subprocess.PIPE)
if process.wait() != 0:
delete_if_exists(dest + '.tmp')
create_broken_image(broken_file + ext, dest)
logger.error('PNG optimisation (pngcrush) failed: %s' % process.communicate()[0])
return 4
delete_if_exists(dest)
process = subprocess.Popen(['optipng', '-o9', '-preserve', '--force', '-out', dest, dest + '.tmp'], stdout=subprocess.PIPE)
if process.wait() != 0:
delete_if_exists(dest + '.tmp')
create_broken_image(broken_file + ext, dest)
logger.error('PNG optimisation (optipng) failed: %s' % process.communicate()[0])
return 4
delete_if_exists(dest + '.tmp')
process = subprocess.Popen(['advpng', '--recompress', '--shrink-insane', dest], stdout=subprocess.PIPE)
if process.wait() != 0:
create_broken_image(broken_file + ext, dest)
logger.error('PNG optimisation (advpng) failed: %s' % process.communicate()[0])
return 4
elif 'GIF' == img_format:
process = subprocess.Popen(['gifsicle', '-O2', '-b', dest], stdout=subprocess.PIPE)
if process.wait() != 0:
create_broken_image(broken_file + ext, dest)
logger.error('GIF optimisation failed: %s' % process.communicate()[0])
return 4
else:
logger.error('Unexpected error while cropping')
return 5
return 0
def _fit_init(model, batch_provider,
X_train, y_train, X_valid,
y_valid, keep_training, em_train=False):
start_epoch = 0
best_epoch = 0
best_loss = np.inf
best_params = model.get_params()
validate = X_valid is not None and y_valid is not None
epoch_idx = 0
if em_train:
start_cycle = 0
best_cycle = 0
epoch_idx = 1
cycle_idx = 0
train_loss_fn = os.path.join(model.out_dir, 'train_loss.txt')
if not keep_training:
delete_if_exists(train_loss_fn)
if validate:
valid_loss_fn = os.path.join(model.out_dir, 'valid_loss_reg.txt')
valid_loss_2_fn = os.path.join(model.out_dir, 'valid_loss.txt')
if not keep_training:
delete_if_exists(valid_loss_fn)
delete_if_exists(valid_loss_2_fn)
else:
if os.path.exists(valid_loss_fn):
valid_loss_old = np.loadtxt(valid_loss_fn)
best_loss_idx = np.argmin(valid_loss_old[:, -1])
best_loss = valid_loss_old[best_loss_idx, -1]
best_epoch = valid_loss_old[best_loss_idx, epoch_idx]
start_epoch = int(best_epoch) + 1
if em_train:
best_cycle = valid_loss_old[best_loss_idx, cycle_idx]
start_cycle = int(best_cycle) + 1
elif os.path.exists(train_loss_fn):
train_loss_old = np.loadtxt(train_loss_fn)
best_loss_idx = np.argmin(train_loss_old[:, -1])
best_loss = train_loss_old[best_loss_idx, -1]
best_epoch = train_loss_old[best_loss_idx, epoch_idx]
start_epoch = int(best_epoch) + 1
if em_train:
best_cycle = train_loss_old[best_loss_idx, cycle_idx]
start_cycle = int(best_cycle) + 1
train_batch_provider = batch_provider(theano.config.floatX)
train_batch_provider.store_data(X_train, y_train)
if validate:
valid_batch_provider = batch_provider(theano.config.floatX)
valid_batch_provider.store_data(X_valid, y_valid)
else:
valid_batch_provider = None
total_train_instances = np.sum(len(x) for x in X_train)
if em_train:
return (train_batch_provider, valid_batch_provider,
start_epoch, best_loss, best_params,
total_train_instances,
train_loss_fn, valid_loss_fn, valid_loss_2_fn,
validate, best_epoch, start_cycle, best_cycle)
else:
return (train_batch_provider, valid_batch_provider,
start_epoch, best_loss, best_params,
total_train_instances,
train_loss_fn, valid_loss_fn, valid_loss_2_fn,
validate, best_epoch)
def fit(self, X_train, y_train,
X_valid=None, y_valid=None,
n_epochs=100, batch_size=10,
max_epochs_from_best=10,
keep_training=False):
if self.gradient_steps < 1:
seq_length = 1
else:
seq_length = 2 * self.gradient_steps
start_epoch = 0
best_epoch = 0
best_loss = np.inf
best_params = self.get_params()
validate = X_valid is not None and y_valid is not None
train_loss_fn = os.path.join(self.out_dir, 'train_loss.txt')
if not keep_training:
delete_if_exists(train_loss_fn)
if validate:
valid_loss_fn = os.path.join(self.out_dir, 'valid_loss_reg.txt')
valid_loss_2_fn = os.path.join(self.out_dir, 'valid_loss.txt')
if not keep_training:
delete_if_exists(valid_loss_fn)
delete_if_exists(valid_loss_2_fn)
else:
if os.path.exists(valid_loss_fn):
valid_loss_old = np.loadtxt(valid_loss_fn)
best_loss_idx = np.argmin(valid_loss_old[:, 1])
best_loss = valid_loss_old[best_loss_idx, 1]
best_epoch = valid_loss_old[best_loss_idx, 0]
start_epoch = int(best_epoch) + 1
elif os.path.exists(train_loss_fn):
train_loss_old = np.loadtxt(train_loss_fn)
best_loss_idx = np.argmin(train_loss_old[:, 1])
best_loss = train_loss_old[best_loss_idx, 1]
best_epoch = train_loss_old[best_loss_idx, 0]
start_epoch = int(best_epoch) + 1
else:
named_valid_results = ()
train_batch_provider = BatchProvider(theano.config.floatX)
train_batch_provider.store_data(X_train, y_train)
valid_batch_provider = BatchProvider(theano.config.floatX)
valid_batch_provider.store_data(X_valid, y_valid)
total_train_instances = np.sum(len(x) for x in X_train)
n_train_batches_per_epoch = max(
1, 2 * total_train_instances / (batch_size * seq_length))
LOGGER.info('Batch size: {}; Batches per epoch: {}'
.format(batch_size, n_train_batches_per_epoch))
# variables to hold data batches; reusing rather than recreating the
# arrays saves (a little bit of) time
X_t = train_batch_provider.make_X_batch_array(batch_size, seq_length)
y_t = train_batch_provider.make_Y_batch_array(batch_size, seq_length)
if isinstance(self.train_fun, (tuple, list)):
train_mode_selector = ParameterUpdate(start_epoch, n_epochs)
else:
train_mode_selector = SimpleParameterUpdate()
# Initialize valid loss (in case there is no validation set)
valid_loss = np.array([0 for o in self.valid_fun.outputs])
try:
for epoch in xrange(start_epoch,
n_epochs):
# train_results = []
mode = train_mode_selector.select_mode(epoch)
LOGGER.info('Training {0} params'.format(mode))
train_loss, named_train_results = _train_loss(
self, train_batch_provider, batch_size, seq_length,
X_t, y_t, train_loss_fn, epoch,
n_train_batches_per_epoch, mode=mode)
if validate and (
np.mod(epoch - start_epoch, 5) == 0 or
epoch == n_epochs - 1):
valid_results = []
for i, (X_v, y_v) in enumerate(
valid_batch_provider.iter_pieces()):
valid_results.append(self.valid_fun(X_v, y_v))
valid_loss = np.nanmean(valid_results, axis=0)
write_append(
valid_loss_fn, epoch, valid_loss[0])
write_append(
valid_loss_2_fn, epoch, valid_loss[-1])
named_valid_results = zip([o.variable.name for o in
self.valid_fun.outputs],
valid_loss)
LOGGER.info(
("Epoch: {0}/{3}, "
"train: {1}, "
"validate: {2} ")
.format(epoch,
'; '.join(
'{0} ={1: .3f}'.format(k, v) for k, v in
named_train_results),
'; '.join(
'{0} ={1: .3f}'.format(k, v) for k, v in
named_valid_results),
n_epochs))
params = self.get_params()
# Early stopping
if validate:
es_loss = valid_loss[0]
else:
es_loss = train_loss[0]
if es_loss < best_loss:
best_params = params
best_loss = es_loss
best_epoch = epoch
# Make a backup every 100 epochs (Astrud is sometimes
# unreliable)
if np.mod(epoch - start_epoch, 100) == 0:
LOGGER.info('Backing parameters up!')
save_pyc_bz(best_params,
os.path.join(self.out_dir, 'backup_params.pyc.bz'))
early_stop = (
epoch > (best_epoch + max_epochs_from_best))
if early_stop:
break
except KeyboardInterrupt:
print('Training interrupted')
if best_loss < np.inf:
print('Reloading best self (epoch = {0}, {2} loss = {1:.3f})'
.format(best_epoch + 1, best_loss,
'validation' if validate else 'training'))
self.set_params(best_params)
return self.get_params()
def create_broken_image(broken, dest):
delete_if_exists(dest)
os.symlink(broken, dest)
def main(argv=None):
if argv is None:
argv = sys.argv
gearman_workload = sys.stdin.read()
params = json.loads(gearman_workload)
file_hash = params['file_hash']
file_format = params['format']
links = params['links']
# Validate inputs
if not is_hex(file_hash):
LOGGER.error('file_hash is not a hexadecimal value')
return 1
if file_format != 'jpg' and file_format != 'png' and file_format != 'gif':
LOGGER.error('file_format is not recognized')
return 1
if not isinstance(links, list):
LOGGER.error('links is not a list')
return 1
for l in links:
if not is_hash_pair(l):
LOGGER.error('links is not a list of hash pairs')
return 1
filename = "%s.%s" % (file_hash, file_format)
source = settings.READY_FILES_ROOT + filename
dest = settings.USER_FILES_ROOT + filename
# Sanity checks
if os.path.isfile(dest):
LOGGER.warning('Destination already exists')
return 0
if not os.path.isfile(source):
LOGGER.error('Source file not found')
return 1
# Change file ownership so that it can no longer be modified
try:
os.chown(source, 0, 0) # root:root
except:
LOGGER.error('Cannot change file ownership')
return 3
# Remove from /ready and move to /user
try:
shutil.move(source, dest)
except:
LOGGER.error('Cannot move file')
return 2
# All done, we can delete the original file as uploaded by the user
uploaded_file = settings.UPLOADED_FILES_ROOT + filename
delete_if_exists(uploaded_file)
# Finally, create any links to email hashes that were requested
gm_client = gearman.GearmanClient(settings.GEARMAN_SERVERS)
for hashes in links:
params = {'photo_hash': file_hash, 'photo_format': file_format,
'md5_hash': hashes[0], 'sha256_hash': hashes[1]}
gm_client.submit_job('changephoto', json.dumps(params),
background=True, wait_until_complete=False)
return 0
def main(argv=None):
if argv is None:
argv = sys.argv
gearman_workload = sys.stdin.read()
params = json.loads(gearman_workload)
file_hash = params['file_hash']
file_format = params['format']
links = params['links']
# Validate inputs
if not is_hex(file_hash):
logger.error('file_hash is not a hexadecimal value')
return 1
if file_format != 'jpg' and file_format != 'png' and file_format != 'gif':
logger.error('file_format is not recognized')
return 1
if not isinstance(links, list):
logger.error('links is not a list')
return 1
for l in links:
if not is_hash_pair(l):
logger.error('links is not a list of hash pairs')
return 1
filename = "%s.%s" % (file_hash, file_format)
source = settings.READY_FILES_ROOT + filename
dest = settings.USER_FILES_ROOT + filename
# Sanity checks
if os.path.isfile(dest):
logger.warning('Destination already exists')
return 0
if not os.path.isfile(source):
logger.error('Source file not found')
return 1
# Remove from /ready and move to /user
try:
shutil.move(source, dest)
except:
logger.error('Cannot move file')
return 2
# All done, we can delete the original file as uploaded by the user
uploaded_file = settings.UPLOADED_FILES_ROOT + filename
delete_if_exists(uploaded_file)
# Finally, create any links to email hashes that were requested
gm_client = libgearman.Client()
for server in settings.GEARMAN_SERVERS:
gm_client.add_server(server)
for hashes in links:
params = {'photo_hash': file_hash, 'photo_format': file_format,
'md5_hash': hashes[0], 'sha256_hash': hashes[1]}
gm_client.do_background('changephoto', json.dumps(params))
return 0
def main(argv=None):
if argv is None:
argv = sys.argv
gearman_workload = sys.stdin.read()
params = json.loads(gearman_workload)
file_hash = params['file_hash']
file_format = params['format']
links = params['links']
# Validate inputs
if not is_hex(file_hash):
LOGGER.error('file_hash is not a hexadecimal value')
return 1
if file_format != 'jpg' and file_format != 'png' and file_format != 'gif':
LOGGER.error('file_format is not recognized')
return 1
if not isinstance(links, list):
LOGGER.error('links is not a list')
return 1
for l in links:
if not is_hash_pair(l):
LOGGER.error('links is not a list of hash pairs')
return 1
filename = "%s.%s" % (file_hash, file_format)
source = settings.READY_FILES_ROOT + filename
dest = settings.USER_FILES_ROOT + filename
# Sanity checks
if os.path.isfile(dest):
LOGGER.warning('Destination already exists')
return 0
if not os.path.isfile(source):
LOGGER.error('Source file not found')
return 1
# Change file ownership so that it can no longer be modified
try:
os.chown(source, 0, 0) # root:root
except:
LOGGER.error('Cannot change file ownership')
return 3
# Remove from /ready and move to /user
try:
shutil.move(source, dest)
except:
LOGGER.error('Cannot move file')
return 2
# All done, we can delete the original file as uploaded by the user
uploaded_file = settings.UPLOADED_FILES_ROOT + filename
delete_if_exists(uploaded_file)
# Finally, create any links to email hashes that were requested
gm_client = gearman.GearmanClient(settings.GEARMAN_SERVERS)
for hashes in links:
params = {
'photo_hash': file_hash,
'photo_format': file_format,
'md5_hash': hashes[0],
'sha256_hash': hashes[1]
}
gm_client.submit_job('changephoto',
json.dumps(params),
background=True,
wait_until_complete=False)
return 0
def create_broken_image(broken, dest):
delete_if_exists(dest)
os.symlink(broken, dest)
def main(argv=None):
if argv is None:
argv = sys.argv
gearman_workload = sys.stdin.read()
params = json.loads(gearman_workload)
file_hash = params['file_hash']
file_format = params['format']
links = params['links']
# Validate inputs
if not is_hex(file_hash):
logger.error('file_hash is not a hexadecimal value')
return 1
if file_format != 'jpg' and file_format != 'png' and file_format != 'gif':
logger.error('file_format is not recognized')
return 1
if not isinstance(links, list):
logger.error('links is not a list')
return 1
for l in links:
if not is_hash_pair(l):
logger.error('links is not a list of hash pairs')
return 1
filename = "%s.%s" % (file_hash, file_format)
source = settings.READY_FILES_ROOT + filename
dest = settings.USER_FILES_ROOT + filename
# Sanity checks
if os.path.isfile(dest):
logger.warning('Destination already exists')
return 0
if not os.path.isfile(source):
logger.error('Source file not found')
return 1
# Remove from /ready and move to /user
try:
shutil.move(source, dest)
except:
logger.error('Cannot move file')
return 2
# All done, we can delete the original file as uploaded by the user
uploaded_file = settings.UPLOADED_FILES_ROOT + filename
delete_if_exists(uploaded_file)
# Finally, create any links to email hashes that were requested
gm_client = libgearman.Client()
for server in settings.GEARMAN_SERVERS:
gm_client.add_server(server)
for hashes in links:
params = {
'photo_hash': file_hash,
'photo_format': file_format,
'md5_hash': hashes[0],
'sha256_hash': hashes[1]
}
gm_client.do_background('changephoto', json.dumps(params))
return 0
