def post_table(self, params, files):
if files:
bottle.abort(400, 'Table does not support files')
params = dict((base64.b64decode(k), base64.b64decode(v))
for k, v in params.items())
path = params['path']
with thrift_lock() as thrift:
manager = PicarusManager(db=thrift)
if path.startswith('model/'):
return _create_model_from_params(manager, self.owner, path,
params)
elif path.startswith('factory/'):
table = params['table']
start_stop_rows = parse_slices()
data_table = get_table(self._auth_user, table)
for start_row, stop_row in start_stop_rows:
data_table._slice_validate(start_row, stop_row, 'r')
try:
factory = FACTORIES[path]
except KeyError:
bottle.abort(400, 'Invalid parameter value [path]')
slices = [
base64.b64encode(start_row) + ',' +
base64.b64encode(stop_row)
for start_row, stop_row in start_stop_rows
]
job_row = JOBS.add_task('model', self.owner, {
'slices': ';'.join(slices),
'table': self.table,
'path': path
}, {})
return _create_model_from_factory(self.owner, thrift, path,
factory, params,
start_stop_rows,
data_table.table, job_row)
def post_slice(self, start_row, stop_row, params, files):
if files:
bottle.abort(400, 'Table does not support files')
params = dict((k, base64.b64decode(v)) for k, v in params.items())
action = params['action']
with thrift_lock() as thrift:
manager = PicarusManager(db=thrift)
if action == 'io/thumbnail':
self._slice_validate(start_row, stop_row, 'rw')
# Makes 150x150 thumbnails from the data:image column
model = [{
'name': 'picarus.ImagePreprocessor',
'kw': {
'method': 'force_square',
'size': 150,
'compression': 'jpg'
}
}]
job_row = JOBS.add_task(
'process', self.owner, {
'startRow': base64.b64encode(start_row),
'stopRow': base64.b64encode(stop_row),
'table': self.table,
'action': action
}, {})
thrift.takeout_chain_job('images',
model,
'data:image',
'thum:image_150sq',
start_row=start_row,
stop_row=stop_row,
job_row=job_row)
return dict((base64.b64encode(k), base64.b64encode(v))
for k, v in {
'row': job_row,
'table': 'jobs'
}.items())
elif action == 'io/exif':
self._slice_validate(start_row, stop_row, 'rw')
job_row = JOBS.add_task(
'process', self.owner, {
'startRow': base64.b64encode(start_row),
'stopRow': base64.b64encode(stop_row),
'table': self.table,
'action': action
}, {})
thrift.exif_job(start_row=start_row,
stop_row=stop_row,
job_row=job_row)
return dict((base64.b64encode(k), base64.b64encode(v))
for k, v in {
'row': job_row,
'table': 'jobs'
}.items())
elif action == 'io/copy':
self._slice_validate(start_row, stop_row, 'rw')
input_column = params['inputColumn']
output_column = params['outputColumn']
self._column_write_validate(output_column)
job_row = JOBS.add_task(
'process', self.owner, {
'startRow': base64.b64encode(start_row),
'stopRow': base64.b64encode(stop_row),
'inputColumn': base64.b64encode(input_column),
'outputColumn': base64.b64encode(output_column),
'table': self.table,
'action': action
}, {})
thrift.copy_job('images',
input_column=input_column,
output_column=output_column,
start_row=start_row,
stop_row=stop_row,
job_row=job_row)
return dict((base64.b64encode(k), base64.b64encode(v))
for k, v in {
'row': job_row,
'table': 'jobs'
}.items())
elif action == 'io/link':
self._slice_validate(start_row, stop_row, 'rw')
model_key = params['model']
chain_input, model_link = _takeout_input_model_link_from_key(
manager, model_key)
job_row = JOBS.add_task(
'process', self.owner, {
'startRow': base64.b64encode(start_row),
'stopRow': base64.b64encode(stop_row),
'table': self.table,
'action': action
}, {})
thrift.takeout_chain_job('images', [model_link],
chain_input,
model_key,
start_row=start_row,
stop_row=stop_row,
job_row=job_row)
return dict((base64.b64encode(k), base64.b64encode(v))
for k, v in {
'row': job_row,
'table': 'jobs'
}.items())
elif action == 'io/chain':
self._slice_validate(start_row, stop_row, 'rw')
model_key = params['model']
chain_inputs, model_chain = zip(
*_takeout_input_model_chain_from_key(manager, model_key))
job_row = JOBS.add_task(
'process', self.owner, {
'startRow': base64.b64encode(start_row),
'stopRow': base64.b64encode(stop_row),
'table': self.table,
'action': action
}, {})
thrift.takeout_chain_job('images',
list(model_chain),
chain_inputs[0],
model_key,
start_row=start_row,
stop_row=stop_row,
job_row=job_row)
return dict((base64.b64encode(k), base64.b64encode(v))
for k, v in {
'row': job_row,
'table': 'jobs'
}.items())
elif action == 'o/crawl/flickr':
self._slice_validate(start_row, stop_row, 'w')
job_row = JOBS.add_task(
'crawl', self.owner, {
'startRow': base64.b64encode(start_row),
'stopRow': base64.b64encode(stop_row),
'table': self.table,
'action': action
}, {})
thrift.flickr_job(params, start_row, stop_row, job_row)
return dict((base64.b64encode(k), base64.b64encode(v))
for k, v in {
'row': job_row,
'table': 'jobs'
}.items())
elif action == 'o/crawl/streetview':
self._slice_validate(start_row, stop_row, 'w')
job_row = JOBS.add_task(
'crawl', self.owner, {
'startRow': base64.b64encode(start_row),
'stopRow': base64.b64encode(stop_row),
'table': self.table,
'action': action
}, {})
thrift.street_view_job(params, start_row, stop_row, job_row)
return dict((base64.b64encode(k), base64.b64encode(v))
for k, v in {
'row': job_row,
'table': 'jobs'
}.items())
else:
bottle.abort(400, 'Invalid parameter value [action]')
def post_row(self, row, params, files):
if files:
bottle.abort(400, 'Table does not support files')
params = dict((k, base64.b64decode(v)) for k, v in params.items())
action = params['action']
with thrift_lock() as thrift:
manager = PicarusManager(db=thrift)
if action in ('i/link', 'i/chain', 'io/link', 'io/chain'):
model_key = params['model']
write_result = action.startswith('io/')
if write_result:
self._row_validate(row, 'rw')
else:
self._row_validate(row, 'r')
if action.endswith('/link'):
chain_input, model_link = _takeout_input_model_link_from_key(
manager, model_key)
binary_input = thrift.get_column(self.table, row,
chain_input)
model = picarus_takeout.ModelChain(
msgpack.dumps([model_link]))
else:
chain_inputs, model_chain = zip(
*_takeout_input_model_chain_from_key(
manager, model_key))
binary_input = thrift.get_column(self.table, row,
chain_inputs[0])
model = picarus_takeout.ModelChain(
msgpack.dumps(list(model_chain)))
bottle.response.headers["Content-type"] = "application/json"
model_out = model.process_binary(binary_input)
if write_result:
thrift.mutate_row(self.table, row, {model_key: model_out})
return json.dumps(
{base64.b64encode(model_key): base64.b64encode(model_out)})
elif action in ('io/thumbnail', 'i/thumbnail'):
# TODO: Refactor this, it shares code with link/chain
write_result = action.startswith('io/')
if write_result:
self._row_validate(row, 'rw')
else:
self._row_validate(row, 'r')
# Makes 150x150 thumbnails from the data:image column
model_chain = [{
'name': 'picarus.ImagePreprocessor',
'kw': {
'method': 'force_square',
'size': 150,
'compression': 'jpg'
}
}]
model = picarus_takeout.ModelChain(
msgpack.dumps(list(model_chain)))
bottle.response.headers["Content-type"] = "application/json"
model_out = model.process_binary(
thrift.get_column(self.table, row, 'data:image'))
if write_result:
thrift.mutate_row(self.table, row,
{'thum:image_150sq': model_out})
return json.dumps({
base64.b64encode('thum:image_150sq'):
base64.b64encode(model_out)
})
else:
bottle.abort(400, 'Invalid parameter value [action]')
