def _depr_upload_dataset(self, dataset_name, image_data):
'''
DEPRECATED
Uploads multiple images and groups them into a dataset
@dataset_name: name of the dataset
@image_data: list of pairs of (filename, metadata) input
'''
ds_list = []
for filename, metadata in image_data:
image = self.upload_image(filename=filename, metadata=metadata)
if image is not None:
ds_list.append((image.name, image.uri))
self._debug_print('Dataset ' + dataset_name + ' contans '+ str(len(ds_list)) + ' element(s)')
dataset = BQDataset (name=dataset_name)
dataset.value = [ (uri, 'object') for image_name, uri in ds_list ]
self.session = self._authenticate(self.bqs)
if self.session.save(dataset):
self.logger.info('Dataset ' + dataset_name + ' uploaded successfully!')
self._debug_print('Dataset ' + dataset_name + ' uploaded successfully!')
else:
self.logger.error('Dataset ' + dataset_name + ' failed to upload')
self._debug_print('Dataset ' + dataset_name + ' failed to upload')
def _depr_upload_dataset(self, dataset_name, image_data):
'''
DEPRECATED
Uploads multiple images and groups them into a dataset
@dataset_name: name of the dataset
@image_data: list of pairs of (filename, metadata) input
'''
ds_list = []
for filename, metadata in image_data:
image = self.upload_image(filename=filename, metadata=metadata)
if image is not None:
ds_list.append((image.name, image.uri))
self._debug_print('Dataset ' + dataset_name + ' contans ' +
str(len(ds_list)) + ' element(s)')
dataset = BQDataset(name=dataset_name)
dataset.value = [(uri, 'object') for image_name, uri in ds_list]
self.session = self._authenticate(self.bqs)
if self.session.save(dataset):
self.logger.info('Dataset ' + dataset_name +
' uploaded successfully!')
self._debug_print('Dataset ' + dataset_name +
' uploaded successfully!')
else:
self.logger.error('Dataset ' + dataset_name + ' failed to upload')
self._debug_print('Dataset ' + dataset_name + ' failed to upload')
def upload_dataset(session, dataset_name, uri_list):
"""Save a dataset
Create a data setup from the uploaded list
@param session: a bqsession
@param dataset_name : name the new dataset
@param url_list : array of uris (strings)
@return: None
"""
dataset = BQDataset(name="%s-%s" % (dataset_name, str(uuid.uuid4())[:4]))
# force dataset values to be interpreted as object references
dataset.value = [(url, 'object') for url in uri_list]
if session.args.verbose:
six.print_("DATASET ", BQFactory.to_string(dataset))
if not session.args.dryrun:
session.save(dataset)
def upload_dataset(self, dataset_name, image_data, num_threads=4):
'''
Uploads multiple images and groups them into a dataset via threading
@dataset_name: name of the dataset
@image_data: list of pairs of (filename, metadata) input
'''
pool = multiprocessing.Pool(num_threads)
result = pool.map_async(self._upload_image_helper,
[(image[0], image[1]) for image in image_data])
# OLD Method of async
#async_results = [ pool.apply_async(self.upload_image, (idata[0],idata[1])) for idata in image_data ]
#print async_results
#map(ApplyResult.wait, async_results)
#ds_list = [r.get() for r in async_results]
while not result.ready():
self._debug_print("Chunks Remaining: >={}".format(
result._number_left))
time.sleep(30)
# Build dataset
ds_list = result.get()
pool.close()
self._debug_print('Dataset ' + dataset_name + ' contans ' +
str(len(ds_list)) + ' element(s)')
dataset = BQDataset(name=dataset_name)
dataset.value = [(uri, 'object') for image_name, uri in ds_list]
# Refresh authentication in cases of long uploads
local_session = self._authenticate()
if local_session.save(dataset):
self._debug_print('Dataset ' + dataset_name +
' uploaded successfully!')
else:
self._debug_print('Dataset ' + dataset_name + ' failed to upload',
log="error")
return ds_list
def upload_dataset(self, dataset_name, image_data, num_threads=4):
'''
Uploads multiple images and groups them into a dataset via threading
@dataset_name: name of the dataset
@image_data: list of pairs of (filename, metadata) input
'''
pool = multiprocessing.Pool(num_threads)
result = pool.map_async(self._upload_image_helper, [(image[0], image[1]) for image in image_data] )
# OLD Method of async
#async_results = [ pool.apply_async(self.upload_image, (idata[0],idata[1])) for idata in image_data ]
#print async_results
#map(ApplyResult.wait, async_results)
#ds_list = [r.get() for r in async_results]
while not result.ready():
self._debug_print("Chunks Remaining: >={}".format(result._number_left))
time.sleep(30)
# Build dataset
ds_list = result.get()
pool.close()
self._debug_print('Dataset ' + dataset_name + ' contans '+ str(len(ds_list)) + ' element(s)')
dataset = BQDataset (name=dataset_name)
dataset.value = [ (uri, 'object') for image_name, uri in ds_list ]
# Refresh authentication in cases of long uploads
local_session = self._authenticate()
if local_session.save(dataset):
self._debug_print('Dataset ' + dataset_name + ' uploaded successfully!')
else:
self._debug_print('Dataset ' + dataset_name + ' failed to upload', log="error")
return ds_list
