def get_activity(self, team_id,
filter_user_id=None, filter_project_id=None, filter_job_id=None, filter_actions=None,
progress_cb=None, start_date=None, end_date=None):
from datetime import datetime, timedelta
filters = []
if filter_user_id is not None:
filters.append({"field": ApiField.USER_ID, "operator": "=", "value": filter_user_id})
if filter_project_id is not None:
filters.append({"field": ApiField.PROJECT_ID, "operator": "=", "value": filter_project_id})
if filter_job_id is not None:
filters.append({"field": ApiField.JOB_ID, "operator": "=", "value": filter_job_id})
if filter_actions is not None:
if type(filter_actions) is not list:
raise TypeError(
"type(filter_actions) is {!r}. But has to be of type {!r}".format(type(filter_actions), list))
filters.append({"field": ApiField.TYPE, "operator": "in", "value": filter_actions})
def _add_dt_filter(filters, dt, op):
dt_iso = None
if dt is None:
return
if type(dt) is str:
dt_iso = dt
elif type(dt) is datetime:
dt_iso = dt.isoformat()
else:
raise TypeError('DT type must be string in ISO8601 format or datetime, not {}'.format(type(dt)))
filters.append({"field": ApiField.DATE, "operator": op, "value": dt_iso})
_add_dt_filter(filters, start_date, ">=")
_add_dt_filter(filters, end_date, "<=")
method = 'teams.activity'
data = {ApiField.TEAM_ID: team_id, ApiField.FILTER: filters}
first_response = self._api.post(method, data)
first_response = first_response.json()
total = first_response['total']
per_page = first_response['perPage']
pages_count = first_response['pagesCount']
results = first_response['entities']
if progress_cb is not None:
progress_cb(len(results), total)
if pages_count == 1 and len(first_response['entities']) == total:
pass
else:
for page_idx in range(2, pages_count + 1):
temp_resp = self._api.post(method, {**data, 'page': page_idx, 'per_page': per_page})
temp_items = temp_resp.json()['entities']
results.extend(temp_items)
if progress_cb is not None:
progress_cb(len(results), total)
if len(results) != total:
logger.warn(f"Method '{method}': new events were created during pagination, "
f"downloaded={len(results)}, total={total}")
return results
def add_pair(self, ann_gt, ann_pred):
labels_gt = filter_labels_by_name(ann_gt.labels, self._gt_to_pred_class_mapping)
all_labels_pred = [label for label in filter_labels_by_name(ann_pred.labels, self._pred_to_gt_class_mapping)]
labels_pred = []
for label in all_labels_pred:
label_confidence = self._get_confidence_value(label)
if label_confidence is None:
logger.warn(f'Found a label with class {label.obj_class.name!r} that does not have a '
f'{self._confidence_tag_name!r} tag attached. Skipping this object for metric computation.')
elif label_confidence >= self._confidence_threshold:
labels_pred.append(label)
match_result = match_labels_by_iou(labels_1=labels_gt, labels_2=labels_pred, img_size=ann_gt.img_size,
iou_threshold=self._iou_threshold)
for match in match_result.matches:
gt_class = match.label_1.obj_class.name
label_pred = match.label_2
self._counters[gt_class][MATCHES].append(
MatchWithConfidence(is_correct=(label_pred.obj_class.name == self._gt_to_pred_class_mapping[gt_class]),
confidence=self._get_confidence_value(label_pred)))
# Add unmatched predictions to the list as false positive matches.
for umatched_pred in match_result.unmatched_labels_2:
gt_class = self._pred_to_gt_class_mapping[umatched_pred.obj_class.name]
self._counters[gt_class][MATCHES].append(
MatchWithConfidence(is_correct=False, confidence=self._get_confidence_value(umatched_pred)))
for label_1 in labels_gt:
self._counters[label_1.obj_class.name][TOTAL_GROUND_TRUTH] += 1
def _upload_data_bulk(self,
func_item_to_byte_stream,
items_hashes,
retry_cnt=3,
progress_cb=None,
item_progress=None):
hash_to_items = {i_hash: item for item, i_hash in items_hashes}
unique_hashes = set(hash_to_items.keys())
remote_hashes = set(self.check_existing_hashes(
list(unique_hashes))) # existing -- from server
if progress_cb:
progress_cb(len(remote_hashes))
#pending_hashes = unique_hashes #- remote_hashes #@TODO: only fo debug!
pending_hashes = unique_hashes - remote_hashes
for retry_idx in range(retry_cnt):
# single attempt to upload all data which is not uploaded yet
for hashes in batched(list(pending_hashes)):
pending_hashes_items = [(h, hash_to_items[h]) for h in hashes]
hashes_rcv = self._upload_uniq_videos_single_req(
func_item_to_byte_stream, pending_hashes_items,
item_progress)
pending_hashes -= set(hashes_rcv)
if set(hashes_rcv) - set(hashes):
logger.warn('Hash inconsistency in images bulk upload.',
extra={
'sent': hashes,
'received': hashes_rcv
})
if progress_cb:
progress_cb(len(hashes_rcv))
if not pending_hashes:
return
logger.warn('Unable to upload videos (data).',
extra={
'retry_idx':
retry_idx,
'items':
[(h, hash_to_items[h]) for h in pending_hashes]
})
# now retry it for the case if it is a shadow server/connection error
raise RuntimeError(
"Unable to upload videos (data). "
"Please check if videos are in supported format and if ones aren't corrupted."
)
def get_data_dir():
key = "SLY_APP_DATA_DIR"
dir = None
try:
dir = os.environ[key]
except KeyError as e:
raise KeyError(f"Environment variable {key} is not defined")
if dir_exists(dir) is False:
logger.warn(
f"App data directory {dir} doesn't exist. Will be made automatically."
)
mkdir(dir)
return dir
def crop(self, rect):
'''
Crop the current Polygon with a given rectangle, if polygon cat't be cropped it generate exception error
:param rect: Rectangle class object
:return: list of Poligon class objects
'''
from supervisely.geometry.point_location import PointLocation
try:
# points = [
# PointLocation(row=rect.top, col=rect.left),
# PointLocation(row=rect.top, col=rect.right + 1),
# PointLocation(row=rect.bottom + 1, col=rect.right + 1),
# PointLocation(row=rect.bottom + 1, col=rect.left)
# ]
points = [
PointLocation(row=rect.top, col=rect.left),
PointLocation(row=rect.top, col=rect.right),
PointLocation(row=rect.bottom, col=rect.right),
PointLocation(row=rect.bottom, col=rect.left)
]
#points = rect.corners # old implementation with 1 pixel error (right bottom)
# #@TODO: investigate here (critical issue)
clipping_window_shpl = ShapelyPolygon(points_to_row_col_list(points))
self_shpl = ShapelyPolygon(self.exterior_np, holes=self.interior_np)
intersections_shpl = self_shpl.buffer(0).intersection(clipping_window_shpl)
mapping_shpl = mapping(intersections_shpl)
except Exception:
logger.warn('Polygon cropping exception, shapely.', exc_info=True)
# raise
# if polygon is invalid, just print warning and skip it
# @TODO: need more investigation here
return []
intersections = shapely_figure_to_coords_list(mapping_shpl)
# Check for bad cropping cases (e.g. empty points list)
out_polygons = []
for intersection in intersections:
if isinstance(intersection, list) and len(intersection) > 0 and len(intersection[0]) >= 3:
exterior = row_col_list_to_points(intersection[0], do_round=True)
interiors = []
for interior_contour in intersection[1:]:
if len(interior_contour) > 2:
interiors.append(row_col_list_to_points(interior_contour, do_round=True))
out_polygons.append(Polygon(exterior, interiors))
return out_polygons
def _upload_data_bulk(self, func_item_to_byte_stream, items_hashes, retry_cnt=3, progress_cb=None):
"""
Upload images (binary data) to server. Works with already existing or duplicating images.
:param func_item_to_byte_stream: converter for "item" to byte stream
:param items_hashes: iterable of pairs (item, hash) where "item" is a some descriptor (e.g. image file path)
for image data, and "hash" is a hash for the image binary data
:param retry_cnt: int, number of retries to send the whole set of items
:param progress_cb: callback to account progress (in number of items)
"""
hash_to_items = {i_hash: item for item, i_hash in items_hashes}
unique_hashes = set(hash_to_items.keys())
remote_hashes = set(self.check_existing_hashes(list(unique_hashes))) # existing -- from server
if progress_cb:
progress_cb(len(remote_hashes))
pending_hashes = unique_hashes - remote_hashes
# @TODO: some correlation with sly.io.network_exceptions. Should we perform retries here?
for retry_idx in range(retry_cnt):
# single attempt to upload all data which is not uploaded yet
for hashes in batched(list(pending_hashes)):
pending_hashes_items = [(h, hash_to_items[h]) for h in hashes]
hashes_rcv = self._upload_uniq_images_single_req(func_item_to_byte_stream, pending_hashes_items)
pending_hashes -= set(hashes_rcv)
if set(hashes_rcv) - set(hashes):
logger.warn('Hash inconsistency in images bulk upload.',
extra={'sent': hashes, 'received': hashes_rcv})
if progress_cb:
progress_cb(len(hashes_rcv))
if not pending_hashes:
return
logger.warn('Unable to upload images (data).', extra={
'retry_idx': retry_idx,
'items': [(h, hash_to_items[h]) for h in pending_hashes]
})
# now retry it for the case if it is a shadow server/connection error
raise RuntimeError("Unable to upload images (data). "
"Please check if images are in supported format and if ones aren't corrupted.")
def __init__(self, exterior, interior,
sly_id=None, class_id=None, labeler_login=None, updated_at=None, created_at=None):
'''
:param exterior: list of PointLocation objects, the object contour is defined with these points
:param interior: list of elements that has the same structure like the "exterior" field. This is the list of polygons that define object holes.
'''
if len(exterior) < 3:
exterior.extend([exterior[-1]] * (3 - len(exterior)))
logger.warn('"{}" field must contain at least 3 points to create "Polygon" object.'.format(EXTERIOR))
#raise ValueError('"{}" field must contain at least 3 points to create "Polygon" object.'.format(EXTERIOR))
for element in interior:
if len(element) < 3:
logger.warn('"{}" interior field must contain at least 3 points to create "Polygon" object.'.format(element))
element.extend([element[-1]] * (3 - len(element)))
#if any(len(element) < 3 for element in interior):
# raise ValueError('"{}" element must contain at least 3 points.'.format(INTERIOR))
super().__init__(exterior, interior, sly_id=sly_id, class_id=class_id, labeler_login=labeler_login,
updated_at=updated_at, created_at=created_at)
def clean_dir(dir_: str, ignore_errors=True):
"""
Recursively delete a directory tree, but save root directory.
Args:
dir_: Target directory path.
"""
# old implementation
#shutil.rmtree(dir_, ignore_errors=True)
#mkdir(dir_)
for filename in os.listdir(dir_):
file_path = os.path.join(dir_, filename)
try:
if os.path.isfile(file_path) or os.path.islink(file_path):
os.unlink(file_path)
elif os.path.isdir(file_path):
shutil.rmtree(file_path)
except Exception as e:
logger.warn(f"Failed to delete {file_path}. Reason: {repr(e)}")
if ignore_errors is False:
raise e
def _upload_uniq_images_single_req(self, func_item_to_byte_stream, hashes_items_to_upload):
"""
Upload images (binary data) to server with single request.
Expects unique images that aren't exist at server.
:param func_item_to_byte_stream: converter for "item" to byte stream
:param hashes_items_to_upload: list of pairs (hash, item)
:return: list of hashes for successfully uploaded items
"""
content_dict = {}
for idx, (_, item) in enumerate(hashes_items_to_upload):
content_dict["{}-file".format(idx)] = (str(idx), func_item_to_byte_stream(item), 'image/*')
encoder = MultipartEncoder(fields=content_dict)
resp = self._api.post('images.bulk.upload', encoder)
resp_list = json.loads(resp.text)
remote_hashes = [d['hash'] for d in resp_list if 'hash' in d]
if len(remote_hashes) != len(hashes_items_to_upload):
problem_items = [(hsh, item, resp['errors'])
for (hsh, item), resp in zip(hashes_items_to_upload, resp_list) if resp.get('errors')]
logger.warn('Not all images were uploaded within request.', extra={
'total_cnt': len(hashes_items_to_upload), 'ok_cnt': len(remote_hashes), 'items': problem_items})
return remote_hashes
def _upload_uniq_videos_single_req(self,
func_item_to_byte_stream,
hashes_items_to_upload,
progress_cb=None):
content_dict = {}
for idx, (_, item) in enumerate(hashes_items_to_upload):
content_dict["{}-file".format(idx)] = (
str(idx), func_item_to_byte_stream(item), 'video/*')
encoder = MultipartEncoder(fields=content_dict)
if progress_cb is not None:
def _callback(monitor, progress):
progress(monitor.bytes_read)
callback = partial(_callback, progress=progress_cb)
monitor = MultipartEncoderMonitor(encoder, callback)
resp = self._api.post('videos.bulk.upload', monitor)
else:
resp = self._api.post('videos.bulk.upload', encoder)
resp_list = json.loads(resp.text)
remote_hashes = [d['hash'] for d in resp_list if 'hash' in d]
if len(remote_hashes) != len(hashes_items_to_upload):
problem_items = [
(hsh, item, resp['errors'])
for (hsh, item), resp in zip(hashes_items_to_upload, resp_list)
if resp.get('errors')
]
logger.warn('Not all images were uploaded within request.',
extra={
'total_cnt': len(hashes_items_to_upload),
'ok_cnt': len(remote_hashes),
'items': problem_items
})
return remote_hashes
