def on_connection_error(self, unused_connection, excpt): if self._connection_tries < 5: log.warning('Fail to connect - {}. Wait 5 seconds'.format(repr(excpt))) time.sleep(5) self._connection_tries += 1 self.connect() self._connection.ioloop.start() else: log.error('Fail to connect to broker') exit(1)
def get_dataset(app_token, dataset_id): try: log.info(f"Get dataset {dataset_id}") endpoint = jwt.decode(app_token, options={"verify_signature": False})['endpoint'] msg_headers = {'Authorization': f'Bearer {app_token}'} response = requests.get(f"{endpoint}/dataset/{dataset_id}", headers=msg_headers) if response.status_code != 200: log.error(f"Failing get dataset: {response.json()}") return None dataset = response.json() if dataset["dataset_parms"]: return dataset else: log.warning(f"Failing get dataset: {response.json()}") return None except requests.ConnectionError as error: log.error( f'Failing get dataset_id: {dataset_id}. Connection error: {error}') return None except requests.Timeout as error: log.error(f'Failing get dataset_id: {dataset_id}. Timeout: {error}') return None except Exception as excp: log.error(f'Failing get dataset_id: {dataset_id} - {excp}') return None
def get_flow(app_token, flow_id): try: log.info(f"Get flow {flow_id}") if not Path(CONFIG["flow_folder"]).is_dir(): Path(CONFIG["flow_folder"]).mkdir(parents=True, exist_ok=True) local_cache = os.path.join(CONFIG["flow_folder"], flow_id + '.json') endpoint = jwt.decode(app_token, options={"verify_signature": False})['endpoint'] msg_headers = {'Authorization': f'Bearer {app_token}'} response = requests.get(f"{endpoint}/flow/{flow_id}", headers=msg_headers) if response.status_code != 200: log.error(f"Failing get flow from edge: {response.json()}") if os.path.isfile(local_cache): with open(local_cache) as fp: flow = json.load(fp) return flow return None flow = response.json()["flow"] if "_id" in flow: if os.path.isfile(local_cache): os.remove(local_cache) with open(local_cache, 'w') as fp: json.dump(flow, fp, default=str) return flow else: log.warning(f"Failing get flow: {response.json()}") return None except requests.ConnectionError as error: log.error(f'Failing get flow_id: {flow_id}. Connection error: {error}') return None except requests.Timeout as error: log.error(f'Failing get flow_id: {flow_id}. Timeout: {error}') return None except Exception as excp: log.error(f'Failing get flow_id: {flow_id} - {excp}') return None
def get_train(app_token, dataset_id, train_id, train_folder): try: log.info(f"Get train {dataset_id}-{train_id}") folder_path = Path(train_folder + '/' + dataset_id + '/' + train_id) if not folder_path.is_dir(): folder_path.mkdir(parents=True, exist_ok=True) endpoint = jwt.decode(app_token, options={"verify_signature": False})['endpoint'] url = f"{endpoint}/model-hist/{dataset_id}/{train_id}" msg_headers = {'Authorization': f'Bearer {app_token}'} payload = {"download_url": True} response = requests.get(url, headers=msg_headers, params=payload) if response.status_code != 200: log.error(f"Failing get train: {response.json()}") return None train_doc = response.json() dest_filename = os.path.join(str(folder_path), train_id + ".tar.gz") download_file(train_doc["download_url"], dest_filename) # expand_file call(["tar", "-xf", dest_filename, "--directory", str(folder_path)]) os.remove(dest_filename) return train_id except requests.ConnectionError as error: log.error( f'Failing get train_id: {train_id}. Connection error: {error}') return None except requests.Timeout as error: log.error(f'Failing get train_id: {train_id}. Timeout: {error}') return None except Exception as excp: log.error(f'Failing get train_id: {train_id} - {excp}') return None
def get_edge_data(app_token): try: log.info(f"Get edge_data") endpoint = jwt.decode(app_token, options={"verify_signature": False})['endpoint'] msg_headers = {'Authorization': f'Bearer {app_token}'} response = requests.get(f"{endpoint}", headers=msg_headers) if response.status_code != 200: log.error(f"Failing get edge_data: {response.json()}") return None return response.json()["edge_data"] except requests.ConnectionError as error: log.error(f'Failing get edge_data. Connection error: {error}') return None except requests.Timeout as error: log.error(f'Failing get edge_data. Timeout: {error}') return None except Exception as excp: log.error(f'Failing get edge_data - {excp}') return None
def upload_video(app_token, video_id, video_file_annotation, video_file, output_file, video_folder): try: endpoint = jwt.decode(app_token, options={"verify_signature": False})['endpoint'] msg_headers = {'Authorization': f'Bearer {app_token}'} url = f"{endpoint}/video/{video_id}" video_filename = os.path.join(video_folder, video_file) output_filename = os.path.join(video_folder, output_file) files = { 'video': open(video_filename, 'rb'), 'output': open(output_filename, 'rb') } values = { "annotations": json.dumps(video_file_annotation, default=str) } response = requests.post(url, files=files, data=values, headers=msg_headers) if response.status_code != 201: raise Exception(f"Failing upload video: {response.json()}") return video_id except requests.ConnectionError as error: log.error( f'Failing uploading video: {video_file}-{video_id}. Connection error: {error}' ) return None except requests.Timeout as error: log.error( f'Failing uploading video: {video_file}-{video_id}. Timeout: {error}' ) return None except Exception as excp: log.error(f'Failing uploading video: {video_file}-{video_id} - {excp}') return None
def insert_train_event(app_token, event): try: endpoint = jwt.decode(app_token, options={"verify_signature": False})['endpoint'] msg_headers = {'Authorization': f'Bearer {app_token}'} url = f"{endpoint}/train/event" data = {"event": json.dumps(event, default=str)} response = requests.post(url, data=data, headers=msg_headers) if response.status_code != 201: raise Exception(f"Failing insert event: {response.json()}") return True except requests.ConnectionError as error: log.error(f'Failing inserting train event. Connection error: {error}') return None except requests.Timeout as error: log.error(f'Failing inserting train event. Timeout: {error}') return None except Exception as excp: log.error(f'Failing inserting train event - {excp}') return None
def get_video(app_token, video_id, video_folder): local_doc = None try: folder_path = Path(video_folder) if not folder_path.is_dir(): folder_path.mkdir(parents=True, exist_ok=True) local_cache = os.path.join(video_folder, video_id + '.json') if os.path.isfile(local_cache): with open(local_cache) as fp: local_doc = json.load(fp) endpoint = jwt.decode(app_token, options={"verify_signature": False})['endpoint'] url = f"{endpoint}/video/{video_id}" msg_headers = {'Authorization': f'Bearer {app_token}'} payload = {"download_url": False} response = requests.get(url, headers=msg_headers, params=payload) if response.status_code != 200: if local_doc: return local_doc log.error(f"Failing get video: {response.json()}") return None video_doc = response.json() if local_doc and video_doc["date"] == local_doc["date"]: return local_doc payload = {"download_url": True} response = requests.get(url, headers=msg_headers, params=payload) if response.status_code != 200: if local_doc: return local_doc log.error(f"Failing get model: {response.json()}") return None video_doc = response.json() dest_filename = os.path.join(video_folder, video_id + ".mp4") download_file(video_doc["download_url"], dest_filename) if os.path.isfile(local_cache): os.remove(local_cache) with open(local_cache, 'w') as fp: json.dump(video_doc, fp, default=str) return video_doc except requests.ConnectionError as error: if local_doc: return local_doc log.error(f'Failing get video: {video_id}. Connection error: {error}') return None except requests.Timeout as error: if local_doc: return local_doc log.error(f'Failing get video: {video_id}. Timeout: {error}') return None except Exception as excp: if local_doc: return local_doc log.error(f'Failing get video: {video_id} - {excp}') return None
def upload_extract(app_token, dataset_id, extract_folder, max_files=MAX_EXTRACT_FILES, thumb_size=THUMB_SIZE): try: log.info(f"Upload extract {dataset_id}") folder_path = os.path.join(extract_folder, dataset_id) if not os.path.isdir(folder_path): raise Exception(f"Extract folder doesn't exists: {folder_path}") clear_log(folder_path, max_files) generate_extract_thumbs(folder_path, thumb_size) files_data = [] file_list = [f for f in os.listdir(folder_path)] for filename in file_list: exp_id = filename[:24] if filename.endswith('_data.json') and ( exp_id + ".jpg") in file_list and ( exp_id + "_thumb.jpg") in file_list: try: filepath = os.path.join(folder_path, filename) with open(filepath, 'r') as json_file: data = json.load(json_file) if "date" in data: data["date"] = {"$date": data["date"]} if "_id" in data: data["_id"] = {"$oid": data["_id"]} files_data.append(data) except: pass if not files_data: log.warning( f'Cannot upload post upload_extract: {dataset_id}. No files.') return dataset_id extract_files = {"files_data": files_data} with open(os.path.join(folder_path, 'extract_files.json'), 'w', newline='', encoding='utf8') as file_p: json.dump(extract_files, file_p, ensure_ascii=False, default=str) dest_filename = os.path.join(extract_folder, dataset_id + ".tar.gz") if os.path.isfile(dest_filename): os.remove(dest_filename) wd = os.getcwd() os.chdir(folder_path) with tarfile.open(dest_filename, "w:gz") as tar: for filename in os.listdir(folder_path): if filename.endswith( '.jpg') or filename == 'extract_files.json': tar.add(filename) os.chdir(wd) endpoint = jwt.decode(app_token, options={"verify_signature": False})['endpoint'] msg_headers = {'Authorization': f'Bearer {app_token}'} url = f"{endpoint}/dataset/{dataset_id}/extract" files = {'extract': open(dest_filename, 'rb')} values = {'dataset_id': dataset_id, 'extract_files': extract_files} response = requests.post(url, files=files, data=values, headers=msg_headers) if response.status_code != 201: raise Exception(f"Failing upload extract files: {response.json()}") os.remove(dest_filename) return dataset_id except requests.ConnectionError as error: log.error( f'Failing post upload_extract: {dataset_id}. Connection error: {error}' ) return None except requests.Timeout as error: log.error( f'Failing post upload_extract: {dataset_id}. Timeout: {error}') return None except Exception as excp: log.error(f'Failing post upload_extract: {dataset_id} - {excp}') return None
def get_model_component(app_token, model_component_id, model_component_folder): local_doc = None try: log.info(f"Get model_component {model_component_id}") folder_path = Path(model_component_folder + '/' + model_component_id) if not folder_path.is_dir(): folder_path.mkdir(parents=True, exist_ok=True) local_cache = os.path.join(model_component_folder, model_component_id + '.json') if os.path.isfile(local_cache): with open(local_cache) as fp: local_doc = json.load(fp) endpoint = jwt.decode(app_token, options={"verify_signature": False})['endpoint'] url = f"{endpoint}/model-component/{model_component_id}" msg_headers = {'Authorization': f'Bearer {app_token}'} payload = {"download_url": False} response = requests.get(url, headers=msg_headers, params=payload) if response.status_code != 200: if local_doc: return local_doc log.error(f"Failing get model_component: {response.json()}") return None model_component_doc = response.json() if local_doc and model_component_doc["version"] == local_doc["version"]: return local_doc payload = {"download_url": True} response = requests.get(url, headers=msg_headers, params=payload) if response.status_code != 200: if local_doc: return local_doc log.error(f"Failing get model: {response.json()}") return None model_component_doc = response.json() dest_filename = os.path.join(folder_path, model_component_id + ".tar.gz") download_file(model_component_doc["download_url"], dest_filename) # expand_file call(["tar", "-xf", dest_filename, "--directory", folder_path]) os.remove(dest_filename) if os.path.isfile(local_cache): os.remove(local_cache) with open(local_cache, 'w') as fp: json.dump(model_component_doc, fp, default=str) return model_component_id except requests.ConnectionError as error: if local_doc: return local_doc log.error( f'Failing get model_component: {model_component_id}. Connection error: {error}' ) return None except requests.Timeout as error: if local_doc: return local_doc log.error( f'Failing get model_component: {model_component_id}. Timeout: {error}' ) return None except Exception as excp: if local_doc: return local_doc log.error( f'Failing get model_component: {model_component_id} - {excp}') return None
def upload_model(app_token, dataset_id, model_info, model_folder, train_id, train_info, hist_folder): try: model_filename = os.path.join(model_folder, dataset_id + ".tar.gz") if os.path.isfile(model_filename): os.remove(model_filename) folder_path = os.path.join(model_folder, dataset_id) wd = os.getcwd() os.chdir(folder_path) files_list = get_list_files_info("./") with tarfile.open(model_filename, "w:gz") as tar: for filename in files_list: tar.add(filename) os.chdir(wd) hist_filename = os.path.join(hist_folder, train_id + ".tar.gz") if os.path.isfile(hist_filename): os.remove(hist_filename) wd = os.getcwd() os.chdir(hist_folder) files_list = get_list_files_info("./") with tarfile.open(hist_filename, "w:gz") as tar: for filename in files_list: tar.add(filename) os.chdir(wd) endpoint = jwt.decode(app_token, options={"verify_signature": False})['endpoint'] msg_headers = {'Authorization': f'Bearer {app_token}'} url = f"{endpoint}/model/{dataset_id}/{train_id}" files = { 'model_file': open(model_filename, 'rb'), 'train_file': open(hist_filename, 'rb') } test_batch_filename = os.path.join(hist_folder, "test_batch-" + dataset_id + ".jpg") if os.path.isfile(test_batch_filename): files['test_batch'] = open(test_batch_filename, 'rb') test_augmentation_filename = os.path.join( hist_folder, "test_augmentation-" + dataset_id + ".jpg") if os.path.isfile(test_augmentation_filename): files['test_augmentation'] = open(test_augmentation_filename, 'rb') model_info["size"] = os.stat(model_filename).st_size train_info["size"] = os.stat(hist_filename).st_size values = { 'model_info': json.dumps(model_info, default=str), 'train_info': json.dumps(train_info, default=str) } response = requests.post(url, files=files, data=values, headers=msg_headers) if response.status_code != 201: raise Exception(f"Failing upload model: {response.json()}") os.remove(model_filename) os.remove(hist_filename) return dataset_id except requests.ConnectionError as error: log.error( f'Failing uploading model: {dataset_id}. Connection error: {error}' ) return None except requests.Timeout as error: log.error(f'Failing uploading model: {dataset_id}. Timeout: {error}') return None except Exception as excp: log.error(f'Failing uploading model: {dataset_id} - {excp}') return None
def sync_files(self, origin, file_list=None): """ Synchronizes files between local and cloud folder If file_list then synchronize only files in this list """ if self._cloud_obj is None: raise Exception("Cloud provider not defined") if self._cloud_folder is None: raise Exception("Cloud folder not defined") if origin not in ["cloud", "local", "both"]: raise Exception( f"'origin'={origin} must be one of: ('cloud', 'local', 'both')" ) try: local_files = self._get_list_files_info(self._local_folder) cloud_files = self._cloud_obj.list_files_info( folder=self._cloud_folder, resource_id=self.resource_id) if origin in ["cloud", "both"]: download_files = [] for c_file in cloud_files: if file_list is not None and c_file[ "filename"] not in file_list: continue for l_file in local_files: if l_file["filename"] == c_file["filename"] and \ l_file["file_size"] == c_file["file_size"] and \ l_file["modified_date"] >= c_file["modified_date"]: break else: download_files.append(c_file) if download_files: log.info( f"Storage: {self.storage} - Resource-ID: {self.resource_id} - Download {len(download_files)} files" ) for c_file in download_files: file_data = self._cloud_obj.download_file( folder=self._cloud_folder, resource_id=self.resource_id, filename=c_file["filename"]) file_path = os.path.join(self._local_folder, c_file["filename"]) dirname = os.path.dirname(file_path) if not os.path.isdir(dirname): os.makedirs(dirname, exist_ok=True) with open(file_path, "wb") as fp: fp.write(file_data) if origin in ["local", "both"]: upload_files = [] for l_file in local_files: if file_list is not None and l_file[ "filename"] not in file_list: continue for c_file in cloud_files: if l_file["filename"] == c_file["filename"] and \ l_file["file_size"] == c_file["file_size"] and \ l_file["modified_date"] <= c_file["modified_date"]: break else: upload_files.append(l_file) if upload_files: log.info( f"Storage: {self.storage} - Resource-ID: {self.resource_id} - Upload {len(upload_files)} files" ) for l_file in upload_files: file_path = os.path.join(self._local_folder, l_file["filename"]) with open(file_path, "rb") as fp: file_data = fp.read() if self._cloud_obj.is_file( folder=self._cloud_folder, resource_id=self.resource_id, filename=l_file["filename"], ): self._cloud_obj.delete_file( folder=self._cloud_folder, resource_id=self.resource_id, filename=l_file["filename"], ) file_data = self._cloud_obj.upload_file( folder=self._cloud_folder, resource_id=self.resource_id, filename=l_file["filename"], data=file_data) # Set local file modified time to now() to prevent download the same file after date = datetime.datetime.now() mod_time = time.mktime(date.timetuple()) try: os.utime(file_path, (mod_time, mod_time)) except: pass except Exception as excp: log.error(traceback.format_exc()) log.error(f"sync_files error: {excp}") raise excp