def test_create_file():
files_src = [
FileMetadata(metadata={}, id=1007, asset_ids=[3], mime_type="pdf"),
FileMetadata(metadata={},
id=2007,
asset_ids=[7],
mime_type="application/pdf"),
FileMetadata(metadata={}, asset_ids=[5]),
FileMetadata(),
]
runtime = int(time.time() * 1000)
id_mapping = {3: 333, 7: 777, 5: 555}
created_files = []
for i, event in enumerate(files_src):
created_files.append(
create_file(event, id_mapping, "source project name", runtime))
assert len(created_files) == len(files_src)
for i, file_obj in enumerate(created_files):
assert file_obj.metadata["_replicatedInternalId"] == files_src[i].id
assert file_obj.metadata["_replicatedSource"] == "source project name"
assert (file_obj.asset_ids is None) == (files_src[i].asset_ids is None)
if file_obj.asset_ids is not None:
assert len(file_obj.asset_ids) == len(files_src[i].asset_ids)
for asset_id in files_src[i].asset_ids:
assert id_mapping[asset_id] in file_obj.asset_ids
assert created_files[0].mime_type == "application/pdf"
assert created_files[1].mime_type == "application/pdf"
assert created_files[2].mime_type is None
def create_file(src_file: FileMetadata, src_dst_ids_assets: Dict[int, int],
project_src: str, runtime: int) -> FileMetadata:
"""
Makes a new copy of the file to be replicated based on a source file.
Args:
src_file: The file from the source to be replicated to destination.
src_dst_ids_assets: A dictionary of all the mappings of source asset id to destination asset id.
project_src: The name of the project the object is being replicated from.
runtime: The timestamp to be used in the new replicated metadata.
Returns:
The replicated file to be created in the destination.
"""
logging.debug(f"Creating a new file based on source file id {src_file.id}")
mime_type = mimetypes.types_map.get(f".{src_file.mime_type}",
src_file.mime_type)
return FileMetadata(
external_id=src_file.external_id,
name=src_file.name,
source=src_file.source,
mime_type=mime_type,
metadata=replication.new_metadata(src_file, project_src, runtime),
asset_ids=replication.get_asset_ids(src_file.asset_ids,
src_dst_ids_assets),
)
def _upload_file_from_path(self, file: FileMetadata, file_path: str,
overwrite: bool):
with open(file_path, "rb") as fh:
file_metadata = self.upload_bytes(fh,
overwrite=overwrite,
**file.dump())
return file_metadata
def update_file(src_file: FileMetadata, dst_file: FileMetadata,
src_dst_ids_assets: Dict[int, int], project_src: str,
runtime: int) -> FileMetadata:
"""
Makes an updated version of the destination file based on the corresponding source file.
Args:
src_file: The file from the source to be replicated.
dst_file: The file from the destination that needs to be updated to reflect changes made to its source file.
src_dst_ids_assets: A dictionary of all the mappings of source asset id to destination asset id.
project_src: The name of the project the object is being replicated from.
runtime: The timestamp to be used in the new replicated metadata.
Returns:
The updated file object for the replication destination.
"""
logging.debug(
f"Updating existing file {dst_file.id} based on source file id {src_file.id}"
)
dst_file.external_id = src_file.external_id
dst_file.source = src_file.source
dst_file.metadata = replication.new_metadata(src_file, project_src,
runtime)
dst_file.asset_ids = replication.get_asset_ids(src_file.asset_ids,
src_dst_ids_assets)
dst_file.source_created_time = src_file.source_created_time
dst_file.source_modified_time = src_file.source_modified_time
return dst_file
def test_update_file():
files_src = [
FileMetadata(metadata={}, id=1007, asset_ids=[3], mime_type="pdf"),
FileMetadata(metadata={},
id=2007,
asset_ids=[7],
mime_type="application/pdf"),
FileMetadata(metadata={}, asset_ids=[5]),
]
files_dst = [
FileMetadata(metadata={}, id=1007, asset_ids=[3], mime_type="tiff"),
FileMetadata(metadata={}, id=2007, mime_type="jpg"),
FileMetadata(metadata={"dummy": True}, asset_ids=[3]),
]
runtime = time.time() * 1000
id_mapping = {3: 333, 7: 777, 5: 555}
updated_files = [
update_file(files_src[i],
files_dst[i], id_mapping, "source project name",
int(runtime)) for i in range(len(files_src))
]
assert len(updated_files) == len(files_src)
for i, file_obj in enumerate(updated_files):
assert file_obj.metadata["_replicatedInternalId"] == files_src[i].id
assert file_obj.metadata["_replicatedSource"] == "source project name"
assert (file_obj.asset_ids is None) == (files_src[i].asset_ids is None)
if file_obj.asset_ids is not None:
assert len(file_obj.asset_ids) == len(files_src[i].asset_ids)
for asset_id in files_src[i].asset_ids:
assert id_mapping[asset_id] in file_obj.asset_ids
def create(self,
file_metadata: FileMetadata,
overwrite: bool = False) -> Tuple[FileMetadata, str]:
"""Create file without uploading content.
Args:
file_metadata (FileMetaData): File metadata for the file to create.
overwrite (bool): If 'overwrite' is set to true, and the POST body content specifies a 'externalId' field,
fields for the file found for externalId can be overwritten. The default setting is false.
If metadata is included in the request body, all of the original metadata will be overwritten.
File-Asset mappings only change if explicitly stated in the assetIds field of the POST json body.
Do not set assetIds in request body if you want to keep the current file-asset mappings.
Returns:
Tuple[FileMetaData, str]: Tuple containing the file metadata and upload url of the created file.
Examples:
Create a file::
>>> from cognite.client import CogniteClient
>>> from cognite.client.data_classes import FileMetadata
>>> c = CogniteClient()
>>> file_metadata = FileMetadata(name="MyFile")
>>> res = c.files.create(file_metadata)
"""
res = self._post(url_path=self._RESOURCE_PATH,
json=file_metadata.dump(camel_case=True),
params={"overwrite": overwrite})
returned_file_metadata = res.json()
upload_url = returned_file_metadata.pop("uploadUrl")
file_metadata = FileMetadata._load(returned_file_metadata)
return (file_metadata, upload_url)
def test_file_uploader(self, MockCogniteClient):
client: CogniteClient = MockCogniteClient()
client.config.max_workers = 5
post_upload_test = {"value": 0}
def post(x):
post_upload_test["value"] += 1
queue = FileUploadQueue(client,
max_upload_interval=2,
post_upload_function=post)
queue.start()
queue.add_to_upload_queue(FileMetadata(name="hello.txt"), None)
time.sleep(2.1)
self.assertEqual(post_upload_test["value"], 1)
def test_create(self):
file_metadata = FileMetadata(name="mytestfile")
returned_file_metadata, upload_url = COGNITE_CLIENT.files.create(
file_metadata)
assert returned_file_metadata.uploaded is False
COGNITE_CLIENT.files.delete(id=returned_file_metadata.id)
def upload_bytes(
self,
content: Union[str, bytes, TextIO, BinaryIO],
name: str,
external_id: str = None,
source: str = None,
mime_type: str = None,
metadata: Dict[str, Any] = None,
asset_ids: List[int] = None,
source_created_time: int = None,
source_modified_time: int = None,
overwrite: bool = False,
):
"""Upload bytes or string.
You can also pass a file handle to content.
Args:
content (Union[str, bytes, TextIO, BinaryIO]): The content to upload.
name (str): Name of the file.
external_id (str): The external ID provided by the client. Must be unique within the project.
source (str): The source of the file.
mime_type (str): File type. E.g. text/plain, application/pdf,...
metadata (Dict[str, Any]): Customizable extra data about the file. String key -> String value.
asset_ids (List[int]): No description.
source_created_time (int): The timestamp for when the file was originally created in the source system.
source_modified_time (int): The timestamp for when the file was last modified in the source system.
overwrite (bool): If 'overwrite' is set to true, and the POST body content specifies a 'externalId' field,
fields for the file found for externalId can be overwritten. The default setting is false.
If metadata is included in the request body, all of the original metadata will be overwritten.
The actual file will be overwritten after successful upload. If there is no successful upload, the
current file contents will be kept.
File-Asset mappings only change if explicitly stated in the assetIds field of the POST json body.
Do not set assetIds in request body if you want to keep the current file-asset mappings.
Examples:
Upload a file from memory::
>>> from cognite.client import CogniteClient
>>> c = CogniteClient()
>>> res = c.files.upload_bytes(b"some content", name="my_file", asset_ids=[1,2,3])
"""
file_metadata = FileMetadata(
name=name,
external_id=external_id,
source=source,
mime_type=mime_type,
metadata=metadata,
asset_ids=asset_ids,
source_created_time=source_created_time,
source_modified_time=source_modified_time,
)
res = self._post(url_path=self._RESOURCE_PATH,
json=file_metadata.dump(camel_case=True),
params={"overwrite": overwrite})
returned_file_metadata = res.json()
upload_url = returned_file_metadata.pop("uploadUrl")
headers = {"X-Upload-Content-Type": file_metadata.mime_type}
self._request_session.put(upload_url, data=content, headers=headers)
return FileMetadata._load(returned_file_metadata)
def upload(
self,
path: str,
external_id: str = None,
name: str = None,
source: str = None,
mime_type: str = None,
metadata: Dict[str, Any] = None,
asset_ids: List[int] = None,
source_created_time: int = None,
source_modified_time: int = None,
recursive: bool = False,
overwrite: bool = False,
) -> Union[FileMetadata, FileMetadataList]:
"""Upload a file
Args:
path (str): Path to the file you wish to upload. If path is a directory, this method will upload all files in that directory.
external_id (str): The external ID provided by the client. Must be unique within the project.
name (str): Name of the file.
source (str): The source of the file.
mime_type (str): File type. E.g. text/plain, application/pdf, ...
metadata (Dict[str, Any]): Customizable extra data about the file. String key -> String value.
asset_ids (List[int]): No description.
source_created_time (int): The timestamp for when the file was originally created in the source system.
source_modified_time (int): The timestamp for when the file was last modified in the source system.
recursive (bool): If path is a directory, upload all contained files recursively.
overwrite (bool): If 'overwrite' is set to true, and the POST body content specifies a 'externalId' field,
fields for the file found for externalId can be overwritten. The default setting is false.
If metadata is included in the request body, all of the original metadata will be overwritten.
The actual file will be overwritten after successful upload. If there is no successful upload, the
current file contents will be kept.
File-Asset mappings only change if explicitly stated in the assetIds field of the POST json body.
Do not set assetIds in request body if you want to keep the current file-asset mappings.
Returns:
Union[FileMetadata, FileMetadataList]: The file metadata of the uploaded file(s).
Examples:
Upload a file in a given path::
>>> from cognite.client import CogniteClient
>>> c = CogniteClient()
>>> res = c.files.upload("/path/to/file", name="my_file")
If name is omitted, this method will use the name of the file
>>> from cognite.client import CogniteClient
>>> c = CogniteClient()
>>> res = c.files.upload("/path/to/file")
You can also upload all files in a directory by setting path to the path of a directory::
>>> from cognite.client import CogniteClient
>>> c = CogniteClient()
>>> res = c.files.upload("/path/to/my/directory")
"""
file_metadata = FileMetadata(
name=name,
external_id=external_id,
source=source,
mime_type=mime_type,
metadata=metadata,
asset_ids=asset_ids,
source_created_time=source_created_time,
source_modified_time=source_modified_time,
)
if os.path.isfile(path):
if not name:
file_metadata.name = os.path.basename(path)
return self._upload_file_from_path(file_metadata, path, overwrite)
elif os.path.isdir(path):
tasks = []
if recursive:
for root, _, files in os.walk(path):
for file in files:
file_path = os.path.join(root, file)
basename = os.path.basename(file_path)
tasks.append((FileMetadata(name=basename), file_path,
overwrite))
else:
for file_name in os.listdir(path):
file_path = os.path.join(path, file_name)
if os.path.isfile(file_path):
tasks.append((FileMetadata(name=file_name), file_path,
overwrite))
tasks_summary = utils._concurrency.execute_tasks_concurrently(
self._upload_file_from_path, tasks, self._config.max_workers)
tasks_summary.raise_compound_exception_if_failed_tasks(
task_unwrap_fn=lambda x: x[0].name)
return FileMetadataList(tasks_summary.results)
raise ValueError("path '{}' does not exist".format(path))
