def _infoFromMetadata(self, metadata): # pylint: disable=no-self-use isFolder = (metadata["mimeType"] == _folderMimeType) rfc3339 = "%Y-%m-%dT%H:%M:%S.%fZ" rawInfo = { "basic": { "name": metadata["name"], "is_dir": isFolder, }, "details": { "accessed": None, # not supported by Google Drive API "created": datetime_to_epoch( datetime.strptime(metadata["createdTime"], rfc3339)), "metadata_changed": None, # not supported by Google Drive API "modified": datetime_to_epoch( datetime.strptime(metadata["modifiedTime"], rfc3339)), "size": metadata["size"] if isFolder is False else None, # folders have no size "type": 1 if isFolder else 0 } } # there is also file-type-specific metadata like imageMediaMetadata return Info(rawInfo)
def _info_from_object(self, obj, namespaces): """Make an info dict from an s3 Object.""" key = obj.key path = self._key_to_path(key) name = basename(path.rstrip('/')) is_dir = key.endswith(self.delimiter) info = {"basic": {"name": name, "is_dir": is_dir}} if 'details' in namespaces: _type = int( ResourceType.directory if is_dir else ResourceType.file) info['details'] = { 'accessed': None, 'modified': datetime_to_epoch(obj.last_modified), 'size': obj.content_length, 'type': _type } if 's3' in namespaces: s3info = info['s3'] = {} for name in self._object_attributes: value = getattr(obj, name, None) if isinstance(value, datetime): value = datetime_to_epoch(value) s3info[name] = value if 'urls' in namespaces: url = self.client.generate_presigned_url(ClientMethod='get_object', Params={ 'Bucket': self._bucket_name, 'Key': key }) info['urls'] = {'download': url} return info
def _infoFromMetadata(self, metadata): # pylint: disable=no-self-use isRoot = isinstance(metadata, list) or metadata == _rootMetadata isFolder = isRoot or (metadata["mimeType"] == _folderMimeType) rfc3339 = "%Y-%m-%dT%H:%M:%S.%fZ" permissions = metadata.get("permissions", None) rawInfo = { "basic": { "name": "" if isRoot else metadata["name"], "is_dir": isFolder }, "details": { "accessed": None, # not supported by Google Drive API "created": None if isRoot else datetime_to_epoch(datetime.strptime(metadata["createdTime"], rfc3339)), "metadata_changed": None, # not supported by Google Drive API "modified": None if isRoot else datetime_to_epoch(datetime.strptime(metadata["modifiedTime"], rfc3339)), "size": int(metadata["size"]) if "size" in metadata else None, # folders, native google documents etc have no size "type": ResourceType.directory if isFolder else ResourceType.file }, "sharing": { "id": None if isRoot else metadata["id"], "permissions": permissions, "is_shared": len(permissions) > 1 if permissions is not None else None } } if "contentHints" in metadata and "indexableText" in metadata["contentHints"]: rawInfo.update({"google": {"indexableText": metadata["contentHints"]["indexableText"]}}) if "appProperties" in metadata: rawInfo.update({"google": {"appProperties": metadata["appProperties"]}}) # there is also file-type-specific metadata like imageMediaMetadata return Info(rawInfo)
def _info_from_object(self, obj, namespaces): """Make an info dict from an s3 Object.""" key = obj.key path = self._key_to_path(key) name = basename(path.rstrip("/")) is_dir = key.endswith(self.delimiter) info = {"basic": {"name": name, "is_dir": is_dir}} if "details" in namespaces: _type = int(ResourceType.directory if is_dir else ResourceType.file) info["details"] = { "accessed": None, "modified": datetime_to_epoch(obj.last_modified), "size": obj.content_length, "type": _type, } if "s3" in namespaces: s3info = info["s3"] = {} for name in self._object_attributes: value = getattr(obj, name, None) if isinstance(value, datetime): value = datetime_to_epoch(value) s3info[name] = value if "urls" in namespaces: url = self.client.generate_presigned_url( ClientMethod="get_object", Params={"Bucket": self._bucket_name, "Key": key}, ) info["urls"] = {"download": url} return info
def _info_from_object(self, obj, namespaces): """Make an info dict from an s3 Object.""" key = obj.key path = self._key_to_path(key) name = basename(path.rstrip('/')) is_dir = key.endswith(self.delimiter) info = {"basic": {"name": name, "is_dir": is_dir}} if 'details' in namespaces: _type = int( ResourceType.directory if is_dir else ResourceType.file) info['details'] = { 'accessed': None, 'modified': datetime_to_epoch(obj.last_modified), 'size': obj.content_length, 'type': _type } if 's3' in namespaces: s3info = info['s3'] = {} for name in self._object_attributes: value = getattr(obj, name, None) if isinstance(value, datetime): value = datetime_to_epoch(value) s3info[name] = value return info
def _infoFromMetadata(self, metadata): # pylint: disable=no-self-use isFolder = (metadata["mimeType"] == _folderMimeType) rfc3339 = "%Y-%m-%dT%H:%M:%S.%fZ" rawInfo = { "basic": { "name": metadata["name"], "is_dir": isFolder }, "details": { "accessed": None, # not supported by Google Drive API "created": datetime_to_epoch( datetime.strptime(metadata["createdTime"], rfc3339)), "metadata_changed": None, # not supported by Google Drive API "modified": datetime_to_epoch( datetime.strptime(metadata["modifiedTime"], rfc3339)), "size": int(metadata["size"]) if isFolder is False else None, # folders have no size "type": ResourceType.directory if isFolder else ResourceType.file }, "sharing": { "id": metadata["id"], "permissions": metadata["permissions"], "is_shared": True if len(metadata["permissions"]) > 1 else False } } # there is also file-type-specific metadata like imageMediaMetadata return Info(rawInfo)
def getattr(self, path, fh=None): try: info = self.fs.getinfo(path, namespaces=['access', 'details']) st = { 'st_atime': datetime_to_epoch(info.accessed) if info.accessed else 0, 'st_ctime': datetime_to_epoch(info.created) if info.created else 0, 'st_mtime': datetime_to_epoch(info.modified) if info.modified else 0, 'st_size': info.size or 0, 'st_uid': self.uid, 'st_gid': self.gid } if info.type == ResourceType.directory: st['st_nlink'] = 2 st['st_mode'] = (stat.S_IFDIR | 0o755) elif info.type == ResourceType.file: st['st_nlink'] = 1 st['st_mode'] = (stat.S_IFREG | 0o644) return (st) except fs.errors.ResourceNotFound: raise FuseOSError(errno.ENOENT)
def _infoFromMetadata(self, metadata): # pylint: disable=no-self-use rawInfo = { "basic": { "name": metadata.name, "is_dir": isinstance(metadata, FolderMetadata), } } if isinstance(metadata, FileMetadata): rawInfo.update({ "details": { "accessed": None, # not supported by Dropbox API "created": None, # not supported by Dropbox API?, "metadata_changed": None, # not supported by Dropbox "modified": datetime_to_epoch(metadata.server_modified), # API documentation says that this is reliable "size": metadata.size, "type": 0 }, "dropbox": { "content_hash": metadata.content_hash, # see https://www.dropbox.com/developers/reference/content-hash "rev": metadata.rev, "client_modified": metadata.client_modified # unverified value coming from dropbox clients } }) if metadata.media_info is not None and metadata.media_info.is_metadata() is True: media_info_metadata = metadata.media_info.get_metadata() if media_info_metadata.time_taken is not None: rawInfo.update({ "media_info": { "taken_date_time": datetime_to_epoch(media_info_metadata.time_taken) } }) if media_info_metadata.location is not None: rawInfo.update({ "media_info": { "location_latitude": media_info_metadata.location.latitude, "location_longitude": media_info_metadata.location.longitude } }) # Dropbox doesn't parse some jpgs properly if media_info_metadata.dimensions is not None: rawInfo.update({ "media_info": { "dimensions_height": media_info_metadata.dimensions.height, "dimensions_width": media_info_metadata.dimensions.width } }) elif isinstance(metadata, FolderMetadata): rawInfo.update({ "details": { "accessed": None, # not supported by Dropbox API "created": None, # not supported by Dropbox API, "metadata_changed": None, # not supported by Dropbox "modified": None, # not supported for folders "size": None, # not supported for folders "type": 1 }}) else: assert False, f"{metadata.name}, {metadata}, {type(metadata)}" return Info(rawInfo)
def test_setinfo2(self): self.fs.touch("file") modifiedTime = datetime(2000, 1, 1, 14, 42, 42) self.fs.setinfo("file", {"details": {"modified": datetime_to_epoch(modifiedTime)}}) info_ = self.fs.getinfo("file") assert datetime_to_epoch(info_.modified) == datetime_to_epoch(modifiedTime), f"{info_.modified}" createdTime = datetime(1999, 1, 1, 14, 42, 42) with self.fs.openbin("file2", "wb", createdDateTime=createdTime) as f: f.write(b"file2") info_ = self.fs.getinfo("file2") assert datetime_to_epoch(info_.created) == datetime_to_epoch(createdTime), f"{info_.created}"
def _InfoFromMetadata(metadata): isRoot = (metadata == _rootMetadata) isFolder = (metadata['mimeType'] == _folderMimeType) rfc3339 = '%Y-%m-%dT%H:%M:%S.%fZ' permissions = metadata.get('permissions', None) rawInfo = { 'basic': { 'name': '' if isRoot else metadata['name'], 'is_dir': isFolder }, 'details': { 'accessed': None, # not supported by Google Drive API 'created': None if isRoot else datetime_to_epoch( datetime.strptime(metadata['createdTime'], rfc3339)), 'metadata_changed': None, # not supported by Google Drive API 'modified': None if isRoot else datetime_to_epoch( datetime.strptime(metadata['modifiedTime'], rfc3339)), 'size': int(metadata['size']) if 'size' in metadata else None, # folders, native google documents etc have no size 'type': ResourceType.directory if isFolder else ResourceType.file }, 'sharing': { 'id': metadata['id'], 'permissions': permissions, 'is_shared': len(permissions) > 1 if permissions is not None else False } } googleMetadata = {} if 'contentHints' in metadata and 'indexableText' in metadata[ 'contentHints']: googleMetadata.update( {'indexableText': metadata['contentHints']['indexableText']}) if 'appProperties' in metadata: googleMetadata.update({'appProperties': metadata['appProperties']}) if 'md5Checksum' in metadata: rawInfo.update({'hashes': {'MD5': metadata['md5Checksum']}}) if 'mimeType' in metadata: googleMetadata.update( {'isShortcut': metadata['mimeType'] == _shortcutMimeType}) rawInfo.update({'google': googleMetadata}) # there is also file-type-specific metadata like imageMediaMetadata return Info(rawInfo)
def write(self, file_source): """Given a DownloadFileSource, write each entry to the tarfile. Args: file_source (DownloadFileSource): The file source """ for target in file_source: try: fileobj = file_source.open(target) try: tarinfo = tarfile.TarInfo() tarinfo.name = target.dst_path.lstrip('/') tarinfo.size = target.size tarinfo.mtime = datetime_to_epoch(target.modified) self._addfile(tarinfo, fileobj) finally: fileobj.close() except OSError: msg = ("Error happened during sending file content in archive stream, file path: %s, " "container: %s/%s, archive path: %s" % (target.src_path, target.container_type, target.container_id, target.dst_path)) self.log.critical(msg) self.log.exception("Error opening file for streaming") # Write a placeholder instead tarinfo = tarfile.TarInfo() tarinfo.name = target.dst_path + '.MISSING' self._tarfile.addfile(tarinfo)
def test_details(self): dates = [ datetime.datetime(2016, 7, 5, tzinfo=pytz.UTC), datetime.datetime(2016, 7, 6, tzinfo=pytz.UTC), datetime.datetime(2016, 7, 7, tzinfo=pytz.UTC), datetime.datetime(2016, 7, 8, tzinfo=pytz.UTC) ] epochs = [datetime_to_epoch(d) for d in dates] info = Info({ "details": { "accessed": epochs[0], "modified": epochs[1], "created": epochs[2], "metadata_changed": epochs[3], "type": int(ResourceType.file) } }) self.assertEqual(info.accessed, dates[0]) self.assertEqual(info.modified, dates[1]) self.assertEqual(info.created, dates[2]) self.assertEqual(info.metadata_changed, dates[3]) self.assertIsInstance(info.type, ResourceType) self.assertEqual(info.type, ResourceType.file) self.assertEqual(info.type, 2)
def getinfo(self, path, namespaces=None): if path == "/": return Info({ "basic": { "name": "", "is_dir": True }, "details": { "type": int(ResourceType.directory) } }) f = self._c.get_file_by_path(self._project_id, path) is_dir = f.mime_type == "directory" return Info({ "basic": { "name": f.name, "is_dir": is_dir }, "modified": datetime_to_epoch(f.mtime), "size": f.size, "type": int(ResourceType.directory if is_dir else ResourceType.file) })
def _itemInfo(self, item): # pylint: disable=no-self-use # Looks like the dates returned directly in item.file_system_info (i.e. not file_system_info) are UTC naive-datetimes # We're supposed to return timestamps, which the framework can convert to UTC aware-datetimes rawInfo = { "basic": { "name": item.name, "is_dir": item.folder is not None, }, "details": { "accessed": None, # not supported by OneDrive "created": datetime_to_epoch(item.created_date_time), "metadata_changed": None, # not supported by OneDrive "modified": datetime_to_epoch(item.last_modified_date_time), "size": item.size, "type": 1 if item.folder is not None else 0, }, "file_system_info": { "client_created": datetime_to_epoch(item.file_system_info.created_date_time), "client_modified": datetime_to_epoch(item.file_system_info.last_modified_date_time) } } if item.photo is not None: rawInfo.update({"photo": { "camera_make": item.photo.camera_make, "camera_model": item.photo.camera_model, "exposure_denominator": item.photo.exposure_denominator, "exposure_numerator": item.photo.exposure_numerator, "focal_length": item.photo.focal_length, "f_number": item.photo.f_number, "taken_date_time": item.photo.taken_date_time, "iso": item.photo.iso }}) if item.location is not None: rawInfo.update({"location": { "altitude": item.location.altitude, "latitude": item.location.latitude, "longitude": item.location.longitude }}) if item.tags is not None: # doesn't work rawInfo.update({"tags": { "tags": list(item.tags.tags) }}) return Info(rawInfo)
def test_setinfo2(self): self.fs.touch('file') modifiedTime = datetime(2000, 1, 1, 14, 42, 42) self.fs.setinfo( 'file', {'details': { 'modified': datetime_to_epoch(modifiedTime) }}) info_ = self.fs.getinfo('file') assert datetime_to_epoch(info_.modified) == datetime_to_epoch( modifiedTime), f'{info_.modified}' createdTime = datetime(1999, 1, 1, 14, 42, 42) with self.fs.openbin('file2', 'wb', createdDateTime=createdTime) as f: f.write(b'file2') info_ = self.fs.getinfo('file2') assert datetime_to_epoch(info_.created) == datetime_to_epoch( createdTime), f'{info_.created}'
def _info_from_blob(blob: Blob, namespaces: Optional[List[str]] = None) -> Info: """Make an info dict from a GCS object.""" path = blob.name name = basename(path.rstrip("/")) info = {"basic": {"name": name, "is_dir": False}} if "details" in namespaces: info["details"] = { "accessed": None, "modified": datetime_to_epoch(blob.updated), "size": blob.size, "type": int(ResourceType.file) } # TODO more namespaces: basic, urls, gcs, ... return Info(info)
def archivestream(self, ticket): stream = cStringIO.StringIO() with tarfile.open(mode='w|', fileobj=stream): for filepath, arcpath, cont_name, cont_id, f_size, f_modified in ticket[ 'target']: tarinfo = TarInfo() tarinfo.name = arcpath.lstrip('/') tarinfo.size = f_size tarinfo.mtime = datetime_to_epoch(f_modified) tarinfo_buf = tarinfo.tobuf() signed_url = None try: signed_url = files.get_signed_url(filepath, config.fs) except fs.errors.ResourceNotFound: pass if signed_url: content_generator = self.stream_file_signed_url( signed_url, tarinfo_buf, (filepath, cont_name, cont_id, arcpath)) else: content_generator = self.stream_regular_file( filepath, tarinfo_buf, (filepath, cont_name, cont_id, arcpath)) for chunk in content_generator: yield chunk self.log_user_access( AccessType.download_file, cont_name=cont_name, cont_id=cont_id, filename=os.path.basename(arcpath), origin_override=ticket['origin'], download_ticket=ticket['_id']) # log download yield stream.getvalue() # get tar stream trailer stream.close()
def _itemInfo(self, item): # pylint: disable=no-self-use # Looks like the dates returned directly in item.file_system_info (i.e. not file_system_info) are UTC naive-datetimes # We're supposed to return timestamps, which the framework can convert to UTC aware-datetimes rawInfo = { "basic": { "name": item["name"], "is_dir": "folder" in item, }, "details": { "accessed": None, # not supported by OneDrive "created": datetime_to_epoch(_ParseDateTime(item["createdDateTime"])), "metadata_changed": None, # not supported by OneDrive "modified": datetime_to_epoch(_ParseDateTime( item["lastModifiedDateTime"])), "size": item["size"], "type": ResourceType.directory if "folder" in item else ResourceType.file, }, "file_system_info": { "client_created": datetime_to_epoch( _ParseDateTime(item["fileSystemInfo"]["createdDateTime"])), "client_modified": datetime_to_epoch( _ParseDateTime( item["fileSystemInfo"]["lastModifiedDateTime"])) } } if "photo" in item: rawInfo["photo"] = {} rawInfo["photo"].update( _UpdateDict(item["photo"], "cameraMake", "camera_make")) rawInfo["photo"].update( _UpdateDict(item["photo"], "cameraModel", "camera_model")) rawInfo["photo"].update( _UpdateDict(item["photo"], "exposureDenominator", "exposure_denominator")) rawInfo["photo"].update( _UpdateDict(item["photo"], "exposureNumerator", "exposure_numerator")) rawInfo["photo"].update( _UpdateDict(item["photo"], "focalLength", "focal_length")) rawInfo["photo"].update( _UpdateDict(item["photo"], "fNumber", "f_number")) rawInfo["photo"].update( _UpdateDict(item["photo"], "takenDateTime", "taken_date_time", _ParseDateTime)) rawInfo["photo"].update(_UpdateDict(item["photo"], "iso", "iso")) if "image" in item: rawInfo["image"] = {} rawInfo["image"].update( _UpdateDict(item["image"], "width", "width")) rawInfo["image"].update( _UpdateDict(item["image"], "height", "height")) if "location" in item: rawInfo["location"] = {} rawInfo["location"].update( _UpdateDict(item["location"], "altitude", "altitude")) rawInfo["location"].update( _UpdateDict(item["location"], "latitude", "latitude")) rawInfo["location"].update( _UpdateDict(item["location"], "longitude", "longitude")) if "file" in item: if "hashes" in item["file"]: rawInfo["hashes"] = {} # The spec is at https://docs.microsoft.com/en-us/onedrive/developer/rest-api/resources/hashes?view=odsp-graph-online # CRC32 appears in the spec but not in the implementation rawInfo["hashes"].update( _UpdateDict(item["file"]["hashes"], "crc32Hash", "CRC32")) # Standard SHA1 rawInfo["hashes"].update( _UpdateDict(item["file"]["hashes"], "sha1Hash", "SHA1")) # proprietary hash for change detection rawInfo["hashes"].update( _UpdateDict(item["file"]["hashes"], "quickXorHash", "quickXorHash")) if "tags" in item: # doesn't work rawInfo.update({"tags": {"tags": item["tags"]["tags"]}}) return Info(rawInfo)
def test_datetime_to_epoch(self): self.assertEqual( datetime_to_epoch(datetime(1974, 7, 5, tzinfo=pytz.UTC)), 142214400)
def test_datetime_to_epoch(self): self.assertEqual( datetime_to_epoch(datetime(1974, 7, 5, tzinfo=timezone.utc)), 142214400)