async def read_file( id: bytes, metadata_location: str, version: Optional[DropVersion] = None, get_latest: bool = False, ) -> Optional['DropMetadata']: """Read a drop metadata file from disk :param id: the drop id :param metadata_location: where to look for the file :param version: the drop version :return: A DropMetadata object, or maybe None """ logger.debug("reading from file") if version is None: logger.debug( "Version is None, looking it up in %s", metadata_location, ) if get_latest: logger.info("reading latest") file_name = await DropMetadata.read_latest( id, metadata_location, ) else: logger.info("reading current") file_name = await DropMetadata.read_current( id, metadata_location, ) else: logger.debug("Getting version %s", version) file_name = DropMetadata.make_filename(id, version) if file_name is None: logger.warning( "current drop metadata not found for %s", crypto_util.b64encode(id), ) return None if not os.path.isfile(os.path.join(metadata_location, file_name)): logger.warning( "drop metadata not found for %s", crypto_util.b64encode(id), ) return None async with aiofiles.open( os.path.join(metadata_location, file_name), 'rb', ) as f: b = b'' while True: data = await f.read() if not data: break b += data return await DropMetadata.decode(b)
async def write_chunk( filepath: str, position: int, contents: bytes, chunk_hash: bytes, chunk_size: int = DEFAULT_CHUNK_SIZE, ) -> None: """ Takes a filepath, position, contents, and contents hash and writes it to a file correctly. Assumes the file has been created. Will check the hash, and raise a VerificationException if the provided chunk_hash doesn't match. May raise relevant IO exceptions. If the file extension indicates the file is complete, does nothing. :param filepath: the path of the file to write to :param position: the posiiton in the file to write to :param contents: the contents to write :param chunk_hash: the expected hash of contents :param chunk_size: (optional) override the chunk size, used to calculate \ the position in the file :raises crypto_util.VerificationException: When the hash of the provided \ bytes does not match the provided hash :return: None """ if is_complete(filepath): logger.info("file %s already done, not writing", filepath) return filepath += DEFAULT_INCOMPLETE_EXT computed_hash = await crypto_util.hash(contents) if computed_hash != chunk_hash: raise crypto_util.VerificationException( "Computed: %s, expected: %s" % ( crypto_util.b64encode(computed_hash), crypto_util.b64encode(chunk_hash), ), ) logger.debug( "writing chunk with filepath %s and hash %s", filepath, crypto_util.b64encode(chunk_hash), ) await write_locks[filepath].acquire() async with aiofiles.open(filepath, 'r+b') as f: pos_bytes = position * chunk_size await f.seek(pos_bytes) await f.write(contents) await f.flush() write_locks[filepath].release()
async def read_file( file_id: bytes, metadata_location: str, file_name: str, ) -> Optional['FileMetadata']: """Read a file metadata file and return FileMetadata :param file_id: The hash of the file to read :param metadata_location: drop location with default metadata location :return: a FileMetadata object or None if it does not exist """ logger.debug("reading from file") file_name = crypto_util.b64encode(file_id).decode("utf-8") if not os.path.exists(os.path.join(metadata_location, file_name)): return None async with aiofiles.open( os.path.join(metadata_location, file_name), 'rb', ) as f: b = b'' while True: data = await f.read() if not data: break b += data return FileMetadata.decode(b)
async def start_drop_from_id(drop_id: bytes, save_dir: str) -> None: """Given a drop_id and save directory, sets up the directory for syncing and adds the info to the global dir Should be followed by calls to `get_drop_metadata`, `get_file_metadata` for each file, and `sync_file_contents` :param drop_id: The drop id to add :param save_dir: where to download the drop to """ logger.info( "Adding drop from id %s to %s", crypto_util.b64encode(drop_id), save_dir, ) os.makedirs( os.path.join(save_dir, DEFAULT_DROP_METADATA_LOCATION), exist_ok=True, ) os.makedirs( os.path.join(save_dir, DEFAULT_FILE_METADATA_LOCATION), exist_ok=True, ) await save_drop_location(drop_id, save_dir)
async def send_drops_to_dps( ip: str, port: int, shutdown_flag: threading.Event, ) -> None: """For each drop tell the dps that ip/port has that drop :param ip: The ip/address to tell the dps :param port: The port to tell the dps :param shutdown_flag: Stop when this is set """ this_node_id = await node_id_from_private_key( await load_private_key_from_disk(), ) dps = await get_drop_peer_store(this_node_id) while not shutdown_flag.is_set(): drops = list_drops() logger.info("Sending drops to dps") for drop in drops: logger.debug("Sending drop %s", crypto_util.b64encode(drop)) await dps.add_drop_peer(drop, ip, port) sleep_time = TRACKER_DROP_AVAILABILITY_TTL / 2 - 1 logger.debug("Sleeping for %s", sleep_time) await asyncio.sleep(sleep_time)
def make_filename( id: bytes, version: Union[str, DropVersion], ) -> str: """Make the filename for a drop metadata""" return "%s_%s" % ( crypto_util.b64encode(id).decode("utf-8"), str(version), )
def unsubscribe(self) -> None: """Removes the refrence in the .5yncr folder therefore preventing future updates :return: None """ save_path = _get_save_path() encoded_drop_id = crypto_util.b64encode(self.id).decode('utf-8') drop_loc_file = os.path.join(save_path, encoded_drop_id) logger.info("removing file: ", drop_loc_file) os.remove(drop_loc_file)
async def drop_metadata_to_response(md: DropMetadata) -> Dict[str, Any]: """ Converts dropMetadata object into frontend readable dictionary. :param md: DropMetadata object :return: Dictionary for frontend """ files = await get_file_names_percent(md.id) response = { 'drop_id': crypto_util.b64encode(md.id), 'name': md.name, 'version': "%s" % md.version, 'previous_versions': ["%s" % v for v in md.previous_versions], 'primary_owner': crypto_util.b64encode(md.owner), 'other_owners': [crypto_util.b64encode(o) for o in md.other_owners], 'signed_by': crypto_util.b64encode(md.signed_by), 'files': {n: int(p * 100) for n, p in files.items()}, } return response
async def send_my_pub_key() -> None: """Send the pub key for this node the the Key Store""" this_node_id = await node_id_from_private_key( await load_private_key_from_disk(), ) logger.info( "Sending pub key for %s to tracker", crypto_util.b64encode(this_node_id), ) public_key_store = await get_public_key_store(this_node_id) pub_key = (await load_private_key_from_disk()).public_key() pub_key_bytes = crypto_util.dump_public_key(pub_key) await public_key_store.set_key(pub_key_bytes)
def log(self) -> logging.Logger: """ A logger for this object :return: a logger object for this class """ if self._log is None: self._log = get_logger( '.'.join([ __name__, self.__class__.__name__, crypto_util.b64encode(self.id).decode('utf-8'), ]), ) return self._log
async def get_drop_location(drop_id: bytes) -> str: """Get a drop's location from the central data dir :param drop_id: The drop id to look up :return: The drops save dir """ save_path = _get_save_path() encoded_drop_id = crypto_util.b64encode(drop_id).decode('utf-8') async with aiofiles.open( os.path.join(save_path, encoded_drop_id), 'r', ) as f: return await f.read()
async def write_file( self, metadata_location: str, ) -> None: """Write this file metadata to a file :param metadata_location: where to save it """ self.log.debug("writing file") file_name = crypto_util.b64encode(self.file_id).decode("utf-8") if not os.path.exists(metadata_location): os.makedirs(metadata_location) async with aiofiles.open( os.path.join(metadata_location, file_name), 'wb', ) as f: await f.write(self.encode())
async def save_drop_location(drop_id: bytes, location: str) -> None: """Save a drop's location in the central data dir :param drop_id: The unencoded drop id :param location: Where the drop is located on disk """ save_path = _get_save_path() encoded_drop_id = crypto_util.b64encode(drop_id).decode('utf-8') if not os.path.exists(save_path): os.makedirs(save_path) async with aiofiles.open( os.path.join(save_path, encoded_drop_id), 'w', ) as f: await f.write(location)
async def get_file_metadata( drop_id: bytes, file_id: bytes, save_dir: str, file_name: str, peers: List[Tuple[str, int]], ) -> FileMetadata: """Get file metadata, given a file id, drop id and save dir. If the file metadata is not on disk already, attempt to download from peers. :param drop_id: the drop id :param file_id: the file id :param save_dir: where the drop is saved :param peers: where to look on the network for data :return: A file metadata object """ logger.info("getting file metadata for %s", crypto_util.b64encode(file_id)) metadata_dir = os.path.join(save_dir, DEFAULT_FILE_METADATA_LOCATION) metadata = await FileMetadata.read_file( file_id=file_id, metadata_location=metadata_dir, file_name=file_name, ) if metadata is None: logger.debug("file metadata not on disk, getting from network") if not peers: peers = await get_drop_peers(drop_id) metadata = await send_requests.do_request( request_fun=send_requests.send_file_metadata_request, peers=peers, fun_args={ 'drop_id': drop_id, 'file_id': file_id }, ) if metadata is None: raise Exception await metadata.write_file(metadata_dir) return metadata
async def send_drops_once( ip: str, port: int, dps: Optional['DropPeerStore'] = None, ) -> None: """For each drop tell the dps our ip and port, then exit :param ip: ip/address to tell the dps :param port: port to tell the dps :param dps: if provided, use this dps, else get it from config """ if dps is None: this_node_id = await node_id_from_private_key( await load_private_key_from_disk(), ) dps = await get_drop_peer_store(this_node_id) drops = list_drops() logger.info("Sending drops to dps") for drop in drops: logger.debug("Sending drop %s", crypto_util.b64encode(drop)) await dps.add_drop_peer(drop, ip, port)
async def get_drop_peers(drop_id: bytes) -> List[Tuple[str, int]]: """ Gets the peers that have a drop. Also shuffles the list :param drop_id: id of drop :raises PeerStoreError: If peers cannot be found :return: A list of peers in format (ip, port) """ priv_key = await node_init.load_private_key_from_disk() node_id = await crypto_util.node_id_from_public_key(priv_key.public_key()) drop_peer_store_instance = await drop_peer_store.get_drop_peer_store( node_id, ) success, drop_peers = await drop_peer_store_instance.request_peers(drop_id) if not success: encoded_id = crypto_util.b64encode(drop_id) raise PeerStoreError("No peers found for drop %s" % encoded_id) peers = [(ip, int(port)) for peer_name, ip, port in drop_peers] shuffle(peers) return peers
async def get_drop_metadata( drop_id: bytes, peers: List[Tuple[str, int]], save_dir: Optional[str] = None, version: Optional[DropVersion] = None, ) -> DropMetadata: """Get drop metadata, given a drop id and save dir. If the drop metadata is not on disk already, attempt to download from peers. :param drop_id: the drop id :param peers: where to look on the network for data :param save_dir: where the drop is saved :return: A drop metadata object """ logger.info("getting drop metadata for %s", crypto_util.b64encode(drop_id)) if save_dir is None: logger.debug("save_dir not set, trying to look it up") save_dir = await get_drop_location(drop_id) logger.debug("save_dir is %s", save_dir) metadata_dir = os.path.join(save_dir, DEFAULT_DROP_METADATA_LOCATION) metadata = await DropMetadata.read_file( id=drop_id, metadata_location=metadata_dir, ) if metadata is None: logger.debug("drop metadata not on disk, getting from network") metadata = await do_metadata_request(drop_id, peers, version) # mypy can't figure out that this won't be None metadata = cast(DropMetadata, metadata) await metadata.write_file( is_latest=True, metadata_location=metadata_dir, ) return metadata
async def initialize_drop(directory: str) -> bytes: """ Initialize a drop from a directory. Generates the necesssary drop and file metadata files and writes the drop location to the central config dif :param directory: The directory to initialize a drop from :return: The b64 encoded id of the created drop """ logger.info("initializing drop in dir %s", directory) priv_key = await node_init.load_private_key_from_disk() node_id = await crypto_util.node_id_from_public_key(priv_key.public_key()) (drop_m, files_m) = await make_drop_metadata( path=directory, drop_name=os.path.basename(directory), owner=node_id, ) await drop_m.write_file( is_current=True, is_latest=True, metadata_location=os.path.join( directory, DEFAULT_DROP_METADATA_LOCATION, ), ) for f_m in files_m.values(): await f_m.write_file( os.path.join(directory, DEFAULT_FILE_METADATA_LOCATION), ) await save_drop_location(drop_m.id, directory) logger.info("drop initialized with %s files", len(files_m)) scanned_files = await fileio_util.scan_current_files(directory) await fileio_util.write_timestamp_file( scanned_files, directory, ) return crypto_util.b64encode(drop_m.id)
async def get_pub_key(node_id: bytes) -> crypto_util.rsa.RSAPublicKey: """ Gets the public key from disk if possible otherwise request it from PublicKeyStore :param node_id: bytes for the node you want public key of :raises VerificationException: If the pub key cannot be retrieved :return: PublicKey """ init_directory = get_full_init_directory(None) pub_key_directory = os.path.join( init_directory, DEFAULT_PUB_KEY_LOOKUP_LOCATION, ) if not os.path.isdir(pub_key_directory): os.makedirs(pub_key_directory) key_file_name = "{}.pub".format(crypto_util.b64encode(node_id)) key_path = os.path.join(pub_key_directory, key_file_name) if os.path.isfile(key_path): async with aiofiles.open(key_path, 'rb') as pub_file: pub_key = await pub_file.read() return load_public_key(pub_key) else: key_bytes = await load_private_key_from_disk() this_node_id = await node_id_from_private_key(key_bytes) public_key_store = await get_public_key_store(this_node_id) key_request = await public_key_store.request_key(node_id) if key_request[0] and key_request[1] is not None: pub_key = key_request[1].encode('utf-8') await _save_key_to_disk(key_path, pub_key) return load_public_key(pub_key) else: raise VerificationException()
async def read_chunk( filepath: str, position: int, file_hash: Optional[bytes] = None, chunk_size: int = DEFAULT_CHUNK_SIZE, ) -> Tuple[bytes, bytes]: """Reads a chunk for a file, returning the contents and its hash. May raise relevant IO exceptions If file_hash is provided, will check the chunk that is read :param filepath: the path of the file to read from :param position: where to read from :param file_hash: if provided, will check the file hash :param chunk_size: (optional) override the chunk size :raises crypto_util.VerificationException: If the hash of the bytes read \ does not match the provided hash :return: a double of (contents, hash), both bytes """ if not is_complete(filepath): logger.debug("file %s not done, adding extention", filepath) filepath += DEFAULT_INCOMPLETE_EXT async with aiofiles.open(filepath, 'rb') as f: logger.debug("async reading %s", filepath) pos_bytes = position * chunk_size await f.seek(pos_bytes) data = await f.read(chunk_size) h = await crypto_util.hash(data) logger.debug("async read hash: %s", crypto_util.b64encode(h)) if file_hash is not None: logger.info("input file_hash is not None, checking") if h != file_hash: raise crypto_util.VerificationException() return (data, h)