class TaskMixin: logger = logger_factory.get_logger('Tasks') def __init__(self, task_base=None, drive=None, items_store=None, task_pool=None): self.drive = drive if task_base is None else task_base.drive self.items_store = items_store if task_base is None else task_base.items_store self.task_pool = task_pool if task_base is None else task_base.task_pool @property def drive(self): return self._drive @drive.setter def drive(self, d): self._drive = d @property def items_store(self): return self._items_store @items_store.setter def items_store(self, s): self._items_store = s @property def task_pool(self): return self._task_pool @task_pool.setter def task_pool(self, p): self._task_pool = p
def mock_loggers(): for cls in [ drives.DriveObject, account_db.AccountStorage, account_db.AccountStorage, drives_db.DriveStorage, drive_config.DriveConfig ]: cls.logger = logger_factory.get_logger('ut_' + cls.__name__, min_level=logging.CRITICAL)
def main(): global logger args = fix_log_args(parse_args()) logger = logger_factory.get_logger('Main') check_config_dir() load_user_config() load_item_storage() load_task_storage() start_task_workers() refill_tasks()
class NetworkMonitor(threading.Thread): THREAD_NAME = "netmon" logger = logger_factory.get_logger(__name__) def __init__(self, test_uri='https://onedrive.com', retry_delay_sec=30, proxies=None): """ :param str test_uri: The url to use in testing internet connectivity. :param int retry_delay_sec: The amount of seconds to wait before retry. :param dict[str, str] proxies: A dict of protocol-url pairs. """ super().__init__() self.name = NetworkMonitor.THREAD_NAME self.daemon = True self.test_uri = test_uri self.retry_delay = retry_delay_sec self.proxies = proxies self.queue = queue.Queue() self.conditions = {} self.logger.info("Initialized.") def suspend_caller(self): """Put the calling thread into suspension queue.""" me = threading.current_thread() cond = self.conditions[me.ident] = threading.Condition() self.queue.put(me) self.logger.info("Suspended due to network failure.") cond.acquire() # Put the caller thread to sleep cond.wait() # Thread is waken up by manager cond.release() del self.conditions[me.ident] self.logger.info("Resumed.") def is_connected(self): """ Test if internet connection is OK by connecting to the test URI provided. If proxy setting is set in the client, it will be used as well. :return: True if internet connection is on; False otherwise. """ try: requests.head(self.test_uri, proxies=self.proxies) return True except requests.ConnectionError: return False def run(self): while True: th = self.queue.get() # blocking call while not self.is_connected(): time.sleep(self.retry_delay) self.conditions[th.ident].acquire() self.conditions[th.ident].notify() self.conditions[th.ident].release()
class TaskConsumer(threading.Thread): terminate_sign = threading.Event() logger = logger_factory.get_logger('TaskConsumer') def __init__(self, task_pool): """ :param onedrive_d.store.task_pool.TaskPool task_pool: """ super().__init__() self.daemon = True self.task_pool = task_pool def run(self): self.logger.debug('Started.') while True: self.task_pool.semaphore.acquire() if self.terminate_sign.is_set(): break task = self.task_pool.pop_task() self.logger.debug('Acquired task of type "%s" on path "%s"', task.__class__.__name__, task.local_parent_path + '/' + task.name) task.handle() self.logger.debug('Stopped.')
class ItemStorage: """ Local storage for items under ONE drive. """ logger = logger_factory.get_logger('ItemStorage') def __init__(self, db_path, drive): """ :param str db_path: A unique path for the database to store items for the target drive. :param onedrive_d.api.drives.DriveObject drive: The underlying drive object. """ if not hasattr(drive, 'storage_lock'): drive.storage_lock = rwlock.RWLock() self.lock = drive.storage_lock self._conn = sqlite3.connect(db_path, isolation_level=None, check_same_thread=False) self.drive = drive self._cursor = self._conn.cursor() self._cursor.execute(get_content('onedrive_items.sql')) self._conn.commit() atexit.register(self.close) def close(self): self._cursor.close() self._conn.close() def local_path_to_remote_path(self, local_path): return self.drive.drive_path + '/root:' + local_path def get_items_by_id(self, item_id=None, parent_path=None, item_name=None, local_parent_path=None): """ FInd all qualified records from database by ID or path. :param str item_id: ID of the target item. :param str parent_path: Path reference of the target item's parent. Used with item_name. :param str item_name: Name of the item. Used with parent_path or local_parent_path. :param str local_parent_path: Path relative to drive's local root. If at root, use ''. :return dict[str, onedrive_d.store.items_db.ItemRecord]: All qualified records index by item ID. """ if local_parent_path is not None: parent_path = self.local_path_to_remote_path(local_parent_path) args = {'item_id': item_id, 'parent_path': parent_path, 'item_name': item_name} return self.get_items(args) def get_items_by_hash(self, crc32_hash=None, sha1_hash=None): """ Find all qualified records from database whose hash values match either parameter. :param str crc32_hash: CRC32 hash of the target item. :param str sha1_hash: SHA-1 hash of the target item. :return dict[str, onedrive_d.store.items_db.ItemRecord]: All qualified records index by item ID. """ args = {'crc32_hash': crc32_hash, 'sha1_hash': sha1_hash} return self.get_items(args, 'OR') @staticmethod def _get_where_clause(args, relation='AND'): """ Form a where clause in SQL query and the tuples for the filler values. :param dict[str, str | int]] args: Keys are where conditions and values are the filler values. :param str relation: Either 'AND' or 'OR'. :return (str, ()): """ keys = [] values = [] for k, v in args.items(): if v is not None: keys.append(k + '=?') values.append(v) relation = ' ' + relation + ' ' return relation.join(keys), tuple(values) def get_items(self, args, relation='AND'): """ :param dict[str, int | str] args: Criteria used to construct SQL query. :param str relation: Relation of the criteria. :return dict[str, onedrive_d.store.items_db.ItemRecord]: All matching rows in the form of ItemRecord. """ where, values = self._get_where_clause(args, relation) ret = {} self.lock.reader_acquire() q = self._cursor.execute('SELECT item_id, type, item_name, parent_id, parent_path, etag, ctag, size, ' 'created_time, modified_time, status, crc32_hash, sha1_hash FROM items WHERE ' + where, values) for row in q.fetchall(): item = ItemRecord(row) ret[item.item_id] = item self.lock.reader_release() return ret def update_item(self, item, status=ItemRecordStatuses.OK): """ :param onedrive_d.api.items.OneDriveItem item: :param str status: :return: """ if item.is_folder: crc32_hash = None sha1_hash = None else: file_facet = item.file_props crc32_hash = file_facet.hashes.crc32 sha1_hash = file_facet.hashes.sha1 parent_ref = item.parent_reference created_time_str = datetime_to_str(item.created_time) modified_time_str = datetime_to_str(item.modified_time) self.lock.writer_acquire() self._cursor.execute( 'INSERT OR REPLACE INTO items (item_id, type, item_name, parent_id, parent_path, etag, ' 'ctag, size, created_time, modified_time, status, crc32_hash, sha1_hash)' ' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', (item.id, item.type, item.name, parent_ref.id, parent_ref.path, item.e_tag, item.c_tag, item.size, created_time_str, modified_time_str, status, crc32_hash, sha1_hash)) self._conn.commit() self.lock.writer_release() def delete_item(self, item_id=None, parent_path=None, item_name=None, local_parent_path=None, is_folder=False): """ Delete the specified item from database. If the item is a directory, then also delete all its children items. :param str item_id: ID of the target item. :param str parent_path: Path reference of the target item's parent. Used with item_name. :param str item_name: Name of the item. Used with parent_path or local_parent_path. :param str local_parent_path: Path relative to drive's local root. If at root, use ''. :param True | False is_folder: True to indicate that the item is a folder (delete all children). """ if local_parent_path is not None: parent_path = self.local_path_to_remote_path(local_parent_path) where, values = self._get_where_clause({'item_id': item_id, 'parent_path': parent_path, 'item_name': item_name}) self.lock.writer_acquire() if is_folder: # Translate ID reference to path and name reference. q = self._cursor.execute('SELECT item_id, parent_path, item_name FROM items WHERE ' + where, values) row = q.fetchone() if row is None: self.logger.warning('The folder to delete does not exist: %s, %s', where, str(values)) else: item_id, parent_path, item_name = row self._cursor.execute('DELETE FROM items WHERE parent_id=? OR parent_path LIKE ?', (item_id, parent_path + '/' + item_name + '/%')) self._cursor.execute('DELETE FROM items WHERE ' + where, values) self._conn.commit() self.lock.writer_release() def update_status(self, status, item_id=None, parent_path=None, item_name=None, local_parent_path=None): """ Update the status tag of the target item. :param str item_id: ID of the target item. :param str parent_path: Path reference of the target item's parent. Used with item_name. :param str item_name: Name of the item. Used with parent_path or local_parent_path. :param str local_parent_path: Path relative to drive's local root. If at root, use ''. """ if local_parent_path is not None: parent_path = self.local_path_to_remote_path(local_parent_path) where, values = self._get_where_clause({'item_id': item_id, 'parent_path': parent_path, 'item_name': item_name}) values = (status,) + values self.lock.writer_acquire() self._cursor.execute('UPDATE items SET status=? WHERE ' + where, values) self._conn.commit() self.lock.writer_release()
class AccountStorage: """ A SQLite-based storage layer for account and drive objects. """ logger = logger_factory.get_logger('AccountStorage') def __init__(self, db_path, personal_client=None, business_client=None): self._conn = sqlite3.connect(db_path, isolation_level=None) self._cursor = self._conn.cursor() self._cursor.execute(get_content('onedrive_accounts.sql')) self._conn.commit() self.personal_client = personal_client self.business_client = business_client self._all_accounts = {} atexit.register(self.close) def parse_account_type(self, account_type): """ :param str account_type: :return onedrive_d.api.accounts.PersonalAccount | onedrive_d.accounts.accounts.BusinessAccount: """ if account_type == accounts.AccountTypes.PERSONAL: return accounts.PersonalAccount, self.personal_client else: return accounts.BusinessAccount, self.business_client def deserialize_account_row(self, account_id, account_type, account_dump, profile_dump, container): account_cls, client = self.parse_account_type(account_type) if client is None: self.logger.warning( 'Account %s was not loaded since no client of that type provided.', account_id) return try: account = account_cls.load(client, account_dump) container[(account_id, account_type)] = account except ValueError as e: self.logger.warning('Failed to deserialize account %s: %s', account_id, e) else: try: profile = resources.UserProfile.load(profile_dump) except ValueError as e: self.logger.warning( 'Failed to deserialize user profile for account %s: %s', account_id, e) profile = account.profile account.profile = profile def get_all_accounts(self): self._conn.commit() q = self._cursor.execute( 'SELECT account_id, account_type, account_dump, profile_dump FROM accounts' ) for row in q.fetchall(): account_id, account_type, account_dump, profile_dump = row self.deserialize_account_row(account_id, account_type, account_dump, profile_dump, self._all_accounts) return self._all_accounts def get_account(self, account_id, account_type): key = (account_id, account_type) if key in self._all_accounts: return self._all_accounts[key] else: return None def add_account(self, account): profile = account.profile if (profile.user_id, account.TYPE) not in self._all_accounts: self._all_accounts[(profile.user_id, account.TYPE)] = account def insert_record(self, account): profile = account.profile params = (profile.user_id, account.TYPE, account.dump(), profile.dump()) self._cursor.execute( 'INSERT OR REPLACE INTO accounts (account_id, account_type, account_dump, profile_dump) ' 'VALUES (?, ?, ?, ?)', params) def delete_account(self, account): key = (account.profile.user_id, account.TYPE) del self._all_accounts[key] self._cursor.execute( 'DELETE FROM accounts WHERE account_id=? AND account_type=?', key) def close(self): # Write all data back to database self.logger.info('Writing account information back to storage...') for account_key, account in self._all_accounts.items(): self.insert_record(account) self._conn.commit() # Close database connection self.logger.info('Closing account storage...') self._cursor.close() self._conn.close() self.logger.info('Account storage was closed.')
class DriveStorage: logger = logger_factory.get_logger('DriveStorage') def __init__(self, db_path, account_store): """ :param str db_path: Path to Drive database. :param onedrive_d.store.account_db.AccountStorage account_store: """ self._conn = sqlite3.connect(db_path, isolation_level=None) self._cursor = self._conn.cursor() self._cursor.execute(get_content('onedrive_drives.sql')) self._conn.commit() self._all_drives = {} self.account_store = account_store self._drive_roots = { k: drives.DriveRoot(account) for k, account in account_store.get_all_accounts().items() } atexit.register(self.close) @staticmethod def _get_key(drive_id, account_id, account_type): return drive_id, account_id, account_type def assemble_drive_record(self, row, container): drive_id, account_id, account_type, drive_dump = row try: drive_root = self.get_drive_root(account_id, account_type) except KeyError: self.logger.warning( 'The %s account %s for drive %s was not registered.', account_type, account_id, drive_id) return try: drive = drives.DriveObject.load(drive_root, account_id, account_type, drive_dump) container[self._get_key(drive.drive_id, account_id, account_type)] = drive except ValueError as e: self.logger.warning('Cannot load drive %s from database: %s', drive_id, e) def get_drive_root(self, account_id, account_type): return self._drive_roots[(account_id, account_type)] def get_all_drives(self): self._conn.commit() q = self._cursor.execute( 'SELECT drive_id, account_id, account_type, drive_dump FROM drives' ) for row in q.fetchall(): self.assemble_drive_record(row, self._all_drives) return self._all_drives def add_record(self, drive): account = drive.root.account params = (drive.drive_id, account.profile.user_id, account.TYPE, drive.config.local_root, drive.dump()) self._cursor.execute( 'INSERT OR REPLACE INTO drives (drive_id, account_id, account_type, local_root, ' 'drive_dump) VALUES (?,?,?,?,?)', params) def close(self): self._conn.commit() self._cursor.close() self._conn.close()
class DriveObject: """ Abstracts a specific Drive resource. All items.OneDriveItem objects are generated by DriveObject API. """ VERSION_KEY = '@version' VERSION_VALUE = 0 logger = logger_factory.get_logger('DriveObject') def __init__(self, root, data, config): """ :param onedrive_d.api.drives.OneDriveRoot root: The parent root object. :param dict[str, str | int | dict] data: The deserialized Drive dictionary. :param onedrive_d.api.common.drive_config.DriveConfig config: Drive configuration. """ self._data = data self.config = config self.root = root self.is_default = root.account.profile.user_id.lower( ) == self.drive_id.lower() if self.is_default: self.drive_path = '/drive' else: self.drive_path = '/drives/' + data['id'] self.drive_uri = root.account.client.API_URI @property def drive_id(self): """ Return the drive ID. :rtype: str """ return self._data['id'] @property def type(self): """ Return a string representing the drive's type. {'personal', 'business'} :rtype: str """ return self._data['driveType'] @property def quota(self): """ :rtype: onedrive_d.api.facets.QuotaFacet """ return facets.QuotaFacet(self._data['quota']) def refresh(self): """ Refresh metadata of the drive object. """ self.root.purge_drive_cache(self.drive_id) new_drive = self.root.get_drive(self.drive_id) # noinspection PyProtectedMember self._data = new_drive._data del new_drive def get_item_uri(self, item_id=None, item_path=None): """ Generate URL to the specified item. If both item_id and item_path are None, return root item. :param str | None item_id: (Optional) ID of the specified item. :param str | None item_path: (Optional) Path to the specified item. :rtype: str """ uri = self.drive_uri if item_id is not None: uri += self.drive_path + '/items/' + item_id elif item_path is not None and item_path != self.drive_path + '/root:': uri += item_path else: uri += self.drive_path + '/root' return uri def get_root_dir(self, list_children=True): return self.get_item(None, None, list_children) def build_item(self, data): return items.OneDriveItem(self, data) def get_item(self, item_id=None, item_path=None, list_children=True): """ Retrieve the metadata of an item from OneDrive server. :param str | None item_id: ID of the item. Required if item_path is None. :param str | None item_path: Path to the item relative to drive root. Required if item_id is None. :rtype: onedrive_d.api.items.OneDriveItem """ uri = self.get_item_uri(item_id, item_path) if list_children: uri += '?expand=children' request = self.root.account.session.get(uri) return items.OneDriveItem(self, request.json()) def get_children(self, item_id=None, item_path=None): """ Assuming the target item is a directory, return a collection of all its children items. :param str | None item_id: (Optional) ID of the target directory. :param str | None item_path: (Optional) Path to the target directory. :rtype: onedrive_d.api.items.ItemCollection """ uri = self.get_item_uri(item_id, item_path) if item_path is not None: uri += ':' uri += '/children' request = self.root.account.session.get(uri) return items.ItemCollection(self, request.json()) def create_dir(self, name, parent_id=None, parent_path=None, conflict_behavior=options.NameConflictBehavior.DEFAULT): """ Create a new directory under the specified parent directory. :param str name: Name of the new directory. :param str | None parent_id: (Optional) ID of the parent directory item. :param str | None parent_path: (Optional) Path to the parent directory item. :param str conflict_behavior: (Optional) What to do if name exists. One value from options.nameConflictBehavior. :rtype: onedrive_d.api.items.OneDriveItem """ data = { 'name': name, 'folder': {}, '@name.conflictBehavior': conflict_behavior } uri = self.get_item_uri(parent_id, parent_path) + '/children' request = self.root.account.session.post( uri, json=data, ok_status_code=requests.codes.created) return items.OneDriveItem(self, request.json()) def upload_file(self, filename, data, size, parent_id=None, parent_path=None, conflict_behavior=options.NameConflictBehavior.REPLACE): """ Upload a file object to the specified parent directory, the method of which is determined by file size. :param str filename: Name of the remote file. :param file data: An opened file object available for reading. :param int size: Size of the content to upload. :param str | None parent_id: (Optional) ID of the parent directory. :param str | None parent_path: (Optional) Path to the parent directory. :param str conflict_behavior: (Optional) Specify the behavior to use if the file already exists. :rtype: onedrive_d.api.items.OneDriveItem """ if size <= self.config.max_put_size_bytes: return self.put_file(filename, data, parent_id, parent_path, conflict_behavior) else: return self.put_large_file(filename, data, size, parent_id, parent_path, conflict_behavior) def put_large_file(self, filename, data, size, parent_id=None, parent_path=None, conflict_behavior=options.NameConflictBehavior.REPLACE): """ Upload a large file by splitting it into fragments. https://github.com/OneDrive/onedrive-api-docs/blob/master/items/upload_large_files.md :param str filename: Name of the remote file. :param file data: An opened file object available for reading. :param int size: Size of the content to upload. :param str | None parent_id: (Optional) ID of the parent directory. :param str | None parent_path: (Optional) Path to the parent directory. :param str conflict_behavior: (Optional) Specify the behavior to use if the file already exists. :rtype: onedrive_d.api.items.OneDriveItem """ # Create an upload session. if parent_id is not None: parent_id += ':' uri = self.get_item_uri( parent_id, parent_path) + '/' + filename + ':/upload.createSession' payload = {'item': {'name': filename}} if conflict_behavior != options.NameConflictBehavior.REPLACE: payload['item']['@name.conflictBehavior'] = conflict_behavior size_str = str(size) request = self.root.account.session.post(uri, json=payload) current_session = resources.UploadSession(request.json()) # Upload content. expected_ranges = [ (0, size - 1) ] # Use local value rather than that given in session. while len(expected_ranges) > 0: # Ranges must come in order f, t = expected_ranges.pop(0) # Both inclusive if t is None or t >= size: t = size - 1 next_cursor = f + self.config.max_put_size_bytes if t >= next_cursor: expected_ranges.insert(0, (next_cursor, t)) t = next_cursor - 1 data.seek(f) chunk = data.read(t - f + 1) headers = {'Content-Range': str(f) + '-' + str(t) + '/' + size_str} request = self.root.account.session.put( current_session.upload_url, data=chunk, headers=headers, ok_status_code=requests.codes.accepted) current_session.update(request.json()) # TODO: handle timeout error # https://github.com/OneDrive/onedrive-api-docs/blob/master/items/upload_large_files.md#request-upload-status def put_file(self, filename, data, parent_id=None, parent_path=None, conflict_behavior=options.NameConflictBehavior.REPLACE): """ Use HTTP PUT to upload a file that is relatively small (less than 100M). :param str filename: Name of the remote file. :param file data: An opened file object available for reading. :param str | None parent_id: (Optional) ID of the parent directory. :param str | None parent_path: (Optional) Path to the parent directory. :param str conflict_behavior: (Optional) Specify the behavior to use if the file already exists. :rtype: onedrive_d.api.items.OneDriveItem """ if parent_id is not None: parent_id += ':' uri = self.get_item_uri(parent_id, parent_path) + '/' + filename + ':/content' if conflict_behavior != options.NameConflictBehavior.REPLACE: uri += '[email protected]=' + conflict_behavior request = self.root.account.session.put( uri, data=data, ok_status_code=requests.codes.created) return items.OneDriveItem(self, request.json()) def download_file(self, file, size, item_id=None, item_path=None): """ Download the target item to target file object. If the file is too large, download by fragments. :param file file: An open file object available for writing binary data. :param int size: Expected size of the item. :param str | None item_id: ID of the target file. :param str | None item_path: Path to the target file. """ if size <= self.config.max_get_size_bytes: self.get_file_content(item_id, item_path, file=file) return t = 0 while t < size: f = t t += self.config.max_get_size_bytes - 1 # Both inclusive. if t >= size: t = size - 1 self.get_file_content(item_id, item_path, range_bytes=(f, t), file=file) t += 1 def get_file_content(self, item_id=None, item_path=None, range_bytes=None, file=None): """ Get the content of an item. :param str | None item_id: ID of the target file. :param str | None item_path: Path to the target file. :param (int, int) | None range_bytes: Range of the bytes to download. :param file | None file: An opened file object. If set, write the content there. Otherwise return the content. :rtype: bytes """ uri = self.get_item_uri(item_id, item_path) + '/content' if range_bytes is None: headers = None ok_status_code = requests.codes.ok else: headers = {'Range': 'bytes=%d-%d' % range_bytes} ok_status_code = requests.codes.partial request = self.root.account.session.get(uri, headers=headers, ok_status_code=ok_status_code) if file is not None: file.write(request.content) else: return request.content def delete_item(self, item_id=None, item_path=None): """ https://github.com/OneDrive/onedrive-api-docs/blob/master/items/delete.md Delete the specified item on OneDrive server. :param str | None item_id: ID of the item. Required if item_path is None. :param str | None item_path: Path to the item relative to drive root. Required if item_id is None. """ uri = self.get_item_uri(item_id, item_path) self.root.account.session.delete( uri, ok_status_code=requests.codes.no_content) def update_item(self, item_id=None, item_path=None, new_name=None, new_description=None, new_parent_reference=None, new_file_system_info=None): """ Update the metadata of the specified item. :param str | None item_id: (Optional) ID of the target item. :param str | None item_path: (Optional) Path to the target item. :param str | None new_name: (Optional) If set, update the item metadata with the new name. :param str | None new_description: (Optional) If set, update the item metadata with the new description. :param onedrive_d.api.resources.ItemReference | None new_parent_reference: (Optional) If set, move the item. :param onedrive_d.api.facets.FileSystemInfoFacet | None new_file_system_info: (Optional) If set, update the client-wise timestamps. :rtype: onedrive_d.api.items.OneDriveItem """ if item_id is None and item_path is None: raise ValueError('Root is immutable. A specific item is required.') data = {} if new_name is not None: data['name'] = new_name if new_description is not None: data['description'] = new_description if new_parent_reference is not None: data['parentReference'] = new_parent_reference.data if new_file_system_info is not None: data['fileSystemInfo'] = new_file_system_info.data if len(data) == 0: raise ValueError('Nothing is to change.') uri = self.get_item_uri(item_id, item_path) request = self.root.account.session.patch(uri, data) return items.OneDriveItem(self, request.json()) def copy_item(self, dest_reference, item_id=None, item_path=None, new_name=None): """ Copy an item (including any children) on OneDrive under a new parent. :param onedrive_d.api.resources.ItemReference dest_reference: Reference to new parent. :param str | None item_id: (Optional) ID of the source item. Required if item_path is None. :param str | None item_path: (Optional) Path to the source item. Required if item_id is None. :param str | None new_name: (Optional) If set, use this name for the copied item. :rtype: onedrive_d.api.resources.AsyncCopySession """ if not isinstance(dest_reference, resources.ItemReference): raise ValueError('Destination should be an ItemReference object.') if item_id is None and item_path is None: raise ValueError('Source of copy must be specified.') uri = self.get_item_uri(item_id, item_path) if item_path is not None: uri += ':' uri += '/action.copy' data = {'parentReference': dest_reference.data} if new_name is not None: data['name'] = new_name headers = {'Prefer': 'respond-async'} request = self.root.account.session.post(uri, json=data, headers=headers) return resources.AsyncCopySession(self, request.headers) def get_thumbnail(self): raise NotImplementedError('The API feature is not used yet.') def search(self, keyword, select=None, item_id=None, item_path=None): """ Use a keyword to search for items within the specified directory (default: root). :param str keyword: Keyword for the search. :param [str] | None select: Only fetch the specified fields. :param str | None item_id: (Optional) ID of the item to search within. :param str | None item_path: (Optional) Path to the item to search within. :return onedrive_d.api.items.ItemCollection: """ params = {'q': keyword} if select is not None: params['select'] = ','.join(select) uri = self.get_item_uri(item_id, item_path) + '/view.search' request = self.root.account.session.get(uri, params=params) return items.ItemCollection(self, request.json()) def get_changes(self): raise NotImplementedError('The API feature is not used yet.') def get_special_dir(self, name): raise NotImplementedError('The API feature is not used yet.') def dump(self): data = { 'config_dump': self.config.dump(), 'data': self._data, self.VERSION_KEY: self.VERSION_VALUE } return json.dumps(data) @classmethod def load(cls, drive_root, account_id, account_type, s): data = json.loads(s) drive = DriveObject(drive_root, data['data'], drive_config.DriveConfig.load(data['config_dump'])) try: drive_root.add_cached_drive(account_id, account_type, drive) except ValueError as e: cls.logger.warning( 'Faild to register deserialized drive %s to drive root: %s', drive.drive_id, e) return drive
def mock_loggers(): logger_factory.init_logger(min_level=logging.CRITICAL) for cls in [drives.DriveObject, account_db.AccountStorage, account_db.AccountStorage, drives_db.DriveStorage, drive_config.DriveConfig]: cls.logger = logger_factory.get_logger('ut_' + cls.__name__)
class DriveConfig: DEFAULT_VALUES = { 'max_get_size_bytes': 1048576, 'max_put_size_bytes': 524288, 'local_root': None, 'ignore_files': set(), } logger = logger_factory.get_logger('DriveConfig') def __init__(self, data): for k, v in self.DEFAULT_VALUES.items(): if k not in data: data[k] = v if isinstance(data['ignore_files'], list): data['ignore_files'] = set(data['ignore_files']) for item in self.DEFAULT_VALUES['ignore_files']: if item not in data['ignore_files']: data['ignore_files'].add(item) self.data = data @staticmethod def default_config(): return DriveConfig(deepcopy(DriveConfig.DEFAULT_VALUES)) @classmethod def set_default_config(cls, config): """ Set the new config as default, with side-effect of updating all existing configs that use (unsaved) default values. :param onedrive_d.api.drives.DriveConfig config: """ for k, v in cls.DEFAULT_VALUES.items(): v2 = getattr(config, k) if v2 != v: cls.DEFAULT_VALUES[k] = v2 @property def max_get_size_bytes(self): """ :rtype: int """ return self.data['max_get_size_bytes'] @property def max_put_size_bytes(self): """ :rtype: int """ return self.data['max_put_size_bytes'] @property def local_root(self): """ :rtype: str """ return self.data['local_root'] @property def ignore_files(self): """ :rtype: [str] """ return self.data['ignore_files'] # noinspection PyAttributeOutsideInit @property def path_filter(self): if not hasattr(self, '_path_filter'): rules = set() for path in self.ignore_files: try: with open(path, 'r') as f: rules.update(f.read().splitlines()) except Exception as e: self.logger.error('Failed to load ignore list "%s": %s', path, e) self._path_filter = path_filter.PathFilter(rules) return self._path_filter def dump(self, exact_dump=False): data = {} for key in ['max_get_size_bytes', 'max_put_size_bytes', 'local_root']: if exact_dump or getattr(self, key) != self.DEFAULT_VALUES[key]: data[key] = getattr(self, key) ignore_files = [ s for s in self.ignore_files if exact_dump or s not in self.DEFAULT_VALUES['ignore_files'] ] if len(ignore_files) > 0: data['ignore_files'] = ignore_files return data @classmethod def load(cls, d): return DriveConfig(d)
class TaskPool: """ An in-memory, singleton storage for Tasks based on hash maps. """ logger = logger_factory.get_logger('TaskPool') _instance = None @classmethod def get_instance(cls): if cls._instance is None: cls._semaphore = threading.Semaphore(0) cls._lock = rwlock.RWLock() cls._instance = TaskPool() return cls._instance def __init__(self): self._all_tasks = [] self._tasks_by_path = {} def _add_to_list(self, key, table, value): """ Add an item to a dict whose values are lists. :param str key: :param dict[str, T] table: :param T value: """ if key not in table: table[key] = [value] else: table[key].append(value) def get_task_path(self, task): """ Return the local path the task performs on. :param onedrive_d.common.tasks.LocalParentPathMixin task: :return str: """ return task.local_parent_path + '/' + task.name def add_task(self, task): self.logger.debug('Try acquiring writer lock...') self._lock.writer_acquire() self._all_tasks.append(task) self._add_to_list(self.get_task_path(task), self._tasks_by_path, task) self.logger.debug('Added task "%s" on path "%s".', task.__class__.__name__, task.local_parent_path + '/' + task.name) self._lock.writer_release() self.logger.debug('Writer lock released.') self._semaphore.release() @property def semaphore(self): return self._semaphore def pop_task(self, task_class=None): self._lock.writer_acquire() ret = None if len(self._all_tasks) > 0: if task_class is None: ret = self._all_tasks.pop(0) else: for t in self._all_tasks: if isinstance(t, task_class): ret = t self._all_tasks.remove(t) break if ret is not None: self._tasks_by_path[self.get_task_path(ret)].remove(ret) self._lock.writer_release() return ret def has_pending_task(self, local_path): self._lock.reader_acquire() ret = local_path in self._tasks_by_path and len( self._tasks_by_path[local_path]) > 0 self.logger.debug('Item "%s" has pending task? %s.', local_path, str(ret)) self._lock.reader_release() return ret def remove_children_tasks(self, local_parent_path): local_parent_path += '/' self._lock.writer_acquire() for t in self._all_tasks: task_path = self.get_task_path(t) if task_path.startswith(local_parent_path): self._all_tasks.remove(t) self._tasks_by_path[task_path].remove(t) self._lock.writer_release()
class ManagedRESTClient: AUTO_RETRY_SECONDS = 30 RECOVERABLE_STATUS_CODES = {requests.codes.too_many, 500, 502, 503, 504} logger = logger_factory.get_logger(__name__) def __init__(self, session, net_mon, account, proxies=None): """ :param session: Dictate a requests Session object. :param onedrive_d.common.netman.NetworkMonitor net_mon: Network monitor instance. :param onedrive_d.api.accounts.PersonalAccount | onedrive_d.api.accounts.BusinessAccount account: Account. :param dict[str, str] proxies: (Optional) A dictionary of protocol-host pairs. :return: No return value. """ self.session = session self.net_mon = net_mon self.account = account self.proxies = proxies def request(self, method, url, params, ok_status_code, auto_renew): """ Perform a HTTP request call. Do auto-recover as fits. :param str method: One of {GET, POST, PATCH, PUT, DELETE}. :param str url: URL of the HTTP request. :param dict[str, str | dict | bytes] params: Params to send to the request call. :param int ok_status_code: Expected status code for HTTP response. :param True | False auto_renew: If True, auto recover the expired token. :rtype: requests.Response :raise errors.OneDriveError: """ while True: try: request = getattr(self.session, method)(url, **params) bad_status = request.status_code != ok_status_code if isinstance(ok_status_code, int) \ else request.status_code not in ok_status_code if bad_status: if request.status_code in self.RECOVERABLE_STATUS_CODES: if 'Retry-After' in request.headers: retry_after_seconds = int( request.headers['Retry-After']) else: retry_after_seconds = self.AUTO_RETRY_SECONDS self.logger.info( 'Server returned code %d which is assumed recoverable. Retry in %d seconds', request.status_code, retry_after_seconds) raise errors.OneDriveRecoverableError( retry_after_seconds) raise errors.OneDriveError(request.json()) return request except requests.ConnectionError: self.net_mon.suspend_caller() except errors.OneDriveRecoverableError as e: time.sleep(e.retry_after_seconds) except errors.OneDriveTokenExpiredError as e: if auto_renew: self.logger.info('Access token expired. Try refreshing...') self.account.renew_tokens() else: raise e def get(self, url, params=None, headers=None, ok_status_code=requests.codes.ok, auto_renew=True): """ Perform a HTTP GET request. :param str url: URL of the HTTP request. :param dict[str, T] | None params: (Optional) Dictionary to construct query string. :param dict | None headers: (Optional) Additional headers for the HTTP request. :param int ok_status_code: (Optional) Expected status code for the HTTP response. :param True | False auto_renew: (Optional) If True, auto recover from expired token error or Internet failure. :rtype: requests.Response """ args = {'proxies': self.proxies} if params is not None: args['params'] = params if headers is not None: args['headers'] = headers return self.request('get', url, args, ok_status_code=ok_status_code, auto_renew=auto_renew) def post(self, url, data=None, json=None, headers=None, ok_status_code=requests.codes.ok, auto_renew=True): """ Perform a HTTP POST request. :param str url: URL of the HTTP request. :param dict | None data: (Optional) Data in POST body of the request. :param dict | None json: (Optional) Send the dictionary as JSON content in POST body and set proper headers. :param dict | None headers: (Optional) Additional headers for the HTTP request. :param int ok_status_code: (Optional) Expected status code for the HTTP response. :param True | False auto_renew: (Optional) If True, auto recover from expired token error or Internet failure. :rtype: requests.Response """ params = {'proxies': self.proxies} if json is not None: params['json'] = json else: params['data'] = data if headers is not None: params['headers'] = headers return self.request('post', url, params, ok_status_code=ok_status_code, auto_renew=auto_renew) def patch(self, url, json, ok_status_code=requests.codes.ok, auto_renew=True): """ Perform a HTTP PATCH request. :param str url: URL of the HTTP request. :param dict json: Send the dictionary as JSON content in POST body and set proper headers. :param int ok_status_code: (Optional) Expected status code for the HTTP response. :param True | False auto_renew: (Optional) If True, auto recover from expired token error or Internet failure. :rtype: requests.Response """ params = {'proxies': self.proxies, 'json': json} return self.request('patch', url, params, ok_status_code=ok_status_code, auto_renew=auto_renew) def put(self, url, data, headers=None, ok_status_code=requests.codes.ok, auto_renew=True): """ Perform a HTTP PUT request. :param str url: URL of the HTTP request. :param bytes | None data: Binary data to send in the request body. :param dict | None headers: Additional headers for the HTTP request. :param int ok_status_code: (Optional) Expected status code for the HTTP response. :param True | False auto_renew: (Optional) If True, auto recover from expired token error or Internet failure. :rtype: requests.Response """ params = {'proxies': self.proxies, 'data': data} if headers is not None: params['headers'] = headers return self.request('put', url, params=params, ok_status_code=ok_status_code, auto_renew=auto_renew) def delete(self, url, ok_status_code=requests.codes.ok, auto_renew=True): """ Perform a HTTP DELETE request on the specified URL. :param str url: URL of the HTTP request. :param int ok_status_code: (Optional) Expected status code for the HTTP response. :param True | False auto_renew: (Optional) If True, auto recover from expired token error or Internet failure. :rtype: requests.Response """ return self.request('delete', url, {'proxies': self.proxies}, ok_status_code=ok_status_code, auto_renew=auto_renew)