class TaskMixin:
logger = logger_factory.get_logger('Tasks')
def __init__(self, task_base=None, drive=None, items_store=None, task_pool=None):
self.drive = drive if task_base is None else task_base.drive
self.items_store = items_store if task_base is None else task_base.items_store
self.task_pool = task_pool if task_base is None else task_base.task_pool
@property
def drive(self):
return self._drive
@drive.setter
def drive(self, d):
self._drive = d
@property
def items_store(self):
return self._items_store
@items_store.setter
def items_store(self, s):
self._items_store = s
@property
def task_pool(self):
return self._task_pool
@task_pool.setter
def task_pool(self, p):
self._task_pool = p
def mock_loggers():
for cls in [
drives.DriveObject, account_db.AccountStorage,
account_db.AccountStorage, drives_db.DriveStorage,
drive_config.DriveConfig
]:
cls.logger = logger_factory.get_logger('ut_' + cls.__name__,
min_level=logging.CRITICAL)
def main():
global logger
args = fix_log_args(parse_args())
logger = logger_factory.get_logger('Main')
check_config_dir()
load_user_config()
load_item_storage()
load_task_storage()
start_task_workers()
refill_tasks()
def main():
global logger
args = fix_log_args(parse_args())
logger = logger_factory.get_logger('Main')
check_config_dir()
load_user_config()
load_item_storage()
load_task_storage()
start_task_workers()
refill_tasks()
class NetworkMonitor(threading.Thread):
THREAD_NAME = "netmon"
logger = logger_factory.get_logger(__name__)
def __init__(self, test_uri='https://onedrive.com', retry_delay_sec=30, proxies=None):
"""
:param str test_uri: The url to use in testing internet connectivity.
:param int retry_delay_sec: The amount of seconds to wait before retry.
:param dict[str, str] proxies: A dict of protocol-url pairs.
"""
super().__init__()
self.name = NetworkMonitor.THREAD_NAME
self.daemon = True
self.test_uri = test_uri
self.retry_delay = retry_delay_sec
self.proxies = proxies
self.queue = queue.Queue()
self.conditions = {}
self.logger.info("Initialized.")
def suspend_caller(self):
"""Put the calling thread into suspension queue."""
me = threading.current_thread()
cond = self.conditions[me.ident] = threading.Condition()
self.queue.put(me)
self.logger.info("Suspended due to network failure.")
cond.acquire()
# Put the caller thread to sleep
cond.wait()
# Thread is waken up by manager
cond.release()
del self.conditions[me.ident]
self.logger.info("Resumed.")
def is_connected(self):
"""
Test if internet connection is OK by connecting to the test URI provided.
If proxy setting is set in the client, it will be used as well.
:return: True if internet connection is on; False otherwise.
"""
try:
requests.head(self.test_uri, proxies=self.proxies)
return True
except requests.ConnectionError:
return False
def run(self):
while True:
th = self.queue.get() # blocking call
while not self.is_connected():
time.sleep(self.retry_delay)
self.conditions[th.ident].acquire()
self.conditions[th.ident].notify()
self.conditions[th.ident].release()
class TaskConsumer(threading.Thread):
terminate_sign = threading.Event()
logger = logger_factory.get_logger('TaskConsumer')
def __init__(self, task_pool):
"""
:param onedrive_d.store.task_pool.TaskPool task_pool:
"""
super().__init__()
self.daemon = True
self.task_pool = task_pool
def run(self):
self.logger.debug('Started.')
while True:
self.task_pool.semaphore.acquire()
if self.terminate_sign.is_set():
break
task = self.task_pool.pop_task()
self.logger.debug('Acquired task of type "%s" on path "%s"',
task.__class__.__name__,
task.local_parent_path + '/' + task.name)
task.handle()
self.logger.debug('Stopped.')
class ItemStorage:
"""
Local storage for items under ONE drive.
"""
logger = logger_factory.get_logger('ItemStorage')
def __init__(self, db_path, drive):
"""
:param str db_path: A unique path for the database to store items for the target drive.
:param onedrive_d.api.drives.DriveObject drive: The underlying drive object.
"""
if not hasattr(drive, 'storage_lock'):
drive.storage_lock = rwlock.RWLock()
self.lock = drive.storage_lock
self._conn = sqlite3.connect(db_path, isolation_level=None, check_same_thread=False)
self.drive = drive
self._cursor = self._conn.cursor()
self._cursor.execute(get_content('onedrive_items.sql'))
self._conn.commit()
atexit.register(self.close)
def close(self):
self._cursor.close()
self._conn.close()
def local_path_to_remote_path(self, local_path):
return self.drive.drive_path + '/root:' + local_path
def get_items_by_id(self, item_id=None, parent_path=None, item_name=None, local_parent_path=None):
"""
FInd all qualified records from database by ID or path.
:param str item_id: ID of the target item.
:param str parent_path: Path reference of the target item's parent. Used with item_name.
:param str item_name: Name of the item. Used with parent_path or local_parent_path.
:param str local_parent_path: Path relative to drive's local root. If at root, use ''.
:return dict[str, onedrive_d.store.items_db.ItemRecord]: All qualified records index by item ID.
"""
if local_parent_path is not None:
parent_path = self.local_path_to_remote_path(local_parent_path)
args = {'item_id': item_id, 'parent_path': parent_path, 'item_name': item_name}
return self.get_items(args)
def get_items_by_hash(self, crc32_hash=None, sha1_hash=None):
"""
Find all qualified records from database whose hash values match either parameter.
:param str crc32_hash: CRC32 hash of the target item.
:param str sha1_hash: SHA-1 hash of the target item.
:return dict[str, onedrive_d.store.items_db.ItemRecord]: All qualified records index by item ID.
"""
args = {'crc32_hash': crc32_hash, 'sha1_hash': sha1_hash}
return self.get_items(args, 'OR')
@staticmethod
def _get_where_clause(args, relation='AND'):
"""
Form a where clause in SQL query and the tuples for the filler values.
:param dict[str, str | int]] args: Keys are where conditions and values are the filler values.
:param str relation: Either 'AND' or 'OR'.
:return (str, ()):
"""
keys = []
values = []
for k, v in args.items():
if v is not None:
keys.append(k + '=?')
values.append(v)
relation = ' ' + relation + ' '
return relation.join(keys), tuple(values)
def get_items(self, args, relation='AND'):
"""
:param dict[str, int | str] args: Criteria used to construct SQL query.
:param str relation: Relation of the criteria.
:return dict[str, onedrive_d.store.items_db.ItemRecord]: All matching rows in the form of ItemRecord.
"""
where, values = self._get_where_clause(args, relation)
ret = {}
self.lock.reader_acquire()
q = self._cursor.execute('SELECT item_id, type, item_name, parent_id, parent_path, etag, ctag, size, '
'created_time, modified_time, status, crc32_hash, sha1_hash FROM items WHERE ' +
where, values)
for row in q.fetchall():
item = ItemRecord(row)
ret[item.item_id] = item
self.lock.reader_release()
return ret
def update_item(self, item, status=ItemRecordStatuses.OK):
"""
:param onedrive_d.api.items.OneDriveItem item:
:param str status:
:return:
"""
if item.is_folder:
crc32_hash = None
sha1_hash = None
else:
file_facet = item.file_props
crc32_hash = file_facet.hashes.crc32
sha1_hash = file_facet.hashes.sha1
parent_ref = item.parent_reference
created_time_str = datetime_to_str(item.created_time)
modified_time_str = datetime_to_str(item.modified_time)
self.lock.writer_acquire()
self._cursor.execute(
'INSERT OR REPLACE INTO items (item_id, type, item_name, parent_id, parent_path, etag, '
'ctag, size, created_time, modified_time, status, crc32_hash, sha1_hash)'
' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
(item.id, item.type, item.name, parent_ref.id, parent_ref.path, item.e_tag, item.c_tag,
item.size, created_time_str, modified_time_str, status, crc32_hash, sha1_hash))
self._conn.commit()
self.lock.writer_release()
def delete_item(self, item_id=None, parent_path=None, item_name=None, local_parent_path=None, is_folder=False):
"""
Delete the specified item from database. If the item is a directory, then also delete all its children items.
:param str item_id: ID of the target item.
:param str parent_path: Path reference of the target item's parent. Used with item_name.
:param str item_name: Name of the item. Used with parent_path or local_parent_path.
:param str local_parent_path: Path relative to drive's local root. If at root, use ''.
:param True | False is_folder: True to indicate that the item is a folder (delete all children).
"""
if local_parent_path is not None:
parent_path = self.local_path_to_remote_path(local_parent_path)
where, values = self._get_where_clause({'item_id': item_id, 'parent_path': parent_path, 'item_name': item_name})
self.lock.writer_acquire()
if is_folder:
# Translate ID reference to path and name reference.
q = self._cursor.execute('SELECT item_id, parent_path, item_name FROM items WHERE ' + where, values)
row = q.fetchone()
if row is None:
self.logger.warning('The folder to delete does not exist: %s, %s', where, str(values))
else:
item_id, parent_path, item_name = row
self._cursor.execute('DELETE FROM items WHERE parent_id=? OR parent_path LIKE ?',
(item_id, parent_path + '/' + item_name + '/%'))
self._cursor.execute('DELETE FROM items WHERE ' + where, values)
self._conn.commit()
self.lock.writer_release()
def update_status(self, status, item_id=None, parent_path=None, item_name=None, local_parent_path=None):
"""
Update the status tag of the target item.
:param str item_id: ID of the target item.
:param str parent_path: Path reference of the target item's parent. Used with item_name.
:param str item_name: Name of the item. Used with parent_path or local_parent_path.
:param str local_parent_path: Path relative to drive's local root. If at root, use ''.
"""
if local_parent_path is not None:
parent_path = self.local_path_to_remote_path(local_parent_path)
where, values = self._get_where_clause({'item_id': item_id, 'parent_path': parent_path, 'item_name': item_name})
values = (status,) + values
self.lock.writer_acquire()
self._cursor.execute('UPDATE items SET status=? WHERE ' + where, values)
self._conn.commit()
self.lock.writer_release()
class AccountStorage:
"""
A SQLite-based storage layer for account and drive objects.
"""
logger = logger_factory.get_logger('AccountStorage')
def __init__(self, db_path, personal_client=None, business_client=None):
self._conn = sqlite3.connect(db_path, isolation_level=None)
self._cursor = self._conn.cursor()
self._cursor.execute(get_content('onedrive_accounts.sql'))
self._conn.commit()
self.personal_client = personal_client
self.business_client = business_client
self._all_accounts = {}
atexit.register(self.close)
def parse_account_type(self, account_type):
"""
:param str account_type:
:return onedrive_d.api.accounts.PersonalAccount | onedrive_d.accounts.accounts.BusinessAccount:
"""
if account_type == accounts.AccountTypes.PERSONAL:
return accounts.PersonalAccount, self.personal_client
else:
return accounts.BusinessAccount, self.business_client
def deserialize_account_row(self, account_id, account_type, account_dump,
profile_dump, container):
account_cls, client = self.parse_account_type(account_type)
if client is None:
self.logger.warning(
'Account %s was not loaded since no client of that type provided.',
account_id)
return
try:
account = account_cls.load(client, account_dump)
container[(account_id, account_type)] = account
except ValueError as e:
self.logger.warning('Failed to deserialize account %s: %s',
account_id, e)
else:
try:
profile = resources.UserProfile.load(profile_dump)
except ValueError as e:
self.logger.warning(
'Failed to deserialize user profile for account %s: %s',
account_id, e)
profile = account.profile
account.profile = profile
def get_all_accounts(self):
self._conn.commit()
q = self._cursor.execute(
'SELECT account_id, account_type, account_dump, profile_dump FROM accounts'
)
for row in q.fetchall():
account_id, account_type, account_dump, profile_dump = row
self.deserialize_account_row(account_id, account_type,
account_dump, profile_dump,
self._all_accounts)
return self._all_accounts
def get_account(self, account_id, account_type):
key = (account_id, account_type)
if key in self._all_accounts:
return self._all_accounts[key]
else:
return None
def add_account(self, account):
profile = account.profile
if (profile.user_id, account.TYPE) not in self._all_accounts:
self._all_accounts[(profile.user_id, account.TYPE)] = account
def insert_record(self, account):
profile = account.profile
params = (profile.user_id, account.TYPE, account.dump(),
profile.dump())
self._cursor.execute(
'INSERT OR REPLACE INTO accounts (account_id, account_type, account_dump, profile_dump) '
'VALUES (?, ?, ?, ?)', params)
def delete_account(self, account):
key = (account.profile.user_id, account.TYPE)
del self._all_accounts[key]
self._cursor.execute(
'DELETE FROM accounts WHERE account_id=? AND account_type=?', key)
def close(self):
# Write all data back to database
self.logger.info('Writing account information back to storage...')
for account_key, account in self._all_accounts.items():
self.insert_record(account)
self._conn.commit()
# Close database connection
self.logger.info('Closing account storage...')
self._cursor.close()
self._conn.close()
self.logger.info('Account storage was closed.')
class DriveStorage:
logger = logger_factory.get_logger('DriveStorage')
def __init__(self, db_path, account_store):
"""
:param str db_path: Path to Drive database.
:param onedrive_d.store.account_db.AccountStorage account_store:
"""
self._conn = sqlite3.connect(db_path, isolation_level=None)
self._cursor = self._conn.cursor()
self._cursor.execute(get_content('onedrive_drives.sql'))
self._conn.commit()
self._all_drives = {}
self.account_store = account_store
self._drive_roots = {
k: drives.DriveRoot(account)
for k, account in account_store.get_all_accounts().items()
}
atexit.register(self.close)
@staticmethod
def _get_key(drive_id, account_id, account_type):
return drive_id, account_id, account_type
def assemble_drive_record(self, row, container):
drive_id, account_id, account_type, drive_dump = row
try:
drive_root = self.get_drive_root(account_id, account_type)
except KeyError:
self.logger.warning(
'The %s account %s for drive %s was not registered.',
account_type, account_id, drive_id)
return
try:
drive = drives.DriveObject.load(drive_root, account_id,
account_type, drive_dump)
container[self._get_key(drive.drive_id, account_id,
account_type)] = drive
except ValueError as e:
self.logger.warning('Cannot load drive %s from database: %s',
drive_id, e)
def get_drive_root(self, account_id, account_type):
return self._drive_roots[(account_id, account_type)]
def get_all_drives(self):
self._conn.commit()
q = self._cursor.execute(
'SELECT drive_id, account_id, account_type, drive_dump FROM drives'
)
for row in q.fetchall():
self.assemble_drive_record(row, self._all_drives)
return self._all_drives
def add_record(self, drive):
account = drive.root.account
params = (drive.drive_id, account.profile.user_id, account.TYPE,
drive.config.local_root, drive.dump())
self._cursor.execute(
'INSERT OR REPLACE INTO drives (drive_id, account_id, account_type, local_root, '
'drive_dump) VALUES (?,?,?,?,?)', params)
def close(self):
self._conn.commit()
self._cursor.close()
self._conn.close()
class DriveObject:
"""
Abstracts a specific Drive resource. All items.OneDriveItem objects are generated by DriveObject API.
"""
VERSION_KEY = '@version'
VERSION_VALUE = 0
logger = logger_factory.get_logger('DriveObject')
def __init__(self, root, data, config):
"""
:param onedrive_d.api.drives.OneDriveRoot root: The parent root object.
:param dict[str, str | int | dict] data: The deserialized Drive dictionary.
:param onedrive_d.api.common.drive_config.DriveConfig config: Drive configuration.
"""
self._data = data
self.config = config
self.root = root
self.is_default = root.account.profile.user_id.lower(
) == self.drive_id.lower()
if self.is_default:
self.drive_path = '/drive'
else:
self.drive_path = '/drives/' + data['id']
self.drive_uri = root.account.client.API_URI
@property
def drive_id(self):
"""
Return the drive ID.
:rtype: str
"""
return self._data['id']
@property
def type(self):
"""
Return a string representing the drive's type. {'personal', 'business'}
:rtype: str
"""
return self._data['driveType']
@property
def quota(self):
"""
:rtype: onedrive_d.api.facets.QuotaFacet
"""
return facets.QuotaFacet(self._data['quota'])
def refresh(self):
"""
Refresh metadata of the drive object.
"""
self.root.purge_drive_cache(self.drive_id)
new_drive = self.root.get_drive(self.drive_id)
# noinspection PyProtectedMember
self._data = new_drive._data
del new_drive
def get_item_uri(self, item_id=None, item_path=None):
"""
Generate URL to the specified item. If both item_id and item_path are None, return root item.
:param str | None item_id: (Optional) ID of the specified item.
:param str | None item_path: (Optional) Path to the specified item.
:rtype: str
"""
uri = self.drive_uri
if item_id is not None:
uri += self.drive_path + '/items/' + item_id
elif item_path is not None and item_path != self.drive_path + '/root:':
uri += item_path
else:
uri += self.drive_path + '/root'
return uri
def get_root_dir(self, list_children=True):
return self.get_item(None, None, list_children)
def build_item(self, data):
return items.OneDriveItem(self, data)
def get_item(self, item_id=None, item_path=None, list_children=True):
"""
Retrieve the metadata of an item from OneDrive server.
:param str | None item_id: ID of the item. Required if item_path is None.
:param str | None item_path: Path to the item relative to drive root. Required if item_id is None.
:rtype: onedrive_d.api.items.OneDriveItem
"""
uri = self.get_item_uri(item_id, item_path)
if list_children:
uri += '?expand=children'
request = self.root.account.session.get(uri)
return items.OneDriveItem(self, request.json())
def get_children(self, item_id=None, item_path=None):
"""
Assuming the target item is a directory, return a collection of all its children items.
:param str | None item_id: (Optional) ID of the target directory.
:param str | None item_path: (Optional) Path to the target directory.
:rtype: onedrive_d.api.items.ItemCollection
"""
uri = self.get_item_uri(item_id, item_path)
if item_path is not None:
uri += ':'
uri += '/children'
request = self.root.account.session.get(uri)
return items.ItemCollection(self, request.json())
def create_dir(self,
name,
parent_id=None,
parent_path=None,
conflict_behavior=options.NameConflictBehavior.DEFAULT):
"""
Create a new directory under the specified parent directory.
:param str name: Name of the new directory.
:param str | None parent_id: (Optional) ID of the parent directory item.
:param str | None parent_path: (Optional) Path to the parent directory item.
:param str conflict_behavior: (Optional) What to do if name exists. One value from options.nameConflictBehavior.
:rtype: onedrive_d.api.items.OneDriveItem
"""
data = {
'name': name,
'folder': {},
'@name.conflictBehavior': conflict_behavior
}
uri = self.get_item_uri(parent_id, parent_path) + '/children'
request = self.root.account.session.post(
uri, json=data, ok_status_code=requests.codes.created)
return items.OneDriveItem(self, request.json())
def upload_file(self,
filename,
data,
size,
parent_id=None,
parent_path=None,
conflict_behavior=options.NameConflictBehavior.REPLACE):
"""
Upload a file object to the specified parent directory, the method of which is determined by file size.
:param str filename: Name of the remote file.
:param file data: An opened file object available for reading.
:param int size: Size of the content to upload.
:param str | None parent_id: (Optional) ID of the parent directory.
:param str | None parent_path: (Optional) Path to the parent directory.
:param str conflict_behavior: (Optional) Specify the behavior to use if the file already exists.
:rtype: onedrive_d.api.items.OneDriveItem
"""
if size <= self.config.max_put_size_bytes:
return self.put_file(filename, data, parent_id, parent_path,
conflict_behavior)
else:
return self.put_large_file(filename, data, size, parent_id,
parent_path, conflict_behavior)
def put_large_file(self,
filename,
data,
size,
parent_id=None,
parent_path=None,
conflict_behavior=options.NameConflictBehavior.REPLACE):
"""
Upload a large file by splitting it into fragments.
https://github.com/OneDrive/onedrive-api-docs/blob/master/items/upload_large_files.md
:param str filename: Name of the remote file.
:param file data: An opened file object available for reading.
:param int size: Size of the content to upload.
:param str | None parent_id: (Optional) ID of the parent directory.
:param str | None parent_path: (Optional) Path to the parent directory.
:param str conflict_behavior: (Optional) Specify the behavior to use if the file already exists.
:rtype: onedrive_d.api.items.OneDriveItem
"""
# Create an upload session.
if parent_id is not None:
parent_id += ':'
uri = self.get_item_uri(
parent_id, parent_path) + '/' + filename + ':/upload.createSession'
payload = {'item': {'name': filename}}
if conflict_behavior != options.NameConflictBehavior.REPLACE:
payload['item']['@name.conflictBehavior'] = conflict_behavior
size_str = str(size)
request = self.root.account.session.post(uri, json=payload)
current_session = resources.UploadSession(request.json())
# Upload content.
expected_ranges = [
(0, size - 1)
] # Use local value rather than that given in session.
while len(expected_ranges) > 0: # Ranges must come in order
f, t = expected_ranges.pop(0) # Both inclusive
if t is None or t >= size:
t = size - 1
next_cursor = f + self.config.max_put_size_bytes
if t >= next_cursor:
expected_ranges.insert(0, (next_cursor, t))
t = next_cursor - 1
data.seek(f)
chunk = data.read(t - f + 1)
headers = {'Content-Range': str(f) + '-' + str(t) + '/' + size_str}
request = self.root.account.session.put(
current_session.upload_url,
data=chunk,
headers=headers,
ok_status_code=requests.codes.accepted)
current_session.update(request.json())
# TODO: handle timeout error
# https://github.com/OneDrive/onedrive-api-docs/blob/master/items/upload_large_files.md#request-upload-status
def put_file(self,
filename,
data,
parent_id=None,
parent_path=None,
conflict_behavior=options.NameConflictBehavior.REPLACE):
"""
Use HTTP PUT to upload a file that is relatively small (less than 100M).
:param str filename: Name of the remote file.
:param file data: An opened file object available for reading.
:param str | None parent_id: (Optional) ID of the parent directory.
:param str | None parent_path: (Optional) Path to the parent directory.
:param str conflict_behavior: (Optional) Specify the behavior to use if the file already exists.
:rtype: onedrive_d.api.items.OneDriveItem
"""
if parent_id is not None:
parent_id += ':'
uri = self.get_item_uri(parent_id,
parent_path) + '/' + filename + ':/content'
if conflict_behavior != options.NameConflictBehavior.REPLACE:
uri += '[email protected]=' + conflict_behavior
request = self.root.account.session.put(
uri, data=data, ok_status_code=requests.codes.created)
return items.OneDriveItem(self, request.json())
def download_file(self, file, size, item_id=None, item_path=None):
"""
Download the target item to target file object. If the file is too large, download by fragments.
:param file file: An open file object available for writing binary data.
:param int size: Expected size of the item.
:param str | None item_id: ID of the target file.
:param str | None item_path: Path to the target file.
"""
if size <= self.config.max_get_size_bytes:
self.get_file_content(item_id, item_path, file=file)
return
t = 0
while t < size:
f = t
t += self.config.max_get_size_bytes - 1 # Both inclusive.
if t >= size:
t = size - 1
self.get_file_content(item_id,
item_path,
range_bytes=(f, t),
file=file)
t += 1
def get_file_content(self,
item_id=None,
item_path=None,
range_bytes=None,
file=None):
"""
Get the content of an item.
:param str | None item_id: ID of the target file.
:param str | None item_path: Path to the target file.
:param (int, int) | None range_bytes: Range of the bytes to download.
:param file | None file: An opened file object. If set, write the content there. Otherwise return the content.
:rtype: bytes
"""
uri = self.get_item_uri(item_id, item_path) + '/content'
if range_bytes is None:
headers = None
ok_status_code = requests.codes.ok
else:
headers = {'Range': 'bytes=%d-%d' % range_bytes}
ok_status_code = requests.codes.partial
request = self.root.account.session.get(uri,
headers=headers,
ok_status_code=ok_status_code)
if file is not None:
file.write(request.content)
else:
return request.content
def delete_item(self, item_id=None, item_path=None):
"""
https://github.com/OneDrive/onedrive-api-docs/blob/master/items/delete.md
Delete the specified item on OneDrive server.
:param str | None item_id: ID of the item. Required if item_path is None.
:param str | None item_path: Path to the item relative to drive root. Required if item_id is None.
"""
uri = self.get_item_uri(item_id, item_path)
self.root.account.session.delete(
uri, ok_status_code=requests.codes.no_content)
def update_item(self,
item_id=None,
item_path=None,
new_name=None,
new_description=None,
new_parent_reference=None,
new_file_system_info=None):
"""
Update the metadata of the specified item.
:param str | None item_id: (Optional) ID of the target item.
:param str | None item_path: (Optional) Path to the target item.
:param str | None new_name: (Optional) If set, update the item metadata with the new name.
:param str | None new_description: (Optional) If set, update the item metadata with the new description.
:param onedrive_d.api.resources.ItemReference | None new_parent_reference: (Optional) If set,
move the item.
:param onedrive_d.api.facets.FileSystemInfoFacet | None new_file_system_info: (Optional) If set, update the
client-wise timestamps.
:rtype: onedrive_d.api.items.OneDriveItem
"""
if item_id is None and item_path is None:
raise ValueError('Root is immutable. A specific item is required.')
data = {}
if new_name is not None:
data['name'] = new_name
if new_description is not None:
data['description'] = new_description
if new_parent_reference is not None:
data['parentReference'] = new_parent_reference.data
if new_file_system_info is not None:
data['fileSystemInfo'] = new_file_system_info.data
if len(data) == 0:
raise ValueError('Nothing is to change.')
uri = self.get_item_uri(item_id, item_path)
request = self.root.account.session.patch(uri, data)
return items.OneDriveItem(self, request.json())
def copy_item(self,
dest_reference,
item_id=None,
item_path=None,
new_name=None):
"""
Copy an item (including any children) on OneDrive under a new parent.
:param onedrive_d.api.resources.ItemReference dest_reference: Reference to new parent.
:param str | None item_id: (Optional) ID of the source item. Required if item_path is None.
:param str | None item_path: (Optional) Path to the source item. Required if item_id is None.
:param str | None new_name: (Optional) If set, use this name for the copied item.
:rtype: onedrive_d.api.resources.AsyncCopySession
"""
if not isinstance(dest_reference, resources.ItemReference):
raise ValueError('Destination should be an ItemReference object.')
if item_id is None and item_path is None:
raise ValueError('Source of copy must be specified.')
uri = self.get_item_uri(item_id, item_path)
if item_path is not None:
uri += ':'
uri += '/action.copy'
data = {'parentReference': dest_reference.data}
if new_name is not None:
data['name'] = new_name
headers = {'Prefer': 'respond-async'}
request = self.root.account.session.post(uri,
json=data,
headers=headers)
return resources.AsyncCopySession(self, request.headers)
def get_thumbnail(self):
raise NotImplementedError('The API feature is not used yet.')
def search(self, keyword, select=None, item_id=None, item_path=None):
"""
Use a keyword to search for items within the specified directory (default: root).
:param str keyword: Keyword for the search.
:param [str] | None select: Only fetch the specified fields.
:param str | None item_id: (Optional) ID of the item to search within.
:param str | None item_path: (Optional) Path to the item to search within.
:return onedrive_d.api.items.ItemCollection:
"""
params = {'q': keyword}
if select is not None:
params['select'] = ','.join(select)
uri = self.get_item_uri(item_id, item_path) + '/view.search'
request = self.root.account.session.get(uri, params=params)
return items.ItemCollection(self, request.json())
def get_changes(self):
raise NotImplementedError('The API feature is not used yet.')
def get_special_dir(self, name):
raise NotImplementedError('The API feature is not used yet.')
def dump(self):
data = {
'config_dump': self.config.dump(),
'data': self._data,
self.VERSION_KEY: self.VERSION_VALUE
}
return json.dumps(data)
@classmethod
def load(cls, drive_root, account_id, account_type, s):
data = json.loads(s)
drive = DriveObject(drive_root, data['data'],
drive_config.DriveConfig.load(data['config_dump']))
try:
drive_root.add_cached_drive(account_id, account_type, drive)
except ValueError as e:
cls.logger.warning(
'Faild to register deserialized drive %s to drive root: %s',
drive.drive_id, e)
return drive
def mock_loggers():
logger_factory.init_logger(min_level=logging.CRITICAL)
for cls in [drives.DriveObject, account_db.AccountStorage, account_db.AccountStorage, drives_db.DriveStorage,
drive_config.DriveConfig]:
cls.logger = logger_factory.get_logger('ut_' + cls.__name__)
class DriveConfig:
DEFAULT_VALUES = {
'max_get_size_bytes': 1048576,
'max_put_size_bytes': 524288,
'local_root': None,
'ignore_files': set(),
}
logger = logger_factory.get_logger('DriveConfig')
def __init__(self, data):
for k, v in self.DEFAULT_VALUES.items():
if k not in data:
data[k] = v
if isinstance(data['ignore_files'], list):
data['ignore_files'] = set(data['ignore_files'])
for item in self.DEFAULT_VALUES['ignore_files']:
if item not in data['ignore_files']:
data['ignore_files'].add(item)
self.data = data
@staticmethod
def default_config():
return DriveConfig(deepcopy(DriveConfig.DEFAULT_VALUES))
@classmethod
def set_default_config(cls, config):
"""
Set the new config as default, with side-effect of updating all existing configs that use (unsaved) default
values.
:param onedrive_d.api.drives.DriveConfig config:
"""
for k, v in cls.DEFAULT_VALUES.items():
v2 = getattr(config, k)
if v2 != v:
cls.DEFAULT_VALUES[k] = v2
@property
def max_get_size_bytes(self):
"""
:rtype: int
"""
return self.data['max_get_size_bytes']
@property
def max_put_size_bytes(self):
"""
:rtype: int
"""
return self.data['max_put_size_bytes']
@property
def local_root(self):
"""
:rtype: str
"""
return self.data['local_root']
@property
def ignore_files(self):
"""
:rtype: [str]
"""
return self.data['ignore_files']
# noinspection PyAttributeOutsideInit
@property
def path_filter(self):
if not hasattr(self, '_path_filter'):
rules = set()
for path in self.ignore_files:
try:
with open(path, 'r') as f:
rules.update(f.read().splitlines())
except Exception as e:
self.logger.error('Failed to load ignore list "%s": %s',
path, e)
self._path_filter = path_filter.PathFilter(rules)
return self._path_filter
def dump(self, exact_dump=False):
data = {}
for key in ['max_get_size_bytes', 'max_put_size_bytes', 'local_root']:
if exact_dump or getattr(self, key) != self.DEFAULT_VALUES[key]:
data[key] = getattr(self, key)
ignore_files = [
s for s in self.ignore_files
if exact_dump or s not in self.DEFAULT_VALUES['ignore_files']
]
if len(ignore_files) > 0:
data['ignore_files'] = ignore_files
return data
@classmethod
def load(cls, d):
return DriveConfig(d)
class TaskPool:
"""
An in-memory, singleton storage for Tasks based on hash maps.
"""
logger = logger_factory.get_logger('TaskPool')
_instance = None
@classmethod
def get_instance(cls):
if cls._instance is None:
cls._semaphore = threading.Semaphore(0)
cls._lock = rwlock.RWLock()
cls._instance = TaskPool()
return cls._instance
def __init__(self):
self._all_tasks = []
self._tasks_by_path = {}
def _add_to_list(self, key, table, value):
"""
Add an item to a dict whose values are lists.
:param str key:
:param dict[str, T] table:
:param T value:
"""
if key not in table:
table[key] = [value]
else:
table[key].append(value)
def get_task_path(self, task):
"""
Return the local path the task performs on.
:param onedrive_d.common.tasks.LocalParentPathMixin task:
:return str:
"""
return task.local_parent_path + '/' + task.name
def add_task(self, task):
self.logger.debug('Try acquiring writer lock...')
self._lock.writer_acquire()
self._all_tasks.append(task)
self._add_to_list(self.get_task_path(task), self._tasks_by_path, task)
self.logger.debug('Added task "%s" on path "%s".',
task.__class__.__name__,
task.local_parent_path + '/' + task.name)
self._lock.writer_release()
self.logger.debug('Writer lock released.')
self._semaphore.release()
@property
def semaphore(self):
return self._semaphore
def pop_task(self, task_class=None):
self._lock.writer_acquire()
ret = None
if len(self._all_tasks) > 0:
if task_class is None:
ret = self._all_tasks.pop(0)
else:
for t in self._all_tasks:
if isinstance(t, task_class):
ret = t
self._all_tasks.remove(t)
break
if ret is not None:
self._tasks_by_path[self.get_task_path(ret)].remove(ret)
self._lock.writer_release()
return ret
def has_pending_task(self, local_path):
self._lock.reader_acquire()
ret = local_path in self._tasks_by_path and len(
self._tasks_by_path[local_path]) > 0
self.logger.debug('Item "%s" has pending task? %s.', local_path,
str(ret))
self._lock.reader_release()
return ret
def remove_children_tasks(self, local_parent_path):
local_parent_path += '/'
self._lock.writer_acquire()
for t in self._all_tasks:
task_path = self.get_task_path(t)
if task_path.startswith(local_parent_path):
self._all_tasks.remove(t)
self._tasks_by_path[task_path].remove(t)
self._lock.writer_release()
class ManagedRESTClient:
AUTO_RETRY_SECONDS = 30
RECOVERABLE_STATUS_CODES = {requests.codes.too_many, 500, 502, 503, 504}
logger = logger_factory.get_logger(__name__)
def __init__(self, session, net_mon, account, proxies=None):
"""
:param session: Dictate a requests Session object.
:param onedrive_d.common.netman.NetworkMonitor net_mon: Network monitor instance.
:param onedrive_d.api.accounts.PersonalAccount | onedrive_d.api.accounts.BusinessAccount account: Account.
:param dict[str, str] proxies: (Optional) A dictionary of protocol-host pairs.
:return: No return value.
"""
self.session = session
self.net_mon = net_mon
self.account = account
self.proxies = proxies
def request(self, method, url, params, ok_status_code, auto_renew):
"""
Perform a HTTP request call. Do auto-recover as fits.
:param str method: One of {GET, POST, PATCH, PUT, DELETE}.
:param str url: URL of the HTTP request.
:param dict[str, str | dict | bytes] params: Params to send to the request call.
:param int ok_status_code: Expected status code for HTTP response.
:param True | False auto_renew: If True, auto recover the expired token.
:rtype: requests.Response
:raise errors.OneDriveError:
"""
while True:
try:
request = getattr(self.session, method)(url, **params)
bad_status = request.status_code != ok_status_code if isinstance(ok_status_code, int) \
else request.status_code not in ok_status_code
if bad_status:
if request.status_code in self.RECOVERABLE_STATUS_CODES:
if 'Retry-After' in request.headers:
retry_after_seconds = int(
request.headers['Retry-After'])
else:
retry_after_seconds = self.AUTO_RETRY_SECONDS
self.logger.info(
'Server returned code %d which is assumed recoverable. Retry in %d seconds',
request.status_code, retry_after_seconds)
raise errors.OneDriveRecoverableError(
retry_after_seconds)
raise errors.OneDriveError(request.json())
return request
except requests.ConnectionError:
self.net_mon.suspend_caller()
except errors.OneDriveRecoverableError as e:
time.sleep(e.retry_after_seconds)
except errors.OneDriveTokenExpiredError as e:
if auto_renew:
self.logger.info('Access token expired. Try refreshing...')
self.account.renew_tokens()
else:
raise e
def get(self,
url,
params=None,
headers=None,
ok_status_code=requests.codes.ok,
auto_renew=True):
"""
Perform a HTTP GET request.
:param str url: URL of the HTTP request.
:param dict[str, T] | None params: (Optional) Dictionary to construct query string.
:param dict | None headers: (Optional) Additional headers for the HTTP request.
:param int ok_status_code: (Optional) Expected status code for the HTTP response.
:param True | False auto_renew: (Optional) If True, auto recover from expired token error or Internet failure.
:rtype: requests.Response
"""
args = {'proxies': self.proxies}
if params is not None:
args['params'] = params
if headers is not None:
args['headers'] = headers
return self.request('get',
url,
args,
ok_status_code=ok_status_code,
auto_renew=auto_renew)
def post(self,
url,
data=None,
json=None,
headers=None,
ok_status_code=requests.codes.ok,
auto_renew=True):
"""
Perform a HTTP POST request.
:param str url: URL of the HTTP request.
:param dict | None data: (Optional) Data in POST body of the request.
:param dict | None json: (Optional) Send the dictionary as JSON content in POST body and set proper headers.
:param dict | None headers: (Optional) Additional headers for the HTTP request.
:param int ok_status_code: (Optional) Expected status code for the HTTP response.
:param True | False auto_renew: (Optional) If True, auto recover from expired token error or Internet failure.
:rtype: requests.Response
"""
params = {'proxies': self.proxies}
if json is not None:
params['json'] = json
else:
params['data'] = data
if headers is not None:
params['headers'] = headers
return self.request('post',
url,
params,
ok_status_code=ok_status_code,
auto_renew=auto_renew)
def patch(self,
url,
json,
ok_status_code=requests.codes.ok,
auto_renew=True):
"""
Perform a HTTP PATCH request.
:param str url: URL of the HTTP request.
:param dict json: Send the dictionary as JSON content in POST body and set proper headers.
:param int ok_status_code: (Optional) Expected status code for the HTTP response.
:param True | False auto_renew: (Optional) If True, auto recover from expired token error or Internet failure.
:rtype: requests.Response
"""
params = {'proxies': self.proxies, 'json': json}
return self.request('patch',
url,
params,
ok_status_code=ok_status_code,
auto_renew=auto_renew)
def put(self,
url,
data,
headers=None,
ok_status_code=requests.codes.ok,
auto_renew=True):
"""
Perform a HTTP PUT request.
:param str url: URL of the HTTP request.
:param bytes | None data: Binary data to send in the request body.
:param dict | None headers: Additional headers for the HTTP request.
:param int ok_status_code: (Optional) Expected status code for the HTTP response.
:param True | False auto_renew: (Optional) If True, auto recover from expired token error or Internet failure.
:rtype: requests.Response
"""
params = {'proxies': self.proxies, 'data': data}
if headers is not None:
params['headers'] = headers
return self.request('put',
url,
params=params,
ok_status_code=ok_status_code,
auto_renew=auto_renew)
def delete(self, url, ok_status_code=requests.codes.ok, auto_renew=True):
"""
Perform a HTTP DELETE request on the specified URL.
:param str url: URL of the HTTP request.
:param int ok_status_code: (Optional) Expected status code for the HTTP response.
:param True | False auto_renew: (Optional) If True, auto recover from expired token error or Internet failure.
:rtype: requests.Response
"""
return self.request('delete',
url, {'proxies': self.proxies},
ok_status_code=ok_status_code,
auto_renew=auto_renew)
