def sync(creds, sourceid, destid): drive = get_drive(creds) copy_jobs, delete_jobs = process_recursively(drive, sourceid, destid, compare_function, new_folder_function) if len(copy_jobs) > 0: copy_tasks = [] asyncio.run(async_copy(creds, copy_jobs)) else: print('nothing to copy') if len(delete_jobs) > 0: for i in progressbar.progressbar(delete_jobs, widgets=[ 'deleting files ', progressbar.Counter(), '/' + str(len(delete_jobs)), ' ', progressbar.Bar(), ' ', progressbar.AdaptiveETA() ]): apicall(drive.files().delete(fileId=i, supportsAllDrives=True)) else: print('nothing to delete')
def sync(drive, sourceid, destid): copy_jobs, delete_jobs = process_recursively(drive, sourceid, destid, compare_function, new_folder_function) if len(copy_jobs) > 0: for i in progressbar.progressbar(copy_jobs, widgets=[ "copying files ", progressbar.Counter(), "/" + str(len(copy_jobs)), " ", progressbar.Bar(), " ", progressbar.AdaptiveETA() ]): apicall(drive.files().copy(fileId=i[0], body={"parents": [i[1]]}, supportsAllDrives=True)) else: print("nothing to copy") if len(delete_jobs) > 0: for i in progressbar.progressbar(delete_jobs, widgets=[ "deleting files ", progressbar.Counter(), "/" + str(len(delete_jobs)), " ", progressbar.Bar(), " ", progressbar.AdaptiveETA() ]): apicall(drive.files().delete(fileId=i, supportsAllDrives=True)) else: print("nothing to delete")
def sync(drive, source, destid): upload_jobs, delete_jobs = process_recursively(drive, source, destid, compare_function, new_folder_function) if len(upload_jobs) > 0: for i in upload_jobs: filesize = i[0].stat().st_size if filesize == 0: apicall(drive.files().create(body={ 'name': i[0].name, 'parents': [i[1]] })) elif filesize <= 5242880: upload_multipart(drive, i[0], i[1]) else: upload_resumable(drive, i[0], i[1]) else: print('nothing to upload') if len(delete_jobs) > 0: for i in progressbar.progressbar(delete_jobs, widgets=[ 'deleting files ', progressbar.Counter(), '/' + str(len(delete_jobs)), ' ', progressbar.Bar(), ' ', progressbar.AdaptiveETA() ]): apicall(drive.files().delete(fileId=i, supportsAllDrives=True)) else: print('nothing to delete')
def new_folder_function(drive, folder_name, folder_parent): return apicall(drive.files().create(body={ "mimeType": "application/vnd.google-apps.folder", "name": folder_name, "parents": [folder_parent] }, supportsAllDrives=True))["id"]
def new_folder_function(drive, folder_name, folder_parent): return apicall(drive.files().create(body={ 'mimeType': 'application/vnd.google-apps.folder', 'name': folder_name, 'parents': [folder_parent] }, supportsAllDrives=True))['id']
def upload_multipart(drive, to_upload, parent): media = MediaFileUpload(to_upload) request = drive.files().create(media_body=media, supportsAllDrives=True, body={ 'name': to_upload.name, 'parents': [parent], 'modifiedTime': get_time_rfc3339(to_upload) }) print(f'uploading {to_upload.as_posix()}') resp = apicall(request) return resp
def determine_folder(drive, source_dir, dest_dir, compare_function): source_files = list_path(drive, source_dir) dest_files = list_path(drive, dest_dir) from_local = isinstance(source_dir, Path) to_local = isinstance(dest_dir, Path) if from_local: source_filename = source_dir.name else: source_filename = apicall(drive.files().get(fileId=source_dir))['name'] to_create = set() to_delete = set() to_process_length = len(source_files) + len(dest_files) count = 0 with progressbar.ProgressBar(0, to_process_length, [ 'processing files (' + source_filename + ') ', progressbar.Counter(), '/' + str(to_process_length), ' ', progressbar.Bar() ]).start() as pbar: for source_file in source_files: for dest_file in dest_files: match_found, add_item, delete_item = compare_function( drive, source_file, dest_file, dest_dir) if match_found: if add_item: to_create.add(add_item) if delete_item: to_delete.add(delete_item) break else: if from_local: to_create.add((source_file, dest_dir)) elif to_local: # hacky solution, i know to_create.add((source_file['id'], dest_dir, source_file['name'], source_file['size'])) else: to_create.add((source_file['id'], dest_dir)) count += 1 pbar.update(count) for dest_file in dest_files: if to_local: if dest_file in to_delete: continue else: if dest_file['id'] in to_delete: continue for source_file in source_files: if from_local: source_filename = source_file.name else: source_filename = source_file['name'] if to_local: dest_filename = dest_file.name else: dest_filename = dest_file['name'] if source_filename == dest_filename: break else: if to_local: to_delete.add(dest_file) else: to_delete.add(dest_file['id']) count += 1 pbar.update(count) return to_create, to_delete
def get_path(self, path_string, creds): try: path_type, path_dirs = path_string.split(':') except ValueError: print('error processing path') return None if path_type == 'ld': path_final = Path(path_dirs) if path_final.exists(): return path_final else: print(f'path not found: {path_final.resolve()}') return None elif path_type in ['md', 'sd', 'fi' ] or path_type in self.config['path_aliases']: drive = get_drive(creds) to_traverse = [i.strip() for i in path_dirs.strip('/').split('/')] to_traverse.reverse() if path_type == 'md': current_dir = apicall(drive.files().get(fileId='root'))['id'] elif path_type == 'sd': drives = lsdrives(drive) current_dir = None for i in drives: if i['name'] == to_traverse[-1]: current_dir = i['id'] break if current_dir == None: print('drive not found: ' + to_traverse[-1]) return None to_traverse.pop() elif path_type == 'fi': current_dir = apicall( drive.files().get(fileId=to_traverse.pop()))['id'] else: alias_path = self.config['path_aliases'][path_type] if alias_path[0] == 'local': return Path(alias_path[1]) else: current_dir = apicall( drive.files().get(fileId=alias_path[1]))['id'] if len(to_traverse) != 1 or len(to_traverse[0]) != 0: while len(to_traverse) != 0: search_for = to_traverse.pop() found = False for i in ls(drive, current_dir): if i['name'] == search_for: current_dir = i['id'] found = True break if not found: print(f"couldn't find {search_for}!") return None return current_dir else: print(f"unrecognized path type '{path_type}'")
def get_path(self, path_string, creds): try: path_type, path_dirs = path_string.split(":") except ValueError: print("error processing path") return None if path_type == "ld": path_final = Path(path_dirs) if path_final.exists(): return path_final else: print(f"path not found: {path_final.resolve()}") return None elif path_type in ["md", "sd", "fi"] or path_type in self.config["path_aliases"]: drive = get_drive(creds) to_traverse = [i.strip() for i in path_dirs.strip("/").split("/")] to_traverse.reverse() if path_type == "md": current_dir = apicall(drive.files().get(fileId="root"))["id"] elif path_type == "sd": drives = lsdrives(drive) current_dir = None for i in drives: if i["name"] == to_traverse[-1]: current_dir = i["id"] break if current_dir == None: print("drive not found: " + to_traverse[-1]) return None to_traverse.pop() elif path_type == "fi": current_dir = apicall(drive.files().get(fileId=to_traverse.pop()))["id"] else: alias_path = self.config["path_aliases"][path_type] if alias_path[0] == "local": return Path(alias_path[1]) else: current_dir = apicall(drive.files().get(fileId=alias_path[1]))["id"] if len(to_traverse) != 1 or len(to_traverse[0]) != 0: while len(to_traverse) != 0: search_for = to_traverse.pop() found = False for i in ls(drive, current_dir): if i["name"] == search_for: current_dir = i["id"] found = True break if not found: print(f"couldn't find {search_for}!") return None return current_dir else: print(f'unrecognized path type "{path_type}"')