def index(system_id, path, username, pems = False, bottom_up = False, levels = 0): try: user = get_user_model().objects.get(username = username) ag = user.agave_oauth.client mngr = FileManager(agave_client = ag) mngr.index(system_id, path, username, bottom_up = bottom_up, levels = levels) if pems: index_permissions.delay(system_id, path, username, bottom_up = bottom_up, levels = levels) except ObjectDoesNotExist: logger.exception('Unable to locate local user=%s' % username)
def check_or_create_agave_home_dir(username): logger.info("Checking home directory for user=%s on default storage systemId=%s" % ( username, settings.AGAVE_STORAGE_SYSTEM)) try: # TODO should use DS Files API for this ag = Agave(api_server=settings.AGAVE_TENANT_BASEURL, token=settings.AGAVE_SUPER_TOKEN) body = {'action': 'mkdir', 'path': username} ag.files.manage(systemId=settings.AGAVE_STORAGE_SYSTEM, filePath='', body=body) # add dir to index fm = FileManager(agave_client=ag) fm.index(settings.AGAVE_STORAGE_SYSTEM, username, username, levels=1) except (HTTPError, AgaveException): logger.exception('Failed to create home directory.', extra={'user': username, 'systemId': settings.AGAVE_STORAGE_SYSTEM})
def get(self, request, *args, **kwargs): self.set_context_props(request, **kwargs) mgr = FileManager(agave_client = self.agave_client) if self.file_path == request.user.username: mgr.check_shared_folder(system_id = self.filesystem, username = request.user.username) if request.GET.get('refresh', None): mgr.index(self.filesystem, self.file_path, request.user.username, levels = 1) l = mgr.list_path(system_id = self.filesystem, path = self.file_path, username = request.user.username, special_dir = self.special_dir, is_public = self.is_public) if isinstance(l, GeneratorType): response = [o.to_dict(get_id = True) for o in l] else: tasks.index.delay(self.filesystem, self.file_path, request.user.username, pems = True, levels = 1) response = [o.to_dict(pems = False) for o in l] return self.render_to_json_response(response, status = 200)
def copy_box_item(self, username, box_item_type, box_item_id, target_system_id, target_path): user = get_user_model().objects.get(username=username) client = user.box_user_token.client try: op = getattr(client, box_item_type) item = op(box_item_id).get() except AttributeError as e: logger.error('Invalid box_item_type') self.update_state(state='FAILURE', meta={'exc': e}) return agave_client = user.agave_oauth.client fm = FileManager(agave_client=agave_client) if box_item_type == 'file': try: import_url = item.get_shared_link_download_url() safe_name = get_valid_filename(item.name) import_resp = agave_client.files.importData(systemId=target_system_id, filePath=target_path, fileName=safe_name, urlToIngest=import_url) async_resp = AgaveAsyncResponse(agave_client, import_resp) async_status = async_resp.result(600) if async_status == 'FAILED': logger.error('Box File Transfer failed: %s' % target_path) else: file_path = '%s/%s' % (target_path, item.name) logger.info('Indexing Box File Transfer %s' % file_path) fm.index(settings.AGAVE_STORAGE_SYSTEM, file_path, username, levels=1) except BoxAPIException as e: logger.error('Unable to get download link from Box') self.update_state(state='FAILURE', meta={'exc': e}) except HTTPError as e: logger.error('Agave.files.importData raised HTTPError') self.update_state(state='FAILURE', meta={'exc': e}) except (TimeoutError, Error) as e: logger.error('Agave.files.importData failed to complete') self.update_state(state='FAILURE', meta={'exc': e}) else: # box_item_type == 'folder' # create directory for the folder try: safe_name = get_valid_filename(item.name) mf, f = fm.mkdir(target_path, safe_name, target_system_id, username, raise_if_exists=False) logger.info('Created directory "{0}"; scheduling transfer of contents', f.full_path) except HTTPError as e: logger.error('Agave.files.manage(mkdir) failed') self.update_state(state='FAILURE', meta={'exc': e}) # schedule to copy all of it's items limit = 10 offset = 0 while True: item_collection = item.get_items(limit=limit, offset=offset) for it in item_collection: args = (username, it.type, it.object_id, target_system_id, f.full_path) copy_box_item.apply_async(args=args, countdown=offset*2) if len(item_collection) == limit: offset += limit else: break