Esempio n. 1
0
def external_resource_upload(self, username, dest_resource, src_file_id, dest_file_id):
    """
    :param self:
    :param username:
    :param dest_resource:
    :param src_file_id:
    :param dest_file_id:
    :return:
    """
    logger.debug('Initializing external_resource_upload. username: %s, src_file_id: %s, dest_resource: %s, dest_file_id: %s ', username, src_file_id, dest_resource, dest_file_id)

    from designsafe.apps.api.external_resources.box.filemanager.manager \
            import FileManager as BoxFileManager
    from designsafe.apps.api.external_resources.dropbox.filemanager.manager \
            import FileManager as DropboxFileManager
    from designsafe.apps.api.external_resources.googledrive.filemanager.manager \
            import FileManager as GoogleDriveFileManager

    user = get_user_model().objects.get(username=username)

    if dest_resource == 'box':
        fmgr = BoxFileManager(user)
    elif dest_resource == 'dropbox':
        fmgr = DropboxFileManager(user)
    elif dest_resource == 'googledrive':
        fmgr = GoogleDriveFileManager(user)

    logger.debug('fmgr.upload( %s, %s, %s)', username, src_file_id, dest_file_id)
    fmgr.upload(username, src_file_id, dest_file_id)
Esempio n. 2
0
def external_resource_download(self, file_mgr_name, username, src_file_id, dest_file_id):
    """
    :param self:
    :param username:
    :param src_file_id:
    :param dest_file_id:
    :return:
    """
    logger.debug('Downloading %s://%s for user %s to %s',
                 file_mgr_name, src_file_id, username, dest_file_id)

    from designsafe.apps.api.external_resources.box.filemanager.manager \
            import FileManager as BoxFileManager
    from designsafe.apps.api.external_resources.dropbox.filemanager.manager \
            import FileManager as DropboxFileManager
    from designsafe.apps.api.external_resources.googledrive.filemanager.manager \
            import FileManager as GoogleDriveFileManager

    user = get_user_model().objects.get(username=username)

    if file_mgr_name == 'box':
        fmgr = BoxFileManager(user)
    elif file_mgr_name == 'dropbox':
        fmgr = DropboxFileManager(user)
    elif file_mgr_name == 'googledrive':
        fmgr = GoogleDriveFileManager(user)

    fmgr.copy(username, src_file_id, dest_file_id)
Esempio n. 3
0
def box_upload(self, username, src_resource, src_file_id, dest_resource, dest_file_id):
    """
    :param self:
    :param username:
    :param src_resource: should be 'agave'
    :param src_file_id: the agave file id
    :param dest_resource: should be 'box'
    :param dest_file_id: the box id of the destination folder
    :return:
    """
    logger.debug('Importing file %s://%s for user %s to %s://%s' % (
        src_resource, src_file_id, username, dest_resource, dest_file_id))

    try:
        n = Notification(event_type = 'data',
                         status = Notification.INFO,
                         operation = 'box_upload_start',
                         message = 'Starting import of file %s into box.' % src_file_id,
                         user = username,
                         # extra = {'target_path': '%s%s/%s' %(reverse('designsafe_data:data_depot'), src_resource, src_file_id)})
                         extra={'id': src_file_id})
        n.save()
        user = get_user_model().objects.get(username=username)

        from designsafe.apps.api.data import BoxFileManager
        box_fm = BoxFileManager(user)
        file_type, file_id = box_fm.parse_file_id(dest_file_id)

        if file_type != 'folder':
            logger.warn('Cannot import to a file destination!')
            raise Exception('You can only import files to a folder!', status=400,
                            extra={
                                'src_resource': src_resource,
                                'src_file_id': src_file_id,
                                'dest_resource': dest_resource,
                                'dest_file_id': dest_file_id,
                            })

        if src_resource == 'agave' or src_resource == 'public':
            try:
                logger.debug('Starting upload to Box...')
                from designsafe.apps.api.data import lookup_file_manager
                agave_fm = lookup_file_manager(src_resource)(user)
                file_real_path = agave_fm.get_file_real_path(src_file_id)
                if os.path.isfile(file_real_path):
                    box_upload_file(box_fm, file_id, file_real_path)
                elif os.path.isdir(file_real_path):
                    box_upload_directory(box_fm, file_id, file_real_path)
                else:
                    logger.error('Unable to upload %s: file does not exist!',
                                 file_real_path)
            except:
                logger.exception('Upload to Box failed!')

        logger.debug('Box upload task complete.')

        n = Notification(event_type = 'data',
                         status = Notification.SUCCESS,
                         operation = 'box_upload_end',
                         message = 'File(s) %s succesfully uploaded into box!' % src_file_id,
                         user = username,
                         extra={'id': src_file_id})
                         # extra = {'target_path': '%s%s/%s' %(reverse('designsafe_data:data_depot'), dest_resource, dest_file_id)})
        n.save()
    except:
        logger.exception('Unexpected task failure: box_upload', extra={
            'username': username,
            'src_resource': src_resource,
            'src_file_id': src_file_id,
            'dest_resource': dest_resource,
            'dest_file_id': dest_file_id,
        })
        n = Notification(event_type = 'data',
                         status = Notification.ERROR,
                         operation = 'box_download_error',
                         message = 'We were unable to get the specified file from box. Please try again...',
                         user = username,
                         extra={
                                'src_resource': src_resource,
                                'id': src_file_id,
                                'dest_resource': dest_resource,
                                'dest_file_id': dest_file_id,
                            })
                         # extra = {'target_path': '%s%s/%s' %(reverse('designsafe_data:data_depot'), src_resource, src_file_id)})
        n.save()
Esempio n. 4
0
def box_download(self, username, src_resource, src_file_id, dest_resource, dest_file_id):
    """
    :param self:
    :param username:
    :param src_resource:
    :param src_file_id:
    :param dest_resource:
    :param dest_file_id:
    :return:
    """

    logger.debug('Downloading %s://%s for user %s to %s://%s',
                 src_resource, src_file_id, username, dest_resource, dest_file_id)

    try:
        target_path = reverse('designsafe_data:data_depot',
                              args=[src_resource, src_file_id])
        n = Notification(event_type='data',
                         status=Notification.INFO,
                         operation='box_download_start',
                         message='Starting download of file %s from box.' % (src_file_id,),
                         user=username,
                         # extra={'target_path': target_path})
                         extra={'id': dest_file_id}
                         )
        n.save()

        user = get_user_model().objects.get(username=username)

        from designsafe.apps.api.data import BoxFileManager, AgaveFileManager
        agave_fm = AgaveFileManager(user)
        dest_real_path = agave_fm.get_file_real_path(dest_file_id)

        box_fm = BoxFileManager(user)
        file_type, file_id = box_fm.parse_file_id(src_file_id)

        levels = 0
        downloaded_file_path = None
        if file_type == 'file':
            downloaded_file_path = box_download_file(box_fm, file_id, dest_real_path)
            levels = 1
        elif file_type == 'folder':
            downloaded_file_path = box_download_folder(box_fm, file_id, dest_real_path)

        if downloaded_file_path is not None:
            downloaded_file_id = agave_fm.from_file_real_path(downloaded_file_path)
            system_id, file_user, file_path = agave_fm.parse_file_id(downloaded_file_id)
            agave_fm.indexer.index(system_id, file_path, file_user, full_indexing=True,
                                   pems_indexing=True, index_full_path=True,
                                   levels=levels)

        target_path = reverse('designsafe_data:data_depot',
                              args=[dest_resource, dest_file_id])
        n = Notification(event_type='data',
                         status=Notification.SUCCESS,
                         operation='box_download_end',
                         message='File %s was copied from box successfully!' % (src_file_id, ),
                         user=username,
                         extra={'id': dest_file_id}
                         )
        n.save()
    except:
        logger.exception('Unexpected task failure: box_download', extra={
            'username': username,
            'box_file_id': src_file_id,
            'to_resource': dest_resource,
            'dest_file_id': dest_file_id
        })
        target_path = reverse('designsafe_data:data_depot',
                              args=[src_resource, src_file_id])
        n = Notification(event_type='data',
                         status=Notification.ERROR,
                         operation='box_download_error',
                         message='We were unable to get the specified file from box. '
                                 'Please try again...',
                         user=username,
                         extra={'system': dest_resource,
                                'id': dest_file_id,
                                'src_file_id': src_file_id,
                                'src_resource': src_resource
                         })
        n.save()
Esempio n. 5
0
def box_resource_download(self, username, src_file_id, dest_file_id):
    """
    :param self:
    :param username:
    :param src_file_id:
    :param dest_file_id:
    :return:
    """

    logger.debug('Downloading box://%s for user %s to %s',
                 src_file_id, username, dest_file_id)

    try:
        n = Notification(event_type='data',
                         status=Notification.INFO,
                         operation='box_download_start',
                         message='Starting download file %s from box.' % (src_file_id,),
                         user=username,
                         extra={})
        n.save()
        logger.debug('username: {}, src_file_id: {}, dest_file_id: {}'.format(username, src_file_id, dest_file_id))
        user = get_user_model().objects.get(username=username)

        from designsafe.apps.api.external_resources.box.filemanager.manager import \
             FileManager as BoxFileManager
        from designsafe.apps.api.agave.filemanager.agave import AgaveFileManager
        # Initialize agave filemanager
        agave_fm = AgaveFileManager(agave_client=user.agave_oauth.client)
        # Split destination file path
        dest_file_path_comps = dest_file_id.strip('/').split('/')
        # If it is an agave file id then the first component is a system id
        agave_system_id = dest_file_path_comps[0]
        # Start construction the actual real path into the NSF mount
        if dest_file_path_comps[1:]:
            dest_real_path = os.path.join(*dest_file_path_comps[1:])
        else:
            dest_real_path = '/'
        # Get what the system id maps to
        base_mounted_path = agave_fm.base_mounted_path(agave_system_id)
        # Add actual path
        if re.search(r'^project-', agave_system_id):
            project_dir = agave_system_id.replace('project-', '', 1)
            dest_real_path = os.path.join(base_mounted_path, project_dir, dest_real_path.strip('/'))
        else:
            dest_real_path = os.path.join(base_mounted_path, dest_real_path.strip('/'))
        logger.debug('dest_real_path: {}'.format(dest_real_path))

        box_fm = BoxFileManager(user)
        box_file_type, box_file_id = box_fm.parse_file_id(file_id=src_file_id)

        levels = 0
        downloaded_file_path = None
        if box_file_type == 'file':
            downloaded_file_path = box_fm.download_file(box_file_id, dest_real_path)
            levels = 1
        elif box_file_type == 'folder':
            downloaded_file_path = box_fm.download_folder(box_file_id, dest_real_path)

        #if downloaded_file_path is not None:
        #    downloaded_file_id = agave_fm.from_file_real_path(downloaded_file_path)
        #    system_id, file_user, file_path = agave_fm.parse_file_id(downloaded_file_id)

        n = Notification(event_type='data',
                         status=Notification.SUCCESS,
                         operation='box_download_end',
                         message='File %s has been copied from box successfully!' % (src_file_id, ),
                         user=username,
                         extra={})
        n.save()
        if re.search(r'^project-', agave_system_id):
            project_dir = agave_system_id.replace('project-', '', 1)
            project_dir = os.path.join(base_mounted_path.strip('/'), project_dir)
            agave_file_path = downloaded_file_path.replace(project_dir, '', 1).strip('/')
        else:
            agave_file_path = downloaded_file_path.replace(base_mounted_path, '', 1).strip('/')

        reindex_agave.apply_async(kwargs={
                                  'username': user.username,
                                  'file_id': '{}/{}'.format(agave_system_id, agave_file_path)
                                  })
    except:
        logger.exception('Unexpected task failure: box_download', extra={
            'username': username,
            'box_file_id': src_file_id,
            'dest_file_id': dest_file_id
        })
        n = Notification(event_type='data',
                         status=Notification.ERROR,
                         operation='box_download_error',
                         message='We were unable to get the specified file from box. '
                                 'Please try again...',
                         user=username,
                         extra={})
        n.save()
Esempio n. 6
0
def box_upload(self, username, src_resource, src_file_id, dest_resource, dest_file_id):
    """
    :param self:
    :param username:
    :param src_resource: should be 'agave'
    :param src_file_id: the agave file id
    :param dest_resource: should be 'box'
    :param dest_file_id: the box id of the destination folder
    :return:
    """
    logger.debug('Importing file %s://%s for user %s to %s://%s' % (
        src_resource, src_file_id, username, dest_resource, dest_file_id))

    try:
        n = Notification(event_type = 'data',
                         status = Notification.INFO,
                         operation = 'box_upload_start',
                         message = 'Starting import file %s into box.' % src_file_id,
                         user = username,
                         extra = {'target_path': '%s%s/%s' %(reverse('designsafe_data:data_browser'), src_resource, src_file_id)})
        n.save()
        user = get_user_model().objects.get(username=username)

        from designsafe.apps.api.data import BoxFileManager
        box_fm = BoxFileManager(user)
        file_type, file_id = box_fm.parse_file_id(dest_file_id)

        if file_type != 'folder':
            logger.warn('Cannot import to a file destination!')
            raise Exception('You can only import files to a folder!', status=400,
                            extra={
                                'src_resource': src_resource,
                                'src_file_id': src_file_id,
                                'dest_resource': dest_resource,
                                'dest_file_id': dest_file_id,
                            })

        if src_resource == 'agave' or src_resource == 'public':
            try:
                logger.debug('Starting upload to Box...')
                from designsafe.apps.api.data import lookup_file_manager
                agave_fm = lookup_file_manager(src_resource)(user)
                file_real_path = agave_fm.get_file_real_path(src_file_id)
                if os.path.isfile(file_real_path):
                    box_upload_file(box_fm, file_id, file_real_path)
                elif os.path.isdir(file_real_path):
                    box_upload_directory(box_fm, file_id, file_real_path)
                else:
                    logger.error('Unable to upload %s: file does not exist!',
                                 file_real_path)
            except:
                logger.exception('Upload to Box failed!')

        logger.debug('Box upload task complete.')

        n = Notification(event_type = 'data',
                         status = Notification.SUCCESS,
                         operation = 'box_upload_end',
                         message = 'File(s) %s succesfully uploaded into box!' % src_file_id,
                         user = username,
                         extra = {'target_path': '%s%s/%s' %(reverse('designsafe_data:data_browser'), dest_resource, dest_file_id)})
        n.save()
    except:
        logger.exception('Unexpected task failure: box_upload', extra={
            'username': username,
            'src_resource': src_resource,
            'src_file_id': src_file_id,
            'dest_resource': dest_resource,
            'dest_file_id': dest_file_id,
        })
        n = Notification(event_type = 'data',
                         status = Notification.ERROR,
                         operation = 'box_download_error',
                         message = 'We were unable to get the specified file from box. Please try again...',
                         user = username,
                         extra = {'target_path': '%s%s/%s' %(reverse('designsafe_data:data_browser'), src_resource, src_file_id)})
        n.save()
Esempio n. 7
0
def box_download(self, username, src_resource, src_file_id, dest_resource, dest_file_id):
    """
    :param self:
    :param username:
    :param src_resource:
    :param src_file_id:
    :param dest_resource:
    :param dest_file_id:
    :return:
    """

    logger.debug('Downloading %s://%s for user %s to %s://%s',
                 src_resource, src_file_id, username, dest_resource, dest_file_id)

    try:
        target_path = reverse('designsafe_data:data_browser',
                              args=[src_resource, src_file_id])
        n = Notification(event_type='data',
                         status=Notification.INFO,
                         operation='box_download_start',
                         message='Starting download file %s from box.' % (src_file_id,),
                         user=username,
                         extra={'target_path': target_path})
        n.save()

        user = get_user_model().objects.get(username=username)

        from designsafe.apps.api.data import BoxFileManager, AgaveFileManager
        agave_fm = AgaveFileManager(user)
        dest_real_path = agave_fm.get_file_real_path(dest_file_id)

        box_fm = BoxFileManager(user)
        file_type, file_id = box_fm.parse_file_id(src_file_id)

        levels = 0
        downloaded_file_path = None
        if file_type == 'file':
            downloaded_file_path = box_download_file(box_fm, file_id, dest_real_path)
            levels = 1
        elif file_type == 'folder':
            downloaded_file_path = box_download_folder(box_fm, file_id, dest_real_path)

        if downloaded_file_path is not None:
            downloaded_file_id = agave_fm.from_file_real_path(downloaded_file_path)
            system_id, file_user, file_path = agave_fm.parse_file_id(downloaded_file_id)
            agave_fm.indexer.index(system_id, file_path, file_user, full_indexing=True,
                                   pems_indexing=True, index_full_path=True,
                                   levels=levels)

        target_path = reverse('designsafe_data:data_browser',
                              args=[dest_resource, dest_file_id])
        n = Notification(event_type='data',
                         status=Notification.SUCCESS,
                         operation='box_download_end',
                         message='File %s has been copied from box successfully!' % (src_file_id, ),
                         user=username,
                         extra={'target_path': target_path})
        n.save()
    except:
        logger.exception('Unexpected task failure: box_download', extra={
            'username': username,
            'box_file_id': src_file_id,
            'to_resource': dest_resource,
            'dest_file_id': dest_file_id
        })
        target_path = reverse('designsafe_data:data_browser',
                              args=[src_resource, src_file_id])
        n = Notification(event_type='data',
                         status=Notification.ERROR,
                         operation='box_download_error',
                         message='We were unable to get the specified file from box. '
                                 'Please try again...',
                         user=username,
                         extra={'target_path': target_path})
        n.save()