示例#1
0
def generate_export(export_type,
                    extension,
                    username,
                    id_string,
                    export_id=None,
                    filter_query=None,
                    group_delimiter='/',
                    split_select_multiples=True,
                    binary_select_multiples=False):
    """
    Create appropriate export object given the export type
    """

    export_type_func_map = {
        Export.XLS_EXPORT: 'to_xls_export',
        Export.CSV_EXPORT: 'to_flat_csv_export',
        Export.CSV_ZIP_EXPORT: 'to_zipped_csv',
        Export.SAV_ZIP_EXPORT: 'to_zipped_sav',
        Export.ANALYSER_EXPORT: 'to_analyser_export'
    }

    xform = XForm.objects.get(user__username__iexact=username,
                              id_string__exact=id_string)

    # query mongo for the cursor
    records = query_mongo(username, id_string, filter_query)

    export_builder = ExportBuilder()
    export_builder.GROUP_DELIMITER = group_delimiter
    export_builder.SPLIT_SELECT_MULTIPLES = split_select_multiples
    export_builder.BINARY_SELECT_MULTIPLES = binary_select_multiples
    export_builder.set_survey(xform.data_dictionary().survey)

    prefix = slugify('{}_export__{}__{}'.format(export_type, username,
                                                id_string))
    temp_file = NamedTemporaryFile(prefix=prefix, suffix=("." + extension))

    # get the export function by export type
    func = getattr(export_builder, export_type_func_map[export_type])

    func.__call__(temp_file.name, records, username, id_string, filter_query)

    # generate filename
    basename = "%s_%s" % (id_string,
                          datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
    if export_type == Export.ANALYSER_EXPORT:
        # Analyser exports should be distinguished by more than just their file extension.
        basename = '{}_ANALYSER_{}'.format(
            id_string,
            datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
    filename = basename + "." + extension

    # check filename is unique
    while not Export.is_filename_unique(xform, filename):
        filename = increment_index_in_filename(filename)

    file_path = os.path.join(username, 'exports', id_string, export_type,
                             filename)

    # TODO: if s3 storage, make private - how will we protect local storage??
    storage = get_storage_class()()
    # seek to the beginning as required by storage classes
    temp_file.seek(0)
    export_filename = storage.save(file_path, File(temp_file, file_path))
    temp_file.close()

    dir_name, basename = os.path.split(export_filename)

    # get or create export object
    if export_id:
        export = Export.objects.get(id=export_id)
    else:
        export = Export(xform=xform, export_type=export_type)
    export.filedir = dir_name
    export.filename = basename
    export.internal_status = Export.SUCCESSFUL
    # dont persist exports that have a filter
    if filter_query is None:
        export.save()
    return export
示例#2
0
def generate_export(export_type, extension, username, id_string,
                    export_id=None, filter_query=None, group_delimiter='/',
                    split_select_multiples=True,
                    binary_select_multiples=False):
    """
    Create appropriate export object given the export type
    """
    # TODO resolve circular import
    from onadata.apps.viewer.models.export import Export
    export_type_func_map = {
        Export.XLS_EXPORT: 'to_xls_export',
        Export.CSV_EXPORT: 'to_flat_csv_export',
        Export.CSV_ZIP_EXPORT: 'to_zipped_csv',
        Export.SAV_ZIP_EXPORT: 'to_zipped_sav',
    }

    xform = XForm.objects.get(
        user__username__iexact=username, id_string__iexact=id_string)

    # query mongo for the cursor
    records = query_mongo(username, id_string, filter_query)

    export_builder = ExportBuilder()
    export_builder.GROUP_DELIMITER = group_delimiter
    export_builder.SPLIT_SELECT_MULTIPLES = split_select_multiples
    export_builder.BINARY_SELECT_MULTIPLES = binary_select_multiples
    export_builder.set_survey(xform.data_dictionary().survey)

    temp_file = NamedTemporaryFile(suffix=("." + extension))

    # get the export function by export type
    func = getattr(export_builder, export_type_func_map[export_type])

    func.__call__(
        temp_file.name, records, username, id_string, filter_query)

    # generate filename
    basename = "%s_%s" % (
        id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
    filename = basename + "." + extension

    # check filename is unique
    while not Export.is_filename_unique(xform, filename):
        filename = increment_index_in_filename(filename)

    file_path = os.path.join(
        username,
        'exports',
        id_string,
        export_type,
        filename)

    # TODO: if s3 storage, make private - how will we protect local storage??
    storage = get_storage_class()()
    # seek to the beginning as required by storage classes
    temp_file.seek(0)
    export_filename = storage.save(
        file_path,
        File(temp_file, file_path))
    temp_file.close()

    dir_name, basename = os.path.split(export_filename)

    # get or create export object
    if export_id:
        export = Export.objects.get(id=export_id)
    else:
        export = Export(xform=xform, export_type=export_type)
    export.filedir = dir_name
    export.filename = basename
    export.internal_status = Export.SUCCESSFUL
    # dont persist exports that have a filter
    if filter_query is None:
        export.save()
    return export
示例#3
0
def generate_export(export_type,
                    extension,
                    username,
                    id_string,
                    export_id=None,
                    filter_query=None,
                    group_delimiter='/',
                    split_select_multiples=True,
                    binary_select_multiples=False,
                    start=None,
                    end=None,
                    remove_group_name=False):
    """
    Create appropriate export object given the export type
    """
    # TODO resolve circular import
    from onadata.apps.viewer.models.export import Export
    export_type_func_map = {
        Export.XLS_EXPORT: 'to_xls_export',
        Export.CSV_EXPORT: 'to_flat_csv_export',
        Export.CSV_ZIP_EXPORT: 'to_zipped_csv',
        Export.SAV_ZIP_EXPORT: 'to_zipped_sav',
    }

    xform = XForm.objects.get(user__username__iexact=username,
                              id_string__iexact=id_string)

    records = ParsedInstance.query_data(xform,
                                        query=filter_query,
                                        start=start,
                                        end=end)

    export_builder = ExportBuilder()

    export_builder.TRUNCATE_GROUP_TITLE = remove_group_name
    export_builder.GROUP_DELIMITER = group_delimiter
    export_builder.SPLIT_SELECT_MULTIPLES = split_select_multiples
    export_builder.BINARY_SELECT_MULTIPLES = binary_select_multiples
    export_builder.set_survey(xform.data_dictionary().survey)

    temp_file = NamedTemporaryFile(suffix=("." + extension))

    # get the export function by export type
    func = getattr(export_builder, export_type_func_map[export_type])
    try:
        func.__call__(temp_file.name,
                      records,
                      username,
                      id_string,
                      filter_query,
                      start=start,
                      end=end)
    except NoRecordsFoundError:
        pass

    # generate filename
    basename = "%s_%s" % (id_string,
                          datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
    filename = basename + "." + extension

    # check filename is unique
    while not Export.is_filename_unique(xform, filename):
        filename = increment_index_in_filename(filename)

    file_path = os.path.join(username, 'exports', id_string, export_type,
                             filename)

    # TODO: if s3 storage, make private - how will we protect local storage??
    storage = get_storage_class()()
    # seek to the beginning as required by storage classes
    temp_file.seek(0)
    export_filename = storage.save(file_path, File(temp_file, file_path))
    temp_file.close()

    dir_name, basename = os.path.split(export_filename)

    # get or create export object
    if export_id:
        export = Export.objects.get(id=export_id)
    else:
        export = Export(xform=xform, export_type=export_type)
    export.filedir = dir_name
    export.filename = basename
    export.internal_status = Export.SUCCESSFUL
    # dont persist exports that have a filter
    if filter_query is None and start is None and end is None:
        export.save()
    return export
示例#4
0
def generate_export(export_type, extension, username, id_string,
                    export_id=None, filter_query=None, group_delimiter='/',
                    split_select_multiples=True,
                    binary_select_multiples=False,
                    sync_to_gsuit=False, user=None):
    """
    Create appropriate export object given the export type
    """
    time.sleep(5)
    export_type_func_map = {
        Export.XLS_EXPORT: 'to_xls_export',
        Export.CSV_EXPORT: 'to_flat_csv_export',
        Export.CSV_ZIP_EXPORT: 'to_zipped_csv',
        Export.SAV_ZIP_EXPORT: 'to_zipped_sav',
        Export.ANALYSER_EXPORT: 'to_analyser_export'
    }

    xform = XForm.objects.get(
        user__username__iexact=username, id_string__exact=id_string)

    # query mongo for the cursor
    records = query_mongo(username, id_string, filter_query)

    export_builder = ExportBuilder()
    export_builder.GROUP_DELIMITER = group_delimiter
    export_builder.SPLIT_SELECT_MULTIPLES = split_select_multiples
    export_builder.BINARY_SELECT_MULTIPLES = binary_select_multiples
    __version__ = "0"
    try:
        __version__ = filter_query['$and'][0]['__version__']
    except Exception as e:
        print(str(e))
    if __version__:
        survey = build_survey_from_history(xform, __version__)
        if not survey:
            export_builder.set_survey(xform.data_dictionary().survey)
        else:
            export_builder.set_survey(survey)
    else:
        export_builder.set_survey(xform.data_dictionary().survey)

    prefix = slugify('{}_export__{}__{}'.format(export_type, username, id_string))
    temp_file = NamedTemporaryFile(prefix=prefix, suffix=("." + extension))

    # get the export function by export type
    func = getattr(export_builder, export_type_func_map[export_type])

    func.__call__(
        temp_file.name, records, username, id_string, filter_query)

    # generate filename
    basename = "%s_%s" % (
        id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
    if export_type == Export.ANALYSER_EXPORT:
        # Analyser exports should be distinguished by more than just their file extension.
        basename= '{}_ANALYSER_{}'.format(id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
    filename = basename + "." + extension

    # check filename is unique
    while not Export.is_filename_unique(xform, filename):
        filename = increment_index_in_filename(filename)

    file_path = os.path.join(
        username,
        'exports',
        id_string,
        export_type,
        filename)

    # TODO: if s3 storage, make private - how will we protect local storage??
    storage = get_storage_class()()
    # seek to the beginning as required by storage classes
    
    print sync_to_gsuit, 'file_url--------->', temp_file, filter_query

    try:
        if sync_to_gsuit == True and '__version__' not in filter_query['$and'][0]:
            if not os.path.exists("media/forms/"):
                os.makedirs("media/forms/")

            temporarylocation="media/forms/submissions_{}.xls".format(id_string)
            import shutil
            shutil.copy(temp_file.name, temporarylocation)
            fxf_form = FieldSightXF.objects.get(pk=filter_query['$and'][0]['fs_project_uuid'])
            upload_to_drive(temporarylocation, str(fxf_form.id) + '_' +id_string, None, fxf_form.project, user)
        
            os.remove(temporarylocation)
        
    except Exception as e:
        print e.__dict__
    # get or create export object
    temp_file.seek(0)
    export_filename = storage.save(
        file_path,
        File(temp_file, file_path))
    
    dir_name, basename = os.path.split(export_filename)
    temp_file.close()
    if export_id:
        export = Export.objects.get(id=export_id)
    else:
        fsxf = filter_query.values()[0]
        # print("fsxf", fsxf)
        export = Export(xform=xform, export_type=export_type, fsxf_id=fsxf)
    export.filedir = dir_name
    export.filename = basename
    export.internal_status = Export.SUCCESSFUL
    # dont persist exports that have a filter
    if filter_query is None:
        export.save()
    export.save()
    return export
def generate_export(export_type,
                    extension,
                    username,
                    id_string,
                    export_id=None,
                    filter_query=None,
                    group_delimiter='/',
                    split_select_multiples=True,
                    binary_select_multiples=False,
                    show_label=False):
    """
    Create appropriate export object given the export type
    """
    # TODO resolve circular import
    from onadata.apps.viewer.models.export import Export
    export_type_func_map = {
        Export.XLS_EXPORT: 'to_xls_export',
        Export.CSV_EXPORT: 'to_flat_csv_export',
        Export.CSV_ZIP_EXPORT: 'to_zipped_csv',
        Export.SAV_ZIP_EXPORT: 'to_zipped_sav',
        Export.DB_EXPORT: 'to_postgres_sav',
    }

    xform = XForm.objects.get(user__username__iexact=username,
                              id_string__exact=id_string)

    # query mongo for the cursor
    records = query_mongo(username, id_string, filter_query)

    export_builder = ExportBuilder()
    export_builder.GROUP_DELIMITER = group_delimiter
    export_builder.SPLIT_SELECT_MULTIPLES = split_select_multiples
    export_builder.BINARY_SELECT_MULTIPLES = binary_select_multiples
    export_builder.SHOW_LABEL = show_label
    export_builder.set_survey(xform.data_dictionary().survey)
    print "xform.data_dictionary().survey"
    print xform.data_dictionary().survey
    export_builder.FORM_OWNER = xform.user.username

    temp_file = NamedTemporaryFile(suffix=("." + extension))

    # get the export function by export type
    func = getattr(export_builder, export_type_func_map[export_type])

    func.__call__(temp_file.name, records, username, id_string, filter_query)

    # generate filename
    basename = "%s_%s" % (id_string,
                          datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
    filename = basename + "." + extension

    # check filename is unique
    while not Export.is_filename_unique(xform, filename):
        filename = increment_index_in_filename(filename)

    file_path = os.path.join(username, 'exports', id_string, export_type,
                             filename)

    # TODO: if s3 storage, make private - how will we protect local storage??
    storage = get_storage_class()()
    # seek to the beginning as required by storage classes
    temp_file.seek(0)
    export_filename = storage.save(file_path, File(temp_file, file_path))
    temp_file.close()

    dir_name, basename = os.path.split(export_filename)

    # get or create export object
    if export_id:
        export = Export.objects.get(id=export_id)
    else:
        export = Export(xform=xform, export_type=export_type)
    export.filedir = dir_name
    export.filename = basename
    export.internal_status = Export.SUCCESSFUL
    # start: original code commented out
    # dont persist exports that have a filter
    # if filter_query is None:
    #     export.save()
    # end: original code commented out
    export.save()
    return export