def test_csv_returns_all_users(self):
        self.register(fullname="Manolita")
        self.signout()
        self.register(fullname="Juan Jose",
                      name="juan",
                      email="*****@*****.**",
                      password="******")
        self.signout()
        self.register(fullname="Juan Jose2",
                      name="juan2",
                      email="*****@*****.**",
                      password="******")
        self.signin()

        res = self.app.get('/admin/users/export?format=csv',
                           follow_redirects=True)
        data = res.data
        csv_content = StringIO.StringIO(data)
        csvreader = unicode_csv_reader(csv_content)

        # number of users is -1 because the first row in csv are the headers
        number_of_users = -1
        for row in csvreader:
            number_of_users += 1

        assert number_of_users == 3, number_of_users
Example #2
0
def import_task(short_name):
    app = App.query.filter_by(short_name=short_name)\
            .first_or_404()

    form = BulkTaskImportForm(request.form)
    if form.validate_on_submit():
        r = requests.get(form.csv_url.data)
        if r.status_code == 403:
            flash("Oops! It looks like you don't have permission to access"
                  " that file!", 'error')
            return render_template('/applications/import.html',
                    app=app, form=form)
        if (not 'text/plain' in r.headers['content-type'] and not 'text/csv'
                in r.headers['content-type']):
            flash("Oops! That file doesn't look like a CSV file.", 'error')
            return render_template('/applications/import.html',
                    app=app, form=form)
        empty = True
        csvcontent = StringIO(r.text)
        csvreader = unicode_csv_reader(csvcontent)
        # TODO: check for errors
        headers = []
        fields = set(['state', 'quorum', 'calibration', 'priority_0',
                'n_answers'])
        field_header_index = []
        try:
            for row in csvreader:
                if not headers:
                    headers = row
                    if len(headers) != len(set(headers)):
                        flash('The CSV file you uploaded has two headers with'
                              ' the same name.', 'error')
                        return render_template('/applications/import.html',
                            app=app, form=form)
                    field_headers = set(headers) & fields
                    for field in field_headers:
                        field_header_index.append(headers.index(field))
                else:
                    info = {}
                    task = model.Task(app=app)
                    for index, cell in enumerate(row):
                        if index in field_header_index:
                            setattr(task, headers[index], cell)
                        else:
                            info[headers[index]] = cell
                    task.info = info
                    db.session.add(task)
                    db.session.commit()
                    empty = False
            if empty:
                flash('Oops! It looks like the CSV file is empty.', 'error')
                return render_template('/applications/import.html',
                    app=app, form=form)
            flash('Tasks imported successfully!', 'success')
            return redirect(url_for('.details', short_name=app.short_name))
        except:
            flash('Oops! Looks like there was an error with processing '
                  'that file!', 'error')
    return render_template('/applications/import.html',
            app=app, form=form)
    def test_csv_returns_all_users(self, password_needed, send):
        password_needed.return_value = False
        restricted = UserFactory.create(restrict=True, id=5000016)
        self.register(fullname="Manolita")
        project = self.create_project_and_tasks()
        self.signin()
        self.contribute(project)
        self.signout()

        self.register(fullname="Juan Jose", name="juan",
                      email="*****@*****.**", password="******")
        self.signin(email="*****@*****.**", password="******")
        self.contribute(project)
        self.signout()

        self.register(fullname="Juan Jose2", name="juan2",
                      email="*****@*****.**", password="******")
        self.signin(email="*****@*****.**", password="******")
        self.contribute(project)
        self.signout()
        self.signin()

        export_all_users('csv', '*****@*****.**')
        args, _ = send.call_args
        data = args[0]['attachments'][0].data

        assert restricted.name not in data
        csv_content = StringIO.StringIO(data)
        csvreader = unicode_csv_reader(csv_content)
        # number of users is -1 because the first row in csv are the headers
        number_of_users = -1
        for row in csvreader:
            number_of_users += 1

        assert number_of_users == 4, number_of_users # user report returning all users also returns [email protected]
Example #4
0
 def test_unicode_csv_reader(self):
     """Test unicode_csv_reader works."""
     fake_csv = ['one, two, three']
     err_msg = "Each cell should be encoded as Unicode"
     for row in util.unicode_csv_reader(fake_csv):
         for item in row:
             assert type(item) == unicode, err_msg
Example #5
0
 def test_unicode_csv_reader(self):
     """Test unicode_csv_reader works."""
     fake_csv = ['one, two, three']
     err_msg = "Each cell should be encoded as Unicode"
     for row in util.unicode_csv_reader(fake_csv):
         for item in row:
             assert type(item) == unicode, err_msg
Example #6
0
    def _get_csv_data_from_request(self, csv_filename):
        if csv_filename is None:
            msg = ("Not a valid csv file for import")
            raise BulkImportException(gettext(msg), 'error')

        csv_file = FileStorage(io.open(csv_filename, encoding='utf-8-sig'))    #utf-8-sig to ignore BOM

        csvcontent = io.StringIO(csv_file.stream.read().decode("UTF8"))
        csvreader = unicode_csv_reader(csvcontent)
        return self._import_csv_users(csvreader)
Example #7
0
class BulkUserCSVImport(BulkUserImport):

    """Class to import users from CSV in bulk."""

    importer_id = "userimport"

    def __init__(self, **form_data):
       self.form_data = form_data

    def _get_data(self):
        """Get data."""
        return self.form_data['csv_filename']

    def count_users(self):
        return len([user for user in self.users()])

    def users(self):
        """Get users from a given URL."""
        csv_filename = self._get_data()
        return self._get_csv_data_from_request(csv_filename)
        
    def _get_csv_data_from_request(self, csv_filename):
        if csv_filename is None:
            msg = ("Not a valid csv file for import")
            raise BulkImportException(gettext(msg), 'error')

        retry = 0
        csv_file = None
        while retry < 5:
            try:
                csv_file = FileStorage(open(csv_filename, 'r'))
                break
            except IOError, e:
                time.sleep(1)
                retry += 1
                
        if csv_file is None:
           if (('text/plain' not in request.headers['content-type']) and
                   ('text/csv' not in request.headers['content-type']) and
                   ('multipart/form-data' not in request.headers['content-type'])):
               msg = gettext("Oops! That file doesn't look like the right file.")
               raise BulkImportException(msg, 'error')

           request.encoding = 'utf-8'
           csv_file = request.files['file']
           if csv_file is None or csv_file.stream is None:
               msg = ("Not a valid csv file for import")
               raise BulkImportException(gettext(msg), 'error')

        csv_file.stream.seek(0)
        csvcontent = io.StringIO(csv_file.stream.read().decode("UTF8"))
        csvreader = unicode_csv_reader(csvcontent)
        return self._import_csv_users(csvreader)
Example #8
0
    def get_csv_data_from_request(self, r):
        if r.status_code == 403:
            msg = "Oops! It looks like you don't have permission to access" \
                " that file"
            raise BulkImportException(gettext(msg), 'error')
        if ((not 'text/plain' in r.headers['content-type'])
                and (not 'text/csv' in r.headers['content-type'])):
            msg = gettext("Oops! That file doesn't look like the right file.")
            raise BulkImportException(msg, 'error')

        csvcontent = StringIO(r.text)
        csvreader = unicode_csv_reader(csvcontent)
        return self.import_csv_tasks(csvreader)
Example #9
0
    def _get_csv_data_from_request(self, r):
        if r.status_code == 403:
            msg = "Oops! It looks like you don't have permission to access" \
                " that file"
            raise BulkImportException(gettext(msg), 'error')
        if ((not 'text/plain' in r.headers['content-type']) and
                (not 'text/csv' in r.headers['content-type'])):
            msg = gettext("Oops! That file doesn't look like the right file.")
            raise BulkImportException(msg, 'error')

        csvcontent = StringIO(r.text)
        csvreader = unicode_csv_reader(csvcontent)
        return self._import_csv_tasks(csvreader)
Example #10
0
    def _get_csv_data_from_request(self, r):
        """Get CSV data from a request."""
        if r.status_code == 403:
            msg = ("Oops! It looks like you don't have permission to access"
                   " that file")
            raise BulkImportException(gettext(msg), 'error')
        if (('text/plain' not in r.headers['content-type'])
                and ('text/csv' not in r.headers['content-type'])):
            msg = gettext("Oops! That file doesn't look like the right file.")
            raise BulkImportException(msg, 'error')

        r.encoding = 'utf-8'
        csvcontent = StringIO(r.text)
        csvreader = unicode_csv_reader(csvcontent)
        return self._import_csv_tasks(csvreader)
Example #11
0
    def _get_csv_data_from_request(self, csv_filename):
        if csv_filename is None:
            msg = ("Not a valid csv file for import")
            raise BulkImportException(gettext(msg), 'error')

        datafile = get_import_csv_file(csv_filename)
        csv_file = FileStorage(io.open(datafile.name, encoding='utf-8-sig'))    #utf-8-sig to ignore BOM

        if csv_file is None or csv_file.stream is None:
            msg = ("Unable to load csv file for import, file {0}".format(csv_filename))
            raise BulkImportException(gettext(msg), 'error')

        csv_file.stream.seek(0)
        csvcontent = io.StringIO(csv_file.stream.read())
        csvreader = unicode_csv_reader(csvcontent)
        return list(self._import_csv_tasks(csvreader))
Example #12
0
class _BulkTaskLocalCSVImport(_BulkTaskCSVImport):
    """Class to import CSV tasks in bulk from local file."""

    importer_id = "localcsv"

    def _get_data(self, **form_data):
        """Get data."""
        return form_data['csv_filename']

    def _get_csv_data_from_request(self, csv_filename):
        if csv_filename is None:
            msg = ("Not a valid csv file for import")
            raise BulkImportException(gettext(msg), 'error')

        retry = 0
        csv_file = None
        while retry < 5:
            try:
                csv_file = FileStorage(open(csv_filename, 'r'))
                break
            except IOError, e:
                time.sleep(1)
                retry += 1

        if csv_file is None:
            if (('text/plain' not in request.headers['content-type'])
                    and ('text/csv' not in request.headers['content-type'])
                    and ('multipart/form-data'
                         not in request.headers['content-type'])):
                msg = gettext(
                    "Oops! That file doesn't look like the right file.")
                raise BulkImportException(msg, 'error')

            request.encoding = 'utf-8'
            csv_file = request.files['file']
            if csv_file is None or file.stream is None:
                msg = ("Not a valid csv file for import")
                raise BulkImportException(gettext(msg), 'error')

        csv_file.stream.seek(0)
        csvcontent = io.StringIO(csv_file.stream.read().decode("UTF8"))
        csvreader = unicode_csv_reader(csvcontent)
        return self._import_csv_tasks(csvreader)
    def test_csv_returns_all_users(self):
        self.register(fullname="Manolita")
        self.signout()
        self.register(fullname="Juan Jose", username="******", email="*****@*****.**", password="******")
        self.signout()
        self.register(fullname="Juan Jose2", username="******", email="*****@*****.**", password="******")
        self.signin()

        res = self.app.get("/admin/users/export?format=csv", follow_redirects=True)
        data = res.data
        csv_content = StringIO.StringIO(data)
        csvreader = unicode_csv_reader(csv_content)

        # number of users is -1 because the first row in csv are the headers
        number_of_users = -1
        for row in csvreader:
            number_of_users += 1

        assert number_of_users == 3, number_of_users
Example #14
0
    def _get_csv_data_from_request(self, csv_filename):
        if csv_filename is None:
            msg = ("Not a valid csv file for import")
            raise BulkImportException(gettext(msg), 'error')

        if (('text/plain' not in request.headers['content-type']) and
                ('text/csv' not in request.headers['content-type']) and
                ('multipart/form-data' not in request.headers['content-type'])):
            msg = gettext("Oops! That file doesn't look like the right file.")
            raise BulkImportException(msg, 'error')

        request.encoding = 'utf-8'
        file = request.files['file']
        if file is None or file.stream is None:
            msg = ("Not a valid csv file for import")
            raise BulkImportException(gettext(msg), 'error')

        file.stream.seek(0)
        csvcontent = io.StringIO(file.stream.read().decode("UTF8")) #csvcontent = StringIO(file.stream.read())
        csvreader = unicode_csv_reader(csvcontent)
        return self._import_csv_tasks(csvreader)
Example #15
0
    def _get_csv_data_from_request(self, csv_filename):
        if csv_filename is None:
            msg = ("Not a valid csv file for import")
            raise BulkImportException(gettext(msg), 'error')

        retry = 0
        csv_file = None
        datafile = self.get_local_csv_import_file_from_s3(csv_filename)
        if not datafile:
            return []

        csv_file = FileStorage(open(datafile.name, 'r'))
        if csv_file is None or csv_file.stream is None:
            msg = ("Unable to load csv file for import, file {0}".format(
                csv_filename))
            raise BulkImportException(gettext(msg), 'error')

        csv_file.stream.seek(0)
        csvcontent = io.StringIO(csv_file.stream.read().decode("UTF8"))
        csvreader = unicode_csv_reader(csvcontent)
        return list(self._import_csv_tasks(csvreader))
Example #16
0
class BulkTaskLocalCSVImport(BulkTaskCSVImport):
    """Class to import CSV tasks in bulk from local file."""

    importer_id = "localCSV"

    def __init__(self, **form_data):
        self.form_data = form_data

    def _get_data(self):
        """Get data."""
        return self.form_data['csv_filename']

    def count_tasks(self):
        return len([task for task in self.tasks()])

    def _get_csv_data_from_request(self, csv_filename):
        if csv_filename is None:
            msg = ("Not a valid csv file for import")
            raise BulkImportException(gettext(msg), 'error')

        retry = 0
        csv_file = None
        while retry < 10:
            try:
                csv_file = FileStorage(
                    io.open(csv_filename, encoding='utf-8-sig'))
                break
            except IOError, e:
                time.sleep(2)
                retry += 1

        if csv_file is None or csv_file.stream is None:
            msg = ("Unable to load csv file for import, file {0}".format(
                csv_filename))
            raise BulkImportException(gettext(msg), 'error')

        csv_file.stream.seek(0)
        csvcontent = io.StringIO(csv_file.stream.read())
        csvreader = unicode_csv_reader(csvcontent)
        return list(self._import_csv_tasks(csvreader))
Example #17
0
    def test_csv_returns_all_users(self, password_needed):
        password_needed.return_value = False
        restricted = UserFactory.create(restrict=True, id=5000016)
        self.register(fullname="Manolita")
        project = self.create_project_and_tasks()
        self.signin()
        self.contribute(project)
        self.signout()

        self.register(fullname="Juan Jose",
                      name="juan",
                      email="*****@*****.**",
                      password="******")
        self.signin(email="*****@*****.**", password="******")
        self.contribute(project)
        self.signout()

        self.register(fullname="Juan Jose2",
                      name="juan2",
                      email="*****@*****.**",
                      password="******")
        self.signin(email="*****@*****.**", password="******")
        self.contribute(project)
        self.signout()
        self.signin()

        res = self.app.get('/admin/users/export?format=csv',
                           follow_redirects=True)
        data = res.data
        assert restricted.name not in data
        csv_content = StringIO.StringIO(data)
        csvreader = unicode_csv_reader(csv_content)
        # number of users is -1 because the first row in csv are the headers
        number_of_users = -1
        for row in csvreader:
            number_of_users += 1

        assert number_of_users == 4, number_of_users  # user report returning all users also returns [email protected]
Example #18
0
def import_task(short_name):
    app = App.query.filter_by(short_name=short_name)\
            .first_or_404()

    form = BulkTaskImportForm(request.form)
    if form.validate_on_submit():
        r = requests.get(form.csv_url.data)
        if r.status_code == 403:
            flash(
                "Oops! It looks like you don't have permission to access"
                " that file!", 'error')
            return render_template('/applications/import.html',
                                   app=app,
                                   form=form)
        if (not 'text/plain' in r.headers['content-type']
                and not 'text/csv' in r.headers['content-type']):
            flash("Oops! That file doesn't look like a CSV file.", 'error')
            return render_template('/applications/import.html',
                                   app=app,
                                   form=form)
        empty = True
        csvcontent = StringIO(r.text)
        csvreader = unicode_csv_reader(csvcontent)
        # TODO: check for errors
        headers = []
        fields = set(
            ['state', 'quorum', 'calibration', 'priority_0', 'n_answers'])
        field_header_index = []
        try:
            for row in csvreader:
                if not headers:
                    headers = row
                    if len(headers) != len(set(headers)):
                        flash(
                            'The CSV file you uploaded has two headers with'
                            ' the same name.', 'error')
                        return render_template('/applications/import.html',
                                               app=app,
                                               form=form)
                    field_headers = set(headers) & fields
                    for field in field_headers:
                        field_header_index.append(headers.index(field))
                else:
                    info = {}
                    task = model.Task(app=app)
                    for index, cell in enumerate(row):
                        if index in field_header_index:
                            setattr(task, headers[index], cell)
                        else:
                            info[headers[index]] = cell
                    task.info = info
                    db.session.add(task)
                    db.session.commit()
                    empty = False
            if empty:
                flash('Oops! It looks like the CSV file is empty.', 'error')
                return render_template('/applications/import.html',
                                       app=app,
                                       form=form)
            flash('Tasks imported successfully!', 'success')
            return redirect(url_for('.details', short_name=app.short_name))
        except:
            flash(
                'Oops! Looks like there was an error with processing '
                'that file!', 'error')
    return render_template('/applications/import.html', app=app, form=form)
Example #19
0
def import_task(short_name):
    app = App.query.filter_by(short_name=short_name).first_or_404()
    title = "Applications: %s &middot; Import Tasks" % app.name

    dataurl = None
    csvform = BulkTaskCSVImportForm(request.form)
    gdform = BulkTaskGDImportForm(request.form)
    if app.tasks or (request.args.get('template') or request.method == 'POST'):
        if request.args.get('template') == 'image':
            gdform.googledocs_url.data = \
                    "https://docs.google.com/spreadsheet/ccc" \
                    "?key=0AsNlt0WgPAHwdHFEN29mZUF0czJWMUhIejF6dWZXdkE" \
                    "&usp=sharing"
        elif request.args.get('template') == 'map':
            gdform.googledocs_url.data = \
                    "https://docs.google.com/spreadsheet/ccc" \
                    "?key=0AsNlt0WgPAHwdGZnbjdwcnhKRVNlN1dGXy0tTnNWWXc" \
                    "&usp=sharing"
        elif request.args.get('template') == 'pdf':
            gdform.googledocs_url.data = \
                    "https://docs.google.com/spreadsheet/ccc" \
                    "?key=0AsNlt0WgPAHwdEVVamc0R0hrcjlGdXRaUXlqRXlJMEE" \
                    "&usp=sharing"
        else:
            pass
        if 'csv_url' in request.form and csvform.validate_on_submit():
            dataurl = csvform.csv_url.data
        elif 'googledocs_url' in request.form and gdform.validate_on_submit():
            dataurl = ''.join([gdform.googledocs_url.data, '&output=csv'])
        if dataurl:
            print "dataurl found"
            r = requests.get(dataurl)
            if r.status_code == 403:
                flash("Oops! It looks like you don't have permission to access"
                      " that file!", 'error')
                return render_template('/applications/import.html',
                                       title=title,
                                       app=app,
                                       csvform=csvform,
                                       gdform=gdform)
            if (not 'text/plain' in r.headers['content-type'] and not 'text/csv'
                    in r.headers['content-type']):
                flash("Oops! That file doesn't look like the right file.", 'error')
                return render_template('/applications/import.html',
                                       title=title,
                                       app=app,
                                       csvform=csvform,
                                       gdform=gdform)
            empty = True
            csvcontent = StringIO(r.text)
            csvreader = unicode_csv_reader(csvcontent)
            # TODO: check for errors
            headers = []
            fields = set(['state', 'quorum', 'calibration', 'priority_0',
                          'n_answers'])
            field_header_index = []
            try:
                for row in csvreader:
                    if not headers:
                        headers = row
                        if len(headers) != len(set(headers)):
                            flash('The file you uploaded has two headers with'
                                  ' the same name.', 'error')
                            return render_template('/applications/import.html',
                                                   title=title,
                                                   app=app,
                                                   csvform=csvform,
                                                   gdform=gdform)
                        field_headers = set(headers) & fields
                        for field in field_headers:
                            field_header_index.append(headers.index(field))
                    else:
                        info = {}
                        task = model.Task(app=app)
                        for idx, cell in enumerate(row):
                            if idx in field_header_index:
                                setattr(task, headers[idx], cell)
                            else:
                                info[headers[idx]] = cell
                        task.info = info
                        db.session.add(task)
                        db.session.commit()
                        empty = False
                if empty:
                    flash('Oops! It looks like the file is empty.', 'error')
                    return render_template('/applications/import.html',
                                           title=title,
                                           app=app,
                                           csvform=csvform,
                                           gdform=gdform)
                flash('Tasks imported successfully!', 'success')
                return redirect(url_for('.details', short_name=app.short_name))
            except:
                flash('Oops! Looks like there was an error with processing '
                      'that file!', 'error')
        return render_template('/applications/import.html',
                               title=title,
                               app=app,
                               csvform=csvform,
                               gdform=gdform)
    else:
        return render_template('/applications/import_options.html',
                        title=title,
                        app=app,
                        csvform=csvform,
                        gdform=gdform)