def test_csv_returns_all_users(self):
self.register(fullname="Manolita")
self.signout()
self.register(fullname="Juan Jose",
name="juan",
email="*****@*****.**",
password="******")
self.signout()
self.register(fullname="Juan Jose2",
name="juan2",
email="*****@*****.**",
password="******")
self.signin()
res = self.app.get('/admin/users/export?format=csv',
follow_redirects=True)
data = res.data
csv_content = StringIO.StringIO(data)
csvreader = unicode_csv_reader(csv_content)
# number of users is -1 because the first row in csv are the headers
number_of_users = -1
for row in csvreader:
number_of_users += 1
assert number_of_users == 3, number_of_users
def import_task(short_name):
app = App.query.filter_by(short_name=short_name)\
.first_or_404()
form = BulkTaskImportForm(request.form)
if form.validate_on_submit():
r = requests.get(form.csv_url.data)
if r.status_code == 403:
flash("Oops! It looks like you don't have permission to access"
" that file!", 'error')
return render_template('/applications/import.html',
app=app, form=form)
if (not 'text/plain' in r.headers['content-type'] and not 'text/csv'
in r.headers['content-type']):
flash("Oops! That file doesn't look like a CSV file.", 'error')
return render_template('/applications/import.html',
app=app, form=form)
empty = True
csvcontent = StringIO(r.text)
csvreader = unicode_csv_reader(csvcontent)
# TODO: check for errors
headers = []
fields = set(['state', 'quorum', 'calibration', 'priority_0',
'n_answers'])
field_header_index = []
try:
for row in csvreader:
if not headers:
headers = row
if len(headers) != len(set(headers)):
flash('The CSV file you uploaded has two headers with'
' the same name.', 'error')
return render_template('/applications/import.html',
app=app, form=form)
field_headers = set(headers) & fields
for field in field_headers:
field_header_index.append(headers.index(field))
else:
info = {}
task = model.Task(app=app)
for index, cell in enumerate(row):
if index in field_header_index:
setattr(task, headers[index], cell)
else:
info[headers[index]] = cell
task.info = info
db.session.add(task)
db.session.commit()
empty = False
if empty:
flash('Oops! It looks like the CSV file is empty.', 'error')
return render_template('/applications/import.html',
app=app, form=form)
flash('Tasks imported successfully!', 'success')
return redirect(url_for('.details', short_name=app.short_name))
except:
flash('Oops! Looks like there was an error with processing '
'that file!', 'error')
return render_template('/applications/import.html',
app=app, form=form)
def test_csv_returns_all_users(self, password_needed, send):
password_needed.return_value = False
restricted = UserFactory.create(restrict=True, id=5000016)
self.register(fullname="Manolita")
project = self.create_project_and_tasks()
self.signin()
self.contribute(project)
self.signout()
self.register(fullname="Juan Jose", name="juan",
email="*****@*****.**", password="******")
self.signin(email="*****@*****.**", password="******")
self.contribute(project)
self.signout()
self.register(fullname="Juan Jose2", name="juan2",
email="*****@*****.**", password="******")
self.signin(email="*****@*****.**", password="******")
self.contribute(project)
self.signout()
self.signin()
export_all_users('csv', '*****@*****.**')
args, _ = send.call_args
data = args[0]['attachments'][0].data
assert restricted.name not in data
csv_content = StringIO.StringIO(data)
csvreader = unicode_csv_reader(csv_content)
# number of users is -1 because the first row in csv are the headers
number_of_users = -1
for row in csvreader:
number_of_users += 1
assert number_of_users == 4, number_of_users # user report returning all users also returns [email protected]
def test_unicode_csv_reader(self):
"""Test unicode_csv_reader works."""
fake_csv = ['one, two, three']
err_msg = "Each cell should be encoded as Unicode"
for row in util.unicode_csv_reader(fake_csv):
for item in row:
assert type(item) == unicode, err_msg
def test_unicode_csv_reader(self):
"""Test unicode_csv_reader works."""
fake_csv = ['one, two, three']
err_msg = "Each cell should be encoded as Unicode"
for row in util.unicode_csv_reader(fake_csv):
for item in row:
assert type(item) == unicode, err_msg
def _get_csv_data_from_request(self, csv_filename):
if csv_filename is None:
msg = ("Not a valid csv file for import")
raise BulkImportException(gettext(msg), 'error')
csv_file = FileStorage(io.open(csv_filename, encoding='utf-8-sig')) #utf-8-sig to ignore BOM
csvcontent = io.StringIO(csv_file.stream.read().decode("UTF8"))
csvreader = unicode_csv_reader(csvcontent)
return self._import_csv_users(csvreader)
class BulkUserCSVImport(BulkUserImport):
"""Class to import users from CSV in bulk."""
importer_id = "userimport"
def __init__(self, **form_data):
self.form_data = form_data
def _get_data(self):
"""Get data."""
return self.form_data['csv_filename']
def count_users(self):
return len([user for user in self.users()])
def users(self):
"""Get users from a given URL."""
csv_filename = self._get_data()
return self._get_csv_data_from_request(csv_filename)
def _get_csv_data_from_request(self, csv_filename):
if csv_filename is None:
msg = ("Not a valid csv file for import")
raise BulkImportException(gettext(msg), 'error')
retry = 0
csv_file = None
while retry < 5:
try:
csv_file = FileStorage(open(csv_filename, 'r'))
break
except IOError, e:
time.sleep(1)
retry += 1
if csv_file is None:
if (('text/plain' not in request.headers['content-type']) and
('text/csv' not in request.headers['content-type']) and
('multipart/form-data' not in request.headers['content-type'])):
msg = gettext("Oops! That file doesn't look like the right file.")
raise BulkImportException(msg, 'error')
request.encoding = 'utf-8'
csv_file = request.files['file']
if csv_file is None or csv_file.stream is None:
msg = ("Not a valid csv file for import")
raise BulkImportException(gettext(msg), 'error')
csv_file.stream.seek(0)
csvcontent = io.StringIO(csv_file.stream.read().decode("UTF8"))
csvreader = unicode_csv_reader(csvcontent)
return self._import_csv_users(csvreader)
def get_csv_data_from_request(self, r):
if r.status_code == 403:
msg = "Oops! It looks like you don't have permission to access" \
" that file"
raise BulkImportException(gettext(msg), 'error')
if ((not 'text/plain' in r.headers['content-type'])
and (not 'text/csv' in r.headers['content-type'])):
msg = gettext("Oops! That file doesn't look like the right file.")
raise BulkImportException(msg, 'error')
csvcontent = StringIO(r.text)
csvreader = unicode_csv_reader(csvcontent)
return self.import_csv_tasks(csvreader)
def _get_csv_data_from_request(self, r):
if r.status_code == 403:
msg = "Oops! It looks like you don't have permission to access" \
" that file"
raise BulkImportException(gettext(msg), 'error')
if ((not 'text/plain' in r.headers['content-type']) and
(not 'text/csv' in r.headers['content-type'])):
msg = gettext("Oops! That file doesn't look like the right file.")
raise BulkImportException(msg, 'error')
csvcontent = StringIO(r.text)
csvreader = unicode_csv_reader(csvcontent)
return self._import_csv_tasks(csvreader)
def _get_csv_data_from_request(self, r):
"""Get CSV data from a request."""
if r.status_code == 403:
msg = ("Oops! It looks like you don't have permission to access"
" that file")
raise BulkImportException(gettext(msg), 'error')
if (('text/plain' not in r.headers['content-type'])
and ('text/csv' not in r.headers['content-type'])):
msg = gettext("Oops! That file doesn't look like the right file.")
raise BulkImportException(msg, 'error')
r.encoding = 'utf-8'
csvcontent = StringIO(r.text)
csvreader = unicode_csv_reader(csvcontent)
return self._import_csv_tasks(csvreader)
def _get_csv_data_from_request(self, csv_filename):
if csv_filename is None:
msg = ("Not a valid csv file for import")
raise BulkImportException(gettext(msg), 'error')
datafile = get_import_csv_file(csv_filename)
csv_file = FileStorage(io.open(datafile.name, encoding='utf-8-sig')) #utf-8-sig to ignore BOM
if csv_file is None or csv_file.stream is None:
msg = ("Unable to load csv file for import, file {0}".format(csv_filename))
raise BulkImportException(gettext(msg), 'error')
csv_file.stream.seek(0)
csvcontent = io.StringIO(csv_file.stream.read())
csvreader = unicode_csv_reader(csvcontent)
return list(self._import_csv_tasks(csvreader))
class _BulkTaskLocalCSVImport(_BulkTaskCSVImport):
"""Class to import CSV tasks in bulk from local file."""
importer_id = "localcsv"
def _get_data(self, **form_data):
"""Get data."""
return form_data['csv_filename']
def _get_csv_data_from_request(self, csv_filename):
if csv_filename is None:
msg = ("Not a valid csv file for import")
raise BulkImportException(gettext(msg), 'error')
retry = 0
csv_file = None
while retry < 5:
try:
csv_file = FileStorage(open(csv_filename, 'r'))
break
except IOError, e:
time.sleep(1)
retry += 1
if csv_file is None:
if (('text/plain' not in request.headers['content-type'])
and ('text/csv' not in request.headers['content-type'])
and ('multipart/form-data'
not in request.headers['content-type'])):
msg = gettext(
"Oops! That file doesn't look like the right file.")
raise BulkImportException(msg, 'error')
request.encoding = 'utf-8'
csv_file = request.files['file']
if csv_file is None or file.stream is None:
msg = ("Not a valid csv file for import")
raise BulkImportException(gettext(msg), 'error')
csv_file.stream.seek(0)
csvcontent = io.StringIO(csv_file.stream.read().decode("UTF8"))
csvreader = unicode_csv_reader(csvcontent)
return self._import_csv_tasks(csvreader)
def test_csv_returns_all_users(self):
self.register(fullname="Manolita")
self.signout()
self.register(fullname="Juan Jose", username="******", email="*****@*****.**", password="******")
self.signout()
self.register(fullname="Juan Jose2", username="******", email="*****@*****.**", password="******")
self.signin()
res = self.app.get("/admin/users/export?format=csv", follow_redirects=True)
data = res.data
csv_content = StringIO.StringIO(data)
csvreader = unicode_csv_reader(csv_content)
# number of users is -1 because the first row in csv are the headers
number_of_users = -1
for row in csvreader:
number_of_users += 1
assert number_of_users == 3, number_of_users
def _get_csv_data_from_request(self, csv_filename):
if csv_filename is None:
msg = ("Not a valid csv file for import")
raise BulkImportException(gettext(msg), 'error')
if (('text/plain' not in request.headers['content-type']) and
('text/csv' not in request.headers['content-type']) and
('multipart/form-data' not in request.headers['content-type'])):
msg = gettext("Oops! That file doesn't look like the right file.")
raise BulkImportException(msg, 'error')
request.encoding = 'utf-8'
file = request.files['file']
if file is None or file.stream is None:
msg = ("Not a valid csv file for import")
raise BulkImportException(gettext(msg), 'error')
file.stream.seek(0)
csvcontent = io.StringIO(file.stream.read().decode("UTF8")) #csvcontent = StringIO(file.stream.read())
csvreader = unicode_csv_reader(csvcontent)
return self._import_csv_tasks(csvreader)
def _get_csv_data_from_request(self, csv_filename):
if csv_filename is None:
msg = ("Not a valid csv file for import")
raise BulkImportException(gettext(msg), 'error')
retry = 0
csv_file = None
datafile = self.get_local_csv_import_file_from_s3(csv_filename)
if not datafile:
return []
csv_file = FileStorage(open(datafile.name, 'r'))
if csv_file is None or csv_file.stream is None:
msg = ("Unable to load csv file for import, file {0}".format(
csv_filename))
raise BulkImportException(gettext(msg), 'error')
csv_file.stream.seek(0)
csvcontent = io.StringIO(csv_file.stream.read().decode("UTF8"))
csvreader = unicode_csv_reader(csvcontent)
return list(self._import_csv_tasks(csvreader))
class BulkTaskLocalCSVImport(BulkTaskCSVImport):
"""Class to import CSV tasks in bulk from local file."""
importer_id = "localCSV"
def __init__(self, **form_data):
self.form_data = form_data
def _get_data(self):
"""Get data."""
return self.form_data['csv_filename']
def count_tasks(self):
return len([task for task in self.tasks()])
def _get_csv_data_from_request(self, csv_filename):
if csv_filename is None:
msg = ("Not a valid csv file for import")
raise BulkImportException(gettext(msg), 'error')
retry = 0
csv_file = None
while retry < 10:
try:
csv_file = FileStorage(
io.open(csv_filename, encoding='utf-8-sig'))
break
except IOError, e:
time.sleep(2)
retry += 1
if csv_file is None or csv_file.stream is None:
msg = ("Unable to load csv file for import, file {0}".format(
csv_filename))
raise BulkImportException(gettext(msg), 'error')
csv_file.stream.seek(0)
csvcontent = io.StringIO(csv_file.stream.read())
csvreader = unicode_csv_reader(csvcontent)
return list(self._import_csv_tasks(csvreader))
def test_csv_returns_all_users(self, password_needed):
password_needed.return_value = False
restricted = UserFactory.create(restrict=True, id=5000016)
self.register(fullname="Manolita")
project = self.create_project_and_tasks()
self.signin()
self.contribute(project)
self.signout()
self.register(fullname="Juan Jose",
name="juan",
email="*****@*****.**",
password="******")
self.signin(email="*****@*****.**", password="******")
self.contribute(project)
self.signout()
self.register(fullname="Juan Jose2",
name="juan2",
email="*****@*****.**",
password="******")
self.signin(email="*****@*****.**", password="******")
self.contribute(project)
self.signout()
self.signin()
res = self.app.get('/admin/users/export?format=csv',
follow_redirects=True)
data = res.data
assert restricted.name not in data
csv_content = StringIO.StringIO(data)
csvreader = unicode_csv_reader(csv_content)
# number of users is -1 because the first row in csv are the headers
number_of_users = -1
for row in csvreader:
number_of_users += 1
assert number_of_users == 4, number_of_users # user report returning all users also returns [email protected]
def import_task(short_name):
app = App.query.filter_by(short_name=short_name)\
.first_or_404()
form = BulkTaskImportForm(request.form)
if form.validate_on_submit():
r = requests.get(form.csv_url.data)
if r.status_code == 403:
flash(
"Oops! It looks like you don't have permission to access"
" that file!", 'error')
return render_template('/applications/import.html',
app=app,
form=form)
if (not 'text/plain' in r.headers['content-type']
and not 'text/csv' in r.headers['content-type']):
flash("Oops! That file doesn't look like a CSV file.", 'error')
return render_template('/applications/import.html',
app=app,
form=form)
empty = True
csvcontent = StringIO(r.text)
csvreader = unicode_csv_reader(csvcontent)
# TODO: check for errors
headers = []
fields = set(
['state', 'quorum', 'calibration', 'priority_0', 'n_answers'])
field_header_index = []
try:
for row in csvreader:
if not headers:
headers = row
if len(headers) != len(set(headers)):
flash(
'The CSV file you uploaded has two headers with'
' the same name.', 'error')
return render_template('/applications/import.html',
app=app,
form=form)
field_headers = set(headers) & fields
for field in field_headers:
field_header_index.append(headers.index(field))
else:
info = {}
task = model.Task(app=app)
for index, cell in enumerate(row):
if index in field_header_index:
setattr(task, headers[index], cell)
else:
info[headers[index]] = cell
task.info = info
db.session.add(task)
db.session.commit()
empty = False
if empty:
flash('Oops! It looks like the CSV file is empty.', 'error')
return render_template('/applications/import.html',
app=app,
form=form)
flash('Tasks imported successfully!', 'success')
return redirect(url_for('.details', short_name=app.short_name))
except:
flash(
'Oops! Looks like there was an error with processing '
'that file!', 'error')
return render_template('/applications/import.html', app=app, form=form)
def import_task(short_name):
app = App.query.filter_by(short_name=short_name).first_or_404()
title = "Applications: %s · Import Tasks" % app.name
dataurl = None
csvform = BulkTaskCSVImportForm(request.form)
gdform = BulkTaskGDImportForm(request.form)
if app.tasks or (request.args.get('template') or request.method == 'POST'):
if request.args.get('template') == 'image':
gdform.googledocs_url.data = \
"https://docs.google.com/spreadsheet/ccc" \
"?key=0AsNlt0WgPAHwdHFEN29mZUF0czJWMUhIejF6dWZXdkE" \
"&usp=sharing"
elif request.args.get('template') == 'map':
gdform.googledocs_url.data = \
"https://docs.google.com/spreadsheet/ccc" \
"?key=0AsNlt0WgPAHwdGZnbjdwcnhKRVNlN1dGXy0tTnNWWXc" \
"&usp=sharing"
elif request.args.get('template') == 'pdf':
gdform.googledocs_url.data = \
"https://docs.google.com/spreadsheet/ccc" \
"?key=0AsNlt0WgPAHwdEVVamc0R0hrcjlGdXRaUXlqRXlJMEE" \
"&usp=sharing"
else:
pass
if 'csv_url' in request.form and csvform.validate_on_submit():
dataurl = csvform.csv_url.data
elif 'googledocs_url' in request.form and gdform.validate_on_submit():
dataurl = ''.join([gdform.googledocs_url.data, '&output=csv'])
if dataurl:
print "dataurl found"
r = requests.get(dataurl)
if r.status_code == 403:
flash("Oops! It looks like you don't have permission to access"
" that file!", 'error')
return render_template('/applications/import.html',
title=title,
app=app,
csvform=csvform,
gdform=gdform)
if (not 'text/plain' in r.headers['content-type'] and not 'text/csv'
in r.headers['content-type']):
flash("Oops! That file doesn't look like the right file.", 'error')
return render_template('/applications/import.html',
title=title,
app=app,
csvform=csvform,
gdform=gdform)
empty = True
csvcontent = StringIO(r.text)
csvreader = unicode_csv_reader(csvcontent)
# TODO: check for errors
headers = []
fields = set(['state', 'quorum', 'calibration', 'priority_0',
'n_answers'])
field_header_index = []
try:
for row in csvreader:
if not headers:
headers = row
if len(headers) != len(set(headers)):
flash('The file you uploaded has two headers with'
' the same name.', 'error')
return render_template('/applications/import.html',
title=title,
app=app,
csvform=csvform,
gdform=gdform)
field_headers = set(headers) & fields
for field in field_headers:
field_header_index.append(headers.index(field))
else:
info = {}
task = model.Task(app=app)
for idx, cell in enumerate(row):
if idx in field_header_index:
setattr(task, headers[idx], cell)
else:
info[headers[idx]] = cell
task.info = info
db.session.add(task)
db.session.commit()
empty = False
if empty:
flash('Oops! It looks like the file is empty.', 'error')
return render_template('/applications/import.html',
title=title,
app=app,
csvform=csvform,
gdform=gdform)
flash('Tasks imported successfully!', 'success')
return redirect(url_for('.details', short_name=app.short_name))
except:
flash('Oops! Looks like there was an error with processing '
'that file!', 'error')
return render_template('/applications/import.html',
title=title,
app=app,
csvform=csvform,
gdform=gdform)
else:
return render_template('/applications/import_options.html',
title=title,
app=app,
csvform=csvform,
gdform=gdform)
