def csv_export(short_name):
"""Export project results as a CSV file.
:param short_name: The short name of the project.
"""
project = project_repo.get_by_shortname(short_name)
if project is None: # pragma: no cover
abort(404)
si = StringIO.StringIO()
writer = UnicodeWriter(si)
exporter = Exporter()
name = exporter._project_name_latin_encoded(project)
secure_name = secure_filename('{0}_{1}.csv'.format(name, 'results'))
results = result_repo.filter_by(project_id=project.id)
data = []
for r in results:
row = {k: v for k, v in r.dictize().items()}
if isinstance(row['info'], dict): # Explode info
keys = row['info'].keys()
for k in keys:
row['info_{0}'.format(k)] = row['info'][k]
data.append(row)
headers = set(itertools.chain(*[row.keys() for row in data]))
writer.writerow([h for h in headers])
for row in data:
writer.writerow([row.get(h, '') for h in headers])
fn = "filename={0}".format(secure_name)
resp = make_response(si.getvalue())
resp.headers["Content-Disposition"] = "attachment; {0}".format(fn)
resp.headers["Content-type"] = "text/csv"
resp.headers['Cache-Control'] = "no-store, no-cache, must-revalidate, \
post-check=0, pre-check=0, max-age=0"
return resp
def respond_csv(ty):
# Export Task(/Runs) to CSV
types = {
"task":
(model.Task, handle_task, (lambda x: True),
lazy_gettext("Oops, the application does not have tasks to \
export, if you are the owner add some tasks")),
"task_run":
(model.TaskRun, handle_task_run, (lambda x: type(x.info) == dict),
lazy_gettext("Oops, there are no Task Runs yet to export, invite \
some users to participate"))
}
try:
table, handle_row, test, msg = types[ty]
except KeyError:
return abort(404)
out = StringIO()
writer = UnicodeWriter(out)
t = db.session.query(table)\
.filter_by(app_id=app.id)\
.first()
if t is not None:
if test(t):
writer.writerow(t.info.keys())
return Response(get_csv(out, writer, table, handle_row),
mimetype='text/csv')
else:
flash(msg, 'info')
return render_template('/applications/export.html',
title=title,
app=app)
def _make_zip(self, project, ty):
print '****** entered _make_zip *******'
# json_buffer = self._respond_cust_exp_json(project.id)
# if json_buffer is None:
# logging.debug('buffer is null')
# logging.debug('****** exit create_zip *******')
# return
# detailed csv
file_full = tempfile.NamedTemporaryFile()
wfull = UnicodeWriter(file_full)
self._write_csv(wfull)
file_full.flush()
#summarized csv
file_summary = tempfile.NamedTemporaryFile()
wfull = UnicodeWriter(file_summary)
self._write_csv_summary(wfull)
file_summary.flush()
print '********** file names, file_full: %s, file_summary: %s' % (file_full, file_summary)
name = self._project_name_latin_encoded(project)
#datafile = tempfile.NamedTemporaryFile()
try:
# for line in json_buffer:
# datafile.write(str(line))
# datafile.flush()
zipped_datafile = tempfile.NamedTemporaryFile()
try:
zip = self._zip_factory(zipped_datafile.name)
print '******** zip filename: %s' % zip
#zip.write(datafile.name, secure_filename('%s_%s.json' % (name, ty)))
zip.write(file_full.name, secure_filename('%s_%s_full.csv' % (name, ty)))
zip.write(file_summary.name, secure_filename('%s_%s_summary.csv' % (name, ty)))
zip.close()
container = "user_%d" % project.owner_id
file = FileStorage(filename=self.download_name(project, ty), stream=zipped_datafile)
uploader.upload_file(file, container=container)
print '****** zip file %s upload complete *******' % file
finally:
zipped_datafile.close()
finally:
#datafile.close()
file_full.close()
file_summary.close()
# create response; copied from Exporter:get_zip()
filename = self.download_name(project, ty)
if isinstance(uploader, local.LocalUploader):
filepath = self._download_path(project)
res = send_file(filename_or_fp=safe_join(filepath, filename),
mimetype='application/octet-stream',
as_attachment=True,
attachment_filename=filename)
return res
else:
return redirect(url_for('rackspace', filename=filename,
container=self._container(project),
_external=True))
def respond_csv(ty):
# Export Task(/Runs) to CSV
types = {
"task": (
model.Task, handle_task,
(lambda x: True),
gettext(
"Oops, the application does not have tasks to \
export, if you are the owner add some tasks")),
"task_run": (
model.TaskRun, handle_task_run,
(lambda x: type(x.info) == dict),
gettext(
"Oops, there are no Task Runs yet to export, invite \
some users to participate"))}
try:
table, handle_row, test, msg = types[ty]
except KeyError:
return abort(404)
out = StringIO()
writer = UnicodeWriter(out)
t = db.session.query(table)\
.filter_by(app_id=app.id)\
.first()
if t is not None:
if test(t):
writer.writerow(sorted(t.info.keys()))
return Response(get_csv(out, writer, table, handle_row),
mimetype='text/csv')
else:
flash(msg, 'info')
return respond()
def _respond_csv(self, ty, id):
out = tempfile.TemporaryFile()
writer = UnicodeWriter(out)
t = getattr(task_repo, 'get_%s_by' % ty)(project_id=id)
if t is not None:
headers = self._format_headers(t, ty)
writer.writerow(headers)
return self._get_csv(out, writer, ty, id)
else:
def empty_csv(out):
yield out.read()
return empty_csv(out)
def _respond_csv(self, ty, id):
out = tempfile.TemporaryFile()
writer = UnicodeWriter(out)
t = getattr(task_repo, 'get_%s_by' % ty)(project_id=id)
if t is not None:
headers = self._format_headers(t, ty)
writer.writerow(headers)
return self._get_csv(out, writer, ty, id)
else:
def empty_csv(out):
yield out.read()
return empty_csv(out)
def respond_csv():
out = StringIO()
writer = UnicodeWriter(out)
tmp = 'attachment; filename=all_users.csv'
res = Response(gen_csv(out, writer, write_user), mimetype='text/csv')
res.headers['Content-Disposition'] = tmp
return res
def _respond_csv(self, ty, base_url, info_only=True):
out = tempfile.TemporaryFile()
writer = UnicodeWriter(out)
t = project_repo.get_projects_report(base_url)
if t is not None:
headers = self._format_headers(next(iter(t), None), ty)
writer.writerow(headers)
return self._get_csv(out, writer, ty, base_url, t)
else:
def empty_csv(out):
yield out.read()
return empty_csv(out)
def _respond_csv(self, ty, id):
try:
# Export Task(/Runs) to CSV
types = {
"task": (
model.task.Task, self._handle_task,
(lambda x: True),
gettext(
"Oops, the project does not have tasks to \
export, if you are the owner add some tasks")),
"task_run": (
model.task_run.TaskRun, self._handle_task_run,
(lambda x: True),
gettext(
"Oops, there are no Task Runs yet to export, invite \
some users to participate"))}
try:
table, handle_row, test, msg = types[ty]
except KeyError:
return abort(404) # TODO!
out = tempfile.TemporaryFile()
writer = UnicodeWriter(out)
t = getattr(task_repo, 'get_%s_by' % ty)(project_id=id)
if t is not None:
if test(t):
tmp = t.dictize().keys()
task_keys = []
for k in tmp:
k = "%s__%s" % (ty, k)
task_keys.append(k)
if (type(t.info) == dict):
task_info_keys = []
tmp = t.info.keys()
for k in tmp:
k = "%sinfo__%s" % (ty, k)
task_info_keys.append(k)
else:
task_info_keys = []
keys = task_keys + task_info_keys
writer.writerow(sorted(keys))
return self._get_csv(out, writer, ty, handle_row, id)
else:
pass # TODO
except: # pragma: no cover
raise
def _respond_csv(self, ty, project_id, expanded=False, filters=None):
out = tempfile.TemporaryFile()
writer = UnicodeWriter(out)
if filters:
return self._get_csv_with_filters(out, writer, ty, project_id,
expanded, filters)
else:
return self._get_csv(out, writer, ty, project_id, expanded)
def _respond_csv(self, ty, id):
try:
# Export Task(/Runs) to CSV
types = {
"task": (model.task.Task, self._handle_task, (lambda x: True),
gettext("Oops, the project does not have tasks to \
export, if you are the owner add some tasks")),
"task_run":
(model.task_run.TaskRun, self._handle_task_run,
(lambda x: True),
gettext("Oops, there are no Task Runs yet to export, invite \
some users to participate"))
}
try:
table, handle_row, test, msg = types[ty]
except KeyError:
return abort(404) # TODO!
out = tempfile.TemporaryFile()
writer = UnicodeWriter(out)
t = getattr(task_repo, 'get_%s_by' % ty)(project_id=id)
if t is not None:
if test(t):
tmp = t.dictize().keys()
task_keys = []
for k in tmp:
k = "%s__%s" % (ty, k)
task_keys.append(k)
if (type(t.info) == dict):
task_info_keys = []
tmp = t.info.keys()
for k in tmp:
k = "%sinfo__%s" % (ty, k)
task_info_keys.append(k)
else:
task_info_keys = []
keys = task_keys + task_info_keys
writer.writerow(sorted(keys))
return self._get_csv(out, writer, ty, handle_row, id)
else:
pass # TODO
except: # pragma: no cover
raise
def _respond_csv(self,
ty,
project_id,
expanded=False,
filters=None,
disclose_gold=False):
out = tempfile.TemporaryFile()
writer = UnicodeWriter(out)
return self._get_csv_with_filters(out, writer, ty, project_id,
expanded, filters, disclose_gold)
def _respond_csv(self, ty, project_id, expanded=False, **filters):
out = tempfile.TemporaryFile()
writer = UnicodeWriter(out)
try:
if filters:
return self._get_csv_with_filters(out, writer, ty, project_id,
expanded, **filters)
else:
return self._get_csv(out, writer, ty, project_id, expanded)
except:
def empty_csv(out):
yield out.read()
return empty_csv(out)
def get_csv_response(results):
"""Return a CSV response containing result data."""
si = StringIO.StringIO()
writer = UnicodeWriter(si)
data = []
for r in results:
row = {k: v for k, v in r.dictize().items()}
if isinstance(row['info'], dict): # Explode info
keys = row['info'].keys()
for k in keys:
row['info_{0}'.format(k)] = row['info'][k]
data.append(row)
headers = set(itertools.chain(*[row.keys() for row in data]))
writer.writerow([h for h in headers])
for row in data:
writer.writerow([row.get(h, '') for h in headers])
return make_response(si.getvalue())
def _respond_csv(self, ty, id, info_only=False):
out = tempfile.TemporaryFile()
writer = UnicodeWriter(out)
empty_row = []
p = project_repo.get(id)
if p is not None:
project_section = ['Project Statistics']
project_header = ['Id', 'Name', 'Short Name', 'Total Tasks', 'First Task Submission',
'Last Task Submission', 'Average Time Spend Per Task', 'Task Redundancy']
writer.writerow(project_section)
writer.writerow(project_header)
project_data = get_project_report_projectdata(id)
writer.writerow(project_data)
writer.writerow(empty_row)
user_section = ['User Statistics']
user_header = ['Id', 'Name', 'Fullname', 'Email', 'Admin', 'Subadmin', 'Enabled', 'Languages',
'Locations', 'Start Time', 'End Time', 'Timezone', 'Type of User',
'Additional Comments', 'Total Tasks Completed', 'Percent Tasks Completed',
'First Task Submission', 'Last Task Submission', 'Average Time Per Task']
writer.writerow(user_section)
try:
users_project_data = get_project_report_userdata(id)
if users_project_data:
writer.writerow(user_header)
else:
writer.writerow(['No user data'])
except Exception:
current_app.logger.exception('Error in get_project_report_userdata. project_id: {}'.format(id))
raise BadRequest("Failed to obtain user Statistics")
for user_data in users_project_data:
writer.writerow(user_data)
return self._get_csv(out, writer)
else:
def empty_csv(out):
yield out.read()
return empty_csv(out)
def respond_csv():
out = StringIO()
writer = UnicodeWriter(out)
return Response(gen_csv(out, writer, write_user), mimetype='text/csv')
def _respond_csv(self, ty, id, info_only=False):
out = tempfile.TemporaryFile()
writer = UnicodeWriter(out)
empty_row = []
p = project_repo.get(id)
if p is not None:
project_section = ['Project Statistics']
project_header = [
'Id', 'Name', 'Short Name', 'Total Tasks',
'First Task Submission', 'Last Task Submission',
'Average Time Spend Per Task', 'Task Redundancy'
]
writer.writerow(project_section)
writer.writerow(project_header)
project_data = get_project_report_projectdata(id)
writer.writerow(project_data)
writer.writerow(empty_row)
user_section = ['User Statistics']
user_header = [
'Id', 'Name', 'Fullname', 'Email', 'Admin', 'Subadmin',
'Enabled', 'Languages', 'Locations', 'Start Time', 'End Time',
'Timezone', 'Type of User', 'Additional Comments',
'Total Tasks Completed', 'Percent Tasks Completed',
'First Task Submission', 'Last Task Submission',
'Average Time Per Task'
]
writer.writerow(user_section)
users_project_data = get_project_report_userdata(id)
if users_project_data:
writer.writerow(user_header)
for user_data in users_project_data:
writer.writerow(user_data)
else:
writer.writerow(['No user data'])
return self._get_csv(out, writer)
def export_to(short_name):
"""Export Tasks and TaskRuns in the given format"""
app = App.query.filter_by(short_name=short_name).first_or_404()
if app:
title = "Application: %s · Export" % app.name
else:
title = "Application not found"
if request.args.get('format') and request.args.get('type'):
if request.args.get('format') == 'json':
if request.args.get('type') == 'task':
def gen_json_tasks():
n = db.session.query(model.Task)\
.filter_by(app_id=app.id).count()
i = 0
yield "["
for t in db.session.query(model.Task)\
.filter_by(app_id=app.id).yield_per(1):
i += 1
if (i != n):
yield json.dumps(t.dictize()) + ", "
else:
yield json.dumps(t.dictize())
yield "]"
return Response(gen_json_tasks(), mimetype='application/json')
elif request.args.get('type') == 'task_run':
def gen_json_task_runs():
n = db.session.query(model.TaskRun)\
.filter_by(app_id=app.id).count()
i = 0
yield "["
for tr in db.session.query(model.TaskRun)\
.filter_by(app_id=app.id).yield_per(1):
i += 1
if (i != n):
yield json.dumps(tr.dictize()) + ", "
else:
yield json.dumps(tr.dictize())
yield "]"
return Response(gen_json_task_runs(),
mimetype='application/json')
else:
return abort(404)
elif request.args.get('format') == 'csv':
# Export Tasks to CSV
if request.args.get('type') == 'task':
out = StringIO()
#writer = csv.writer(out)
writer = UnicodeWriter(out)
t = db.session.query(model.Task)\
.filter_by(app_id=app.id)\
.first()
if t is not None:
writer.writerow(t.info.keys())
def get_csv_task():
for t in db.session.query(model.Task)\
.filter_by(app_id=app.id)\
.yield_per(1):
writer.writerow(t.info.values())
yield out.getvalue()
return Response(get_csv_task(), mimetype='text/csv')
else:
msg = "Oops, the application does not have tasks to \
export, if you are the owner add some tasks"
flash(msg, 'info')
return render_template('/applications/export.html',
title=title,
app=app)
# Export Task Runs to CSV
elif request.args.get('type') == 'task_run':
out = StringIO()
writer = UnicodeWriter(out)
tr = db.session.query(model.TaskRun)\
.filter_by(app_id=app.id)\
.first()
if tr is not None:
if (type(tr.info) == dict):
writer.writerow(tr.info.keys())
def get_csv_task_run():
for tr in db.session.query(model.TaskRun)\
.filter_by(app_id=app.id)\
.yield_per(1):
if (type(tr.info) == dict):
writer.writerow(tr.info.values())
else:
writer.writerow([tr.info])
yield out.getvalue()
return Response(get_csv_task_run(), mimetype='text/csv')
else:
msg = "Oops, there are no Task Runs yet to export, invite \
some users to participate"
flash(msg, 'info')
return render_template('/applications/export.html',
title=title,
app=app)
else:
abort(404)
else:
abort(404)
elif len(request.args) >= 1:
abort(404)
else:
return render_template('/applications/export.html',
title=title,
app=app)
