def upload(project_id):
'''
Uploads files to a normalization project. (NB: cannot upload directly to
a link type project).
Also creates the mini version of the project
GET:
- project_id: ID of the normalization project
POST:
file: (csv file) A csv to upload to the chosen normalization project
NB: the "filename" property will be used to name the file
json:
- module_params:
- make_mini: (default True) Set to False to NOT create a mini version of the file
- sample_size
- randomize
'''
# Load project
proj = ESNormalizer(project_id=project_id)
_, module_params = _parse_request()
if module_params is None:
module_params = {}
make_mini = module_params.get('make_mini', True) # TODO: can remove ?
# Upload data
def custom_stream_factory(total_content_length, filename, content_type, content_length=None):
tmpfile = tempfile.NamedTemporaryFile('wb+', prefix='flaskapp')
app.logger.info("start receiving file ... filename => " + str(tmpfile.name))
return tmpfile
_, _, files = werkzeug.formparser.parse_form_data(flask.request.environ, stream_factory=custom_stream_factory)
# Upload data
file_name = files['file'].filename
stream = files['file'].stream
_, run_info = proj.upload_init_data(stream, file_name)
# Make mini
if make_mini:
proj.load_data('INIT', run_info['file_name'])
proj.make_mini(module_params)
# Write transformations and log # TODO: not clean
if proj.metadata['has_mini']:
proj.write_data()
else:
proj._write_metadata()
return jsonify(run_info=run_info, project_id=proj.project_id)
if __name__ == '__main__':
assert False
source_file_name = 'source.csv'
source_user_given_name = 'my_source.csv'
ref_file_name = 'ref.csv'
# Create source
proj = ESNormalizer(None, create_new=True)
source_proj_id = proj.project_id
# Upload files to normalize
file_path = os.path.join('local_test_data', source_file_name)
with open(file_path, 'rb') as f:
proj.upload_init_data(f, source_file_name, source_user_given_name)
# Create ref
proj = ESNormalizer(None, create_new=True)
ref_proj_id = proj.project_id
# Upload files to normalize
file_path = os.path.join('local_test_data', ref_file_name)
with open(file_path, 'rb') as f:
proj.upload_init_data(f, ref_file_name, ref_file_name)
# Try deduping
proj = ESLinker(create_new=True)
proj.add_selected_project('source', False, source_proj_id)
proj.add_selected_project('ref', False, ref_proj_id)