def approve_dataset(source_url_hash): # get the MetaTable row and change the approved_status and bounce back to view-datasets. meta = session.query(MetaTable).get(source_url_hash) json_data_types = None if ((not meta.is_socrata_source) and meta.contributed_data_types): json_data_types = json.loads(meta.contributed_data_types) add_dataset_task.delay(source_url_hash, data_types=json_data_types) upd = {'approved_status': 'true'} meta.approved_status = 'true' session.commit() # Email the user who submitted that their dataset has been approved. # email the response to somebody msg_body = """Hello %s,\r\n \r\n Your dataset has been approved and added to Plenar.io:\r\n \r\n %s\r\n \r\n It should appear on http://plenar.io within 24 hours.\r\n \r\n Thank you!\r\n The Plenario Team\r\n http://plenar.io""" % (meta.contributor_name, meta.human_name) send_mail(subject="Your dataset has been added to Plenar.io", recipient=meta.contributor_email, body=msg_body)
def submit(context): form = request.form is_shapefile = context['is_shapefile'] is_admin = context['is_admin'] try: # Store the metadata if is_shapefile: if shape_already_submitted(form['dataset_name']): msg = 'A Shapefile with this name has already been submitted' raise RuntimeError(msg) else: meta = shape_meta_from_submit_form(form, is_approved=is_admin) else: meta = point_meta_from_submit_form(form, is_approved=is_admin) except RuntimeError as e: context['error_msg'] = e.message return render_with_context(context) else: # Successfully stored the metadata # Now fire ingestion task... if is_admin: if is_shapefile: add_shape_task.delay(meta.dataset_name) else: add_dataset_task.delay(meta.source_url_hash) # or send thankyou email else: return send_submission_email(meta.human_name, meta.contributor_name, meta.contributor_email) return view_datasets()
def approve_dataset(source_url_hash): # get the MetaTable row and change the approved_status and bounce back to view-datasets. meta = session.query(MetaTable).get(source_url_hash) json_data_types = None if ((not meta.is_socrata_source) and meta.contributed_data_types): json_data_types = json.loads(meta.contributed_data_types) add_dataset_task.delay(source_url_hash, data_types=json_data_types) upd = { 'approved_status': 'true' } meta.approved_status = 'true' session.commit() # Email the user who submitted that their dataset has been approved. # email the response to somebody msg_body = """Hello %s,\r\n \r\n Your dataset has been approved and added to Plenar.io:\r\n \r\n %s\r\n \r\n It should appear on http://plenar.io within 24 hours.\r\n \r\n Thank you!\r\n The Plenario Team\r\n http://plenar.io""" % (meta.contributor_name, meta.human_name) send_mail(subject="Your dataset has been added to Plenar.io", recipient=meta.contributor_email, body=msg_body)
def approve_dataset(source_url_hash): # Approve it meta = session.query(MetaTable).get(source_url_hash) meta.approved_status = True session.commit() # Ingest it add_dataset_task.delay(source_url_hash) send_approval_email(meta.human_name, meta.contributor_name, meta.contributor_email)
def add_dataset(): dataset_info = {} errors = [] socrata_source = False url = "" dataset_id = None md = None if request.args.get('dataset_url'): url = request.args.get('dataset_url') (dataset_info, errors, socrata_source) = get_context_for_new_dataset(url) # populate contributor info from session user = session.query(User).get(flask_session['user_id']) dataset_info['contributor_name'] = user.name dataset_info['contributor_organization'] = 'Plenario Admin' dataset_info['contributor_email'] = user.email # check if dataset with the same URL has already been loaded dataset_id = md5(url).hexdigest() md = session.query(MetaTable).get(dataset_id) if md: errors.append( "A dataset with that URL has already been loaded: '%s'" % md.human_name) if request.method == 'POST' and not md: md = add_dataset_to_metatable(request, url, dataset_id, dataset_info, socrata_source, approved_status=True) json_data_types = None if ((not md.is_socrata_source) and md.contributed_data_types): json_data_types = json.loads(md.contributed_data_types) add_dataset_task.delay(md.source_url_hash, data_types=json_data_types) flash('%s added successfully!' % md.human_name, 'success') return redirect(url_for('views.view_datasets')) context = { 'dataset_info': dataset_info, 'errors': errors, 'socrata_source': socrata_source } return render_template('admin/add-dataset.html', **context)
def add_dataset(): dataset_info = {} errors = [] socrata_source = False url = "" dataset_id = None md = None if request.args.get('dataset_url'): url = request.args.get('dataset_url') (dataset_info, errors, socrata_source) = get_context_for_new_dataset(url) # populate contributor info from session user = session.query(User).get(flask_session['user_id']) dataset_info['contributor_name'] = user.name dataset_info['contributor_organization'] = 'Plenario Admin' dataset_info['contributor_email'] = user.email # check if dataset with the same URL has already been loaded dataset_id = md5(url).hexdigest() md = session.query(MetaTable).get(dataset_id) if md: errors.append("A dataset with that URL has already been loaded: '%s'" % md.human_name) if request.method == 'POST' and not md: md = add_dataset_to_metatable(request, url, dataset_id, dataset_info, socrata_source, approved_status=True) json_data_types = None if ((not md.is_socrata_source) and md.contributed_data_types): json_data_types = json.loads(md.contributed_data_types) add_dataset_task.delay(md.source_url_hash, data_types=json_data_types) flash('%s added successfully!' % md.human_name, 'success') return redirect(url_for('views.view_datasets')) context = {'dataset_info': dataset_info, 'errors': errors, 'socrata_source': socrata_source} return render_template('admin/add-dataset.html', **context)
def submit_dataset(): # Slightly dumb way to make sure that POSTs are only coming from # originating domain for the time being referer = request.headers.get('Referer') if referer: referer = urlparse(referer).netloc req_url = urlparse(request.url).netloc if referer != req_url: abort(401) else: abort(401) resp = {'status': 'ok', 'message': ''} status_code = 200 errors = [] post = request.form.get('data') if not post: try: post = request.form.keys()[0] except IndexError: resp['status'] = 'error' resp['message'] = 'Unable to decode POST data' status_code = 400 if status_code == 200: post = json.loads(post) if post.get('view_url'): if post.get('socrata'): source_domain = urlparse(post['view_url']).netloc four_by_four = re.findall(r'/([a-z0-9]{4}-[a-z0-9]{4})', post['view_url'])[-1] view_url = 'http://%s/api/views/%s' % (source_domain, four_by_four) dataset_info, errors, status_code = get_socrata_data_info(view_url) source_url = '%s/rows.csv?accessType=DOWNLOAD' % view_url else: dataset_info = { 'attribution': '', 'description': '', } source_url = post['view_url'] dataset_info['name'] = urlparse(source_url).path.split('/')[-1] if errors: resp['message'] = ', '.join([e for e in errors]) resp['status'] = 'error' status_code = 400 else: dataset_id = md5(source_url).hexdigest() md = session.query(MetaTable).get(dataset_id) if not md: d = { 'dataset_name': slugify(dataset_info['name'], delim=u'_'), 'human_name': dataset_info['name'], 'attribution': dataset_info['attribution'], 'description': dataset_info['description'], 'source_url': source_url, 'source_url_hash': dataset_id, 'update_freq': post['update_frequency'], 'business_key': post['field_definitions']['id_field'], 'observed_date': post['field_definitions']['date_field'], 'latitude': post['field_definitions'].get('latitude'), 'longitude': post['field_definitions'].get('longitude'), 'location': post['field_definitions'].get('location') } if len(d['dataset_name']) > 49: d['dataset_name'] = d['dataset_name'][:50] md = MetaTable(**d) session.add(md) session.commit() add_dataset.delay(md.source_url_hash, data_types=post.get('data_types')) resp['message'] = 'Dataset %s submitted successfully' % dataset_info['name'] else: resp['status'] = 'error' resp['message'] = 'Must provide a url where data can be downloaded' status_code = 400 resp = make_response(json.dumps(resp, default=dthandler), status_code) resp.headers['Content-Type'] = 'application/json' return resp
def submit_dataset(): # Slightly dumb way to make sure that POSTs are only coming from # originating domain for the time being referer = request.headers.get('Referer') if referer: referer = urlparse(referer).netloc req_url = urlparse(request.url).netloc if referer != req_url: abort(401) else: abort(401) resp = {'status': 'ok', 'message': ''} status_code = 200 errors = [] post = request.form.get('data') if not post: try: post = request.form.keys()[0] except IndexError: resp['status'] = 'error' resp['message'] = 'Unable to decode POST data' status_code = 400 if status_code == 200: post = json.loads(post) if post.get('view_url'): if post.get('socrata'): source_domain = urlparse(post['view_url']).netloc four_by_four = re.findall(r'/([a-z0-9]{4}-[a-z0-9]{4})', post['view_url'])[-1] view_url = 'http://%s/api/views/%s' % (source_domain, four_by_four) dataset_info, errors, status_code = get_socrata_data_info( view_url) source_url = '%s/rows.csv?accessType=DOWNLOAD' % view_url else: dataset_info = { 'attribution': '', 'description': '', } source_url = post['view_url'] dataset_info['name'] = urlparse(source_url).path.split('/')[-1] if errors: resp['message'] = ', '.join([e for e in errors]) resp['status'] = 'error' status_code = 400 else: dataset_id = md5(source_url).hexdigest() md = session.query(MetaTable).get(dataset_id) if not md: d = { 'dataset_name': slugify(dataset_info['name'], delim=u'_'), 'human_name': dataset_info['name'], 'attribution': dataset_info['attribution'], 'description': dataset_info['description'], 'source_url': source_url, 'source_url_hash': dataset_id, 'update_freq': post['update_frequency'], 'business_key': post['field_definitions']['id_field'], 'observed_date': post['field_definitions']['date_field'], 'latitude': post['field_definitions'].get('latitude'), 'longitude': post['field_definitions'].get('longitude'), 'location': post['field_definitions'].get('location') } if len(d['dataset_name']) > 49: d['dataset_name'] = d['dataset_name'][:50] md = MetaTable(**d) session.add(md) session.commit() add_dataset.delay(md.source_url_hash, data_types=post.get('data_types')) resp[ 'message'] = 'Dataset %s submitted successfully' % dataset_info[ 'name'] else: resp['status'] = 'error' resp['message'] = 'Must provide a url where data can be downloaded' status_code = 400 resp = make_response(json.dumps(resp, default=dthandler), status_code) resp.headers['Content-Type'] = 'application/json' return resp