def update_tree(): from warehouse import get_warehouse wh = get_warehouse() parcels = [p for p in wh.get_all_parcels() if not p.uploading] parcels.sort(key=lambda p: p.metadata['upload_time']) for p in parcels: print p.name, (p.link_in_tree() or '[already linked]')
def delete_file(name, filename): wh = get_warehouse() parcel = get_or_404(wh.get_parcel, name, _exc=KeyError) if not authorize_for_parcel(parcel): flask.abort(403) if not parcel.uploading: flask.abort(403) if flask.request.method == 'POST': filename = secure_filename(filename) file_path = flask.safe_join(parcel.get_path(), filename) wh.logger.info("Delete file %r for parcel %r (user %s)", filename, parcel.name, _current_user()) try: os.unlink(file_path) parcel_file_deleted.send(parcel) flask.flash("File %s was deleted." % name, 'system') except OSError: flask.flash("File %s was not deleted." % name, 'system') return flask.redirect(flask.url_for('parcel.view', name=name)) else: return flask.render_template('parcel_file_delete.html', parcel=parcel, filename=filename)
def delete(name): wh = get_warehouse() app = flask.current_app parcel = get_or_404(wh.get_parcel, name, _exc=KeyError) if not app.config['ALLOW_PARCEL_DELETION']: flask.abort(403) if not auth.authorize(['ROLE_ADMIN']): return flask.abort(403) if flask.request.method == 'POST': delete_parcel_and_followers(wh, parcel.name) flask.flash("Parcel %s was deleted." % name, 'system') return flask.redirect(flask.url_for('parcel.index')) else: will_remove = list(walk_parcels(wh, name)) will_not_remove = list(walk_parcels(wh, name, 'prev_parcel'))[1:] will_not_remove.reverse() return flask.render_template( 'parcel_delete.html', **{ 'parcel': parcel, 'will_remove': will_remove, 'will_not_remove': will_not_remove, })
def upload(name): wh = get_warehouse() parcel = get_or_404(wh.get_parcel, name, _exc=KeyError) form = flask.request.form.to_dict() if not authorize_for_upload(parcel): flask.abort(403) identifier = form['resumableIdentifier'] chunk_number = int(form['resumableChunkNumber']) parcel_path = parcel.get_path() temp = parcel_path.joinpath(identifier) if not os.path.isdir(temp): os.makedirs(temp) filename = secure_filename(form['resumableFilename']) if parcel_path.joinpath(filename).exists(): return "File already exists", 415 posted_file = flask.request.files['file'] tmp_file = tempfile.NamedTemporaryFile(dir=temp, delete=False) tmp = path(tmp_file.name) tmp_file.close() posted_file.save(tmp) with exclusive_lock(): chunk_path = temp.joinpath('%s_%s' % (chunk_number, identifier)) wh.logger.info("Begin chunked upload file %r for parcel %r (user %s)", filename, parcel.name, _current_user()) tmp.rename(chunk_path) return flask.jsonify({'status': 'success'})
def finalize(name): wh = get_warehouse() parcel = get_or_404(wh.get_parcel, name, _exc=KeyError) stage_def = STAGES[parcel.metadata['stage']] if not authorize_for_parcel(parcel): return flask.abort(403) if stage_def.get('last'): return flask.abort(403) if not parcel.uploading: return flask.abort(403) if stage_def.get('reject'): reject = bool(flask.request.values.get('reject')) else: reject = None if flask.request.values.get('reject'): flask.abort(403) if flask.request.method == "POST": finalize_parcel(wh, parcel, reject) url = flask.url_for('parcel.view', name=parcel.name) return flask.redirect(url) else: return flask.render_template("parcel_confirm_finalize.html", parcel=parcel, reject=reject)
def new(): if flask.request.method == 'POST': if not authorize_for_parcel(None): return flask.abort(403) wh = get_warehouse() form = flask.request.form.to_dict() if form['extent'] == 'full': form['coverage'] = '' metadata = {k: form.get(k, '') for k in EDITABLE_METADATA} metadata['stage'] = INITIAL_STAGE parcel = wh.new_parcel() if not validate_metadata(metadata): flask.abort(400) parcel.save_metadata(metadata) parcel.add_history_item("New upload", datetime.utcnow(), flask.g.username, "") parcel_created.send(parcel) url = flask.url_for('parcel.view', name=parcel.name) return flask.redirect(url) else: return flask.render_template('parcel_new.html')
def search(): wh = get_warehouse() parcels = list(filter_parcels(chain_tails(wh), **get_filter_arguments())) parcels.sort(key=lambda p: p.last_modified, reverse=True) return flask.render_template('search.html', **{ 'parcels': parcels, })
def get_parcels_by_stage(name): wh = get_warehouse() stages_with_parcels = dict([(stage, None) for stage in STAGES]) for parcel in walk_parcels(wh, name, 'prev_parcel'): stage = parcel.metadata['stage'] if not stages_with_parcels[stage]: stages_with_parcels[stage] = parcel return stages_with_parcels
def test_get_parcel_metadata(self): import warehouse name = self.new_parcel() resp = self.get_json('/api/parcel/' + name) with self.app.test_request_context(): wh = warehouse.get_warehouse() parcel = wh.get_parcel(name) self.assertEqual(resp['metadata'], parcel.metadata)
def download_report_file(report_id): wh = get_warehouse() report = get_or_404(wh.get_report, report_id, _exc=KeyError) file_path = safe_join(wh.reports_path, report.filename) if not path(file_path).isfile(): flask.abort(404) return flask.send_file(file_path, as_attachment=True, attachment_filename=report.filename)
def comment(name): wh = get_warehouse() parcel = get_or_404(wh.get_parcel, name, _exc=KeyError) comment = flask.request.form.get("comment", "").strip() if comment: add_history_item_and_notify( parcel, 'comment', "Comment", datetime.utcnow(), flask.g.username, escape(comment)) return flask.redirect(flask.url_for('parcel.view', name=name))
def comment(name): wh = get_warehouse() parcel = get_or_404(wh.get_parcel, name, _exc=KeyError) comment = flask.request.form.get("comment", "").strip() if comment: add_history_item_and_notify(parcel, 'comment', "Comment", datetime.utcnow(), flask.g.username, escape(comment)) return flask.redirect(flask.url_for('parcel.view', name=name))
def download(name, filename): wh = get_warehouse() parcel = get_or_404(wh.get_parcel, name, _exc=KeyError) file_path = safe_join(parcel.get_path(), filename) if not path(file_path).isfile(): flask.abort(404) return flask.send_file(file_path, as_attachment=True, attachment_filename=filename)
def create_parcel(self, stage, finalize_and_link=False): with self.app.test_request_context(): wh = warehouse.get_warehouse() parcel = wh.new_parcel() parcel.save_metadata(self.PARCEL_METADATA) parcel.save_metadata({'stage': stage}) if finalize_and_link: parcel.finalize() parcel.link_in_tree() return parcel.name
def test_repeated_calls_to_link_in_tree_dont_create_more_links(self): stage = 'enh' name = self.create_parcel(stage, True) parent_path = self.symlink_path(self.PARCEL_METADATA, stage) with self.app.test_request_context(): wh = warehouse.get_warehouse() parcel = wh.get_parcel(name) parcel.link_in_tree() self.assertEqual(parent_path.listdir(), [parent_path / '1'])
def download(name, filename): from werkzeug.security import safe_join wh = get_warehouse() parcel = get_or_404(wh.get_parcel, name, _exc=KeyError) file_path = safe_join(parcel.get_path(), filename) if not path(file_path).isfile(): flask.abort(404) return flask.send_file(file_path, as_attachment=True, attachment_filename=filename)
def stream(code): wh = get_warehouse() all_parcels = [p for p in chain_tails(wh) if p.metadata['delivery_type'] == STREAM and p.metadata['lot'] == code] grouped_parcels = group_parcels(all_parcels) return flask.render_template('stream.html', **{ 'code': code, 'grouped_parcels': grouped_parcels, })
def country(code): wh = get_warehouse() all_parcels = [p for p in chain_tails(wh) if p.metadata['country'] == code] grouped_parcels = group_parcels(all_parcels) return flask.render_template( 'country.html', **{ 'code': code, 'grouped_parcels': grouped_parcels, })
def test_symlink_generator_skips_over_broken_symlinks(self): stage = 'enh' name1 = self.create_parcel(stage, True) with self.app.test_request_context(): wh = warehouse.get_warehouse() wh.delete_parcel(name1) name2 = self.create_parcel(stage, True) symlink_path_2 = self.symlink_path(self.PARCEL_METADATA, stage, 2) self.assertTrue(symlink_path_2.islink()) self.assertEqual(symlink_path_2.readlink(), self.wh_path / 'parcels' / name2)
def migrate_prev_parcel_to_list(): from warehouse import get_warehouse wh = get_warehouse() parcels = wh.get_all_parcels() for parcel in parcels: prev_parcel = parcel.metadata.get('prev_parcel') if prev_parcel: parcel.save_metadata({'prev_parcel_list': [prev_parcel]}) del parcel.metadata['prev_parcel']
def country(code): wh = get_warehouse() all_parcels = [p for p in chain_tails(wh) if p.metadata['delivery_type'] == COUNTRY and p.metadata['country'] == code] grouped_parcels = group_parcels(all_parcels) return flask.render_template('country.html', **{ 'code': code, 'grouped_parcels': grouped_parcels, })
def test_rollback_on_error(self): @self.app.route('/change_something', methods=['POST']) def change_something(): wh = warehouse.get_warehouse() wh.test_value = 'asdf' raise ValueError self.assertRaises(ValueError, self.client.post, '/change_something') with self.app.test_request_context(): wh = warehouse.get_warehouse() self.assertFalse(hasattr(wh, 'test_value'))
def test_commit_on_success(self): @self.app.route('/change_something', methods=['POST']) def change_something(): wh = warehouse.get_warehouse() wh.test_value = 'asdf' return 'ok' self.client.post('/change_something') with self.app.test_request_context(): wh = warehouse.get_warehouse() self.assertEqual(wh.test_value, 'asdf')
def fsck(): from warehouse import get_warehouse, checksum wh = get_warehouse() parcels = wh.get_all_parcels() for parcel in parcels: folder_path = parcel.get_path() files_checksum = checksum(folder_path) if not files_checksum == getattr(parcel, 'checksum', []): print "Checksum for parcel %r is wrong" % parcel.name print "Finished checking for parcel checksums"
def save(self): data = dict(self.data) data['stage'] = INITIAL_STAGE[self.DELIVERY_TYPE] data['delivery_type'] = self.DELIVERY_TYPE wh = get_warehouse() parcel = wh.new_parcel() parcel.save_metadata(data) parcel.add_history_item('New upload', datetime.utcnow(), g.username, '') return parcel
def files(name): wh = get_warehouse() parcel = get_or_404(wh.get_parcel, name, _exc=KeyError) app = flask.current_app if parcel.uploading and authorize_for_parcel(parcel): file_authorize = True else: file_authorize = False template = app.jinja_env.get_template('bits.html') return template.module.files_table(parcel, delete_buttons=file_authorize)
def files(name): wh = get_warehouse() parcel = get_or_404(wh.get_parcel, name, _exc=KeyError) app = flask.current_app if (parcel.uploading and parcel.file_uploading and authorize_for_parcel(parcel)): file_authorize = True else: file_authorize = False template = app.jinja_env.get_template('bits.html') return template.module.files_table(parcel, delete_buttons=file_authorize)
def create_parcel(self, stage=None): with patch('auth.authorize'): post_resp = self.client.post('/parcel/new', data=self.PARCEL_METADATA) self.assertEqual(post_resp.status_code, 302) parcel_name = post_resp.location.rsplit('/', 1)[-1] if stage is not None: with self.app.test_request_context(): wh = warehouse.get_warehouse() wh.get_parcel(parcel_name).metadata['stage'] = stage return parcel_name
def lot(code): wh = get_warehouse() all_parcels = [p for p in chain_tails(wh) if p.metadata['delivery_type'] == LOT and p.metadata['lot'] == code] all_reports = [r for r in wh.get_all_reports() if r.lot == code] grouped_parcels = group_parcels(all_parcels) return flask.render_template('lot.html', **{ 'code': code, 'grouped_parcels': grouped_parcels, 'all_reports': all_reports, })
def get_parcels_by_stage(name): wh = get_warehouse() parcel = get_or_404(wh.get_parcel, name, _exc=KeyError) DELIVERY_STAGES, _ = _get_stages_for_parcel(parcel) stages_with_parcels = {stage: None for stage in DELIVERY_STAGES} for parcel in walk_parcels(wh, name, forward=False): stage = parcel.metadata['stage'] if not stages_with_parcels[stage]: stages_with_parcels[stage] = parcel prev_parcel_list = parcel.metadata.get('prev_parcel_list', []) if len(prev_parcel_list) > 1: prev_parcel = wh.get_parcel(prev_parcel_list[0]) stages_with_parcels[prev_parcel.metadata['stage']] = \ 'Merged with %s other parcels.' % len(prev_parcel_list) return stages_with_parcels
def chain(name): wh = get_warehouse() first_parcel = get_or_404(wh.get_parcel, name, _exc=KeyError) previous_parcels = list(walk_parcels(wh, name, 'prev_parcel')) if len(previous_parcels) > 1: first_parcel = previous_parcels[-1] url = flask.url_for('parcel.chain', name=first_parcel.name) return flask.redirect(url) workflow_parcels = list(walk_parcels(wh, name)) return flask.render_template( 'parcel_chain.html', **{ 'first_parcel': first_parcel, 'workflow_parcels': workflow_parcels, })
def delete_report(report_id): if not auth.authorize(['ROLE_ADMIN']): return flask.abort(403) wh = get_warehouse() report = get_or_404(wh.get_report, report_id, _exc=KeyError) lot_code = report.lot if flask.request.method == 'POST': file_path = safe_join(wh.reports_path, report.filename) if file_path.exists(): file_path.unlink() wh.delete_report(report_id) flask.flash('Report was deleted.', 'system') url = flask.url_for('parcel.lot', code=lot_code) return flask.redirect(url) return flask.render_template('report_delete.html', report=report)
def search(delivery_type): wh = get_warehouse() filter_arguments = get_filter_arguments() all_reports = [] # right now the reports are showed only when a lot is selected if 'lot' in filter_arguments and delivery_type == LOT: all_reports = [r for r in wh.get_all_reports() if r.lot == filter_arguments['lot']] parcels = list(filter_parcels(chain_tails(wh), **filter_arguments)) parcels = [p for p in parcels if p.metadata.get('delivery_type', COUNTRY) == delivery_type] parcels.sort(key=lambda p: p.last_modified, reverse=True) return flask.render_template('search.html', **{ 'parcels': parcels, 'all_reports': all_reports, 'delivery_type': delivery_type, })
def create_parcel(self, stage=None, delivery_type=LOT): with patch("auth.authorize"): if delivery_type == COUNTRY: post_resp = self.client.post("parcel/new/country", data=self.COUNTRY_METADATA) elif delivery_type == STREAM: post_resp = self.client.post("parcel/new/stream", data=self.STREAM_METADATA) else: post_resp = self.client.post("/parcel/new/lot", data=self.LOT_METADATA) self.assertEqual(post_resp.status_code, 302) parcel_name = post_resp.location.rsplit("/", 1)[-1] if stage is not None: with self.app.test_request_context(): wh = warehouse.get_warehouse() wh.get_parcel(parcel_name).metadata["stage"] = stage return parcel_name
def finalize_upload(name): wh = get_warehouse() parcel = get_or_404(wh.get_parcel, name, _exc=KeyError) response = {'status': 'success'} form = flask.request.form.to_dict() filename = secure_filename(form['resumableFilename']) identifier = form['resumableIdentifier'] total_size = int(form['resumableTotalSize']) temp = parcel.get_path().joinpath(identifier) if all_chunks_uploaded(temp, total_size): create_file_from_chunks(parcel, temp, filename) wh.logger.info("Finished chunked upload %r for parcel %r (user %s)", filename, parcel.name, _current_user()) else: response['status'] = 'error' response['message'] = "Upload didn't finalize. an error occurred" return flask.jsonify(response)
def shell(warehouse=False): def run(): code.interact('', local=context) app = flask._request_ctx_stack.top.app context = {'app': app} if warehouse: import warehouse import transaction context['wh'] = warehouse.get_warehouse() context['transaction'] = transaction try: run() finally: if warehouse: transaction.abort()
def check_chunk(name): wh = get_warehouse() parcel = get_or_404(wh.get_parcel, name, _exc=KeyError) if not authorize_for_upload(parcel): flask.abort(403) form = flask.request.args.to_dict() chunk_number = form['resumableChunkNumber'] chunk_size = int(form['resumableChunkSize']) identifier = form['resumableIdentifier'] temp = parcel.get_path().joinpath(identifier) chunk_path = temp.joinpath('%s_%s' % (chunk_number, identifier)) if not chunk_path.exists(): flask.abort(404) if not chunk_path.stat().st_size >= chunk_size: flask.abort(404) return flask.Response()
def dispatch_request(self, name, *args, **kwargs): self.wh = get_warehouse() self.parcel = get_or_404(self.wh.get_parcel, name, _exc=KeyError) DELIVERY_STAGES, _ = _get_stages_for_parcel(self.parcel) if DELIVERY_STAGES: stage_def = DELIVERY_STAGES[self.parcel.metadata['stage']] else: flask.abort(400) if (not authorize_for_parcel(self.parcel) or stage_def.get('last') or not self.parcel.uploading): flask.abort(403) if stage_def.get('reject'): self.reject = bool(flask.request.values.get('reject')) else: self.reject = None if flask.request.values.get('reject'): flask.abort(403) return super(Finalize, self).dispatch_request(name, *args, **kwargs)
def chain(name): wh = get_warehouse() first_parcel = get_or_404(wh.get_parcel, name, _exc=KeyError) previous_parcels = list(walk_parcels(wh, name, forward=False)) if len(previous_parcels) > 1: first_parcel = previous_parcels[-1] url = flask.url_for('parcel.chain', name=first_parcel.name) return flask.redirect(url) workflow_parcels = list(walk_parcels(wh, name)) prev_parcels = [] if workflow_parcels: prev_parcel_list = workflow_parcels[0].metadata.get( 'prev_parcel_list', []) if len(prev_parcel_list) > 1: prev_parcels = [wh.get_parcel(p) for p in prev_parcel_list] return flask.render_template('parcel_chain.html', **{ 'first_parcel': first_parcel, 'workflow_parcels': workflow_parcels, 'prev_parcels': prev_parcels, })
def upload_single_file(name): wh = get_warehouse() parcel = get_or_404(wh.get_parcel, name, _exc=KeyError) if not authorize_for_upload(parcel): flask.abort(403) posted_file = flask.request.files['file'] filename = secure_filename(posted_file.filename) file_path = parcel.get_path().joinpath(filename) if file_path.exists(): flask.flash("File %s already exists." % filename, 'system') else: if posted_file: posted_file.save(file_path) file_uploaded.send(parcel, filename=filename) wh.logger.info("Finished upload %r for parcel %r (user %s)", filename, parcel.name, _current_user()) else: flask.flash("Please upload a valid file", 'system') return flask.redirect(flask.url_for('parcel.view', name=name))
def new_report(): if not authorize_for_cdr(): return flask.abort(403) if flask.request.method == 'POST': wh = get_warehouse() form = flask.request.form.to_dict() metadata = {k: form.get(k, '') for k in REPORT_METADATA} data_map = zip(REPORT_METADATA, [LOTS, COUNTRY_LOT_PRODUCTS.get(metadata['lot'], ())]) if not validate_metadata(metadata, data_map): flask.abort(400) posted_file = flask.request.files.get('file') if posted_file and extension(posted_file.filename) in DOCUMENTS: report = wh.new_report(**metadata) save_report_file(reports_path=wh.reports_path, posted_file=posted_file, report=report) report_created.send(report) url = flask.url_for('parcel.lot', code=report.lot) return flask.redirect(url) else: flask.flash("File field is missing or it's not a document.", 'system') return flask.render_template('report_new.html')
def change_something(): wh = warehouse.get_warehouse() wh.test_value = 'asdf' raise ValueError
def ping(): warehouse.get_warehouse() return 'gioland is ok'
def parcel_metadata(name): wh = get_warehouse() parcel = get_or_404(wh.get_parcel, name, _exc=KeyError) return flask.jsonify({'metadata': dict(parcel.metadata)})
def api_find_parcels(): wh = get_warehouse() parcels = filter_parcels(wh.get_all_parcels(), **get_filter_arguments()) return flask.jsonify({ 'parcels': [p.name for p in parcels], })
def view(name): wh = get_warehouse() parcel = get_or_404(wh.get_parcel, name, _exc=KeyError) return flask.render_template('parcel.html', parcel=parcel)