예제 #1
0
def index():

    doc_url = oauth.doc_url()
    if doc_url is not None:
        return redirect_to_results(process_link(doc_url), 'link')

    forms = OrderedDict()
    forms['sample'] = WTFCSVSample(g.current_lang)
    forms['upload'] = WTFCSVUpload()
    # forms['link'] = WTFCSVLink()

    if request.method == 'POST':
        btn_value = request.form['btn']
        sample_id = ''
        results = None

        if btn_value == 'upload':
            upload_file = forms['upload'].data['upload']
            logger.debug("New from upload: %s", upload_file.filename)
            results = process_upload(upload_file)
        elif btn_value == 'link':
            doc_url = forms['link'].data['link']
            logger.debug("New from link: %s", doc_url)
            doc = oauth.open_doc_from_url(doc_url, request.url)
            if doc['authenticate'] is not None:
                return redirect(doc['authenticate'])
            elif doc['doc'] is not None:
                results = process_link(doc['doc'])
        elif btn_value == 'sample':
            sample_source = forms['sample'].data['sample']
            sample = filehandler.get_sample(sample_source)
            sample_name = sample['title']
            sample_id = sample_source
            existing_doc_id = mongo.results_for_sample('wtfcsv', sample_id)
            if existing_doc_id is not None:
                logger.debug("Existing from sample: %s", sample_source)
                return redirect(request.url + 'results/' + existing_doc_id)
            logger.debug("New from sample: %s", sample_name)
            sample_path = sample['path']
            logger.debug("  loading from %s", sample_path)
            results = [
                wtfcsvstat.get_summary(sample_path, language=g.current_lang)
            ]
            # logger.info(results)
            results[0]['filename'] = sample_name + '.csv'
            results[0]['biography'] = sample['biography']
        else:
            results = None

        if results:
            return redirect_to_results(results, btn_value, sample_id)

    return render_template('wtfcsv.html',
                           forms=list(forms.items()),
                           tool_name='wtfcsv',
                           max_file_size_in_mb=g.max_file_size_mb)
예제 #2
0
def process_link(sheet):
    file_paths = filehandler.open_workbook(sheet)
    results = []
    for f in file_paths:
        summary = wtfcsvstat.get_summary(f)
        if 'bad_formatting' not in summary:
            summary['sheet_name'] = _get_sheet_name(f)
            summary['filename'] = sheet.sheet1.title
        results.append(summary)
    filehandler.delete_files(file_paths)
    return results
예제 #3
0
파일: wtfcsv.py 프로젝트: c4fcm/DataBasic
def process_link(sheet):
    file_paths = filehandler.open_workbook(sheet)
    results = []
    for f in file_paths:
        summary = wtfcsvstat.get_summary(f)
        if 'bad_formatting' not in summary:
            summary['sheet_name'] = _get_sheet_name(f)
            summary['filename'] = sheet.sheet1.title
        results.append (summary)
    filehandler.delete_files(file_paths)
    return results
예제 #4
0
파일: wtfcsv.py 프로젝트: c4fcm/DataBasic
def index():
    
    doc_url = oauth.doc_url()
    if doc_url is not None:
        return redirect_to_results(process_link(doc_url), 'link')

    tab = 'paste' if not 'tab' in request.args else request.args['tab']
    results = None

    forms = OrderedDict()
    forms['sample'] = WTFCSVSample(g.current_lang)
    forms['upload'] = WTFCSVUpload()
    # forms['link'] = WTFCSVLink()

    if request.method == 'POST':

        btn_value = request.form['btn']
        sample_id = ''

        if btn_value == 'upload':
            upload_file = forms['upload'].data['upload']
            logger.debug("New from upload: %s", upload_file.filename)

            results = process_upload(upload_file)
        elif btn_value == 'link':
            doc_url = forms['link'].data['link']
            logger.debug("New from link: %s", doc_url)
            doc = oauth.open_doc_from_url(doc_url, request.url)
            if doc['authenticate'] is not None:
                return redirect(doc['authenticate'])
            elif doc['doc'] is not None:
                results = process_link(doc['doc'])
        elif btn_value == 'sample':
            sample_source = forms['sample'].data['sample']
            samplename = filehandler.get_sample_title(sample_source)
            sample_id = sample_source
            existing_doc_id = mongo.results_for_sample('wtfcsv',sample_id)
            if existing_doc_id is not None:
                logger.debug("Existing from sample: %s", sample_source)
                return redirect(request.url + 'results/' + existing_doc_id)
            logger.debug("New from sample: %s", samplename)
            sample_path = filehandler.get_sample_path(sample_source)
            logger.debug("  loading from %s", sample_path)
            results = []
            results.append(wtfcsvstat.get_summary(sample_path))
            results[0]['filename'] = samplename + '.csv'

        if btn_value is not None and btn_value is not u'':
            return redirect_to_results(results, btn_value, sample_id)

    return render_template('wtfcsv.html', forms=forms.items(), tool_name='wtfcsv', max_file_size_in_mb = g.max_file_size_mb)
예제 #5
0
파일: wtfcsv.py 프로젝트: c4fcm/DataBasic
def process_upload(csv_file):
    file_path = filehandler.open_doc(csv_file)
    file_size = os.stat(file_path).st_size # because browser might not have sent content_length
    logger.debug("Upload: %d bytes", file_size)
    file_paths = filehandler.convert_to_csv(file_path)
    results = []
    for f in file_paths:
        summary = wtfcsvstat.get_summary(f)
        if 'bad_formatting' not in summary:
            summary['sheet_name'] = _get_sheet_name(f)
            summary['filename'] = csv_file.filename
        results.append(summary)
    filehandler.delete_files(file_paths)
    return results
예제 #6
0
def process_upload(csv_file):
    file_path = filehandler.open_doc(csv_file)
    file_size = os.stat(
        file_path
    ).st_size  # because browser might not have sent content_length
    logger.debug("Upload: %d bytes", file_size)
    file_paths = filehandler.convert_to_csv(file_path)
    results = []
    for f in file_paths:
        summary = wtfcsvstat.get_summary(f)
        if 'bad_formatting' not in summary:
            summary['sheet_name'] = _get_sheet_name(f)
            summary['filename'] = csv_file.filename
        results.append(summary)
    filehandler.delete_files(file_paths)
    return results
예제 #7
0
 def test_get_summary_from_csv(self):
     test_data_path = os.path.join(self._fixtures_dir,'somerville-tree-details.csv')
     results = wtfcsvstat.get_summary(test_data_path)
     self.assertEqual(len(results['columns']), 43)
     self.assertEqual(results['row_count'], 13882)
예제 #8
0
 def test_trailing_comma(self):
     test_data_path = os.path.join(self._fixtures_dir,'trailing-comma.csv')
     results = wtfcsvstat.get_summary(test_data_path)
     self.assertEqual(len(results['columns']), 2)
     self.assertEqual(results['row_count'], 3)
예제 #9
0
 def test_get_summary_from_xls(self):
 	fixture_path = os.path.join(self._fixtures_dir,'HowAmericaInjuresItself_FromNEISS.xlsx')
     csv_file = filehandler.convert_to_csv(fixture_path)[0]
     results = wtfcsvstat.get_summary(csv_file)
     self.assertEqual(len(results['columns']), 19)
     self.assertEqual(results['row_count'], 26303)