示例#1
0
 def test_check_successful_run(self):
     redis.flushdb()
     self.assertEqual(0, redis.dbsize())
     import_data()
     self.assertEqual(2, redis.dbsize())
     self.assertEqual(10, len(redis.lrange('warehouse_whcorp', 0, -1)))
     self.assertEqual(
         10, len(redis.lrange('warehouse_american_storage', 0, -1)))
示例#2
0
def import_shapefile(request):
	if request.method == "GET":
		categories = Category.objects.all();
		return render(request, "import_shapefile.html",{'categories':categories});
	elif request.method == "POST":
		result = {}
		try:
			# se ejecuta la tarea de Celery
			result = importer.import_data(request)
			return HttpResponse(json.dumps(result),content_type='application/json')
		except Exception as e:
			result["error"]=str(e);
			return HttpResponse(json.dumps(result),content_type='application/json')
示例#3
0
文件: views.py 项目: alenares/mtv
def import_shapefile(request):
    template_name = 'index.html'
    
    if request.method == 'POST':
        form = ImportShapefileForm( request.POST,
                                    request.FILES)
        if form.is_valid():
            shapefile = request.FILES['import_file']
            feature_type = request.POST['feature_type']
            
            err_msg = import_data(shapefile, feature_type)
        else:
            err_msg = "Please review the Upload form!"
        
        context = {'form' : form, 'err_msg' : err_msg}
        return render(request, template_name, context)
示例#4
0
def data_mapping(filename=None):
    filepath = path.join(current_app.config["UPLOAD_FOLDER"], filename)

    try:
        data_file = open(filepath, "r")
    except IOError:
        return abort(404)

    data_file = load_from_file(data_file)
    headers = next(data_file)
    lowercase_headers = [h.lower() for h in headers]
    headers_ = [""] + headers
    header_list = zip(headers_, headers_)
    table_columns = Transaction.__table__.columns
    mode = request.args.get("mode", None)

    class NewForm(FieldMappingForm):
        pass

    for field in table_columns:
        if not field.primary_key:
            i = lowercase_headers.index(field.name.lower(
            )) if field.name.lower() in lowercase_headers else -1
            default_ = headers[i] if i >= 0 else None
            NewForm = _generate_field(NewForm,
                                      field,
                                      choices=header_list,
                                      default=default_,
                                      mode=mode)

    form = NewForm(request.form)

    if request.method == "POST" and form.validate():
        total_imported = import_data(data_file, mode,
                                     form["spend_transactions"].data,
                                     form["income_transactions"].data,
                                     form["accountID"].data)
        flash("{} records have been imported".format(total_imported))

        data_file.close()
        remove(filepath)

        return redirect(url_for('admin.list', model_name='Transaction'))

    return render_template("edit_layout.html", form=form, title="Map Fields")
示例#5
0
def import_raster(request):
    if request.method == "GET":
        categories = Category.objects.all()
        return render(request, "import_raster.html",
                      {'categories': categories})
    elif request.method == "POST":
        result = {}
        try:
            print "se ejecuta la tarea en import_raster"
            result = importer.import_data(request)
            print "el resultado de la tarea en import_raster", result
            return HttpResponse(json.dumps(result),
                                content_type='application/json')
        except Exception as e:
            print "El error en import_raster", e
            result["error"] = str(e)
            return HttpResponse(json.dumps(result),
                                content_type='application/json')
示例#6
0
    def get(self):
        user_info = AuthenticatedRequestHandler.authenticate_user(self)
        if user_info:
            imported = importer.import_data()
            users = imported['users']
            prices = imported['prices']
            days = imported['days']

            template_values = {
                'title': 'Group Schedule',
                'page': 'group',
                'nickname': user_info[0],
                'logout_url': user_info[1],
                'users': users,
                'prices': prices,
                'days': days,
            }

            path = os.path.join(os.path.dirname(__file__), 'template.html')
            self.response.out.write(template.render(path, template_values))
示例#7
0
def data_mapping(filename=None):
    filepath = path.join(current_app.config["UPLOAD_FOLDER"], filename)

    try:
        data_file = open(filepath, "r")
    except IOError:
        return abort(404)

    data_file = load_from_file(data_file)
    headers = next(data_file)
    lowercase_headers = [h.lower() for h in headers]
    headers_ = [""] + headers
    header_list = zip(headers_, headers_)
    table_columns = Transaction.__table__.columns
    mode = request.args.get("mode", None)

    class NewForm(FieldMappingForm):
        pass

    for field in table_columns:
        if not field.primary_key:
            i = lowercase_headers.index(field.name.lower()) if field.name.lower() in lowercase_headers else -1
            default_ = headers[i] if i >= 0 else None
            NewForm = _generate_field(NewForm, field, choices=header_list, default=default_, mode=mode)

    form = NewForm(request.form)

    if request.method == "POST" and form.validate():
        total_imported = import_data(data_file,
                                mode,
                                form["spend_transactions"].data,
                                form["income_transactions"].data,
                                form["accountID"].data)
        flash("{} records have been imported".format(total_imported))

        data_file.close()
        remove(filepath)

        return redirect(url_for('admin.list', model_name='Transaction'))

    return render_template("edit_layout.html", form=form, title="Map Fields")
示例#8
0
import sys
import os

# argument passed to runner program is stored in host variable
host = sys.argv[1]
#host = "cs"
param.dbconn(host)

if not os.path.exists(param.newpath):
    os.makedirs(param.newpath)

if sys.argv[1] in ('cs', 'abasence.io'):
    print("Extracting Json for " + str(param.start_date) + " - " +
          str(param.end_date))

else:
    print("No other ETL defiend yet")

if (host == "cs"):
    param.counter = 2
    for i in param.tbl_cs:
        print('Extracting Json for ' + i + ' started')
        runner = Exporter(i)
        runner.start()
else:
    print("no other mongo source is defined yet")

# run the ETL process until all the mentioned collections in the param file are exported.
while param.counter != 0:
    importer.import_data()
示例#9
0
 def import_data(self):
     data = self.cleaned_data['data_file']
     return import_data(data)
def get_data(file_paths):
    return importer.import_data(file_paths[0], file_paths[1])