def test_check_successful_run(self):
redis.flushdb()
self.assertEqual(0, redis.dbsize())
import_data()
self.assertEqual(2, redis.dbsize())
self.assertEqual(10, len(redis.lrange('warehouse_whcorp', 0, -1)))
self.assertEqual(
10, len(redis.lrange('warehouse_american_storage', 0, -1)))
def import_shapefile(request):
if request.method == "GET":
categories = Category.objects.all();
return render(request, "import_shapefile.html",{'categories':categories});
elif request.method == "POST":
result = {}
try:
# se ejecuta la tarea de Celery
result = importer.import_data(request)
return HttpResponse(json.dumps(result),content_type='application/json')
except Exception as e:
result["error"]=str(e);
return HttpResponse(json.dumps(result),content_type='application/json')
def import_shapefile(request):
template_name = 'index.html'
if request.method == 'POST':
form = ImportShapefileForm( request.POST,
request.FILES)
if form.is_valid():
shapefile = request.FILES['import_file']
feature_type = request.POST['feature_type']
err_msg = import_data(shapefile, feature_type)
else:
err_msg = "Please review the Upload form!"
context = {'form' : form, 'err_msg' : err_msg}
return render(request, template_name, context)
def data_mapping(filename=None):
filepath = path.join(current_app.config["UPLOAD_FOLDER"], filename)
try:
data_file = open(filepath, "r")
except IOError:
return abort(404)
data_file = load_from_file(data_file)
headers = next(data_file)
lowercase_headers = [h.lower() for h in headers]
headers_ = [""] + headers
header_list = zip(headers_, headers_)
table_columns = Transaction.__table__.columns
mode = request.args.get("mode", None)
class NewForm(FieldMappingForm):
pass
for field in table_columns:
if not field.primary_key:
i = lowercase_headers.index(field.name.lower(
)) if field.name.lower() in lowercase_headers else -1
default_ = headers[i] if i >= 0 else None
NewForm = _generate_field(NewForm,
field,
choices=header_list,
default=default_,
mode=mode)
form = NewForm(request.form)
if request.method == "POST" and form.validate():
total_imported = import_data(data_file, mode,
form["spend_transactions"].data,
form["income_transactions"].data,
form["accountID"].data)
flash("{} records have been imported".format(total_imported))
data_file.close()
remove(filepath)
return redirect(url_for('admin.list', model_name='Transaction'))
return render_template("edit_layout.html", form=form, title="Map Fields")
def import_raster(request):
if request.method == "GET":
categories = Category.objects.all()
return render(request, "import_raster.html",
{'categories': categories})
elif request.method == "POST":
result = {}
try:
print "se ejecuta la tarea en import_raster"
result = importer.import_data(request)
print "el resultado de la tarea en import_raster", result
return HttpResponse(json.dumps(result),
content_type='application/json')
except Exception as e:
print "El error en import_raster", e
result["error"] = str(e)
return HttpResponse(json.dumps(result),
content_type='application/json')
def get(self):
user_info = AuthenticatedRequestHandler.authenticate_user(self)
if user_info:
imported = importer.import_data()
users = imported['users']
prices = imported['prices']
days = imported['days']
template_values = {
'title': 'Group Schedule',
'page': 'group',
'nickname': user_info[0],
'logout_url': user_info[1],
'users': users,
'prices': prices,
'days': days,
}
path = os.path.join(os.path.dirname(__file__), 'template.html')
self.response.out.write(template.render(path, template_values))
def data_mapping(filename=None):
filepath = path.join(current_app.config["UPLOAD_FOLDER"], filename)
try:
data_file = open(filepath, "r")
except IOError:
return abort(404)
data_file = load_from_file(data_file)
headers = next(data_file)
lowercase_headers = [h.lower() for h in headers]
headers_ = [""] + headers
header_list = zip(headers_, headers_)
table_columns = Transaction.__table__.columns
mode = request.args.get("mode", None)
class NewForm(FieldMappingForm):
pass
for field in table_columns:
if not field.primary_key:
i = lowercase_headers.index(field.name.lower()) if field.name.lower() in lowercase_headers else -1
default_ = headers[i] if i >= 0 else None
NewForm = _generate_field(NewForm, field, choices=header_list, default=default_, mode=mode)
form = NewForm(request.form)
if request.method == "POST" and form.validate():
total_imported = import_data(data_file,
mode,
form["spend_transactions"].data,
form["income_transactions"].data,
form["accountID"].data)
flash("{} records have been imported".format(total_imported))
data_file.close()
remove(filepath)
return redirect(url_for('admin.list', model_name='Transaction'))
return render_template("edit_layout.html", form=form, title="Map Fields")
import sys
import os
# argument passed to runner program is stored in host variable
host = sys.argv[1]
#host = "cs"
param.dbconn(host)
if not os.path.exists(param.newpath):
os.makedirs(param.newpath)
if sys.argv[1] in ('cs', 'abasence.io'):
print("Extracting Json for " + str(param.start_date) + " - " +
str(param.end_date))
else:
print("No other ETL defiend yet")
if (host == "cs"):
param.counter = 2
for i in param.tbl_cs:
print('Extracting Json for ' + i + ' started')
runner = Exporter(i)
runner.start()
else:
print("no other mongo source is defined yet")
# run the ETL process until all the mentioned collections in the param file are exported.
while param.counter != 0:
importer.import_data()
def import_data(self):
data = self.cleaned_data['data_file']
return import_data(data)
def get_data(file_paths):
return importer.import_data(file_paths[0], file_paths[1])