def importdatabygeometrytype(geometrytype, year, datatype):
"""
Import data for a specific geometry type and year
"""
datatypecode = 'ELEC'
if datatype == 1: datatypecode = 'GAS'
geometry_prefix = 'LSOA'
geometrycode_row = 'LSOACode'
multiplier_meter = 1
multiplier_value = 1
if geometrytype == 'msoa':
geometry_prefix = 'MSOA'
geometrycode_row = 'MSOACode'
if geometrytype == 'lau1':
geometry_prefix = 'LAU1'
geometrycode_row = 'LA Code'
multiplier_meter = 1000
multiplier_value = 1000000
filepath = 'BEIS/' + geometry_prefix + '_' + datatypecode + '_' + str(
year) + '.csv'
Data.objects.filter(geometrytype=geometrytype, year=year,
type=datatype).delete()
count = 0
if os.path.isfile(filepath):
with open(filepath, 'r') as fileobj:
reader = csv.DictReader(fileobj)
for row in reader:
count += 1
print("Importing line", count, filepath)
geometrycode = row[geometrycode_row].strip()
if geometrytype == 'lau1':
if row['Total consumption'] == '..': continue
if row['Total consumption'] == ' - ': continue
value = float(non_decimal.sub("",
row['Total consumption']))
meters = float(row['Total number of meters'])
else:
value = float(row['KWH'])
meters = float(row['METERS'])
meters = meters * multiplier_meter
value = value * multiplier_value
data = Data(type=datatype,
year=str(year),
value=value,
meters=meters,
geometrycode=geometrycode,
geometrytype=geometrytype)
data.save()
print("Imported " + geometry_prefix + " for type " + str(datatype) +
" for " + str(year))
else:
print(filepath, "not found")
def add_data(request):
if request.method == "POST":
# print(request.body)
# req=json.loads(request.body,strict=False)
# logging.info(req)
# key_flag=req.get("name") and req.get("data_type")
# if key_flag:
# return JsonResponse({"status":"BS.100","msg":"publish article sucess."})
print("request is ", request)
print("body is ", request.body)
name = request.POST.get("name")
data_type = request.POST.get("data_type")
case_id = request.POST.get("case")
print(name)
if name and data_type:
add_da = Data(name=name, data_type=data_type, case=case_id)
add_da.save()
return JsonResponse({
"status": "BS.200",
"msg": "publish article sucess."
})
else:
return JsonResponse({
"status": "BS.300",
"msg": "publish article sucess."
})
else:
return JsonResponse({"status": "BS.400", "msg": "error."})
def fetch_data_montly():
print("test passed ")
# users = User.objects.all()
# print(users)
daily_campaigns = Campaign.objects.filter(crawlInterval=Campaign.MONTHLY)
for campaign in daily_campaigns:
try:
response = urllib.request.urlopen(campaign.source)
data = json.load(response)
print(data)
Data(campaign=campaign, data=data).save()
except Exception as e:
print("error")
print(e)
def add_data():
api_key = request.headers.get("Authorization", None)
app.logger.info(api_key)
if not api_key:
return jsonify(
message="API Key missing from `Authorization` Header"), 401
try:
project = Project.query.filter_by(api_key=api_key).first()
except Exception as e:
app.logger.info(e)
return jsonify(message="No project exist with given API Key"), 404
username = request.form.get("username", None)
user = User.query.filter_by(username=username).first()
if not user:
return jsonify(message="No user found with given username"), 404
reference_transcription = request.form.get("reference_transcription", None)
is_marked_for_review = bool(request.form.get("is_marked_for_review",
False))
audio_file = request.files["audio_file"]
original_filename = secure_filename(audio_file.filename)
extension = Path(original_filename).suffix.lower()
if len(extension) > 1 and extension[1:] not in ALLOWED_EXTENSIONS:
return jsonify(message="File format is not supported"), 400
filename = f"{str(uuid.uuid4().hex)}{extension}"
file_path = Path(app.config["UPLOAD_FOLDER"]).joinpath(filename)
audio_file.save(file_path.as_posix())
app.logger.info(filename)
try:
data = Data(
project_id=project.id,
filename=filename,
original_filename=original_filename,
reference_transcription=reference_transcription,
is_marked_for_review=is_marked_for_review,
assigned_user_id=user.id,
)
db.session.add(data)
db.session.commit()
db.session.refresh(data)
except Exception as e:
app.logger.error(f"Error adding data to project: {project.name}")
app.logger.error(e)
return (
jsonify(
message=f"Error adding data to project: {project.name}",
type="DATA_CREATION_FAILED",
),
500,
)
return (
jsonify(
data_id=data.id,
message=f"Data uploaded, created and assigned successfully",
type="DATA_CREATED",
),
201,
)
def add_data():
api_key = request.headers.get("Authorization", None)
if not api_key:
raise BadRequest(
description="API Key missing from `Authorization` Header")
project = Project.query.filter_by(api_key=api_key).first()
if not project:
raise NotFound(description="No project exist with given API Key")
username = request.form.getlist("username", None)
username_id = {}
for name in username:
user = User.query.filter_by(username=name).first()
if not user:
raise NotFound(description="No user found with given username")
username_id[name] = user.id
segmentations = request.form.get("segmentations", "[]")
reference_transcription = request.form.get("reference_transcription", None)
is_marked_for_review = bool(request.form.get("is_marked_for_review",
False))
audio_file = request.files["audio_file"]
original_filename = secure_filename(audio_file.filename)
extension = Path(original_filename).suffix.lower()
if len(extension) > 1 and extension[1:] not in ALLOWED_EXTENSIONS:
raise BadRequest(description="File format is not supported")
filename = f"{str(uuid.uuid4().hex)}{extension}"
file_path = Path(app.config["UPLOAD_FOLDER"]).joinpath(filename)
audio_file.save(file_path.as_posix())
try:
data = Data(
project_id=project.id,
filename=filename,
original_filename=original_filename,
reference_transcription=reference_transcription,
is_marked_for_review=is_marked_for_review,
assigned_user_id=username_id,
)
except Exception as e:
#error = "username_id is bad " + username_id
raise BadRequest(description="username_id is bad ")
print("HELLLO THERE ERROR MESSAGE")
db.session.add(data)
db.session.flush()
segmentations = json.loads(segmentations)
new_segmentations = []
for segment in segmentations:
validated = validate_segmentation(segment)
if not validated:
raise BadRequest(description=f"Segmentations have missing keys.")
new_segment = generate_segmentation(
data_id=data.id,
project_id=project.id,
end_time=segment["end_time"],
start_time=segment["start_time"],
annotations=segment.get("annotations", {}),
transcription=segment["transcription"],
)
new_segmentations.append(new_segment)
data.set_segmentations(new_segmentations)
db.session.commit()
db.session.refresh(data)
return (
jsonify(
data_id=data.id,
message=
f"Data uploaded, created and assigned successfully for user",
type="DATA_CREATED",
),
201,
)