def extractor_save(base_path, folder, outputs):
nb_written = 0
name_to_path = {}
wrote_endpoints = False
for name, contents in outputs:
if '.proto' in name:
if folder:
path = base_path / 'protos' / folder / name
else:
path = base_path / name
makedirs(path.parent, exist_ok=True)
with open(path, 'w') as fd:
fd.write(contents)
if name not in name_to_path:
nb_written += 1
name_to_path[name] = str(path)
elif name.endswith('.sample'):
endpoint = contents
name = name.replace('.sample', '.proto')
endpoint['proto_path'] = name_to_path[name]
endpoint['proto_msg'] = name.replace('.proto', '')
wrote_endpoints = True
if folder:
insert_endpoint(base_path / 'endpoints', {'request': endpoint})
else:
insert_endpoint(base_path, {'request': endpoint})
return nb_written, wrote_endpoints
def create_CSV_zone(filepath, x1, y1, x2, y2, out_filepath, nb_samples=-1):
"""create a CSV file with all the samples
for an area specified by x1 an y1 coordinates (top left corner pixel) and x2 an y2 coordinates (bottom right corner pixel)
Arguments:
filepath : {str} -- repertory of the rawls samples
x1 : {int} -- horizontal coordinate of the top left corner pixel
y1 : {int} -- vertical coordinate of the top left corner pixel
x2 : {int} -- horizontal coordinate of the bottom right corner pixel
y2 : {int} -- vertical coordinate of the bottom right corner pixel
out_filepath : {str} -- path where we want to save the CSV file
nb_samples : {int} -- number of the samples we use for create CSV file
Raises:
Exception: Invalid coodinates
Exception: Invalid input filepath, need at least one image rawls
"""
if (x1 > x2) or (y1 > y2):
raise Exception('Invalid coordinates')
if filepath.endswith("/"):
filepath = filepath[:-1]
if out_filepath.endswith("/"):
out_filepath = out_filepath[:-1]
if out_filepath.startswith("/home") == False:
if out_filepath.startswith("/") == False:
out_filepath = "/" + out_filepath
cwd = getcwd()
out_filepath = cwd + out_filepath
#create a temp repertory
temp_out_filepath = "/tmp/" + filepath.split('/')[-1]
for j in range(y2 - y1 + 1):
for i in range(x2 - x1 + 1):
create_CSV(filepath, x1 + i, y1 + j, temp_out_filepath, nb_samples)
if nb_samples == -1:
nb_samples = 0
for name in listdir(filepath):
if name.endswith(".rawls"):
nb_samples += 1
chdir(temp_out_filepath)
file_extension = '.csv'
all_filenames = [i for i in glob.glob(f"*{file_extension}")]
#combine all files in the list
# combined_csv = pd.concat([pd.read_csv(f) for f in all_filenames ], ignore_index=True)
reader = []
for f in all_filenames:
read = pd.read_csv(f)
reader.append(read)
for x in range(len(all_filenames)):
if x != 0:
for column in reader[x].columns:
for row in range(nb_samples):
reader[0][column] = reader[x][column].iloc[row]
combined_csv = reader[0]
file_name_CSV = filepath.split('/')[-1] + "_" + str(x1) + "_" + str(
y1) + "_to_" + str(x2) + "_" + str(y2) + ".csv"
combined_csv.to_csv(file_name_CSV, index=False, encoding='utf-8-sig')
shutil.move(file_name_CSV, out_filepath + "/" + file_name_CSV)
shutil.rmtree(temp_out_filepath)
def clientUploadFile(request):
#Get Attributes
uploadedFile = None
allowAllUsersWrite = None
allowAllUsersRead = None
allowKeyUsersRead = None
allowKeyUsersWrite = None
name = None
integration = None
parent = None
size = None
try:
uploadedFile = request.FILES['file'].read()
name = request.POST.get("name")
allowAllUsersWrite = strtobool(request.POST.get("allowAllUsersWrite"))
allowAllUsersRead = strtobool(request.POST.get("allowAllUsersRead"))
allowKeyUsersRead = strtobool(request.POST.get("allowKeyUsersRead"))
allowKeyUsersWrite = strtobool(request.POST.get("allowKeyUsersWrite"))
integration = request.POST.get("integration")
parent = request.POST.get("parent")
size = request.POST.get("size")
except:
return HttpResponse("woahh - does'nt seem like the data we need")
#check if folder contains unwanted charectors
special_characters = "'" "!@#$%^&*+?=,<>/" "'"
if any(c in special_characters for c in name):
return HttpResponse("1702")
# Get the integration and project
try:
clientsIntegration = Integration.objects.get(
integrationKey=integration)
project = clientsIntegration.project
except:
return HttpResponse("not found")
# Get the parent index object to insert the file to
parentIndexObject = None
try:
if (parent == "root"):
parentIndexObject = IndexObject.objects.get(
name=f"{project.owner.username}.{project.name}")
else:
parentIndexObject = IndexObject.objects.get(id=parent)
except:
return HttpResponse("500")
# check barn
try:
BarnedDeveloperClient.objects.get(
project=project,
client=DeveloperClient.objects.get(user=request.user))
return HttpResponse("denied")
except:
pass
#check if the filename file exists
try:
IndexObject.objects.get(name=name, parent=parentIndexObject)
return HttpResponse("1703")
except:
pass
# Add the file to the parent object
try:
#create an index object
newIndexObject = IndexObject()
newIndexObject.create(request.user,
"FL",
name,
parentIndexObject.project,
parentIndexObject,
size=size) #create first time to get reference
#upload to mongo
mongoUploadFile(uploadedFile, newIndexObject.id, name,
request.user.username)
#update indexObject
fileType = "file"
if (name.endswith(".mp3") or name.endswith(".WAV")):
fileType = "audio"
elif (name.endswith(".mp4") or name.endswith(".MOV")
or name.endswith(".WMV") or name.endswith(".FLV")
or name.endswith(".MKV")):
fileType = "video"
elif (name.endswith(".pdf")):
fileType = "pdf"
newIndexObject.fileReference = newIndexObject.id
newIndexObject.fileType = fileType
newIndexObject.allowKeyUsersWrite = allowKeyUsersWrite
newIndexObject.allowKeyUsersRead = allowKeyUsersRead
newIndexObject.allowAllUsersRead = allowAllUsersRead
newIndexObject.allowAllUsersWrite = allowAllUsersWrite
newIndexObject.save()
return HttpResponse(newIndexObject.id)
except:
return HttpResponse("Boss man! something is seriously wrong")