def build_zipfile_of_videos(self,
filename,
study_uuid,
match,
requesting_user_uuid,
consent_only=False):
from studies.models import Study
from accounts.models import User
study = Study.objects.get(uuid=study_uuid)
requesting_user = User.objects.get(uuid=requesting_user_uuid)
video_qs = (study.consent_videos
if consent_only else study.videos_for_consented_responses)
if not requesting_user.has_study_perms(
StudyPermission.READ_STUDY_RESPONSE_DATA, study):
video_qs = video_qs.filter(response__is_preview=True)
if not requesting_user.has_study_perms(
StudyPermission.READ_STUDY_PREVIEW_DATA, study):
video_qs = video_qs.filter(response__is_preview=False)
if match:
video_qs = video_qs.filter(full_name__contains=match)
m = hashlib.sha256()
for video in video_qs:
m.update(video.full_name.encode("utf-8"))
# create a sha256 of the included filenames
sha = m.hexdigest()
# use that sha in the filename
zip_filename = f"{filename}_{sha}.zip"
# get the gc client
gs_client = gc_storage.client.Client(project=settings.GS_PROJECT_ID)
# get the bucket
gs_private_bucket = gs_client.get_bucket(settings.GS_PRIVATE_BUCKET_NAME)
# instantiate a blob for the file
gs_blob = gc_storage.blob.Blob(zip_filename,
gs_private_bucket,
chunk_size=256 * 1024 * 1024) # 256mb
# if the file exists short circuit and send the email with a 30m link
if not gs_blob.exists():
# if it doesn't exist build the zipfile
with tempfile.TemporaryDirectory() as temp_directory:
zip_file_path = os.path.join(temp_directory, zip_filename)
with zipfile.ZipFile(zip_file_path, "w") as zf:
for video in video_qs:
temporary_file_path = os.path.join(temp_directory,
video.full_name)
file_response = requests.get(video.download_url,
stream=True)
with open(temporary_file_path, mode="w+b") as local_file:
for chunk in file_response.iter_content(8192):
local_file.write(chunk)
zf.write(temporary_file_path, video.full_name)
os.remove(temporary_file_path)
# upload the zip to GoogleCloudStorage
gs_blob.upload_from_filename(zip_file_path)
# then send the email with a 30m link
signed_url = gs_blob.generate_signed_url(
int(time.time() + datetime.timedelta(minutes=30).seconds))
# send an email with the signed url and return
email_context = dict(
signed_url=signed_url,
user=requesting_user,
videos=video_qs,
zip_filename=zip_filename,
)
send_mail(
"download_zip",
"Your video archive has been created",
[requesting_user.username],
from_email=settings.EMAIL_FROM_ADDRESS,
**email_context,
)
def build_framedata_dict(filename, study_uuid, requesting_user_uuid):
from studies.models import Study
from accounts.models import User
from exp.views.responses import build_framedata_dict_csv
requesting_user = User.objects.get(uuid=requesting_user_uuid)
study = Study.objects.get(uuid=study_uuid)
response_qs = study.responses_for_researcher(requesting_user).order_by(
"id")
responses = response_qs.select_related("child", "study").values(
"uuid",
"exp_data",
"child__uuid",
"study__uuid",
"study__salt",
"study__hash_digits",
"global_event_timings",
)
# make filename for this request unique by adding timestamp
timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
csv_filename = f"{filename}_{timestamp}.csv"
# get the gc client
gs_client = gc_storage.client.Client(project=settings.GS_PROJECT_ID)
# get the bucket
gs_private_bucket = gs_client.get_bucket(settings.GS_PRIVATE_BUCKET_NAME)
# instantiate a blob for the file
gs_blob = gc_storage.blob.Blob(csv_filename,
gs_private_bucket,
chunk_size=256 * 1024 * 1024) # 256mb
header_list = [
"column",
"description",
"possible_frame_id",
"frame_description",
"possible_key",
"key_description",
]
# if the file exists short circuit and send the email
if not gs_blob.exists():
# if it doesn't exist build the file
with tempfile.TemporaryDirectory() as temp_directory:
file_path = os.path.join(temp_directory, csv_filename)
with open(file_path, "w") as csv_file:
writer = csv.DictWriter(
csv_file,
quoting=csv.QUOTE_NONNUMERIC,
fieldnames=header_list,
restval="",
extrasaction="ignore",
)
writer.writeheader()
build_framedata_dict_csv(writer, responses)
# upload the csv to GoogleCloudStorage
gs_blob.upload_from_filename(file_path)
# then send the email with a 24h link
signed_url = gs_blob.generate_signed_url(datetime.timedelta(hours=24))
# send an email with the signed url and return
email_context = dict(signed_url=signed_url,
user=requesting_user,
csv_filename=csv_filename)
send_mail(
"download_framedata_dict",
"Your frame data dictionary has been created",
[requesting_user.username],
from_email=settings.EMAIL_FROM_ADDRESS,
**email_context,
)
def build_zipfile_of_videos(self,
filename,
study_uuid,
orderby,
match,
requesting_user_uuid,
consent=False):
from studies.models import Study
from accounts.models import User
# get the study in question
study = Study.objects.get(uuid=study_uuid)
# get the user
requesting_user = User.objects.get(uuid=requesting_user_uuid)
# find the requested attachments
if consent:
attachments = attachment_helpers.get_consent_videos(study.uuid)
else:
attachments = attachment_helpers.get_study_attachments(
study, orderby, match)
m = hashlib.sha256()
for attachment in attachments:
m.update(attachment.key.encode('utf-8'))
# create a sha256 of the included filenames
sha = m.hexdigest()
# use that sha in the filename
zip_filename = f'{filename}_{sha}.zip'
# get the gc client
gs_client = gc_storage.client.Client(project=settings.GS_PROJECT_ID)
# get the bucket
gs_private_bucket = gs_client.get_bucket(settings.GS_PRIVATE_BUCKET_NAME)
# instantiate a blob for the file
gs_blob = gc_storage.blob.Blob(zip_filename, gs_private_bucket)
# if the file exists short circuit and send the email with a 30m link
if not gs_blob.exists():
# if it doesn't exist build the zipfile
with tempfile.TemporaryDirectory() as temp_directory:
zip_file_path = os.path.join(temp_directory, zip_filename)
zip = zipfile.ZipFile(zip_file_path, 'w')
for attachment in attachments:
temporary_file_path = os.path.join(temp_directory,
attachment.key)
file_response = requests.get(
attachment_helpers.get_download_url(attachment.key),
stream=True)
local_file = open(temporary_file_path, mode='w+b')
for chunk in file_response.iter_content(8192):
local_file.write(chunk)
local_file.close()
zip.write(temporary_file_path, attachment.key)
zip.close()
# upload the zip to GoogleCloudStorage
gs_blob.upload_from_filename(zip_file_path)
# then send the email with a 30m link
signed_url = gs_blob.generate_signed_url(
int(time.time() + datetime.timedelta(minutes=30).seconds))
# send an email with the signed url and return
context = dict(signed_url=signed_url,
user=requesting_user,
videos=attachments,
zip_filename=zip_filename)
send_mail('download_zip',
'Your video archive has been created',
settings.EMAIL_FROM_ADDRESS,
bcc=[
requesting_user.username,
],
from_email=settings.EMAIL_FROM_ADDRESS,
**context)