def upload_file(self, file_object, filename=None, use_file_name=False, document_type=None, expand=False, metadata_dict_list=None, user=None, document=None, new_version_data=None, command_line=False):
is_compressed = None
if expand:
try:
cf = CompressedFile(file_object)
count = 1
for fp in cf.children():
if command_line:
print 'Uploading file #%d: %s' % (count, fp)
self.upload_single_file(file_object=fp, filename=None, document_type=document_type, metadata_dict_list=metadata_dict_list, user=user)
fp.close()
count += 1
except NotACompressedFile:
is_compressed = False
logging.debug('Exception: NotACompressedFile')
if command_line:
raise
self.upload_single_file(file_object=file_object, filename=filename, document_type=document_type, metadata_dict_list=metadata_dict_list, user=user)
else:
is_compressed = True
else:
self.upload_single_file(file_object, filename, use_file_name, document_type, metadata_dict_list, user, document, new_version_data)
file_object.close()
return {'is_compressed': is_compressed}
def handle_upload(self, file_object, description=None, document_type=None, expand=False, label=None, language=None, metadata_dict_list=None, metadata_dictionary=None, user=None):
if not document_type:
document_type = self.document_type
kwargs = {
'description': description, 'document_type': document_type,
'label': label, 'language': language,
'metadata_dict_list': metadata_dict_list,
'metadata_dictionary': metadata_dictionary, 'user': user
}
if expand:
try:
compressed_file = CompressedFile(file_object)
for compressed_file_child in compressed_file.children():
kwargs.update({'label': unicode(compressed_file_child)})
self.upload_document(
file_object=File(compressed_file_child), **kwargs
)
compressed_file_child.close()
except NotACompressedFile:
logging.debug('Exception: NotACompressedFile')
self.upload_document(file_object=file_object, **kwargs)
else:
self.upload_document(file_object=file_object, **kwargs)
def handle_upload(self, file_object, description=None, document_type=None, expand=False, label=None, language=None, metadata_dict_list=None, metadata_dictionary=None, user=None):
if not document_type:
document_type = self.document_type
kwargs = {
'description': description, 'document_type': document_type,
'label': label, 'language': language,
'metadata_dict_list': metadata_dict_list,
'metadata_dictionary': metadata_dictionary, 'user': user
}
if expand:
try:
compressed_file = CompressedFile(file_object)
for compressed_file_child in compressed_file.children():
kwargs.update({'label': force_text(compressed_file_child)})
self.upload_document(
file_object=File(compressed_file_child), **kwargs
)
compressed_file_child.close()
except NotACompressedFile:
logging.debug('Exception: NotACompressedFile')
self.upload_document(file_object=file_object, **kwargs)
else:
self.upload_document(file_object=file_object, **kwargs)
def handle_upload(self, file_object, description=None, document_type=None, expand=False, label=None, language=None, metadata_dict_list=None, metadata_dictionary=None, tag_ids=None, user=None):
"""
Handle an upload request from a file object which may be an individual
document or a compressed file containing multiple documents.
"""
if not document_type:
document_type = self.document_type
kwargs = {
'description': description, 'document_type': document_type,
'label': label, 'language': language,
'metadata_dict_list': metadata_dict_list,
'metadata_dictionary': metadata_dictionary, 'tag_ids': tag_ids,
'user': user
}
if expand:
try:
compressed_file = CompressedFile(file_object)
for compressed_file_child in compressed_file.children():
kwargs.update({'label': force_text(compressed_file_child)})
self.upload_document(
file_object=File(compressed_file_child), **kwargs
)
compressed_file_child.close()
except NotACompressedFile:
logging.debug('Exception: NotACompressedFile')
self.upload_document(file_object=file_object, **kwargs)
else:
self.upload_document(file_object=file_object, **kwargs)
def handle_upload(self,
file_object,
description=None,
document_type=None,
expand=False,
label=None,
language=None,
user=None):
"""
Handle an upload request from a file object which may be an individual
document or a compressed file containing multiple documents.
"""
documents = []
if not document_type:
document_type = self.document_type
kwargs = {
'description': description,
'document_type': document_type,
'label': label,
'language': language,
'user': user
}
if expand:
try:
compressed_file = CompressedFile(file_object)
for compressed_file_child in compressed_file.children():
kwargs.update({'label': force_text(compressed_file_child)})
documents.append(
self.upload_document(
file_object=File(compressed_file_child), **kwargs))
compressed_file_child.close()
except NotACompressedFile:
logging.debug('Exception: NotACompressedFile')
documents.append(
self.upload_document(file_object=file_object, **kwargs))
else:
documents.append(
self.upload_document(file_object=file_object, **kwargs))
# Return a list of newly created documents. Used by the email source
# to assign the from and subject metadata values.
return documents
def new_document(self, document_type, file_object, label=None, command_line=False, description=None, expand=False, language=None, user=None):
versions_created = []
if expand:
try:
compressed_file = CompressedFile(file_object)
count = 1
for compressed_file_child in compressed_file.children():
if command_line:
print 'Uploading file #%d: %s' % (count, compressed_file_child)
versions_created.append(self.upload_single_document(document_type=document_type, file_object=compressed_file_child, description=description, label=unicode(compressed_file_child), language=language or LANGUAGE, user=user))
compressed_file_child.close()
count += 1
except NotACompressedFile:
logging.debug('Exception: NotACompressedFile')
if command_line:
raise
versions_created.append(self.upload_single_document(document_type=document_type, file_object=file_object, description=description, label=label, language=language or LANGUAGE, user=user))
else:
versions_created.append(self.upload_single_document(document_type=document_type, file_object=file_object, description=description, label=label, language=language or LANGUAGE, user=user))
return versions_created
def task_source_handle_upload(self,
document_type_id,
shared_uploaded_file_id,
source_id,
description=None,
expand=False,
label=None,
language=None,
metadata_dict_list=None,
skip_list=None,
tag_ids=None,
user_id=None):
SharedUploadedFile = apps.get_model(app_label='common',
model_name='SharedUploadedFile')
DocumentType = apps.get_model(app_label='documents',
model_name='DocumentType')
try:
document_type = DocumentType.objects.get(pk=document_type_id)
shared_upload = SharedUploadedFile.objects.get(
pk=shared_uploaded_file_id)
if not label:
label = shared_upload.filename
except OperationalError as exception:
logger.warning(
'Operational error during attempt to load data to handle source '
'upload: %s. Retrying.', exception)
raise self.retry(exc=exception)
kwargs = {
'description': description,
'document_type_id': document_type.pk,
'label': label,
'language': language,
'metadata_dict_list': metadata_dict_list,
'source_id': source_id,
'tag_ids': tag_ids,
'user_id': user_id
}
if not skip_list:
skip_list = []
with shared_upload.open() as file_object:
if expand:
try:
compressed_file = CompressedFile(file_object)
for compressed_file_child in compressed_file.children():
# TODO: find way to uniquely identify child files
# Use filename in the mean time.
if force_text(compressed_file_child) not in skip_list:
kwargs.update(
{'label': force_text(compressed_file_child)})
try:
child_shared_uploaded_file = SharedUploadedFile.objects.create(
file=File(compressed_file_child))
except OperationalError as exception:
logger.warning(
'Operational error while preparing to upload '
'child document: %s. Rescheduling.', exception)
task_source_handle_upload.delay(
document_type_id=document_type_id,
shared_uploaded_file_id=shared_uploaded_file_id,
source_id=source_id,
description=description,
expand=expand,
label=label,
language=language,
metadata_dict_list=metadata_dict_list,
skip_list=skip_list,
tag_ids=tag_ids,
user_id=user_id)
return
else:
skip_list.append(force_text(compressed_file_child))
task_upload_document.delay(
shared_uploaded_file_id=
child_shared_uploaded_file.pk,
**kwargs)
finally:
compressed_file_child.close()
compressed_file_child.close()
try:
shared_upload.delete()
except OperationalError as exception:
logger.warning(
'Operational error during attempt to delete shared '
'upload file: %s; %s. Retrying.', shared_upload,
exception)
except NotACompressedFile:
logging.debug('Exception: NotACompressedFile')
task_upload_document.delay(
shared_uploaded_file_id=shared_upload.pk, **kwargs)
else:
task_upload_document.delay(
shared_uploaded_file_id=shared_upload.pk, **kwargs)
def task_source_handle_upload(self, document_type_id, shared_uploaded_file_id, source_id, description=None, expand=False, label=None, language=None, metadata_dict_list=None, skip_list=None, tag_ids=None, user_id=None):
SharedUploadedFile = apps.get_model(
app_label='common', model_name='SharedUploadedFile'
)
DocumentType = apps.get_model(
app_label='documents', model_name='DocumentType'
)
try:
document_type = DocumentType.objects.get(pk=document_type_id)
shared_upload = SharedUploadedFile.objects.get(
pk=shared_uploaded_file_id
)
if not label:
label = shared_upload.filename
except OperationalError as exception:
logger.warning(
'Operational error during attempt to load data to handle source '
'upload: %s. Retrying.', exception
)
raise self.retry(exc=exception)
kwargs = {
'description': description, 'document_type_id': document_type.pk,
'label': label, 'language': language,
'metadata_dict_list': metadata_dict_list,
'source_id': source_id, 'tag_ids': tag_ids, 'user_id': user_id
}
if not skip_list:
skip_list = []
with shared_upload.open() as file_object:
if expand:
try:
compressed_file = CompressedFile(file_object)
for compressed_file_child in compressed_file.children():
# TODO: find way to uniquely identify child files
# Use filename in the mean time.
if force_text(compressed_file_child) not in skip_list:
kwargs.update(
{'label': force_text(compressed_file_child)}
)
try:
child_shared_uploaded_file = SharedUploadedFile.objects.create(
file=File(compressed_file_child)
)
except OperationalError as exception:
logger.warning(
'Operational error while preparing to upload '
'child document: %s. Rescheduling.', exception
)
task_source_handle_upload.delay(
document_type_id=document_type_id,
shared_uploaded_file_id=shared_uploaded_file_id,
source_id=source_id, description=description,
expand=expand, label=label,
language=language,
metadata_dict_list=metadata_dict_list,
skip_list=skip_list, tag_ids=tag_ids,
user_id=user_id
)
return
else:
skip_list.append(force_text(compressed_file_child))
task_upload_document.delay(
shared_uploaded_file_id=child_shared_uploaded_file.pk,
**kwargs
)
finally:
compressed_file_child.close()
compressed_file_child.close()
try:
shared_upload.delete()
except OperationalError as exception:
logger.warning(
'Operational error during attempt to delete shared '
'upload file: %s; %s. Retrying.', shared_upload,
exception
)
except NotACompressedFile:
logging.debug('Exception: NotACompressedFile')
task_upload_document.delay(
shared_uploaded_file_id=shared_upload.pk, **kwargs
)
else:
task_upload_document.delay(
shared_uploaded_file_id=shared_upload.pk, **kwargs
)
def task_source_handle_upload(
self,
document_type_id,
shared_uploaded_file_id,
source_id,
description=None,
expand=False,
label=None,
language=None,
metadata_dict_list=None,
skip_list=None,
user_id=None,
):
try:
document_type = DocumentType.objects.get(pk=document_type_id)
shared_upload = SharedUploadedFile.objects.get(pk=shared_uploaded_file_id)
if not label:
label = shared_upload.filename
except OperationalError as exception:
logger.warning(
"Operational error during attempt to load data to handle source " "upload: %s. Retrying.", exception
)
raise self.retry(exc=exception)
kwargs = {
"description": description,
"document_type_id": document_type.pk,
"label": label,
"language": language,
"metadata_dict_list": metadata_dict_list,
"source_id": source_id,
"user_id": user_id,
}
if not skip_list:
skip_list = []
with shared_upload.open() as file_object:
if expand:
try:
compressed_file = CompressedFile(file_object)
for compressed_file_child in compressed_file.children():
# TODO: find way to uniquely identify child files
# Use filename in the mean time.
if unicode(compressed_file_child) not in skip_list:
kwargs.update({"label": unicode(compressed_file_child)})
try:
child_shared_uploaded_file = SharedUploadedFile.objects.create(
file=File(compressed_file_child)
)
except OperationalError as exception:
logger.warning(
"Operational error while preparing to upload " "child document: %s. Rescheduling.",
exception,
)
task_source_handle_upload.delay(
document_type_id=document_type_id,
shared_uploaded_file_id=shared_uploaded_file_id,
source_id=source_id,
description=description,
expand=expand,
label=label,
language=language,
metadata_dict_list=metadata_dict_list,
skip_list=skip_list,
user_id=user_id,
)
return
else:
skip_list.append(unicode(compressed_file_child))
task_upload_document.delay(shared_uploaded_file_id=child_shared_uploaded_file.pk, **kwargs)
finally:
compressed_file_child.close()
compressed_file_child.close()
try:
shared_upload.delete()
except OperationalError as exception:
logger.warning(
"Operational error during attempt to delete shared " "upload file: %s; %s. Retrying.",
shared_upload,
exception,
)
except NotACompressedFile:
logging.debug("Exception: NotACompressedFile")
task_upload_document.delay(shared_uploaded_file_id=shared_upload.pk, **kwargs)
else:
task_upload_document.delay(shared_uploaded_file_id=shared_upload.pk, **kwargs)
