def upload_archive(request):
try:
import tempfile
file_name = unquote(request.META['HTTP_X_FILE_NAME'])
output_file_path = os.path.join('/tmp/', new_id())
if not isinstance(output_file_path, unicode):
file_name = unicode(output_file_path, 'utf-8')
_write_file(request, output_file_path)
if file_name.endswith('.zip'):
import zipfile
archive = zipfile.ZipFile(output_file_path, 'r')
elif file_name.endswith('.tar') or file_name.endswith('.gz'):
import tarfile
archive = tarfile.TarFile(output_file_path, 'r')
extracting_dir = tempfile.mkdtemp()
logger.debug('extracting into %s...'%extracting_dir)
archive.extractall(extracting_dir)
logger.debug('extracted')
import_dir(extracting_dir, request.user, request.session['workspace'], make_copy = True, recursive = True, force_generation = False, link = False, remove_orphans=False)
shutil.rmtree(extracting_dir, True)
return HttpResponse(simplejson.dumps({'success': True}))
except Exception, ex:
logger.exception(ex)
raise ex
def _upload_item(file_name, file_raw, variant, user, tmp_dir, workspace, session_finished = False):
logger.debug('_upload_item %s in %s'%(file_name, tmp_dir))
file_name = new_id() + '_' + file_name
file_name = os.path.join(tmp_dir, file_name)
if not isinstance(file_name , unicode):
file_name = unicode(file_name , 'utf-8')
_write_file(file_raw, file_name)
def _upload_item(file_name, file_raw, variant, user, tmp_dir, workspace, session_finished = False):
logger.debug('_upload_item %s in %s'%(file_name, tmp_dir))
file_name = new_id() + '_' + file_name
file_name = os.path.join(tmp_dir, file_name)
if not isinstance(file_name , unicode):
file_name = unicode(file_name , 'utf-8')
_write_file(file_raw, file_name)
def upload_archive(request):
try:
import tempfile
file_name = unquote(request.META['HTTP_X_FILE_NAME'])
output_file_path = os.path.join('/tmp/', new_id())
if not isinstance(output_file_path, unicode):
file_name = unicode(output_file_path, 'utf-8')
_write_file(request, output_file_path)
if file_name.endswith('.zip'):
import zipfile
archive = zipfile.ZipFile(output_file_path, 'r')
elif file_name.endswith('.tar') or file_name.endswith('.gz'):
import tarfile
archive = tarfile.TarFile(output_file_path, 'r')
extracting_dir = tempfile.mkdtemp()
logger.debug('extracting into %s...'%extracting_dir)
archive.extractall(extracting_dir)
logger.debug('extracted')
import_dir(extracting_dir, request.user, request.session['workspace'], make_copy = True, recursive = True, force_generation = False, link = False, remove_orphans=False)
shutil.rmtree(extracting_dir, True)
# Actually launch all processes added by import_dir() (after
# committing the transaction, to make new data visible by the
# MProcessor)
transaction.commit()
processor.run()
return HttpResponse(simplejson.dumps({'success': True}))
except Exception, ex:
transaction.rollback()
logger.exception(ex)
raise ex
def _create_items(dir_name, variant_name, user, workspace, make_copy=True, recursive = True, force_generation = False, link = False):
"""
"""
logger.debug('########## _create_items')
def copy_file(original_filename, final_path, link, make_copy):
logger.debug('original_filename %s'%original_filename)
logger.debug('final_path %s'%final_path)
if link:
logger.debug('link')
#os.symlink(original_filename, final_path)
if os.path.exists(final_path):
os.remove(final_path)
os.link(original_filename, final_path)
elif make_copy:
shutil.copyfile(original_filename, final_path)
else:
shutil.move(original_filename, final_path)
variant = Variant.objects.get(name = variant_name)
for entry in os.walk(dir_name):
root_dir, sub_dirs, files = entry
files = [os.path.join(root_dir, x) for x in files]
for original_filename in files:
res_id = new_id()
try:
media_type = Type.objects.get_or_create_by_filename(original_filename)
except MimeError, ex:
logger.error(ex)
continue
final_filename = get_storage_file_name(res_id, workspace.pk, variant.name, media_type.ext)
final_path = os.path.join(settings.MEDIADART_STORAGE, final_filename)
upload_filename = os.path.basename(original_filename)
tmp = upload_filename.split('_')
item, created = _create_item(user, workspace, res_id, media_type, original_filename)
logger.debug('created %s'%created)
logger.debug('+++++++++++++ force_generation %s'%force_generation)
if created:
logger.debug('file created')
if len(tmp) > 1:
upload_filename = '_'.join(tmp[1:])
_create_variant(upload_filename, final_filename, media_type, item, workspace, variant)
logger.debug('yielding item %s'%item)
copy_file(original_filename, final_path, link, make_copy)
yield item
elif force_generation:
logger.debug('force_generation')
component = variant.get_component(workspace, item)
dam_hash = md5(component.get_file_path())
dir_hash = md5(original_filename)
#if force_generation or dir_hash != dam_hash:
logger.debug('component.get_file_path() %s'%component.get_file_path())
logger.debug('original_filename %s'%original_filename)
logger.debug('dir_hash != dam_hash %s'%(dir_hash != dam_hash))
if dir_hash != dam_hash:
copy_file(original_filename, component.get_file_path(), link, make_copy)
yield item
if not recursive:
break
def _get_filepath(file_name):
fname, ext = os.path.splitext(file_name)
res_id = new_id() + ext
fpath = os.path.join(settings.MEDIADART_STORAGE, res_id)
return fpath, res_id
def _create_items(dir_name, variant_name, user, workspace, make_copy=True, recursive = True, force_generation = False, link = False):
"""
"""
logger.debug('########## _create_items')
def copy_file(original_filename, final_path, link, make_copy):
logger.debug('original_filename %s'%original_filename)
logger.debug('final_path %s'%final_path)
if link:
logger.debug('link')
#os.symlink(original_filename, final_path)
if os.path.exists(final_path):
os.remove(final_path)
os.link(original_filename, final_path)
elif make_copy:
shutil.copyfile(original_filename, final_path)
else:
shutil.move(original_filename, final_path)
variant = Variant.objects.get(name = variant_name)
for entry in os.walk(dir_name):
root_dir, sub_dirs, files = entry
files = [os.path.join(root_dir, x) for x in files]
for original_filename in files:
res_id = new_id()
try:
media_type = Type.objects.get_or_create_by_filename(original_filename)
except MimeError, ex:
logger.error(ex)
continue
final_filename = get_storage_file_name(res_id, workspace.pk, variant.name, media_type.ext)
final_path = os.path.join(settings.MPROCESSOR_STORAGE, final_filename)
upload_filename = os.path.basename(original_filename)
tmp = upload_filename.split('_')
item, created = _create_item(user, workspace, res_id, media_type, original_filename)
logger.debug('created %s'%created)
logger.debug('+++++++++++++ force_generation %s'%force_generation)
if created:
logger.debug('file created')
if len(tmp) > 1:
upload_filename = '_'.join(tmp[1:])
_create_variant(upload_filename, final_filename, media_type, item, workspace, variant)
logger.debug('yielding item %s'%item)
copy_file(original_filename, final_path, link, make_copy)
yield item
elif force_generation:
logger.debug('force_generation')
component = variant.get_component(workspace, item)
dam_hash = md5(component.get_file_path())
dir_hash = md5(original_filename)
#if force_generation or dir_hash != dam_hash:
logger.debug('component.get_file_path() %s'%component.get_file_path())
logger.debug('original_filename %s'%original_filename)
logger.debug('dir_hash != dam_hash %s'%(dir_hash != dam_hash))
if dir_hash != dam_hash:
copy_file(original_filename, component.get_file_path(), link, make_copy)
yield item
if not recursive:
break
def _get_filepath(file_name):
fname, ext = os.path.splitext(file_name)
res_id = new_id() + ext
fpath = os.path.join(settings.MPROCESSOR_STORAGE, res_id)
return fpath, res_id
