def output_path(self): return os.path.join( djangui_settings.DJANGUI_FILE_DIR, get_valid_filename(self.user.username if self.user is not None else ""), get_valid_filename(self.script.slug if not self.script.save_path else self.script.save_path), str(self.pk), )
def _calculate_remote_path(self, job_exe, input_file_ids): '''Returns the remote path for storing the products :param job_exe: The job execution model (with related job and job_type fields) that is storing the files :type job_exe: :class:`job.models.JobExecution` :param input_file_ids: Set of input file IDs :type input_file_ids: set of int :returns: The remote path for storing the products :rtype: str ''' job_type_path = get_valid_filename(job_exe.job.job_type.name) job_version_path = get_valid_filename(job_exe.job.job_type.version) remote_path = os.path.join(job_type_path, job_version_path) # Try to use data start time from earliest ancestor source file the_date = None for source_file in FileAncestryLink.objects.get_source_ancestors(list(input_file_ids)): if source_file.data_started: if not the_date or source_file.data_started < the_date: the_date = source_file.data_started # No data start time populated, use current time if not the_date: remote_path = os.path.join(remote_path, 'unknown_source_data_time') the_date = now() year_dir = str(the_date.year) month_dir = u'%02d' % the_date.month day_dir = u'%02d' % the_date.day return os.path.join(remote_path, year_dir, month_dir, day_dir, 'job_exe_%i' % job_exe.id)
def content_filename(instance, filename): return os.path.join( settings.MEDIA_ROOT, 'auth_app', 'files', get_valid_filename(type(instance).__name__.lower()), get_valid_filename(filename), )
def webcam_upload(request, run_uuid): """ Receives upload requests for video/audio chunks during the experiment and merges these chunks into a file. """ fs = FileSystemStorage(location=settings.WEBCAM_ROOT) # Upload request if request.method == 'POST' and request.FILES.get('file'): webcam_file = request.FILES.get('file') # webcam_file_type = request.POST.get('type') # Delete existing file if fs.exists(webcam_file.name): fs.delete(webcam_file.name) fs.save(get_valid_filename(webcam_file.name), webcam_file) logger.info('Received upload request of %s.' % webcam_file.name) return HttpResponse(status=204) # Merge request elif request.method == 'POST' and request.POST.get('trialResultId'): # Get base filename, by removing chunk number at the end base_filename = request.POST.get('filename') base_filename = get_valid_filename(base_filename) logger.info('Received last file of %s, merge files.' % base_filename) # Find and merge individual chunks webcam_files = find_files(base_filename) merge_files(base_filename + '.webm', webcam_files) # Delete chunks for webcam_file in webcam_files: fs.delete(webcam_file) # Add filename to trial result trial_result_id = 0 try: trial_result_id = int(request.POST.get('trialResultId')) except ValueError as e: logger.exception('Failed to retrieve trial result ID: ' + str(e)) raise Http404('Invalid trialResultId.') trial_result = get_object_or_404(TrialResult, pk=trial_result_id, subject=run_uuid) trial_result.webcam_file = base_filename + '.webm' trial_result.save() logger.info('Successfully saved webcam file to trial result.') return HttpResponse(status=204) else: logger.error('Failed to upload webcam file.') raise Http404('Page not found.')
def output_path(self): return os.path.join( wooey_settings.WOOEY_FILE_DIR, get_valid_filename(self.user.username if self.user is not None else ""), get_valid_filename( self.script_version.script.slug if not self.script_version.script.save_path else self.script_version.script.save_path ), str(self.uuid), )
def sdrfile_name(instance, filename): resourcetype = instance.__class__.__name__.lower() if resourcetype == Feature.__name__.lower(): resourcetype = get_valid_filename( Truncator(instance.featuretype.name).words(4, truncate='')) humanname = get_valid_filename( Truncator(instance.sdr.name_short).words(4, truncate='')) root, ext = os.path.splitext(filename) instancefile = '%s-%s-%s-%s%s' % (instance.sdr_id, humanname, resourcetype, instance.pk, ext) return instancefile
def test_successful_recipe_path(self, mock_upload_files, mock_create_file_ancestry_links): """Tests calling ProductDataFileType.store_files() successfully with a job that is in a recipe""" job_exe_in_recipe = job_utils.create_job_exe(status='RUNNING') recipe = recipe_utils.create_recipe() _recipe_job = recipe_utils.create_recipe_job(recipe=recipe, job_name='My Job', job=job_exe_in_recipe.job) remote_base_path_with_recipe = os.path.join('recipes', get_valid_filename(recipe.recipe_type.name), get_valid_filename(recipe.recipe_type.version), 'jobs', get_valid_filename(job_exe_in_recipe.job.job_type.name), get_valid_filename(job_exe_in_recipe.job.job_type.version)) local_path_1 = os.path.join('my', 'path', 'one', 'my_test.txt') media_type_1 = 'text/plain' local_path_2 = os.path.join('my', 'path', 'one', 'my_test.json') media_type_2 = 'application/json' local_path_3 = os.path.join('my', 'path', 'three', 'my_test.png') media_type_3 = 'image/png' local_path_4 = os.path.join('my', 'path', 'four', 'my_test.xml') media_type_4 = None # Set up mocks def new_upload_files(upload_dir, work_dir, file_entries, input_file_ids, job_exe, workspace): results = [] for file_entry in file_entries: # Check base remote path for recipe type and job type information self.assertTrue(file_entry[1].startswith(remote_base_path_with_recipe)) if file_entry[0] == local_path_1: mock_1 = MagicMock() mock_1.id = 1 results.append(mock_1) elif file_entry[0] == local_path_2: mock_2 = MagicMock() mock_2.id = 2 results.append(mock_2) elif file_entry[0] == local_path_3: mock_3 = MagicMock() mock_3.id = 3 results.append(mock_3) elif file_entry[0] == local_path_4: mock_4 = MagicMock() mock_4.id = 4 results.append(mock_4) return results mock_upload_files.side_effect = new_upload_files data_files = {self.workspace_1.id: [(local_path_1, media_type_1), (local_path_2, media_type_2)], self.workspace_2.id: [(local_path_3, media_type_3), (local_path_4, media_type_4)]} parent_ids = {98, 99} # Dummy values upload_dir = 'upload_dir' ProductDataFileStore().store_files(upload_dir, 'work_dir', data_files, parent_ids, job_exe_in_recipe)
def test_successful_recipe_path(self, mock_upload_files, mock_create_file_ancestry_links): """Tests calling ProductDataFileType.store_files() successfully with a job that is in a recipe""" job_exe_in_recipe = job_utils.create_job_exe(status='RUNNING') recipe = recipe_utils.create_recipe() _recipe_job = recipe_utils.create_recipe_job(recipe=recipe, job_name='My Job', job=job_exe_in_recipe.job) remote_base_path_with_recipe = os.path.join('recipes', get_valid_filename(recipe.recipe_type.name), get_valid_filename(recipe.recipe_type.version), 'jobs', get_valid_filename(job_exe_in_recipe.job.job_type.name), get_valid_filename(job_exe_in_recipe.job.job_type.version)) local_path_1 = os.path.join('my', 'path', 'one', 'my_test.txt') media_type_1 = 'text/plain' local_path_2 = os.path.join('my', 'path', 'one', 'my_test.json') media_type_2 = 'application/json' local_path_3 = os.path.join('my', 'path', 'three', 'my_test.png') media_type_3 = 'image/png' local_path_4 = os.path.join('my', 'path', 'four', 'my_test.xml') media_type_4 = None # Set up mocks def new_upload_files(file_entries, input_file_ids, job_exe, workspace): results = [] for file_entry in file_entries: # Check base remote path for recipe type and job type information self.assertTrue(file_entry[1].startswith(remote_base_path_with_recipe)) if file_entry[0] == local_path_1: mock_1 = MagicMock() mock_1.id = 1 results.append(mock_1) elif file_entry[0] == local_path_2: mock_2 = MagicMock() mock_2.id = 2 results.append(mock_2) elif file_entry[0] == local_path_3: mock_3 = MagicMock() mock_3.id = 3 results.append(mock_3) elif file_entry[0] == local_path_4: mock_4 = MagicMock() mock_4.id = 4 results.append(mock_4) return results mock_upload_files.side_effect = new_upload_files data_files = {self.workspace_1.id: [(local_path_1, media_type_1), (local_path_2, media_type_2)], self.workspace_2.id: [(local_path_3, media_type_3), (local_path_4, media_type_4)]} parent_ids = {98, 99} # Dummy values ProductDataFileStore().store_files(data_files, parent_ids, job_exe_in_recipe)
def make_upload_path(instance, filename): """Return a string like pdf/2010/08/13/foo_v._var.pdf, with the date set as the date_filed for the case.""" # this code NOT cross platform. Use os.path.join or similar to fix. mimetype = filename.split('.')[-1] + '/' try: path = mimetype + instance.date_filed.strftime("%Y/%m/%d/") + \ get_valid_filename(filename) except AttributeError: # The date is unknown for the case. Use today's date. path = mimetype + instance.time_retrieved.strftime("%Y/%m/%d/") + \ get_valid_filename(filename) return path
def _calculate_remote_path(self, job_exe, input_file_ids): """Returns the remote path for storing the products :param job_exe: The job execution model (with related job and job_type fields) that is storing the files :type job_exe: :class:`job.models.JobExecution` :param input_file_ids: Set of input file IDs :type input_file_ids: set of int :returns: The remote path for storing the products :rtype: str """ remote_path = '' job_recipe = Recipe.objects.get_recipe_for_job(job_exe.job_id) if job_recipe: recipe = job_recipe.recipe recipe_type_path = get_valid_filename(recipe.recipe_type.name) recipe_revision = RecipeTypeRevision.objects.get_revision( recipe.recipe_type.name, recipe.recipe_type.revision_num).revision_num recipe_version_path = get_valid_filename( 'revision_%i' % recipe.recipe_type.revision_num) remote_path = os.path.join(remote_path, 'recipes', recipe_type_path, recipe_version_path) job_type_path = get_valid_filename(job_exe.job.job_type.name) job_version_path = get_valid_filename(job_exe.job.job_type.version) remote_path = os.path.join(remote_path, 'jobs', job_type_path, job_version_path) # Try to use source start time from the job the_date = job_exe.job.source_started if not the_date: # Try to grab source started the old way through the source ancestor file for source_file in FileAncestryLink.objects.get_source_ancestors( list(input_file_ids)): if source_file.data_started: if not the_date or source_file.data_started < the_date: the_date = source_file.data_started # No data start time populated, use current time if not the_date: remote_path = os.path.join(remote_path, 'unknown_source_data_time') the_date = now() year_dir = str(the_date.year) month_dir = '%02d' % the_date.month day_dir = '%02d' % the_date.day return os.path.join(remote_path, year_dir, month_dir, day_dir, 'job_exe_%i' % job_exe.id)
def setUp(self): django.setup() self.workspace_1 = Workspace.objects.create(name='Test workspace 1') self.workspace_2 = Workspace.objects.create(name='Test workspace 2', is_active=False) interface = {'version': '1.0', 'command': 'my command', 'command_arguments': 'command arguments'} job_type = job_utils.create_job_type(name='Type 1', version='1.0', interface=interface) event = TriggerEvent.objects.create_trigger_event('TEST', None, {}, now()) self.job = job_utils.create_job(job_type=job_type, event=event, status='RUNNING', last_status_change=now()) self.job_exe = job_utils.create_job_exe(job=self.job, status='RUNNING', timeout=1, queued=now()) self.remote_base_path = os.path.join('jobs', get_valid_filename(self.job.job_type.name), get_valid_filename(self.job.job_type.version))
def setUp(self): django.setup() self.workspace_1 = Workspace.objects.create(name='Test workspace 1') self.workspace_2 = Workspace.objects.create(name='Test workspace 2', is_active=False) interface = {'version': '1.0', 'command': 'my command'} job_type = job_utils.create_job_type(name='Type 1', version='1.0', interface=interface) event = TriggerEvent.objects.create_trigger_event('TEST', None, {}, now()) self.job = job_utils.create_job(job_type=job_type, event=event, status='RUNNING', last_status_change=now()) self.job_exe = job_utils.create_job_exe(job=self.job, status='RUNNING', timeout=1, queued=now()) self.remote_base_path = os.path.join('jobs', get_valid_filename(self.job.job_type.name), get_valid_filename(self.job.job_type.version))
async def upload_file(self, request, user, directory): if request.method != "POST" or request.content_type != "multipart/form-data": AccessError(400, "invalid") reader = await request.multipart() field = await reader.next() if field.name != "file": AccessError(400, "invalid") fname = text.get_valid_filename(os.path.basename(field.filename)) name = os.path.join(directory, fname) fullname = os.path.join(config.get("global", "basedir"), name) if not fname: self.redirect(request, user, directory, "invalid filename") if os.access(fullname, os.R_OK): raise AccessError(403, "duplicate") with open(fullname, 'wb') as f: while True: chunk = await field.read_chunk() # 8192 bytes by default. if not chunk: break f.write(chunk) log(__name__).info("%s - - File %s uploaded by %s", self.peername(request), name, user) size = os.lstat(fullname).st_size self.redirect(request, user, directory, "%d bytes saved as %s" % (size, fname))
def zip_file(self, request, *args, **kwargs): socket = self.get_object() real_file_list = { f_key: request.build_absolute_uri(Socket.get_storage().url(f_val['file'])) for f_key, f_val in socket.file_list.items() if not f_key.startswith('<') } # File list with full urls can get quite big so we pass it through tempfile with tempfile.NamedTemporaryFile(delete=False, suffix='.file_list', mode="w") as list_file: json.dump(real_file_list, list_file) try: propagate_uwsgi_params(get_tracing_attrs()) uwsgi.add_var('OFFLOAD_HANDLER', 'apps.sockets.handlers.SocketZipHandler') uwsgi.add_var('LIST_FILE', list_file.name) uwsgi.add_var( 'FILE_NAME', get_valid_filename('{}_{}'.format(socket.name, socket.version))) except ValueError: os.unlink(list_file.name) raise UwsgiValueError() return HttpResponse()
def set_file_name(self, file_name): path = os.path.join("contents", file_name) # get file size if this content was saved individually if (self.content_file): self.filesize = self.content_file.size self.file_name = get_valid_filename(file_name) return path
def __init__(self, input_image_file, input_image_filename, cache_folder=None): """ :Args: - self (:class:`BkImageEditor`): BkImageEditor instance. """ self._input_image_file = input_image_file self._input_image_filename = get_valid_filename(input_image_filename) self._cache_folder = cache_folder self._image_width = None self._image_height = None self._image_translate_x = None self._image_translate_y = None self._image_flip_x = None self._image_flip_y = None self._image_rotate_degree = 0 self._image_contrast = None self._image_brightness = None self._image_blur = None self._image_saturate = None self._image_opacity = None self._frame_width = None self._frame_height = None
def generate_upload_filename(instance, filename): return os.path.join( settings.JQFILEUPLOAD_UPLOAD_SUBIDRECTORY, f"{instance.file_id}", get_valid_filename( f"{instance.start_byte}-{instance.end_byte}-{filename}"), )
def open_school(name, records): school = find_school(name, records) school = school[0] filename = 'data/schools/%s.html' % get_valid_filename(school['school_name']) with open(filename) as f: s1 = BS(f.read()) return s1
def upload_action(request): if request.method == "POST": for key in request.FILES.keys(): new_photo = Photo(photo=request.FILES[key], owner=request.user) new_photo.save() message = {"status": "success"} return HttpResponse(json.dumps(message), content_type="application/json") elif request.method == "DELETE": try: req_data = json.loads(request.body) if "filename" in req_data: f = get_valid_filename(req_data["filename"]) files = Photo.objects.filter(photo=f) for fileobj in files: if fileobj.display == False: fileobj.delete() message = {"status": "success"} return HttpResponse(json.dumps(message), content_type="application/json") else: return error() except ValueError: return error() return error()
def handle_attachment(message, content, related=False): # r = '' # if related: # r = '(r)' filename, encoding = decode_header(content.get_filename())[0] if encoding: filename = filename.decode(encoding, errors='replace') #if not related: # print "saving attachment [%s] of type %s from message %d %s" % (filename, content.get_content_type(), message.id, r) a = Attachment() a.filename = filename # TODO need to parse weird strings from this if not a.filename: a.filename = str(uuid.uuid4()) a.content_type = content.get_content_type() a.stored_location = os.path.join(files_dir, str(message.id), get_valid_filename(a.filename)) # probably want to fix this too a.mime_related = related # load the file file_content = content.get_payload(decode=1) a.file_md5 = hashlib.md5(file_content).hexdigest() # again, probably a better way to do this than all in memory # actually write it do disk - should wrap this in a try except too if not os.path.exists(os.path.join(files_dir, str(message.id))): os.makedirs(os.path.join(files_dir, str(message.id))) with open(a.stored_location, 'wb') as fp: fp.write(file_content) a.message = message a.save()
def generate_document(self, tempdirectory): return pdflatex(base_filename=get_valid_filename(self.description), template=self.template, context=self.__dict__, tempdirectory=tempdirectory, additional_files=self.additional_files )
def pdflatex(base_filename, template, context, tempdirectory, additional_files=None): base_filename = get_valid_filename(base_filename) tex_file = path.join(tempdirectory, "{}.tex".format(base_filename)) with open(tex_file, 'w') as tex: tex.write(get_template(template).render(Context(context))) if additional_files: for additional_file in additional_files: needed_dir = path.join(tempdirectory, path.dirname(additional_file[1])) if not path.isdir(needed_dir): mkdir(needed_dir) copyfile(path.join(settings.BASE_DIR, additional_file[0]), path.join(tempdirectory, additional_file[1])) call(['pdflatex', '-interaction', 'nonstopmode', tex_file], cwd=tempdirectory) # return path.join(tempdirectory, "{}.tex".format(base_filename)) pdf_file = path.join(tempdirectory, "{}.pdf".format(base_filename)) if path.isfile(pdf_file): return pdf_file log_file = path.join(tempdirectory, "{}.log".format(base_filename)) if path.isfile(log_file): return log_file return None
def parse_rankings(category): # open each school file, extract data from html table # add to record dict and save all info to new csv file list_file = os.path.join('data', category, 'list_%s.csv' % category) records = get_records(list_file) print('PARSING RANKINGS FOR %s' % category) for i, rec in enumerate(records): schoolfile = get_valid_filename(rec['school_name']) filename = 'data/%s/schools/%s.html' % (category, schoolfile) print('processing %s. %i of %i' % (filename, i+1, len(records))) with open(filename) as f: s1 = BS(f.read()) trs = s1.find_all('tr') for row in trs[5:]: tds = row.find_all('td') key = slugify(tds[0].text.strip()).replace('-', '_') value = tds[1].text.strip().split('\n')[0] rec[key] = value with open(list_file, 'w') as outfile: writer = csv.DictWriter(outfile, fieldnames=records[0].keys()) writer.writeheader() for rec in records: writer.writerow(rec)
def download_song_from_url(self, song_pk): Song = apps.get_model('songs.Song') SongTask = apps.get_model('songs.SongTask') song = Song.objects.get(pk=song_pk) task = SongTask.objects.create(task_id=self.request.id, description="Download song from URL", song=song) outdir = tempfile.mkdtemp() opts = dict(**settings.YOUTUBE_DL_OPTS, outtmpl=f"{outdir}/%(title)s-%(id)s.%(ext)s") with YoutubeDL(opts) as ydl: data = ydl.extract_info(song.url) filename = f"{data['title']}-{data['id']}.mp3" output = os.path.join(outdir, filename) with open(output, 'rb') as fd: song.file = File(fd, name=get_valid_filename(filename)) song.name = data['title'] song.save() shutil.rmtree(outdir)
def srt_to_ass_process(video_id, srt_file_dir): """ 将中英字幕合并成的srt字幕文件转换为ass格式字幕文件 :param video_id: :param srt_file_dir: :return: """ video = Video.objects.get(video_id=video_id) ass_filename = '%s-%s.zh-Hans.en.ass' % (get_valid_filename( video.title), video_id) ass_subs_dir = os.path.join(YOUTUBE_DOWNLOAD_DIR, ass_filename) convert_subtilte_format(srt_file_dir, ass_subs_dir) # 如果成功生成srt_file_dir文件,则将字幕文件地址返回 if os.path.isfile(ass_subs_dir): video.subtitle_merge = ass_subs_dir video.save(update_fields=[ 'subtitle_merge', ]) return ass_subs_dir else: return False
def _make_package_filename(instance, filename): if instance.contest: contest_name = instance.contest.id else: contest_name = 'no_contest' return 'package/%s/%s' % (contest_name, get_valid_filename(os.path.basename(filename)))
def get_download_filename(title, preset, extension): """ Return a valid filename to be downloaded as. """ filename = "{} ({}).{}".format(title, preset, extension) valid_filename = get_valid_filename(filename) return valid_filename
def test_success_new(self, mock_mkdir, mock_getsize, mock_execute): """Tests calling SourceFileManager.store_file() successfully with a new source file""" def new_getsize(path): return 100 mock_getsize.side_effect = new_getsize work_dir = 'work' workspace = storage_utils.create_workspace() remote_path = u'my/remote/path/file.txt' local_path = u'my/local/path/file.txt' workspace.cleanup_upload_dir = MagicMock() workspace.upload_files = MagicMock() workspace.setup_upload_dir = MagicMock() workspace.delete_files = MagicMock() wksp_upload_dir = os.path.join(work_dir, 'upload') wksp_work_dir = os.path.join(work_dir, 'work', 'workspaces', get_valid_filename(workspace.name)) src_file = SourceFile.objects.store_file(work_dir, local_path, [], workspace, remote_path) workspace.upload_files.assert_called_once_with(wksp_upload_dir, wksp_work_dir, [('file.txt', remote_path)]) self.assertListEqual(workspace.delete_files.call_args_list, []) self.assertEqual(u'file.txt', src_file.file_name) self.assertEqual(u'3d8e577bddb17db339eae0b3d9bcf180', src_file.uuid) self.assertEqual(remote_path, src_file.file_path) self.assertEqual(u'text/plain', src_file.media_type) self.assertEqual(workspace.id, src_file.workspace_id)
def _save(self, name, content): """ Lifted partially from django/core/files/storage.py """ full_path = self.path(name) directory = os.path.dirname(full_path) if not os.path.exists(directory): os.makedirs(directory) elif not os.path.isdir(directory): raise IOError("%s exists and is not a directory." % directory) # This file has a file path that we can move. if hasattr(content, 'temporary_file_path'): temp_data_location = content.temporary_file_path() else: tmp_prefix = "tmp_%s" % (get_valid_filename(name), ) temp_data_location = tempfile.mktemp(prefix=tmp_prefix, dir=self.location) try: # This is a normal uploadedfile that we can stream. # This fun binary flag incantation makes os.open throw an # OSError if the file already exists before we open it. fd = os.open( temp_data_location, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0)) locks.lock(fd, locks.LOCK_EX) for chunk in content.chunks(): os.write(fd, chunk) locks.unlock(fd) os.close(fd) except Exception, e: if os.path.exists(temp_data_location): os.remove(temp_data_location) raise
def make_recap_path(instance, filename): """Make a path to a good location on the local system for RECAP files. This dumps them all into the same directory, which seems to be OK, at least so far. """ return "recap/%s" % get_valid_filename(filename)
def _backup_handler(packet, encryption_passphrase=None): """Performs backup operation on packet.""" if not os.path.exists(MANIFESTS_FOLDER): os.makedirs(MANIFESTS_FOLDER) manifest_path = os.path.join(MANIFESTS_FOLDER, get_valid_filename(packet.path) + '.json') manifests = { 'apps': [{ 'name': component.app.app_id, 'version': component.app.info.version, 'backup': component.manifest } for component in packet.components] } with open(manifest_path, 'w') as manifest_file: json.dump(manifests, manifest_file) paths = packet.directories + packet.files paths.append(manifest_path) arguments = ['create-archive', '--path', packet.path] if packet.archive_comment: arguments += ['--comment', packet.archive_comment] arguments += ['--paths'] + paths input_data = '' if encryption_passphrase: input_data = json.dumps( {'encryption_passphrase': encryption_passphrase}) actions.superuser_run('backups', arguments, input=input_data.encode())
def test_fails(self, mock_makedirs, mock_getsize): """Tests calling ScaleFileManager.upload_files() when Workspace.upload_files() fails""" def new_getsize(path): return 100 mock_getsize.side_effect = new_getsize upload_dir = os.path.join('upload', 'dir') work_dir = os.path.join('work', 'dir') workspace = storage_test_utils.create_workspace() file_1 = ScaleFile() file_1.media_type = None # Scale should auto-detect text/plain remote_path_1 = 'my/remote/path/file.txt' local_path_1 = 'my/local/path/file.txt' file_2 = ScaleFile() file_2.media_type = 'application/json' remote_path_2 = 'my/remote/path/2/file.json' local_path_2 = 'my/local/path/2/file.json' workspace.upload_files = MagicMock() workspace.upload_files.side_effect = Exception workspace.delete_files = MagicMock() delete_work_dir = os.path.join(work_dir, 'delete', get_valid_filename(workspace.name)) files = [(file_1, local_path_1, remote_path_1), (file_2, local_path_2, remote_path_2)] self.assertRaises(Exception, ScaleFile.objects.upload_files, upload_dir, work_dir, workspace, files)
def create_zipname(user, query): """ Returns a (valid) filename for the zipfile containing the exported data. """ date_created = query.date_created.strftime('%Y.%m.%d-%H.%M.%S') return get_valid_filename('_'.join( [user.username, query.title, date_created]))
def _save(self, name, content): """ Lifted partially from django/core/files/storage.py """ full_path = self.path(name) directory = os.path.dirname(full_path) if not os.path.exists(directory): os.makedirs(directory) elif not os.path.isdir(directory): raise IOError("%s exists and is not a directory." % directory) # This file has a file path that we can move. if hasattr(content, 'temporary_file_path'): temp_data_location = content.temporary_file_path() else: tmp_prefix = "tmp_%s" %(get_valid_filename(name), ) temp_data_location = tempfile.mktemp(prefix=tmp_prefix, dir=self.location) try: # This is a normal uploadedfile that we can stream. # This fun binary flag incantation makes os.open throw an # OSError if the file already exists before we open it. fd = os.open(temp_data_location, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0)) locks.lock(fd, locks.LOCK_EX) for chunk in content.chunks(): os.write(fd, chunk) locks.unlock(fd) os.close(fd) except Exception, e: if os.path.exists(temp_data_location): os.remove(temp_data_location) raise
def save(self, request, obj, *args, **kwargs): self.instance.creator = request.user self.instance.content_type = ContentType.objects.get_for_model(obj) self.instance.object_id = obj.pk self.instance.name = get_valid_filename( self.instance.attachment_file.name) super(AttachmentForm, self).save(*args, **kwargs)
def do(self, options): self.directory = options.get("directory") self.mode = options.get("mode") if self.mode == "export" and not os.path.isdir(self.directory): os.mkdir(self.directory) self.logger.info('Directory created: %s' % self.directory) elif self.mode == "import" and os.path.isdir(self.directory): files = { os.path.splitext(get_valid_filename(f))[0].upper(): f for f in os.listdir(self.directory) if os.path.isfile(os.path.join(self.directory, f)) } self.logger.info('Directory found: %s' % self.directory) else: raise CommandError('Please check mode (%s) or directory (%s)' % (self.mode, self.directory)) self.totalrows = self.model.objects.all().count() self.inf = 0 for obj in self.model.objects.all(): self.current_row += 1 if self.pbar: self.progressBar(self.current_row, self.totalrows) else: self.logger.info( 'Update object %s/%s "%s"' % (self.current_row, self.totalrows, obj.display)) if self.mode == "export" and obj.valid_imagename not in self.excludes: self.export_image(obj) else: self.import_image(obj, files) print(self.inf)
def download_latest_forecasts(request, pk): """ :return: `latest_forecast_cols_for_project()` output as CSV. for now just does returns a list of the 2-tuples: (Forecast.id, Forecast.source), but later may generalize to allow passing specific columns in `request` """ project = get_object_or_404(Project, pk=pk) if (not request.user.is_authenticated) or not is_user_ok_view_project(request.user, project): return HttpResponseForbidden() response = HttpResponse(content_type='text/csv') csv_filename = get_valid_filename(f"project-{project.name}-latest-forecasts.csv") response['Content-Disposition'] = 'attachment; filename="{}"'.format(str(csv_filename)) # for now just does returns a list of the 2-tuples: (Forecast.id, Forecast.source) rows = latest_forecast_cols_for_project(project, is_incl_fm_id=False, is_incl_tz_id=False, is_incl_issued_at=False, is_incl_created_at=False, is_incl_source=True, is_incl_notes=False) writer = csv.writer(response) writer.writerow(['forecast_id', 'source']) # header # process rows, cleaning up for csv: # - [maybe later] render date and datetime objects as strings: 'issued_at', 'created_at' # - remove \n from free form text: 'source', [maybe later] 'notes' for f_id, source in rows: writer.writerow([f_id, source.replace('\n', '_')]) return response
def _download_job_data_request(job): """ :param job: a Job :return: the data file corresponding to `job` as a CSV file """ # imported here so that tests can patch via mock: from utils.cloud_file import download_file, _file_name_for_object with tempfile.TemporaryFile() as cloud_file_fp: # <class '_io.BufferedRandom'> try: download_file(job, cloud_file_fp) cloud_file_fp.seek(0) # yes you have to do this! # https://stackoverflow.com/questions/16538210/downloading-files-from-amazon-s3-using-django csv_filename = get_valid_filename(f'job-{_file_name_for_object(job)}-data.csv') wrapper = FileWrapper(cloud_file_fp) response = HttpResponse(wrapper, content_type='text/csv') # response['Content-Length'] = os.path.getsize('/tmp/'+fname) response['Content-Disposition'] = 'attachment; filename="{}"'.format(str(csv_filename)) return response except (BotoCoreError, Boto3Error, ClientError, ConnectionClosedError) as aws_exc: logger.debug(f"download_job_data(): AWS error: {aws_exc!r}. job={job}") return HttpResponseNotFound(f"AWS error: {aws_exc!r}, job={job}") except Exception as ex: logger.debug(f"download_job_data(): error: {ex!r}. job={job}") return HttpResponseNotFound(f"error downloading job data. ex={ex!r}, job={job}")
def newFolder(parent): name = get_valid_filename( simpledialog.askstring("Input", "Please enter the file name:", parent=parent)) if not os.path.exists(name): os.makedirs(name)
def import_image(self, obj, files): found = 0 sfile = get_valid_filename(obj) if sfile in files: ffile = files[sfile] elif sfile.split("_")[0] in files: ffile = files[sfile.split("_")[0]] elif "".join(sfile.split("_")) in files: ffile = files["".join(sfile.split("_"))] else: found = 1 splitfile = re.split(r"[, \-!?:_]+", sfile) for sf in splitfile: if len(sf) > 3: for f in files: if sf in f or self.similar_text(sf, f) > 80: ffile = files[f] found = 0 break if found: self.error.add('Logo', 'not found for: %s' % sfile) self.inf += found else: tfile = File(open("%s%s" % (self.directory, ffile), 'rb')) obj.image.save(ffile, tfile)
def add_subtitle_to_video_process(video_id, mode,sub_lang_type='zh-Hans'): """ 将video_id对应的视频的vtt字幕转为ass格式,然后硬入到对应的视频中 :param video_id: :param subtitle_type: (en,zh-Hans,zh-Hans_en) :param mode:指定使用soft还是使用hard的模式将字幕写入视频文件 :return: """ video = Video.objects.get(pk=video_id) # 如果要求写入的中文字幕,而且中文字幕vtt存在 # 则将中文vtt字幕先转为srt,再转为ass,添加式样 if sub_lang_type == 'zh-Hans' and video.subtitle_cn.name: subtitle_file = video.subtitle_cn.path ass_filename = '%s-%s.zh-Hans.ass' % ( get_valid_filename(video.title), video_id) ass_subs_dir = os.path.join(YOUTUBE_DOWNLOAD_DIR, ass_filename) # 则将中文vtt字幕先转为srt,再转为ass,添加式样 subtitle_file = convert_subtilte_format(subtitle_file, ass_subs_dir) subtitle_file = edit_cn_ass_subtitle_style(subtitle_file) # youtube上的 英文vtt字幕包含格式,导致转换成srt字幕再和中文srt字幕合并后有代码 # 暂时不知道该如何处理,所以只合并中文字幕到视频 # elif sub_lang_type == 'en' and video.subtitle_en.name: # subtitle_file = video.subtitle_en.path elif sub_lang_type == 'zh-Hans_en' and video.subtitle_merge.name: # 如果要求写入的中文和英文的合并字幕,而且合并字幕存在 subtitle_file = video.subtitle_merge.path else: # 如果获取不到subtitle_file,则返回False return False if (video.file.name): # 获取到视频文件名称 file_basename = os.path.basename(video.file.path) else: return False # 将文件名称分割为名称和后缀 file_basename_list = os.path.splitext(file_basename) subtitle_video = file_basename_list[0] + '.' + sub_lang_type + \ file_basename_list[1] # 加入字幕的视频文件保存到YOUTUBE_DOWNLOAD_DIR 目录下 subtitle_video = os.path.join(YOUTUBE_DOWNLOAD_DIR, subtitle_video) result = add_subtitle_to_video(video.file.path, subtitle_file, subtitle_video, mode) if result == True and os.path.exists(subtitle_video): # 如何将字幕合并到视频成功,则保存视频文件地址到Video module中 video.subtitle_video_file = subtitle_video video.save(update_fields=['subtitle_video_file']) return True else: print(result) return False
def make_problem_filename(instance, filename): if not isinstance(instance, Problem): assert hasattr(instance, 'problem'), 'problem_file_generator used ' \ 'on object %r which does not have \'problem\' attribute' \ % (instance,) instance = getattr(instance, 'problem') return 'problems/%d/%s' % (instance.id, get_valid_filename(os.path.basename(filename)))
def make_contest_filename(instance, filename): if not isinstance(instance, Contest): assert hasattr(instance, 'contest'), 'contest_file_generator used ' \ 'on object %r which does not have \'contest\' attribute' \ % (instance,) instance = getattr(instance, 'contest') return 'contests/%s/%s' % (instance.id, get_valid_filename(os.path.basename(filename)))
def get_download_filename(self): """ Return a valid filename to be downloaded as. """ title = self.contentnode.title filename = "{} ({}).{}".format(title, self.get_preset(), self.extension) valid_filename = get_valid_filename(filename) return valid_filename
def _get_unique_filename(name, db_alias=DEFAULT_CONNECTION_NAME, collection_name="fs"): fs = GridFS(get_db(db_alias), collection_name) file_root, file_ext = os.path.splitext(get_valid_filename(name)) count = itertools.count(1) while fs.exists(filename=name): # file_ext includes the dot. name = os.path.join("%s_%s%s" % (file_root, next(count), file_ext)) return name
def export(test=True): data = gather_data_and_update_flags(test) if data == []: raise Exception(u"%s" % _(u"No data to export!")) elif not data: logger.error(u"%s" % _(u"Error when fetching data - Data empty.")) raise Exception(u"%s" % _(u"Error when fetching data" u" Export cancelled!")) # File name if test: filename = get_valid_filename("test_%s.csv" % date.today()) else: filename = get_valid_filename("%s.csv" % date.today()) mediafilepath = join("invoices/export/", filename) filepath = join(settings.MEDIA_ROOT, mediafilepath) # Ensure that we got an available file name fss = FileSystemStorage() filepath = fss.get_available_name(filepath) # If the file name change, we update the media file path filename = split(filepath)[1] mediafilepath = join("invoices/export/", filename) # Write the file on the FS, and create the Export object if we are not in # test mode with open(filepath, "w") as csvfile: exportwriter = csv.writer(csvfile, delimiter=';', quoting=csv.QUOTE_ALL) # We do this because CSV doesn't support directly Unicode and UTF-8 # http://docs.python.org/2/library/csv.html#examples for row in data: r = [] for field in row: r.append(("%s" % field).strip().encode("utf-8")) exportwriter.writerow(r) if not test: export = Export(date=datetime.today(), file=mediafilepath) export.save() return settings.MEDIA_URL + mediafilepath
def _save(self, name, content): """ Lifted partially from django/core/files/storage.py """ full_path = self.path(name) # Create any intermediate directories that do not exist. # Note that there is a race between os.path.exists and os.makedirs: # if os.makedirs fails with EEXIST, the directory was created # concurrently, and we can continue normally. Refs #16082. directory = os.path.dirname(full_path) if not os.path.exists(directory): try: os.makedirs(directory) except OSError as e: if e.errno != errno.EEXIST: raise if not os.path.isdir(directory): raise IOError("%s exists and is not a directory." % directory) # There's a potential race condition between get_available_name and # saving the file; it's possible that two threads might return the # same name, at which point all sorts of fun happens. So we need to # try to create the file, but if it already exists we have to go back # to get_available_name() and try again. # This file has a file path that we can move. if hasattr(content, 'temporary_file_path'): temp_data_location = content.temporary_file_path() else: tmp_prefix = "tmp_%s" % (get_valid_filename(name), ) temp_data_location = tempfile.mktemp(prefix=tmp_prefix, dir=self.location) try: # This is a normal uploadedfile that we can stream. # This fun binary flag incantation makes os.open throw an # OSError if the file already exists before we open it. fd = os.open(temp_data_location, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0)) locks.lock(fd, locks.LOCK_EX) for chunk in content.chunks(): os.write(fd, chunk) locks.unlock(fd) os.close(fd) except Exception: if os.path.exists(temp_data_location): os.remove(temp_data_location) raise file_move_safe(temp_data_location, full_path, allow_overwrite=True) content.close() if settings.FILE_UPLOAD_PERMISSIONS is not None: os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS) return name
def _get_delete_work_dir(self, work_dir, workspace): """Returns a work sub-directory used to delete files from the given workspace :param work_dir: Absolute path to a local work directory available to Scale :type work_dir: str :param workspace: The workspace :type workspace: :class:`storage.models.Workspace` """ return os.path.join(self._get_delete_root_dir(work_dir), get_valid_filename(workspace.name))
def _get_workspace_work_dir(self, work_dir, workspace): '''Returns a work sub-directory for the given workspace :param work_dir: Absolute path to a local work directory available to Scale :type work_dir: str :param workspace: The workspace :type workspace: :class:`storage.models.Workspace` ''' return os.path.join(os.path.normpath(work_dir), 'workspaces', get_valid_filename(workspace.name))
def clean_projectname(self): dirname = "%05d_%s"%(self.user.pk, get_valid_filename(self.cleaned_data['projectname'])) logger.debug(dirname) logger.debug(self.user) try: prj = Project.objects.get(owner = self.user, directory = dirname) logger.debug('exists') raise forms.ValidationError("project already exists") except Project.DoesNotExist: return self.cleaned_data['projectname']
def _upload_to(filename): upload_path = getattr(settings, 'MULTIUPLOADER_FILES_FOLDER', DEFAULTS.MULTIUPLOADER_FILES_FOLDER) if upload_path[-1] != '/': upload_path += '/' filename = get_valid_filename(os.path.basename(filename)) filename, ext = os.path.splitext(filename) hash = sha1(str(time.time())).hexdigest() fullname = os.path.join(upload_path, "%s.%s%s" % (filename, hash, ext)) return fullname
def _upload_to(instance, filename): upload_path = getattr(settings, 'BUCKET_FILES_FOLDER') if upload_path[-1] != '/': upload_path += '/' filename = get_valid_filename(os.path.basename(filename)) filename, ext = os.path.splitext(filename) hash = sha1(str(time.time())).hexdigest() fullname = os.path.join(upload_path, "%s%s" % (hash, ext)) return fullname
def get_attachment_file_path(instance, filename): basename = path.basename(filename) basename = get_valid_filename(basename) hs = hashlib.sha256() hs.update(force_bytes(timezone.now().isoformat())) hs.update(os.urandom(1024)) p1, p2, p3, p4, *p5 = split_by_n(hs.hexdigest(), 1) hash_part = path.join(p1, p2, p3, p4, "".join(p5)) return path.join("attachments", hash_part, basename)
def cumulus_upload_to(self, filename): """ Simple, custom upload_to because Cloud Files doesn't support nested containers (directories). Actually found this out from @minter: @richleland The Cloud Files APIs do support pseudo-subdirectories, by creating zero-byte files with type application/directory. May implement in a future version. """ return get_valid_filename(filename)