def dumpfiles(self, collection): mime = MimeTypes() db = MongoClient(host=self.host, port=self.port)[self.db] uploadsCollection = db[collection] fs = gridfs.GridFSBucket(db, bucket_name=collection) uploads = uploadsCollection.find({}, no_cursor_timeout=True) for upload in uploads: if upload["store"] == "GridFS:Uploads": if "complete" in upload and upload["complete"] is True: path = upload["path"] pathSegments = path.split("/") gridfsId = pathSegments[3] for res in fs.find({"_id": gridfsId}): data = res.read() fileext = "" if "extension" in upload: fileext = upload["extension"] else: fileext = mime.guess_extension(res.content_type) if fileext is not None and fileext != "": filename = gridfsId + "." + fileext else: filename = gridfsId file = open(self.outDir + "/" + filename, "wb") file.write(data) file.close() self.addtolog(gridfsId, filename, collection, res.md5) else: print(upload) self.writelog()
def dumpfiles(self, collection, store): mime = MimeTypes() db = self.getdb() uploadsCollection = db[collection] fs = gridfs.GridFSBucket(db, bucket_name=collection) uploads = uploadsCollection.find({}, no_cursor_timeout=True) i = 0 for upload in uploads: if upload["store"] == "GridFS:Uploads": gridfsId = upload['_id'] if "complete" in upload and upload["complete"] is True: for res in fs.find({"_id": gridfsId}): data = res.read() filename = gridfsId fileext = "" if "extension" in upload and upload["extension"] != "": fileext = "." + upload["extension"] else: fileext = mime.guess_extension(res.content_type) if fileext is not None and fileext != "": filename = filename + fileext i += 1 print("%i. Dumping %s %s" % (i, gridfsId, upload['name'])) key = store.put(filename, data, upload) print("%i. Finished dumping %s %s" % (i, gridfsId, upload['name'])) logitem = { "id": gridfsId, "file": filename, "collection": collection, "md5": res.md5, "key": key } self.updateDbEntry(logitem, i, db) self.removeBlobsEntry(logitem, i, db) self.addtolog(logitem) self.writelog() self.log.pop() else: print("[Warning] Skipping incomplete upload %s" % (gridfsId), file=sys.stderr)
def _getExtension( episode ): # get extension from url urlExt = os.path.splitext( episode.url )[ 1 ] # get extension from media type mime = MimeTypes() mimeExtList = mime.guess_all_extensions( episode.type ) # check if url extension matches any item in list if urlExt in mimeExtList: # yep, then url extension is good return urlExt # otherwise, use recommendation from MimeTypes, as default we use .mp3 mimeExt = mime.guess_extension( episode.type ) if mimeExt == None: mimeExt = ".mp3" return mimeExt
def upload_new_song(user, song_id, file_path, signature=None): # Get song info print("Upload new song \nSong path: ", file_path) song = Song.objects.get(pk=song_id) name = song.name author = song.author price = song.price extension = song.extension = open(file_path).name.rsplit('.', 1)[1] print("Extension: ", extension) mime_type = MimeTypes() content_type = mime_type.guess_extension(file_path) print("Mime type: ", content_type) if not user.is_superuser: # if normal user, upload to their own directory if user.profile.drive_folder_id: folder_id = user.profile.drive_folder_id else: folder_id = drive_api.createFolder(user.username) user.profile.drive_folder_id = folder_id user.profile.save() else: # if superuser upload to shiro store directory folder_id = drive_api.shiro_store_folder_id output_filename = name + " - " + author + "." + extension file_id = drive_api.uploadFile(output_filename, file_path, content_type, folder_id=folder_id) # Build new song with info new_song = Song(id=file_id, name=name, author=author, extension=extension, price=price) if signature: new_song.signature = signature if not user.is_superuser: new_song.owner = user new_song.save() user.profile.songs.add( new_song) # Update Archived Song to Profile user.profile.save() else: new_song.save() return file_id
def dumpfiles(self, collection): mime = MimeTypes() db = MongoClient().rocketchat fs = gridfs.GridFSBucket(db, collection) for grid_out in fs.find({}, no_cursor_timeout=True): data = grid_out.read() pprint(grid_out.filename) pprint(grid_out._id) fileext = mime.guess_extension(grid_out.content_type) filename = grid_out.filename filename = '.'.join(filename.split('.')[:1]) if fileext: filename = filename+fileext print(filename) file = open(self.outDir+"/"+filename, "wb") file.write(data) file.close() self.addtolog(grid_out._id, filename, collection, grid_out.md5) self.writelog()
def upload_file(request): if request.method == 'POST': form = UploadFileForm(request.POST, request.FILES) if form.is_valid(): mime = MimeTypes() uploaded_file = request.FILES['file'] mime_type = mime.guess_type(uploaded_file.name) # Get mime type string from tuple if mime_type[0]: mime_type = mime_type[0] else: return HttpResponseBadRequest(json.dumps({'status':'failed', 'reason':'Invalid file.'}), 'application/json') if validate_upload_file(uploaded_file, mime_type) and uploaded_file.size <= settings.MAX_ARCHIVE_FILE_SIZE: link = Link(submitted_url=form.cleaned_data['url'], submitted_title=form.cleaned_data['title'], created_by = request.user) link.save() asset = Asset(link=link) file_name = 'cap' + mime.guess_extension(mime_type) file_path = os.path.join(asset.base_storage_path, file_name) uploaded_file.file.seek(0) file_name = default_storage.store_file(uploaded_file, file_path) if mime_type == 'application/pdf': asset.pdf_capture = file_name else: asset.image_capture = file_name asset.save() response_object = {'status':'success', 'linky_id':link.guid, 'linky_hash':link.guid} return HttpResponse(json.dumps(response_object), 'application/json', 201) # '201 Created' status else: return HttpResponseBadRequest(json.dumps({'status':'failed', 'reason':'Invalid file.'}), 'application/json') else: return HttpResponseBadRequest(json.dumps({'status':'failed', 'reason':'Missing file.'}), 'application/json') return HttpResponseBadRequest(json.dumps({'status':'failed', 'reason':'No file submitted.'}), 'application/json')
def upload_file(request): if request.method == 'POST': form = UploadFileForm(request.POST, request.FILES) if form.is_valid(): mime = MimeTypes() uploaded_file = request.FILES['file'] mime_type = mime.guess_type(uploaded_file.name) # Get mime type string from tuple if mime_type[0]: mime_type = mime_type[0] else: return HttpResponseBadRequest( json.dumps({ 'status': 'failed', 'reason': 'Invalid file.' }), 'application/json') if validate_upload_file( uploaded_file, mime_type ) and uploaded_file.size <= settings.MAX_ARCHIVE_FILE_SIZE: link = Link(submitted_url=form.cleaned_data['url'], submitted_title=form.cleaned_data['title'], created_by=request.user) link.save() asset = Asset(link=link) file_name = 'cap' + mime.guess_extension(mime_type) file_path = os.path.join(asset.base_storage_path, file_name) uploaded_file.file.seek(0) file_name = default_storage.store_file(uploaded_file, file_path) if mime_type == 'application/pdf': asset.pdf_capture = file_name else: asset.image_capture = file_name asset.save() response_object = { 'status': 'success', 'linky_id': link.guid, 'linky_hash': link.guid } return HttpResponse(json.dumps(response_object), 'application/json', 201) # '201 Created' status else: return HttpResponseBadRequest( json.dumps({ 'status': 'failed', 'reason': 'Invalid file.' }), 'application/json') else: return HttpResponseBadRequest( json.dumps({ 'status': 'failed', 'reason': 'Missing file.' }), 'application/json') return HttpResponseBadRequest( json.dumps({ 'status': 'failed', 'reason': 'No file submitted.' }), 'application/json')
def forge_email(fromaddr, toaddr, subject, content, html_content='', html_images=None, usebcc=False, header=None, footer=None, html_header=None, html_footer=None, ln=None, charset=None, replytoaddr="", attachments=None, bccaddr=""): """Prepare email. Add header and footer if needed. @param fromaddr: [string] sender @param toaddr: [string or list-of-strings] list of receivers (if string, then receivers are separated by ',') @param usebcc: [bool] True for using Bcc in place of To @param subject: [string] subject of the email @param content: [string] content of the email @param html_content: [string] html version of the email @param html_images: [dict] dictionary of image id, image path @param header: [string] None for the default header @param footer: [string] None for the default footer @param ln: language @charset: [string] the content charset. By default is None which means to try to encode the email as ascii, then latin1 then utf-8. @param replytoaddr: [string or list-of-strings] to be used for the reply-to header of the email (if string, then receivers are separated by ',') @param attachments: list of paths of files to be attached. Alternatively, every element of the list could be a tuple: (filename, mimetype) @param bccaddr: [string or list-of-strings] to be used for BCC header of the email (if string, then receivers are separated by ',') @return: forged email as an EmailMessage object""" from invenio_ext.template import render_template_to_string ln = default_ln(ln) if html_images is None: html_images = {} content = render_template_to_string('mail_text.tpl', content=unicodifier(content), header=unicodifier(header), footer=unicodifier(footer) ).encode('utf8') if isinstance(toaddr, list): toaddr = ','.join(toaddr) if isinstance(bccaddr, list): bccaddr = ','.join(bccaddr) if isinstance(replytoaddr, list): replytoaddr = ','.join(replytoaddr) toaddr = remove_temporary_emails(toaddr) headers = {} kwargs = {'to': [], 'cc': [], 'bcc': []} if replytoaddr: headers['Reply-To'] = replytoaddr if usebcc: headers['Bcc'] = bccaddr kwargs['bcc'] = toaddr.split(',') + bccaddr.split(',') kwargs['to'] = ['Undisclosed.Recipients:'] else: kwargs['to'] = toaddr.split(',') headers['From'] = fromaddr headers['Date'] = formatdate(localtime=True) headers['User-Agent'] = 'Invenio %s at %s' % (cfg['CFG_VERSION'], cfg['CFG_SITE_URL']) if html_content: html_content = render_template_to_string( 'mail_html.tpl', content=unicodifier(html_content), header=unicodifier(html_header), footer=unicodifier(html_footer) ).encode('utf8') msg_root = EmailMultiAlternatives(subject=subject, body=content, from_email=fromaddr, headers=headers, **kwargs) msg_root.attach_alternative(html_content, "text/html") # if not html_images: # # No image? Attach the HTML to the root # msg_root.attach(msg_text) # else: if html_images: # Image(s)? Attach the HTML and image(s) as children of a # "related" block msg_related = MIMEMultipart('related') # msg_related.attach(msg_text) for image_id, image_path in iteritems(html_images): attach_embed_image(msg_related, image_id, image_path) msg_root.attach(msg_related) else: msg_root = EmailMessage(subject=subject, body=content, from_email=fromaddr, headers=headers, **kwargs) if attachments: _mimes = MimeTypes(strict=False) for attachment in attachments: try: mime = None if type(attachment) in (list, tuple): attachment, mime = attachment if mime is None: # Automatic guessing of mimetype mime = _mimes.guess_type(attachment)[0] if mime is None: ext = _mimes.guess_extension(getContentType(attachment)) mime = _mimes.guess_type("foo" + ext)[0] if not mime: mime = 'application/octet-stream' part = MIMEBase(*mime.split('/', 1)) part.set_payload(open(attachment, 'rb').read()) Encoders.encode_base64(part) part.add_header( 'Content-Disposition', 'attachment; filename="%s"' % os.path.basename(attachment)) msg_root.attach(part) except: from invenio_ext.logging import register_exception register_exception( alert_admin=True, prefix="Can't attach %s" % attachment) return msg_root
def upload_file(request): if request.method == 'POST': form = UploadFileForm(request.POST, request.FILES) if form.is_valid(): mime = MimeTypes() mime_type = mime.guess_type(request.FILES['file'].name) # Get mime type string from tuple if mime_type[0]: mime_type = mime_type[0] else: return HttpResponseBadRequest(json.dumps({'status':'failed', 'reason':'Invalid file.'}), 'application/json') if validate_upload_file(request.FILES['file'], mime_type) and request.FILES['file'].size <= settings.MAX_ARCHIVE_FILE_SIZE: link = Link(submitted_url=form.cleaned_data['url'], submitted_title=form.cleaned_data['title']) if request.user.is_authenticated(): link.created_by = request.user link.save() now = dt = datetime.now() time_tuple = now.timetuple() path_elements = [str(time_tuple.tm_year), str(time_tuple.tm_mon), str(time_tuple.tm_mday), str(time_tuple.tm_hour), str(time_tuple.tm_min), link.guid] linky_home_disk_path = settings.GENERATED_ASSETS_STORAGE + '/' + os.path.sep.join(path_elements) if not os.path.exists(linky_home_disk_path): os.makedirs(linky_home_disk_path) asset, created = Asset.objects.get_or_create(link=link) asset.base_storage_path = os.path.sep.join(path_elements) asset.save() file_name = '/cap' + mime.guess_extension(mime_type) if mime_type == 'application/pdf': asset.pdf_capture = file_name else: asset.image_capture = file_name #print linky_home_disk_path + file_name #png = PythonMagick.Image(linky_home_disk_path + file_name) #png.write("file_out.png") #params = ['convert', linky_home_disk_path + file_name, 'out.png'] #subprocess.check_call(params) asset.save() request.FILES['file'].file.seek(0) f = open(linky_home_disk_path + file_name, 'w') f.write(request.FILES['file'].file.read()) os.fsync(f) f.close() response_object = {'status':'success', 'linky_id':link.guid, 'linky_hash':link.guid} """try: get_source.delay(link.guid, target_url, os.path.sep.join(path_elements), request.META['HTTP_USER_AGENT']) store_text_cap.delay(target_url, target_title, asset) except Exception, e: # TODO: Log the failed url asset.pdf_capture = 'failed' asset.save()""" return HttpResponse(json.dumps(response_object), 'application/json') else: return HttpResponseBadRequest(json.dumps({'status':'failed', 'reason':'Invalid file.'}), 'application/json') else: return HttpResponseBadRequest(json.dumps({'status':'failed', 'reason':'Missing file.'}), 'application/json')
def dumpfiles(self, collection): mime = MimeTypes() db = MongoClient(host=self.host, port=self.port, username=self.username, password=self.password)[self.db] uploadsCollection = db[collection] fs = gridfs.GridFSBucket(db, bucket_name=collection) uploads = uploadsCollection.find({}, no_cursor_timeout=True) instanceId = db["rocketchat_settings"].find_one({"_id": "uniqueID"})["value"] for upload in uploads: if upload["store"] == "GridFS:Uploads": if "complete" in upload and upload["complete"] is True: path = upload["path"] pathSegments = path.split("/") gridfsId = pathSegments[3] for res in fs.find({"_id": gridfsId}): fileext = "" if "extension" in upload: fileext = upload["extension"] else: fileext = mime.guess_extension(res.content_type) if fileext is not None and fileext != "": filename = gridfsId + "." + fileext else: filename = gridfsId print(upload["path"]) userVisitorId = None if "userId" in upload: objKey = instanceId + "/uploads/" + upload[ "rid"] + "/" + upload["userId"] + "/" + gridfsId userVisitorId = upload["userId"] else: objKey = instanceId + "/uploads/" + upload[ "rid"] + "/" + upload[ "visitorToken"] + "/" + gridfsId userVisitorId = upload["visitorToken"] try: objHead = s3.head_object(Bucket=self.bucket, Key=objKey) print("file already exists " + upload["name"]) except: print("uploading " + upload["name"] + " " + str(round(upload["size"] / 1024 / 1024, 2)) + " MB") uploadType = "" if "type" in upload: uploadType = upload["type"] s3.upload_fileobj(res, self.bucket, objKey, ExtraArgs={ "ContentDisposition": "inline; filename=\"" + upload["name"] + "\"", "ContentType": uploadType }) self.addtolog(gridfsId, filename, collection, res.md5, upload["rid"], userVisitorId, upload["name"]) else: print(upload) self.writelog()
def forge_email(fromaddr, toaddr, subject, content, html_content='', html_images=None, usebcc=False, header=None, footer=None, html_header=None, html_footer=None, ln=None, charset=None, replytoaddr="", attachments=None, bccaddr=""): """Prepare email. Add header and footer if needed. @param fromaddr: [string] sender @param toaddr: [string or list-of-strings] list of receivers (if string, then receivers are separated by ',') @param usebcc: [bool] True for using Bcc in place of To @param subject: [string] subject of the email @param content: [string] content of the email @param html_content: [string] html version of the email @param html_images: [dict] dictionary of image id, image path @param header: [string] None for the default header @param footer: [string] None for the default footer @param ln: language @charset: [string] the content charset. By default is None which means to try to encode the email as ascii, then latin1 then utf-8. @param replytoaddr: [string or list-of-strings] to be used for the reply-to header of the email (if string, then receivers are separated by ',') @param attachments: list of paths of files to be attached. Alternatively, every element of the list could be a tuple: (filename, mimetype) @param bccaddr: [string or list-of-strings] to be used for BCC header of the email (if string, then receivers are separated by ',') @return: forged email as an EmailMessage object""" from invenio_ext.template import render_template_to_string ln = default_ln(ln) if html_images is None: html_images = {} content = render_template_to_string( 'mail_text.tpl', content=unicodifier(content), header=unicodifier(header), footer=unicodifier(footer)).encode('utf8') if isinstance(toaddr, list): toaddr = ','.join(toaddr) if isinstance(bccaddr, list): bccaddr = ','.join(bccaddr) if isinstance(replytoaddr, list): replytoaddr = ','.join(replytoaddr) toaddr = remove_temporary_emails(toaddr) headers = {} kwargs = {'to': [], 'cc': [], 'bcc': []} if replytoaddr: headers['Reply-To'] = replytoaddr if usebcc: headers['Bcc'] = bccaddr kwargs['bcc'] = toaddr.split(',') + bccaddr.split(',') kwargs['to'] = ['Undisclosed.Recipients:'] else: kwargs['to'] = toaddr.split(',') headers['From'] = fromaddr headers['Date'] = formatdate(localtime=True) headers['User-Agent'] = 'Invenio %s at %s' % (cfg['CFG_VERSION'], cfg['CFG_SITE_URL']) if html_content: html_content = render_template_to_string( 'mail_html.tpl', content=unicodifier(html_content), header=unicodifier(html_header), footer=unicodifier(html_footer)).encode('utf8') msg_root = EmailMultiAlternatives(subject=subject, body=content, from_email=fromaddr, headers=headers, **kwargs) msg_root.attach_alternative(html_content, "text/html") # if not html_images: # # No image? Attach the HTML to the root # msg_root.attach(msg_text) # else: if html_images: # Image(s)? Attach the HTML and image(s) as children of a # "related" block msg_related = MIMEMultipart('related') # msg_related.attach(msg_text) for image_id, image_path in iteritems(html_images): attach_embed_image(msg_related, image_id, image_path) msg_root.attach(msg_related) else: msg_root = EmailMessage(subject=subject, body=content, from_email=fromaddr, headers=headers, **kwargs) if attachments: _mimes = MimeTypes(strict=False) for attachment in attachments: try: mime = None if type(attachment) in (list, tuple): attachment, mime = attachment if mime is None: # Automatic guessing of mimetype mime = _mimes.guess_type(attachment)[0] if mime is None: ext = _mimes.guess_extension(getContentType(attachment)) mime = _mimes.guess_type("foo" + ext)[0] if not mime: mime = 'application/octet-stream' part = MIMEBase(*mime.split('/', 1)) part.set_payload(open(attachment, 'rb').read()) Encoders.encode_base64(part) part.add_header( 'Content-Disposition', 'attachment; filename="%s"' % os.path.basename(attachment)) msg_root.attach(part) except: from invenio_ext.logging import register_exception register_exception(alert_admin=True, prefix="Can't attach %s" % attachment) return msg_root