def dumpfiles(self, collection):
mime = MimeTypes()
db = MongoClient(host=self.host, port=self.port)[self.db]
uploadsCollection = db[collection]
fs = gridfs.GridFSBucket(db, bucket_name=collection)
uploads = uploadsCollection.find({}, no_cursor_timeout=True)
for upload in uploads:
if upload["store"] == "GridFS:Uploads":
if "complete" in upload and upload["complete"] is True:
path = upload["path"]
pathSegments = path.split("/")
gridfsId = pathSegments[3]
for res in fs.find({"_id": gridfsId}):
data = res.read()
fileext = ""
if "extension" in upload:
fileext = upload["extension"]
else:
fileext = mime.guess_extension(res.content_type)
if fileext is not None and fileext != "":
filename = gridfsId + "." + fileext
else:
filename = gridfsId
file = open(self.outDir + "/" + filename, "wb")
file.write(data)
file.close()
self.addtolog(gridfsId, filename, collection, res.md5)
else:
print(upload)
self.writelog()
def dumpfiles(self, collection, store):
mime = MimeTypes()
db = self.getdb()
uploadsCollection = db[collection]
fs = gridfs.GridFSBucket(db, bucket_name=collection)
uploads = uploadsCollection.find({}, no_cursor_timeout=True)
i = 0
for upload in uploads:
if upload["store"] == "GridFS:Uploads":
gridfsId = upload['_id']
if "complete" in upload and upload["complete"] is True:
for res in fs.find({"_id": gridfsId}):
data = res.read()
filename = gridfsId
fileext = ""
if "extension" in upload and upload["extension"] != "":
fileext = "." + upload["extension"]
else:
fileext = mime.guess_extension(res.content_type)
if fileext is not None and fileext != "":
filename = filename + fileext
i += 1
print("%i. Dumping %s %s" %
(i, gridfsId, upload['name']))
key = store.put(filename, data, upload)
print("%i. Finished dumping %s %s" %
(i, gridfsId, upload['name']))
logitem = {
"id": gridfsId,
"file": filename,
"collection": collection,
"md5": res.md5,
"key": key
}
self.updateDbEntry(logitem, i, db)
self.removeBlobsEntry(logitem, i, db)
self.addtolog(logitem)
self.writelog()
self.log.pop()
else:
print("[Warning] Skipping incomplete upload %s" %
(gridfsId),
file=sys.stderr)
def _getExtension( episode ): # get extension from url urlExt = os.path.splitext( episode.url )[ 1 ] # get extension from media type mime = MimeTypes() mimeExtList = mime.guess_all_extensions( episode.type ) # check if url extension matches any item in list if urlExt in mimeExtList: # yep, then url extension is good return urlExt # otherwise, use recommendation from MimeTypes, as default we use .mp3 mimeExt = mime.guess_extension( episode.type ) if mimeExt == None: mimeExt = ".mp3" return mimeExt
def upload_new_song(user, song_id, file_path, signature=None):
# Get song info
print("Upload new song \nSong path: ", file_path)
song = Song.objects.get(pk=song_id)
name = song.name
author = song.author
price = song.price
extension = song.extension = open(file_path).name.rsplit('.', 1)[1]
print("Extension: ", extension)
mime_type = MimeTypes()
content_type = mime_type.guess_extension(file_path)
print("Mime type: ", content_type)
if not user.is_superuser: # if normal user, upload to their own directory
if user.profile.drive_folder_id:
folder_id = user.profile.drive_folder_id
else:
folder_id = drive_api.createFolder(user.username)
user.profile.drive_folder_id = folder_id
user.profile.save()
else: # if superuser upload to shiro store directory
folder_id = drive_api.shiro_store_folder_id
output_filename = name + " - " + author + "." + extension
file_id = drive_api.uploadFile(output_filename,
file_path,
content_type,
folder_id=folder_id)
# Build new song with info
new_song = Song(id=file_id,
name=name,
author=author,
extension=extension,
price=price)
if signature:
new_song.signature = signature
if not user.is_superuser:
new_song.owner = user
new_song.save()
user.profile.songs.add(
new_song) # Update Archived Song to Profile
user.profile.save()
else:
new_song.save()
return file_id
def dumpfiles(self, collection):
mime = MimeTypes()
db = MongoClient().rocketchat
fs = gridfs.GridFSBucket(db, collection)
for grid_out in fs.find({}, no_cursor_timeout=True):
data = grid_out.read()
pprint(grid_out.filename)
pprint(grid_out._id)
fileext = mime.guess_extension(grid_out.content_type)
filename = grid_out.filename
filename = '.'.join(filename.split('.')[:1])
if fileext:
filename = filename+fileext
print(filename)
file = open(self.outDir+"/"+filename, "wb")
file.write(data)
file.close()
self.addtolog(grid_out._id, filename, collection, grid_out.md5)
self.writelog()
def upload_file(request):
if request.method == 'POST':
form = UploadFileForm(request.POST, request.FILES)
if form.is_valid():
mime = MimeTypes()
uploaded_file = request.FILES['file']
mime_type = mime.guess_type(uploaded_file.name)
# Get mime type string from tuple
if mime_type[0]:
mime_type = mime_type[0]
else:
return HttpResponseBadRequest(json.dumps({'status':'failed', 'reason':'Invalid file.'}), 'application/json')
if validate_upload_file(uploaded_file, mime_type) and uploaded_file.size <= settings.MAX_ARCHIVE_FILE_SIZE:
link = Link(submitted_url=form.cleaned_data['url'], submitted_title=form.cleaned_data['title'], created_by = request.user)
link.save()
asset = Asset(link=link)
file_name = 'cap' + mime.guess_extension(mime_type)
file_path = os.path.join(asset.base_storage_path, file_name)
uploaded_file.file.seek(0)
file_name = default_storage.store_file(uploaded_file, file_path)
if mime_type == 'application/pdf':
asset.pdf_capture = file_name
else:
asset.image_capture = file_name
asset.save()
response_object = {'status':'success', 'linky_id':link.guid, 'linky_hash':link.guid}
return HttpResponse(json.dumps(response_object), 'application/json', 201) # '201 Created' status
else:
return HttpResponseBadRequest(json.dumps({'status':'failed', 'reason':'Invalid file.'}), 'application/json')
else:
return HttpResponseBadRequest(json.dumps({'status':'failed', 'reason':'Missing file.'}), 'application/json')
return HttpResponseBadRequest(json.dumps({'status':'failed', 'reason':'No file submitted.'}), 'application/json')
def upload_file(request):
if request.method == 'POST':
form = UploadFileForm(request.POST, request.FILES)
if form.is_valid():
mime = MimeTypes()
uploaded_file = request.FILES['file']
mime_type = mime.guess_type(uploaded_file.name)
# Get mime type string from tuple
if mime_type[0]:
mime_type = mime_type[0]
else:
return HttpResponseBadRequest(
json.dumps({
'status': 'failed',
'reason': 'Invalid file.'
}), 'application/json')
if validate_upload_file(
uploaded_file, mime_type
) and uploaded_file.size <= settings.MAX_ARCHIVE_FILE_SIZE:
link = Link(submitted_url=form.cleaned_data['url'],
submitted_title=form.cleaned_data['title'],
created_by=request.user)
link.save()
asset = Asset(link=link)
file_name = 'cap' + mime.guess_extension(mime_type)
file_path = os.path.join(asset.base_storage_path, file_name)
uploaded_file.file.seek(0)
file_name = default_storage.store_file(uploaded_file,
file_path)
if mime_type == 'application/pdf':
asset.pdf_capture = file_name
else:
asset.image_capture = file_name
asset.save()
response_object = {
'status': 'success',
'linky_id': link.guid,
'linky_hash': link.guid
}
return HttpResponse(json.dumps(response_object),
'application/json',
201) # '201 Created' status
else:
return HttpResponseBadRequest(
json.dumps({
'status': 'failed',
'reason': 'Invalid file.'
}), 'application/json')
else:
return HttpResponseBadRequest(
json.dumps({
'status': 'failed',
'reason': 'Missing file.'
}), 'application/json')
return HttpResponseBadRequest(
json.dumps({
'status': 'failed',
'reason': 'No file submitted.'
}), 'application/json')
def forge_email(fromaddr, toaddr, subject, content, html_content='',
html_images=None, usebcc=False, header=None, footer=None,
html_header=None, html_footer=None, ln=None,
charset=None, replytoaddr="", attachments=None, bccaddr=""):
"""Prepare email. Add header and footer if needed.
@param fromaddr: [string] sender
@param toaddr: [string or list-of-strings] list of receivers (if string, then
receivers are separated by ',')
@param usebcc: [bool] True for using Bcc in place of To
@param subject: [string] subject of the email
@param content: [string] content of the email
@param html_content: [string] html version of the email
@param html_images: [dict] dictionary of image id, image path
@param header: [string] None for the default header
@param footer: [string] None for the default footer
@param ln: language
@charset: [string] the content charset. By default is None which means
to try to encode the email as ascii, then latin1 then utf-8.
@param replytoaddr: [string or list-of-strings] to be used for the
reply-to header of the email (if string, then
receivers are separated by ',')
@param attachments: list of paths of files to be attached. Alternatively,
every element of the list could be a tuple: (filename, mimetype)
@param bccaddr: [string or list-of-strings] to be used for BCC header of the email
(if string, then receivers are separated by ',')
@return: forged email as an EmailMessage object"""
from invenio_ext.template import render_template_to_string
ln = default_ln(ln)
if html_images is None:
html_images = {}
content = render_template_to_string('mail_text.tpl',
content=unicodifier(content),
header=unicodifier(header),
footer=unicodifier(footer)
).encode('utf8')
if isinstance(toaddr, list):
toaddr = ','.join(toaddr)
if isinstance(bccaddr, list):
bccaddr = ','.join(bccaddr)
if isinstance(replytoaddr, list):
replytoaddr = ','.join(replytoaddr)
toaddr = remove_temporary_emails(toaddr)
headers = {}
kwargs = {'to': [], 'cc': [], 'bcc': []}
if replytoaddr:
headers['Reply-To'] = replytoaddr
if usebcc:
headers['Bcc'] = bccaddr
kwargs['bcc'] = toaddr.split(',') + bccaddr.split(',')
kwargs['to'] = ['Undisclosed.Recipients:']
else:
kwargs['to'] = toaddr.split(',')
headers['From'] = fromaddr
headers['Date'] = formatdate(localtime=True)
headers['User-Agent'] = 'Invenio %s at %s' % (cfg['CFG_VERSION'],
cfg['CFG_SITE_URL'])
if html_content:
html_content = render_template_to_string(
'mail_html.tpl',
content=unicodifier(html_content),
header=unicodifier(html_header),
footer=unicodifier(html_footer)
).encode('utf8')
msg_root = EmailMultiAlternatives(subject=subject, body=content,
from_email=fromaddr,
headers=headers, **kwargs)
msg_root.attach_alternative(html_content, "text/html")
# if not html_images:
# # No image? Attach the HTML to the root
# msg_root.attach(msg_text)
# else:
if html_images:
# Image(s)? Attach the HTML and image(s) as children of a
# "related" block
msg_related = MIMEMultipart('related')
# msg_related.attach(msg_text)
for image_id, image_path in iteritems(html_images):
attach_embed_image(msg_related, image_id, image_path)
msg_root.attach(msg_related)
else:
msg_root = EmailMessage(subject=subject, body=content,
from_email=fromaddr, headers=headers, **kwargs)
if attachments:
_mimes = MimeTypes(strict=False)
for attachment in attachments:
try:
mime = None
if type(attachment) in (list, tuple):
attachment, mime = attachment
if mime is None:
# Automatic guessing of mimetype
mime = _mimes.guess_type(attachment)[0]
if mime is None:
ext = _mimes.guess_extension(getContentType(attachment))
mime = _mimes.guess_type("foo" + ext)[0]
if not mime:
mime = 'application/octet-stream'
part = MIMEBase(*mime.split('/', 1))
part.set_payload(open(attachment, 'rb').read())
Encoders.encode_base64(part)
part.add_header(
'Content-Disposition',
'attachment; filename="%s"' %
os.path.basename(attachment))
msg_root.attach(part)
except:
from invenio_ext.logging import register_exception
register_exception(
alert_admin=True,
prefix="Can't attach %s" %
attachment)
return msg_root
def upload_file(request):
if request.method == 'POST':
form = UploadFileForm(request.POST, request.FILES)
if form.is_valid():
mime = MimeTypes()
mime_type = mime.guess_type(request.FILES['file'].name)
# Get mime type string from tuple
if mime_type[0]:
mime_type = mime_type[0]
else:
return HttpResponseBadRequest(json.dumps({'status':'failed', 'reason':'Invalid file.'}), 'application/json')
if validate_upload_file(request.FILES['file'], mime_type) and request.FILES['file'].size <= settings.MAX_ARCHIVE_FILE_SIZE:
link = Link(submitted_url=form.cleaned_data['url'], submitted_title=form.cleaned_data['title'])
if request.user.is_authenticated():
link.created_by = request.user
link.save()
now = dt = datetime.now()
time_tuple = now.timetuple()
path_elements = [str(time_tuple.tm_year), str(time_tuple.tm_mon), str(time_tuple.tm_mday), str(time_tuple.tm_hour), str(time_tuple.tm_min), link.guid]
linky_home_disk_path = settings.GENERATED_ASSETS_STORAGE + '/' + os.path.sep.join(path_elements)
if not os.path.exists(linky_home_disk_path):
os.makedirs(linky_home_disk_path)
asset, created = Asset.objects.get_or_create(link=link)
asset.base_storage_path = os.path.sep.join(path_elements)
asset.save()
file_name = '/cap' + mime.guess_extension(mime_type)
if mime_type == 'application/pdf':
asset.pdf_capture = file_name
else:
asset.image_capture = file_name
#print linky_home_disk_path + file_name
#png = PythonMagick.Image(linky_home_disk_path + file_name)
#png.write("file_out.png")
#params = ['convert', linky_home_disk_path + file_name, 'out.png']
#subprocess.check_call(params)
asset.save()
request.FILES['file'].file.seek(0)
f = open(linky_home_disk_path + file_name, 'w')
f.write(request.FILES['file'].file.read())
os.fsync(f)
f.close()
response_object = {'status':'success', 'linky_id':link.guid, 'linky_hash':link.guid}
"""try:
get_source.delay(link.guid, target_url, os.path.sep.join(path_elements), request.META['HTTP_USER_AGENT'])
store_text_cap.delay(target_url, target_title, asset)
except Exception, e:
# TODO: Log the failed url
asset.pdf_capture = 'failed'
asset.save()"""
return HttpResponse(json.dumps(response_object), 'application/json')
else:
return HttpResponseBadRequest(json.dumps({'status':'failed', 'reason':'Invalid file.'}), 'application/json')
else:
return HttpResponseBadRequest(json.dumps({'status':'failed', 'reason':'Missing file.'}), 'application/json')
def dumpfiles(self, collection):
mime = MimeTypes()
db = MongoClient(host=self.host,
port=self.port,
username=self.username,
password=self.password)[self.db]
uploadsCollection = db[collection]
fs = gridfs.GridFSBucket(db, bucket_name=collection)
uploads = uploadsCollection.find({}, no_cursor_timeout=True)
instanceId = db["rocketchat_settings"].find_one({"_id":
"uniqueID"})["value"]
for upload in uploads:
if upload["store"] == "GridFS:Uploads":
if "complete" in upload and upload["complete"] is True:
path = upload["path"]
pathSegments = path.split("/")
gridfsId = pathSegments[3]
for res in fs.find({"_id": gridfsId}):
fileext = ""
if "extension" in upload:
fileext = upload["extension"]
else:
fileext = mime.guess_extension(res.content_type)
if fileext is not None and fileext != "":
filename = gridfsId + "." + fileext
else:
filename = gridfsId
print(upload["path"])
userVisitorId = None
if "userId" in upload:
objKey = instanceId + "/uploads/" + upload[
"rid"] + "/" + upload["userId"] + "/" + gridfsId
userVisitorId = upload["userId"]
else:
objKey = instanceId + "/uploads/" + upload[
"rid"] + "/" + upload[
"visitorToken"] + "/" + gridfsId
userVisitorId = upload["visitorToken"]
try:
objHead = s3.head_object(Bucket=self.bucket,
Key=objKey)
print("file already exists " + upload["name"])
except:
print("uploading " + upload["name"] + " " +
str(round(upload["size"] / 1024 / 1024, 2)) +
" MB")
uploadType = ""
if "type" in upload:
uploadType = upload["type"]
s3.upload_fileobj(res,
self.bucket,
objKey,
ExtraArgs={
"ContentDisposition":
"inline; filename=\"" +
upload["name"] + "\"",
"ContentType":
uploadType
})
self.addtolog(gridfsId, filename, collection, res.md5,
upload["rid"], userVisitorId,
upload["name"])
else:
print(upload)
self.writelog()
def forge_email(fromaddr,
toaddr,
subject,
content,
html_content='',
html_images=None,
usebcc=False,
header=None,
footer=None,
html_header=None,
html_footer=None,
ln=None,
charset=None,
replytoaddr="",
attachments=None,
bccaddr=""):
"""Prepare email. Add header and footer if needed.
@param fromaddr: [string] sender
@param toaddr: [string or list-of-strings] list of receivers (if string, then
receivers are separated by ',')
@param usebcc: [bool] True for using Bcc in place of To
@param subject: [string] subject of the email
@param content: [string] content of the email
@param html_content: [string] html version of the email
@param html_images: [dict] dictionary of image id, image path
@param header: [string] None for the default header
@param footer: [string] None for the default footer
@param ln: language
@charset: [string] the content charset. By default is None which means
to try to encode the email as ascii, then latin1 then utf-8.
@param replytoaddr: [string or list-of-strings] to be used for the
reply-to header of the email (if string, then
receivers are separated by ',')
@param attachments: list of paths of files to be attached. Alternatively,
every element of the list could be a tuple: (filename, mimetype)
@param bccaddr: [string or list-of-strings] to be used for BCC header of the email
(if string, then receivers are separated by ',')
@return: forged email as an EmailMessage object"""
from invenio_ext.template import render_template_to_string
ln = default_ln(ln)
if html_images is None:
html_images = {}
content = render_template_to_string(
'mail_text.tpl',
content=unicodifier(content),
header=unicodifier(header),
footer=unicodifier(footer)).encode('utf8')
if isinstance(toaddr, list):
toaddr = ','.join(toaddr)
if isinstance(bccaddr, list):
bccaddr = ','.join(bccaddr)
if isinstance(replytoaddr, list):
replytoaddr = ','.join(replytoaddr)
toaddr = remove_temporary_emails(toaddr)
headers = {}
kwargs = {'to': [], 'cc': [], 'bcc': []}
if replytoaddr:
headers['Reply-To'] = replytoaddr
if usebcc:
headers['Bcc'] = bccaddr
kwargs['bcc'] = toaddr.split(',') + bccaddr.split(',')
kwargs['to'] = ['Undisclosed.Recipients:']
else:
kwargs['to'] = toaddr.split(',')
headers['From'] = fromaddr
headers['Date'] = formatdate(localtime=True)
headers['User-Agent'] = 'Invenio %s at %s' % (cfg['CFG_VERSION'],
cfg['CFG_SITE_URL'])
if html_content:
html_content = render_template_to_string(
'mail_html.tpl',
content=unicodifier(html_content),
header=unicodifier(html_header),
footer=unicodifier(html_footer)).encode('utf8')
msg_root = EmailMultiAlternatives(subject=subject,
body=content,
from_email=fromaddr,
headers=headers,
**kwargs)
msg_root.attach_alternative(html_content, "text/html")
# if not html_images:
# # No image? Attach the HTML to the root
# msg_root.attach(msg_text)
# else:
if html_images:
# Image(s)? Attach the HTML and image(s) as children of a
# "related" block
msg_related = MIMEMultipart('related')
# msg_related.attach(msg_text)
for image_id, image_path in iteritems(html_images):
attach_embed_image(msg_related, image_id, image_path)
msg_root.attach(msg_related)
else:
msg_root = EmailMessage(subject=subject,
body=content,
from_email=fromaddr,
headers=headers,
**kwargs)
if attachments:
_mimes = MimeTypes(strict=False)
for attachment in attachments:
try:
mime = None
if type(attachment) in (list, tuple):
attachment, mime = attachment
if mime is None:
# Automatic guessing of mimetype
mime = _mimes.guess_type(attachment)[0]
if mime is None:
ext = _mimes.guess_extension(getContentType(attachment))
mime = _mimes.guess_type("foo" + ext)[0]
if not mime:
mime = 'application/octet-stream'
part = MIMEBase(*mime.split('/', 1))
part.set_payload(open(attachment, 'rb').read())
Encoders.encode_base64(part)
part.add_header(
'Content-Disposition',
'attachment; filename="%s"' % os.path.basename(attachment))
msg_root.attach(part)
except:
from invenio_ext.logging import register_exception
register_exception(alert_admin=True,
prefix="Can't attach %s" % attachment)
return msg_root
