def mime_type(self): m = MimeTypes() m.read(LIASIS_DIR+MIME_TYPES) if m.guess_type(self.tfile): return m.guess_type(self.tfile)[0] else: return "text/plain"
def getMimeType(self,buffre,url,mtype): if '?' in url: url = url.split('?')[0] mime = MimeTypes() ext = os.path.splitext(url)[1] if mtype == 'text/html' and ext == '': if url[-1] == '/': l = len(url)-1 url = url[0:-1] url = url+'/index.html' ext = '.html' #ext1 = mime.guess_extension(mtype,True) #print ext1 mime_type = mime.guess_type(url) #print url if ext: #print url u = urlparse.urlparse(url) #print u.netloc,u.path print self.host if self.host: root_dir = self.root_dir+"/"+self.host file_path = os.path.join(root_dir,u.netloc+u.path) print file_path #if not os.path.isfile(file_path): makeDir(os.path.dirname(file_path)) f = open(file_path,"wb") f.write(buffre)
def upload_attachment(self, file_path): __, file_name = os.path.split(file_path) mime = MimeTypes() url = urllib.pathname2url(file_path) mime_type, __ = mime.guess_type(url) data = { 'file_name': file_name, 'file_type': mime_type } url = urljoin(self.api_url, 'attachment/upload') response = self.post(url, data=data) with open(file_path) as fh: file_data = fh.read() upload_response = requests.put( response['upload_url'], data=file_data, headers={'content-type': mime_type}, params={'file': file_path} ) upload_response.raise_for_status() return { 'file_url': response['file_url'], 'file_type': mime_type, }
def download(request, filename): #down_file = File.objects.get(name = filename) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) DOWNLOAD_URL = BASE_DIR+"/download/" file_path = DOWNLOAD_URL + filename file_name = filename fp = open(file_path, 'rb') response = HttpResponse(fp.read()) fp.close() mime = MimeTypes() type, encoding = mime.guess_type(file_name) if type is None: type = 'application/octet-stream' response['Content-Type'] = type response['Content-Length'] = str(os.stat(file_path).st_size) if encoding is not None: response['Content-Encoding'] = encoding if u'WebKit' in request.META['HTTP_USER_AGENT']: filename_header = 'filename=%s' % file_name.encode('utf-8') elif u'MSIE' in request.META['HTTP_USER_AGENT']: filename_header = '' else: filename_header = 'filename*=UTF-8\'\'%s' % urllib.quote(file_name.encode('utf-8')) response['Content-Disposition'] = 'attachment; ' + filename_header # 記錄系統事件 if is_event_open(request) : log = Log(user_id=request.user.id, event=u'下載檔案<'+filename+'>') log.save() return response
def update_community(self, properties): pc = api.portal.get_tool('portal_catalog') brain = pc.unrestrictedSearchResults(portal_type='ulearn.community', community_hash=self.params['community']) if not brain: brain = pc.unrestrictedSearchResults(portal_type='ulearn.community', gwuuid=self.params['community']) if brain: community = brain[0].getObject() if properties['title'] is not None: community.title = properties['title'] if properties['description'] is not None: community.description = properties['description'] if properties['image'] is not None: imageObj = '' mime = MimeTypes() mime_type = mime.guess_type(properties['image']) imgName = (properties['image'].split('/')[-1]).decode('utf-8') imgData = requests.get(properties['image']).content imageObj = NamedBlobImage(data=imgData, filename=imgName, contentType=mime_type[0]) community.image = imageObj if properties['activity_view'] is not None: community.activity_view = properties['activity_view'] if properties['twitter_hashtag'] is not None: community.twitter_hashtag = properties['twitter_hashtag'] if properties['notify_activity_via_push'] is not None: community.notify_activity_via_push = True if properties['notify_activity_via_push'] == 'True' else None if properties['notify_activity_via_push_comments_too'] is not None: community.notify_activity_via_push_comments_too = True if properties['notify_activity_via_push_comments_too'] == 'True' else None community.reindexObject() return True else: return False
def get_mime_type(path): mime = MimeTypes() mime_type = mime.guess_type(path)[0] if not mime_type: mime_type = "text/{0}".format(os.path.splitext(path)[1]) return mime_type
def GetRFC(rfc_id,file_name,peer_ip,peer_port): global LRFCdata global Server s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.settimeout(None) peer_ipaddr=peer_ip s.connect((peer_ipaddr,int(peer_port))) message = ["GETRFC",str(rfc_id),"P2P-DI/1.0","Host: ",IP,"Port: ",PORT,"Title: ",file_name,"Client ID: ",str(self_id)] s.send(pickle.dumps(message)) os.chdir(os.getcwd()) filename=file_name+".pdf" file1=open(filename,'wb') q=s.recv(4096) q=pickle.loads(q) if 'FILE NOT FOUND' in q: code = '404' phrase = 'FILE NOT FOUND' reply = ["P2P-DI/1.0 ",str(code),str(phrase)] PrintResponse(reply,'FILERESPN') else: if 'FILE FOUND' in q: last_modified = q[2] message = ["OK"] s.send(pickle.dumps(message)) while True: q=s.recv(4096) if q: file1.write(q) else: code = '200' phrase = 'OK' mime = MimeTypes() filesize = os.stat(filename).st_size mime_type = mime.guess_type(filename) reply = ["P2P-DI/1.0 ",str(code),str(phrase),"Last-Modified: ",str(last_modified),"Content-Length: ",str(filesize),"Content-Type: ",str(mime_type[0])] PrintResponse(reply,'FILESENT') file1.close() break serverIP = Server[0] serverPort = Server[1] message=["PQUERY","P2P-DI/1.0 ",str(self_id),"Host: ",IP,"Port: ",str(PORT)] reply = client(message,serverIP,serverPort) PrintResponse(reply,'PQUERY') LocalRFC = LocalRFCRecord(rfc_id,file_name) LRFCdata.append(LocalRFC.getLocalRFC()) Local_linked_list.append(LocalRFC.getLocalRFC()) active_list = reply[4] active_list=RFCStore(active_list,[[rfc_id,file_name]]) s.close()
def export_warc(self): # by using select_for_update and checking for existence of this file, # we make sure that we won't accidentally try to create the file multiple # times in parallel. asset = self.assets.select_for_update().first() if not asset: return # this is not an old-style Link if default_storage.exists(self.warc_storage_file()): return guid = self.guid out = self.open_warc_for_writing() def write_resource_record(file_path, url, content_type): self.write_warc_resource_record( default_storage.open(file_path), url.encode('utf8'), content_type, default_storage.created_time(file_path), out) def write_metadata_record(metadata, target_headers): concurrent_to = (v for k, v in target_headers if k == warctools.WarcRecord.ID).next() warc_date = (v for k, v in target_headers if k == warctools.WarcRecord.DATE).next() url = (v for k, v in target_headers if k == warctools.WarcRecord.URL).next() self.write_warc_metadata_record(metadata, url, concurrent_to, warc_date, out) # write PDF capture if asset.pdf_capture and ('cap' in asset.pdf_capture or 'upload' in asset.pdf_capture): file_path = os.path.join(asset.base_storage_path, asset.pdf_capture) headers = write_resource_record(file_path, "file:///%s/%s" % (guid, asset.pdf_capture), 'application/pdf') #write_metadata_record({'role':'primary', 'user_upload':asset.user_upload}, headers) # write image capture (if it's not a PDF thumbnail) elif (asset.image_capture and ('cap' in asset.image_capture or 'upload' in asset.image_capture)): file_path = os.path.join(asset.base_storage_path, asset.image_capture) mime_type = get_mime_type(asset.image_capture) write_resource_record(file_path, "file:///%s/%s" % (guid, asset.image_capture), mime_type) if asset.warc_capture: # write WARC capture if asset.warc_capture == 'archive.warc.gz': file_path = os.path.join(asset.base_storage_path, asset.warc_capture) self.write_warc_raw_data(default_storage.open(file_path), out) # write wget capture elif asset.warc_capture == 'source/index.html': mime = MimeTypes() for root, dirs, files in default_storage.walk(os.path.join(asset.base_storage_path, 'source')): rel_path = root.split(asset.base_storage_path, 1)[-1] for file_name in files: mime_type = mime.guess_type(file_name)[0] write_resource_record(os.path.join(root, file_name), "file:///%s%s/%s" % (guid, rel_path, file_name), mime_type) self.close_warc_after_writing(out) # regenerate CDX index self.cdx_lines.all().delete()
def searchImages(rootDir): imageList = [] mime = MimeTypes() for root, subFolders, files in os.walk(rootDir): for file in files: mt = mime.guess_type(file)[0] if mt and mt.startswith('image/'): imageList = imageList + [os.path.join(root,file)] return imageList
def get_extension(url): from mimetypes import MimeTypes mime_types = MimeTypes() (type, encoding) = mime_types.guess_type(url) extensions = mime_types.guess_all_extensions(type) extension = extensions[-1] return extension
def add_documents(request, category_id): if request.is_ajax(): files = request.GET.getlist('files', False) cat = Category.objects.get(id=category_id) l_doc = [] l_pdf = [] cmds = [] paths = [] for f in list(files): mime = MimeTypes() path = os.path.join(settings.MEDIA_ROOT, settings.UPLOAD_DIR, f) m = mime.guess_type(path)[0] d = Document(name=f.encode('ascii', 'ignore'), owner=request.user, refer_category=cat) d.save() cat.add_doc(d) if m == 'application/pdf': l_pdf.append(([cat], path, f, [d])) elif m in ['image/png', 'image/jpeg', 'image/bmp']: im = Image.open(path) w, h = im.size new_filename = str(d.id) + '_' + f new_path = os.path.join(cat.get_absolute_path(), new_filename) shutil.copy2(path, new_path) d.add_page(d.get_npages() + 1, new_filename, w, h) for fu in FileUpload.objects.all(): if fu.file.path == path: fu.delete() d.complete = True d.save() remove_fileupload([path]) elif m in ['application/msword', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document']: p = re.compile(r'.[Dd][Oo][Cc][xX]?$') new_f = p.sub('.pdf', f) new_path = path.replace(f, new_f) cmd = 'soffice --headless --convert-to pdf %s --outdir %s/upload' % (path, settings.MEDIA_ROOT) cmds.append(cmd) paths.append(path) l_doc.append(([cat], new_path, new_f, [d])) elif m in ['application/vnd.ms-excel', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet']: p = re.compile(r'.[Xx][Ll][Ss][xX]?$') new_f = p.sub('.pdf', f) new_path = path.replace(f, new_f) cmd = 'soffice --headless --convert-to pdf %s --outdir %s/upload' % (path, settings.MEDIA_ROOT) cmds.append(cmd) paths.append(path) l_doc.append(([cat], new_path, new_f, [d])) else: print 'ERREUR FORMAT FICHIER' if len(l_doc): thread1 = Timer(0, manage_convert_doc_to_pdf, (cmds, paths, l_doc,)) thread1.start() if len(l_pdf): thread = Timer(0, manage_convert_pdf_to_jpg, (l_pdf,)) thread.start() results = {'doc_list': [d.as_json() for d in cat.get_docs()], 'n': cat.count_docs()} return HttpResponse(json.dumps(results))
def uploadFileToS3(self ,filename,content): try : mime= MimeTypes.guess_type(filename)[0] k=Key(S3UploadManager.S3Bucket) k.key= filename k.set_metadata("Content-Type", mime) k.set_contents_from_string(content) k.set_acl("public-read") except Exception as e: raise type(e)(e.message)
def showfile(file): mime = MimeTypes() mime_type = mime.guess_type(file) if mime_type[0] is not None and mime_type[0].startswith("image"): return render_template("image.html", title="pyle :: preview", filepath="/data/" + file, filename=os.path.basename(file)) else: return readfile(file)
def mime(self): if self.mMimeType is not None: return self.mMimeType mime = MimeTypes() url = urllib.pathname2url(self.mFullPathName) self.mMimeType = mime.guess_type(url)[0] if self.mMimeType is None: self.mMimeType = "Unkown/None" return self.mMimeType
def make_headers(filename): mime = MimeTypes() mime.add_type('text/plain', '.log') mime.add_type('text/x-yaml', '.yaml') content_type, encoding = mime.guess_type(filename) headers = {"Content-Type": "application/octet-stream"} if content_type: headers['Content-Type'] = content_type if encoding: headers['Content-Encoding'] = encoding return headers
def get_mime_type(self, file_path): #Get mime type try: mime = MimeTypes() murl = urllib.pathname2url(file_path) mime_type = mime.guess_type(murl)[0] if mime_type == None: mime_type = 'text/plain' except: mime_type = 'text/plain' return mime_type
def show_media(self, url, title = "", desc ="", loop = 0): mime = MimeTypes() mime_type = mime.guess_type(url) payload = { "target": url, "title":title, "description":desc, "mimeType": mime_type[0], "iconSrc":"", "loop":loop} command = '{"id":"images","type":"request","uri":"ssap://media.viewer/open","payload":'+ json.dumps(payload)+'}' self.send_command(command)
def addFileOutput(self, name, filepath): data = open(filepath).read() filename = os.path.basename(filepath) mime = MimeTypes() mime_guess = mime.guess_type(filename)[0] self.result[name] = { "filename": filename, "content-type": mime_guess, "data": base64.b64encode(data) } return self
def transform_import_values(self, value): ''' # TODO: Following commented code can be used if user does not already have file in final location using django ORM: request = HttpRequest() # request.FILES['file-list_' + str(nodeid)] = None files = [] # request_list = [] for val in value.split(','): val_dict = {} val_dict['content'] = val val_dict['name'] = val.split('/')[-1].split('.')[0] val_dict['url'] = None # val_dict['size'] = None # val_dict['width'] = None # val_dict['height'] = None files.append(val_dict) f = open(val, 'rb') django_file = InMemoryUploadedFile(f,'file',val.split('/')[-1].split('.')[0],None,None,None) request.FILES.appendlist('file-list_' + str(nodeid), django_file) print request.FILES value = files ''' mime = MimeTypes() tile_data = [] for file_path in value.split(','): try: file_stats = os.stat(file_path) tile_file['lastModified'] = file_stats.st_mtime tile_file['size'] = file_stats.st_size except: pass tile_file = {} tile_file['file_id'] = str(uuid.uuid4()) tile_file['status'] = "" tile_file['name'] = file_path.split('/')[-1] tile_file['url'] = settings.MEDIA_URL + 'uploadedfiles/' + str(tile_file['name']) # tile_file['index'] = 0 # tile_file['height'] = 960 # tile_file['content'] = None # tile_file['width'] = 1280 # tile_file['accepted'] = True tile_file['type'] = mime.guess_type(file_path)[0] tile_file['type'] = '' if tile_file['type'] == None else tile_file['type'] tile_data.append(tile_file) file_path = 'uploadedfiles/' + str(tile_file['name']) fileid = tile_file['file_id'] models.File.objects.get_or_create(fileid=fileid, path=file_path) result = json.loads(json.dumps(tile_data)) return result
def update_data(self, data, content_type, size): """ """ if not data: # still index self.index_object() return passed, virus_name = DataVirusCheckAdapter(data).process() if not passed: log.warn('found virus %s, rejecting file' % virus_name) raise XWFFileError('found virus %s, rejecting file' % virus_name) # this isn't necessarily an error, on init we get called without data base_files_dir = self._base_files_dir self.size = size self.set_modificationTime() if base_files_dir: fileId = os.path.join(base_files_dir, self.getId()) f = file(fileId, 'wb+') if hasattr(data, '__class__') and data.__class__ is Pdata: while data is not None: f.write(data.data) data = data.next else: f.write(data) f.close() # fix the title title = to_unicode_or_bust(self.title) title = removePathsFromFilenames(title) self.title = title if content_type is not None: self.content_type = content_type else: mtypes = MimeTypes() mime_type = mtypes.guess_type(title) if mime_type: self.content_type = mime_type self.ZCacheable_invalidate() self.ZCacheable_set(None) self.http__refreshEtag() # index ourselves into the catalog self.index_object()
def get_base64_image(url): template_image = '' mime = MimeTypes() mime_type = mime.guess_type(url)[0] # Steam the image from the url request = requests.get(url) image_buffer = StringIO(request.content) if image_buffer: template_image = image_buffer.getvalue().encode('base64') template_image = u'data:%s;base64,%s' % (mime_type, template_image) return template_image
def batch_upload_response(self, request): output_fields = flatten_fieldsets(self.fieldsets) media_file_name = get_media_file_name(self, self.model) #Disabling exception handling here @olivierdalang's feedback: # try: latest_log_entry = LogEntry.objects.filter(action_flag=ADDITION).order_by('-action_time')[0] ct = ContentType.objects.get_for_id(latest_log_entry.content_type_id) obj = ct.get_object_for_this_type(pk=latest_log_entry.object_id) if obj: object_data = {} mime = MimeTypes() media_file = get_media_file(self, self.model, obj) media_file_url = media_file.url #urllib.pathname2url(media_file.url) #Not sure why i had this, but it's escaping the URL mime_type = mime.guess_type(media_file_url) edit_url = reverse('admin:%s_%s_change' %(obj._meta.app_label, obj._meta.model_name), args=[obj.id] ) object_data['media_file_url'] = media_file_url object_data['media_file_size'] = media_file.size object_data['media_file_type'] = mime_type[0] object_data['edit_url'] = mark_safe(edit_url) field_values = {} for output_field in output_fields: value = str(self.get_field_contents(output_field, obj)) label = str(label_for_field(output_field, self.model, self)) field_values[output_field] = { 'label':label, 'value':value } object_data['field_values'] = field_values data = { "success":True, "files":[ object_data ] } json_dumped = json.dumps(data) return HttpResponse(json_dumped, content_type='application/json')
def write(self, filename, contents, encoding='utf8'): """ Write a file to the data store. """ mime = MimeTypes() mime.add_type('text/x-yaml', '.yaml') content_type, _ = mime.guess_type(filename) key = self.bucket.new_key(self._path(filename)) key.set_contents_from_string(contents.encode(encoding), { 'Content-Type': content_type or 'text/plain', 'Content-Encoding': encoding, }) if self.public: key.set_canned_acl('public-read')
def upload(self, filepath): filename = filepath.split("/")[-1] mime = MimeTypes() mime_type = mime.guess_type(filename) data = {"filename": filename, "mime_type": mime_type} response = self.session.post("%s/files/" % self.url, data=json.dumps(data), headers=HEADERS) if response.status_code == requests.codes.created: f = File(self, response.json()) with open(filepath, "rb") as localfile: response = requests.put(f.upload_url, data=localfile) return f else: print(response.text)
def filetype(file): name= '' mime = MimeTypes() mime_type = mime.guess_type(file) #print(mime_type) if "application/x-zip-compressed" in mime_type: name= "zip" elif "gzip" in mime_type: name= "gzip" elif "application/x-tar" in mime_type: name= "tar" return name
def Email(subject, body, path=0): msg = Message(subject, sender="*****@*****.**", recipients=["*****@*****.**"]) msg.body = body if path!=0: mim=MimeTypes() for fn in os.listdir(path): mimty=mim.guess_type(fn) with app.open_resource(path+"\\"+fn) as fp: msg.attach(fn,mimty[0], fp.read()) shutil.rmtree(path) mail.send(msg)
def POST(self): params = {} params['nom'] = self.params.pop('title') params['community_type'] = self.params.pop('community_type') params['description'] = self.params.pop('description', None) params['image'] = self.params.pop('image', None) params['activity_view'] = self.params.pop('activity_view', None) params['twitter_hashtag'] = self.params.pop('twitter_hashtag', None) params['notify_activity_via_push'] = self.params.pop('notify_activity_via_push', None) params['notify_activity_via_push_comments_too'] = self.params.pop('notify_activity_via_push_comments_too', None) pc = api.portal.get_tool('portal_catalog') nom = safe_unicode(params['nom']) util = queryUtility(IIDNormalizer) id_normalized = util.normalize(nom, max_length=500) result = pc.unrestrictedSearchResults(portal_type='ulearn.community', id=id_normalized) imageObj = '' if params['image']: mime = MimeTypes() mime_type = mime.guess_type(params['image']) imgName = (params['image'].split('/')[-1]).decode('utf-8') imgData = requests.get(params['image']).content imageObj = NamedBlobImage(data=imgData, filename=imgName, contentType=mime_type[0]) if result: # community = result[0].getObject() success_response = 'Community already exists.' status = 200 else: new_community_id = self.context.invokeFactory('ulearn.community', id_normalized, title=params['nom'], description=params['description'], image=imageObj, community_type=params['community_type'], activity_view=params['activity_view'], twitter_hashtag=params['twitter_hashtag'], notify_activity_via_push=True if params['notify_activity_via_push'] == 'True' else None, notify_activity_via_push_comments_too=True if params['notify_activity_via_push_comments_too'] == 'True' else None, checkConstraints=False) new_community = self.context[new_community_id] success_response = 'Created community "{}" with hash "{}".'.format(new_community.absolute_url(), sha1(new_community.absolute_url()).hexdigest()) status = 201 logger.info(success_response) return ApiResponse.from_string(success_response, code=status)
def post_media(self, user_id, file_path, sent_by_maker=False): role = "appUser" if sent_by_maker: role = "appMaker" data = {"role": role} mime = MimeTypes() mime_type, _ = mime.guess_type(file_path) file_name = os.path.basename(file_path) files = {'source': (file_name, open(file_path, 'rb'), mime_type)} url = 'appusers/{0}/conversation/images'.format(user_id) return self.ask(url, data, 'post', files)
def handleFile(self, filename): fd = open(filename) content = fd.read() fd.close() self.send_response(200) mime = MimeTypes() mimetype = mime.guess_type(filename) self.send_header("Content-Type", mimetype[0]) if mimetype[1]: self.send_header("Content-Encoding", mimetype[1]) self.send_header("Content-Length", str(len(content))) self.end_headers() self.wfile.write(content) self.wfile.flush()
def view(self, request, **kwargs): ''' allow a file to be viewed as opposed to download. This is particularly needed when a video file is stored in the fileservice and user wants to be able to use a view the video as opposed to having to download it first. It passes the serving of the file to nginx/apache which will return all the proper headers allowing, say, html5's video viewer's 'seek' indicator/knob to work. Otherwise the video is only played sequentially Note that nginx/apache need to be configured accordingly. nginx for example: location /var/lib/geoserver_data/file-service-store/ { # forces requests to be authorized internal; alias /var/lib/geoserver_data/file-service-store/; } for apache, need to install xsendfile module, enable it, set the path and then XSendFile on XSendFilePath /var/lib/geoserver_data/file-service-store example use: /fileservice/view/med.mp4 or /fileservice/med.mp4/view Note that media players tend to require the route to end with the filename like /fileservice/view/med.mp4 ''' # method check to avoid bad requests self.method_check(request, allowed=['get']) # Must be done otherwise endpoint will be wide open self.is_authenticated(request) response = None file_item_name = kwargs.get('name', None) if file_item_name: mime = MimeTypes() url = urllib.pathname2url(file_item_name) mime_type = mime.guess_type(url) response = HttpResponse(content_type=mime_type[0]) file_with_route = smart_str('{}{}'.format(helpers.get_fileservice_dir(), file_item_name)) # apache header response['X-Sendfile'] = file_with_route # nginx header response['X-Accel-Redirect'] = file_with_route if not response: response = self.create_response(request, {'status': 'filename not specified'}) return response
aws_access_key_id=os.environ['PARROT_AWS_KEY_ID'], aws_secret_access_key=os.environ['PARROT_AWS_KEY_SECRET']) cf = boto3.client('cloudfront', aws_access_key_id=os.environ['PARROT_AWS_KEY_ID'], aws_secret_access_key=os.environ['PARROT_AWS_KEY_SECRET']) # Read our targets bucket = os.environ['PARROT_CDN_BUCKET'] dist = os.environ['PARROT_CDN_DIST'] print('Uploading files...') mime = MimeTypes() for filename in tqdm(glob.glob('frontend/build/**/*', recursive=True)): if not os.path.isdir(filename): # Try to fill the MIME type based on the filename. If we cannot make a # guess, then we assume it's a binary file. mime_type = mime.guess_type(filename)[0] if mime_type is None: mime_type = 'application/octet-stream' # Upload the file to the s3 bucket with read permission s3.upload_file( filename, bucket, # The new filename on s3. We want to remove the frontend/build/ # prefix from the name. filename.split('frontend/build/')[-1], ExtraArgs={ 'ACL': 'public-read', 'ContentType': mime_type, }) print('Done.')
return print('Message saved') def saveSent(Mailbox, emailObj={}): Mailbox.select_folder(IMAP_SENT_FOLDER, readonly=False) msg = createMsg(emailObj).as_string() print(Mailbox.append(IMAP_SENT_FOLDER, msg)) return print('Message saved') #=============Data for test to be deleted===== mime = MimeTypes() file = None AttachFilepath = '/Users/macbookpro15/Documents/ReactJS/QuickMail/myapi/рома.png' with open(AttachFilepath, 'rb') as fp: fileData = fp.read() fileMIME = mime.guess_type(AttachFilepath)[0] mainType = fileMIME.split('/')[0] subType = fileMIME.split('/')[1] fileSize=fp.tell() fileName=basename(AttachFilepath) fileObj = {'name': fileName, 'file':fileData , 'size':fileSize, 'MIME':fileMIME} emailObj = { 'id': 1234567, 'Date': datetime.now(), 'From':{'name':'roman.reshetnyak', 'email':'*****@*****.**'}, 'To': [{'name':'vian.vinos', 'email':'*****@*****.**'}], 'Cc': [{'name':'dima.vinos', 'email':'*****@*****.**'}], 'Subject':'На дворе вечерело', 'body':'<div><b>Hello Привет мир</b>версия 2.6</div>',
class ResponseTypes(object): CLASSES = { 'text/html': 'scrapy.http.HtmlResponse', 'application/atom+xml': 'scrapy.http.XmlResponse', 'application/rdf+xml': 'scrapy.http.XmlResponse', 'application/rss+xml': 'scrapy.http.XmlResponse', 'application/xhtml+xml': 'scrapy.http.HtmlResponse', 'application/vnd.wap.xhtml+xml': 'scrapy.http.HtmlResponse', 'application/xml': 'scrapy.http.XmlResponse', 'application/json': 'scrapy.http.TextResponse', 'application/x-json': 'scrapy.http.TextResponse', 'application/json-amazonui-streaming': 'scrapy.http.TextResponse', 'application/javascript': 'scrapy.http.TextResponse', 'application/x-javascript': 'scrapy.http.TextResponse', 'text/xml': 'scrapy.http.XmlResponse', 'text/*': 'scrapy.http.TextResponse', } def __init__(self): self.classes = {} self.mimetypes = MimeTypes() mimedata = get_data('scrapy', 'mime.types').decode('utf8') self.mimetypes.readfp(StringIO(mimedata)) for mimetype, cls in six.iteritems(self.CLASSES): self.classes[mimetype] = load_object(cls) def from_mimetype(self, mimetype): """Return the most appropriate Response class for the given mimetype""" if mimetype is None: return Response elif mimetype in self.classes: return self.classes[mimetype] else: basetype = "%s/*" % mimetype.split('/')[0] return self.classes.get(basetype, Response) def from_content_type(self, content_type, content_encoding=None): """Return the most appropriate Response class from an HTTP Content-Type header """ if content_encoding: return Response mimetype = to_native_str(content_type).split(';')[0].strip().lower() return self.from_mimetype(mimetype) def from_content_disposition(self, content_disposition): try: filename = to_native_str( content_disposition, encoding='latin-1', errors='replace').split(';')[1].split('=')[1] filename = filename.strip('"\'') return self.from_filename(filename) except IndexError: return Response def from_headers(self, headers): """Return the most appropriate Response class by looking at the HTTP headers""" cls = Response if b'Content-Type' in headers: cls = self.from_content_type( content_type=headers[b'Content-type'], content_encoding=headers.get(b'Content-Encoding')) if cls is Response and b'Content-Disposition' in headers: cls = self.from_content_disposition( headers[b'Content-Disposition']) return cls def from_filename(self, filename): """Return the most appropriate Response class from a file name""" mimetype, encoding = self.mimetypes.guess_type(filename) if mimetype and not encoding: return self.from_mimetype(mimetype) else: return Response def from_body(self, body): """Try to guess the appropriate response based on the body content. This method is a bit magic and could be improved in the future, but it's not meant to be used except for special cases where response types cannot be guess using more straightforward methods.""" chunk = body[:5000] chunk = to_bytes(chunk) if not binary_is_text(chunk): return self.from_mimetype('application/octet-stream') elif b"<html>" in chunk.lower(): return self.from_mimetype('text/html') elif b"<?xml" in chunk.lower(): return self.from_mimetype('text/xml') else: return self.from_mimetype('text') def from_args(self, headers=None, url=None, filename=None, body=None): """Guess the most appropriate Response class based on the given arguments.""" cls = Response if headers is not None: cls = self.from_headers(headers) if cls is Response and url is not None: cls = self.from_filename(url) if cls is Response and filename is not None: cls = self.from_filename(filename) if cls is Response and body is not None: cls = self.from_body(body) return cls
def _check_file_mime_type(path): mime = MimeTypes() return mime.guess_type(path)
#!/usr/bin/env python import sys import json import urllib import base64 from mimetypes import MimeTypes import os dir = os.path.dirname(__file__) if len(sys.argv) < 4: print 'Must give the node uuid, image file-name, and output file.' print sys.argv mime = MimeTypes() url = urllib.pathname2url(sys.argv[2]) mime_type = mime.guess_type(url) image_path = os.path.join(os.getcwd(), sys.argv[2]) with open(image_path, "rb") as image_file: encoded_string = base64.b64encode(image_file.read()) data = { "uuid": sys.argv[1], "file": encoded_string, "mimetype": mime_type[0], } output_path = os.path.join(os.getcwd(), sys.argv[3]) with open(output_path, 'w') as outfile: json.dump(data, outfile)
def handle_client_request(self, resp, addr): #print("Handling client request") #print(resp) try: #while True: # data = server.recv(2048) #resp = binascii.a2b_uu(resp) #resp = self.new_break_req(resp) resp = bytes(resp).decode("utf-8") #print(resp) #print(bytes(resp).decode("utf-8")) if self.debugging: print("Processing a new request for the server:") # print(data) # if not data: # break #self.lock.acquire() self.break_req(resp)# break the client request for server to understand self.checksecurity() if self.error_code != 400: self.directory = self.curr_directory + self.path try: if self.action == "GET": self.make_file_name() if os.path.exists(self.directory): flagorig = True flagnew = False else: flagorig = False if os.path.exists(self.file_directory): flagnew = True self.directory = self.file_directory else: flagnew = False if flagorig or flagnew: if self.debugging: print("Valid Path!") self.isDirectory = os.path.isdir(self.directory) if self.isDirectory: self.error_code = 200 files = os.listdir(self.directory) self.req_body = json.dumps(files) if self.debugging: print("Returning a list of the current files in the current data directory!", self.directory) print("Files returned from the server: ") print(self.req_body) else: self.isFile = os.path.isfile(self.directory) if self.isFile: if self.debugging: print("Returning the content of the file in the data directory", self.directory) file_read = open(self.directory, "r") self.req_body = file_read.read() file_read.close() else: if self.debugging: print("Path could not be found : ", self.directory) self.error_code = 404 elif self.action == "POST": pathlib.Path(os.path.dirname(self.directory)).mkdir(parents=True, exist_ok=True) self.make_file_name() if self.patheditflag: self.directory = self.file_directory file_o = open(self.directory, "w") file_o.write(self.client_body) file_o.close() self.error_code = 200 self.req_body = self.client_body self.isFile = True except OSError as err: if self.debugging: print(err) self.error_code = 400 self.req_body = "" except SystemError as err: if self.debugging: print(err) self.error_code = 400 self.req_body = "" if self.isDirectory: self.header_dict["Content-Type"] = "application/json" elif self.isFile: if self.patheditflag: self.directory = self.file_directory #print("File type") mimes_all = MimeTypes() mime_type = mimes_all.guess_type(self.directory) #print(mime_type[0]) self.header_dict["Content-Type"] = mime_type[0] resp = httplib(self.error_code, self.req_body, self.header_dict) self.response = resp.response_head() + self.req_body if self.debugging: print('Response is :\n',self.response) #print('\nrequest body is: \n' + self.req_body) return self.response #server.sendall(response.encode("ascii")) #self.makepacket.makepackets(response, addr) #TODO: check what needs to be passed to send packets back to client #self.error_code = 200 #self.header_dict = {} #self.lock.release() finally: print("")
def sendFile(self): #If we don't need to send files, return if self.mRequestQueue.empty(): return #Get some values request = self.mRequestQueue.get() filepath = request.getURL() connection = "close" if request.containsKey("Connection"): connection = request.getParam("Connection") #Check if file exists if not os.path.isfile(filepath): print filepath self.sendNotFound() if connection.lower() == "close": self.mShouldStop = True return try: #Open file file = open(filepath, "rb") size = os.path.getsize(filepath) #Get mime type mime = MimeTypes() mime_type = mime.guess_type(filepath) if file.closed: self.sendNotFound() if connection.lower() == "close": self.mShouldStop = True return #Send header self.mSocket.send("HTTP/1.1 200 OK\r\nContent-Length: " + str(size) + "\r\nContent-Type: " + str(mime_type[0]) + "\r\n" + "Connection: " + connection + "\r\n\r\n") #Init bufferData = file.read(5024) self.mSocket.send(bufferData) #Keep sending as long as we have data while bufferData: bufferData = file.read(5024) if not bufferData: break self.mSocket.send(bufferData) #Do a non blocking read to support more than one get request per socket self.mSocket.setblocking(0) try: data = self.mSocket.recv(1024) #If we have data, try to parse requests if data: self.mBuffer = self.mBuffer + data self.parseReceive() except: data = False #Enable blocking again self.mSocket.setblocking(1) file.close() if connection.lower() == "close": print "Shutting down connection because of connection: close" self.mSocket.close() self.mShouldStop = True except: self.sendBadRequest() print "Exception in send." #Work off the files recursive because we check for more requests in the send loop self.sendFile()
class Manager(Queue, EncoderListener): def __init__(self): super().__init__() self._listeners = None self._files = None self._encoder = None self._mime = None self._parameters = settings.parameters self._max_cores = utils.cpu_count() self._encoder = None self._lock = Lock() self._working = False def add_file(self, path: Path): #print(path.absolute()) if not self._files: self._files = [] if path.is_file() and path not in self._files and self._is_supported( path): print('supported') self._files.append(path) for listener in self._listeners: listener.on_file_add(path, self._files.__len__()) def remove_file(self, path: Path): if self._files is None: return index = -1 for i, p in self._files: if p == path: index = i break if index != -1: self.remove_file_at(index) def remove_file_at(self, index: int): path = self._files.pop(index) for listener in self._listeners: listener.on_file_remove(path, self._files.__len__()) def add_listener(self, listener: ManagerListener): if not self._listeners: self._listeners = set() self._listeners.add(listener) def get_encoder(self) -> Encoder: return self._encoder def set_encoder(self, encoder: Encoder): self._encoder = encoder def get_parameters(self): return self._parameters def start_job(self): print('start job') with self._lock: if not self._files: print('Nothing to convert') return if not self._working: if self._encoder is None: self._encoder = EncoderManager(self, self._parameters, self, utils.cpu_count()) self._encoder.do_job() self._working = True def stop_job(self): print('stop job') with self._lock: if self._working: self._encoder.abort() self._working = False def _is_supported(self, path: Path) -> bool: if not self._mime: from mimetypes import MimeTypes self._mime = MimeTypes() mime_type = self._mime.guess_type(str(path.absolute())) return mime_type[0] in self.supported_mime_types @property def supported_mime_types(self): return mime_white_list def _get_files(self) -> list: return self._files def on_job_done(self): for listener in self._listeners: listener.on_encoding_finish() def on_file_encode(self, path: Path, success: bool): for listener in self._listeners: listener.on_file_encode(path, success, len(self._files))
def do_GET(self): print("************************************************") mime = MimeTypes() mimeType = mime.guess_type(self.path) print("Request aayi") urlParseData = urlparse(self.path) print("URLPARSEDATA**** ", urlParseData) urlPath = urlParseData[2] query = urlParseData[4] print("QUERY *****", query) if query != '': query = parse_qs(query) print(query) print("name ", query["nm"][0]) newPath = '' if urlPath == '/' or urlPath == '/favicon.ico': newPath = './default/index.html' print("default wale iff me agya") else: path = urlPath path = path.split("/") print(path[1]) if path[1] in siteNames: if len(path) == 2: print('./apps/' + path[1] + '/index.html') if os.path.isfile('./apps/' + path[1] + '/index.html'): print('index.html found') # urlPath='./apps/'+path[1]+'/index.html' newPath = urlPath + '/index.html' else: print('index.html not found') if os.path.isfile('./apps/' + path[1] + '/index.htm'): print('index.htm found') #urlPath='./apps/'+path[1]+'/index.htm' newPath = urlPath + '/index.html' else: print('index.htm not found') if os.path.isfile('./apps/' + path[1] + '/index.py'): print("index.py found") # urlPath='./apps/'+path[1]+'/index.py' newPath = urlPath + '/index.html' else: self.send_error(404, "home page not found") return self.send_response(301) self.send_header('Location', newPath) self.end_headers() return else: if path[2] == 'private': print("Kuch nahi ho sakta") self.wfile.write(bytes("404 NOT FOUND", "utf-8")) else: a = self.path.find("/") print("index ", a) abcd = urlPath[urlPath.find("/", a + 1):] print("aaaaabcdcdbvava", abcd) mapping = siteNames[path[1]].getMapping() if abcd in mapping: print("py exitssssssssssssssssss karti h") a = mapping[abcd].rfind(".") if a != -1: fileName = mapping[abcd][a + 1:] print(fileName) packageName = mapping[abcd][:a] print(packageName) finalpath = 'apps.' + path[ 1] + ".private." + packageName + "." + fileName moduleName = __import__(finalpath, fromlist=fileName) #request=Request self.send_response(200) request = Request(query) response = Response(self, self.wfile, 'text/html') moduleName.process(request, response) #self.send_header('Content-type',response.getContentType()) #self.end_headers() return else: finalpath = 'apps.' + path[ 1] + ".private." + mapping[abcd] moduleName = __import__(finalpath, fromlist=mapping[abcd]) self.send_response(200) request = Request(query) response = Response(self, self.wfile, 'text/html') moduleName.process(request, response) #self.send_header('Content-type',response.getContentType()) #self.end_headers() return else: newPath = './apps' + urlPath #self.path[self.path.find("/"):] mimeType = mime.guess_type(newPath) else: self.send_error(404, '%s Not Found' % (path[1])) try: #mimeType=mime.guess_type(newPath) print(mimeType) print(newPath) f = open(newPath, 'rb') self.send_response(200) self.send_header('Content-type', mimeType) self.end_headers() self.wfile.write(f.read()) f.close() except IOError: print("File does not exist") self.send_error(404, 'File Not Found: %s' % urlPath) # print(urlPath) return
class Sample: """Represents a document analysis. Uses textract to read document into a long string. The methods are various sequences to get information to help make decisions for deliberate academic writing. """ def __init__(self, path): """ Create document instance for analysis. Opens and reads document to string raw_text. Textract interprets the document format and opens to plain text string (docx, pdf, odt, txt) Args: path (str): path to file to open, anaylze, close Public attributes: -user: (str) optional string to set username. -path: (str) relative path to document. -abs_path: (str) the absolute path to the document. -file_name: (str) the file name with extension of document (base name). -mime: tbd -guessed_type: makes best guess of mimetype of document. -file_type: returns index[0] from guessed_type. -raw_text: (str) plain text extracted from .txt, .odt, .pdf, .docx, and .doc. -ptext: (str) raw text after a series of regex expressions to eliminate special characters. -text_no_feed: (str) ptext with most new line characters eliminated /n/n stays intact. -sentence_tokens: list of all sentences in a comma separated list derived by nltk. -sentence_count: (int) count of sentences found in list. -passive_sentences: list of passive sentences identified by the passive module. -passive_sentence_count: count of the passive_sentences list. -percent_passive: (float) ratio of passive sentences to all sentences in percent form. -be_verb_analysis: (int) sum number of occurrences of each to be verb (am, is, are, was, were, be, being been). -be_verb_count: tbd -be_verb_analysis: tbd -weak_sentences_all: (int) sum of be verb analysis. -weak_sentences_set: (set) set of all sentences identified as having to be verbs. -weak_sentences_count: (int) count of items in weak_sentences_set. -weak_verbs_to_sentences: (float) proportion of sentences with to be to all sentences in percent (this might not be sound). -word_tokens: list of discreet words in text that breaks contractions up (default nltk tokenizer). -word_tokens_no_punct: list of all words in text including contractions but otherwise no punctuation. -no_punct: (str) full text string without sentence punctuation. -word_tokens_no_punct: uses white-space tokenizer to create a list of all words. readability_flesch_re: (int) Flesch Reading Ease Score (numeric score) made by textstat module. readability_smog_index: (int) grade level as determined by the SMOG algorithum made by textstat module. readability_flesch_kincaid_grade: (int) Flesch-Kincaid grade level of reader made by textstat module. readability_coleman_liau_index: (int) grade level of reader as made by textstat module. readability_ari: (int) grade leader of reader determined by automated readability index algorithum implemented by textstat. readability_linser_write: FIX SPELLING grade level as determined by Linsear Write algorithum implemented by textstat. readability_dale_chall: (int) grade level based on Dale-Chall readability as determined by textstat. readability_standard: composite grade level based on readability algorithums. -flesch_re_key: list for interpreting Flesch RE Score. -word_count: word count of document based on white space tokener, this word count should be used. -page_length: (float) page length in decimal format given 250 words per page. -paper_count: (int) number of printed pages given 250 words per page. -parts_of_speech: words with parts of speech tags. -pos_counts: values in word, tag couple grouped in a list. -pos_total: (int) sum of pos_counts values -pos_freq: (dict) word, ratio of whole -doc_pages: (float) page length based on 250 words per page (warning, this is the second time this attribute is defined). -freq_words: word frequency count not standardized based on the correct word tokener (not ratio, just count). modal_dist: count of auxillary verbs based on word_tokens_no_punct. sentence_count (int): Count the sentence tokens passive_sentences (list): List of all sentences identified as passive passive_sentence_count (int): count of items in passive_sentences be_verb_count (int): count "to be" verbs in text word_tokens_no_punct (list): words separated, stripped of punctuation, made lower case flesch_re_key (str): reading ease score to description freq_words (list or dict): frequency distribution of all words modal_dist (list): frequency distribution of aux verbs """ self.user = "" self.path = path self.abs_path = os.path.abspath(self.path) if os.path.isfile(self.path): self.file_name = os.path.basename(path) self.mime = MimeTypes() self.guessed_type = self.mime.guess_type(self.path) self.file_type = self.guessed_type[0] self.raw_text = textract.process(self.path) self.ptext = re.sub(u'[\u201c\u201d]', '"', self.raw_text) self.ptext = re.sub(u"\u2014", "--", self.ptext) self.ptext = re.sub(",", ",", self.ptext) self.ptext = re.sub("—", "--", self.ptext) self.ptext = re.sub("…", "...", self.ptext) self.text_no_feed = self.clean_new_lines(self.ptext) self.sentence_tokens = self.sentence_tokenize(self.text_no_feed) self.sentence_count = len(self.sentence_tokens) self.passive_sentences = passive(self.text_no_feed) self.passive_sentence_count = len(self.passive_sentences) self.percent_passive = (100 * (float(self.passive_sentence_count) / float(self.sentence_count))) self.be_verb_analysis = self.count_be_verbs(self.sentence_tokens) self.be_verb_count = self.be_verb_analysis[0] self.weak_sentences_all = self.be_verb_analysis[1] self.weak_sentences_set = set(self.weak_sentences_all) self.weak_sentences_count = len(self.weak_sentences_set) self.weak_verbs_to_sentences = 100 * float(self.weak_sentences_count) / float(self.sentence_count) self.word_tokens = self.word_tokenize(self.text_no_feed) self.word_tokens_no_punct = \ self.word_tokenize_no_punct(self.text_no_feed) self.no_punct = self.strip_punctuation(self.text_no_feed) # use this! It make lower and strips symbols self.word_tokens_no_punct = self.ws_tokenize(self.no_punct) self.readability_flesch_re = \ textstat.flesch_reading_ease(self.text_no_feed) self.readability_smog_index = \ textstat.smog_index(self.text_no_feed) self.readability_flesch_kincaid_grade = \ textstat.flesch_kincaid_grade(self.text_no_feed) self.readability_coleman_liau_index = \ textstat.coleman_liau_index(self.text_no_feed) self.readability_ari = \ textstat.automated_readability_index(self.text_no_feed) self.readability_linser_write = \ textstat.linsear_write_formula(self.text_no_feed) self.readability_dale_chall = \ textstat.dale_chall_readability_score(self.text_no_feed) self.readability_standard = \ textstat.text_standard(self.text_no_feed) self.flesch_re_key = ( "* 90-100 : Very Easy", "* 80-89 : Easy", "* 70-79 : Fairly Easy", "* 60-69 : Standard", "* 50-59 : Fairly Difficult", "* 30-49 : Difficult", "* 0-29 : Very Confusing" ) if self.word_tokens_no_punct: self.word_count = len(self.word_tokens_no_punct) self.page_length = float(self.word_count)/float(250) self.paper_count = int(math.ceil(self.page_length)) self.parts_of_speech = pos_tag(self.word_tokens_no_punct) self.pos_counts = Counter(tag for word, tag in self.parts_of_speech) self.pos_total = sum(self.pos_counts.values()) self.pos_freq = dict((word, float(count)/self.pos_total) for word, count in self.pos_counts.items()) self.doc_pages = float(float(self.word_count)/float(250)) self.freq_words = \ self.word_frequency(self.word_tokens_no_punct) self.modal_dist = self.modal_count(self.word_tokens_no_punct) # self.ws_tokens = self.ws_tokenize(self.text_no_cr) def strip_punctuation(self, string_in): """ Strip punctuation from string and make lower case. Given a string of sentences, translate string to remove some common symbols and conver caps to lower case. Args: string_in (str): Text to strip punctuation from return: str """ string_in = string_in.translate(None, ',.!?\"<>{}[]--@()\'--') return str(string_in.lower()) def ws_tokenize(self, text): """ Given string of words, return word tokens with contractions OK. Other tokenizers tokenize punctuation. The WhitespaceTokenizer is important because of contractions. Args: text (str) returns: list """ self.tokenizer = nltk.tokenize.regexp.WhitespaceTokenizer() return self.tokenizer.tokenize(text) def syllables_per_word(self, text): """ Return count of syllables per word. Loops through all words to add word and syllable count to a list. Args: text (str) Returns: list """ self.word_syllables = [] for word in text: self.word_syllables.append([word, textstat.textstat.syllable_count( word)]) return self.word_syllables def polysyllables(self, text): """ Count polysyllables. Count words in text string that have >= 3 syllables. Args: text(str) Returns: int: polysllable word count in text arg """ return textstat.textstat.polysyllabcount(text) def word_frequency(self, words): """ List 50 most common words in tokenized list. memo: words = [word for word in words if not word.isnumeric()]. Args: text(str) Returns: list """ words = [word.lower() for word in words] self.word_dist = FreqDist(words) return self.word_dist.most_common(50) def word_tokenize_no_punct(self, text): """ Make list of words without listing punctuation. Args: text (str): Plain text string Returns: list of words """ tokenizer = RegexpTokenizer(r'\w+') return tokenizer.tokenize(text) def word_tokenize(self, paragraph): """ Tokenize words from long string that includes sentences. Uses default tokenizer from nltk. Args: paragraph (str) Return: list """ try: self.word_tokens = tokenize.word_tokenize(paragraph) return self.word_tokens except: print("Error: Cannot perform word analyses.") return False def sentence_tokenize(self, paragraphs): """ Tokenize sentences. Uses default sent tokenizer. Args: paragraph (str) Returns: list """ try: self.sentences = tokenize.sent_tokenize(paragraphs) return self.sentences except: print "Could not tokenize text." return False def clean_new_lines(self, paragraphs): """Strip new line characters except for new paragraphs.""" self.text_no_cr =\ paragraphs.replace("\n\n", "TOADIES").replace("\r", "").replace("\n", "").replace( "TOADIES", "\n") return self.text_no_cr def count_be_verbs(self, sentences): """ Count be verbs in each sentence in a list. Loop through sentences to provide weak verb count. If count >= 1, add sentence to list. Args: sentences (str, list) Return: list of be-verb count and stand-out sentences """ self.verbs = [" am ", " is ", " are ", " was ", " were ", " be ",\ "being ", " been "] self.weak_sentences = [] self.verb_count = 0 for sentence in sentences: for verb in self.verbs: if verb in sentence: self.verb_count = self.verb_count + 1 self.weak_sentences.append(sentence) return [self.verb_count, self.weak_sentences] def syllable_count(self, word): """ Count syllables in a word. Uses NLTK dictionary to find word syllabication. Args: word (string) Returns: int syllable count """ self.d = cmudict.dict() return min([len([y for y in x if isdigit(y[-1])]) for x in self.d[str(word).lower()]]) def modal_count(self, text): """ Return FreqDist of modal verbs in text. Args: text (str) Return: list """ fdist = FreqDist(w.lower() for w in text) modals = ['can', 'could', 'shall', 'should', 'will', 'would', 'do', 'does', 'did', 'may', 'might', 'must', 'has', 'have', 'had'] modals_freq = [] for m in modals: modals_freq.append(str(m + ': ' + str(fdist[m]))) return modals_freq
def populateFileInfo(self, scanOption): '''Runs the two scanners and parses their output into the fileInfo object''' ''' Get File Type''' mime = MimeTypes() self.fileType = mime.guess_type(self.filePath)[0] if self.fileType == None: self.fileType = 'Unknown' '''Check to see if file is cached.''' cached = self.isCached() '''If it isn't cached, run scans, else get file from database.''' if cached == -1: if scanOption == 'fossology': '''Run fossology''' '''Fossology doesn't return an exit code of 0 so we must always catch the output.''' try: fossOutput = subprocess.check_output( [settings.FOSSOLOGY_PATH, self.filePath]) except OSError as ose: print "Error running FOSSology nomos, check your path to nomos in settings.py" except Exception as e: fossOutput = str(e.output) '''Parse outputs''' (fileName, fossLicense) = output_parser.foss_parser(fossOutput) self.licenseInfoInFile.append(fossLicense) self.licenseComments = "#FOSSology " self.licenseComments += fossLicense else: '''Scan to find licenses''' '''Run Ninka''' ninkaOutput = subprocess.check_output( [settings.NINKA_PATH, self.filePath], preexec_fn=lambda: signal(SIGPIPE, SIG_DFL)) '''Run fossology''' '''Fossology doesn't return an exit code of 0 so we must always catch the output.''' try: fossOutput = subprocess.check_output( [settings.FOSSOLOGY_PATH, self.filePath]) except OSError as ose: print "Error running FOSSology nomos, check your path to nomos in settings.py" except Exception as e: fossOutput = str(e.output) '''Parse outputs''' (fileName, ninkaLicense) = output_parser.ninka_parser(ninkaOutput) (fileName, fossLicense) = output_parser.foss_parser(fossOutput) '''Get extracted text from ninka "senttok" file''' try: with open(self.filePath + ".senttok", 'r') as f: for line in f: if ninkaLicense in line: line_tok = line.split(';') self.extractedText += line_tok[3] + "\n" self.extractedText += line_tok[4] except Exception as e: '''Do nothing, we just wont have extracted text for this license.''' '''License merging logic.''' fossLicense = fossLicense.upper().strip() ninkaLicense = ninkaLicense.upper().strip() match = output_parser.lic_compare(fossLicense, ninkaLicense) if match and fossLicense != 'ERROR': self.licenseInfoInFile.append(fossLicense) elif match and fossLicense == 'ERROR': self.licenseInfoInFile.append(ninkaLicense) elif not match and fossLicense == 'UNKNOWN': self.licenseInfoInFile.append(ninkaLicense) else: self.licenseInfoInFile.append("NO ASSERTION") self.licenseComments = "#FOSSology " self.licenseComments += fossLicense self.licenseComments += " #Ninka " self.licenseComments += ninkaLicense else: with MySQLdb.connect(host=settings.database_host, user=settings.database_user, passwd=settings.database_pass, db=settings.database_name) as dbCursor: self.getChecksum() self.getFileInfoFromChecksum(dbCursor)
def get_image(): image_path = request.args.get('image_path') mime = MimeTypes() mimetype, _ = mime.guess_type(image_path) return send_file(os.path.join('..', image_path), mimetype=mimetype)
def create_wagtail_image(img_src: str, image_name: str = None, collection_name: str = None) -> Union[None, Image]: """ Create a Wagtail Image from a given source. It takes an optional file name and collection name. If the collection name is provided, but a collection is not found, a new collection will be created. Examples: create_wagtail_image('/app/source/images/myimage.jpg') create_wagtail_image('/app/source/images/myimage.jpg', image_name='Same Image.jpg') create_wagtail_image('/app/source/images/myimage.jpg', collection_name='Dev test collection') """ mime = MimeTypes() mime_type = mime.guess_type(img_src) if mime_type: mime_type = mime_type[0].split('/')[1].upper() else: # Default to a JPEG mimetype. mime_type = 'JPEG' f = BytesIO() # Copy the image to the local machine before converting it to a Wagtail image. if img_src.startswith("http"): # Download the image from a URL. Requires the requests package. response = requests.get(img_src, stream=True) if response.status_code == 200: # Create an image out of the Cloudinary URL and write it to a PIL Image. pil_image = PILImage.open(response.raw) pil_image.save(f, mime_type) else: # Image URL didn't 200 for us. Nothing we can do about that. Return early. print(f"Could not generate image from url {img_src}") return else: # Save the image from a local source. The requests package is not needed. pil_image = PILImage.open(img_src) pil_image.save(f, mime_type) # If the image is supposed to be part of a collection, look for the collection or create it. collection = Collection.get_first_root_node() if collection_name: specific_collection = Collection.objects.filter( name=collection_name).first() # Use the specific collection if it's found. Otherwise create a new collection # based on the `collection_name` parameter. collection = specific_collection if specific_collection else collection.add_child( name=collection_name) # If an image name was not provided, create one from the img_src.. if not image_name: image_name = ntpath.basename(img_src) # Create the Wagtail Image and return it wagtail_image = Image.objects.create(title=image_name, file=ImageFile(f, name=image_name), collection=collection) return wagtail_image
f_path = os.path.dirname(sys.argv[1]) if f_path != '': f_path = f_path + '/' soup = BeautifulSoup(f, 'html.parser') # encode favicon, styles, images, videos, audios, and scripts for tag, attr in tags.iteritems(): encode(tag, attr) # encode background images styled_tags = soup.find_all(style=True) for tag in styled_tags: if 'background-image' in tag['style']: b_img_url = re.findall(r'background-image:url\((.*?)\);', tag['style'])[0] mimetype = mime.guess_type(b_img_url) with open(f_path + b_img_url, 'rb') as b_img_file: encoded_uri = 'data:' + mimetype[ 0] + ';base64,' + base64.b64encode(b_img_file.read()) tag['style'] = tag['style'].replace(b_img_url, encoded_uri) # encode fonts (they need to be included in an internal CSS style with the "fonts" ID) font_css = soup.find('style', {'id': 'fonts'}) if font_css: fonts_url = re.findall(r'url\((.*?)\)', font_css.contents[0]) for font_url in fonts_url: mimetype = mime.guess_type(font_url) with open(f_path + font_url, 'rb') as font_file: encoded_uri = 'data:' + mimetype[ 0] + ';base64,' + base64.b64encode(font_file.read()) font_css.string = font_css.contents[0].replace(font_url, encoded_uri)
def EscanearArquivo(self, event): processo = ValidadorCodigoBarras() scanner = zbar.Scanner() try: codigos_barras_arquivo = [] temp_url = urllib.pathname2url(event.src_path) mimetypes = MimeTypes() mimetype_file = mimetypes.guess_type(temp_url) del(mimetypes) if mimetype_file[0] != 'application/pdf': EnviarEmailERRO('Formato de arquivo recebido nao reconhecido ou nao aceito. Arquivo: ' + event.src_path + ' - Mimetype: ' + mimetype_file[0]) return arquivo_ftp = event.src_path arquivo_pdf_temp = re.sub('(?i)'+re.escape(diretorio_ftp), lambda m: diretorio_temp, arquivo_ftp) pdfWriter = PyPDF2.PdfFileWriter() pdfFileObj = open(arquivo_ftp, 'rb') pdfReader = PyPDF2.PdfFileReader(pdfFileObj) pageObj = pdfReader.getPage(0) pdfWriter.addPage(pageObj) with open(arquivo_pdf_temp, 'wb') as output: pdfWriter.write(output) pdfFileObj.close() del(pdfWriter) del(pdfFileObj) del(pdfReader) arquivo_imagem = re.sub('(?i)'+re.escape('.pdf'), lambda m: '.jpg', arquivo_pdf_temp) imagem_temp = Image(filename=arquivo_pdf_temp, resolution=(400,400)) imagem_temp = imagem_temp.convert('jpeg') imagem_temp.save(filename=arquivo_imagem) del(imagem_temp) processo.nome_arquivo_pdf = arquivo_ftp processo.nome_arquivo_thumbnail = arquivo_imagem # colorido #print(arquivo_ftp + ' - colorido - sem efeito') codigos_barras = [] imagem_temp = PI.open(arquivo_imagem).convert('L') imagem_array_temp = np.array(imagem_temp) del(imagem_temp) codigo_barras_temp = scanner.scan(imagem_array_temp) del(imagem_array_temp) for codigo_barras in codigo_barras_temp: if codigo_barras.type == 'QR-Code': continue codigos_barras.append(codigo_barras.data) if len(codigos_barras) > 0: for codigo_barras in codigos_barras: processo.codigo_barras = codigo_barras if processo.Processar(True): logger.info(arquivo_ftp + ' - colorido - sem efeito - ' + codigo_barras) os.remove(arquivo_ftp) os.remove(arquivo_pdf_temp) os.remove(arquivo_imagem) return else: if codigo_barras not in codigos_barras_arquivo: codigos_barras_arquivo.append(codigo_barras) #print(arquivo_ftp + ' - colorido - gaussian blur') codigos_barras = [] imagem_temp = cv2.imread(arquivo_imagem) imagem_temp = cv2.GaussianBlur(imagem_temp, (5, 5), 0) imagem_temp = PI.fromarray(imagem_temp).convert('L') imagem_temp = np.array(imagem_temp) codigo_barras_temp = scanner.scan(imagem_temp) for codigo_barras in codigo_barras_temp: if codigo_barras.type == 'QR-Code': continue codigos_barras.append(codigo_barras.data) del(imagem_temp) if len(codigos_barras) > 0: for codigo_barras in codigos_barras: processo.codigo_barras = codigo_barras if processo.Processar(True): logger.info(arquivo_ftp + ' - colorido - gaussian blur - ' + codigo_barras) os.remove(arquivo_ftp) os.remove(arquivo_pdf_temp) os.remove(arquivo_imagem) return else: if codigo_barras not in codigos_barras_arquivo: codigos_barras_arquivo.append(codigo_barras) #print(arquivo_ftp + ' - colorido - median blur') codigos_barras = [] imagem_temp = cv2.imread(arquivo_imagem) imagem_temp = cv2.medianBlur(imagem_temp, 5) imagem_temp = PI.fromarray(imagem_temp).convert('L') imagem_temp = np.array(imagem_temp) codigo_barras_temp = scanner.scan(imagem_temp) for codigo_barras in codigo_barras_temp: if codigo_barras.type == 'QR-Code': continue codigos_barras.append(codigo_barras.data) del(imagem_temp) if len(codigos_barras) > 0: for codigo_barras in codigos_barras: processo.codigo_barras = codigo_barras if processo.Processar(True): logger.info(arquivo_ftp + ' - colorido - median blur - ' + codigo_barras) os.remove(arquivo_ftp) os.remove(arquivo_pdf_temp) os.remove(arquivo_imagem) return else: if codigo_barras not in codigos_barras_arquivo: codigos_barras_arquivo.append(codigo_barras) for angle in range(5, 90, 5): #print(arquivo_ftp + ' - colorido - angulo - ' + str(angle)) codigos_barras = [] imagem_temp = cv2.imread(arquivo_imagem) (oldY,oldX) = imagem_temp.shape[:-1] rotation_matrix = cv2.getRotationMatrix2D((oldX/2,oldY/2),angle,1) (newX,newY) = (oldX*1,oldY*1) r = np.deg2rad(angle) (newX,newY) = (abs(np.sin(r)*newY)+abs(np.cos(r)*newX),abs(np.sin(r)*newX)+abs(np.cos(r)*newY)) (tx,ty) = ((newX-oldX)/2,(newY-oldY)/2) rotation_matrix[0,2] += tx rotation_matrix[1,2] += ty imagem_temp = cv2.warpAffine(imagem_temp, rotation_matrix, dsize=(int(newX),int(newY)), flags=cv2.INTER_LINEAR) imagem_temp = PI.fromarray(imagem_temp).convert('L') imagem_temp = np.array(imagem_temp) codigo_barras_temp = scanner.scan(imagem_temp) for codigo_barras in codigo_barras_temp: if codigo_barras.type == 'QR-Code': continue codigos_barras.append(codigo_barras.data) del(imagem_temp) if len(codigos_barras) > 0: for codigo_barras in codigos_barras: processo.codigo_barras = codigo_barras if processo.Processar(True): logger.info(arquivo_ftp + ' - colorido - angulo - ' + angle + ' - ' + codigo_barras) os.remove(arquivo_ftp) os.remove(arquivo_pdf_temp) os.remove(arquivo_imagem) return else: if codigo_barras not in codigos_barras_arquivo: codigos_barras_arquivo.append(codigo_barras) # preto e branco #print(arquivo_ftp + ' - preto e branco - sem efeito') codigos_barras = [] imagem_temp = cv2.imread(arquivo_imagem) imagem_temp = cv2.cvtColor(imagem_temp, cv2.COLOR_RGB2GRAY) cv2.imwrite(arquivo_imagem, imagem_temp) imagem_temp = PI.open(arquivo_imagem).convert('L') imagem_temp = np.array(imagem_temp) codigo_barras_temp = scanner.scan(imagem_temp) for codigo_barras in codigo_barras_temp: if codigo_barras.type == 'QR-Code': continue codigos_barras.append(codigo_barras.data) del(imagem_temp) if len(codigos_barras) > 0: for codigo_barras in codigos_barras: processo.codigo_barras = codigo_barras if processo.Processar(True): logger.info(arquivo_ftp + ' - preto e branco - sem efeito - ' + codigo_barras) os.remove(arquivo_ftp) os.remove(arquivo_pdf_temp) os.remove(arquivo_imagem) return else: if codigo_barras not in codigos_barras_arquivo: codigos_barras_arquivo.append(codigo_barras) #print(arquivo_ftp + ' - preto e branco - gaussian blur') codigos_barras = [] imagem_temp = cv2.imread(arquivo_imagem) imagem_temp = cv2.GaussianBlur(imagem_temp, (5, 5), 0) imagem_temp = PI.fromarray(imagem_temp).convert('L') imagem_temp = np.array(imagem_temp) codigo_barras_temp = scanner.scan(imagem_temp) for codigo_barras in codigo_barras_temp: if codigo_barras.type == 'QR-Code': continue codigos_barras.append(codigo_barras.data) del(imagem_temp) if len(codigos_barras) > 0: for codigo_barras in codigos_barras: processo.codigo_barras = codigo_barras if processo.Processar(True): logger.info(arquivo_ftp + ' - preto e branco - gaussian blur - ' + codigo_barras) os.remove(arquivo_ftp) os.remove(arquivo_pdf_temp) os.remove(arquivo_imagem) return else: if codigo_barras not in codigos_barras_arquivo: codigos_barras_arquivo.append(codigo_barras) #print(arquivo_ftp + ' - preto e branco - median blur') codigos_barras = [] imagem_temp = cv2.imread(arquivo_imagem) imagem_temp = cv2.medianBlur(imagem_temp, 5) imagem_temp = PI.fromarray(imagem_temp).convert('L') imagem_temp = np.array(imagem_temp) codigo_barras_temp = scanner.scan(imagem_temp) for codigo_barras in codigo_barras_temp: if codigo_barras.type == 'QR-Code': continue codigos_barras.append(codigo_barras.data) del(imagem_temp) if len(codigos_barras) > 0: for codigo_barras in codigos_barras: processo.codigo_barras = codigo_barras if processo.Processar(True): logger.info(arquivo_ftp + ' - preto e branco - median blur - ' + codigo_barras) os.remove(arquivo_ftp) os.remove(arquivo_pdf_temp) os.remove(arquivo_imagem) return else: if codigo_barras not in codigos_barras_arquivo: codigos_barras_arquivo.append(codigo_barras) for angle in range(5, 90, 5): #print(arquivo_ftp + ' - preto e branco - angulo - ' + str(angle)) codigos_barras = [] imagem_temp = cv2.imread(arquivo_imagem) (oldY,oldX) = imagem_temp.shape[:-1] rotation_matrix = cv2.getRotationMatrix2D((oldX/2,oldY/2),angle,1) (newX,newY) = (oldX*1,oldY*1) r = np.deg2rad(angle) (newX,newY) = (abs(np.sin(r)*newY)+abs(np.cos(r)*newX),abs(np.sin(r)*newX)+abs(np.cos(r)*newY)) (tx,ty) = ((newX-oldX)/2,(newY-oldY)/2) rotation_matrix[0,2] += tx rotation_matrix[1,2] += ty imagem_temp = cv2.warpAffine(imagem_temp, rotation_matrix, dsize=(int(newX),int(newY)), flags=cv2.INTER_LINEAR) imagem_temp = PI.fromarray(imagem_temp).convert('L') imagem_temp = np.array(imagem_temp) codigo_barras_temp = scanner.scan(imagem_temp) for codigo_barras in codigo_barras_temp: if codigo_barras.type == 'QR-Code': continue codigos_barras.append(codigo_barras.data) del(imagem_temp) if len(codigos_barras) > 0: for codigo_barras in codigos_barras: processo.codigo_barras = codigo_barras if processo.Processar(True): logger.info(arquivo_ftp + ' - preto e branco - angulo - ' + angle + ' - ' + codigo_barras) os.remove(arquivo_ftp) os.remove(arquivo_pdf_temp) os.remove(arquivo_imagem) return else: if codigo_barras not in codigos_barras_arquivo: codigos_barras_arquivo.append(codigo_barras) imagem_temp = Image(filename=arquivo_pdf_temp, resolution=(400,400)) imagem_temp.convert('jpeg') imagem_temp.save(filename=arquivo_imagem) del(imagem_temp) processo.codigo_barras = '#-#'.join(str(codigo_barras) for codigo_barras in codigos_barras_arquivo) if processo.Processar(False): logger.info(arquivo_ftp + ' - ultimo recurso - ' + processo.codigo_barras) os.remove(arquivo_ftp) os.remove(arquivo_pdf_temp) os.remove(arquivo_imagem) return finally: if isinstance(scanner,zbar.Scanner): del(scanner) if isinstance(processo,ValidadorCodigoBarras): del(processo)
def run(self, path): if path == "/": f = open(os.path.join(config.get_runtime_path(), "platformcode", "template", "page.html"), "rb") self.handler.send_response(200) self.handler.send_header('Content-type', 'text/html') self.handler.end_headers() respuesta = f.read() self.handler.wfile.write(respuesta) f.close() elif path.startswith("/local/"): import base64 if PY3: import urllib.parse as urllib else: import urllib Path = path.replace("/local/", "").split("/")[0] Path = base64.b64decode(urllib.unquote_plus(Path)).decode() Size = int(os.path.getsize(Path)) f = open(Path, "rb") if not self.handler.headers.get("range") == None: if "=" in str(self.handler.headers.get("range")) and "-" in str(self.handler.headers.get("range")): Inicio = int(self.handler.headers.get("range").split("=")[1].split("-")[0]) if self.handler.headers.get("range").split("=")[1].split("-")[1] != "": Fin = int(self.handler.headers.get("range").split("=")[1].split("-")[1]) else: Fin = Size - 1 else: Inicio = 0 Fin = Size - 1 if not Fin > Inicio: Fin = Size - 1 if self.handler.headers.get("range") == None: logger.info("-------------------------------------------------------") logger.info("Solicitando archivo local: " + Path) logger.info("-------------------------------------------------------") self.handler.send_response(200) self.handler.send_header("Content-Disposition", "attachment; filename=video.mp4") self.handler.send_header('Accept-Ranges', 'bytes') self.handler.send_header('Content-Length', str(Size)) self.handler.send_header("Connection", "close") self.handler.end_headers() while True: time.sleep(0.2) buffer = f.read(1024 * 250) if not buffer: break self.handler.wfile.write(buffer) self.handler.wfile.close() f.close() else: logger.info("-------------------------------------------------------") logger.info("Solicitando archivo local: " + Path) logger.info("Rango: " + str(Inicio) + "-" + str(Fin) + "/" + str(Size)) logger.info("-------------------------------------------------------") f.seek(Inicio) self.handler.send_response(206) self.handler.send_header("Content-Disposition", "attachment; filename=video.mp4") self.handler.send_header('Accept-Ranges', 'bytes') self.handler.send_header('Content-Length', str(Fin - Inicio)) self.handler.send_header('Content-Range', str(Inicio) + "-" + str(Fin) + "/" + str(Size)) self.handler.send_header("Connection", "close") self.handler.end_headers() while True: time.sleep(0.2) buffer = f.read(1024 * 250) if not buffer: break self.handler.wfile.write(buffer) self.handler.wfile.close() f.close() elif path.startswith("/media/"): file = os.path.join(config.get_runtime_path(), "platformcode", "template", path[7:]) from mimetypes import MimeTypes mime = MimeTypes() mime_type = mime.guess_type(file) try: mim = mime_type[0] except: mim = "" f = open(file, "rb") self.handler.send_response(200) self.handler.send_header('Content-type', mim) self.handler.end_headers() self.handler.wfile.write(f.read()) f.close()
def handle(self, *args, **options): products = Product.objects.all() buyersguide_page = self.get_or_create_buyers_guide() for product in products: # 1. Create ProductPage out of this product product = product.specific # Get the specific class # Always refresh the buyersguide_page to update treebeards pathing buyersguide_page.refresh_from_db() # Check if ProductPage exists. If it does, continue on. # This check will allow us to run this script more than once if needed if ProductPage.objects.filter(slug=product.slug).exists(): self.debug_print( f"Product '{product.slug}' already exists, skipping.") continue if isinstance(product, SoftwareProduct): new_product_page = SoftwareProductPage() specific_fields = [ 'medical_privacy_compliant', 'easy_to_learn_and_use', 'handles_recordings_how', 'recording_alert', 'recording_alert_helptext', 'medical_privacy_compliant_helptext', 'host_controls', 'easy_to_learn_and_use_helptext' ] elif isinstance(product, GeneralProduct): new_product_page = GeneralProductPage() specific_fields = [ 'camera_device', 'camera_app', 'microphone_device', 'microphone_app', 'location_device', 'location_app', 'personal_data_collected', 'biometric_data_collected', 'social_data_collected', 'how_can_you_control_your_data', 'data_control_policy_is_bad', 'track_record_choices', 'company_track_record', 'track_record_is_bad', 'track_record_details', 'offline_capable', 'offline_use_description', 'uses_ai', 'ai_uses_personal_data', 'ai_is_transparent', 'ai_helptext' ] self.debug_print( f"Treating '{product.slug}' as {new_product_page.__class__.__name__}" ) # Apply the fields that are different or may cause issues if copied directly from one model to another new_product_page.slug_en = product.slug new_product_page.title = product.name new_product_page.title_en = product.name new_product_page.product_url = product.url new_product_page.cloudinary_image = product.cloudinary_image new_product_page.live = not product.draft # If product is draft, it shall not be live. # These are the common fields between SoftwareProductPages and GeneralProductPages fields = specific_fields + [ 'slug', 'privacy_ding', 'adult_content', 'uses_wifi', 'uses_bluetooth', 'review_date', 'company', 'blurb', 'price', 'worst_case', 'signup_requires_email', 'signup_requires_phone', 'signup_requires_third_party_account', 'signup_requirement_explanation', 'how_does_it_use_data_collected', 'data_collection_policy_is_bad', 'user_friendly_privacy_policy', 'show_ding_for_minimum_security_standards', 'meets_minimum_security_standards', 'uses_encryption', 'uses_encryption_helptext', 'security_updates', 'security_updates_helptext', 'strong_password', 'strong_password_helptext', 'manage_vulnerabilities', 'manage_vulnerabilities_helptext', 'privacy_policy', 'privacy_policy_helptext', 'phone_number', 'live_chat', 'email', 'twitter' ] self.debug_print("\tSetting fields:") for field in fields: # Loop through every field for this product and copy the value # from the Product model to the Page model. self.debug_print("\t\t", field, " as ", getattr(product, field)) setattr(new_product_page, field, getattr(product, field)) self.debug_print(f"Product has image? {bool(product.image)}") self.debug_print( f"Product has cloudinary image? {bool(product.cloudinary_image)}" ) # Get the image file field, and convert it into a WagtailImage object if product.image: # Check if there is an image file. If there isn't one, don't try to copy the # FieldFile to a WagtailImage object. try: image_file = product.image.file except FileNotFoundError: image_file = None if image_file: mime = MimeTypes() mime_type = mime.guess_type( product.image.file.name) # -> ('image/jpeg', None) if mime_type: mime_type = mime_type[0].split('/')[1].upper() else: # Default to a JPEG mimetype. mime_type = 'JPEG' # Create an image out of the FileField. pil_image = PILImage.open(product.image.file) f = BytesIO() pil_image.save(f, mime_type) # Store the image as a WagtailImage object new_image_name = ntpath.basename(product.image.file.name) wagtail_image = WagtailImage.objects.create( title=new_image_name, file=ImageFile(f, name=new_image_name)) # Associate new_product_page.image with wagtail_image new_product_page.image = wagtail_image # Add the new page as a child to BuyersGuidePage. This will add a # `path` to the new_product_page and place it in the Wagtail Tree # using Django Treebeard buyersguide_page.add_child(instance=new_product_page) # Save revision and/or publish so we can add Orderables to this page. new_product_page.save() new_product_page.save_revision() self.debug_print("\tCreated", new_product_page) # Loop through all the m2ms and create Orderable objects for this new page type # Add privacy policy links for privacy_link in product.privacy_policy_links.all(): new_orderable = ProductPagePrivacyPolicyLink() new_orderable.page = new_product_page new_orderable.label = privacy_link.label new_orderable.url = privacy_link.url new_orderable.save() new_product_page.privacy_policy_links.add(new_orderable) self.debug_print("\tPrivacy Orderables added") # Add product categories for category in product.product_category.all(): new_orderable = ProductPageCategory() new_orderable.product = new_product_page new_orderable.category = category new_orderable.save() new_product_page.product_categories.add(new_orderable) self.debug_print("\tCategory Orderables added") # Add updates for update in product.updates.all(): new_orderable = ProductUpdates() new_orderable.page = new_product_page new_orderable.update = update new_orderable.save() new_product_page.updates.add(new_orderable) self.debug_print("\tUpdate Orderables added") # Attach a Votes object to each page if `Page.get_or_create_votes()` exists. if hasattr(new_product_page, 'get_or_create_votes'): new_product_page.get_or_create_votes() # Use .to_dict() to pull out the old aggregated votes product_dict = product.to_dict() votes = product_dict.get('votes', None) if votes: votes = votes.get('creepiness').get('vote_breakdown') self.debug_print(votes) values = [x for (i, x) in sorted(votes.items())] product_total = sum([ x * ((i + 1) * 20 - 10) for i, x in enumerate(values) ]) self.debug_print( f'\tOriginal votes: {values} (total score: {product_total})' ) else: # Default vote "bin" values = [0, 0, 0, 0, 0] product_total = 0 new_product_page.votes.set_votes(values) new_product_page.creepiness_value = product_total new_product_page.save() self.debug_print( f'\tNew product votes: {new_product_page.get_or_create_votes()}' ) if not product.draft: new_product_page.live = True new_product_page.save_revision().publish() else: new_product_page.save_revision() # Always good to fresh from db when using Django Treebeard. buyersguide_page.refresh_from_db() time.sleep(1) # Once all the ProductPages are added, add related_products # By writing a secondary for loop we can avoid attaching a legacy_product # to each ProductPage because they'll have slugs in common. self.debug_print("\nFinal step: Adding related products\n") # Loop through every ProductPage we now have. for product_page in ProductPage.objects.all(): # Fetch the PNI Product that this page was created from. try: product = Product.objects.get(slug=product_page.slug) except Product.DoesNotExist: self.debug_print( f"Skipping {product_page} because a ProductPage.slug={product_page.slug} was not found" ) # noqa continue # Loop through all the Product.related_products for related_product in product.related_products.all(): try: # Find the related ProductPage based on the correct slug. related_page = ProductPage.objects.get( slug=related_product.slug) except ProductPage.DoesNotExist: self.debug_print("Missing product page", product_page) continue # Create a new Orderable for the Related Product. This provides # a higher quality editing experience for Wagtail editors/admins. new_related_product = RelatedProducts() new_related_product.page = product_page new_related_product.related_product = related_page new_related_product.save() product_page.related_product_pages.add(new_related_product) self.debug_print("\tAdded related product page:", related_page)
def run_campaign(campaign, domain='demo.django-crm.io', protocol='https'): try: campaign = Campaign.objects.get(id=campaign) attachments = [] if campaign.attachment: file_path = campaign.attachment.path file_name = file_path.split("/")[-1] content = open(file_path, 'rb').read() mime = MimeTypes() mime_type = mime.guess_type(file_path) attachments.append((file_name, content, mime_type[0])) subject = campaign.subject contacts = Contact.objects.filter( contact_list__in=[each_list for each_list in campaign.contact_lists.all()]) default_html = campaign.html_processed for each_contact in contacts: html = default_html campaign_log = CampaignLog.objects.create(contact=each_contact, campaign=campaign) if campaign.reply_to_email: reply_to_email = campaign.reply_to_email else: message_id = get_campaign_message_id(campaign_log) campaign_log.message_id = message_id campaign_log.save() domain_name = 'django-crm.com' if campaign.from_email is not None: from_email = campaign.from_email else: from_email = campaign.created_by.email reply_to_email = str(from_email) + ' <' + \ str(message_id + '@' + domain_name + '') + '>' if not (each_contact.is_bounced or each_contact.is_unsubscribed): # domain_url = settings.URL_FOR_LINKS domain_url = protocol + '://' + domain img_src_url = domain_url + reverse('marketing:campaign_open', kwargs={ 'campaign_log_id': campaign_log.id, 'email_id': each_contact.id}) # images can only be accessed over https link = '<img src={img_src_url} alt="company_logo" title="company_logo" height="1" width="1" />'.format( img_src_url=img_src_url) # link = '<img src="' + domain_url + '/m/cm/track-email/' + \ # str(campaign_log.id) + '/contact/' + \ # str(each_contact.id) + '/" height="1" width="1" alt="company_logo" + \ # title="company_logo"/>' unsubscribe_from_campaign_url = reverse( 'marketing:unsubscribe_from_campaign', kwargs={'contact_id': each_contact.id, 'campaign_id': campaign.id}) unsubscribe_from_campaign_html = "<br><br/><a href={}>Unsubscribe</a>".format( domain_url + unsubscribe_from_campaign_url) names_dict = {'company_name': each_contact.company_name if each_contact.company_name else '', 'last_name': each_contact.last_name if each_contact.last_name else '', 'city': each_contact.city if each_contact.city else '', 'state': each_contact.state if each_contact.state else '', 'first_name': each_contact.name, 'email': each_contact.email, 'email_id': each_contact.id, 'unsubscribe_from_campaign_url': unsubscribe_from_campaign_url} html = Template(html).render(Context(names_dict)) mail_html = html + link + unsubscribe_from_campaign_html from_email = str(campaign.from_name) + "<" + \ str(campaign.from_email) + '>' to_email = [each_contact.email] send_campaign_mail( subject, mail_html, from_email, to_email, [], [reply_to_email], attachments) except Exception as e: print(e) pass
class SauceNao(object): """" small script to work with SauceNao locally """ SEARCH_POST_URL = 'http://saucenao.com/search.php' # basic account allows currently 20 images within 30 seconds # you can increase this value is you have a premium account LIMIT_30_SECONDS = 20 # 0=html, 2=json but json is omitting important data but includes more data about authors # taken from the API documentation(requires login): https://saucenao.com/user.php?page=search-api API_HTML_TYPE = 0 API_JSON_TYPE = 2 CONTENT_CATEGORY_KEY = 'Material' CONTENT_CHARACTERS_KEY = 'Characters' mime = None logger = None def __init__(self, directory, databases=999, minimum_similarity=65, combine_api_types=False, api_key=None, exclude_categories='', move_to_categories=False, output_type=API_HTML_TYPE, start_file=None, log_level=logging.ERROR, title_minimum_similarity=90): """Initializing function :type directory: str :type databases: int :type minimum_similarity: float :type combine_api_types: bool :type api_key: str :type exclude_categories: str :type move_to_categories: bool :type start_file: str :type log_level: int :type title_minimum_similarity: float """ self._directory = directory self._databases = databases self._minimum_similarity = minimum_similarity self._combine_api_types = combine_api_types self._api_key = api_key self._exclude_categories = exclude_categories self._move_to_categories = move_to_categories self._output_type = output_type self._start_file = start_file self._title_minimum_similarity = title_minimum_similarity self._previous_status_code = None self.mime = MimeTypes() logging.basicConfig(level=log_level) self.logger = logging.getLogger("saucenao_logger") def check_file(self, file_name: str) -> list: """Check the given file for results on SauceNAO :type file_name: str :return: """ self.logger.info("checking file: {0:s}".format(file_name)) if self._combine_api_types: result = self.check_image(file_name, self.API_HTML_TYPE) sorted_results = self.parse_results_json(result) additional_result = self.check_image(file_name, self.API_JSON_TYPE) additional_sorted_results = self.parse_results_json(additional_result) sorted_results = self.merge_results(sorted_results, additional_sorted_results) else: result = self.check_image(file_name, self._output_type) sorted_results = self.parse_results_json(result) filtered_results = self.filter_results(sorted_results) return filtered_results def get_http_data(self, file_path: str, output_type: int): """Prepare the http relevant data(files, headers, params) for the given file path and output type :param file_path: :param output_type: :return: """ with open(file_path, 'rb') as file_object: files = {'file': file_object.read()} headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) ' 'Chrome/63.0.3239.84 Safari/537.36', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8', 'Accept-Language': 'en-DE,en-US;q=0.9,en;q=0.8', 'Accept-Encoding': 'gzip, deflate, br', 'DNT': '1', 'Connection': 'keep-alive' } params = { 'file': file_path, 'Content-Type': self.mime.guess_type(file_path), # parameters taken from form on main page: https://saucenao.com/ 'url': None, 'frame': 1, 'hide': 0, # parameters taken from API documentation: https://saucenao.com/user.php?page=search-api 'output_type': output_type, 'db': self._databases, } if self._api_key: params['api_key'] = self._api_key return files, params, headers def check_image(self, file_name: str, output_type: int) -> str: """Check the possible sources for the given file :type output_type: int :type file_name: str :return: """ file_path = os.path.join(self._directory, file_name) files, params, headers = self.get_http_data(file_path=file_path, output_type=output_type) link = requests.post(url=self.SEARCH_POST_URL, files=files, params=params, headers=headers) code, msg = http.verify_status_code(link, file_name) if code == http.STATUS_CODE_SKIP: self.logger.error(msg) return json.dumps({'results': []}) elif code == http.STATUS_CODE_REPEAT: if not self._previous_status_code: self._previous_status_code = code self.logger.info("Received an unexpected status code, repeating after 10 seconds...") time.sleep(10) return self.check_image(file_name, output_type) else: raise UnknownStatusCodeException(msg) else: self._previous_status_code = None if output_type == self.API_HTML_TYPE: return self.parse_results_html_to_json(link.text) return link.text @staticmethod def parse_results_html_to_json(html: str) -> str: """Parse the results and sort them descending by similarity :type html: str :return: """ soup = Soup(html, 'html.parser') # basic format of json API response results = {'header': {}, 'results': []} for res in soup.find_all('td', attrs={"class": "resulttablecontent"}): # type: element.Tag # optional field in SauceNao title_tag = res.find_next('div', attrs={"class": "resulttitle"}) if title_tag: title = title_tag.text else: title = '' # mandatory field in SauceNao similarity = res.find_next('div', attrs={"class": "resultsimilarityinfo"}).text.replace('%', '') alternate_links = [a_tag['href'] for a_tag in res.find_next('div', attrs={"class": "resultmiscinfo"}).find_all('a', href=True)] content_column = [] content_column_tags = res.find_all('div', attrs={"class": "resultcontentcolumn"}) for content_column_tag in content_column_tags: for br in content_column_tag.find_all('br'): br.replace_with('\n') content_column.append(content_column_tag.text) result = { 'header': { 'similarity': similarity }, 'data': { 'title': title, 'content': content_column, 'ext_urls': alternate_links } } results['results'].append(result) return json.dumps(results) @staticmethod def parse_results_json(text: str) -> list: """Parse the results and sort them descending by similarity :type text: str :return: """ result = json.loads(text) results = [res for res in result['results']] return sorted(results, key=lambda k: float(k['header']['similarity']), reverse=True) def filter_results(self, sorted_results) -> list: """Return results with a similarity bigger or the same as the defined similarity from the arguments (default 65%) :type sorted_results: list|tuple|Generator :return: """ filtered_results = [] for res in sorted_results: if float(res['header']['similarity']) >= float(self._minimum_similarity): filtered_results.append(res) else: # we can break here since the results are sorted by similarity anyways break return filtered_results @staticmethod def get_content_value(results, key: str): """Return the first match of Material in content multiple sites have a categorisation which SauceNao utilizes to provide it in the content section :type results: list|tuple|Generator :type key: str :return: """ for result in results: if 'content' in list(result['data'].keys()): for content in result['data']['content']: if re.match('{0:s}: .*'.format(key), content): return ''.join(re.split(r'{0:s}: '.format(key), content)[1:]).rstrip("\n").split('\n') return '' @staticmethod def merge_two_dicts(x: dict, y: dict) -> dict: """Take x dictionary and insert/overwrite y dictionary values :type x: dict :type y: dict :return: """ z = x.copy() z.update(y) return z def merge_results(self, result: list, additional_result: list) -> list: """Merge two result arrays :type result: list :type additional_result: list :return: """ if len(result) <= len(additional_result): length = len(result) else: length = len(additional_result) for i in range(length): for key in list(result[i].keys()): result[i][key] = self.merge_two_dicts(result[i][key], additional_result[i][key]) return result
class ResponseTypes: CLASSES = { "text/html": "scrapy.http.HtmlResponse", "application/atom+xml": "scrapy.http.XmlResponse", "application/rdf+xml": "scrapy.http.XmlResponse", "application/rss+xml": "scrapy.http.XmlResponse", "application/xhtml+xml": "scrapy.http.HtmlResponse", "application/vnd.wap.xhtml+xml": "scrapy.http.HtmlResponse", "application/xml": "scrapy.http.XmlResponse", "application/json": "scrapy.http.TextResponse", "application/x-json": "scrapy.http.TextResponse", "application/json-amazonui-streaming": "scrapy.http.TextResponse", "application/javascript": "scrapy.http.TextResponse", "application/x-javascript": "scrapy.http.TextResponse", "text/xml": "scrapy.http.XmlResponse", "text/*": "scrapy.http.TextResponse", } def __init__(self): self.classes = {} self.mimetypes = MimeTypes() mimedata = get_data("scrapy", "mime.types").decode("utf8") self.mimetypes.readfp(StringIO(mimedata)) for mimetype, cls in self.CLASSES.items(): self.classes[mimetype] = load_object(cls) def from_mimetype(self, mimetype): """Return the most appropriate Response class for the given mimetype""" if mimetype is None: return Response elif mimetype in self.classes: return self.classes[mimetype] else: basetype = f"{mimetype.split('/')[0]}/*" return self.classes.get(basetype, Response) def from_content_type(self, content_type, content_encoding=None): """Return the most appropriate Response class from an HTTP Content-Type header""" if content_encoding: return Response mimetype = to_unicode(content_type).split(";")[0].strip().lower() return self.from_mimetype(mimetype) def from_content_disposition(self, content_disposition): try: filename = (to_unicode( content_disposition, encoding="latin-1", errors="replace").split(";")[1].split("=")[1].strip("\"'")) return self.from_filename(filename) except IndexError: return Response def from_headers(self, headers): """Return the most appropriate Response class by looking at the HTTP headers""" cls = Response if b"Content-Type" in headers: cls = self.from_content_type( content_type=headers[b"Content-Type"], content_encoding=headers.get(b"Content-Encoding"), ) if cls is Response and b"Content-Disposition" in headers: cls = self.from_content_disposition( headers[b"Content-Disposition"]) return cls def from_filename(self, filename): """Return the most appropriate Response class from a file name""" mimetype, encoding = self.mimetypes.guess_type(filename) if mimetype and not encoding: return self.from_mimetype(mimetype) else: return Response def from_body(self, body): """Try to guess the appropriate response based on the body content. This method is a bit magic and could be improved in the future, but it's not meant to be used except for special cases where response types cannot be guess using more straightforward methods.""" chunk = body[:5000] chunk = to_bytes(chunk) if not binary_is_text(chunk): return self.from_mimetype("application/octet-stream") elif b"<html>" in chunk.lower(): return self.from_mimetype("text/html") elif b"<?xml" in chunk.lower(): return self.from_mimetype("text/xml") else: return self.from_mimetype("text") def from_args(self, headers=None, url=None, filename=None, body=None): """Guess the most appropriate Response class based on the given arguments.""" cls = Response if headers is not None: cls = self.from_headers(headers) if cls is Response and url is not None: cls = self.from_filename(url) if cls is Response and filename is not None: cls = self.from_filename(filename) if cls is Response and body is not None: cls = self.from_body(body) return cls
def setanalyzedFileType(self, path): mime = MimeTypes() url = urllib.pathname2url(path) mime_type = mime.guess_type(url) self.fileType = mime_type
def run_campaign(campaign, domain="demo.django-crm.io", protocol="https"): blocked_domains = BlockedDomain.objects.values_list("domain", flat=True) blocked_emails = BlockedEmail.objects.values_list("email", flat=True) try: campaign = Campaign.objects.get(id=campaign) attachments = [] if campaign.attachment: file_path = campaign.attachment.path file_name = file_path.split("/")[-1] content = open(file_path, "rb").read() mime = MimeTypes() mime_type = mime.guess_type(file_path) attachments.append((file_name, content, mime_type[0])) subject = campaign.subject contacts = Contact.objects.filter(contact_list__in=[ each_list for each_list in campaign.contact_lists.all() ]) default_html = campaign.html_processed for each_contact in contacts: html = default_html campaign_log = CampaignLog.objects.create(contact=each_contact, campaign=campaign) if campaign.reply_to_email: reply_to_email = campaign.reply_to_email else: message_id = get_campaign_message_id(campaign_log) campaign_log.message_id = message_id campaign_log.save() domain_name = "django-crm.com" if campaign.from_email is not None: from_email = campaign.from_email else: from_email = campaign.created_by.email reply_to_email = (str(from_email) + " <" + str(message_id + "@" + domain_name + "") + ">") if not (each_contact.is_bounced or each_contact.is_unsubscribed): if (each_contact.email not in blocked_emails) and ( each_contact.email.split("@")[-1] not in blocked_domains): # domain_url = settings.URL_FOR_LINKS domain_url = protocol + "://" + domain img_src_url = domain_url + reverse( "marketing:campaign_open", kwargs={ "campaign_log_id": campaign_log.id, "email_id": each_contact.id, }, ) # images can only be accessed over https link = '<img src={img_src_url} alt="company_logo" title="company_logo" height="1" width="1" />'.format( img_src_url=img_src_url) # link = '<img src="' + domain_url + '/m/cm/track-email/' + \ # str(campaign_log.id) + '/contact/' + \ # str(each_contact.id) + '/" height="1" width="1" alt="company_logo" + \ # title="company_logo"/>' unsubscribe_from_campaign_url = reverse( "marketing:unsubscribe_from_campaign", kwargs={ "contact_id": each_contact.id, "campaign_id": campaign.id, }, ) unsubscribe_from_campaign_html = "<br><br/><a href={}>Unsubscribe</a>".format( domain_url + unsubscribe_from_campaign_url) names_dict = { "company_name": each_contact.company_name if each_contact.company_name else "", "last_name": each_contact.last_name if each_contact.last_name else "", "city": each_contact.city if each_contact.city else "", "state": each_contact.state if each_contact.state else "", "first_name": each_contact.name, "email": each_contact.email, "email_id": each_contact.id, "name": each_contact.name + " " + each_contact.last_name if each_contact.last_name else "", "unsubscribe_from_campaign_url": unsubscribe_from_campaign_url, } html = Template(html).render(Context(names_dict)) mail_html = html + link + unsubscribe_from_campaign_html from_email = (str(campaign.from_name) + "<" + str(campaign.from_email) + ">") to_email = [each_contact.email] send_campaign_mail( subject, mail_html, from_email, to_email, [], [reply_to_email], attachments, ) except Exception as e: print(e) pass
def upload_file(request): if request.method == 'POST': form = UploadFileForm(request.POST, request.FILES) if form.is_valid(): mime = MimeTypes() uploaded_file = request.FILES['file'] mime_type = mime.guess_type(uploaded_file.name) # Get mime type string from tuple if mime_type[0]: mime_type = mime_type[0] else: return HttpResponseBadRequest( json.dumps({ 'status': 'failed', 'reason': 'Invalid file.' }), 'application/json') if validate_upload_file( uploaded_file, mime_type ) and uploaded_file.size <= settings.MAX_ARCHIVE_FILE_SIZE: link = Link(submitted_url=form.cleaned_data['url'], submitted_title=form.cleaned_data['title'], created_by=request.user) link.save() asset = Asset(link=link) file_name = 'cap' + mime.guess_extension(mime_type) file_path = os.path.join(asset.base_storage_path, file_name) uploaded_file.file.seek(0) file_name = default_storage.store_file(uploaded_file, file_path) if mime_type == 'application/pdf': asset.pdf_capture = file_name else: asset.image_capture = file_name asset.save() response_object = { 'status': 'success', 'linky_id': link.guid, 'linky_hash': link.guid } return HttpResponse(json.dumps(response_object), 'application/json', 201) # '201 Created' status else: return HttpResponseBadRequest( json.dumps({ 'status': 'failed', 'reason': 'Invalid file.' }), 'application/json') else: return HttpResponseBadRequest( json.dumps({ 'status': 'failed', 'reason': 'Missing file.' }), 'application/json') return HttpResponseBadRequest( json.dumps({ 'status': 'failed', 'reason': 'No file submitted.' }), 'application/json')
def send_campaign_email_to_admin_contact(campaign, domain="demo.django-crm.io", protocol="https"): try: campaign = Campaign.objects.get(id=campaign) attachments = [] if campaign.attachment: file_path = campaign.attachment.path file_name = file_path.split("/")[-1] content = open(file_path, "rb").read() mime = MimeTypes() mime_type = mime.guess_type(file_path) attachments.append((file_name, content, mime_type[0])) subject = campaign.subject contacts = ContactEmailCampaign.objects.all() default_html = campaign.html_processed blocked_domains = BlockedDomain.objects.values_list("domain", flat=True) blocked_emails = BlockedEmail.objects.values_list("email", flat=True) for each_contact in contacts: if (each_contact.email not in blocked_emails) and ( each_contact.email.split("@")[-1] not in blocked_domains): html = default_html if campaign.reply_to_email: reply_to_email = campaign.reply_to_email else: domain_name = "django-crm.com" if campaign.from_email is not None: from_email = campaign.from_email else: from_email = campaign.created_by.email reply_to_email = (str(from_email) + " <" + str(settings.EMAIL_HOST_USER + "@" + domain_name + "") + ">") # domain_url = settings.URL_FOR_LINKS domain_url = protocol + "://" + domain # img_src_url = domain_url + reverse('marketing:campaign_open', kwargs={ # 'campaign_log_id': campaign_log.id, 'email_id': each_contact.id}) # # images can only be accessed over https # link = '<img src={img_src_url} alt="company_logo" title="company_logo" height="1" width="1" />'.format( # img_src_url=img_src_url) # link = '<img src="' + domain_url + '/m/cm/track-email/' + \ # str(campaign_log.id) + '/contact/' + \ # str(each_contact.id) + '/" height="1" width="1" alt="company_logo" + \ # title="company_logo"/>' # unsubscribe_from_campaign_url = reverse( # 'marketing:unsubscribe_from_campaign', kwargs={'contact_id': each_contact.id, # 'campaign_id': campaign.id}) # unsubscribe_from_campaign_html = "<br><br/><a href={}>Unsubscribe</a>".format( # domain_url + unsubscribe_from_campaign_url) # names_dict = {'company_name': '', 'city': '', 'state': '', # 'last_name': each_contact.last_name if each_contact.last_name else '', # 'email': each_contact.email, 'email_id': each_contact.id, # 'name': each_contact.name + ' ' + each_contact.last_name if each_contact.last_name else '', # } # mail_html = html + link + unsubscribe_from_campaign_html html = Template(html).render( Context({"email_id": each_contact.id})) mail_html = html from_email = (str(campaign.from_name) + "<" + str(campaign.from_email) + ">") to_email = [each_contact.email] send_campaign_mail( subject, mail_html, from_email, to_email, [], [reply_to_email], attachments, ) except Exception as e: print(e) pass
def _get_mime_type(local_file): mime = MimeTypes() url = urllib.pathname2url(local_file) mime_type = mime.guess_type(url) return mime_type[0]
def getFileContent(): try: #Initialize result = {'file' : '', 'mime' : '', 'message' : '', 'message_css' : '', 'download' : '', 'showDownload' : False, 'containerScroll' : True, 'content' : '', 'keywords' : ''} #json_data = request.json json_data = request.get_json(force=True) #file = json_data['path'] file = os.path.join(CURRENT_DIR, json_data['path'].replace("../","")) mime = MimeTypes() mime_type = mime.guess_type(urllib.pathname2url(file))[0] if mime.guess_type(urllib.pathname2url(file))[0] else 'text/plain' result['file'] = file result['mime'] = mime_type download_html = '' download_html+='<div class="">' download_html+='<a class="btn-download btn btn-lg btn-primary" href="' + file + '" target="_blank"><i class="glyphicon glyphicon-download-alt"></i> Download</a>' download_html+='</div>' result['download'] = download_html supported_application = ['application/xml', 'text/x-php', 'text/plain', 'text/xml', 'text/csv'] mime = mime_type.split('/') file_mime = mime[0] if mime_type in supported_application: content = '' words = '' file_ext = file.split('.') if file_ext[len(file_ext)-1] == "properties": with open(file) as f: for line in f: content += line elif file_ext[len(file_ext)-1] == "feature": content = open(file).read() x = open(file, "r") words = [w for w in x.read().split() if w[0] == "@"] elif file_ext[len(file_ext)-1] == "csv": with open(file, 'rb') as csvfile: spamreader = csv.reader(csvfile, delimiter=' ', quotechar='|') for row in spamreader: line = ' '.join(row) content += line + "\n" else: content = open(file).read() content = unicode(content, errors='ignore') result['content'] = content result['containerScroll'] = True result['showDownload'] = False result['keywords'] = words elif (file_mime == 'text'): html_content = '' html_content+= '<object data="' + file + '" type="' + mime_type + '" id="fileObject" style="width:100% ">' html_content+='<embed type="' + mime_type + '" src="' + file + '" style="width:100%">' html_content+='</object>' result['content'] = html_content result['containerScroll'] = False result['showDownload'] = False else: html_content = '' html_content+= '<object data="' + file + '" type="' + mime_type + '" id="fileObject" style="width:100% " class="hidden">' html_content+='<embed type="' + mime_type + '" src="' + file + '" style="width:100%">' html_content+='</object>' result['content'] = html_content result['containerScroll'] = False result['showDownload'] = True return jsonify(status='OK',result=result) except Exception,e: return jsonify(status='ERROR',message=str(e))
def get_file_mime(): mime = MimeTypes() file_path = r'C:\Users\rzhang\Pictures\33.png' print(mime.guess_type(file_path))
def editFileContent(): try: #Initialize result = {'file' : '', 'mime' : '', 'message' : '', 'message_css' : '', 'download' : '', 'showDownload' : False, 'containerScroll' : True, 'content' : '', 'keywords' : ''} #json_data = request.json json_data = request.get_json(force=True) #file = json_data['path'] file = os.path.join(CURRENT_DIR, json_data['path'].replace("../","")) fileContent = json_data['content'].replace("<br>", "\n") # cleanr = re.compile('<.*?>') # cleantext = re.sub(cleanr, '', fileContent) #print "584", cleantext # fileContent = cleantext # Providing Read, Write and Execute permission to selected featureFiles exePermissionStr = "sudo " + "find " + file + " -type " + "f " + "-exec " + "chmod " + "ugo+rwx " + "{} " +"\;" #print "569", exePermissionStr ffPermissionChange = subprocess.Popen(exePermissionStr, stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True) output, err = ffPermissionChange.communicate() temp = ffPermissionChange.returncode #print "572", temp if temp == 0: with open(file, "w") as f: #print "575" f.write(fileContent) else: print "Write permission not granted to the feature file" mime = MimeTypes() mime_type = mime.guess_type(urllib.pathname2url(file))[0] if mime.guess_type(urllib.pathname2url(file))[0] else 'text/plain' result['file'] = file result['mime'] = mime_type download_html = '' download_html+='<div class="">' download_html+='<a class="btn-download btn btn-lg btn-primary" href="' + file + '" target="_blank"><i class="glyphicon glyphicon-download-alt"></i> Download</a>' download_html+='</div>' result['download'] = download_html supported_application = ['application/xml', 'text/x-php', 'text/plain', 'text/xml', 'text/csv' ] mime = mime_type.split('/') file_mime = mime[0] if mime_type in supported_application: content = '' words = '' file_ext = file.split('.') if file_ext[len(file_ext)-1] == "properties": with open(file) as f: for line in f: content += line elif file_ext[len(file_ext)-1] == "feature": #print "569" content = open(file).read() x = open(file, "r") words = [w for w in x.read().split() if w[0] == "@"] else: content = open(file).read() result['content'] = content result['containerScroll'] = True result['showDownload'] = False result['keywords'] = words elif (file_mime == 'text'): html_content = '' html_content+= '<object data="' + file + '" type="' + mime_type + '" id="fileObject" style="width:100% ">' html_content+='<embed type="' + mime_type + '" src="' + file + '" style="width:100%">' html_content+='</object>' result['content'] = html_content result['containerScroll'] = False result['showDownload'] = False else: html_content = '' html_content+= '<object data="' + file + '" type="' + mime_type + '" id="fileObject" style="width:100% " class="hidden">' html_content+='<embed type="' + mime_type + '" src="' + file + '" style="width:100%">' html_content+='</object>' result['content'] = html_content result['containerScroll'] = False result['showDownload'] = True return jsonify(status='OK',result=result) except Exception,e: return jsonify(status='ERROR',message=str(e))
#!/usr/bin/python3 # -*- coding: utf-8 -*- import base64 from mimetypes import MimeTypes try: pathfile = str(input("Digite o caminho do arquivo: ")) except: print("Ex: /home/user/Desktop/img.jpeg") else: with open(pathfile, "rb") as Image: mime = MimeTypes() mime_type = mime.guess_type(pathfile) base64definer = base64.b64encode(Image.read()) base64definer = str(base64definer) base64definer = base64definer.replace("'", "") base64definer = base64definer.replace("b/9j", "/9j") print(f"data:{mime_type[0]};base64,{base64definer}")
continue prev_req_name = req_name prev_req_time = curr_req_time prev_addr = addr[0] logger.info("Connection Accepted: Client: " + str(addr)) # use the below print statement to print the request ( for debugging ) # print(data) logger.info("Requested File: " + urllib.parse.unquote(req_name)) # ? Check whether the request is to download if(req_name[-3:] == "get"): file_path = home_dir + urllib.parse.unquote(req_name[:-3]) # print file path ( for debugging ) # print("file Path: ", file_path) mime = MimeTypes() mime_type = mime.guess_type(file_path)[0] if(mime_type == None): mime_type = 'text/plain' # ? Check whether the request is directory if(os.path.isdir(file_path)): create_zip(file_path.split('/')[-1], file_path) data = '' with open(file_path.split('/')[-1] + '.zip', 'rb') as f: data = f.read() # reading the file in the binary format zip_path = os.getcwd() + '/' + file_path.split('/')[-1] + ".zip" mime_type = mime.guess_type(zip_path)[0] if(mime_type == None): mime_type = 'text/plain' response = get_header(200, 'download', mime_type, os.path.getsize(zip_path), file_path.split('/')[-1] + ".zip").encode() response += data # attaching the binary format of the text to the http response os.remove(zip_path)