def guess_type(path, custom_mimetypes=None): custom_mimetypes = custom_mimetypes or {} if not mimetypes.inited: mimetypes.init() # try to read system mime.types extensions_map = mimetypes.types_map.copy() # @UndefinedVariable extensions_map.update({ "": "application/octet-stream", # Default ".py": "text/plain", ".c": "text/plain", ".h": "text/plain", ".appcache": "text/cache-manifest", ".webapp": "application/x-web-app-manifest+json" }) extensions_map.update(custom_mimetypes) base, ext = posixpath.splitext(path) # @UnusedVariable if ext in extensions_map: return extensions_map[ext] ext = ext.lower() if ext in extensions_map: return extensions_map[ext] else: return extensions_map[""]
def action_download(self, courseid, taskid, path): """ Download a file or a directory """ wanted_path = self.verify_path(courseid, taskid, path) if wanted_path is None: raise web.notfound() # if the user want a dir: if os.path.isdir(wanted_path): tmpfile = tempfile.TemporaryFile() tar = tarfile.open(fileobj=tmpfile, mode='w:gz') for root, _, files in os.walk(wanted_path): for fname in files: info = tarfile.TarInfo(name=os.path.join(os.path.relpath(root, wanted_path), fname)) file_stat = os.stat(os.path.join(root, fname)) info.size = file_stat.st_size info.mtime = file_stat.st_mtime tar.addfile(info, fileobj=open(os.path.join(root, fname), 'r')) tar.close() tmpfile.seek(0) web.header('Content-Type', 'application/x-gzip', unique=True) web.header('Content-Disposition', 'attachment; filename="dir.tgz"', unique=True) return tmpfile else: mimetypes.init() mime_type = mimetypes.guess_type(wanted_path) web.header('Content-Type', mime_type[0]) web.header('Content-Disposition', 'attachment; filename="' + os.path.split(wanted_path)[1] + '"', unique=True) return open(wanted_path, 'r')
def _makeDatastream(self, each): # Identify datastreams folder datastreams_dir = self.obj_dir + "/datastreams" filename = each["mets:fptr"]["@FILEID"] label = each["@LABEL"] order = each["@ORDER"] # get extension, ds_id mimetypes.init() ds_id, ext = os.path.splitext(filename) # create datastream dictionary ds_dict = { "filename": filename, "ds_id": ds_id, "mimetype": mimetypes.types_map[ext], "label": label, "internal_relationships": {}, 'order': order } self.objMeta_handle.datastreams.append(ds_dict) # make symlinks to datastreams on disk bag_location = datastreams_dir + "/" + filename remote_location = self.files_location + "/" + filename os.symlink(remote_location, bag_location) # Set the representative image for the object if order == "1": self.objMeta_handle.isRepresentedBy = ds_id
def ziplib(config, handle): """ Function to unpack a downloaded zip file by exploding it into multiple handles. zip_unpack: Unpack if true """ import zipfile import calendar import mimetypes mimetypes.init() if not config.get("zip_unpack", False): yield handle return logger.info("Extracting since zip_unpack = true"); z = zipfile.ZipFile(handle.fileobj) def convert(dt): """ Convert a ZipInfo date_time into a unix timestamp (compatible with tar). """ return calendar.timegm(dt) for i in z.infolist(): mime = mimetypes.guess_type(i.filename)[0] yield ExtHandle(i.filename, i.file_size, convert(i.date_time), mime, z.open(i)) return
def __init__(self): remuco.PlayerAdapter.__init__( self, "Totem", mime_types=("audio", "video"), volume_known=True, playback_known=True, progress_known=True, file_actions=FILE_ACTIONS, ) self.__to = None self.__signal_ids = () self.__update_item = False self.__md_album = None self.__md_artist = None self.__md_title = None self.__last_mrl = None self.__seek_step_initial = 5000 self.__seek_step = self.__seek_step_initial self.__css_sid = 0 if not mimetypes.inited: mimetypes.init()
def get_content_type_by_filename(file_name): mime_type = "" mime_map = {} mime_map["js"] = "application/javascript" mime_map["xlsx"] = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" mime_map["xltx"] = "application/vnd.openxmlformats-officedocument.spreadsheetml.template" mime_map["potx"] = "application/vnd.openxmlformats-officedocument.presentationml.template" mime_map["ppsx"] = "application/vnd.openxmlformats-officedocument.presentationml.slideshow" mime_map["pptx"] = "application/vnd.openxmlformats-officedocument.presentationml.presentation" mime_map["sldx"] = "application/vnd.openxmlformats-officedocument.presentationml.slide" mime_map["docx"] = "application/vnd.openxmlformats-officedocument.wordprocessingml.document" mime_map["dotx"] = "application/vnd.openxmlformats-officedocument.wordprocessingml.template" mime_map["xlam"] = "application/vnd.ms-excel.addin.macroEnabled.12" mime_map["xlsb"] = "application/vnd.ms-excel.sheet.binary.macroEnabled.12" try: suffix = "" name = os.path.basename(file_name) suffix = name.split('.')[-1] if suffix in mime_map.keys(): mime_type = mime_map[suffix] else: import mimetypes mimetypes.init() mime_type = mimetypes.types_map["." + suffix] except Exception: mime_type = 'application/octet-stream' if not mime_type: mime_type = 'application/octet-stream' return mime_type
def create_server(ui, app): if ui.config('web', 'certificate'): if sys.version_info >= (2, 6): handler = _httprequesthandlerssl else: handler = _httprequesthandleropenssl else: handler = _httprequesthandler if ui.configbool('web', 'ipv6'): cls = IPv6HTTPServer else: cls = MercurialHTTPServer # ugly hack due to python issue5853 (for threaded use) import mimetypes; mimetypes.init() address = ui.config('web', 'address', '') port = util.getport(ui.config('web', 'port', 8000)) try: return cls(ui, app, (address, port), handler) except socket.error, inst: raise util.Abort(_("cannot start server at '%s:%d': %s") % (address, port, inst.args[1]))
def __init__(self): self.secure_cookies_by_default = True self.default_media_type = DEFAULT_MEDIA_TYPE self.media_handlers = Handlers() mimetypes.init() self.static_media_types = mimetypes.types_map
def GoogleSitesLogin(site_name=None, site_domain=None, debug=False): if site_domain is None: raise ValueError("site_domain is not valid") if site_name is None: raise ValueError("site_name is not valid") mimetypes.init() client = gdata.sites.client.SitesClient( source="wiki-push", site=site_name, domain=site_domain ) client.http_client.debug = debug try: gdata.sample_util.authorize_client( client, auth_type=gdata.sample_util.CLIENT_LOGIN, service=client.auth_service, source="wiki-push", scopes=['http://sites.google.com/feeds/', 'https://sites.google.com/feeds/'] ) except gdata.client.BadAuthentication: print "Invalid user credentials given." return None except gdata.client.Error: print "Login Error." return None return client
def __init__(self, email, password, filepath, chunk_size=None, convert=None, host=None, ssl=True, debug=False): self.convert = convert self.debug = debug if chunk_size: self.chunk_size = chunk_size # Authenticate the user with CLientLogin try: self.client = gdata.docs.client.DocsClient(source=APP_NAME) self.client.ssl = ssl # Force all API requests through HTTPS self.client.http_client.debug = self.debug # Set to True for debugging HTTP requests self.client.ClientLogin(email, password, self.client.source); except gdata.client.BadAuthentication: exit('Invalid user credentials given.') except gdata.client.Error: exit('Login Error') mimetypes.init() # Register common mimetypes on system. self.f = open(filepath) content_type = get_mimetype(self.f.name) file_size = os.path.getsize(self.f.name) self.uploader = gdata.client.ResumableUploader( self.client, self.f, content_type, file_size, chunk_size=self.chunk_size, desired_class=gdata.docs.data.DocsEntry)
def get_file_types_list(self): file_list = self.get_file_list() if not file_list[0][1].has_key('mimetype'): mimetypes.init() for item in file_list: item[1]['mimetype'] = mimetypes.guess_type(item[0])[0] return file_list
def __init__(self, filepath, chunk_size=None, convert=None, host=None, ssl=False, debug=False): self.client = gdata.docs.client.DocsClient(source=APP_NAME) self.client.ssl = ssl self.client.http_client.debug = debug self.convert = convert if host: self.client.host = host if chunk_size: self.chunk_size = chunk_size # Authenticate the user with CLientLogin, OAuth, or AuthSub. try: gdata.sample_util.authorize_client( self.client, service=self.client.auth_service, source=APP_NAME, scopes=self.client.auth_scopes) except gdata.client.BadAuthentication: exit('Invalid user credentials given.') except gdata.client.Error: exit('Login Error') mimetypes.init() # Register common mimetypes on system. self.f = open(filepath) content_type = get_mimetype(self.f.name) file_size = os.path.getsize(self.f.name) self.uploader = gdata.client.ResumableUploader( self.client, self.f, content_type, file_size, chunk_size=self.chunk_size, desired_class=gdata.docs.data.DocsEntry)
def handle_noargs(self, **options): mimetypes.init() locked_print("===> Syncing static directory") pool = ThreadPool(20) # Sync every file in the static media dir with S3 def pooled_sync_file(base, filename): pool.apply_async(self.sync_file, args=[base, filename]) self.walk_tree([conf.SIMPLESTATIC_DIR], pooled_sync_file) pool.close() pool.join() locked_print("===> Static directory syncing complete") locked_print("===> Compressing and uploading CSS and JS") pool = ThreadPool(20) # Iterate over every template, looking for SimpleStaticNode def pooled_handle_template(base, filename): pool.apply_async(self.handle_template, args=[base, filename]) self.walk_tree(list(settings.TEMPLATE_DIRS), pooled_handle_template) pool.close() pool.join() locked_print("===> Finished compressing and uploading CSS and JS")
def __init__(self, local_dir, verbose, quick): self.local_dir = local_dir self.verbose = verbose self.quick = quick parser = ConfigParser.SafeConfigParser() config_file_name = os.path.join(local_dir, DOT_S3SITESYNC) with open(config_file_name) as f: config_file_contents = unicode("[s3sitesync]\n" + f.read()) parser.readfp(io.StringIO(config_file_contents), config_file_name) self.keyid = parser.get('s3sitesync', 'awsAccessKeyId') self.key = parser.get('s3sitesync', 'awsSecretAccessKey') self.bucket_name = parser.get('s3sitesync', 'bucketName') self.region = parser.get('s3sitesync', 'awsRegion') if not self.region in API_ENDPOINTS: raise Exception("Invalid region '%s'" % self.region) self.api_endpoint = API_ENDPOINTS[self.region] self.index_doc = parser.get('s3sitesync', 'indexDocument') self.error_doc = parser.get('s3sitesync', 'errorDocument') ignore_file_name = os.path.join(local_dir, DOT_S3IGNORE) self.ignore_patterns = [] if os.path.isfile(ignore_file_name): with open(ignore_file_name) as f: for line in f: line = line.strip() if line: pattern = os.path.join(local_dir, line) self.ignore_patterns.append(pattern) self.ignore_patterns.append(os.path.join(local_dir, DOT_S3SITESYNC)) self.ignore_patterns.append(os.path.join(local_dir, DOT_S3IGNORE)) mimetypes.init() mimetypes.init([os.path.join(local_dir, DOT_S3MIMETYPES)])
def main(): mimetypes.init() for info in scan_dir(sys.argv[1]): #add_file(info) print info['name'], datetime.fromtimestamp(info['mtime']).strftime('%d-%m-%Y')
def _fetch_subtitles(self): if not self._options.fetch_subtitles: return mimetypes.init() for fname in self._options.files: if not os.path.exists(fname): continue if not self._is_video(fname): continue if not self._need_subtitles(fname): continue if self._has_subtitles(fname): continue if self._options.debug or self._options.verbose: msg('fetching subtitles for %s' % fname) for downloader in self._SUBDOWNLOADERS: if self._options.debug or self._options.verbose: msg('trying with %s' % downloader) fn = getattr(self, '_fetch_subtitles_' + downloader) if fn(fname): break else: if self._options.debug or self._options.verbose: msg('no subtitles were found')
def GET(self, courseid, taskid, path): """ GET request """ if User.is_logged_in(): try: course = FrontendCourse(courseid) if not course.is_open_to_user(User.get_username()): return renderer.course_unavailable() task = course.get_task(taskid) if not task.is_visible_by_user(User.get_username()): return renderer.task_unavailable() path_norm = posixpath.normpath(urllib.unquote(path)) public_folder_path = os.path.normpath(os.path.realpath(os.path.join(INGIniousConfiguration["tasks_directory"], courseid, taskid, "public"))) file_path = os.path.normpath(os.path.realpath(os.path.join(public_folder_path, path_norm))) # Verify that we are still inside the public directory if os.path.normpath(os.path.commonprefix([public_folder_path, file_path])) != public_folder_path: raise web.notfound() if os.path.isfile(file_path): mimetypes.init() mime_type = mimetypes.guess_type(file_path) web.header('Content-Type', mime_type[0]) with open(file_path) as static_file: return static_file.read() else: raise web.notfound() except: if web.config.debug: raise else: raise web.notfound() else: return renderer.index(False)
def showpdf(request): sign = os.path.join(settings.MEDIA_ROOT, "signature.png") mimetypes.init() response = None if 'f' in request.GET: fr = open(os.path.join(settings.MEDIA_ROOT,'pdffiles','extracted','%s' % request.GET['f']), "rb") imgTemp = StringIO() imgDoc = canvas.Canvas(imgTemp) if request.GET['o'] == 'l': imgDoc.drawImage(sign, 529, 40, 290/2, 154/2) else: imgDoc.drawImage(sign, 70, 40, 290/2, 154/2) imgDoc.save() overlay = PdfFileReader(StringIO(imgTemp.getvalue())).getPage(0) page = PdfFileReader(fr).getPage(0) page.mergePage(overlay) pdf_out = PdfFileWriter() pdf_out.addPage(page) response = HttpResponse(mimetype='application/pdf') response['Content-Disposition'] = 'attachment; filename=%s' % request.GET['f'] pdf_out.write(response) return response
def _is_video(fp): ''' Check if file is a video file. ''' mimetypes.init() mtype, __ = mimetypes.guess_type(fp) if mtype is None: return False return mtype.startswith('video/')
def download_generic(self): id = self.request.params.get('id', None) filename = self.request.params.get('filename', None) if id is None or filename is None: return HTTPBadRequest() entry = DBSession.query(LuxDownloadUrl).filter( LuxDownloadUrl.id == id).first() if entry is not None: if entry.protected and self.request.user is None: return HTTPUnauthorized() url = entry.url + filename try: data = urllib2.urlopen(url, None, 1800) except Exception as e: log.exception(e) data = None log.debug(url) mimetypes.init() type = "application/octet-stream" mimetype = mimetypes.guess_type(url) if mimetype[0] is not None: type = mimetype[0] headers = {"Content-Type": type, "Content-Disposition": "attachment; filename=\"" + str(filename) + "\""} if data is not None: return Response(data.read(), headers=headers) return HTTPBadRequest()
def media(self,path): ext = path.split('.')[-1] try: mimetypes.init() mimetype = mimetypes.guess_type(path)[0] img = Image.open(self.basepath+'/'+path) width,height = img.size mx = max([width,height]) w,h = width,height if mx > 60: w = width*60/mx h = height*60/mx img = img.resize((w,h), Image.ANTIALIAS) response = HttpResponse(content_type = mimetype or "image/"+ext) response['Cache-Control'] = 'max-age=3600' img.save(response,mimetype.split('/')[1] if mimetype else ext.upper()) return response except Exception as e: imagepath = settings.FILEMANAGER_STATIC_ROOT+'images/icons/'+ext+'.png' if not os.path.exists(imagepath): imagepath = settings.FILEMANAGER_STATIC_ROOT+'images/icons/default.png' img = Image.open(imagepath) width,height = img.size mx = max([width,height]) w,h = width,height if mx > 60: w = width*60/mx h = height*60/mx img = img.resize((w,h), Image.ANTIALIAS) response = HttpResponse(mimetype="image/png") response['Cache-Control'] = 'max-age:3600' img.save(response,'png') return response
def GET(self, courseid, taskid, path): """ GET request """ if self.user_manager.session_logged_in(): try: course = self.course_factory.get_course(courseid) if not self.user_manager.course_is_open_to_user(course): return self.template_helper.get_renderer().course_unavailable() task = course.get_task(taskid) if not self.user_manager.task_is_visible_by_user(task): return self.template_helper.get_renderer().task_unavailable() path_norm = posixpath.normpath(urllib.unquote(path)) public_folder_path = os.path.normpath(os.path.realpath(os.path.join(task.get_directory_path(), "public"))) file_path = os.path.normpath(os.path.realpath(os.path.join(public_folder_path, path_norm))) # Verify that we are still inside the public directory if os.path.normpath(os.path.commonprefix([public_folder_path, file_path])) != public_folder_path: raise web.notfound() if os.path.isfile(file_path): mimetypes.init() mime_type = mimetypes.guess_type(file_path) web.header('Content-Type', mime_type[0]) with open(file_path) as static_file: return static_file.read() else: raise web.notfound() except: if web.config.debug: raise else: raise web.notfound() else: return self.template_helper.get_renderer().index(self.user_manager.get_auth_methods_fields(), False)
def run(self): mimetypes.init() log.debug("Initialized mime type database.") screenshot_tube = self.config.get('beanstalk', 'screenshot_tube') self.beanstalk = politwoops.utils.beanstalk( host=self.config.get('beanstalk', 'host'), port=int(self.config.get('beanstalk', 'port')), watch=screenshot_tube, use=None) log.debug("Connected to queue.") while True: time.sleep(0.2) self.heart.beat() reserve_timeout = max(self.heart.interval.total_seconds() * 0.1, 2) job = self.beanstalk.reserve(timeout=reserve_timeout) if job: try: tweet = anyjson.deserialize(job.body) self.process_entities(tweet) job.delete() except Exception as e: log.error("Exception caught, burying screenshot job for tweet {tweet}: {e}", tweet=tweet.get('id'), e=e) job.bury()
def mimetype(self): try: mimetypes.init() type = mimetypes.guess_type(self.file.name)[0] except: type = None return type
def encoding(self): try: mimetypes.init() encoding = mimetypes.guess_type(self.file.name)[1] except: encoding = None return encoding
def get_resource_content(request, source_name): """ Returns the actual resource. Tries to guess the mimetype. If it fails, returns application/octet-stream as fallback. """ import mimetypes mimetypes.init() resource_id = request.GET.get('resource_id', None) if not resource_id: raise Http404 try: source = Source.objects.get(name=source_name) except Source.DoesNotExist: response_data = {'error': 'Source not found'} else: response_data = Backend(source.backend_id).get_resource_content(source, resource_id) if 'content' in response_data.keys(): content = response_data['content'] mimetype, encoding = mimetypes.guess_type(response_data['name']) if not mimetype: mimetype = 'application/octet-stream' else: content = response_data mimetype = "application/json" return HttpResponse(content, mimetype=mimetype)
def get_file_mimetype(path): """ Returns file mimetype """ filename, file_extension = os.path.splitext(path) mimetypes.init() return mimetypes.types_map[file_extension]
def __call__(self, value, *args, **kwargs): """ :param value: :type value: :class:`list` of :class:`dict` :return: :rtype: :class:`django.forms.Form` """ if value is None: return self.form_cls(*args, **kwargs) post_data = QueryDict("", mutable=True) file_data = QueryDict("", mutable=True) for obj in value: name = obj["name"] value = obj["value"] if name in self.form_cls.base_fields and isinstance( self.form_cls.base_fields[name], FileField ): mimetypes.init() basename = os.path.basename(value) (type_, __) = mimetypes.guess_type(basename) # it's a file => we need to simulate an uploaded one content = InMemoryUploadedFile( io.BytesIO(b"\0"), name, basename, type_ or "application/binary", 1, "utf-8", ) file_data.update({name: content}) else: post_data.update({name: value}) return self.form_cls(post_data, file_data, *args, **kwargs)
def _ContentType(requestedfile): """ Guess the file type based on the extension. This will be used to decide whether to open the file in text or binary mode. """ base, ext = posixpath.splitext(requestedfile) # This uses the standard MIME types mapping library. If it isn't # found, we check to see if it is xhtml (which might not be in the # local mime types library). If the type still is not found, it # defaults to plain text. mimetypes.init() try: ctype = mimetypes.types_map[ext] except: if (ext.lower() == '.xhtml'): ctype = 'application/xhtml+xml' else: ctype = 'text/plain' # We need to augment this information with a set of image # types to use for the file mode. if (ext in ['.png', '.PNG', '.gif', '.GIF', '.jpeg', '.JPEG', '.jpg', '.JPG']): fmode = 'rb' else: fmode = 'r' return fmode, ctype
def sort(dir, tmp_dir = path.abspath('/home/ndhuang/Pictures/'), all = False): mimetypes.init() mimetypes.add_type('image/x-nikon-nef', '.NEF') sort_dir = path.join(tmp_dir, 'sorting') img_files = os.listdir(dir) i = 0 while i < len(img_files): img = img_files[i] if '.pp3' in img: img_files.remove(img) continue elif not all and not re.match('DSC_(\d){4}\.', img): img_files.remove(img) continue mt = mimetypes.guess_type(img)[0] mt = mt.split('/') if mt[0] != 'image': raise RuntimeError('%s is not an image!' %img) else: i += 1 os.mkdir(sort_dir) imgs = [[] for i in img_files] for i in range(len(img_files)): try: imgs[i] = Img(path.join(dir, img_files[i])) except KeyError: print '%s is missing EXIF data!' %img_files[i] # remove empty arrays while [] in imgs: imgs.remove([]) imgs = sorted(imgs) pic_num = 1 copies = 1 for i, img in enumerate(imgs): ext = img.path[-3:] if i != 0 and imgs[i] == imgs[i - 1]: dst = path.join(sort_dir, 'DSC_%04d-%d.%s' %(pic_num - 1, copies, ext)) copies += 1 else: dst = path.join(sort_dir, 'DSC_%04d.%s' %(pic_num, ext)) pic_num += 1 copies = 1 os.rename(img.path, dst) try: os.rename(img.path + '.out.pp3', dst + '.out.pp3') except OSError as err: pass try: os.rename(img.path + '.pp3', dst + '.pp3') except OSError as err: pass for f in os.listdir(dir): os.rename(path.join(dir, f), path.join(sort_dir, f)) os.rmdir(dir) os.rename(sort_dir, dir)
def __init__(self, conf, memcache=None, logger=None, account_ring=None, container_ring=None): if conf is None: conf = {} if logger is None: self.logger = get_logger(conf, log_route='proxy-server') else: self.logger = logger self._error_limiting = {} swift_dir = conf.get('swift_dir', '/etc/swift') self.swift_dir = swift_dir self.node_timeout = int(conf.get('node_timeout', 10)) self.recoverable_node_timeout = int( conf.get('recoverable_node_timeout', self.node_timeout)) self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.client_timeout = int(conf.get('client_timeout', 60)) self.put_queue_depth = int(conf.get('put_queue_depth', 10)) self.object_chunk_size = int(conf.get('object_chunk_size', 65536)) self.client_chunk_size = int(conf.get('client_chunk_size', 65536)) self.trans_id_suffix = conf.get('trans_id_suffix', '') self.post_quorum_timeout = float(conf.get('post_quorum_timeout', 0.5)) self.error_suppression_interval = \ int(conf.get('error_suppression_interval', 60)) self.error_suppression_limit = \ int(conf.get('error_suppression_limit', 10)) self.recheck_container_existence = \ int(conf.get('recheck_container_existence', 60)) self.recheck_account_existence = \ int(conf.get('recheck_account_existence', 60)) self.allow_account_management = \ config_true_value(conf.get('allow_account_management', 'no')) self.object_post_as_copy = \ config_true_value(conf.get('object_post_as_copy', 'true')) self.container_ring = container_ring or Ring(swift_dir, ring_name='container') self.account_ring = account_ring or Ring(swift_dir, ring_name='account') # ensure rings are loaded for all configured storage policies for policy in POLICIES: policy.load_ring(swift_dir) self.obj_controller_router = ObjectControllerRouter() self.memcache = memcache mimetypes.init(mimetypes.knownfiles + [os.path.join(swift_dir, 'mime.types')]) self.account_autocreate = \ config_true_value(conf.get('account_autocreate', 'no')) self.auto_create_account_prefix = ( conf.get('auto_create_account_prefix') or '.') self.expiring_objects_account = self.auto_create_account_prefix + \ (conf.get('expiring_objects_account_name') or 'expiring_objects') self.expiring_objects_container_divisor = \ int(conf.get('expiring_objects_container_divisor') or 86400) self.max_containers_per_account = \ int(conf.get('max_containers_per_account') or 0) self.max_containers_whitelist = [ a.strip() for a in conf.get('max_containers_whitelist', '').split(',') if a.strip()] self.deny_host_headers = [ host.strip() for host in conf.get('deny_host_headers', '').split(',') if host.strip()] self.log_handoffs = config_true_value(conf.get('log_handoffs', 'true')) self.cors_allow_origin = [ a.strip() for a in conf.get('cors_allow_origin', '').split(',') if a.strip()] self.strict_cors_mode = config_true_value( conf.get('strict_cors_mode', 't')) self.node_timings = {} self.timing_expiry = int(conf.get('timing_expiry', 300)) self.sorting_method = conf.get('sorting_method', 'shuffle').lower() self.max_large_object_get_time = float( conf.get('max_large_object_get_time', '86400')) value = conf.get('request_node_count', '2 * replicas').lower().split() if len(value) == 1: rnc_value = int(value[0]) self.request_node_count = lambda replicas: rnc_value elif len(value) == 3 and value[1] == '*' and value[2] == 'replicas': rnc_value = int(value[0]) self.request_node_count = lambda replicas: rnc_value * replicas else: raise ValueError( 'Invalid request_node_count value: %r' % ''.join(value)) try: self._read_affinity = read_affinity = conf.get('read_affinity', '') self.read_affinity_sort_key = affinity_key_function(read_affinity) except ValueError as err: # make the message a little more useful raise ValueError("Invalid read_affinity value: %r (%s)" % (read_affinity, err.message)) try: write_affinity = conf.get('write_affinity', '') self.write_affinity_is_local_fn \ = affinity_locality_predicate(write_affinity) except ValueError as err: # make the message a little more useful raise ValueError("Invalid write_affinity value: %r (%s)" % (write_affinity, err.message)) value = conf.get('write_affinity_node_count', '2 * replicas').lower().split() if len(value) == 1: wanc_value = int(value[0]) self.write_affinity_node_count = lambda replicas: wanc_value elif len(value) == 3 and value[1] == '*' and value[2] == 'replicas': wanc_value = int(value[0]) self.write_affinity_node_count = \ lambda replicas: wanc_value * replicas else: raise ValueError( 'Invalid write_affinity_node_count value: %r' % ''.join(value)) # swift_owner_headers are stripped by the account and container # controllers; we should extend header stripping to object controller # when a privileged object header is implemented. swift_owner_headers = conf.get( 'swift_owner_headers', 'x-container-read, x-container-write, ' 'x-container-sync-key, x-container-sync-to, ' 'x-account-meta-temp-url-key, x-account-meta-temp-url-key-2, ' 'x-container-meta-temp-url-key, x-container-meta-temp-url-key-2, ' 'x-account-access-control') self.swift_owner_headers = [ name.strip().title() for name in swift_owner_headers.split(',') if name.strip()] # Initialization was successful, so now apply the client chunk size # parameter as the default read / write buffer size for the network # sockets. # # NOTE WELL: This is a class setting, so until we get set this on a # per-connection basis, this affects reading and writing on ALL # sockets, those between the proxy servers and external clients, and # those between the proxy servers and the other internal servers. # # ** Because it affects the client as well, currently, we use the # client chunk size as the govenor and not the object chunk size. socket._fileobject.default_bufsize = self.client_chunk_size self.expose_info = config_true_value( conf.get('expose_info', 'yes')) self.disallowed_sections = list_from_csv( conf.get('disallowed_sections', 'swift.valid_api_versions')) self.admin_key = conf.get('admin_key', None) register_swift_info( version=swift_version, strict_cors_mode=self.strict_cors_mode, policies=POLICIES.get_policy_info(), allow_account_management=self.allow_account_management, account_autocreate=self.account_autocreate, **constraints.EFFECTIVE_CONSTRAINTS)
class SimpleHTTPRequestHandler(http.server.BaseHTTPRequestHandler): def do_GET(self): f = self.send_head() if f: for i in f.readlines(): if isinstance(i, str): self.wfile.write(i.encode("utf-8")) else: self.wfile.write(i) f.close() def do_HEAD(self): f = self.send_head() if f: f.close() # 上传结果处理 def do_POST(self): r, info = self.deal_post_data() print(r, info, "by: ", self.client_address) f = StringIO() f.write('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">') f.write( '<meta name="viewport" content="width=device-width" charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0, minimum-scale=1.0, maximum-scale=1.0, user-scalable=no">' ) f.write("<html>\n<title>上传结果</title>\n") f.write("<body>\n<h2>文件上传</h2>") if r: f.write('<strong style="color:#00FF00">成功</strong>\n') else: f.write('<strong style="color:#FF0000">失败</strong>\n') f.write("<hr>\n") f.write(info) f.write("</br><a href=\"%s\">点击返回</a>" % self.headers['referer']) f.write("<hr><small>Powered By: gaowanliang ") f.write("</small></body>\n</html>\n") length = f.tell() f.seek(0) self.send_response(200) self.send_header("Content-type", "text/html") self.send_header("Content-Length", str(length)) self.end_headers() if f: for i in f.readlines(): self.wfile.write(i.encode("utf-8")) f.close() def deal_post_data(self): boundary = str( self.headers["Content-Type"].split("=")[1]).encode("utf-8") remainbytes = int(self.headers['Content-length']) line = self.rfile.readline() remainbytes -= len(line) if not boundary in line: return (False, "Content NOT begin with boundary") line = self.rfile.readline() remainbytes -= len(line) fn = re.findall( r'Content-Disposition.*name="file"; filename="(.*)"'.encode( 'utf-8'), line) if not fn: return (False, "Can't find out file name...") path = str(self.translate_path(self.path)).encode('utf-8') osType = platform.system() try: if osType == "Linux": fn = os.path.join(path, fn[0].decode('gbk').encode('utf-8')) else: fn = os.path.join(path, fn[0]) except Exception as e: return (False, "文件名请不要用中文,或者使用IE上传中文名的文件。{}".format(e)) while os.path.exists(fn): fn += "_".encode("utf-8") line = self.rfile.readline() remainbytes -= len(line) line = self.rfile.readline() remainbytes -= len(line) try: out = open(fn, 'wb') except IOError: return ( False, "Can't create file to write, do you have permission to write?") preline = self.rfile.readline() remainbytes -= len(preline) while remainbytes > 0: line = self.rfile.readline() remainbytes -= len(line) if boundary in line: preline = preline[0:-1] if preline.endswith('\r'.encode("utf-8")): preline = preline[0:-1] out.write(preline) out.close() return (True, "文件 '%s' 上传成功" % fn) else: out.write(preline) preline = line return (False, "Unexpect Ends of data.") def send_head(self): path = self.translate_path(self.path) f = None if os.path.isdir(path): if not self.path.endswith('/'): self.send_response(301) self.send_header("Location", self.path + "/") self.end_headers() return None for index in "index.html", "index.htm": index = os.path.join(path, index) if os.path.exists(index): path = index break else: return self.list_directory(path) ctype = self.guess_type(path) try: f = open(path, 'rb') except IOError: self.send_error(404, "File not found") return None self.send_response(200) self.send_header("Content-type", ctype) fs = os.fstat(f.fileno()) self.send_header("Content-Length", str(fs[6])) self.send_header("Last-Modified", self.date_time_string(fs.st_mtime)) self.end_headers() return f def list_directory(self, path): try: list = os.listdir(path) except os.error: self.send_error(404, "No permission to list directory") return None list.sort(key=lambda a: a.lower()) f = StringIO() displaypath = html.escape(urllib.parse.unquote(self.path)) dirs = '' for name in list: fullname = os.path.join(path, name) colorName = displayname = linkname = name if os.path.isdir(fullname): colorName = '<span style="background-color: #CEFFCE;">' + name + '/</span>' displayname = name linkname = name + "/" if os.path.islink(fullname): colorName = '<span style="background-color: #FFBFFF;">' + name + '@</span>' displayname = name filename = os.getcwd() + '/' + displaypath + displayname dirs += f""" <tr> <td width='60%'><a href='{urllib.parse.quote(linkname)}'>{colorName}</a> </td><td width='20%'>{sizeof_fmt(os.path.getsize(filename))}</td> <td width='20%'>{modification_date(filename)}</td> </tr> """ script_code = """ """ content = f""" <!DOCTYPE html> <html lang="en"> <head> <meta charset="UTF-8"> <title>Title</title> </head> <body> <h2>目录清单 位于{displaypath}</h2> <hr> <form ENCTYPE="multipart/form-data" method="post"> <input name="file" type="file"/> <input type="submit" value="上传"/> <input type="button" value="主目录" onClick="location='/'"> </form> <h2 style="color:#FF0000">请先选择完文件再点上传,不这样做的话可能会出现奇怪的情况</h2> <hr> <table id='wrap'> {dirs} </table> <hr> </body> {script_code} </html> """ f.write(content) length = f.tell() f.seek(0) self.send_response(200) self.send_header("Content-type", "text/html") self.send_header("Content-Length", str(length)) self.end_headers() return f def translate_path(self, path): path = path.split('?', 1)[0] path = path.split('#', 1)[0] path = posixpath.normpath(urllib.parse.unquote(path)) words = path.split('/') words = [_f for _f in words if _f] path = os.getcwd() for word in words: drive, word = os.path.splitdrive(word) head, word = os.path.split(word) if word in (os.curdir, os.pardir): continue path = os.path.join(path, word) return path def copyfile(self, source, outputfile): shutil.copyfileobj(source, outputfile) def guess_type(self, path): base, ext = posixpath.splitext(path) if ext in self.extensions_map: return self.extensions_map[ext] ext = ext.lower() if ext in self.extensions_map: return self.extensions_map[ext] else: return self.extensions_map[''] if not mimetypes.inited: mimetypes.init() # try to read system mime.types extensions_map = mimetypes.types_map.copy() extensions_map.update({ '': 'application/octet-stream', # Default '.py': 'text/plain', '.c': 'text/plain', '.h': 'text/plain', })
import ssl import pathlib import asyncio import json import logging import array import time import os from http import HTTPStatus from http.server import HTTPServer, BaseHTTPRequestHandler import mimetypes import threading import websockets mimetypes.init() logging.basicConfig() # Static file server (compatible with websocket library). # See https://gist.github.com/artizirk/04eb23d957d7916c01ca632bb27d5436 class WebSocketServerProtocolWithHTTP(websockets.WebSocketServerProtocol): """Implements a simple static file server for WebSocketServer""" async def process_request(self, path, request_headers): """Serves a file when doing a GET request with a valid path""" if "Upgrade" in request_headers: return # Probably a WebSocket connection if path == '/': path = '/index.html'
def init(): """ Initialize all channels. """ logger = logging.getLogger(__name__) # Initialize mimetypes library mimetypes.init( [pkg_resources.resource_filename('ehforwarderbot', 'mimetypes')]) # Initialize all channels # (Load libraries and modules and init them with Queue `q`) conf = config.load_config() for i in conf['slave_channels']: logger.log(99, "\x1b[0;37;46m %s \x1b[0m", _("Initializing slave {}...").format(i)) cls = utils.locate_module(i, 'slave') instance_id = i.split('#', 1)[1:] instance_id = (instance_id and instance_id[0]) or None coordinator.add_channel(cls(instance_id=instance_id)) logger.log( 99, "\x1b[0;37;42m %s \x1b[0m", _("Slave channel {name} ({id}) # {instance_id} is initialized."). format(name=cls.channel_name, id=cls.channel_id, instance_id=instance_id)) logger.log(99, "\x1b[0;37;46m %s \x1b[0m", _("Initializing master {}...").format(conf['master_channel'])) instance_id = conf['master_channel'].split('#', 1)[1:] instance_id = (instance_id and instance_id[0]) or None coordinator.add_channel( utils.locate_module(conf['master_channel'], 'master')(instance_id=instance_id)) logger.log( 99, "\x1b[0;37;42m %s \x1b[0m", _("Master channel {name} ({id}) # {instance_id} is initialized."). format(name=coordinator.master.channel_name, id=coordinator.master.channel_id, instance_id=instance_id)) logger.log(99, "\x1b[1;37;42m %s \x1b[0m", _("All channels initialized.")) for i in conf['middlewares']: logger.log(99, "\x1b[0;37;46m %s \x1b[0m", _("Initializing middleware {}...").format(i)) cls = utils.locate_module(i, 'middleware') instance_id = i.split('#', 1)[1:] instance_id = (instance_id and instance_id[0]) or None coordinator.add_middleware(cls(instance_id=instance_id)) logger.log( 99, "\x1b[0;37;42m %s \x1b[0m", _("Middleware {name} ({id}) # {instance_id} is initialized."). format(name=cls.middleware_name, id=cls.middleware_id, instance_id=instance_id)) logger.log(99, "\x1b[1;37;42m %s \x1b[0m", _("All middlewares are initialized.")) coordinator.master_thread = threading.Thread( target=coordinator.master.poll) coordinator.slave_threads = { key: threading.Thread(target=coordinator.slaves[key].poll) for key in coordinator.slaves }
class SimpleHTTPRequestHandler(http.server.BaseHTTPRequestHandler): """Simple HTTP request handler with GET/HEAD/POST commands. This serves files from the current directory and any of its subdirectories. The MIME type for files is determined by calling the .guess_type() method. And can reveive file uploaded by client. The GET/HEAD/POST requests are identical except that the HEAD request omits the actual contents of the file. """ server_version = "SimpleHTTPWithUpload/" + __version__ def do_GET(self): """Serve a GET request.""" f = self.send_head() if f: self.copyfile(f, self.wfile) f.close() def do_HEAD(self): """Serve a HEAD request.""" f = self.send_head() if f: f.close() def do_POST(self): """Serve a POST request.""" r, info = self.deal_post_data() print((r, info, "by: ", self.client_address)) f = BytesIO() f.write(b'<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">') f.write(b"<html>\n<title>Upload Result Page</title>\n") f.write(b"<body>\n<h2>Upload Result Page</h2>\n") f.write(b"<hr>\n") if r: f.write(b"<strong>Success:</strong>") else: f.write(b"<strong>Failed:</strong>") f.write(info.encode()) f.write( ("<br><a href=\"%s\">back</a>" % self.headers['referer']).encode()) f.write(b"<hr><small>Powerd By: bones7456, check new version at ") f.write(b"<a href=\"http://li2z.cn/?s=SimpleHTTPServerWithUpload\">") f.write(b"here</a>.</small></body>\n</html>\n") length = f.tell() f.seek(0) self.send_response(200) self.send_header("Content-type", "text/html") self.send_header("Content-Length", str(length)) self.end_headers() if f: self.copyfile(f, self.wfile) f.close() def deal_post_data(self): content_type = self.headers['content-type'] if not content_type: return (False, "Content-Type header doesn't contain boundary") boundary = content_type.split("=")[1].encode() remainbytes = int(self.headers['content-length']) line = self.rfile.readline() remainbytes -= len(line) if not boundary in line: return (False, "Content NOT begin with boundary") line = self.rfile.readline() remainbytes -= len(line) fn = re.findall(r'Content-Disposition.*name="file"; filename="(.*)"', line.decode()) if not fn: return (False, "Can't find out file name...") path = self.translate_path(self.path) fn = os.path.join(path, fn[0]) line = self.rfile.readline() remainbytes -= len(line) line = self.rfile.readline() remainbytes -= len(line) try: out = open(fn, 'wb') except IOError: return ( False, "Can't create file to write, do you have permission to write?") preline = self.rfile.readline() remainbytes -= len(preline) while remainbytes > 0: line = self.rfile.readline() remainbytes -= len(line) if boundary in line: preline = preline[0:-1] if preline.endswith(b'\r'): preline = preline[0:-1] out.write(preline) out.close() return (True, "File '%s' upload success!" % fn) else: out.write(preline) preline = line return (False, "Unexpect Ends of data.") def send_head(self): """Common code for GET and HEAD commands. This sends the response code and MIME headers. Return value is either a file object (which has to be copied to the outputfile by the caller unless the command was HEAD, and must be closed by the caller under all circumstances), or None, in which case the caller has nothing further to do. """ path = self.translate_path(self.path) f = None if os.path.isdir(path): if not self.path.endswith('/'): # redirect browser - doing basically what apache does self.send_response(301) self.send_header("Location", self.path + "/") self.end_headers() return None for index in "index.html", "index.htm": index = os.path.join(path, index) if os.path.exists(index): path = index break else: return self.list_directory(path) ctype = self.guess_type(path) try: # Always read in binary mode. Opening files in text mode may cause # newline translations, making the actual size of the content # transmitted *less* than the content-length! f = open(path, 'rb') except IOError: self.send_error(404, "File not found") return None self.send_response(200) self.send_header("Content-type", ctype) fs = os.fstat(f.fileno()) self.send_header("Content-Length", str(fs[6])) self.send_header("Last-Modified", self.date_time_string(fs.st_mtime)) self.end_headers() return f def list_directory(self, path): """Helper to produce a directory listing (absent index.html). Return value is either a file object, or None (indicating an error). In either case, the headers are sent, making the interface the same as for send_head(). """ try: list = os.listdir(path) except os.error: self.send_error(404, "No permission to list directory") return None list.sort(key=lambda a: a.lower()) f = BytesIO() displaypath = cgi.escape(urllib.parse.unquote(self.path)) f.write(b'<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">') f.write(("<html>\n<title>Directory listing for %s</title>\n" % displaypath).encode()) f.write(("<body>\n<h2>Directory listing for %s</h2>\n" % displaypath).encode()) f.write(b"<hr>\n") f.write(b"<form ENCTYPE=\"multipart/form-data\" method=\"post\">") f.write(b"<input name=\"file\" type=\"file\"/>") f.write(b"<input type=\"submit\" value=\"upload\"/></form>\n") f.write(b"<hr>\n<ul>\n") for name in list: fullname = os.path.join(path, name) displayname = linkname = name # Append / for directories or @ for symbolic links if os.path.isdir(fullname): displayname = name + "/" linkname = name + "/" if os.path.islink(fullname): displayname = name + "@" # Note: a link to a directory displays with @ and links with / f.write(('<li><a href="%s">%s</a>\n' % (urllib.parse.quote(linkname), cgi.escape(displayname))).encode()) f.write(b"</ul>\n<hr>\n</body>\n</html>\n") length = f.tell() f.seek(0) self.send_response(200) self.send_header("Content-type", "text/html") self.send_header("Content-Length", str(length)) self.end_headers() return f def translate_path(self, path): """Translate a /-separated PATH to the local filename syntax. Components that mean special things to the local file system (e.g. drive or directory names) are ignored. (XXX They should probably be diagnosed.) """ # abandon query parameters path = path.split('?', 1)[0] path = path.split('#', 1)[0] path = posixpath.normpath(urllib.parse.unquote(path)) words = path.split('/') words = [_f for _f in words if _f] path = os.getcwd() for word in words: drive, word = os.path.splitdrive(word) head, word = os.path.split(word) if word in (os.curdir, os.pardir): continue path = os.path.join(path, word) return path def copyfile(self, source, outputfile): """Copy all data between two file objects. The SOURCE argument is a file object open for reading (or anything with a read() method) and the DESTINATION argument is a file object open for writing (or anything with a write() method). The only reason for overriding this would be to change the block size or perhaps to replace newlines by CRLF -- note however that this the default server uses this to copy binary data as well. """ shutil.copyfileobj(source, outputfile) def guess_type(self, path): """Guess the type of a file. Argument is a PATH (a filename). Return value is a string of the form type/subtype, usable for a MIME Content-type header. The default implementation looks the file's extension up in the table self.extensions_map, using application/octet-stream as a default; however it would be permissible (if slow) to look inside the data to make a better guess. """ base, ext = posixpath.splitext(path) if ext in self.extensions_map: return self.extensions_map[ext] ext = ext.lower() if ext in self.extensions_map: return self.extensions_map[ext] else: return self.extensions_map[''] if not mimetypes.inited: mimetypes.init() # try to read system mime.types extensions_map = mimetypes.types_map.copy() extensions_map.update({ '': 'application/octet-stream', # Default '.py': 'text/plain', '.c': 'text/plain', '.h': 'text/plain', })
def guess_mime_type(filename): """Guess mime type based of file extension.""" if not mimetypes.inited: mimetypes.init() return mimetypes.guess_type(filename)[0]
def __init__(self, static_path): mimetypes.init() self.static_path = static_path self.block_loader = BlockLoader()
class SimpleHTTPRequestHandler(BaseHTTPRequestHandler): """Simple HTTP request handler with GET and HEAD commands. This serves files from the current directory and any of its subdirectories. The MIME type for files is determined by calling the .guess_type() method. The GET and HEAD requests are identical except that the HEAD request omits the actual contents of the file. """ server_version = "SimpleHTTP/" + __version__ def do_GET(self): """Serve a GET request.""" f = self.send_head() if f: try: self.copyfile(f, self.wfile) finally: f.close() def do_HEAD(self): """Serve a HEAD request.""" f = self.send_head() if f: f.close() def send_head(self): """Common code for GET and HEAD commands. This sends the response code and MIME headers. Return value is either a file object (which has to be copied to the outputfile by the caller unless the command was HEAD, and must be closed by the caller under all circumstances), or None, in which case the caller has nothing further to do. """ path = self.translate_path(self.path) f = None if os.path.isdir(path): parts = urllib.parse.urlsplit(self.path) if not parts.path.endswith('/'): # redirect browser - doing basically what apache does self.send_response(HTTPStatus.MOVED_PERMANENTLY) new_parts = (parts[0], parts[1], parts[2] + '/', parts[3], parts[4]) new_url = urllib.parse.urlunsplit(new_parts) self.send_header("Location", new_url) self.end_headers() return None for index in "index.html", "index.htm": index = os.path.join(path, index) if os.path.exists(index): path = index break else: return self.list_directory(path) ctype = self.guess_type(path) try: f = open(path, 'rb') except OSError: self.send_error(HTTPStatus.NOT_FOUND, "File not found") return None try: fs = os.fstat(f.fileno()) # Use browser cache if possible if ("If-Modified-Since" in self.headers and "If-None-Match" not in self.headers): # compare If-Modified-Since and time of last file modification try: ims = email.utils.parsedate_to_datetime( self.headers["If-Modified-Since"]) except (TypeError, IndexError, OverflowError, ValueError): # ignore ill-formed values pass else: if ims.tzinfo is None: # obsolete format with no timezone, cf. # https://tools.ietf.org/html/rfc7231#section-7.1.1.1 ims = ims.replace(tzinfo=datetime.timezone.utc) if ims.tzinfo is datetime.timezone.utc: # compare to UTC datetime of last modification last_modif = datetime.datetime.fromtimestamp( fs.st_mtime, datetime.timezone.utc) # remove microseconds, like in If-Modified-Since last_modif = last_modif.replace(microsecond=0) if last_modif <= ims: self.send_response(HTTPStatus.NOT_MODIFIED) self.end_headers() f.close() return None self.send_response(HTTPStatus.OK) self.send_header("Content-type", ctype) self.send_header("Content-Length", str(fs[6])) self.send_header("Last-Modified", self.date_time_string(fs.st_mtime)) self.end_headers() return f except: f.close() raise def list_directory(self, path): """Helper to produce a directory listing (absent index.html). Return value is either a file object, or None (indicating an error). In either case, the headers are sent, making the interface the same as for send_head(). """ try: list = os.listdir(path) except OSError: self.send_error( HTTPStatus.NOT_FOUND, "No permission to list directory") return None list.sort(key=lambda a: a.lower()) r = [] try: displaypath = urllib.parse.unquote(self.path, errors='surrogatepass') except UnicodeDecodeError: displaypath = urllib.parse.unquote(path) displaypath = html.escape(displaypath, quote=False) enc = sys.getfilesystemencoding() title = 'Directory listing for %s' % displaypath r.append('<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" ' '"http://www.w3.org/TR/html4/strict.dtd">') r.append('<html>\n<head>') r.append('<meta http-equiv="Content-Type" ' 'content="text/html; charset=%s">' % enc) r.append('<title>%s</title>\n</head>' % title) r.append('<body>\n<h1>%s</h1>' % title) r.append('<hr>\n<ul>') for name in list: fullname = os.path.join(path, name) displayname = linkname = name # Append / for directories or @ for symbolic links if os.path.isdir(fullname): displayname = name + "/" linkname = name + "/" if os.path.islink(fullname): displayname = name + "@" # Note: a link to a directory displays with @ and links with / r.append('<li><a href="%s">%s</a></li>' % (urllib.parse.quote(linkname, errors='surrogatepass'), html.escape(displayname, quote=False))) r.append('</ul>\n<hr>\n</body>\n</html>\n') encoded = '\n'.join(r).encode(enc, 'surrogateescape') f = io.BytesIO() f.write(encoded) f.seek(0) self.send_response(HTTPStatus.OK) self.send_header("Content-type", "text/html; charset=%s" % enc) self.send_header("Content-Length", str(len(encoded))) self.end_headers() return f def translate_path(self, path): """Translate a /-separated PATH to the local filename syntax. Components that mean special things to the local file system (e.g. drive or directory names) are ignored. (XXX They should probably be diagnosed.) """ # abandon query parameters path = path.split('?',1)[0] path = path.split('#',1)[0] # Don't forget explicit trailing slash when normalizing. Issue17324 trailing_slash = path.rstrip().endswith('/') try: path = urllib.parse.unquote(path, errors='surrogatepass') except UnicodeDecodeError: path = urllib.parse.unquote(path) path = posixpath.normpath(path) words = path.split('/') words = filter(None, words) path = os.getcwd() for word in words: if os.path.dirname(word) or word in (os.curdir, os.pardir): # Ignore components that are not a simple file/directory name continue path = os.path.join(path, word) if trailing_slash: path += '/' return path def copyfile(self, source, outputfile): """Copy all data between two file objects. The SOURCE argument is a file object open for reading (or anything with a read() method) and the DESTINATION argument is a file object open for writing (or anything with a write() method). The only reason for overriding this would be to change the block size or perhaps to replace newlines by CRLF -- note however that this the default server uses this to copy binary data as well. """ shutil.copyfileobj(source, outputfile) def guess_type(self, path): """Guess the type of a file. Argument is a PATH (a filename). Return value is a string of the form type/subtype, usable for a MIME Content-type header. The default implementation looks the file's extension up in the table self.extensions_map, using application/octet-stream as a default; however it would be permissible (if slow) to look inside the data to make a better guess. """ base, ext = posixpath.splitext(path) if ext in self.extensions_map: return self.extensions_map[ext] ext = ext.lower() if ext in self.extensions_map: return self.extensions_map[ext] else: return self.extensions_map[''] if not mimetypes.inited: mimetypes.init() # try to read system mime.types extensions_map = mimetypes.types_map.copy() extensions_map.update({ '': 'application/octet-stream', # Default '.py': 'text/plain', '.c': 'text/plain', '.h': 'text/plain', })
class ResumeHTTPHandler(BaseHTTPRequestHandler): server_version = "SimpleHTTPWithUpload/" + __version__ def do_GET(self): """Serve a GET request.""" f = self.send_head() if f: self.wfile.write(f) templateStr = '''<html> <head> <title>QR Link Generator</title> </head> <body> %s <br> <br> <form action="/qr" name=f method=""><input maxLength=1024 size=70 name=s value="" title="Text to QR Encode"><input type=submit value="Show QR" name=qr> </form> </body> </html> ''' def do_HEAD(self): """Serve a HEAD request.""" f = self.send_head() def do_POST(self): """Serve a POST request.""" r, info = self.deal_post_data() print(info) print("uploaded by:", self.client_address) info = info.replace('\n', '<br>') f = ('<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">') +\ ('<html><head>') +\ ('<meta http-equiv="Content-Type" content="text/html; charset=utf-8">') +\ ('<title>Upload Result Page</title>') +\ ('</head><body>') +\ ('<h1>Upload Result Page</h1>') +\ ('<hr>') if r: f = f + ('<strong>Success:<strong><br/>') + info else: f = f + ('<strong>Failed:<strong>') + info f = f + '<br><a href="%s">back</a>' % self.headers['referer'] +\ '</body></html>' f = f.encode('utf-8') length = len(f) self.send_response(200) self.send_header("Content-type", "text/html") self.send_header("Content-Length", str(length)) self.end_headers() self.wfile.write(f) ''' POST data ------WebKitFormBoundaryLVlRNkjiiJLtNYQE Content-Disposition: form-data; name="file"; filename="file1.txt" Content-Type: text/plain content in file1 hello file1 ------WebKitFormBoundaryLVlRNkjiiJLtNYQE Content-Disposition: form-data; name="file"; filename="file2.txt" Content-Type: text/plain content in file2 hello file2 ------WebKitFormBoundaryLVlRNkjiiJLtNYQE-- ''' def deal_post_data(self): boundary = self.headers["Content-Type"].split("=")[1] boundary_begin = ('--' + boundary + '\r\n').encode('utf-8') boundary_end = ('--' + boundary + '--\r\n').encode('utf-8') return_status = True return_info = '\n' outer = 1 inner = 2 leave = 3 loop_info = outer # 1: outer loop, 2: inner_loop, 3: leave and return # first line # b'------WebKitFormBoundaryLVlRNkjiiJLtNYQE' line = self.rfile.readline() while loop_info == outer: # print(line) line = line if line != boundary_begin: return_status = False return_info += "Content NOT begin with boundary\n" break # get filename # b'Content-Disposition: form-data; name="file"; filename="file1.txt"' line = self.rfile.readline().decode('utf-8').rstrip('\r\n') # print(line) filename = re.findall(r'filename="(.*)"', line)[0] # print(filename) if not filename: return_status = False return_info += "Can't find out file name...\n" loop_info = leave break path = self.translate_path(self.path) filename = os.path.join(path, filename) # if filename alread exists if os.path.exists(filename): filename += "_copy" # second line # b'Content-Type: text/plain' line = self.rfile.readline() # blank line line = self.rfile.readline() loop_info = inner # POST data try: with open(filename, 'wb') as f: while loop_info == inner: line = self.rfile.readline() # print(line) if line == boundary_begin: loop_info = outer # print('out') break elif line == boundary_end: # print('leave') loop_info = leave break else: # line 还是二进制形式, realine() 不会删掉二进制的'\n' f.write(line) except Exception as e: loop_info = leave return_status = False return_info += 'Exception!\n' return_info += filename + '\n' return (return_status, return_info) def send_head(self): """Common code for GET and HEAD commands. This sends the response code and MIME headers. Return value is either a file object (which has to be copied to the outputfile by the caller unless the command was HEAD, and must be closed by the caller under all circumstances), or None, in which case the caller has nothing further to do. """ path = self.translate_path(self.path) f = None if os.path.isdir(path): if not self.path.endswith('/'): # redirect browser - doing basically what apache does self.send_response(301) self.send_header("Location", self.path + "/") self.end_headers() return None for index in "index.html", "index.htm": index = os.path.join(path, index) if os.path.exists(index): path = index break else: return self.list_directory(path) ctype = self.guess_type(path) try: # Always read in binary mode. Opening files in text mode may cause # newline translations, making the actual size of the content # transmitted *less* than the content-length! f = open(path, 'rb') except IOError: self.send_error(404, "File not found") return None self.send_response(200) self.send_header("Content-type", ctype) fs = os.fstat(f.fileno()) self.send_header("Content-Length", str(fs[6])) self.send_header("Last-Modified", self.date_time_string(fs.st_mtime)) self.end_headers() data = f.read() f.close() return data def list_directory(self, path): """Helper to produce a directory listing (absent index.html). Return value is either a file object, or None (indicating an error). In either case, the headers are sent, making the interface the same as for send_head(). """ try: list = os.listdir(path) except os.error: self.send_error(404, "No permission to list directory") return None list.sort(key=lambda a: a.lower()) displaypath = cgi.escape(urllib.parse.unquote(self.path)) f = ('<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">') +\ ('<html><head>') +\ ('<meta http-equiv="Content-Type" content="text/html; charset=utf-8">') +\ ('<title>Directory listing for %s</title>' % displaypath) +\ ('</head><body>') +\ ('<h1>Directory listing for %s</h1>' % displaypath) +\ ('<form ENCTYPE="multipart/form-data" method="post">') +\ ('<input name="file" type="file" multiple="multiple"/>') +\ ('<input type="submit" value="upload"/></form>') +\ ('<hr><ul>') for name in list: fullname = os.path.join(path, name) displayname = linkname = name # Append / for directories or @ for symbolic links if os.path.isdir(fullname): displayname = name + "/" linkname = name + "/" if os.path.islink(fullname): displayname = name + "@" # Note: a link to a directory displays with @ and links with / f = f + ('<li><a href="%s">%s</a>' % (urllib.parse.quote(linkname), cgi.escape(displayname))) f = f + ("</ul><hr></body></html>") f = f.encode('utf-8') length = len(f) self.send_response(200) self.send_header("Content-type", "text/html") self.send_header("Content-Length", str(length)) self.end_headers() return f def translate_path(self, path): """Translate a /-separated PATH to the local filename syntax. Components that mean special things to the local file system (e.g. drive or directory names) are ignored. (XXX They should probably be diagnosed.) """ # abandon query parameters path = path.split('?', 1)[0] path = path.split('#', 1)[0] path = posixpath.normpath(urllib.parse.unquote(path)) words = path.split('/') words = filter(None, words) path = os.getcwd() for word in words: drive, word = os.path.splitdrive(word) head, word = os.path.split(word) if word in (os.curdir, os.pardir): continue path = os.path.join(path, word) return path def guess_type(self, path): """Guess the type of a file. Argument is a PATH (a filename). Return value is a string of the form type/subtype, usable for a MIME Content-type header. The default implementation looks the file's extension up in the table self.extensions_map, using application/octet-stream as a default; however it would be permissible (if slow) to look inside the data to make a better guess. """ base, ext = posixpath.splitext(path) if ext in self.extensions_map: return self.extensions_map[ext] ext = ext.lower() if ext in self.extensions_map: return self.extensions_map[ext] else: return self.extensions_map[''] if not mimetypes.inited: mimetypes.init() # try to read system mime.types extensions_map = mimetypes.types_map.copy() extensions_map.update({ '': 'application/octet-stream', # Default '.py': 'text/plain', '.c': 'text/plain', '.h': 'text/plain', })
class SimpleHTTPRequestHandler(BaseHTTPRequestHandler): """Simple HTTP request handler with GET and HEAD commands. This serves files from the current directory and any of its subdirectories. The MIME type for files is determined by calling the .guess_type() method. The GET and HEAD requests are identical except that the HEAD request omits the actual contents of the file. """ server_version = "SimpleHTTP/" + __version__ def do_GET(self): """Serve a GET request.""" f = self.send_head() if f: self.copyfile(f, self.wfile) f.close() def do_HEAD(self): """Serve a HEAD request.""" f = self.send_head() if f: f.close() def send_head(self): """Common code for GET and HEAD commands. This sends the response code and MIME headers. Return value is either a file object (which has to be copied to the outputfile by the caller unless the command was HEAD, and must be closed by the caller under all circumstances), or None, in which case the caller has nothing further to do. """ path = self.translate_path(self.path) f = None if os.path.isdir(path): if not self.path.endswith('/'): # redirect browser - doing basically what apache does self.send_response(301) self.send_header("Location", self.path + "/") self.end_headers() return None for index in "index.html", "index.htm": index = os.path.join(path, index) if os.path.exists(index): path = index break else: return self.list_directory(path) ctype = self.guess_type(path) try: f = open(path, 'rb') except IOError: self.send_error(404, "File not found") return None self.send_response(200) self.send_header("Content-type", ctype) fs = os.fstat(f.fileno()) self.send_header("Content-Length", str(fs[6])) self.send_header("Last-Modified", self.date_time_string(fs.st_mtime)) self.end_headers() return f def list_directory(self, path): """Helper to produce a directory listing (absent index.html). Return value is either a file object, or None (indicating an error). In either case, the headers are sent, making the interface the same as for send_head(). """ try: list = os.listdir(path) except os.error: self.send_error(404, "No permission to list directory") return None list.sort(key=lambda a: a.lower()) r = [] displaypath = html.escape(urllib.parse.unquote(self.path)) r.append('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">') r.append("<html>\n<title>Directory listing for %s</title>\n" % displaypath) r.append("<body>\n<h2>Directory listing for %s</h2>\n" % displaypath) r.append("<hr>\n<ul>\n") for name in list: fullname = os.path.join(path, name) displayname = linkname = name # Append / for directories or @ for symbolic links if os.path.isdir(fullname): displayname = name + "/" linkname = name + "/" if os.path.islink(fullname): displayname = name + "@" # Note: a link to a directory displays with @ and links with / r.append('<li><a href="%s">%s</a>\n' % (urllib.parse.quote(linkname), html.escape(displayname))) r.append("</ul>\n<hr>\n</body>\n</html>\n") enc = sys.getfilesystemencoding() encoded = ''.join(r).encode(enc) f = io.BytesIO() f.write(encoded) f.seek(0) self.send_response(200) self.send_header("Content-type", "text/html; charset=%s" % enc) self.send_header("Content-Length", str(len(encoded))) self.end_headers() return f def translate_path(self, path): """Translate a /-separated PATH to the local filename syntax. Components that mean special things to the local file system (e.g. drive or directory names) are ignored. (XXX They should probably be diagnosed.) """ # abandon query parameters path = path.split('?', 1)[0] path = path.split('#', 1)[0] path = posixpath.normpath(urllib.parse.unquote(path)) words = path.split('/') words = filter(None, words) path = os.getcwd() for word in words: drive, word = os.path.splitdrive(word) head, word = os.path.split(word) if word in (os.curdir, os.pardir): continue path = os.path.join(path, word) return path def copyfile(self, source, outputfile): """Copy all data between two file objects. The SOURCE argument is a file object open for reading (or anything with a read() method) and the DESTINATION argument is a file object open for writing (or anything with a write() method). The only reason for overriding this would be to change the block size or perhaps to replace newlines by CRLF -- note however that this the default server uses this to copy binary data as well. """ shutil.copyfileobj(source, outputfile) def guess_type(self, path): """Guess the type of a file. Argument is a PATH (a filename). Return value is a string of the form type/subtype, usable for a MIME Content-type header. The default implementation looks the file's extension up in the table self.extensions_map, using application/octet-stream as a default; however it would be permissible (if slow) to look inside the data to make a better guess. """ base, ext = posixpath.splitext(path) if ext in self.extensions_map: return self.extensions_map[ext] ext = ext.lower() if ext in self.extensions_map: return self.extensions_map[ext] else: return self.extensions_map[''] if not mimetypes.inited: mimetypes.init() # try to read system mime.types extensions_map = mimetypes.types_map.copy() extensions_map.update({ '': 'application/octet-stream', # Default '.py': 'text/plain', '.c': 'text/plain', '.h': 'text/plain', })
def mime_attach(body, attachments, charset, body_charset=None): mimetypes.init() message = MIMEMultipart('mixed') bodypart = BetterMIMEText(body, _charset=(body_charset or charset)) bodypart.add_header('Content-Disposition', 'inline') message.preamble = 'This is a multi-part MIME message sent by reportbug.\n\n' message.epilogue = '' message.attach(bodypart) failed = False for attachment in attachments: try: fp = file(attachment) fp.close() except EnvironmentError, x: ewrite("Warning: opening '%s' failed: %s.\n", attachment, x.strerror) failed = True continue ctype = None cset = charset info = Popen(['file', '--mime', '--brief', attachment], stdout=PIPE, stderr=STDOUT).communicate()[0] if info: match = re.match(r'([^;, ]*)(,[^;]+)?(?:; )?(.*)', info) if match: ctype, junk, extras = match.groups() match = re.search(r'charset=([^,]+|"[^,"]+")', extras) if match: cset = match.group(1) # If we didn't get a real MIME type, fall back if '/' not in ctype: ctype = None # If file doesn't work, try to guess based on the extension if not ctype: ctype, encoding = mimetypes.guess_type(attachment, strict=False) if not ctype: ctype = 'application/octet-stream' maintype, subtype = ctype.split('/', 1) if maintype == 'text': fp = file(attachment, 'rU') part = BetterMIMEText(fp.read(), _subtype=subtype, _charset=cset) fp.close() elif maintype == 'message': fp = file(attachment, 'rb') part = MIMEMessage(email.message_from_file(fp), _subtype=subtype) fp.close() elif maintype == 'image': fp = file(attachment, 'rb') part = MIMEImage(fp.read(), _subtype=subtype) fp.close() elif maintype == 'audio': fp = file(attachment, 'rb') part = MIMEAudio(fp.read(), _subtype=subtype) fp.close() else: fp = file(attachment, 'rb') part = MIMEBase(maintype, subtype) part.set_payload(fp.read()) fp.close() email.Encoders.encode_base64(part) part.add_header('Content-Disposition', 'attachment', filename=os.path.basename(attachment)) message.attach(part)
class dragdropserver(http.server.BaseHTTPRequestHandler): # The Queue communicates with the stacoan.py file. It's a communication pipe. q = Queue() def log_request(self, code='-', size='-'): if not any( s in str(self.requestline) for s in ('lootbox.html', '.ico', 'robots.txt', '.js', '.css', 'start.html', '.woff2', '.png', '.jpg')): Logger(self.requestline + " " + str(code) + " " + str(size), Logger.INFO) def log_error(self, format, *args): if not any(s in str(self.requestline) for s in ('lootbox.html', 'robots.txt')): Logger(("%s - - [%s] %s - %s\n" % (self.address_string(), self.log_date_time_string(), format % args, str(self.requestline))), Logger.WARNING) def log_message(self, format, *args): Logger(("%s - - [%s] %s\n" % (self.address_string(), self.log_date_time_string(), format % args)), Logger.INFO) def do_GET(self): """Serve a GET request.""" f = self.send_head() if f: self.copyfile(f, self.wfile) f.close() def do_POST(self): """Serve a POST request.""" if re.findall(r'KILLSERVERCOMMAND', self.requestline): ServerWrapper.dragdropserver.q.put("KILLSERVERCOMMAND") Logger("Server upload killed", Logger.INFO) self.send_response(200) exit(0) return True, "Exit" r, info = self.deal_post_data() Logger((str(r) + str(info) + "by: " + str(self.client_address)), Logger.INFO) f = BytesIO() f.write(b'<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">') f.write(b"<html>\n<title>Upload Result Page</title>\n") f.write(b"<body>\n<h2>Upload Result Page</h2>\n") f.write(b"<hr>\n") if r: f.write(b"<strong>Success:</strong>") else: f.write(b"<strong>Failed:</strong>") f.write(info.encode()) f.write(("<br><a href=\"%s\">back</a>" % self.headers['referer']).encode()) length = f.tell() f.seek(0) self.send_response(200) self.send_header("Content-type", "text/html") self.send_header("Content-Length", str(length)) self.end_headers() if f: self.copyfile(f, self.wfile) f.close() def deal_post_data(self): content_type = self.headers['content-type'] if not content_type: return (False, "Content-Type header doesn't contain boundary") boundary = content_type.split("=")[1].encode() remainbytes = int(self.headers['content-length']) line = self.rfile.readline() remainbytes -= len(line) if not boundary in line: return (False, "Content NOT begin with boundary") for line in self.rfile: remainbytes -= len(line) fn = re.findall( r'Content-Disposition.*name="file"; filename="(.*\S.*)"', line.decode()) if fn: break if not fn: return (False, "Can't find out file name...") # check filetype serverside: if not (str(fn[0]).endswith(tuple(ServerWrapper.apptypes))): self.send_error(408, "Filetype not allowed") return (False, "Filetype not allowed.") path = self.translate_path(self.path) # fn holds an array of files, have to make a for all loop fn = os.path.join(path, fn[0]) line = self.rfile.readline() remainbytes -= len(line) line = self.rfile.readline() remainbytes -= len(line) try: out = open(fn, 'wb') except IOError: return ( False, "Can't create file to write, do you have permission to write?" ) preline = self.rfile.readline() remainbytes -= len(preline) while remainbytes > 0: line = self.rfile.readline() remainbytes -= len(line) if boundary in line: preline = preline[0:-1] if preline.endswith(b'\r'): preline = preline[0:-1] out.write(preline) out.close() # Start stacoan instance (program), by using the queue ServerWrapper.dragdropserver.q.put(fn) return True, "File '%s' upload success!" % fn else: out.write(preline) preline = line return False, "Unexpected End of data." def send_head(self): """Common code for GET and HEAD commands. This sends the response code and MIME headers. Return value is either a file object (which has to be copied to the outputfile by the caller unless the command was HEAD, and must be closed by the caller under all circumstances), or None, in which case the caller has nothing further to do. """ path = self.translate_path(self.path) f = None if os.path.isdir(path): if not self.path.endswith('/'): # redirect browser - doing basically what apache does self.send_response(301) self.send_header("Location", self.path + "/") self.end_headers() return None for index in "index.html", "index.htm": index = os.path.join(path, index) if os.path.exists(index): path = index break else: return self.list_directory(path) ctype = self.guess_type(path) try: # Always read in binary mode. Opening files in text mode may cause # newline translations, making the actual size of the content # transmitted *less* than the content-length! f = open(path, 'rb') except IOError: self.send_error(404, "File not found") return None self.send_response(200) self.send_header("Content-type", ctype) fs = os.fstat(f.fileno()) self.send_header("Content-Length", str(fs[6])) self.send_header("Last-Modified", self.date_time_string(fs.st_mtime)) self.end_headers() return f def list_directory(self, path): """Helper to produce a directory listing (absent index.html). Return value is either a file object, or None (indicating an error). In either case, the headers are sent, making the interface the same as for send_head(). """ try: list = os.listdir(path) except os.error: self.send_error(404, "No permission to list directory") return None list.sort(key=lambda a: a.lower()) f = BytesIO() # ToDo make resources local (now files are loaded externally) f.write(b""" <head> <meta charset="utf-8"> <title>Drag and Drop File Uploading</title> <meta name="viewport" content="width=device-width,initial-scale=1" /> <link rel="stylesheet" href="//fonts.googleapis.com/css?family=Roboto:300,300italic,400" /> <script type="text/javascript" src="report/html/jquery.min.js"></script> <style> body { font-family: Roboto, sans-serif; color: #0f3c4b; background-color: #e5edf1; padding: 5rem 1.25rem; /* 80 20 */ } .container { width: 100%; max-width: 680px; /* 800 */ text-align: center; margin: 0 auto; } .container h1 { font-size: 42px; font-weight: 300; color: #0f3c4b; margin-bottom: 40px; } .container h1 a:hover, .container h1 a:focus { color: #39bfd3; } .container nav { margin-bottom: 40px; } .container nav a { border-bottom: 2px solid #c8dadf; display: inline-block; padding: 4px 8px; margin: 0 5px; } .container nav a.is-selected { font-weight: 700; color: #39bfd3; border-bottom-color: currentColor; } .container nav a:not( .is-selected ):hover, .container nav a:not( .is-selected ):focus { border-bottom-color: #0f3c4b; } .container footer { color: #92b0b3; margin-top: 40px; } .container footer p + p { margin-top: 1em; } .container footer a:hover, .container footer a:focus { color: #39bfd3; } .box { font-size: 1.25rem; /* 20 */ background-color: #c8dadf; position: relative; padding: 100px 20px; } .box.has-advanced-upload { outline: 2px dashed #92b0b3; outline-offset: -10px; -webkit-transition: outline-offset .15s ease-in-out, background-color .15s linear; transition: outline-offset .15s ease-in-out, background-color .15s linear; } .box.is-dragover { outline-offset: -20px; outline-color: #c8dadf; background-color: #fff; } .box__dragndrop, .box__icon { display: none; } .box.has-advanced-upload .box__dragndrop { display: inline; } .box.has-advanced-upload .box__icon { width: 100%; height: 80px; fill: #92b0b3; display: block; margin-bottom: 40px; } .box.is-uploading .box__input, .box.is-success .box__input, .box.is-error .box__input { visibility: hidden; } .box__uploading, .box__success, .box__error { display: none; } .box.is-uploading .box__uploading, .box.is-success .box__success, .box.is-error .box__error { display: block; position: absolute; top: 50%; right: 0; left: 0; -webkit-transform: translateY( -50% ); transform: translateY( -50% ); } .box__uploading { font-style: italic; } .box__success { -webkit-animation: appear-from-inside .25s ease-in-out; animation: appear-from-inside .25s ease-in-out; } @-webkit-keyframes appear-from-inside { from { -webkit-transform: translateY( -50% ) scale( 0 ); } 75% { -webkit-transform: translateY( -50% ) scale( 1.1 ); } to { -webkit-transform: translateY( -50% ) scale( 1 ); } } @keyframes appear-from-inside { from { transform: translateY( -50% ) scale( 0 ); } 75% { transform: translateY( -50% ) scale( 1.1 ); } to { transform: translateY( -50% ) scale( 1 ); } } .box__restart { font-weight: 700; } .box__restart:focus, .box__restart:hover { color: #39bfd3; } .js .box__file { width: 0.1px; height: 0.1px; opacity: 0; overflow: hidden; position: absolute; z-index: -1; } .js .box__file + label { max-width: 80%; text-overflow: ellipsis; white-space: nowrap; cursor: pointer; display: inline-block; overflow: hidden; } .js .box__file + label:hover strong, .box__file:focus + label strong, .box__file.has-focus + label strong { color: #39bfd3; } .js .box__file:focus + label, .js .box__file.has-focus + label { outline: 1px dotted #000; outline: -webkit-focus-ring-color auto 5px; } .js .box__file + label * { /* pointer-events: none; */ /* in case of FastClick lib use */ } .no-js .box__file + label { display: none; } .no-js .box__button { display: block; } .box__button { font-weight: 700; color: #e5edf1; background-color: #39bfd3; display: none; padding: 8px 16px; margin: 40px auto 0; } .box__button:hover, .box__button:focus { background-color: #0f3c4b; } .killserverbutton { -moz-box-shadow:inset 0px 39px 0px -24px #e67a73; -webkit-box-shadow:inset 0px 39px 0px -24px #e67a73; box-shadow:inset 0px 39px 0px -24px #e67a73; background-color:#e4685d; -moz-border-radius:4px; -webkit-border-radius:4px; border-radius:4px; border:1px solid #ffffff; display:inline-block; cursor:pointer; color:#ffffff; font-family:Arial; font-size:15px; padding:6px 15px; text-decoration:none; text-shadow:0px 1px 0px #b23e35; } .killserverbutton:hover { background-color:#eb675e; } .killserverbutton:active { position:relative; top:1px; } </style> <!-- remove this if you use Modernizr --> <script>(function(e,t,n){var r=e.querySelectorAll("html")[0];r.className=r.className.replace(/(^|\s)no-js(\s|$)/,"$1js$2")})(document,window,0);</script> </head> <body> <center> <form id="killserverform" action="/KILLSERVERCOMMAND" target="" method="POST"> <input type="text" name="KILLSERVERCOMMAND" value="Mickey" hidden> <input type="submit" value="KILL SERVER" class="killserverbutton"> </form> <script> $('#killserverform').submit(function() { alert("Server will shut down! Bye!"); return true; // return false to cancel form action }); </script> </center> <div class="container" role="main"> <form action="/" ENCTYPE="multipart/form-data" method="post" novalidate class="box"> <div class="box__input"> <svg class="box__icon" xmlns="http://www.w3.org/2000/svg" width="50" height="43" viewBox="0 0 50 43"><path d="M48.4 26.5c-.9 0-1.7.7-1.7 1.7v11.6h-43.3v-11.6c0-.9-.7-1.7-1.7-1.7s-1.7.7-1.7 1.7v13.2c0 .9.7 1.7 1.7 1.7h46.7c.9 0 1.7-.7 1.7-1.7v-13.2c0-1-.7-1.7-1.7-1.7zm-24.5 6.1c.3.3.8.5 1.2.5.4 0 .9-.2 1.2-.5l10-11.6c.7-.7.7-1.7 0-2.4s-1.7-.7-2.4 0l-7.1 8.3v-25.3c0-.9-.7-1.7-1.7-1.7s-1.7.7-1.7 1.7v25.3l-7.1-8.3c-.7-.7-1.7-.7-2.4 0s-.7 1.7 0 2.4l10 11.6z"/></svg> """) f.write( b"""<input type="file" name="file" id="file" class="box__file" data-multiple-caption="{count} files selected" multiple accept=\"""" ) for apptype in ServerWrapper.apptypes: f.write(bytes(str(apptype) + ", ", 'utf-8')) f.write(b"""\" />""") f.write(b""" <label for="file" id="nameup"><strong>Choose a file</strong><span class="box__dragndrop"> or drag it here</span>.</label> <button type="submit" class="box__button">Upload</button> </div> <div class="box__uploading">Uploading…</div> <div class="box__success">Done! <a href="/" class="box__restart2" id="done_link" onclick="javascript:event.target.port=""" ) f.write(bytearray(str(ServerWrapper.REPORT_SERVER_PORT), 'utf8')) f.write(b"""" role="button" target="_blank">Open report!</a></div> <div class="box__error">Error! <span></span>. <a href="/?" class="box__restart" role="button">Try again!</a></div> </form> </div> <script> 'use strict'; ;( function ( document, window, index ) { // feature detection for drag&drop upload var isAdvancedUpload = function() { var div = document.createElement( 'div' ); return ( ( 'draggable' in div ) || ( 'ondragstart' in div && 'ondrop' in div ) ) && 'FormData' in window && 'FileReader' in window; }(); // applying the effect for every form var forms = document.querySelectorAll( '.box' ); Array.prototype.forEach.call( forms, function( form ) { var input = form.querySelector( 'input[type="file"]' ), label = form.querySelector( 'label' ), errorMsg = form.querySelector( '.box__error span' ), restart = form.querySelectorAll( '.box__restart' ), droppedFiles = false, showFiles = function( files ) { label.textContent = files.length > 1 ? ( input.getAttribute( 'data-multiple-caption' ) || '' ).replace( '{count}', files.length ) : files[ 0 ].name; }, triggerFormSubmit = function() { var event = document.createEvent( 'HTMLEvents' ); event.initEvent( 'submit', true, false ); form.dispatchEvent( event ); }; // letting the server side to know we are going to make an Ajax request var ajaxFlag = document.createElement( 'input' ); ajaxFlag.setAttribute( 'type', 'hidden' ); ajaxFlag.setAttribute( 'name', 'ajax' ); ajaxFlag.setAttribute( 'value', 1 ); form.appendChild( ajaxFlag ); // automatically submit the form on file select input.addEventListener( 'change', function( e ) { showFiles( e.target.files ); triggerFormSubmit(); }); // drag&drop files if the feature is available if( isAdvancedUpload ) { form.classList.add( 'has-advanced-upload' ); // letting the CSS part to know drag&drop is supported by the browser [ 'drag', 'dragstart', 'dragend', 'dragover', 'dragenter', 'dragleave', 'drop' ].forEach( function( event ) { form.addEventListener( event, function( e ) { // preventing the unwanted behaviours e.preventDefault(); e.stopPropagation(); }); }); [ 'dragover', 'dragenter' ].forEach( function( event ) { form.addEventListener( event, function() { form.classList.add( 'is-dragover' ); }); }); [ 'dragleave', 'dragend', 'drop' ].forEach( function( event ) { form.addEventListener( event, function() { form.classList.remove( 'is-dragover' ); }); }); form.addEventListener( 'drop', function( e ) { droppedFiles = e.dataTransfer.files; // the files that were dropped showFiles( droppedFiles ); triggerFormSubmit(); }); } // if the form was submitted form.addEventListener( 'submit', function( e ) { // preventing the duplicate submissions if the current one is in progress if( form.classList.contains( 'is-uploading' ) ) return false; form.classList.add( 'is-uploading' ); form.classList.remove( 'is-error' ); if( isAdvancedUpload ) // ajax file upload for modern browsers { e.preventDefault(); // gathering the form data var ajaxData = new FormData( form ); if( droppedFiles ) { Array.prototype.forEach.call( droppedFiles, function( file ) { ajaxData.append( input.getAttribute( 'name' ), file ); document.getElementById("done_link").setAttribute("href", file.name.replace(/\./g, "_")+"/report/start.html"); }); } // ajax request var ajax = new XMLHttpRequest(); ajax.open( form.getAttribute( 'method' ), form.getAttribute( 'action' ), true ); ajax.onload = function() { form.classList.remove( 'is-uploading' ); if( ajax.status == 408 ) { alert( 'Filetype is not allowed'); } else if( ajax.status >= 200 && ajax.status < 400 ) { var data = ajax.responseText; console.log(data) form.classList.add( data.includes("Success:") == true ? 'is-success' : 'is-error' ); if( !data.includes("Success:") ) errorMsg.textContent = data.error; if (data.includes("Success:") ) document.getElementById("done_link").setAttribute("href", nameup.innerHTML.replace(/\./g, "_")+"/report/start.html"); } else alert( 'Error. Please, contact the webmaster!' ); }; ajax.onerror = function() { form.classList.remove( 'is-uploading' ); alert( 'Error. Please, try again! Filetype is probably not supported.' ); }; ajax.send( ajaxData ); } else // fallback Ajax solution upload for older browsers { alert(1); var iframeName = 'uploadiframe' + new Date().getTime(), iframe = document.createElement( 'iframe' ); $iframe = $( '<iframe name="' + iframeName + '" style="display: none;"></iframe>' ); iframe.setAttribute( 'name', iframeName ); iframe.style.display = 'none'; document.body.appendChild( iframe ); form.setAttribute( 'target', iframeName ); iframe.addEventListener( 'load', function() { var data = JSON.parse( iframe.contentDocument.body.innerHTML ); form.classList.remove( 'is-uploading' ) form.classList.add( data.success == true ? 'is-success' : 'is-error' ) form.removeAttribute( 'target' ); if( !data.success ) errorMsg.textContent = data.error; iframe.parentNode.removeChild( iframe ); }); } }); // restart the form if has a state of error/success Array.prototype.forEach.call( restart, function( entry ) { entry.addEventListener( 'click', function( e ) { e.preventDefault(); form.classList.remove( 'is-error', 'is-success' ); input.click(); }); }); // Firefox focus bug fix for file input input.addEventListener( 'focus', function(){ input.classList.add( 'has-focus' ); }); input.addEventListener( 'blur', function(){ input.classList.remove( 'has-focus' ); }); }); }( document, window, 0 )); </script> </body> </html> """) # f.write(b"<form ENCTYPE=\"multipart/form-data\" method=\"post\">") # f.write(b"<input name=\"file\" type=\"file\"/>") # f.write(b"<input type=\"submit\" value=\"upload\"/></form>\n") length = f.tell() f.seek(0) self.send_response(200) self.send_header("Content-type", "text/html") self.send_header("Content-Length", str(length)) self.end_headers() return f def translate_path(self, path): """Translate a /-separated PATH to the local filename syntax. Components that mean special things to the local file system (e.g. drive or directory names) are ignored. (XXX They should probably be diagnosed.) """ # abandon query parameters path = path.split('?', 1)[0] path = path.split('#', 1)[0] path = posixpath.normpath(urllib.parse.unquote(path)) words = path.split('/') words = [_f for _f in words if _f] path = os.getcwd() for word in words: drive, word = os.path.splitdrive(word) head, word = os.path.split(word) if word in (os.curdir, os.pardir): continue path = os.path.join(path, word) return path def copyfile(self, source, outputfile): """Copy all data between two file objects. The SOURCE argument is a file object open for reading (or anything with a read() method) and the DESTINATION argument is a file object open for writing (or anything with a write() method). The only reason for overriding this would be to change the block size or perhaps to replace newlines by CRLF -- note however that this the default server uses this to copy binary data as well. """ shutil.copyfileobj(source, outputfile) def guess_type(self, path): """Guess the type of a file. Argument is a PATH (a filename). Return value is a string of the form type/subtype, usable for a MIME Content-type header. The default implementation looks the file's extension up in the table self.extensions_map, using application/octet-stream as a default; however it would be permissible (if slow) to look inside the data to make a better guess. """ base, ext = posixpath.splitext(path) if ext in self.extensions_map: return self.extensions_map[ext] ext = ext.lower() if ext in self.extensions_map: return self.extensions_map[ext] else: return self.extensions_map[''] if not mimetypes.inited: mimetypes.init() # try to read system mime.types extensions_map = mimetypes.types_map.copy() extensions_map.update({ '': 'application/octet-stream', # Default '.py': 'text/plain', '.c': 'text/plain', '.h': 'text/plain', })
import os import re import sys import base64 import mimetypes from grit import lazy_re from grit import util # There is a python bug that makes mimetypes crash if the Windows # registry contains non-Latin keys ( http://bugs.python.org/issue9291 # ). Initing manually and blocking external mime-type databases will # prevent that bug and if we add svg manually, it will still give us # the data we need. mimetypes.init([]) mimetypes.add_type('image/svg+xml', '.svg') DIST_DEFAULT = 'chromium' DIST_ENV_VAR = 'CHROMIUM_BUILD' DIST_SUBSTR = '%DISTRIBUTION%' # Matches beginning of an "if" block with trailing spaces. _BEGIN_IF_BLOCK = lazy_re.compile( '<if [^>]*?expr="(?P<expression>[^"]*)"[^>]*?>\s*') # Matches ending of an "if" block with preceding spaces. _END_IF_BLOCK = lazy_re.compile('\s*</if>') # Used by DoInline to replace various links with inline content. _STYLESHEET_RE = lazy_re.compile(
def __init__(self, conf, memcache=None, logger=None, account_ring=None, container_ring=None, object_ring=None): if conf is None: conf = {} if logger is None: self.logger = get_logger(conf, log_route='proxy-server') else: self.logger = logger swift_dir = conf.get('swift_dir', '/etc/swift') self.node_timeout = int(conf.get('node_timeout', 10)) self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.client_timeout = int(conf.get('client_timeout', 60)) self.put_queue_depth = int(conf.get('put_queue_depth', 10)) self.object_chunk_size = int(conf.get('object_chunk_size', 65536)) self.client_chunk_size = int(conf.get('client_chunk_size', 65536)) self.trans_id_suffix = conf.get('trans_id_suffix', '') self.error_suppression_interval = \ int(conf.get('error_suppression_interval', 60)) self.error_suppression_limit = \ int(conf.get('error_suppression_limit', 10)) self.recheck_container_existence = \ int(conf.get('recheck_container_existence', 60)) self.recheck_account_existence = \ int(conf.get('recheck_account_existence', 60)) self.allow_account_management = \ config_true_value(conf.get('allow_account_management', 'no')) self.object_post_as_copy = \ config_true_value(conf.get('object_post_as_copy', 'true')) self.resellers_conf = ConfigParser() self.resellers_conf.read(os.path.join(swift_dir, 'resellers.conf')) self.object_ring = object_ring or Ring(swift_dir, ring_name='object') self.container_ring = container_ring or Ring(swift_dir, ring_name='container') self.account_ring = account_ring or Ring(swift_dir, ring_name='account') self.memcache = memcache mimetypes.init(mimetypes.knownfiles + [os.path.join(swift_dir, 'mime.types')]) self.account_autocreate = \ config_true_value(conf.get('account_autocreate', 'no')) self.expiring_objects_account = \ (conf.get('auto_create_account_prefix') or '.') + \ 'expiring_objects' self.expiring_objects_container_divisor = \ int(conf.get('expiring_objects_container_divisor') or 86400) self.max_containers_per_account = \ int(conf.get('max_containers_per_account') or 0) self.max_containers_whitelist = [ a.strip() for a in conf.get('max_containers_whitelist', '').split(',') if a.strip() ] self.deny_host_headers = [ host.strip() for host in conf.get('deny_host_headers', '').split(',') if host.strip() ] self.rate_limit_after_segment = \ int(conf.get('rate_limit_after_segment', 10)) self.rate_limit_segments_per_sec = \ int(conf.get('rate_limit_segments_per_sec', 1)) self.log_handoffs = config_true_value(conf.get('log_handoffs', 'true')) self.cors_allow_origin = [ a.strip() for a in conf.get('cors_allow_origin', '').split(',') if a.strip() ] self.node_timings = {} self.timing_expiry = int(conf.get('timing_expiry', 300)) self.sorting_method = conf.get('sorting_method', 'shuffle').lower() self.allow_static_large_object = config_true_value( conf.get('allow_static_large_object', 'true')) value = conf.get('request_node_count', '2 * replicas').lower().split() if len(value) == 1: value = int(value[0]) self.request_node_count = lambda r: value elif len(value) == 3 and value[1] == '*' and value[2] == 'replicas': value = int(value[0]) self.request_node_count = lambda r: value * r.replica_count else: raise ValueError('Invalid request_node_count value: %r' % ''.join(value))
def GET(self, courseid, taskid, is_LTI): """ GET request """ username = self.user_manager.session_username() # Fetch the course try: course = self.course_factory.get_course(courseid) except CourseNotFoundException as ex: raise NotFound(description=str(ex)) if is_LTI and not self.user_manager.course_is_user_registered(course): self.user_manager.course_register_user(course, force=True) if not self.user_manager.course_is_open_to_user(course, username, is_LTI): return handle_course_unavailable(self.cp.app.get_homepath(), self.template_helper, self.user_manager, course) is_staff = self.user_manager.has_staff_rights_on_course(course, username) try: task = course.get_task(taskid) if not self.user_manager.task_is_visible_by_user(task, username, is_LTI): return self.template_helper.render("task_unavailable.html") except TaskNotFoundException: raise NotFound() user_task_list = course.get_task_dispenser().get_user_task_list([username])[username] if taskid not in user_task_list: previous_taskid = None next_taskid = None else: # Compute previous and next taskid index = user_task_list.index(taskid) previous_taskid = user_task_list[index - 1] if index > 0 else None next_taskid = user_task_list[index + 1] if index < len(user_task_list) - 1 else None self.user_manager.user_saw_task(username, courseid, taskid) is_staff = self.user_manager.has_staff_rights_on_course(course, username) userinput = flask.request.args if "submissionid" in userinput and "questionid" in userinput: # Download a previously submitted file submission = self.submission_manager.get_submission(userinput["submissionid"], user_check=not is_staff) if submission is None: raise self.cp.app.notfound(message=_("Submission doesn't exist.")) sinput = self.submission_manager.get_input_from_submission(submission, True) if userinput["questionid"] not in sinput: raise NotFound() if isinstance(sinput[userinput["questionid"]], dict): # File uploaded previously mimetypes.init() mime_type = mimetypes.guess_type(urllib.request.pathname2url(sinput[userinput["questionid"]]['filename'])) return Response(response=sinput[userinput["questionid"]]['value'], content_type=mime_type[0]) else: # Other file, download it as text return Response(response=sinput[userinput["questionid"]], content_type='text/plain') else: # Generate random inputs and save it into db random.seed(str(username if username is not None else "") + taskid + courseid + str( time.time() if task.regenerate_input_random() else "")) random_input_list = [random.random() for i in range(task.get_number_input_random())] user_task = self.database.user_tasks.find_one_and_update( { "courseid": task.get_course_id(), "taskid": task.get_id(), "username": self.user_manager.session_username() }, { "$set": {"random": random_input_list} }, return_document=ReturnDocument.AFTER ) submissionid = user_task.get('submissionid', None) eval_submission = self.database.submissions.find_one({'_id': ObjectId(submissionid)}) if submissionid else None students = [self.user_manager.session_username()] if task.is_group_task() and not self.user_manager.has_admin_rights_on_course(course, username): group = self.database.groups.find_one({"courseid": task.get_course_id(), "students": self.user_manager.session_username()}) if group is not None: students = group["students"] # we don't care for the other case, as the student won't be able to submit. submissions = self.submission_manager.get_user_submissions(task) if self.user_manager.session_logged_in() else [] user_info = self.user_manager.get_user_info(username) # Visible tags course_tags = course.get_tags() task_categories = task.get_categories() visible_tags = [course_tags[category] for category in task_categories if course_tags[category].is_visible_for_student() or self.user_manager.has_staff_rights_on_course(course)] # Problem dict pdict = {problem.get_id(): problem.get_type() for problem in task.get_problems()} is_input_list = {problem.get_id(): 1 if problem.input_type() == list else 0 for problem in task.get_problems()} # Display the task itself return self.template_helper.render("task.html", user_info=user_info, course=course, task=task, submissions=submissions, students=students, eval_submission=eval_submission, user_task=user_task, previous_taskid=previous_taskid, next_taskid=next_taskid, webterm_link=self.webterm_link, input_random_list=random_input_list, visible_tags=visible_tags, pdict=pdict, is_input_list=is_input_list)
class JadeRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): def log_message(self, format, *args): return def do_GET(self): path = self.path path = path.split('?', 1)[0] path = path.split('#', 1)[0] path = path.replace('/', '') ctype = self.guess_type(path) try: f = open(path, 'rb') except IOError: self.send_error(404, "File not found") return None try: self.send_response(200) self.send_header("Content-type", ctype) fs = os.fstat(f.fileno()) self.send_header("Content-Length", str(fs[6])) self.send_header("Last-Modified", self.date_time_string(fs.st_mtime)) self.end_headers() shutil.copyfileobj(f, self.wfile) f.close() except: f.close() raise def do_POST(self): # determine key, value ctype, pdict = cgi.parse_header(self.headers.getheader('content-type')) if ctype == 'multipart/form-data': postvars = cgi.parse_multipart(self.rfile, pdict) elif ctype == 'application/x-www-form-urlencoded': length = int(self.headers.getheader('content-length')) postvars = cgi.parse_qs(self.rfile.read(length), keep_blank_values=1) else: postvars = {} key = postvars.get('key', [None])[0] value = postvars.get('value', [None])[0] response = '' if value is None: # return stored value c.execute('select val from key_value where key=?;', (key, )) row = c.fetchone() if row is not None: response = row[0] else: # update stored value c.execute( 'insert or replace into key_value (key,val) values (?,?);', (key, value)) db.commit() self.send_response(200) self.send_header("Content-type", 'text/plain') self.send_header("Content-Length", str(len(response))) self.end_headers() self.wfile.write(response) def guess_type(self, path): base, ext = posixpath.splitext(path) if ext in self.extensions_map: return self.extensions_map[ext] ext = ext.lower() if ext in self.extensions_map: return self.extensions_map[ext] else: return self.extensions_map[''] if not mimetypes.inited: mimetypes.init() # try to read system mime.types extensions_map = mimetypes.types_map.copy() extensions_map.update({ '': 'application/octet-stream', # Default })
def init_mimetypes(mimetypes): # this is a function so it can be unittested if hasattr(mimetypes, 'init'): mimetypes.init() return True return False
import sys import time from tempfile import NamedTemporaryFile import sacred.optional as opt from sacred.commandline_options import CommandLineOption from sacred.dependencies import get_digest from sacred.observers.base import RunObserver from sacred.observers.queue import QueueObserver from sacred.serializer import flatten from sacred.utils import ObserverError DEFAULT_MONGO_PRIORITY = 30 # This ensures consistent mimetype detection across platforms. mimetypes.init(files=[]) def force_valid_bson_key(key): key = str(key) if key.startswith('$'): key = '@' + key[1:] key = key.replace('.', ',') return key def force_bson_encodeable(obj): import bson if isinstance(obj, dict): try: bson.BSON.encode(obj, check_keys=True)
# SPDX-License-Identifier: MIT import os import sys import rq import shutil import tempfile import numpy as np from PIL import Image from traceback import print_exception from ast import literal_eval import mimetypes _SCRIPT_DIR = os.path.realpath(os.path.dirname(__file__)) _MEDIA_MIMETYPES_FILE = os.path.join(_SCRIPT_DIR, "media.mimetypes") mimetypes.init(files=[_MEDIA_MIMETYPES_FILE]) import django_rq from django.conf import settings from django.db import transaction from ffmpy import FFmpeg from pyunpack import Archive from distutils.dir_util import copy_tree from . import models from .log import slogger ############################# Low Level server API def create(tid, data):
def _init_mimetypes(): global _mt_inited import mimetypes mimetypes.init([P('mime.types')]) _mt_inited = True
class TestRequestHandler(BaseHTTPRequestHandler): def __init__(self, request, client_address, server): BaseHTTPRequestHandler.__init__(self, request, client_address, server) self.protocol_version = 'HTTP/1.1' def do_GET(self): self.handle_data() def do_POST(self): self.form = cgi.FieldStorage( fp=self.rfile, headers=self.headers, environ={'REQUEST_METHOD':'POST', 'CONTENT_TYPE':self.headers['Content-Type'], }, keep_blank_values=True, strict_parsing=False) self.handle_data() def handle_data(self): if self.path == '/': p = '/html/fp.html' else: p = self.path path = self.translate_path(p) if not os.path.exists(path): p = '/html'+p path = self.translate_path(p) ctype = self.guess_type(path) try: f = open(path) except IOError: print 'File not found %s' % path self.send_error(404, 'File not found') return self.send_response(200) self.send_header('Content-type', ctype) self.send_header('Last-Modified', self.date_time_string()) self.end_headers() self.copyfile(f, self.wfile) f.close() def translate_path(self, path): path = path.decode('utf-8') path = urlparse.urlparse(path)[2] path = posixpath.normpath(urllib.unquote(path)) words = path.split('/') words = filter(None, words) path = os.getcwd() for word in words: drive, word = os.path.splitdrive(word) head, word = os.path.split(word) if word in (os.curdir, os.pardir): continue path = os.path.join(path, word) return path def copyfile(self, source, outputfile): shutil.copyfileobj(source, outputfile) def guess_type(self, path): base, ext = posixpath.splitext(path) if ext in self.extensions_map: return self.extensions_map[ext] ext = ext.lower() if ext in self.extensions_map: return self.extensions_map[ext] else: return self.extensions_map[''] if not mimetypes.inited: mimetypes.init() # try to read system mime.types extensions_map = mimetypes.types_map.copy() extensions_map.update({ '': 'application/octet-stream', # Default })
class SimpleHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): """Simple HTTP request handler with GET/HEAD/POST commands. This serves files from the current directory and any of its subdirectories. The MIME type for files is determined by calling the .guess_type() method. And can reveive file uploaded by client. The GET/HEAD/POST requests are identical except that the HEAD request omits the actual contents of the file. """ counter = Counter() def do_HEAD(self): """Serve a HEAD request.""" f = self.send_head() if f: f.close() def is_authenticated(self): auth_header = self.headers.getheader('Authorization') return auth_header and auth_header == 'Basic ' + key() def do_AUTHHEAD(self): self.send_response(401) self.send_header('WWW-Authenticate', 'Basic realm=\"Test\"') self.send_header('Content-type', 'text/html') self.end_headers() def try_authenticate(self): if not self.is_authenticated(): self.do_AUTHHEAD() print 'not authenticated' self.wfile.write('not authenticated') return False return True def do_GET(self): if not self.try_authenticate(): return print 'authenticated' f = self.send_head() if f: self.copyfile(f, self.wfile) f.close() def do_POST(self): if not self.try_authenticate(): return print 'authenticated' """Serve a POST request.""" r, info = self.deal_post_data() print r, info, "by: ", self.client_address f = StringIO() f.write('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">') f.write("<html>\n<title>Upload Result Page</title>\n") f.write("<body>\n<h2>Upload Result Page</h2>\n") f.write("<hr>\n") if r: f.write("<strong>Success:</strong>") else: f.write("<strong>Failed:</strong>") f.write(info) f.write("<br><a href=\"%s\">back</a>" % urllib.quote(self.headers['referer'])) f.write("<hr><small>Powerd By: bones7456, check new version at ") f.write("<a href=\"http://li2z.cn/?s=SimpleHTTPServerWithUpload\">") f.write("</body>\n</html>\n") length = f.tell() f.seek(0) self.send_response(200) self.send_header("Content-type", "text/html") self.send_header("Content-Length", str(length)) self.end_headers() if f: self.copyfile(f, self.wfile) f.close() def deal_post_data(self): boundary = self.headers.plisttext.split("=")[1] remainbytes = int(self.headers['content-length']) line = self.rfile.readline() remainbytes -= len(line) if not boundary in line: return (False, "Content NOT begin with boundary") line = self.rfile.readline() remainbytes -= len(line) fn = re.findall(r'Content-Disposition.*name="file"; filename="(.*)"', line) if not fn: return (False, "Can't find out file name...") path = self.url_path_to_file_path(self.path) fn = os.path.join(path, fn[0].replace('\\', '/').split('/')[-1]) if os.path.exists(fn): return (False, "The path already exists, you cannot overwrite it.") line = self.rfile.readline() remainbytes -= len(line) line = self.rfile.readline() remainbytes -= len(line) try: out = open(fn, 'wb') except IOError: return ( False, "Can't create file to write, do you have permission to write?") preline = self.rfile.readline() remainbytes -= len(preline) while remainbytes > 0: line = self.rfile.readline() remainbytes -= len(line) if boundary in line: preline = preline[0:-1] if preline.endswith('\r'): preline = preline[0:-1] out.write(preline) out.close() return (True, "File '%s' upload success!" % fn) else: out.write(preline) preline = line return (False, "Unexpect Ends of data.") def send_head(self): """Common code for GET and HEAD commands. This sends the response code and MIME headers. Return value is either a file object (which has to be copied to the outputfile by the caller unless the command was HEAD, and must be closed by the caller under all circumstances), or None, in which case the caller has nothing further to do. """ print 'url_path', self.path file_path = self.url_path_to_file_path(self.path) print 'file_path', file_path f = None if os.path.isdir(file_path): if not self.path.endswith('/'): # redirect browser - doing basically what apache does self.send_response(301) self.send_header("Location", self.path + "/") self.end_headers() return None for index in "index.html", "index.htm": index = os.path.join(file_path, index) if os.path.exists(index): file_path = index break self.counter.incr_counter(file_path) if os.path.isdir(file_path): return self.list_directory(file_path) ctype = self.guess_type(file_path) try: # Always read in binary mode. Opening files in text mode may cause # newline translations, making the actual size of the content # transmitted *less* than the content-length! f = open(file_path, 'rb') except IOError: self.send_error(404, "File not found " + file_path) return None self.send_response(200) self.send_header("Content-type", ctype) fs = os.fstat(f.fileno()) self.send_header("Content-Length", str(fs[6])) self.send_header("Last-Modified", self.date_time_string(fs.st_mtime)) self.end_headers() return f def list_directory(self, dir_path): """Helper to produce a directory listing (absent index.html). Return value is either a file object, or None (indicating an error). In either case, the headers are sent, making the interface the same as for send_head(). """ try: list = os.listdir(dir_path) except os.error: self.send_error(404, "No permission to list directory") return None list.sort(key=lambda a: a.lower()) if dir_path != '/': list = ['..'] + list f = StringIO() displaypath = cgi.escape(urllib.unquote(self.path)) f.write('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">') f.write("<html>\n<title>Directory listing for %s</title>\n" % displaypath) f.write( "<body>\n<h2>Directory listing for %s (frequently used directories are more reddish)</h2>\n" % displaypath) f.write("<hr>\n") f.write("<form ENCTYPE=\"multipart/form-data\" method=\"post\">") f.write("<input name=\"file\" type=\"file\"/>") f.write("<input type=\"submit\" value=\"upload\"/></form>\n") f.write("<hr>\n<ul>\n") tot_counts = 0 for name in list: child_file_path = posixpath.normpath(os.path.join(dir_path, name)) counts = self.counter.read_counter(child_file_path) print child_file_path, counts tot_counts += counts # avoid divide by zero error if tot_counts == 0: tot_counts += 1 for name in list: child_file_path = posixpath.normpath(os.path.join(dir_path, name)) displayname = linkname = name # Append / for directories or @ for symbolic links if os.path.isdir(child_file_path): displayname = name + "/" linkname = name + "/" if os.path.islink(child_file_path): displayname = name + "@" # Note: a link to a directory displays with @ and links with / counts = self.counter.read_counter(child_file_path) # red portion of rgb value. with **0.2, it's overall more reddish rgb_r = 255 * (float(counts) / tot_counts)**0.2 f.write('<li><a style="color:rgb(%d,0,0)" href="%s">%s</a>\n' % (rgb_r, urllib.quote(linkname), cgi.escape(displayname))) f.write("</ul>\n<hr>\n</body>\n</html>\n") length = f.tell() f.seek(0) self.send_response(200) self.send_header("Content-type", "text/html") self.send_header("Content-Length", str(length)) self.end_headers() return f def url_path_to_file_path(self, url_path): # 'base_url/' => '/' # 'base_url/home/' => '/home' # 'base_url/home/test/' => '/home/test' # 'base_url/home/test.zip' => '/home/test.zip' # 'base_url/home/%22test%22/' => '/home/"test"' url_path = url_path[len(settings.base_url):] # abandon query parameters url_path = url_path.split('?', 1)[0] url_path = url_path.split('#', 1)[0] url_path = posixpath.normpath(urllib.unquote(url_path)) return url_path def copyfile(self, source, outputfile): """Copy all data between two file objects. The SOURCE argument is a file object open for reading (or anything with a read() method) and the DESTINATION argument is a file object open for writing (or anything with a write() method). The only reason for overriding this would be to change the block size or perhaps to replace newlines by CRLF -- note however that this the default server uses this to copy binary data as well. """ shutil.copyfileobj(source, outputfile) def guess_type(self, path): """Guess the type of a file. Argument is a PATH (a filename). Return value is a string of the form type/subtype, usable for a MIME Content-type header. The default implementation looks the file's extension up in the table self.extensions_map, using application/octet-stream as a default; however it would be permissible (if slow) to look inside the data to make a better guess. """ base, ext = posixpath.splitext(path) if ext in self.extensions_map: return self.extensions_map[ext] ext = ext.lower() if ext in self.extensions_map: return self.extensions_map[ext] else: return self.extensions_map[''] if not mimetypes.inited: mimetypes.init() # try to read system mime.types extensions_map = mimetypes.types_map.copy() extensions_map.update({ '': 'application/octet-stream', # Default '.py': 'text/plain', '.c': 'text/plain', '.h': 'text/plain', })
class RangeHTTPRequestHandler(http.server.BaseHTTPRequestHandler): """Simple HTTP request handler with GET and HEAD commands. This serves files from the current directory and any of its subdirectories. The MIME type for files is determined by calling the .guess_type() method. The GET and HEAD requests are identical except that the HEAD request omits the actual contents of the file. """ server_version = "RangeHTTP/" + __version__ def do_GET(self): """Serve a GET request.""" f, start_range, end_range = self.send_head() print("Got values of {} and {}".format(start_range, end_range)) if f: f.seek(start_range, 0) chunk = 0x1000 total = 0 while chunk > 0: if start_range + chunk > end_range: chunk = end_range - start_range if _bandwidth != 0: time_to_sleep = float(float(chunk) / float(_bandwidth)) time.sleep(time_to_sleep) try: self.wfile.write(f.read(chunk)) except Exception: break total += chunk start_range += chunk f.close() def do_HEAD(self): """Serve a HEAD request.""" f, start_range, end_range = self.send_head() if f: f.close() def send_head(self): """Common code for GET and HEAD commands. This sends the response code and MIME headers. Return value is either a file object (which has to be copied to the outputfile by the caller unless the command was HEAD, and must be closed by the caller under all circumstances), or None, in which case the caller has nothing further to do. """ path = self.translate_path(self.path) f = None if os.path.isdir(path): if not self.path.endswith("/"): # redirect browser self.send_response(301) self.send_header("Location", self.path + "/") self.end_headers() return (None, 0, 0) for index in "index.html", "index.html": index = os.path.join(path, index) if os.path.exists(index): path = index break else: return self.list_directory(path) ctype = self.guess_type(path) try: # Always read in binary mode. Opening files in text mode may cause # newline translations, making the actual size of the content # transmitted *less* than the content-length! f = open(path, "rb") except IOError: self.send_error(404, "File not found") return (None, 0, 0) if "Range" in self.headers: self.send_response(206) # partial content response else: self.send_response(200) self.send_header("Content-type", ctype) file_size = os.path.getsize(path) start_range = 0 end_range = file_size self.send_header("Accept-Ranges", "bytes") if "Range" in self.headers: s, e = self.headers['range'][6:].split('-', 1) # bytes:%d-%d sl = len(s) el = len(e) if sl: start_range = int(s) if el: end_range = int(e) + 1 elif el: start_range = file_size - min(file_size, int(e)) self.send_header( "Content-Range", "bytes {}-{}/{}".format(start_range, end_range, file_size)) self.send_header("Content-Length", end_range - start_range) self.end_headers() print("Sending bytes {} to {}...".format(start_range, end_range)) return (f, start_range, end_range) def list_directory(self, path): """Helper to produce a directory listing (absent index.html). Return value is either a file object, or None (indicating an error). In either case, the headers are sent, making the interface the same as for send_head(). """ try: lst = os.listdir(path) except OSError: self.send_error(404, "Access Forbidden") return None lst.sort(key=lambda file_name: file_name.lower()) html_text = [] displaypath = html.escape(urllib.parse.unquote(self.path)) html_text.append( '<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">') html_text.append( "<html>\n<title>Directory listing for {}</title>\n".format( displaypath)) html_text.append( "<body>\n<h2>Directory listing for {}</h2>\n".format(displaypath)) html_text.append("<hr>\n<ul>\n") for name in lst: fullname = os.path.join(path, name) displayname = linkname = name if os.path.isdir(fullname): displayname = name + "/" linkname = name + "/" if os.path.islink(fullname): displayname = name + "@" html_text.append('<li><a href = "{}">{}</a>\n'.format( urllib.parse.quote(linkname), html.escape(displayname))) html_text.append('</ul>\n</hr>\n</body>\n</html>\n') byte_encoded_string = "\n".join(html_text).encode( "utf-8", "surrogateescape") f = io.BytesIO() f.write(byte_encoded_string) length = len(byte_encoded_string) f.seek(0) self.send_response(200) self.send_header("Content-type", "text/html") self.send_header("Content-length", str(length)) self.end_headers() return (f, 0, length) def translate_path(self, path): """Translate a /-separated PATH to the local filename syntax. Components that mean special things to the local file system (e.g. drive or directory names) are ignored. (XXX They should probably be diagnosed.) """ # abandon query parameters path = path.split("?", 1)[0] path = path.split("#", 1)[0] path = posixpath.normpath(urllib.parse.unquote(path)) words = path.split("/") words = filter(None, words) path = os.getcwd() for word in words: drive, word = os.path.splitdrive(word) head, word = os.path.split(word) if word in (os.curdir, os.pardir): continue path = os.path.join(path, word) return path def guess_type(self, path): """Guess the type of a file. Argument is a PATH (a filename). Return value is a string of the form type/subtype, usable for a MIME Content-type header. The default implementation looks the file's extension up in the table self.extensions_map, using application/octet-stream as a default; however it would be permissible (if slow) to look inside the data to make a better guess. """ base, ext = posixpath.splitext(path) if ext in self.extension_map: return self.extension_map[ext] ext = ext.lower() if ext in self.extension_map: return self.extension_map[ext] else: return self.extension_map[''] if not mimetypes.inited: mimetypes.init() extension_map = mimetypes.types_map.copy() extension_map.update({ '': 'application/octet-stream', # Default '.py': 'text/plain', '.c': 'text/plain', '.h': 'text/plain', '.mp4': 'video/mp4', '.ogg': 'video/ogg', '.java': 'text/plain', })
def _mimetypes(): import mimetypes mimetypes.init() return mimetypes
def setUp(self): # ensure all entries actually come from the Windows registry self.original_types_map = mimetypes.types_map.copy() mimetypes.types_map.clear() mimetypes.init() self.db = mimetypes.MimeTypes()
import Foundation from PyObjCTools import AppHelper from Foundation import NSBundle assert Foundation.NSThread.isMultiThreaded() # Don't load Python modules from system paths remove_paths = list(path for path in sys.path if 'site-packages' in path) for path in remove_paths: sys.path.remove(path) # Make mimetypes use our copy of the file in order to work with sandboxing resource_path = str(Foundation.NSBundle.mainBundle().resourcePath()) mime_path = os.path.join(resource_path, "mime.types") mimetypes.init(files=[mime_path]) class NSLogger(object): closed = False encoding = 'UTF-8' mode = 'w' name = '<NSLogger>' newlines = None softspace = 0 def close(self): pass def flush(self): pass
def _put_get_user_stage(tmpdir, conn_cnx, db_parameters, number_of_files=1, number_of_lines=1, from_path=True): # sanity check assert 'AWS_ACCESS_KEY_ID' in os.environ, 'AWS_ACCESS_KEY_ID is missing' assert 'AWS_SECRET_ACCESS_KEY' in os.environ, \ 'AWS_SECRET_ACCESS_KEY is missing' if not from_path: assert number_of_files == 1 tmp_dir = generate_k_lines_of_n_files(number_of_lines, number_of_files, tmp_dir=str(tmpdir.mkdir('data'))) files = os.path.join(tmp_dir, 'file*' if from_path else os.listdir(tmp_dir)[0]) file_stream = None if from_path else open(files, 'rb') stage_name = db_parameters['name'] + '_stage_{}_{}'.format( number_of_files, number_of_lines) with conn_cnx(user=db_parameters['user'], account=db_parameters['account'], password=db_parameters['password']) as cnx: cnx.cursor().execute(""" create or replace table {name} ( aa int, dt date, ts timestamp, tsltz timestamp_ltz, tsntz timestamp_ntz, tstz timestamp_tz, pct float, ratio number(6,2)) """.format(name=db_parameters['name'])) user_bucket = os.getenv('SF_AWS_USER_BUCKET', "sfc-dev1-regression/{}/reg".format(getuser())) cnx.cursor().execute(""" create or replace stage {stage_name} url='s3://{user_bucket}/{stage_name}-{number_of_files}-{number_of_lines}' credentials=( AWS_KEY_ID='{aws_key_id}' AWS_SECRET_KEY='{aws_secret_key}' ) """.format(stage_name=stage_name, user_bucket=user_bucket, aws_key_id=os.getenv('AWS_ACCESS_KEY_ID'), aws_secret_key=os.getenv('AWS_SECRET_ACCESS_KEY'), number_of_files=number_of_files, number_of_lines=number_of_lines)) try: with conn_cnx(user=db_parameters['user'], account=db_parameters['account'], password=db_parameters['password']) as cnx: cnx.cursor().execute( "alter session set disable_put_and_get_on_external_stage = false" ) cnx.cursor().execute( "rm @{stage_name}".format(stage_name=stage_name)) put(cnx.cursor(), files, stage_name, from_path, file_stream=file_stream) cnx.cursor().execute("copy into {name} from @{stage_name}".format( name=db_parameters['name'], stage_name=stage_name)) c = cnx.cursor() try: c.execute("select count(*) from {name}".format( name=db_parameters['name'])) rows = 0 for rec in c: rows += rec[0] assert rows == number_of_files * number_of_lines, \ 'Number of rows' finally: c.close() cnx.cursor().execute( "rm @{stage_name}".format(stage_name=stage_name)) cnx.cursor().execute("copy into @{stage_name} from {name}".format( name=db_parameters['name'], stage_name=stage_name)) tmp_dir_user = str(tmpdir.mkdir('put_get_stage')) cnx.cursor().execute( "get @{stage_name}/ file://{tmp_dir_user}/".format( stage_name=stage_name, tmp_dir_user=tmp_dir_user)) for _, _, files in os.walk(tmp_dir_user): for file in files: mimetypes.init() _, encoding = mimetypes.guess_type(file) assert encoding == 'gzip', "exported file type" finally: if file_stream: file_stream.close() with conn_cnx(user=db_parameters['user'], account=db_parameters['account'], password=db_parameters['password']) as cnx: cnx.cursor().execute( "rm @{stage_name}".format(stage_name=stage_name)) cnx.cursor().execute("drop stage if exists {stage_name}".format( stage_name=stage_name)) cnx.cursor().execute("drop table if exists {name}".format( name=db_parameters['name']))
class CustomHandler(BaseHTTPServer.BaseHTTPRequestHandler): """Custom HTTP request handler with GET and HEAD and POST commands. This serves files from the current directory and any of its subdirectories. The MIME type for files is determined by calling the .guess_type() method. The GET and HEAD requests are identical except that the HEAD request omits the actual contents of the file. """ post_listeners = set() @classmethod def add_post_listener(cls, function): cls.post_listeners.add(function) @classmethod def inform_post_listeners(cls, json_string): for func in cls.post_listeners: func(json_string) def do_GET(self): """Serve a GET request.""" f = self.send_head() if f: try: self.copyfile(f, self.wfile) finally: f.close() def do_HEAD(self): """Serve a HEAD request.""" f = self.send_head() if f: f.close() def do_POST(self): """Serve a POST request. only specific to API.""" if self.path.rstrip().endswith('/api/setMapping'): if not self.rfile.closed: content_len = int(self.headers.getheader('content-length', 0)) post_body = self.rfile.read(content_len) json_string = post_body self.__class__.inform_post_listeners(json_string) response_text = 'updated the Mapping' self.send_response(200, 'responseMessage') self.send_header('Content-Type', 'text/plain') self.send_header('Content-Length', len(response_text)) self.end_headers() self.wfile.write(response_text) else: self.send_error(403, 'no no') self.wfile.close() self.rfile.close() def send_head(self): """Common code for GET and HEAD commands. This sends the response code and MIME headers. Return value is either a file object (which has to be copied to the outputfile by the caller unless the command was HEAD, and must be closed by the caller under all circumstances), or None, in which case the caller has nothing further to do. """ path = self.translate_path(self.path) f = None if os.path.isdir(path): parts = urlparse.urlsplit(self.path) if not parts.path.endswith('/'): # redirect browser - doing basically what apache does self.send_response(301) new_parts = (parts[0], parts[1], parts[2] + '/', parts[3], parts[4]) new_url = urlparse.urlunsplit(new_parts) self.send_header("Location", new_url) self.end_headers() return None for index in "index.html", "index.htm": index = os.path.join(path, index) if os.path.exists(index): path = index break else: return self.list_directory(path) ctype = self.guess_type(path) try: # Always read in binary mode. Opening files in text mode may cause # newline translations, making the actual size of the content # transmitted *less* than the content-length! f = open(path, 'rb') except IOError: self.send_error(404, "File not found") return None try: self.send_response(200) self.send_header("Content-type", ctype) fs = os.fstat(f.fileno()) self.send_header("Content-Length", str(fs[6])) self.send_header("Last-Modified", self.date_time_string(fs.st_mtime)) self.end_headers() return f except: f.close() raise def list_directory(self, path): """Helper to produce a directory listing (absent index.html). Return value is either a file object, or None (indicating an error). In either case, the headers are sent, making the interface the same as for send_head(). """ try: list = os.listdir(path) except os.error: self.send_error(404, "No permission to list directory") return None list.sort(key=lambda a: a.lower()) f = StringIO() displaypath = cgi.escape(urllib.unquote(self.path)) f.write('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">') f.write("<html>\n<title>Directory listing for %s</title>\n" % displaypath) f.write("<body>\n<h2>Directory listing for %s</h2>\n" % displaypath) f.write("<hr>\n<ul>\n") for name in list: fullname = os.path.join(path, name) displayname = linkname = name # Append / for directories or @ for symbolic links if os.path.isdir(fullname): displayname = name + "/" linkname = name + "/" if os.path.islink(fullname): displayname = name + "@" # Note: a link to a directory displays with @ and links with / f.write('<li><a href="%s">%s</a>\n' % (urllib.quote(linkname), cgi.escape(displayname))) f.write("</ul>\n<hr>\n</body>\n</html>\n") length = f.tell() f.seek(0) self.send_response(200) encoding = sys.getfilesystemencoding() self.send_header("Content-type", "text/html; charset=%s" % encoding) self.send_header("Content-Length", str(length)) self.end_headers() return f def translate_path(self, path): """Translate a /-separated PATH to the local filename syntax. Components that mean special things to the local file system (e.g. drive or directory names) are ignored. (XXX They should probably be diagnosed.) """ # abandon query parameters path = path.split('?', 1)[0] path = path.split('#', 1)[0] # Don't forget explicit trailing slash when normalizing. Issue17324 trailing_slash = path.rstrip().endswith('/') path = posixpath.normpath(urllib.unquote(path)) words = path.split('/') words = filter(None, words) path = os.getcwd() for word in words: if os.path.dirname(word) or word in (os.curdir, os.pardir): # Ignore components that are not a simple file/directory name continue path = os.path.join(path, word) if trailing_slash: path += '/' return path def copyfile(self, source, outputfile): """Copy all data between two file objects. The SOURCE argument is a file object open for reading (or anything with a read() method) and the DESTINATION argument is a file object open for writing (or anything with a write() method). The only reason for overriding this would be to change the block size or perhaps to replace newlines by CRLF -- note however that this the default server uses this to copy binary data as well. """ shutil.copyfileobj(source, outputfile) def guess_type(self, path): """Guess the type of a file. Argument is a PATH (a filename). Return value is a string of the form type/subtype, usable for a MIME Content-type header. The default implementation looks the file's extension up in the table self.extensions_map, using application/octet-stream as a default; however it would be permissible (if slow) to look inside the data to make a better guess. """ base, ext = posixpath.splitext(path) if ext in self.extensions_map: return self.extensions_map[ext] ext = ext.lower() if ext in self.extensions_map: return self.extensions_map[ext] else: return self.extensions_map[''] if not mimetypes.inited: mimetypes.init() # try to read system mime.types extensions_map = mimetypes.types_map.copy() extensions_map.update({ '': 'application/octet-stream', # Default '.py': 'text/plain', '.c': 'text/plain', '.h': 'text/plain', })
def test_put_get_duplicated_data_user_stage(is_public_test, tmpdir, conn_cnx, db_parameters, number_of_files=5, number_of_lines=100): """[s3] Puts and Gets Duplicated Data using User Stage.""" if is_public_test or 'AWS_ACCESS_KEY_ID' not in os.environ: pytest.skip('This test requires to change the internal parameter') logger = getLogger(__name__) assert 'AWS_ACCESS_KEY_ID' in os.environ, 'AWS_ACCESS_KEY_ID is missing' assert 'AWS_SECRET_ACCESS_KEY' in os.environ, \ 'AWS_SECRET_ACCESS_KEY is missing' tmp_dir = generate_k_lines_of_n_files(number_of_lines, number_of_files, tmp_dir=str(tmpdir.mkdir('data'))) files = os.path.join(tmp_dir, 'file*') stage_name = db_parameters['name'] + '_stage' with conn_cnx(user=db_parameters['user'], account=db_parameters['account'], password=db_parameters['password']) as cnx: cnx.cursor().execute(""" create or replace table {name} ( aa int, dt date, ts timestamp, tsltz timestamp_ltz, tsntz timestamp_ntz, tstz timestamp_tz, pct float, ratio number(6,2)) """.format(name=db_parameters['name'])) user_bucket = os.getenv('SF_AWS_USER_BUCKET', "sfc-dev1-regression/{}/reg".format(getuser())) cnx.cursor().execute(""" create or replace stage {stage_name} url='s3://{user_bucket}/{stage_name}-{number_of_files}-{number_of_lines}' credentials=( AWS_KEY_ID='{aws_key_id}' AWS_SECRET_KEY='{aws_secret_key}' ) """.format(stage_name=stage_name, user_bucket=user_bucket, aws_key_id=os.getenv('AWS_ACCESS_KEY_ID'), aws_secret_key=os.getenv('AWS_SECRET_ACCESS_KEY'), number_of_files=number_of_files, number_of_lines=number_of_lines)) try: with conn_cnx(user=db_parameters['user'], account=db_parameters['account'], password=db_parameters['password']) as cnx: c = cnx.cursor() try: for rec in c.execute( "rm @{stage_name}".format(stage_name=stage_name)): logger.info('rec=%s', rec) finally: c.close() success_cnt = 0 skipped_cnt = 0 c = cnx.cursor() c.execute( "alter session set disable_put_and_get_on_external_stage = false" ) try: for rec in c.execute("put file://{file} @{stage_name}".format( file=files, stage_name=stage_name)): logger.info('rec=%s', rec) if rec[6] == 'UPLOADED': success_cnt += 1 elif rec[6] == 'SKIPPED': skipped_cnt += 1 finally: c.close() assert success_cnt == number_of_files, 'uploaded files' assert skipped_cnt == 0, 'skipped files' logger.info( 'deleting files in {stage_name}'.format(stage_name=stage_name)) deleted_cnt = 0 cnx.cursor().execute( "rm @{stage_name}/file0".format(stage_name=stage_name)) deleted_cnt += 1 cnx.cursor().execute( "rm @{stage_name}/file1".format(stage_name=stage_name)) deleted_cnt += 1 cnx.cursor().execute( "rm @{stage_name}/file2".format(stage_name=stage_name)) deleted_cnt += 1 success_cnt = 0 skipped_cnt = 0 c = cnx.cursor() try: for rec in c.execute("put file://{file} @{stage_name}".format( file=files, stage_name=stage_name), _raise_put_get_error=False): logger.info('rec=%s', rec) if rec[6] == 'UPLOADED': success_cnt += 1 elif rec[6] == 'SKIPPED': skipped_cnt += 1 assert success_cnt == deleted_cnt, \ 'uploaded files in the second time' assert skipped_cnt == number_of_files - deleted_cnt, \ 'skipped files in the second time' finally: c.close() time.sleep(5) cnx.cursor().execute("copy into {name} from @{stage_name}".format( name=db_parameters['name'], stage_name=stage_name)) c = cnx.cursor() try: c.execute("select count(*) from {name}".format( name=db_parameters['name'])) rows = 0 for rec in c: rows += rec[0] assert rows == number_of_files * number_of_lines, 'Number of rows' finally: c.close() cnx.cursor().execute( "rm @{stage_name}".format(stage_name=stage_name)) cnx.cursor().execute("copy into @{stage_name} from {name}".format( name=db_parameters['name'], stage_name=stage_name)) tmp_dir_user = str(tmpdir.mkdir('stage2')) cnx.cursor().execute( "get @{stage_name}/ file://{tmp_dir_user}/".format( stage_name=stage_name, tmp_dir_user=tmp_dir_user)) for _, _, files in os.walk(tmp_dir_user): for file in files: mimetypes.init() _, encoding = mimetypes.guess_type(file) assert encoding == 'gzip', "exported file type" finally: with conn_cnx(user=db_parameters['user'], account=db_parameters['account'], password=db_parameters['password']) as cnx: cnx.cursor().execute("drop stage if exists {stage_name}".format( stage_name=stage_name)) cnx.cursor().execute("drop table if exists {name}".format( name=db_parameters['name']))
class MediaRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): server_version = "MediaTest/" + __version__ def do_GET(self): if VERBOSE: print "GET", self.path f = self.send_head() if f: self.copyfile(f, self.wfile) f.close() def do_HEAD(self): f = self.send_head() if f: f.close() def send_head(self): path = self.translate_path(self.path) f = None if os.path.isdir(path): return None ctype = self.guess_type(path) try: f = open(path, 'rb') except IOError: self.send_error(404, "File not found") return None self.send_response(200) self.send_header("Content-type", ctype) fs = os.fstat(f.fileno()) self.send_header("Content-Length", str(fs[6])) self.send_header("Last-Modified", self.date_time_string(fs.st_mtime)) self.end_headers() return f def translate_path(self, path): path = urlparse.urlparse(path)[2] path = posixpath.normpath(urllib.unquote(path)) words = path.split('/') words = filter(None, words) path = os.getcwd() for word in words: drive, word = os.path.splitdrive(word) head, word = os.path.split(word) if word in (os.curdir, os.pardir): continue path = os.path.join(path, word) return path def copyfile(self, source, outputfile): global NEXT_TIME start = time.time() #print "START", start total=0 elapsed = 0 while True: chunk = source.read(4096) #print "READ", len(chunk) if len(chunk) == 0: break now = time.time() if NEXT_TIME == -1: NEXT_TIME = now #print 'NOW =',now, ' NEXT = ', NEXT_TIME delay = 0 if now < NEXT_TIME: delay = NEXT_TIME-now NEXT_TIME += float(len(chunk)*8)/BANDWIDTH if delay: time.sleep(delay) try: outputfile.write(chunk) except: break total += len(chunk) now = time.time() elapsed = now-start rate = total/elapsed #print "STAT", total, elapsed, rate #print "END", time.time() #print "ELAPSED", elapsed print "BPS =", (8*total)/elapsed def guess_type(self, path): base, ext = posixpath.splitext(path) if ext in self.extensions_map: return self.extensions_map[ext] ext = ext.lower() if ext in self.extensions_map: return self.extensions_map[ext] else: return self.extensions_map[''] if not mimetypes.inited: mimetypes.init() # try to read system mime.types extensions_map = mimetypes.types_map.copy() extensions_map.update({ '': 'application/octet-stream', # Default '.py': 'text/plain', '.c': 'text/plain', '.h': 'text/plain', '.mp4': 'video/mp4', '.mp3': 'audio/mpeg' })