def guess_type(path, custom_mimetypes=None):
custom_mimetypes = custom_mimetypes or {}
if not mimetypes.inited:
mimetypes.init() # try to read system mime.types
extensions_map = mimetypes.types_map.copy() # @UndefinedVariable
extensions_map.update({
"": "application/octet-stream", # Default
".py": "text/plain",
".c": "text/plain",
".h": "text/plain",
".appcache": "text/cache-manifest",
".webapp": "application/x-web-app-manifest+json"
})
extensions_map.update(custom_mimetypes)
base, ext = posixpath.splitext(path) # @UnusedVariable
if ext in extensions_map:
return extensions_map[ext]
ext = ext.lower()
if ext in extensions_map:
return extensions_map[ext]
else:
return extensions_map[""]
def action_download(self, courseid, taskid, path):
""" Download a file or a directory """
wanted_path = self.verify_path(courseid, taskid, path)
if wanted_path is None:
raise web.notfound()
# if the user want a dir:
if os.path.isdir(wanted_path):
tmpfile = tempfile.TemporaryFile()
tar = tarfile.open(fileobj=tmpfile, mode='w:gz')
for root, _, files in os.walk(wanted_path):
for fname in files:
info = tarfile.TarInfo(name=os.path.join(os.path.relpath(root, wanted_path), fname))
file_stat = os.stat(os.path.join(root, fname))
info.size = file_stat.st_size
info.mtime = file_stat.st_mtime
tar.addfile(info, fileobj=open(os.path.join(root, fname), 'r'))
tar.close()
tmpfile.seek(0)
web.header('Content-Type', 'application/x-gzip', unique=True)
web.header('Content-Disposition', 'attachment; filename="dir.tgz"', unique=True)
return tmpfile
else:
mimetypes.init()
mime_type = mimetypes.guess_type(wanted_path)
web.header('Content-Type', mime_type[0])
web.header('Content-Disposition', 'attachment; filename="' + os.path.split(wanted_path)[1] + '"', unique=True)
return open(wanted_path, 'r')
def _makeDatastream(self, each):
# Identify datastreams folder
datastreams_dir = self.obj_dir + "/datastreams"
filename = each["mets:fptr"]["@FILEID"]
label = each["@LABEL"]
order = each["@ORDER"]
# get extension, ds_id
mimetypes.init()
ds_id, ext = os.path.splitext(filename)
# create datastream dictionary
ds_dict = {
"filename": filename,
"ds_id": ds_id,
"mimetype": mimetypes.types_map[ext],
"label": label,
"internal_relationships": {},
'order': order
}
self.objMeta_handle.datastreams.append(ds_dict)
# make symlinks to datastreams on disk
bag_location = datastreams_dir + "/" + filename
remote_location = self.files_location + "/" + filename
os.symlink(remote_location, bag_location)
# Set the representative image for the object
if order == "1":
self.objMeta_handle.isRepresentedBy = ds_id
def ziplib(config, handle):
"""
Function to unpack a downloaded zip file by exploding it into multiple handles.
zip_unpack: Unpack if true
"""
import zipfile
import calendar
import mimetypes
mimetypes.init()
if not config.get("zip_unpack", False):
yield handle
return
logger.info("Extracting since zip_unpack = true");
z = zipfile.ZipFile(handle.fileobj)
def convert(dt):
"""
Convert a ZipInfo date_time into a unix timestamp (compatible with tar).
"""
return calendar.timegm(dt)
for i in z.infolist():
mime = mimetypes.guess_type(i.filename)[0]
yield ExtHandle(i.filename, i.file_size, convert(i.date_time), mime, z.open(i))
return
def __init__(self):
remuco.PlayerAdapter.__init__(
self,
"Totem",
mime_types=("audio", "video"),
volume_known=True,
playback_known=True,
progress_known=True,
file_actions=FILE_ACTIONS,
)
self.__to = None
self.__signal_ids = ()
self.__update_item = False
self.__md_album = None
self.__md_artist = None
self.__md_title = None
self.__last_mrl = None
self.__seek_step_initial = 5000
self.__seek_step = self.__seek_step_initial
self.__css_sid = 0
if not mimetypes.inited:
mimetypes.init()
def get_content_type_by_filename(file_name):
mime_type = ""
mime_map = {}
mime_map["js"] = "application/javascript"
mime_map["xlsx"] = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
mime_map["xltx"] = "application/vnd.openxmlformats-officedocument.spreadsheetml.template"
mime_map["potx"] = "application/vnd.openxmlformats-officedocument.presentationml.template"
mime_map["ppsx"] = "application/vnd.openxmlformats-officedocument.presentationml.slideshow"
mime_map["pptx"] = "application/vnd.openxmlformats-officedocument.presentationml.presentation"
mime_map["sldx"] = "application/vnd.openxmlformats-officedocument.presentationml.slide"
mime_map["docx"] = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
mime_map["dotx"] = "application/vnd.openxmlformats-officedocument.wordprocessingml.template"
mime_map["xlam"] = "application/vnd.ms-excel.addin.macroEnabled.12"
mime_map["xlsb"] = "application/vnd.ms-excel.sheet.binary.macroEnabled.12"
try:
suffix = ""
name = os.path.basename(file_name)
suffix = name.split('.')[-1]
if suffix in mime_map.keys():
mime_type = mime_map[suffix]
else:
import mimetypes
mimetypes.init()
mime_type = mimetypes.types_map["." + suffix]
except Exception:
mime_type = 'application/octet-stream'
if not mime_type:
mime_type = 'application/octet-stream'
return mime_type
def create_server(ui, app):
if ui.config('web', 'certificate'):
if sys.version_info >= (2, 6):
handler = _httprequesthandlerssl
else:
handler = _httprequesthandleropenssl
else:
handler = _httprequesthandler
if ui.configbool('web', 'ipv6'):
cls = IPv6HTTPServer
else:
cls = MercurialHTTPServer
# ugly hack due to python issue5853 (for threaded use)
import mimetypes; mimetypes.init()
address = ui.config('web', 'address', '')
port = util.getport(ui.config('web', 'port', 8000))
try:
return cls(ui, app, (address, port), handler)
except socket.error, inst:
raise util.Abort(_("cannot start server at '%s:%d': %s")
% (address, port, inst.args[1]))
def __init__(self):
self.secure_cookies_by_default = True
self.default_media_type = DEFAULT_MEDIA_TYPE
self.media_handlers = Handlers()
mimetypes.init()
self.static_media_types = mimetypes.types_map
def GoogleSitesLogin(site_name=None, site_domain=None, debug=False):
if site_domain is None:
raise ValueError("site_domain is not valid")
if site_name is None:
raise ValueError("site_name is not valid")
mimetypes.init()
client = gdata.sites.client.SitesClient( source="wiki-push",
site=site_name,
domain=site_domain )
client.http_client.debug = debug
try:
gdata.sample_util.authorize_client( client,
auth_type=gdata.sample_util.CLIENT_LOGIN,
service=client.auth_service,
source="wiki-push",
scopes=['http://sites.google.com/feeds/',
'https://sites.google.com/feeds/'] )
except gdata.client.BadAuthentication:
print "Invalid user credentials given."
return None
except gdata.client.Error:
print "Login Error."
return None
return client
def __init__(self, email, password, filepath, chunk_size=None,
convert=None, host=None, ssl=True, debug=False):
self.convert = convert
self.debug = debug
if chunk_size:
self.chunk_size = chunk_size
# Authenticate the user with CLientLogin
try:
self.client = gdata.docs.client.DocsClient(source=APP_NAME)
self.client.ssl = ssl # Force all API requests through HTTPS
self.client.http_client.debug = self.debug # Set to True for debugging HTTP requests
self.client.ClientLogin(email, password, self.client.source);
except gdata.client.BadAuthentication:
exit('Invalid user credentials given.')
except gdata.client.Error:
exit('Login Error')
mimetypes.init() # Register common mimetypes on system.
self.f = open(filepath)
content_type = get_mimetype(self.f.name)
file_size = os.path.getsize(self.f.name)
self.uploader = gdata.client.ResumableUploader(
self.client, self.f, content_type, file_size,
chunk_size=self.chunk_size, desired_class=gdata.docs.data.DocsEntry)
def get_file_types_list(self):
file_list = self.get_file_list()
if not file_list[0][1].has_key('mimetype'):
mimetypes.init()
for item in file_list:
item[1]['mimetype'] = mimetypes.guess_type(item[0])[0]
return file_list
def __init__(self, filepath, chunk_size=None, convert=None,
host=None, ssl=False, debug=False):
self.client = gdata.docs.client.DocsClient(source=APP_NAME)
self.client.ssl = ssl
self.client.http_client.debug = debug
self.convert = convert
if host:
self.client.host = host
if chunk_size:
self.chunk_size = chunk_size
# Authenticate the user with CLientLogin, OAuth, or AuthSub.
try:
gdata.sample_util.authorize_client(
self.client, service=self.client.auth_service, source=APP_NAME,
scopes=self.client.auth_scopes)
except gdata.client.BadAuthentication:
exit('Invalid user credentials given.')
except gdata.client.Error:
exit('Login Error')
mimetypes.init() # Register common mimetypes on system.
self.f = open(filepath)
content_type = get_mimetype(self.f.name)
file_size = os.path.getsize(self.f.name)
self.uploader = gdata.client.ResumableUploader(
self.client, self.f, content_type, file_size,
chunk_size=self.chunk_size, desired_class=gdata.docs.data.DocsEntry)
def handle_noargs(self, **options):
mimetypes.init()
locked_print("===> Syncing static directory")
pool = ThreadPool(20)
# Sync every file in the static media dir with S3
def pooled_sync_file(base, filename):
pool.apply_async(self.sync_file, args=[base, filename])
self.walk_tree([conf.SIMPLESTATIC_DIR], pooled_sync_file)
pool.close()
pool.join()
locked_print("===> Static directory syncing complete")
locked_print("===> Compressing and uploading CSS and JS")
pool = ThreadPool(20)
# Iterate over every template, looking for SimpleStaticNode
def pooled_handle_template(base, filename):
pool.apply_async(self.handle_template, args=[base, filename])
self.walk_tree(list(settings.TEMPLATE_DIRS), pooled_handle_template)
pool.close()
pool.join()
locked_print("===> Finished compressing and uploading CSS and JS")
def __init__(self, local_dir, verbose, quick):
self.local_dir = local_dir
self.verbose = verbose
self.quick = quick
parser = ConfigParser.SafeConfigParser()
config_file_name = os.path.join(local_dir, DOT_S3SITESYNC)
with open(config_file_name) as f:
config_file_contents = unicode("[s3sitesync]\n" + f.read())
parser.readfp(io.StringIO(config_file_contents), config_file_name)
self.keyid = parser.get('s3sitesync', 'awsAccessKeyId')
self.key = parser.get('s3sitesync', 'awsSecretAccessKey')
self.bucket_name = parser.get('s3sitesync', 'bucketName')
self.region = parser.get('s3sitesync', 'awsRegion')
if not self.region in API_ENDPOINTS:
raise Exception("Invalid region '%s'" % self.region)
self.api_endpoint = API_ENDPOINTS[self.region]
self.index_doc = parser.get('s3sitesync', 'indexDocument')
self.error_doc = parser.get('s3sitesync', 'errorDocument')
ignore_file_name = os.path.join(local_dir, DOT_S3IGNORE)
self.ignore_patterns = []
if os.path.isfile(ignore_file_name):
with open(ignore_file_name) as f:
for line in f:
line = line.strip()
if line:
pattern = os.path.join(local_dir, line)
self.ignore_patterns.append(pattern)
self.ignore_patterns.append(os.path.join(local_dir, DOT_S3SITESYNC))
self.ignore_patterns.append(os.path.join(local_dir, DOT_S3IGNORE))
mimetypes.init()
mimetypes.init([os.path.join(local_dir, DOT_S3MIMETYPES)])
def main():
mimetypes.init()
for info in scan_dir(sys.argv[1]):
#add_file(info)
print info['name'], datetime.fromtimestamp(info['mtime']).strftime('%d-%m-%Y')
def _fetch_subtitles(self):
if not self._options.fetch_subtitles:
return
mimetypes.init()
for fname in self._options.files:
if not os.path.exists(fname):
continue
if not self._is_video(fname):
continue
if not self._need_subtitles(fname):
continue
if self._has_subtitles(fname):
continue
if self._options.debug or self._options.verbose:
msg('fetching subtitles for %s' % fname)
for downloader in self._SUBDOWNLOADERS:
if self._options.debug or self._options.verbose:
msg('trying with %s' % downloader)
fn = getattr(self, '_fetch_subtitles_' + downloader)
if fn(fname):
break
else:
if self._options.debug or self._options.verbose:
msg('no subtitles were found')
def GET(self, courseid, taskid, path):
""" GET request """
if User.is_logged_in():
try:
course = FrontendCourse(courseid)
if not course.is_open_to_user(User.get_username()):
return renderer.course_unavailable()
task = course.get_task(taskid)
if not task.is_visible_by_user(User.get_username()):
return renderer.task_unavailable()
path_norm = posixpath.normpath(urllib.unquote(path))
public_folder_path = os.path.normpath(os.path.realpath(os.path.join(INGIniousConfiguration["tasks_directory"], courseid, taskid, "public")))
file_path = os.path.normpath(os.path.realpath(os.path.join(public_folder_path, path_norm)))
# Verify that we are still inside the public directory
if os.path.normpath(os.path.commonprefix([public_folder_path, file_path])) != public_folder_path:
raise web.notfound()
if os.path.isfile(file_path):
mimetypes.init()
mime_type = mimetypes.guess_type(file_path)
web.header('Content-Type', mime_type[0])
with open(file_path) as static_file:
return static_file.read()
else:
raise web.notfound()
except:
if web.config.debug:
raise
else:
raise web.notfound()
else:
return renderer.index(False)
def showpdf(request):
sign = os.path.join(settings.MEDIA_ROOT, "signature.png")
mimetypes.init()
response = None
if 'f' in request.GET:
fr = open(os.path.join(settings.MEDIA_ROOT,'pdffiles','extracted','%s' % request.GET['f']), "rb")
imgTemp = StringIO()
imgDoc = canvas.Canvas(imgTemp)
if request.GET['o'] == 'l':
imgDoc.drawImage(sign, 529, 40, 290/2, 154/2)
else:
imgDoc.drawImage(sign, 70, 40, 290/2, 154/2)
imgDoc.save()
overlay = PdfFileReader(StringIO(imgTemp.getvalue())).getPage(0)
page = PdfFileReader(fr).getPage(0)
page.mergePage(overlay)
pdf_out = PdfFileWriter()
pdf_out.addPage(page)
response = HttpResponse(mimetype='application/pdf')
response['Content-Disposition'] = 'attachment; filename=%s' % request.GET['f']
pdf_out.write(response)
return response
def _is_video(fp):
''' Check if file is a video file. '''
mimetypes.init()
mtype, __ = mimetypes.guess_type(fp)
if mtype is None:
return False
return mtype.startswith('video/')
def download_generic(self):
id = self.request.params.get('id', None)
filename = self.request.params.get('filename', None)
if id is None or filename is None:
return HTTPBadRequest()
entry = DBSession.query(LuxDownloadUrl).filter(
LuxDownloadUrl.id == id).first()
if entry is not None:
if entry.protected and self.request.user is None:
return HTTPUnauthorized()
url = entry.url + filename
try:
data = urllib2.urlopen(url, None, 1800)
except Exception as e:
log.exception(e)
data = None
log.debug(url)
mimetypes.init()
type = "application/octet-stream"
mimetype = mimetypes.guess_type(url)
if mimetype[0] is not None:
type = mimetype[0]
headers = {"Content-Type": type,
"Content-Disposition": "attachment; filename=\""
+ str(filename) + "\""}
if data is not None:
return Response(data.read(), headers=headers)
return HTTPBadRequest()
def media(self,path):
ext = path.split('.')[-1]
try:
mimetypes.init()
mimetype = mimetypes.guess_type(path)[0]
img = Image.open(self.basepath+'/'+path)
width,height = img.size
mx = max([width,height])
w,h = width,height
if mx > 60:
w = width*60/mx
h = height*60/mx
img = img.resize((w,h), Image.ANTIALIAS)
response = HttpResponse(content_type = mimetype or "image/"+ext)
response['Cache-Control'] = 'max-age=3600'
img.save(response,mimetype.split('/')[1] if mimetype else ext.upper())
return response
except Exception as e:
imagepath = settings.FILEMANAGER_STATIC_ROOT+'images/icons/'+ext+'.png'
if not os.path.exists(imagepath):
imagepath = settings.FILEMANAGER_STATIC_ROOT+'images/icons/default.png'
img = Image.open(imagepath)
width,height = img.size
mx = max([width,height])
w,h = width,height
if mx > 60:
w = width*60/mx
h = height*60/mx
img = img.resize((w,h), Image.ANTIALIAS)
response = HttpResponse(mimetype="image/png")
response['Cache-Control'] = 'max-age:3600'
img.save(response,'png')
return response
def GET(self, courseid, taskid, path):
""" GET request """
if self.user_manager.session_logged_in():
try:
course = self.course_factory.get_course(courseid)
if not self.user_manager.course_is_open_to_user(course):
return self.template_helper.get_renderer().course_unavailable()
task = course.get_task(taskid)
if not self.user_manager.task_is_visible_by_user(task):
return self.template_helper.get_renderer().task_unavailable()
path_norm = posixpath.normpath(urllib.unquote(path))
public_folder_path = os.path.normpath(os.path.realpath(os.path.join(task.get_directory_path(), "public")))
file_path = os.path.normpath(os.path.realpath(os.path.join(public_folder_path, path_norm)))
# Verify that we are still inside the public directory
if os.path.normpath(os.path.commonprefix([public_folder_path, file_path])) != public_folder_path:
raise web.notfound()
if os.path.isfile(file_path):
mimetypes.init()
mime_type = mimetypes.guess_type(file_path)
web.header('Content-Type', mime_type[0])
with open(file_path) as static_file:
return static_file.read()
else:
raise web.notfound()
except:
if web.config.debug:
raise
else:
raise web.notfound()
else:
return self.template_helper.get_renderer().index(self.user_manager.get_auth_methods_fields(), False)
def run(self):
mimetypes.init()
log.debug("Initialized mime type database.")
screenshot_tube = self.config.get('beanstalk', 'screenshot_tube')
self.beanstalk = politwoops.utils.beanstalk(
host=self.config.get('beanstalk', 'host'),
port=int(self.config.get('beanstalk', 'port')),
watch=screenshot_tube,
use=None)
log.debug("Connected to queue.")
while True:
time.sleep(0.2)
self.heart.beat()
reserve_timeout = max(self.heart.interval.total_seconds() * 0.1, 2)
job = self.beanstalk.reserve(timeout=reserve_timeout)
if job:
try:
tweet = anyjson.deserialize(job.body)
self.process_entities(tweet)
job.delete()
except Exception as e:
log.error("Exception caught, burying screenshot job for tweet {tweet}: {e}",
tweet=tweet.get('id'), e=e)
job.bury()
def mimetype(self):
try:
mimetypes.init()
type = mimetypes.guess_type(self.file.name)[0]
except:
type = None
return type
def encoding(self):
try:
mimetypes.init()
encoding = mimetypes.guess_type(self.file.name)[1]
except:
encoding = None
return encoding
def get_resource_content(request, source_name):
"""
Returns the actual resource.
Tries to guess the mimetype. If it fails, returns
application/octet-stream as fallback.
"""
import mimetypes
mimetypes.init()
resource_id = request.GET.get('resource_id', None)
if not resource_id:
raise Http404
try:
source = Source.objects.get(name=source_name)
except Source.DoesNotExist:
response_data = {'error': 'Source not found'}
else:
response_data = Backend(source.backend_id).get_resource_content(source, resource_id)
if 'content' in response_data.keys():
content = response_data['content']
mimetype, encoding = mimetypes.guess_type(response_data['name'])
if not mimetype:
mimetype = 'application/octet-stream'
else:
content = response_data
mimetype = "application/json"
return HttpResponse(content, mimetype=mimetype)
def get_file_mimetype(path):
"""
Returns file mimetype
"""
filename, file_extension = os.path.splitext(path)
mimetypes.init()
return mimetypes.types_map[file_extension]
def __call__(self, value, *args, **kwargs):
"""
:param value:
:type value: :class:`list` of :class:`dict`
:return:
:rtype: :class:`django.forms.Form`
"""
if value is None:
return self.form_cls(*args, **kwargs)
post_data = QueryDict("", mutable=True)
file_data = QueryDict("", mutable=True)
for obj in value:
name = obj["name"]
value = obj["value"]
if name in self.form_cls.base_fields and isinstance(
self.form_cls.base_fields[name], FileField
):
mimetypes.init()
basename = os.path.basename(value)
(type_, __) = mimetypes.guess_type(basename)
# it's a file => we need to simulate an uploaded one
content = InMemoryUploadedFile(
io.BytesIO(b"\0"),
name,
basename,
type_ or "application/binary",
1,
"utf-8",
)
file_data.update({name: content})
else:
post_data.update({name: value})
return self.form_cls(post_data, file_data, *args, **kwargs)
def _ContentType(requestedfile): """ Guess the file type based on the extension. This will be used to decide whether to open the file in text or binary mode. """ base, ext = posixpath.splitext(requestedfile) # This uses the standard MIME types mapping library. If it isn't # found, we check to see if it is xhtml (which might not be in the # local mime types library). If the type still is not found, it # defaults to plain text. mimetypes.init() try: ctype = mimetypes.types_map[ext] except: if (ext.lower() == '.xhtml'): ctype = 'application/xhtml+xml' else: ctype = 'text/plain' # We need to augment this information with a set of image # types to use for the file mode. if (ext in ['.png', '.PNG', '.gif', '.GIF', '.jpeg', '.JPEG', '.jpg', '.JPG']): fmode = 'rb' else: fmode = 'r' return fmode, ctype
def sort(dir, tmp_dir = path.abspath('/home/ndhuang/Pictures/'), all = False):
mimetypes.init()
mimetypes.add_type('image/x-nikon-nef', '.NEF')
sort_dir = path.join(tmp_dir, 'sorting')
img_files = os.listdir(dir)
i = 0
while i < len(img_files):
img = img_files[i]
if '.pp3' in img:
img_files.remove(img)
continue
elif not all and not re.match('DSC_(\d){4}\.', img):
img_files.remove(img)
continue
mt = mimetypes.guess_type(img)[0]
mt = mt.split('/')
if mt[0] != 'image':
raise RuntimeError('%s is not an image!' %img)
else:
i += 1
os.mkdir(sort_dir)
imgs = [[] for i in img_files]
for i in range(len(img_files)):
try:
imgs[i] = Img(path.join(dir, img_files[i]))
except KeyError:
print '%s is missing EXIF data!' %img_files[i]
# remove empty arrays
while [] in imgs:
imgs.remove([])
imgs = sorted(imgs)
pic_num = 1
copies = 1
for i, img in enumerate(imgs):
ext = img.path[-3:]
if i != 0 and imgs[i] == imgs[i - 1]:
dst = path.join(sort_dir,
'DSC_%04d-%d.%s' %(pic_num - 1, copies, ext))
copies += 1
else:
dst = path.join(sort_dir, 'DSC_%04d.%s' %(pic_num, ext))
pic_num += 1
copies = 1
os.rename(img.path, dst)
try:
os.rename(img.path + '.out.pp3', dst + '.out.pp3')
except OSError as err:
pass
try:
os.rename(img.path + '.pp3', dst + '.pp3')
except OSError as err:
pass
for f in os.listdir(dir):
os.rename(path.join(dir, f), path.join(sort_dir, f))
os.rmdir(dir)
os.rename(sort_dir, dir)
def __init__(self, conf, memcache=None, logger=None, account_ring=None,
container_ring=None):
if conf is None:
conf = {}
if logger is None:
self.logger = get_logger(conf, log_route='proxy-server')
else:
self.logger = logger
self._error_limiting = {}
swift_dir = conf.get('swift_dir', '/etc/swift')
self.swift_dir = swift_dir
self.node_timeout = int(conf.get('node_timeout', 10))
self.recoverable_node_timeout = int(
conf.get('recoverable_node_timeout', self.node_timeout))
self.conn_timeout = float(conf.get('conn_timeout', 0.5))
self.client_timeout = int(conf.get('client_timeout', 60))
self.put_queue_depth = int(conf.get('put_queue_depth', 10))
self.object_chunk_size = int(conf.get('object_chunk_size', 65536))
self.client_chunk_size = int(conf.get('client_chunk_size', 65536))
self.trans_id_suffix = conf.get('trans_id_suffix', '')
self.post_quorum_timeout = float(conf.get('post_quorum_timeout', 0.5))
self.error_suppression_interval = \
int(conf.get('error_suppression_interval', 60))
self.error_suppression_limit = \
int(conf.get('error_suppression_limit', 10))
self.recheck_container_existence = \
int(conf.get('recheck_container_existence', 60))
self.recheck_account_existence = \
int(conf.get('recheck_account_existence', 60))
self.allow_account_management = \
config_true_value(conf.get('allow_account_management', 'no'))
self.object_post_as_copy = \
config_true_value(conf.get('object_post_as_copy', 'true'))
self.container_ring = container_ring or Ring(swift_dir,
ring_name='container')
self.account_ring = account_ring or Ring(swift_dir,
ring_name='account')
# ensure rings are loaded for all configured storage policies
for policy in POLICIES:
policy.load_ring(swift_dir)
self.obj_controller_router = ObjectControllerRouter()
self.memcache = memcache
mimetypes.init(mimetypes.knownfiles +
[os.path.join(swift_dir, 'mime.types')])
self.account_autocreate = \
config_true_value(conf.get('account_autocreate', 'no'))
self.auto_create_account_prefix = (
conf.get('auto_create_account_prefix') or '.')
self.expiring_objects_account = self.auto_create_account_prefix + \
(conf.get('expiring_objects_account_name') or 'expiring_objects')
self.expiring_objects_container_divisor = \
int(conf.get('expiring_objects_container_divisor') or 86400)
self.max_containers_per_account = \
int(conf.get('max_containers_per_account') or 0)
self.max_containers_whitelist = [
a.strip()
for a in conf.get('max_containers_whitelist', '').split(',')
if a.strip()]
self.deny_host_headers = [
host.strip() for host in
conf.get('deny_host_headers', '').split(',') if host.strip()]
self.log_handoffs = config_true_value(conf.get('log_handoffs', 'true'))
self.cors_allow_origin = [
a.strip()
for a in conf.get('cors_allow_origin', '').split(',')
if a.strip()]
self.strict_cors_mode = config_true_value(
conf.get('strict_cors_mode', 't'))
self.node_timings = {}
self.timing_expiry = int(conf.get('timing_expiry', 300))
self.sorting_method = conf.get('sorting_method', 'shuffle').lower()
self.max_large_object_get_time = float(
conf.get('max_large_object_get_time', '86400'))
value = conf.get('request_node_count', '2 * replicas').lower().split()
if len(value) == 1:
rnc_value = int(value[0])
self.request_node_count = lambda replicas: rnc_value
elif len(value) == 3 and value[1] == '*' and value[2] == 'replicas':
rnc_value = int(value[0])
self.request_node_count = lambda replicas: rnc_value * replicas
else:
raise ValueError(
'Invalid request_node_count value: %r' % ''.join(value))
try:
self._read_affinity = read_affinity = conf.get('read_affinity', '')
self.read_affinity_sort_key = affinity_key_function(read_affinity)
except ValueError as err:
# make the message a little more useful
raise ValueError("Invalid read_affinity value: %r (%s)" %
(read_affinity, err.message))
try:
write_affinity = conf.get('write_affinity', '')
self.write_affinity_is_local_fn \
= affinity_locality_predicate(write_affinity)
except ValueError as err:
# make the message a little more useful
raise ValueError("Invalid write_affinity value: %r (%s)" %
(write_affinity, err.message))
value = conf.get('write_affinity_node_count',
'2 * replicas').lower().split()
if len(value) == 1:
wanc_value = int(value[0])
self.write_affinity_node_count = lambda replicas: wanc_value
elif len(value) == 3 and value[1] == '*' and value[2] == 'replicas':
wanc_value = int(value[0])
self.write_affinity_node_count = \
lambda replicas: wanc_value * replicas
else:
raise ValueError(
'Invalid write_affinity_node_count value: %r' % ''.join(value))
# swift_owner_headers are stripped by the account and container
# controllers; we should extend header stripping to object controller
# when a privileged object header is implemented.
swift_owner_headers = conf.get(
'swift_owner_headers',
'x-container-read, x-container-write, '
'x-container-sync-key, x-container-sync-to, '
'x-account-meta-temp-url-key, x-account-meta-temp-url-key-2, '
'x-container-meta-temp-url-key, x-container-meta-temp-url-key-2, '
'x-account-access-control')
self.swift_owner_headers = [
name.strip().title()
for name in swift_owner_headers.split(',') if name.strip()]
# Initialization was successful, so now apply the client chunk size
# parameter as the default read / write buffer size for the network
# sockets.
#
# NOTE WELL: This is a class setting, so until we get set this on a
# per-connection basis, this affects reading and writing on ALL
# sockets, those between the proxy servers and external clients, and
# those between the proxy servers and the other internal servers.
#
# ** Because it affects the client as well, currently, we use the
# client chunk size as the govenor and not the object chunk size.
socket._fileobject.default_bufsize = self.client_chunk_size
self.expose_info = config_true_value(
conf.get('expose_info', 'yes'))
self.disallowed_sections = list_from_csv(
conf.get('disallowed_sections', 'swift.valid_api_versions'))
self.admin_key = conf.get('admin_key', None)
register_swift_info(
version=swift_version,
strict_cors_mode=self.strict_cors_mode,
policies=POLICIES.get_policy_info(),
allow_account_management=self.allow_account_management,
account_autocreate=self.account_autocreate,
**constraints.EFFECTIVE_CONSTRAINTS)
class SimpleHTTPRequestHandler(http.server.BaseHTTPRequestHandler):
def do_GET(self):
f = self.send_head()
if f:
for i in f.readlines():
if isinstance(i, str):
self.wfile.write(i.encode("utf-8"))
else:
self.wfile.write(i)
f.close()
def do_HEAD(self):
f = self.send_head()
if f:
f.close()
# 上传结果处理
def do_POST(self):
r, info = self.deal_post_data()
print(r, info, "by: ", self.client_address)
f = StringIO()
f.write('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">')
f.write(
'<meta name="viewport" content="width=device-width" charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0, minimum-scale=1.0, maximum-scale=1.0, user-scalable=no">'
)
f.write("<html>\n<title>上传结果</title>\n")
f.write("<body>\n<h2>文件上传</h2>")
if r:
f.write('<strong style="color:#00FF00">成功</strong>\n')
else:
f.write('<strong style="color:#FF0000">失败</strong>\n')
f.write("<hr>\n")
f.write(info)
f.write("</br><a href=\"%s\">点击返回</a>" % self.headers['referer'])
f.write("<hr><small>Powered By: gaowanliang ")
f.write("</small></body>\n</html>\n")
length = f.tell()
f.seek(0)
self.send_response(200)
self.send_header("Content-type", "text/html")
self.send_header("Content-Length", str(length))
self.end_headers()
if f:
for i in f.readlines():
self.wfile.write(i.encode("utf-8"))
f.close()
def deal_post_data(self):
boundary = str(
self.headers["Content-Type"].split("=")[1]).encode("utf-8")
remainbytes = int(self.headers['Content-length'])
line = self.rfile.readline()
remainbytes -= len(line)
if not boundary in line:
return (False, "Content NOT begin with boundary")
line = self.rfile.readline()
remainbytes -= len(line)
fn = re.findall(
r'Content-Disposition.*name="file"; filename="(.*)"'.encode(
'utf-8'), line)
if not fn:
return (False, "Can't find out file name...")
path = str(self.translate_path(self.path)).encode('utf-8')
osType = platform.system()
try:
if osType == "Linux":
fn = os.path.join(path, fn[0].decode('gbk').encode('utf-8'))
else:
fn = os.path.join(path, fn[0])
except Exception as e:
return (False, "文件名请不要用中文，或者使用IE上传中文名的文件。{}".format(e))
while os.path.exists(fn):
fn += "_".encode("utf-8")
line = self.rfile.readline()
remainbytes -= len(line)
line = self.rfile.readline()
remainbytes -= len(line)
try:
out = open(fn, 'wb')
except IOError:
return (
False,
"Can't create file to write, do you have permission to write?")
preline = self.rfile.readline()
remainbytes -= len(preline)
while remainbytes > 0:
line = self.rfile.readline()
remainbytes -= len(line)
if boundary in line:
preline = preline[0:-1]
if preline.endswith('\r'.encode("utf-8")):
preline = preline[0:-1]
out.write(preline)
out.close()
return (True, "文件 '%s' 上传成功" % fn)
else:
out.write(preline)
preline = line
return (False, "Unexpect Ends of data.")
def send_head(self):
path = self.translate_path(self.path)
f = None
if os.path.isdir(path):
if not self.path.endswith('/'):
self.send_response(301)
self.send_header("Location", self.path + "/")
self.end_headers()
return None
for index in "index.html", "index.htm":
index = os.path.join(path, index)
if os.path.exists(index):
path = index
break
else:
return self.list_directory(path)
ctype = self.guess_type(path)
try:
f = open(path, 'rb')
except IOError:
self.send_error(404, "File not found")
return None
self.send_response(200)
self.send_header("Content-type", ctype)
fs = os.fstat(f.fileno())
self.send_header("Content-Length", str(fs[6]))
self.send_header("Last-Modified", self.date_time_string(fs.st_mtime))
self.end_headers()
return f
def list_directory(self, path):
try:
list = os.listdir(path)
except os.error:
self.send_error(404, "No permission to list directory")
return None
list.sort(key=lambda a: a.lower())
f = StringIO()
displaypath = html.escape(urllib.parse.unquote(self.path))
dirs = ''
for name in list:
fullname = os.path.join(path, name)
colorName = displayname = linkname = name
if os.path.isdir(fullname):
colorName = '<span style="background-color: #CEFFCE;">' + name + '/</span>'
displayname = name
linkname = name + "/"
if os.path.islink(fullname):
colorName = '<span style="background-color: #FFBFFF;">' + name + '@</span>'
displayname = name
filename = os.getcwd() + '/' + displaypath + displayname
dirs += f"""
<tr>
<td width='60%'><a href='{urllib.parse.quote(linkname)}'>{colorName}</a>
</td><td width='20%'>{sizeof_fmt(os.path.getsize(filename))}</td>
<td width='20%'>{modification_date(filename)}</td>
</tr>
"""
script_code = """
"""
content = f"""
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Title</title>
</head>
<body>
<h2>目录清单 位于{displaypath}</h2>
<hr>
<form ENCTYPE="multipart/form-data" method="post">
<input name="file" type="file"/>
<input type="submit" value="上传"/>
<input type="button" value="主目录" onClick="location='/'">
</form>
<h2 style="color:#FF0000">请先选择完文件再点上传，不这样做的话可能会出现奇怪的情况</h2>
<hr>
<table id='wrap'>
{dirs}
</table>
<hr>
</body>
{script_code}
</html>
"""
f.write(content)
length = f.tell()
f.seek(0)
self.send_response(200)
self.send_header("Content-type", "text/html")
self.send_header("Content-Length", str(length))
self.end_headers()
return f
def translate_path(self, path):
path = path.split('?', 1)[0]
path = path.split('#', 1)[0]
path = posixpath.normpath(urllib.parse.unquote(path))
words = path.split('/')
words = [_f for _f in words if _f]
path = os.getcwd()
for word in words:
drive, word = os.path.splitdrive(word)
head, word = os.path.split(word)
if word in (os.curdir, os.pardir):
continue
path = os.path.join(path, word)
return path
def copyfile(self, source, outputfile):
shutil.copyfileobj(source, outputfile)
def guess_type(self, path):
base, ext = posixpath.splitext(path)
if ext in self.extensions_map:
return self.extensions_map[ext]
ext = ext.lower()
if ext in self.extensions_map:
return self.extensions_map[ext]
else:
return self.extensions_map['']
if not mimetypes.inited:
mimetypes.init() # try to read system mime.types
extensions_map = mimetypes.types_map.copy()
extensions_map.update({
'': 'application/octet-stream', # Default
'.py': 'text/plain',
'.c': 'text/plain',
'.h': 'text/plain',
})
import ssl
import pathlib
import asyncio
import json
import logging
import array
import time
import os
from http import HTTPStatus
from http.server import HTTPServer, BaseHTTPRequestHandler
import mimetypes
import threading
import websockets
mimetypes.init()
logging.basicConfig()
# Static file server (compatible with websocket library).
# See https://gist.github.com/artizirk/04eb23d957d7916c01ca632bb27d5436
class WebSocketServerProtocolWithHTTP(websockets.WebSocketServerProtocol):
"""Implements a simple static file server for WebSocketServer"""
async def process_request(self, path, request_headers):
"""Serves a file when doing a GET request with a valid path"""
if "Upgrade" in request_headers:
return # Probably a WebSocket connection
if path == '/':
path = '/index.html'
def init():
"""
Initialize all channels.
"""
logger = logging.getLogger(__name__)
# Initialize mimetypes library
mimetypes.init(
[pkg_resources.resource_filename('ehforwarderbot', 'mimetypes')])
# Initialize all channels
# (Load libraries and modules and init them with Queue `q`)
conf = config.load_config()
for i in conf['slave_channels']:
logger.log(99, "\x1b[0;37;46m %s \x1b[0m",
_("Initializing slave {}...").format(i))
cls = utils.locate_module(i, 'slave')
instance_id = i.split('#', 1)[1:]
instance_id = (instance_id and instance_id[0]) or None
coordinator.add_channel(cls(instance_id=instance_id))
logger.log(
99, "\x1b[0;37;42m %s \x1b[0m",
_("Slave channel {name} ({id}) # {instance_id} is initialized.").
format(name=cls.channel_name,
id=cls.channel_id,
instance_id=instance_id))
logger.log(99, "\x1b[0;37;46m %s \x1b[0m",
_("Initializing master {}...").format(conf['master_channel']))
instance_id = conf['master_channel'].split('#', 1)[1:]
instance_id = (instance_id and instance_id[0]) or None
coordinator.add_channel(
utils.locate_module(conf['master_channel'],
'master')(instance_id=instance_id))
logger.log(
99, "\x1b[0;37;42m %s \x1b[0m",
_("Master channel {name} ({id}) # {instance_id} is initialized.").
format(name=coordinator.master.channel_name,
id=coordinator.master.channel_id,
instance_id=instance_id))
logger.log(99, "\x1b[1;37;42m %s \x1b[0m", _("All channels initialized."))
for i in conf['middlewares']:
logger.log(99, "\x1b[0;37;46m %s \x1b[0m",
_("Initializing middleware {}...").format(i))
cls = utils.locate_module(i, 'middleware')
instance_id = i.split('#', 1)[1:]
instance_id = (instance_id and instance_id[0]) or None
coordinator.add_middleware(cls(instance_id=instance_id))
logger.log(
99, "\x1b[0;37;42m %s \x1b[0m",
_("Middleware {name} ({id}) # {instance_id} is initialized.").
format(name=cls.middleware_name,
id=cls.middleware_id,
instance_id=instance_id))
logger.log(99, "\x1b[1;37;42m %s \x1b[0m",
_("All middlewares are initialized."))
coordinator.master_thread = threading.Thread(
target=coordinator.master.poll)
coordinator.slave_threads = {
key: threading.Thread(target=coordinator.slaves[key].poll)
for key in coordinator.slaves
}
class SimpleHTTPRequestHandler(http.server.BaseHTTPRequestHandler):
"""Simple HTTP request handler with GET/HEAD/POST commands.
This serves files from the current directory and any of its
subdirectories. The MIME type for files is determined by
calling the .guess_type() method. And can reveive file uploaded
by client.
The GET/HEAD/POST requests are identical except that the HEAD
request omits the actual contents of the file.
"""
server_version = "SimpleHTTPWithUpload/" + __version__
def do_GET(self):
"""Serve a GET request."""
f = self.send_head()
if f:
self.copyfile(f, self.wfile)
f.close()
def do_HEAD(self):
"""Serve a HEAD request."""
f = self.send_head()
if f:
f.close()
def do_POST(self):
"""Serve a POST request."""
r, info = self.deal_post_data()
print((r, info, "by: ", self.client_address))
f = BytesIO()
f.write(b'<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">')
f.write(b"<html>\n<title>Upload Result Page</title>\n")
f.write(b"<body>\n<h2>Upload Result Page</h2>\n")
f.write(b"<hr>\n")
if r:
f.write(b"<strong>Success:</strong>")
else:
f.write(b"<strong>Failed:</strong>")
f.write(info.encode())
f.write(
("<br><a href=\"%s\">back</a>" % self.headers['referer']).encode())
f.write(b"<hr><small>Powerd By: bones7456, check new version at ")
f.write(b"<a href=\"http://li2z.cn/?s=SimpleHTTPServerWithUpload\">")
f.write(b"here</a>.</small></body>\n</html>\n")
length = f.tell()
f.seek(0)
self.send_response(200)
self.send_header("Content-type", "text/html")
self.send_header("Content-Length", str(length))
self.end_headers()
if f:
self.copyfile(f, self.wfile)
f.close()
def deal_post_data(self):
content_type = self.headers['content-type']
if not content_type:
return (False, "Content-Type header doesn't contain boundary")
boundary = content_type.split("=")[1].encode()
remainbytes = int(self.headers['content-length'])
line = self.rfile.readline()
remainbytes -= len(line)
if not boundary in line:
return (False, "Content NOT begin with boundary")
line = self.rfile.readline()
remainbytes -= len(line)
fn = re.findall(r'Content-Disposition.*name="file"; filename="(.*)"',
line.decode())
if not fn:
return (False, "Can't find out file name...")
path = self.translate_path(self.path)
fn = os.path.join(path, fn[0])
line = self.rfile.readline()
remainbytes -= len(line)
line = self.rfile.readline()
remainbytes -= len(line)
try:
out = open(fn, 'wb')
except IOError:
return (
False,
"Can't create file to write, do you have permission to write?")
preline = self.rfile.readline()
remainbytes -= len(preline)
while remainbytes > 0:
line = self.rfile.readline()
remainbytes -= len(line)
if boundary in line:
preline = preline[0:-1]
if preline.endswith(b'\r'):
preline = preline[0:-1]
out.write(preline)
out.close()
return (True, "File '%s' upload success!" % fn)
else:
out.write(preline)
preline = line
return (False, "Unexpect Ends of data.")
def send_head(self):
"""Common code for GET and HEAD commands.
This sends the response code and MIME headers.
Return value is either a file object (which has to be copied
to the outputfile by the caller unless the command was HEAD,
and must be closed by the caller under all circumstances), or
None, in which case the caller has nothing further to do.
"""
path = self.translate_path(self.path)
f = None
if os.path.isdir(path):
if not self.path.endswith('/'):
# redirect browser - doing basically what apache does
self.send_response(301)
self.send_header("Location", self.path + "/")
self.end_headers()
return None
for index in "index.html", "index.htm":
index = os.path.join(path, index)
if os.path.exists(index):
path = index
break
else:
return self.list_directory(path)
ctype = self.guess_type(path)
try:
# Always read in binary mode. Opening files in text mode may cause
# newline translations, making the actual size of the content
# transmitted *less* than the content-length!
f = open(path, 'rb')
except IOError:
self.send_error(404, "File not found")
return None
self.send_response(200)
self.send_header("Content-type", ctype)
fs = os.fstat(f.fileno())
self.send_header("Content-Length", str(fs[6]))
self.send_header("Last-Modified", self.date_time_string(fs.st_mtime))
self.end_headers()
return f
def list_directory(self, path):
"""Helper to produce a directory listing (absent index.html).
Return value is either a file object, or None (indicating an
error). In either case, the headers are sent, making the
interface the same as for send_head().
"""
try:
list = os.listdir(path)
except os.error:
self.send_error(404, "No permission to list directory")
return None
list.sort(key=lambda a: a.lower())
f = BytesIO()
displaypath = cgi.escape(urllib.parse.unquote(self.path))
f.write(b'<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">')
f.write(("<html>\n<title>Directory listing for %s</title>\n" %
displaypath).encode())
f.write(("<body>\n<h2>Directory listing for %s</h2>\n" %
displaypath).encode())
f.write(b"<hr>\n")
f.write(b"<form ENCTYPE=\"multipart/form-data\" method=\"post\">")
f.write(b"<input name=\"file\" type=\"file\"/>")
f.write(b"<input type=\"submit\" value=\"upload\"/></form>\n")
f.write(b"<hr>\n<ul>\n")
for name in list:
fullname = os.path.join(path, name)
displayname = linkname = name
# Append / for directories or @ for symbolic links
if os.path.isdir(fullname):
displayname = name + "/"
linkname = name + "/"
if os.path.islink(fullname):
displayname = name + "@"
# Note: a link to a directory displays with @ and links with /
f.write(('<li><a href="%s">%s</a>\n' %
(urllib.parse.quote(linkname),
cgi.escape(displayname))).encode())
f.write(b"</ul>\n<hr>\n</body>\n</html>\n")
length = f.tell()
f.seek(0)
self.send_response(200)
self.send_header("Content-type", "text/html")
self.send_header("Content-Length", str(length))
self.end_headers()
return f
def translate_path(self, path):
"""Translate a /-separated PATH to the local filename syntax.
Components that mean special things to the local file system
(e.g. drive or directory names) are ignored. (XXX They should
probably be diagnosed.)
"""
# abandon query parameters
path = path.split('?', 1)[0]
path = path.split('#', 1)[0]
path = posixpath.normpath(urllib.parse.unquote(path))
words = path.split('/')
words = [_f for _f in words if _f]
path = os.getcwd()
for word in words:
drive, word = os.path.splitdrive(word)
head, word = os.path.split(word)
if word in (os.curdir, os.pardir): continue
path = os.path.join(path, word)
return path
def copyfile(self, source, outputfile):
"""Copy all data between two file objects.
The SOURCE argument is a file object open for reading
(or anything with a read() method) and the DESTINATION
argument is a file object open for writing (or
anything with a write() method).
The only reason for overriding this would be to change
the block size or perhaps to replace newlines by CRLF
-- note however that this the default server uses this
to copy binary data as well.
"""
shutil.copyfileobj(source, outputfile)
def guess_type(self, path):
"""Guess the type of a file.
Argument is a PATH (a filename).
Return value is a string of the form type/subtype,
usable for a MIME Content-type header.
The default implementation looks the file's extension
up in the table self.extensions_map, using application/octet-stream
as a default; however it would be permissible (if
slow) to look inside the data to make a better guess.
"""
base, ext = posixpath.splitext(path)
if ext in self.extensions_map:
return self.extensions_map[ext]
ext = ext.lower()
if ext in self.extensions_map:
return self.extensions_map[ext]
else:
return self.extensions_map['']
if not mimetypes.inited:
mimetypes.init() # try to read system mime.types
extensions_map = mimetypes.types_map.copy()
extensions_map.update({
'': 'application/octet-stream', # Default
'.py': 'text/plain',
'.c': 'text/plain',
'.h': 'text/plain',
})
def guess_mime_type(filename):
"""Guess mime type based of file extension."""
if not mimetypes.inited:
mimetypes.init()
return mimetypes.guess_type(filename)[0]
def __init__(self, static_path):
mimetypes.init()
self.static_path = static_path
self.block_loader = BlockLoader()
class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
"""Simple HTTP request handler with GET and HEAD commands.
This serves files from the current directory and any of its
subdirectories. The MIME type for files is determined by
calling the .guess_type() method.
The GET and HEAD requests are identical except that the HEAD
request omits the actual contents of the file.
"""
server_version = "SimpleHTTP/" + __version__
def do_GET(self):
"""Serve a GET request."""
f = self.send_head()
if f:
try:
self.copyfile(f, self.wfile)
finally:
f.close()
def do_HEAD(self):
"""Serve a HEAD request."""
f = self.send_head()
if f:
f.close()
def send_head(self):
"""Common code for GET and HEAD commands.
This sends the response code and MIME headers.
Return value is either a file object (which has to be copied
to the outputfile by the caller unless the command was HEAD,
and must be closed by the caller under all circumstances), or
None, in which case the caller has nothing further to do.
"""
path = self.translate_path(self.path)
f = None
if os.path.isdir(path):
parts = urllib.parse.urlsplit(self.path)
if not parts.path.endswith('/'):
# redirect browser - doing basically what apache does
self.send_response(HTTPStatus.MOVED_PERMANENTLY)
new_parts = (parts[0], parts[1], parts[2] + '/',
parts[3], parts[4])
new_url = urllib.parse.urlunsplit(new_parts)
self.send_header("Location", new_url)
self.end_headers()
return None
for index in "index.html", "index.htm":
index = os.path.join(path, index)
if os.path.exists(index):
path = index
break
else:
return self.list_directory(path)
ctype = self.guess_type(path)
try:
f = open(path, 'rb')
except OSError:
self.send_error(HTTPStatus.NOT_FOUND, "File not found")
return None
try:
fs = os.fstat(f.fileno())
# Use browser cache if possible
if ("If-Modified-Since" in self.headers
and "If-None-Match" not in self.headers):
# compare If-Modified-Since and time of last file modification
try:
ims = email.utils.parsedate_to_datetime(
self.headers["If-Modified-Since"])
except (TypeError, IndexError, OverflowError, ValueError):
# ignore ill-formed values
pass
else:
if ims.tzinfo is None:
# obsolete format with no timezone, cf.
# https://tools.ietf.org/html/rfc7231#section-7.1.1.1
ims = ims.replace(tzinfo=datetime.timezone.utc)
if ims.tzinfo is datetime.timezone.utc:
# compare to UTC datetime of last modification
last_modif = datetime.datetime.fromtimestamp(
fs.st_mtime, datetime.timezone.utc)
# remove microseconds, like in If-Modified-Since
last_modif = last_modif.replace(microsecond=0)
if last_modif <= ims:
self.send_response(HTTPStatus.NOT_MODIFIED)
self.end_headers()
f.close()
return None
self.send_response(HTTPStatus.OK)
self.send_header("Content-type", ctype)
self.send_header("Content-Length", str(fs[6]))
self.send_header("Last-Modified",
self.date_time_string(fs.st_mtime))
self.end_headers()
return f
except:
f.close()
raise
def list_directory(self, path):
"""Helper to produce a directory listing (absent index.html).
Return value is either a file object, or None (indicating an
error). In either case, the headers are sent, making the
interface the same as for send_head().
"""
try:
list = os.listdir(path)
except OSError:
self.send_error(
HTTPStatus.NOT_FOUND,
"No permission to list directory")
return None
list.sort(key=lambda a: a.lower())
r = []
try:
displaypath = urllib.parse.unquote(self.path,
errors='surrogatepass')
except UnicodeDecodeError:
displaypath = urllib.parse.unquote(path)
displaypath = html.escape(displaypath, quote=False)
enc = sys.getfilesystemencoding()
title = 'Directory listing for %s' % displaypath
r.append('<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" '
'"http://www.w3.org/TR/html4/strict.dtd">')
r.append('<html>\n<head>')
r.append('<meta http-equiv="Content-Type" '
'content="text/html; charset=%s">' % enc)
r.append('<title>%s</title>\n</head>' % title)
r.append('<body>\n<h1>%s</h1>' % title)
r.append('<hr>\n<ul>')
for name in list:
fullname = os.path.join(path, name)
displayname = linkname = name
# Append / for directories or @ for symbolic links
if os.path.isdir(fullname):
displayname = name + "/"
linkname = name + "/"
if os.path.islink(fullname):
displayname = name + "@"
# Note: a link to a directory displays with @ and links with /
r.append('<li><a href="%s">%s</a></li>'
% (urllib.parse.quote(linkname,
errors='surrogatepass'),
html.escape(displayname, quote=False)))
r.append('</ul>\n<hr>\n</body>\n</html>\n')
encoded = '\n'.join(r).encode(enc, 'surrogateescape')
f = io.BytesIO()
f.write(encoded)
f.seek(0)
self.send_response(HTTPStatus.OK)
self.send_header("Content-type", "text/html; charset=%s" % enc)
self.send_header("Content-Length", str(len(encoded)))
self.end_headers()
return f
def translate_path(self, path):
"""Translate a /-separated PATH to the local filename syntax.
Components that mean special things to the local file system
(e.g. drive or directory names) are ignored. (XXX They should
probably be diagnosed.)
"""
# abandon query parameters
path = path.split('?',1)[0]
path = path.split('#',1)[0]
# Don't forget explicit trailing slash when normalizing. Issue17324
trailing_slash = path.rstrip().endswith('/')
try:
path = urllib.parse.unquote(path, errors='surrogatepass')
except UnicodeDecodeError:
path = urllib.parse.unquote(path)
path = posixpath.normpath(path)
words = path.split('/')
words = filter(None, words)
path = os.getcwd()
for word in words:
if os.path.dirname(word) or word in (os.curdir, os.pardir):
# Ignore components that are not a simple file/directory name
continue
path = os.path.join(path, word)
if trailing_slash:
path += '/'
return path
def copyfile(self, source, outputfile):
"""Copy all data between two file objects.
The SOURCE argument is a file object open for reading
(or anything with a read() method) and the DESTINATION
argument is a file object open for writing (or
anything with a write() method).
The only reason for overriding this would be to change
the block size or perhaps to replace newlines by CRLF
-- note however that this the default server uses this
to copy binary data as well.
"""
shutil.copyfileobj(source, outputfile)
def guess_type(self, path):
"""Guess the type of a file.
Argument is a PATH (a filename).
Return value is a string of the form type/subtype,
usable for a MIME Content-type header.
The default implementation looks the file's extension
up in the table self.extensions_map, using application/octet-stream
as a default; however it would be permissible (if
slow) to look inside the data to make a better guess.
"""
base, ext = posixpath.splitext(path)
if ext in self.extensions_map:
return self.extensions_map[ext]
ext = ext.lower()
if ext in self.extensions_map:
return self.extensions_map[ext]
else:
return self.extensions_map['']
if not mimetypes.inited:
mimetypes.init() # try to read system mime.types
extensions_map = mimetypes.types_map.copy()
extensions_map.update({
'': 'application/octet-stream', # Default
'.py': 'text/plain',
'.c': 'text/plain',
'.h': 'text/plain',
})
class ResumeHTTPHandler(BaseHTTPRequestHandler):
server_version = "SimpleHTTPWithUpload/" + __version__
def do_GET(self):
"""Serve a GET request."""
f = self.send_head()
if f:
self.wfile.write(f)
templateStr = '''<html>
<head>
<title>QR Link Generator</title>
</head>
<body>
%s
<br>
<br>
<form action="/qr" name=f method=""><input maxLength=1024 size=70
name=s value="" title="Text to QR Encode"><input type=submit
value="Show QR" name=qr>
</form>
</body>
</html> '''
def do_HEAD(self):
"""Serve a HEAD request."""
f = self.send_head()
def do_POST(self):
"""Serve a POST request."""
r, info = self.deal_post_data()
print(info)
print("uploaded by:", self.client_address)
info = info.replace('\n', '<br>')
f = ('<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">') +\
('<html><head>') +\
('<meta http-equiv="Content-Type" content="text/html; charset=utf-8">') +\
('<title>Upload Result Page</title>') +\
('</head><body>') +\
('<h1>Upload Result Page</h1>') +\
('<hr>')
if r:
f = f + ('<strong>Success:<strong><br/>') + info
else:
f = f + ('<strong>Failed:<strong>') + info
f = f + '<br><a href="%s">back</a>' % self.headers['referer'] +\
'</body></html>'
f = f.encode('utf-8')
length = len(f)
self.send_response(200)
self.send_header("Content-type", "text/html")
self.send_header("Content-Length", str(length))
self.end_headers()
self.wfile.write(f)
'''
POST data
------WebKitFormBoundaryLVlRNkjiiJLtNYQE
Content-Disposition: form-data; name="file"; filename="file1.txt"
Content-Type: text/plain
content in file1
hello file1
------WebKitFormBoundaryLVlRNkjiiJLtNYQE
Content-Disposition: form-data; name="file"; filename="file2.txt"
Content-Type: text/plain
content in file2
hello file2
------WebKitFormBoundaryLVlRNkjiiJLtNYQE--
'''
def deal_post_data(self):
boundary = self.headers["Content-Type"].split("=")[1]
boundary_begin = ('--' + boundary + '\r\n').encode('utf-8')
boundary_end = ('--' + boundary + '--\r\n').encode('utf-8')
return_status = True
return_info = '\n'
outer = 1
inner = 2
leave = 3
loop_info = outer # 1: outer loop, 2: inner_loop, 3: leave and return
# first line
# b'------WebKitFormBoundaryLVlRNkjiiJLtNYQE'
line = self.rfile.readline()
while loop_info == outer:
# print(line)
line = line
if line != boundary_begin:
return_status = False
return_info += "Content NOT begin with boundary\n"
break
# get filename
# b'Content-Disposition: form-data; name="file"; filename="file1.txt"'
line = self.rfile.readline().decode('utf-8').rstrip('\r\n')
# print(line)
filename = re.findall(r'filename="(.*)"', line)[0]
# print(filename)
if not filename:
return_status = False
return_info += "Can't find out file name...\n"
loop_info = leave
break
path = self.translate_path(self.path)
filename = os.path.join(path, filename)
# if filename alread exists
if os.path.exists(filename):
filename += "_copy"
# second line
# b'Content-Type: text/plain'
line = self.rfile.readline()
# blank line
line = self.rfile.readline()
loop_info = inner
# POST data
try:
with open(filename, 'wb') as f:
while loop_info == inner:
line = self.rfile.readline()
# print(line)
if line == boundary_begin:
loop_info = outer
# print('out')
break
elif line == boundary_end:
# print('leave')
loop_info = leave
break
else:
# line 还是二进制形式, realine() 不会删掉二进制的'\n'
f.write(line)
except Exception as e:
loop_info = leave
return_status = False
return_info += 'Exception!\n'
return_info += filename + '\n'
return (return_status, return_info)
def send_head(self):
"""Common code for GET and HEAD commands.
This sends the response code and MIME headers. Return value is either a file object (which has to be copied
to the outputfile by the caller unless the command was HEAD, and must be closed by the caller under all
circumstances), or None, in which case the caller has nothing further to do.
"""
path = self.translate_path(self.path)
f = None
if os.path.isdir(path):
if not self.path.endswith('/'):
# redirect browser - doing basically what apache does
self.send_response(301)
self.send_header("Location", self.path + "/")
self.end_headers()
return None
for index in "index.html", "index.htm":
index = os.path.join(path, index)
if os.path.exists(index):
path = index
break
else:
return self.list_directory(path)
ctype = self.guess_type(path)
try:
# Always read in binary mode. Opening files in text mode may cause
# newline translations, making the actual size of the content
# transmitted *less* than the content-length!
f = open(path, 'rb')
except IOError:
self.send_error(404, "File not found")
return None
self.send_response(200)
self.send_header("Content-type", ctype)
fs = os.fstat(f.fileno())
self.send_header("Content-Length", str(fs[6]))
self.send_header("Last-Modified", self.date_time_string(fs.st_mtime))
self.end_headers()
data = f.read()
f.close()
return data
def list_directory(self, path):
"""Helper to produce a directory listing (absent index.html). Return value is either a file object, or None (indicating an
error). In either case, the headers are sent, making the interface the same as for send_head().
"""
try:
list = os.listdir(path)
except os.error:
self.send_error(404, "No permission to list directory")
return None
list.sort(key=lambda a: a.lower())
displaypath = cgi.escape(urllib.parse.unquote(self.path))
f = ('<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">') +\
('<html><head>') +\
('<meta http-equiv="Content-Type" content="text/html; charset=utf-8">') +\
('<title>Directory listing for %s</title>' % displaypath) +\
('</head><body>') +\
('<h1>Directory listing for %s</h1>' % displaypath) +\
('<form ENCTYPE="multipart/form-data" method="post">') +\
('<input name="file" type="file" multiple="multiple"/>') +\
('<input type="submit" value="upload"/></form>') +\
('<hr><ul>')
for name in list:
fullname = os.path.join(path, name)
displayname = linkname = name
# Append / for directories or @ for symbolic links
if os.path.isdir(fullname):
displayname = name + "/"
linkname = name + "/"
if os.path.islink(fullname):
displayname = name + "@"
# Note: a link to a directory displays with @ and links with /
f = f + ('<li><a href="%s">%s</a>' %
(urllib.parse.quote(linkname), cgi.escape(displayname)))
f = f + ("</ul><hr></body></html>")
f = f.encode('utf-8')
length = len(f)
self.send_response(200)
self.send_header("Content-type", "text/html")
self.send_header("Content-Length", str(length))
self.end_headers()
return f
def translate_path(self, path):
"""Translate a /-separated PATH to the local filename syntax. Components that mean special things to the local file system
(e.g. drive or directory names) are ignored. (XXX They should probably be diagnosed.)
"""
# abandon query parameters
path = path.split('?', 1)[0]
path = path.split('#', 1)[0]
path = posixpath.normpath(urllib.parse.unquote(path))
words = path.split('/')
words = filter(None, words)
path = os.getcwd()
for word in words:
drive, word = os.path.splitdrive(word)
head, word = os.path.split(word)
if word in (os.curdir, os.pardir): continue
path = os.path.join(path, word)
return path
def guess_type(self, path):
"""Guess the type of a file.
Argument is a PATH (a filename). Return value is a string of the form type/subtype,
usable for a MIME Content-type header. The default implementation looks the file's extension
up in the table self.extensions_map, using application/octet-stream as a default; however it
would be permissible (if slow) to look inside the data to make a better guess.
"""
base, ext = posixpath.splitext(path)
if ext in self.extensions_map:
return self.extensions_map[ext]
ext = ext.lower()
if ext in self.extensions_map:
return self.extensions_map[ext]
else:
return self.extensions_map['']
if not mimetypes.inited:
mimetypes.init() # try to read system mime.types
extensions_map = mimetypes.types_map.copy()
extensions_map.update({
'': 'application/octet-stream', # Default
'.py': 'text/plain',
'.c': 'text/plain',
'.h': 'text/plain',
})
class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
"""Simple HTTP request handler with GET and HEAD commands.
This serves files from the current directory and any of its
subdirectories. The MIME type for files is determined by
calling the .guess_type() method.
The GET and HEAD requests are identical except that the HEAD
request omits the actual contents of the file.
"""
server_version = "SimpleHTTP/" + __version__
def do_GET(self):
"""Serve a GET request."""
f = self.send_head()
if f:
self.copyfile(f, self.wfile)
f.close()
def do_HEAD(self):
"""Serve a HEAD request."""
f = self.send_head()
if f:
f.close()
def send_head(self):
"""Common code for GET and HEAD commands.
This sends the response code and MIME headers.
Return value is either a file object (which has to be copied
to the outputfile by the caller unless the command was HEAD,
and must be closed by the caller under all circumstances), or
None, in which case the caller has nothing further to do.
"""
path = self.translate_path(self.path)
f = None
if os.path.isdir(path):
if not self.path.endswith('/'):
# redirect browser - doing basically what apache does
self.send_response(301)
self.send_header("Location", self.path + "/")
self.end_headers()
return None
for index in "index.html", "index.htm":
index = os.path.join(path, index)
if os.path.exists(index):
path = index
break
else:
return self.list_directory(path)
ctype = self.guess_type(path)
try:
f = open(path, 'rb')
except IOError:
self.send_error(404, "File not found")
return None
self.send_response(200)
self.send_header("Content-type", ctype)
fs = os.fstat(f.fileno())
self.send_header("Content-Length", str(fs[6]))
self.send_header("Last-Modified", self.date_time_string(fs.st_mtime))
self.end_headers()
return f
def list_directory(self, path):
"""Helper to produce a directory listing (absent index.html).
Return value is either a file object, or None (indicating an
error). In either case, the headers are sent, making the
interface the same as for send_head().
"""
try:
list = os.listdir(path)
except os.error:
self.send_error(404, "No permission to list directory")
return None
list.sort(key=lambda a: a.lower())
r = []
displaypath = html.escape(urllib.parse.unquote(self.path))
r.append('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">')
r.append("<html>\n<title>Directory listing for %s</title>\n" %
displaypath)
r.append("<body>\n<h2>Directory listing for %s</h2>\n" % displaypath)
r.append("<hr>\n<ul>\n")
for name in list:
fullname = os.path.join(path, name)
displayname = linkname = name
# Append / for directories or @ for symbolic links
if os.path.isdir(fullname):
displayname = name + "/"
linkname = name + "/"
if os.path.islink(fullname):
displayname = name + "@"
# Note: a link to a directory displays with @ and links with /
r.append('<li><a href="%s">%s</a>\n' %
(urllib.parse.quote(linkname), html.escape(displayname)))
r.append("</ul>\n<hr>\n</body>\n</html>\n")
enc = sys.getfilesystemencoding()
encoded = ''.join(r).encode(enc)
f = io.BytesIO()
f.write(encoded)
f.seek(0)
self.send_response(200)
self.send_header("Content-type", "text/html; charset=%s" % enc)
self.send_header("Content-Length", str(len(encoded)))
self.end_headers()
return f
def translate_path(self, path):
"""Translate a /-separated PATH to the local filename syntax.
Components that mean special things to the local file system
(e.g. drive or directory names) are ignored. (XXX They should
probably be diagnosed.)
"""
# abandon query parameters
path = path.split('?', 1)[0]
path = path.split('#', 1)[0]
path = posixpath.normpath(urllib.parse.unquote(path))
words = path.split('/')
words = filter(None, words)
path = os.getcwd()
for word in words:
drive, word = os.path.splitdrive(word)
head, word = os.path.split(word)
if word in (os.curdir, os.pardir): continue
path = os.path.join(path, word)
return path
def copyfile(self, source, outputfile):
"""Copy all data between two file objects.
The SOURCE argument is a file object open for reading
(or anything with a read() method) and the DESTINATION
argument is a file object open for writing (or
anything with a write() method).
The only reason for overriding this would be to change
the block size or perhaps to replace newlines by CRLF
-- note however that this the default server uses this
to copy binary data as well.
"""
shutil.copyfileobj(source, outputfile)
def guess_type(self, path):
"""Guess the type of a file.
Argument is a PATH (a filename).
Return value is a string of the form type/subtype,
usable for a MIME Content-type header.
The default implementation looks the file's extension
up in the table self.extensions_map, using application/octet-stream
as a default; however it would be permissible (if
slow) to look inside the data to make a better guess.
"""
base, ext = posixpath.splitext(path)
if ext in self.extensions_map:
return self.extensions_map[ext]
ext = ext.lower()
if ext in self.extensions_map:
return self.extensions_map[ext]
else:
return self.extensions_map['']
if not mimetypes.inited:
mimetypes.init() # try to read system mime.types
extensions_map = mimetypes.types_map.copy()
extensions_map.update({
'': 'application/octet-stream', # Default
'.py': 'text/plain',
'.c': 'text/plain',
'.h': 'text/plain',
})
def mime_attach(body, attachments, charset, body_charset=None):
mimetypes.init()
message = MIMEMultipart('mixed')
bodypart = BetterMIMEText(body, _charset=(body_charset or charset))
bodypart.add_header('Content-Disposition', 'inline')
message.preamble = 'This is a multi-part MIME message sent by reportbug.\n\n'
message.epilogue = ''
message.attach(bodypart)
failed = False
for attachment in attachments:
try:
fp = file(attachment)
fp.close()
except EnvironmentError, x:
ewrite("Warning: opening '%s' failed: %s.\n", attachment,
x.strerror)
failed = True
continue
ctype = None
cset = charset
info = Popen(['file', '--mime', '--brief', attachment],
stdout=PIPE,
stderr=STDOUT).communicate()[0]
if info:
match = re.match(r'([^;, ]*)(,[^;]+)?(?:; )?(.*)', info)
if match:
ctype, junk, extras = match.groups()
match = re.search(r'charset=([^,]+|"[^,"]+")', extras)
if match:
cset = match.group(1)
# If we didn't get a real MIME type, fall back
if '/' not in ctype:
ctype = None
# If file doesn't work, try to guess based on the extension
if not ctype:
ctype, encoding = mimetypes.guess_type(attachment, strict=False)
if not ctype:
ctype = 'application/octet-stream'
maintype, subtype = ctype.split('/', 1)
if maintype == 'text':
fp = file(attachment, 'rU')
part = BetterMIMEText(fp.read(), _subtype=subtype, _charset=cset)
fp.close()
elif maintype == 'message':
fp = file(attachment, 'rb')
part = MIMEMessage(email.message_from_file(fp), _subtype=subtype)
fp.close()
elif maintype == 'image':
fp = file(attachment, 'rb')
part = MIMEImage(fp.read(), _subtype=subtype)
fp.close()
elif maintype == 'audio':
fp = file(attachment, 'rb')
part = MIMEAudio(fp.read(), _subtype=subtype)
fp.close()
else:
fp = file(attachment, 'rb')
part = MIMEBase(maintype, subtype)
part.set_payload(fp.read())
fp.close()
email.Encoders.encode_base64(part)
part.add_header('Content-Disposition',
'attachment',
filename=os.path.basename(attachment))
message.attach(part)
class dragdropserver(http.server.BaseHTTPRequestHandler):
# The Queue communicates with the stacoan.py file. It's a communication pipe.
q = Queue()
def log_request(self, code='-', size='-'):
if not any(
s in str(self.requestline)
for s in ('lootbox.html', '.ico', 'robots.txt', '.js',
'.css', 'start.html', '.woff2', '.png', '.jpg')):
Logger(self.requestline + " " + str(code) + " " + str(size),
Logger.INFO)
def log_error(self, format, *args):
if not any(s in str(self.requestline)
for s in ('lootbox.html', 'robots.txt')):
Logger(("%s - - [%s] %s - %s\n" %
(self.address_string(), self.log_date_time_string(),
format % args, str(self.requestline))),
Logger.WARNING)
def log_message(self, format, *args):
Logger(("%s - - [%s] %s\n" %
(self.address_string(), self.log_date_time_string(),
format % args)), Logger.INFO)
def do_GET(self):
"""Serve a GET request."""
f = self.send_head()
if f:
self.copyfile(f, self.wfile)
f.close()
def do_POST(self):
"""Serve a POST request."""
if re.findall(r'KILLSERVERCOMMAND', self.requestline):
ServerWrapper.dragdropserver.q.put("KILLSERVERCOMMAND")
Logger("Server upload killed", Logger.INFO)
self.send_response(200)
exit(0)
return True, "Exit"
r, info = self.deal_post_data()
Logger((str(r) + str(info) + "by: " + str(self.client_address)),
Logger.INFO)
f = BytesIO()
f.write(b'<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">')
f.write(b"<html>\n<title>Upload Result Page</title>\n")
f.write(b"<body>\n<h2>Upload Result Page</h2>\n")
f.write(b"<hr>\n")
if r:
f.write(b"<strong>Success:</strong>")
else:
f.write(b"<strong>Failed:</strong>")
f.write(info.encode())
f.write(("<br><a href=\"%s\">back</a>" %
self.headers['referer']).encode())
length = f.tell()
f.seek(0)
self.send_response(200)
self.send_header("Content-type", "text/html")
self.send_header("Content-Length", str(length))
self.end_headers()
if f:
self.copyfile(f, self.wfile)
f.close()
def deal_post_data(self):
content_type = self.headers['content-type']
if not content_type:
return (False, "Content-Type header doesn't contain boundary")
boundary = content_type.split("=")[1].encode()
remainbytes = int(self.headers['content-length'])
line = self.rfile.readline()
remainbytes -= len(line)
if not boundary in line:
return (False, "Content NOT begin with boundary")
for line in self.rfile:
remainbytes -= len(line)
fn = re.findall(
r'Content-Disposition.*name="file"; filename="(.*\S.*)"',
line.decode())
if fn:
break
if not fn:
return (False, "Can't find out file name...")
# check filetype serverside:
if not (str(fn[0]).endswith(tuple(ServerWrapper.apptypes))):
self.send_error(408, "Filetype not allowed")
return (False, "Filetype not allowed.")
path = self.translate_path(self.path)
# fn holds an array of files, have to make a for all loop
fn = os.path.join(path, fn[0])
line = self.rfile.readline()
remainbytes -= len(line)
line = self.rfile.readline()
remainbytes -= len(line)
try:
out = open(fn, 'wb')
except IOError:
return (
False,
"Can't create file to write, do you have permission to write?"
)
preline = self.rfile.readline()
remainbytes -= len(preline)
while remainbytes > 0:
line = self.rfile.readline()
remainbytes -= len(line)
if boundary in line:
preline = preline[0:-1]
if preline.endswith(b'\r'):
preline = preline[0:-1]
out.write(preline)
out.close()
# Start stacoan instance (program), by using the queue
ServerWrapper.dragdropserver.q.put(fn)
return True, "File '%s' upload success!" % fn
else:
out.write(preline)
preline = line
return False, "Unexpected End of data."
def send_head(self):
"""Common code for GET and HEAD commands.
This sends the response code and MIME headers.
Return value is either a file object (which has to be copied
to the outputfile by the caller unless the command was HEAD,
and must be closed by the caller under all circumstances), or
None, in which case the caller has nothing further to do.
"""
path = self.translate_path(self.path)
f = None
if os.path.isdir(path):
if not self.path.endswith('/'):
# redirect browser - doing basically what apache does
self.send_response(301)
self.send_header("Location", self.path + "/")
self.end_headers()
return None
for index in "index.html", "index.htm":
index = os.path.join(path, index)
if os.path.exists(index):
path = index
break
else:
return self.list_directory(path)
ctype = self.guess_type(path)
try:
# Always read in binary mode. Opening files in text mode may cause
# newline translations, making the actual size of the content
# transmitted *less* than the content-length!
f = open(path, 'rb')
except IOError:
self.send_error(404, "File not found")
return None
self.send_response(200)
self.send_header("Content-type", ctype)
fs = os.fstat(f.fileno())
self.send_header("Content-Length", str(fs[6]))
self.send_header("Last-Modified",
self.date_time_string(fs.st_mtime))
self.end_headers()
return f
def list_directory(self, path):
"""Helper to produce a directory listing (absent index.html).
Return value is either a file object, or None (indicating an
error). In either case, the headers are sent, making the
interface the same as for send_head().
"""
try:
list = os.listdir(path)
except os.error:
self.send_error(404, "No permission to list directory")
return None
list.sort(key=lambda a: a.lower())
f = BytesIO()
# ToDo make resources local (now files are loaded externally)
f.write(b"""
<head>
<meta charset="utf-8">
<title>Drag and Drop File Uploading</title>
<meta name="viewport" content="width=device-width,initial-scale=1" />
<link rel="stylesheet" href="//fonts.googleapis.com/css?family=Roboto:300,300italic,400" />
<script type="text/javascript" src="report/html/jquery.min.js"></script>
<style>
body
{
font-family: Roboto, sans-serif;
color: #0f3c4b;
background-color: #e5edf1;
padding: 5rem 1.25rem; /* 80 20 */
}
.container
{
width: 100%;
max-width: 680px; /* 800 */
text-align: center;
margin: 0 auto;
}
.container h1
{
font-size: 42px;
font-weight: 300;
color: #0f3c4b;
margin-bottom: 40px;
}
.container h1 a:hover,
.container h1 a:focus
{
color: #39bfd3;
}
.container nav
{
margin-bottom: 40px;
}
.container nav a
{
border-bottom: 2px solid #c8dadf;
display: inline-block;
padding: 4px 8px;
margin: 0 5px;
}
.container nav a.is-selected
{
font-weight: 700;
color: #39bfd3;
border-bottom-color: currentColor;
}
.container nav a:not( .is-selected ):hover,
.container nav a:not( .is-selected ):focus
{
border-bottom-color: #0f3c4b;
}
.container footer
{
color: #92b0b3;
margin-top: 40px;
}
.container footer p + p
{
margin-top: 1em;
}
.container footer a:hover,
.container footer a:focus
{
color: #39bfd3;
}
.box
{
font-size: 1.25rem; /* 20 */
background-color: #c8dadf;
position: relative;
padding: 100px 20px;
}
.box.has-advanced-upload
{
outline: 2px dashed #92b0b3;
outline-offset: -10px;
-webkit-transition: outline-offset .15s ease-in-out, background-color .15s linear;
transition: outline-offset .15s ease-in-out, background-color .15s linear;
}
.box.is-dragover
{
outline-offset: -20px;
outline-color: #c8dadf;
background-color: #fff;
}
.box__dragndrop,
.box__icon
{
display: none;
}
.box.has-advanced-upload .box__dragndrop
{
display: inline;
}
.box.has-advanced-upload .box__icon
{
width: 100%;
height: 80px;
fill: #92b0b3;
display: block;
margin-bottom: 40px;
}
.box.is-uploading .box__input,
.box.is-success .box__input,
.box.is-error .box__input
{
visibility: hidden;
}
.box__uploading,
.box__success,
.box__error
{
display: none;
}
.box.is-uploading .box__uploading,
.box.is-success .box__success,
.box.is-error .box__error
{
display: block;
position: absolute;
top: 50%;
right: 0;
left: 0;
-webkit-transform: translateY( -50% );
transform: translateY( -50% );
}
.box__uploading
{
font-style: italic;
}
.box__success
{
-webkit-animation: appear-from-inside .25s ease-in-out;
animation: appear-from-inside .25s ease-in-out;
}
@-webkit-keyframes appear-from-inside
{
from { -webkit-transform: translateY( -50% ) scale( 0 ); }
75% { -webkit-transform: translateY( -50% ) scale( 1.1 ); }
to { -webkit-transform: translateY( -50% ) scale( 1 ); }
}
@keyframes appear-from-inside
{
from { transform: translateY( -50% ) scale( 0 ); }
75% { transform: translateY( -50% ) scale( 1.1 ); }
to { transform: translateY( -50% ) scale( 1 ); }
}
.box__restart
{
font-weight: 700;
}
.box__restart:focus,
.box__restart:hover
{
color: #39bfd3;
}
.js .box__file
{
width: 0.1px;
height: 0.1px;
opacity: 0;
overflow: hidden;
position: absolute;
z-index: -1;
}
.js .box__file + label
{
max-width: 80%;
text-overflow: ellipsis;
white-space: nowrap;
cursor: pointer;
display: inline-block;
overflow: hidden;
}
.js .box__file + label:hover strong,
.box__file:focus + label strong,
.box__file.has-focus + label strong
{
color: #39bfd3;
}
.js .box__file:focus + label,
.js .box__file.has-focus + label
{
outline: 1px dotted #000;
outline: -webkit-focus-ring-color auto 5px;
}
.js .box__file + label *
{
/* pointer-events: none; */ /* in case of FastClick lib use */
}
.no-js .box__file + label
{
display: none;
}
.no-js .box__button
{
display: block;
}
.box__button
{
font-weight: 700;
color: #e5edf1;
background-color: #39bfd3;
display: none;
padding: 8px 16px;
margin: 40px auto 0;
}
.box__button:hover,
.box__button:focus
{
background-color: #0f3c4b;
}
.killserverbutton {
-moz-box-shadow:inset 0px 39px 0px -24px #e67a73;
-webkit-box-shadow:inset 0px 39px 0px -24px #e67a73;
box-shadow:inset 0px 39px 0px -24px #e67a73;
background-color:#e4685d;
-moz-border-radius:4px;
-webkit-border-radius:4px;
border-radius:4px;
border:1px solid #ffffff;
display:inline-block;
cursor:pointer;
color:#ffffff;
font-family:Arial;
font-size:15px;
padding:6px 15px;
text-decoration:none;
text-shadow:0px 1px 0px #b23e35;
}
.killserverbutton:hover {
background-color:#eb675e;
}
.killserverbutton:active {
position:relative;
top:1px;
}
</style>
<!-- remove this if you use Modernizr -->
<script>(function(e,t,n){var r=e.querySelectorAll("html")[0];r.className=r.className.replace(/(^|\s)no-js(\s|$)/,"$1js$2")})(document,window,0);</script>
</head>
<body>
<center>
<form id="killserverform" action="/KILLSERVERCOMMAND" target="" method="POST">
<input type="text" name="KILLSERVERCOMMAND" value="Mickey" hidden>
<input type="submit" value="KILL SERVER" class="killserverbutton">
</form>
<script>
$('#killserverform').submit(function() {
alert("Server will shut down! Bye!");
return true; // return false to cancel form action
});
</script>
</center>
<div class="container" role="main">
<form action="/" ENCTYPE="multipart/form-data" method="post" novalidate class="box">
<div class="box__input">
<svg class="box__icon" xmlns="http://www.w3.org/2000/svg" width="50" height="43" viewBox="0 0 50 43"><path d="M48.4 26.5c-.9 0-1.7.7-1.7 1.7v11.6h-43.3v-11.6c0-.9-.7-1.7-1.7-1.7s-1.7.7-1.7 1.7v13.2c0 .9.7 1.7 1.7 1.7h46.7c.9 0 1.7-.7 1.7-1.7v-13.2c0-1-.7-1.7-1.7-1.7zm-24.5 6.1c.3.3.8.5 1.2.5.4 0 .9-.2 1.2-.5l10-11.6c.7-.7.7-1.7 0-2.4s-1.7-.7-2.4 0l-7.1 8.3v-25.3c0-.9-.7-1.7-1.7-1.7s-1.7.7-1.7 1.7v25.3l-7.1-8.3c-.7-.7-1.7-.7-2.4 0s-.7 1.7 0 2.4l10 11.6z"/></svg>
""")
f.write(
b"""<input type="file" name="file" id="file" class="box__file" data-multiple-caption="{count} files selected" multiple accept=\""""
)
for apptype in ServerWrapper.apptypes:
f.write(bytes(str(apptype) + ", ", 'utf-8'))
f.write(b"""\" />""")
f.write(b"""
<label for="file" id="nameup"><strong>Choose a file</strong><span class="box__dragndrop"> or drag it here</span>.</label>
<button type="submit" class="box__button">Upload</button>
</div>
<div class="box__uploading">Uploading…</div>
<div class="box__success">Done! <a href="/" class="box__restart2" id="done_link" onclick="javascript:event.target.port="""
)
f.write(bytearray(str(ServerWrapper.REPORT_SERVER_PORT), 'utf8'))
f.write(b"""" role="button" target="_blank">Open report!</a></div>
<div class="box__error">Error! <span></span>. <a href="/?" class="box__restart" role="button">Try again!</a></div>
</form>
</div>
<script>
'use strict';
;( function ( document, window, index )
{
// feature detection for drag&drop upload
var isAdvancedUpload = function()
{
var div = document.createElement( 'div' );
return ( ( 'draggable' in div ) || ( 'ondragstart' in div && 'ondrop' in div ) ) && 'FormData' in window && 'FileReader' in window;
}();
// applying the effect for every form
var forms = document.querySelectorAll( '.box' );
Array.prototype.forEach.call( forms, function( form )
{
var input = form.querySelector( 'input[type="file"]' ),
label = form.querySelector( 'label' ),
errorMsg = form.querySelector( '.box__error span' ),
restart = form.querySelectorAll( '.box__restart' ),
droppedFiles = false,
showFiles = function( files )
{
label.textContent = files.length > 1 ? ( input.getAttribute( 'data-multiple-caption' ) || '' ).replace( '{count}', files.length ) : files[ 0 ].name;
},
triggerFormSubmit = function()
{
var event = document.createEvent( 'HTMLEvents' );
event.initEvent( 'submit', true, false );
form.dispatchEvent( event );
};
// letting the server side to know we are going to make an Ajax request
var ajaxFlag = document.createElement( 'input' );
ajaxFlag.setAttribute( 'type', 'hidden' );
ajaxFlag.setAttribute( 'name', 'ajax' );
ajaxFlag.setAttribute( 'value', 1 );
form.appendChild( ajaxFlag );
// automatically submit the form on file select
input.addEventListener( 'change', function( e )
{
showFiles( e.target.files );
triggerFormSubmit();
});
// drag&drop files if the feature is available
if( isAdvancedUpload )
{
form.classList.add( 'has-advanced-upload' ); // letting the CSS part to know drag&drop is supported by the browser
[ 'drag', 'dragstart', 'dragend', 'dragover', 'dragenter', 'dragleave', 'drop' ].forEach( function( event )
{
form.addEventListener( event, function( e )
{
// preventing the unwanted behaviours
e.preventDefault();
e.stopPropagation();
});
});
[ 'dragover', 'dragenter' ].forEach( function( event )
{
form.addEventListener( event, function()
{
form.classList.add( 'is-dragover' );
});
});
[ 'dragleave', 'dragend', 'drop' ].forEach( function( event )
{
form.addEventListener( event, function()
{
form.classList.remove( 'is-dragover' );
});
});
form.addEventListener( 'drop', function( e )
{
droppedFiles = e.dataTransfer.files; // the files that were dropped
showFiles( droppedFiles );
triggerFormSubmit();
});
}
// if the form was submitted
form.addEventListener( 'submit', function( e )
{
// preventing the duplicate submissions if the current one is in progress
if( form.classList.contains( 'is-uploading' ) ) return false;
form.classList.add( 'is-uploading' );
form.classList.remove( 'is-error' );
if( isAdvancedUpload ) // ajax file upload for modern browsers
{
e.preventDefault();
// gathering the form data
var ajaxData = new FormData( form );
if( droppedFiles )
{
Array.prototype.forEach.call( droppedFiles, function( file )
{
ajaxData.append( input.getAttribute( 'name' ), file );
document.getElementById("done_link").setAttribute("href", file.name.replace(/\./g, "_")+"/report/start.html");
});
}
// ajax request
var ajax = new XMLHttpRequest();
ajax.open( form.getAttribute( 'method' ), form.getAttribute( 'action' ), true );
ajax.onload = function()
{
form.classList.remove( 'is-uploading' );
if( ajax.status == 408 )
{
alert( 'Filetype is not allowed');
}
else if( ajax.status >= 200 && ajax.status < 400 )
{
var data = ajax.responseText;
console.log(data)
form.classList.add( data.includes("Success:") == true ? 'is-success' : 'is-error' );
if( !data.includes("Success:") ) errorMsg.textContent = data.error;
if (data.includes("Success:") ) document.getElementById("done_link").setAttribute("href", nameup.innerHTML.replace(/\./g, "_")+"/report/start.html");
}
else alert( 'Error. Please, contact the webmaster!' );
};
ajax.onerror = function()
{
form.classList.remove( 'is-uploading' );
alert( 'Error. Please, try again! Filetype is probably not supported.' );
};
ajax.send( ajaxData );
}
else // fallback Ajax solution upload for older browsers
{
alert(1);
var iframeName = 'uploadiframe' + new Date().getTime(),
iframe = document.createElement( 'iframe' );
$iframe = $( '<iframe name="' + iframeName + '" style="display: none;"></iframe>' );
iframe.setAttribute( 'name', iframeName );
iframe.style.display = 'none';
document.body.appendChild( iframe );
form.setAttribute( 'target', iframeName );
iframe.addEventListener( 'load', function()
{
var data = JSON.parse( iframe.contentDocument.body.innerHTML );
form.classList.remove( 'is-uploading' )
form.classList.add( data.success == true ? 'is-success' : 'is-error' )
form.removeAttribute( 'target' );
if( !data.success ) errorMsg.textContent = data.error;
iframe.parentNode.removeChild( iframe );
});
}
});
// restart the form if has a state of error/success
Array.prototype.forEach.call( restart, function( entry )
{
entry.addEventListener( 'click', function( e )
{
e.preventDefault();
form.classList.remove( 'is-error', 'is-success' );
input.click();
});
});
// Firefox focus bug fix for file input
input.addEventListener( 'focus', function(){ input.classList.add( 'has-focus' ); });
input.addEventListener( 'blur', function(){ input.classList.remove( 'has-focus' ); });
});
}( document, window, 0 ));
</script>
</body>
</html>
""")
# f.write(b"<form ENCTYPE=\"multipart/form-data\" method=\"post\">")
# f.write(b"<input name=\"file\" type=\"file\"/>")
# f.write(b"<input type=\"submit\" value=\"upload\"/></form>\n")
length = f.tell()
f.seek(0)
self.send_response(200)
self.send_header("Content-type", "text/html")
self.send_header("Content-Length", str(length))
self.end_headers()
return f
def translate_path(self, path):
"""Translate a /-separated PATH to the local filename syntax.
Components that mean special things to the local file system
(e.g. drive or directory names) are ignored. (XXX They should
probably be diagnosed.)
"""
# abandon query parameters
path = path.split('?', 1)[0]
path = path.split('#', 1)[0]
path = posixpath.normpath(urllib.parse.unquote(path))
words = path.split('/')
words = [_f for _f in words if _f]
path = os.getcwd()
for word in words:
drive, word = os.path.splitdrive(word)
head, word = os.path.split(word)
if word in (os.curdir, os.pardir): continue
path = os.path.join(path, word)
return path
def copyfile(self, source, outputfile):
"""Copy all data between two file objects.
The SOURCE argument is a file object open for reading
(or anything with a read() method) and the DESTINATION
argument is a file object open for writing (or
anything with a write() method).
The only reason for overriding this would be to change
the block size or perhaps to replace newlines by CRLF
-- note however that this the default server uses this
to copy binary data as well.
"""
shutil.copyfileobj(source, outputfile)
def guess_type(self, path):
"""Guess the type of a file.
Argument is a PATH (a filename).
Return value is a string of the form type/subtype,
usable for a MIME Content-type header.
The default implementation looks the file's extension
up in the table self.extensions_map, using application/octet-stream
as a default; however it would be permissible (if
slow) to look inside the data to make a better guess.
"""
base, ext = posixpath.splitext(path)
if ext in self.extensions_map:
return self.extensions_map[ext]
ext = ext.lower()
if ext in self.extensions_map:
return self.extensions_map[ext]
else:
return self.extensions_map['']
if not mimetypes.inited:
mimetypes.init() # try to read system mime.types
extensions_map = mimetypes.types_map.copy()
extensions_map.update({
'': 'application/octet-stream', # Default
'.py': 'text/plain',
'.c': 'text/plain',
'.h': 'text/plain',
})
import os
import re
import sys
import base64
import mimetypes
from grit import lazy_re
from grit import util
# There is a python bug that makes mimetypes crash if the Windows
# registry contains non-Latin keys ( http://bugs.python.org/issue9291
# ). Initing manually and blocking external mime-type databases will
# prevent that bug and if we add svg manually, it will still give us
# the data we need.
mimetypes.init([])
mimetypes.add_type('image/svg+xml', '.svg')
DIST_DEFAULT = 'chromium'
DIST_ENV_VAR = 'CHROMIUM_BUILD'
DIST_SUBSTR = '%DISTRIBUTION%'
# Matches beginning of an "if" block with trailing spaces.
_BEGIN_IF_BLOCK = lazy_re.compile(
'<if [^>]*?expr="(?P<expression>[^"]*)"[^>]*?>\s*')
# Matches ending of an "if" block with preceding spaces.
_END_IF_BLOCK = lazy_re.compile('\s*</if>')
# Used by DoInline to replace various links with inline content.
_STYLESHEET_RE = lazy_re.compile(
def __init__(self,
conf,
memcache=None,
logger=None,
account_ring=None,
container_ring=None,
object_ring=None):
if conf is None:
conf = {}
if logger is None:
self.logger = get_logger(conf, log_route='proxy-server')
else:
self.logger = logger
swift_dir = conf.get('swift_dir', '/etc/swift')
self.node_timeout = int(conf.get('node_timeout', 10))
self.conn_timeout = float(conf.get('conn_timeout', 0.5))
self.client_timeout = int(conf.get('client_timeout', 60))
self.put_queue_depth = int(conf.get('put_queue_depth', 10))
self.object_chunk_size = int(conf.get('object_chunk_size', 65536))
self.client_chunk_size = int(conf.get('client_chunk_size', 65536))
self.trans_id_suffix = conf.get('trans_id_suffix', '')
self.error_suppression_interval = \
int(conf.get('error_suppression_interval', 60))
self.error_suppression_limit = \
int(conf.get('error_suppression_limit', 10))
self.recheck_container_existence = \
int(conf.get('recheck_container_existence', 60))
self.recheck_account_existence = \
int(conf.get('recheck_account_existence', 60))
self.allow_account_management = \
config_true_value(conf.get('allow_account_management', 'no'))
self.object_post_as_copy = \
config_true_value(conf.get('object_post_as_copy', 'true'))
self.resellers_conf = ConfigParser()
self.resellers_conf.read(os.path.join(swift_dir, 'resellers.conf'))
self.object_ring = object_ring or Ring(swift_dir, ring_name='object')
self.container_ring = container_ring or Ring(swift_dir,
ring_name='container')
self.account_ring = account_ring or Ring(swift_dir,
ring_name='account')
self.memcache = memcache
mimetypes.init(mimetypes.knownfiles +
[os.path.join(swift_dir, 'mime.types')])
self.account_autocreate = \
config_true_value(conf.get('account_autocreate', 'no'))
self.expiring_objects_account = \
(conf.get('auto_create_account_prefix') or '.') + \
'expiring_objects'
self.expiring_objects_container_divisor = \
int(conf.get('expiring_objects_container_divisor') or 86400)
self.max_containers_per_account = \
int(conf.get('max_containers_per_account') or 0)
self.max_containers_whitelist = [
a.strip()
for a in conf.get('max_containers_whitelist', '').split(',')
if a.strip()
]
self.deny_host_headers = [
host.strip()
for host in conf.get('deny_host_headers', '').split(',')
if host.strip()
]
self.rate_limit_after_segment = \
int(conf.get('rate_limit_after_segment', 10))
self.rate_limit_segments_per_sec = \
int(conf.get('rate_limit_segments_per_sec', 1))
self.log_handoffs = config_true_value(conf.get('log_handoffs', 'true'))
self.cors_allow_origin = [
a.strip() for a in conf.get('cors_allow_origin', '').split(',')
if a.strip()
]
self.node_timings = {}
self.timing_expiry = int(conf.get('timing_expiry', 300))
self.sorting_method = conf.get('sorting_method', 'shuffle').lower()
self.allow_static_large_object = config_true_value(
conf.get('allow_static_large_object', 'true'))
value = conf.get('request_node_count', '2 * replicas').lower().split()
if len(value) == 1:
value = int(value[0])
self.request_node_count = lambda r: value
elif len(value) == 3 and value[1] == '*' and value[2] == 'replicas':
value = int(value[0])
self.request_node_count = lambda r: value * r.replica_count
else:
raise ValueError('Invalid request_node_count value: %r' %
''.join(value))
def GET(self, courseid, taskid, is_LTI):
""" GET request """
username = self.user_manager.session_username()
# Fetch the course
try:
course = self.course_factory.get_course(courseid)
except CourseNotFoundException as ex:
raise NotFound(description=str(ex))
if is_LTI and not self.user_manager.course_is_user_registered(course):
self.user_manager.course_register_user(course, force=True)
if not self.user_manager.course_is_open_to_user(course, username, is_LTI):
return handle_course_unavailable(self.cp.app.get_homepath(), self.template_helper, self.user_manager, course)
is_staff = self.user_manager.has_staff_rights_on_course(course, username)
try:
task = course.get_task(taskid)
if not self.user_manager.task_is_visible_by_user(task, username, is_LTI):
return self.template_helper.render("task_unavailable.html")
except TaskNotFoundException:
raise NotFound()
user_task_list = course.get_task_dispenser().get_user_task_list([username])[username]
if taskid not in user_task_list:
previous_taskid = None
next_taskid = None
else:
# Compute previous and next taskid
index = user_task_list.index(taskid)
previous_taskid = user_task_list[index - 1] if index > 0 else None
next_taskid = user_task_list[index + 1] if index < len(user_task_list) - 1 else None
self.user_manager.user_saw_task(username, courseid, taskid)
is_staff = self.user_manager.has_staff_rights_on_course(course, username)
userinput = flask.request.args
if "submissionid" in userinput and "questionid" in userinput:
# Download a previously submitted file
submission = self.submission_manager.get_submission(userinput["submissionid"], user_check=not is_staff)
if submission is None:
raise self.cp.app.notfound(message=_("Submission doesn't exist."))
sinput = self.submission_manager.get_input_from_submission(submission, True)
if userinput["questionid"] not in sinput:
raise NotFound()
if isinstance(sinput[userinput["questionid"]], dict):
# File uploaded previously
mimetypes.init()
mime_type = mimetypes.guess_type(urllib.request.pathname2url(sinput[userinput["questionid"]]['filename']))
return Response(response=sinput[userinput["questionid"]]['value'], content_type=mime_type[0])
else:
# Other file, download it as text
return Response(response=sinput[userinput["questionid"]], content_type='text/plain')
else:
# Generate random inputs and save it into db
random.seed(str(username if username is not None else "") + taskid + courseid + str(
time.time() if task.regenerate_input_random() else ""))
random_input_list = [random.random() for i in range(task.get_number_input_random())]
user_task = self.database.user_tasks.find_one_and_update(
{
"courseid": task.get_course_id(),
"taskid": task.get_id(),
"username": self.user_manager.session_username()
},
{
"$set": {"random": random_input_list}
},
return_document=ReturnDocument.AFTER
)
submissionid = user_task.get('submissionid', None)
eval_submission = self.database.submissions.find_one({'_id': ObjectId(submissionid)}) if submissionid else None
students = [self.user_manager.session_username()]
if task.is_group_task() and not self.user_manager.has_admin_rights_on_course(course, username):
group = self.database.groups.find_one({"courseid": task.get_course_id(),
"students": self.user_manager.session_username()})
if group is not None:
students = group["students"]
# we don't care for the other case, as the student won't be able to submit.
submissions = self.submission_manager.get_user_submissions(task) if self.user_manager.session_logged_in() else []
user_info = self.user_manager.get_user_info(username)
# Visible tags
course_tags = course.get_tags()
task_categories = task.get_categories()
visible_tags = [course_tags[category] for category in task_categories if
course_tags[category].is_visible_for_student() or self.user_manager.has_staff_rights_on_course(course)]
# Problem dict
pdict = {problem.get_id(): problem.get_type() for problem in task.get_problems()}
is_input_list = {problem.get_id(): 1 if problem.input_type() == list else 0 for problem in task.get_problems()}
# Display the task itself
return self.template_helper.render("task.html", user_info=user_info, course=course, task=task,
submissions=submissions, students=students,
eval_submission=eval_submission, user_task=user_task,
previous_taskid=previous_taskid, next_taskid=next_taskid,
webterm_link=self.webterm_link, input_random_list=random_input_list,
visible_tags=visible_tags, pdict=pdict, is_input_list=is_input_list)
class JadeRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
def log_message(self, format, *args):
return
def do_GET(self):
path = self.path
path = path.split('?', 1)[0]
path = path.split('#', 1)[0]
path = path.replace('/', '')
ctype = self.guess_type(path)
try:
f = open(path, 'rb')
except IOError:
self.send_error(404, "File not found")
return None
try:
self.send_response(200)
self.send_header("Content-type", ctype)
fs = os.fstat(f.fileno())
self.send_header("Content-Length", str(fs[6]))
self.send_header("Last-Modified",
self.date_time_string(fs.st_mtime))
self.end_headers()
shutil.copyfileobj(f, self.wfile)
f.close()
except:
f.close()
raise
def do_POST(self):
# determine key, value
ctype, pdict = cgi.parse_header(self.headers.getheader('content-type'))
if ctype == 'multipart/form-data':
postvars = cgi.parse_multipart(self.rfile, pdict)
elif ctype == 'application/x-www-form-urlencoded':
length = int(self.headers.getheader('content-length'))
postvars = cgi.parse_qs(self.rfile.read(length),
keep_blank_values=1)
else:
postvars = {}
key = postvars.get('key', [None])[0]
value = postvars.get('value', [None])[0]
response = ''
if value is None:
# return stored value
c.execute('select val from key_value where key=?;', (key, ))
row = c.fetchone()
if row is not None:
response = row[0]
else:
# update stored value
c.execute(
'insert or replace into key_value (key,val) values (?,?);',
(key, value))
db.commit()
self.send_response(200)
self.send_header("Content-type", 'text/plain')
self.send_header("Content-Length", str(len(response)))
self.end_headers()
self.wfile.write(response)
def guess_type(self, path):
base, ext = posixpath.splitext(path)
if ext in self.extensions_map:
return self.extensions_map[ext]
ext = ext.lower()
if ext in self.extensions_map:
return self.extensions_map[ext]
else:
return self.extensions_map['']
if not mimetypes.inited:
mimetypes.init() # try to read system mime.types
extensions_map = mimetypes.types_map.copy()
extensions_map.update({
'': 'application/octet-stream', # Default
})
def init_mimetypes(mimetypes):
# this is a function so it can be unittested
if hasattr(mimetypes, 'init'):
mimetypes.init()
return True
return False
import sys
import time
from tempfile import NamedTemporaryFile
import sacred.optional as opt
from sacred.commandline_options import CommandLineOption
from sacred.dependencies import get_digest
from sacred.observers.base import RunObserver
from sacred.observers.queue import QueueObserver
from sacred.serializer import flatten
from sacred.utils import ObserverError
DEFAULT_MONGO_PRIORITY = 30
# This ensures consistent mimetype detection across platforms.
mimetypes.init(files=[])
def force_valid_bson_key(key):
key = str(key)
if key.startswith('$'):
key = '@' + key[1:]
key = key.replace('.', ',')
return key
def force_bson_encodeable(obj):
import bson
if isinstance(obj, dict):
try:
bson.BSON.encode(obj, check_keys=True)
# SPDX-License-Identifier: MIT import os import sys import rq import shutil import tempfile import numpy as np from PIL import Image from traceback import print_exception from ast import literal_eval import mimetypes _SCRIPT_DIR = os.path.realpath(os.path.dirname(__file__)) _MEDIA_MIMETYPES_FILE = os.path.join(_SCRIPT_DIR, "media.mimetypes") mimetypes.init(files=[_MEDIA_MIMETYPES_FILE]) import django_rq from django.conf import settings from django.db import transaction from ffmpy import FFmpeg from pyunpack import Archive from distutils.dir_util import copy_tree from . import models from .log import slogger ############################# Low Level server API def create(tid, data):
def _init_mimetypes():
global _mt_inited
import mimetypes
mimetypes.init([P('mime.types')])
_mt_inited = True
class TestRequestHandler(BaseHTTPRequestHandler):
def __init__(self, request, client_address, server):
BaseHTTPRequestHandler.__init__(self, request, client_address, server)
self.protocol_version = 'HTTP/1.1'
def do_GET(self):
self.handle_data()
def do_POST(self):
self.form = cgi.FieldStorage(
fp=self.rfile,
headers=self.headers,
environ={'REQUEST_METHOD':'POST',
'CONTENT_TYPE':self.headers['Content-Type'],
},
keep_blank_values=True,
strict_parsing=False)
self.handle_data()
def handle_data(self):
if self.path == '/':
p = '/html/fp.html'
else:
p = self.path
path = self.translate_path(p)
if not os.path.exists(path):
p = '/html'+p
path = self.translate_path(p)
ctype = self.guess_type(path)
try:
f = open(path)
except IOError:
print 'File not found %s' % path
self.send_error(404, 'File not found')
return
self.send_response(200)
self.send_header('Content-type', ctype)
self.send_header('Last-Modified', self.date_time_string())
self.end_headers()
self.copyfile(f, self.wfile)
f.close()
def translate_path(self, path):
path = path.decode('utf-8')
path = urlparse.urlparse(path)[2]
path = posixpath.normpath(urllib.unquote(path))
words = path.split('/')
words = filter(None, words)
path = os.getcwd()
for word in words:
drive, word = os.path.splitdrive(word)
head, word = os.path.split(word)
if word in (os.curdir, os.pardir): continue
path = os.path.join(path, word)
return path
def copyfile(self, source, outputfile):
shutil.copyfileobj(source, outputfile)
def guess_type(self, path):
base, ext = posixpath.splitext(path)
if ext in self.extensions_map:
return self.extensions_map[ext]
ext = ext.lower()
if ext in self.extensions_map:
return self.extensions_map[ext]
else:
return self.extensions_map['']
if not mimetypes.inited:
mimetypes.init() # try to read system mime.types
extensions_map = mimetypes.types_map.copy()
extensions_map.update({
'': 'application/octet-stream', # Default
})
class SimpleHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
"""Simple HTTP request handler with GET/HEAD/POST commands.
This serves files from the current directory and any of its
subdirectories. The MIME type for files is determined by
calling the .guess_type() method. And can reveive file uploaded
by client.
The GET/HEAD/POST requests are identical except that the HEAD
request omits the actual contents of the file.
"""
counter = Counter()
def do_HEAD(self):
"""Serve a HEAD request."""
f = self.send_head()
if f:
f.close()
def is_authenticated(self):
auth_header = self.headers.getheader('Authorization')
return auth_header and auth_header == 'Basic ' + key()
def do_AUTHHEAD(self):
self.send_response(401)
self.send_header('WWW-Authenticate', 'Basic realm=\"Test\"')
self.send_header('Content-type', 'text/html')
self.end_headers()
def try_authenticate(self):
if not self.is_authenticated():
self.do_AUTHHEAD()
print 'not authenticated'
self.wfile.write('not authenticated')
return False
return True
def do_GET(self):
if not self.try_authenticate():
return
print 'authenticated'
f = self.send_head()
if f:
self.copyfile(f, self.wfile)
f.close()
def do_POST(self):
if not self.try_authenticate():
return
print 'authenticated'
"""Serve a POST request."""
r, info = self.deal_post_data()
print r, info, "by: ", self.client_address
f = StringIO()
f.write('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">')
f.write("<html>\n<title>Upload Result Page</title>\n")
f.write("<body>\n<h2>Upload Result Page</h2>\n")
f.write("<hr>\n")
if r:
f.write("<strong>Success:</strong>")
else:
f.write("<strong>Failed:</strong>")
f.write(info)
f.write("<br><a href=\"%s\">back</a>" %
urllib.quote(self.headers['referer']))
f.write("<hr><small>Powerd By: bones7456, check new version at ")
f.write("<a href=\"http://li2z.cn/?s=SimpleHTTPServerWithUpload\">")
f.write("</body>\n</html>\n")
length = f.tell()
f.seek(0)
self.send_response(200)
self.send_header("Content-type", "text/html")
self.send_header("Content-Length", str(length))
self.end_headers()
if f:
self.copyfile(f, self.wfile)
f.close()
def deal_post_data(self):
boundary = self.headers.plisttext.split("=")[1]
remainbytes = int(self.headers['content-length'])
line = self.rfile.readline()
remainbytes -= len(line)
if not boundary in line:
return (False, "Content NOT begin with boundary")
line = self.rfile.readline()
remainbytes -= len(line)
fn = re.findall(r'Content-Disposition.*name="file"; filename="(.*)"',
line)
if not fn:
return (False, "Can't find out file name...")
path = self.url_path_to_file_path(self.path)
fn = os.path.join(path, fn[0].replace('\\', '/').split('/')[-1])
if os.path.exists(fn):
return (False, "The path already exists, you cannot overwrite it.")
line = self.rfile.readline()
remainbytes -= len(line)
line = self.rfile.readline()
remainbytes -= len(line)
try:
out = open(fn, 'wb')
except IOError:
return (
False,
"Can't create file to write, do you have permission to write?")
preline = self.rfile.readline()
remainbytes -= len(preline)
while remainbytes > 0:
line = self.rfile.readline()
remainbytes -= len(line)
if boundary in line:
preline = preline[0:-1]
if preline.endswith('\r'):
preline = preline[0:-1]
out.write(preline)
out.close()
return (True, "File '%s' upload success!" % fn)
else:
out.write(preline)
preline = line
return (False, "Unexpect Ends of data.")
def send_head(self):
"""Common code for GET and HEAD commands.
This sends the response code and MIME headers.
Return value is either a file object (which has to be copied
to the outputfile by the caller unless the command was HEAD,
and must be closed by the caller under all circumstances), or
None, in which case the caller has nothing further to do.
"""
print 'url_path', self.path
file_path = self.url_path_to_file_path(self.path)
print 'file_path', file_path
f = None
if os.path.isdir(file_path):
if not self.path.endswith('/'):
# redirect browser - doing basically what apache does
self.send_response(301)
self.send_header("Location", self.path + "/")
self.end_headers()
return None
for index in "index.html", "index.htm":
index = os.path.join(file_path, index)
if os.path.exists(index):
file_path = index
break
self.counter.incr_counter(file_path)
if os.path.isdir(file_path):
return self.list_directory(file_path)
ctype = self.guess_type(file_path)
try:
# Always read in binary mode. Opening files in text mode may cause
# newline translations, making the actual size of the content
# transmitted *less* than the content-length!
f = open(file_path, 'rb')
except IOError:
self.send_error(404, "File not found " + file_path)
return None
self.send_response(200)
self.send_header("Content-type", ctype)
fs = os.fstat(f.fileno())
self.send_header("Content-Length", str(fs[6]))
self.send_header("Last-Modified", self.date_time_string(fs.st_mtime))
self.end_headers()
return f
def list_directory(self, dir_path):
"""Helper to produce a directory listing (absent index.html).
Return value is either a file object, or None (indicating an
error). In either case, the headers are sent, making the
interface the same as for send_head().
"""
try:
list = os.listdir(dir_path)
except os.error:
self.send_error(404, "No permission to list directory")
return None
list.sort(key=lambda a: a.lower())
if dir_path != '/':
list = ['..'] + list
f = StringIO()
displaypath = cgi.escape(urllib.unquote(self.path))
f.write('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">')
f.write("<html>\n<title>Directory listing for %s</title>\n" %
displaypath)
f.write(
"<body>\n<h2>Directory listing for %s (frequently used directories are more reddish)</h2>\n"
% displaypath)
f.write("<hr>\n")
f.write("<form ENCTYPE=\"multipart/form-data\" method=\"post\">")
f.write("<input name=\"file\" type=\"file\"/>")
f.write("<input type=\"submit\" value=\"upload\"/></form>\n")
f.write("<hr>\n<ul>\n")
tot_counts = 0
for name in list:
child_file_path = posixpath.normpath(os.path.join(dir_path, name))
counts = self.counter.read_counter(child_file_path)
print child_file_path, counts
tot_counts += counts
# avoid divide by zero error
if tot_counts == 0:
tot_counts += 1
for name in list:
child_file_path = posixpath.normpath(os.path.join(dir_path, name))
displayname = linkname = name
# Append / for directories or @ for symbolic links
if os.path.isdir(child_file_path):
displayname = name + "/"
linkname = name + "/"
if os.path.islink(child_file_path):
displayname = name + "@"
# Note: a link to a directory displays with @ and links with /
counts = self.counter.read_counter(child_file_path)
# red portion of rgb value. with **0.2, it's overall more reddish
rgb_r = 255 * (float(counts) / tot_counts)**0.2
f.write('<li><a style="color:rgb(%d,0,0)" href="%s">%s</a>\n' %
(rgb_r, urllib.quote(linkname), cgi.escape(displayname)))
f.write("</ul>\n<hr>\n</body>\n</html>\n")
length = f.tell()
f.seek(0)
self.send_response(200)
self.send_header("Content-type", "text/html")
self.send_header("Content-Length", str(length))
self.end_headers()
return f
def url_path_to_file_path(self, url_path):
# 'base_url/' => '/'
# 'base_url/home/' => '/home'
# 'base_url/home/test/' => '/home/test'
# 'base_url/home/test.zip' => '/home/test.zip'
# 'base_url/home/%22test%22/' => '/home/"test"'
url_path = url_path[len(settings.base_url):]
# abandon query parameters
url_path = url_path.split('?', 1)[0]
url_path = url_path.split('#', 1)[0]
url_path = posixpath.normpath(urllib.unquote(url_path))
return url_path
def copyfile(self, source, outputfile):
"""Copy all data between two file objects.
The SOURCE argument is a file object open for reading
(or anything with a read() method) and the DESTINATION
argument is a file object open for writing (or
anything with a write() method).
The only reason for overriding this would be to change
the block size or perhaps to replace newlines by CRLF
-- note however that this the default server uses this
to copy binary data as well.
"""
shutil.copyfileobj(source, outputfile)
def guess_type(self, path):
"""Guess the type of a file.
Argument is a PATH (a filename).
Return value is a string of the form type/subtype,
usable for a MIME Content-type header.
The default implementation looks the file's extension
up in the table self.extensions_map, using application/octet-stream
as a default; however it would be permissible (if
slow) to look inside the data to make a better guess.
"""
base, ext = posixpath.splitext(path)
if ext in self.extensions_map:
return self.extensions_map[ext]
ext = ext.lower()
if ext in self.extensions_map:
return self.extensions_map[ext]
else:
return self.extensions_map['']
if not mimetypes.inited:
mimetypes.init() # try to read system mime.types
extensions_map = mimetypes.types_map.copy()
extensions_map.update({
'': 'application/octet-stream', # Default
'.py': 'text/plain',
'.c': 'text/plain',
'.h': 'text/plain',
})
class RangeHTTPRequestHandler(http.server.BaseHTTPRequestHandler):
"""Simple HTTP request handler with GET and HEAD commands.
This serves files from the current directory and any of its
subdirectories. The MIME type for files is determined by
calling the .guess_type() method.
The GET and HEAD requests are identical except that the HEAD
request omits the actual contents of the file.
"""
server_version = "RangeHTTP/" + __version__
def do_GET(self):
"""Serve a GET request."""
f, start_range, end_range = self.send_head()
print("Got values of {} and {}".format(start_range, end_range))
if f:
f.seek(start_range, 0)
chunk = 0x1000
total = 0
while chunk > 0:
if start_range + chunk > end_range:
chunk = end_range - start_range
if _bandwidth != 0:
time_to_sleep = float(float(chunk) / float(_bandwidth))
time.sleep(time_to_sleep)
try:
self.wfile.write(f.read(chunk))
except Exception:
break
total += chunk
start_range += chunk
f.close()
def do_HEAD(self):
"""Serve a HEAD request."""
f, start_range, end_range = self.send_head()
if f:
f.close()
def send_head(self):
"""Common code for GET and HEAD commands.
This sends the response code and MIME headers.
Return value is either a file object (which has to be copied
to the outputfile by the caller unless the command was HEAD,
and must be closed by the caller under all circumstances), or
None, in which case the caller has nothing further to do.
"""
path = self.translate_path(self.path)
f = None
if os.path.isdir(path):
if not self.path.endswith("/"):
# redirect browser
self.send_response(301)
self.send_header("Location", self.path + "/")
self.end_headers()
return (None, 0, 0)
for index in "index.html", "index.html":
index = os.path.join(path, index)
if os.path.exists(index):
path = index
break
else:
return self.list_directory(path)
ctype = self.guess_type(path)
try:
# Always read in binary mode. Opening files in text mode may cause
# newline translations, making the actual size of the content
# transmitted *less* than the content-length!
f = open(path, "rb")
except IOError:
self.send_error(404, "File not found")
return (None, 0, 0)
if "Range" in self.headers:
self.send_response(206) # partial content response
else:
self.send_response(200)
self.send_header("Content-type", ctype)
file_size = os.path.getsize(path)
start_range = 0
end_range = file_size
self.send_header("Accept-Ranges", "bytes")
if "Range" in self.headers:
s, e = self.headers['range'][6:].split('-', 1) # bytes:%d-%d
sl = len(s)
el = len(e)
if sl:
start_range = int(s)
if el:
end_range = int(e) + 1
elif el:
start_range = file_size - min(file_size, int(e))
self.send_header(
"Content-Range", "bytes {}-{}/{}".format(start_range, end_range,
file_size))
self.send_header("Content-Length", end_range - start_range)
self.end_headers()
print("Sending bytes {} to {}...".format(start_range, end_range))
return (f, start_range, end_range)
def list_directory(self, path):
"""Helper to produce a directory listing (absent index.html).
Return value is either a file object, or None (indicating an
error). In either case, the headers are sent, making the
interface the same as for send_head().
"""
try:
lst = os.listdir(path)
except OSError:
self.send_error(404, "Access Forbidden")
return None
lst.sort(key=lambda file_name: file_name.lower())
html_text = []
displaypath = html.escape(urllib.parse.unquote(self.path))
html_text.append(
'<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">')
html_text.append(
"<html>\n<title>Directory listing for {}</title>\n".format(
displaypath))
html_text.append(
"<body>\n<h2>Directory listing for {}</h2>\n".format(displaypath))
html_text.append("<hr>\n<ul>\n")
for name in lst:
fullname = os.path.join(path, name)
displayname = linkname = name
if os.path.isdir(fullname):
displayname = name + "/"
linkname = name + "/"
if os.path.islink(fullname):
displayname = name + "@"
html_text.append('<li><a href = "{}">{}</a>\n'.format(
urllib.parse.quote(linkname), html.escape(displayname)))
html_text.append('</ul>\n</hr>\n</body>\n</html>\n')
byte_encoded_string = "\n".join(html_text).encode(
"utf-8", "surrogateescape")
f = io.BytesIO()
f.write(byte_encoded_string)
length = len(byte_encoded_string)
f.seek(0)
self.send_response(200)
self.send_header("Content-type", "text/html")
self.send_header("Content-length", str(length))
self.end_headers()
return (f, 0, length)
def translate_path(self, path):
"""Translate a /-separated PATH to the local filename syntax.
Components that mean special things to the local file system
(e.g. drive or directory names) are ignored. (XXX They should
probably be diagnosed.)
"""
# abandon query parameters
path = path.split("?", 1)[0]
path = path.split("#", 1)[0]
path = posixpath.normpath(urllib.parse.unquote(path))
words = path.split("/")
words = filter(None, words)
path = os.getcwd()
for word in words:
drive, word = os.path.splitdrive(word)
head, word = os.path.split(word)
if word in (os.curdir, os.pardir):
continue
path = os.path.join(path, word)
return path
def guess_type(self, path):
"""Guess the type of a file.
Argument is a PATH (a filename).
Return value is a string of the form type/subtype,
usable for a MIME Content-type header.
The default implementation looks the file's extension
up in the table self.extensions_map, using application/octet-stream
as a default; however it would be permissible (if
slow) to look inside the data to make a better guess.
"""
base, ext = posixpath.splitext(path)
if ext in self.extension_map:
return self.extension_map[ext]
ext = ext.lower()
if ext in self.extension_map:
return self.extension_map[ext]
else:
return self.extension_map['']
if not mimetypes.inited:
mimetypes.init()
extension_map = mimetypes.types_map.copy()
extension_map.update({
'': 'application/octet-stream', # Default
'.py': 'text/plain',
'.c': 'text/plain',
'.h': 'text/plain',
'.mp4': 'video/mp4',
'.ogg': 'video/ogg',
'.java': 'text/plain',
})
def _mimetypes():
import mimetypes
mimetypes.init()
return mimetypes
def setUp(self):
# ensure all entries actually come from the Windows registry
self.original_types_map = mimetypes.types_map.copy()
mimetypes.types_map.clear()
mimetypes.init()
self.db = mimetypes.MimeTypes()
import Foundation
from PyObjCTools import AppHelper
from Foundation import NSBundle
assert Foundation.NSThread.isMultiThreaded()
# Don't load Python modules from system paths
remove_paths = list(path for path in sys.path if 'site-packages' in path)
for path in remove_paths:
sys.path.remove(path)
# Make mimetypes use our copy of the file in order to work with sandboxing
resource_path = str(Foundation.NSBundle.mainBundle().resourcePath())
mime_path = os.path.join(resource_path, "mime.types")
mimetypes.init(files=[mime_path])
class NSLogger(object):
closed = False
encoding = 'UTF-8'
mode = 'w'
name = '<NSLogger>'
newlines = None
softspace = 0
def close(self):
pass
def flush(self):
pass
def _put_get_user_stage(tmpdir,
conn_cnx,
db_parameters,
number_of_files=1,
number_of_lines=1,
from_path=True):
# sanity check
assert 'AWS_ACCESS_KEY_ID' in os.environ, 'AWS_ACCESS_KEY_ID is missing'
assert 'AWS_SECRET_ACCESS_KEY' in os.environ, \
'AWS_SECRET_ACCESS_KEY is missing'
if not from_path:
assert number_of_files == 1
tmp_dir = generate_k_lines_of_n_files(number_of_lines,
number_of_files,
tmp_dir=str(tmpdir.mkdir('data')))
files = os.path.join(tmp_dir,
'file*' if from_path else os.listdir(tmp_dir)[0])
file_stream = None if from_path else open(files, 'rb')
stage_name = db_parameters['name'] + '_stage_{}_{}'.format(
number_of_files, number_of_lines)
with conn_cnx(user=db_parameters['user'],
account=db_parameters['account'],
password=db_parameters['password']) as cnx:
cnx.cursor().execute("""
create or replace table {name} (
aa int,
dt date,
ts timestamp,
tsltz timestamp_ltz,
tsntz timestamp_ntz,
tstz timestamp_tz,
pct float,
ratio number(6,2))
""".format(name=db_parameters['name']))
user_bucket = os.getenv('SF_AWS_USER_BUCKET',
"sfc-dev1-regression/{}/reg".format(getuser()))
cnx.cursor().execute("""
create or replace stage {stage_name}
url='s3://{user_bucket}/{stage_name}-{number_of_files}-{number_of_lines}'
credentials=(
AWS_KEY_ID='{aws_key_id}'
AWS_SECRET_KEY='{aws_secret_key}'
)
""".format(stage_name=stage_name,
user_bucket=user_bucket,
aws_key_id=os.getenv('AWS_ACCESS_KEY_ID'),
aws_secret_key=os.getenv('AWS_SECRET_ACCESS_KEY'),
number_of_files=number_of_files,
number_of_lines=number_of_lines))
try:
with conn_cnx(user=db_parameters['user'],
account=db_parameters['account'],
password=db_parameters['password']) as cnx:
cnx.cursor().execute(
"alter session set disable_put_and_get_on_external_stage = false"
)
cnx.cursor().execute(
"rm @{stage_name}".format(stage_name=stage_name))
put(cnx.cursor(),
files,
stage_name,
from_path,
file_stream=file_stream)
cnx.cursor().execute("copy into {name} from @{stage_name}".format(
name=db_parameters['name'], stage_name=stage_name))
c = cnx.cursor()
try:
c.execute("select count(*) from {name}".format(
name=db_parameters['name']))
rows = 0
for rec in c:
rows += rec[0]
assert rows == number_of_files * number_of_lines, \
'Number of rows'
finally:
c.close()
cnx.cursor().execute(
"rm @{stage_name}".format(stage_name=stage_name))
cnx.cursor().execute("copy into @{stage_name} from {name}".format(
name=db_parameters['name'], stage_name=stage_name))
tmp_dir_user = str(tmpdir.mkdir('put_get_stage'))
cnx.cursor().execute(
"get @{stage_name}/ file://{tmp_dir_user}/".format(
stage_name=stage_name, tmp_dir_user=tmp_dir_user))
for _, _, files in os.walk(tmp_dir_user):
for file in files:
mimetypes.init()
_, encoding = mimetypes.guess_type(file)
assert encoding == 'gzip', "exported file type"
finally:
if file_stream:
file_stream.close()
with conn_cnx(user=db_parameters['user'],
account=db_parameters['account'],
password=db_parameters['password']) as cnx:
cnx.cursor().execute(
"rm @{stage_name}".format(stage_name=stage_name))
cnx.cursor().execute("drop stage if exists {stage_name}".format(
stage_name=stage_name))
cnx.cursor().execute("drop table if exists {name}".format(
name=db_parameters['name']))
class CustomHandler(BaseHTTPServer.BaseHTTPRequestHandler):
"""Custom HTTP request handler with GET and HEAD and POST commands.
This serves files from the current directory and any of its
subdirectories. The MIME type for files is determined by
calling the .guess_type() method.
The GET and HEAD requests are identical except that the HEAD
request omits the actual contents of the file.
"""
post_listeners = set()
@classmethod
def add_post_listener(cls, function):
cls.post_listeners.add(function)
@classmethod
def inform_post_listeners(cls, json_string):
for func in cls.post_listeners:
func(json_string)
def do_GET(self):
"""Serve a GET request."""
f = self.send_head()
if f:
try:
self.copyfile(f, self.wfile)
finally:
f.close()
def do_HEAD(self):
"""Serve a HEAD request."""
f = self.send_head()
if f:
f.close()
def do_POST(self):
"""Serve a POST request. only specific to API."""
if self.path.rstrip().endswith('/api/setMapping'):
if not self.rfile.closed:
content_len = int(self.headers.getheader('content-length', 0))
post_body = self.rfile.read(content_len)
json_string = post_body
self.__class__.inform_post_listeners(json_string)
response_text = 'updated the Mapping'
self.send_response(200, 'responseMessage')
self.send_header('Content-Type', 'text/plain')
self.send_header('Content-Length', len(response_text))
self.end_headers()
self.wfile.write(response_text)
else:
self.send_error(403, 'no no')
self.wfile.close()
self.rfile.close()
def send_head(self):
"""Common code for GET and HEAD commands.
This sends the response code and MIME headers.
Return value is either a file object (which has to be copied
to the outputfile by the caller unless the command was HEAD,
and must be closed by the caller under all circumstances), or
None, in which case the caller has nothing further to do.
"""
path = self.translate_path(self.path)
f = None
if os.path.isdir(path):
parts = urlparse.urlsplit(self.path)
if not parts.path.endswith('/'):
# redirect browser - doing basically what apache does
self.send_response(301)
new_parts = (parts[0], parts[1], parts[2] + '/', parts[3],
parts[4])
new_url = urlparse.urlunsplit(new_parts)
self.send_header("Location", new_url)
self.end_headers()
return None
for index in "index.html", "index.htm":
index = os.path.join(path, index)
if os.path.exists(index):
path = index
break
else:
return self.list_directory(path)
ctype = self.guess_type(path)
try:
# Always read in binary mode. Opening files in text mode may cause
# newline translations, making the actual size of the content
# transmitted *less* than the content-length!
f = open(path, 'rb')
except IOError:
self.send_error(404, "File not found")
return None
try:
self.send_response(200)
self.send_header("Content-type", ctype)
fs = os.fstat(f.fileno())
self.send_header("Content-Length", str(fs[6]))
self.send_header("Last-Modified",
self.date_time_string(fs.st_mtime))
self.end_headers()
return f
except:
f.close()
raise
def list_directory(self, path):
"""Helper to produce a directory listing (absent index.html).
Return value is either a file object, or None (indicating an
error). In either case, the headers are sent, making the
interface the same as for send_head().
"""
try:
list = os.listdir(path)
except os.error:
self.send_error(404, "No permission to list directory")
return None
list.sort(key=lambda a: a.lower())
f = StringIO()
displaypath = cgi.escape(urllib.unquote(self.path))
f.write('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">')
f.write("<html>\n<title>Directory listing for %s</title>\n" %
displaypath)
f.write("<body>\n<h2>Directory listing for %s</h2>\n" % displaypath)
f.write("<hr>\n<ul>\n")
for name in list:
fullname = os.path.join(path, name)
displayname = linkname = name
# Append / for directories or @ for symbolic links
if os.path.isdir(fullname):
displayname = name + "/"
linkname = name + "/"
if os.path.islink(fullname):
displayname = name + "@"
# Note: a link to a directory displays with @ and links with /
f.write('<li><a href="%s">%s</a>\n' %
(urllib.quote(linkname), cgi.escape(displayname)))
f.write("</ul>\n<hr>\n</body>\n</html>\n")
length = f.tell()
f.seek(0)
self.send_response(200)
encoding = sys.getfilesystemencoding()
self.send_header("Content-type", "text/html; charset=%s" % encoding)
self.send_header("Content-Length", str(length))
self.end_headers()
return f
def translate_path(self, path):
"""Translate a /-separated PATH to the local filename syntax.
Components that mean special things to the local file system
(e.g. drive or directory names) are ignored. (XXX They should
probably be diagnosed.)
"""
# abandon query parameters
path = path.split('?', 1)[0]
path = path.split('#', 1)[0]
# Don't forget explicit trailing slash when normalizing. Issue17324
trailing_slash = path.rstrip().endswith('/')
path = posixpath.normpath(urllib.unquote(path))
words = path.split('/')
words = filter(None, words)
path = os.getcwd()
for word in words:
if os.path.dirname(word) or word in (os.curdir, os.pardir):
# Ignore components that are not a simple file/directory name
continue
path = os.path.join(path, word)
if trailing_slash:
path += '/'
return path
def copyfile(self, source, outputfile):
"""Copy all data between two file objects.
The SOURCE argument is a file object open for reading
(or anything with a read() method) and the DESTINATION
argument is a file object open for writing (or
anything with a write() method).
The only reason for overriding this would be to change
the block size or perhaps to replace newlines by CRLF
-- note however that this the default server uses this
to copy binary data as well.
"""
shutil.copyfileobj(source, outputfile)
def guess_type(self, path):
"""Guess the type of a file.
Argument is a PATH (a filename).
Return value is a string of the form type/subtype,
usable for a MIME Content-type header.
The default implementation looks the file's extension
up in the table self.extensions_map, using application/octet-stream
as a default; however it would be permissible (if
slow) to look inside the data to make a better guess.
"""
base, ext = posixpath.splitext(path)
if ext in self.extensions_map:
return self.extensions_map[ext]
ext = ext.lower()
if ext in self.extensions_map:
return self.extensions_map[ext]
else:
return self.extensions_map['']
if not mimetypes.inited:
mimetypes.init() # try to read system mime.types
extensions_map = mimetypes.types_map.copy()
extensions_map.update({
'': 'application/octet-stream', # Default
'.py': 'text/plain',
'.c': 'text/plain',
'.h': 'text/plain',
})
def test_put_get_duplicated_data_user_stage(is_public_test,
tmpdir,
conn_cnx,
db_parameters,
number_of_files=5,
number_of_lines=100):
"""[s3] Puts and Gets Duplicated Data using User Stage."""
if is_public_test or 'AWS_ACCESS_KEY_ID' not in os.environ:
pytest.skip('This test requires to change the internal parameter')
logger = getLogger(__name__)
assert 'AWS_ACCESS_KEY_ID' in os.environ, 'AWS_ACCESS_KEY_ID is missing'
assert 'AWS_SECRET_ACCESS_KEY' in os.environ, \
'AWS_SECRET_ACCESS_KEY is missing'
tmp_dir = generate_k_lines_of_n_files(number_of_lines,
number_of_files,
tmp_dir=str(tmpdir.mkdir('data')))
files = os.path.join(tmp_dir, 'file*')
stage_name = db_parameters['name'] + '_stage'
with conn_cnx(user=db_parameters['user'],
account=db_parameters['account'],
password=db_parameters['password']) as cnx:
cnx.cursor().execute("""
create or replace table {name} (
aa int,
dt date,
ts timestamp,
tsltz timestamp_ltz,
tsntz timestamp_ntz,
tstz timestamp_tz,
pct float,
ratio number(6,2))
""".format(name=db_parameters['name']))
user_bucket = os.getenv('SF_AWS_USER_BUCKET',
"sfc-dev1-regression/{}/reg".format(getuser()))
cnx.cursor().execute("""
create or replace stage {stage_name}
url='s3://{user_bucket}/{stage_name}-{number_of_files}-{number_of_lines}'
credentials=(
AWS_KEY_ID='{aws_key_id}'
AWS_SECRET_KEY='{aws_secret_key}'
)
""".format(stage_name=stage_name,
user_bucket=user_bucket,
aws_key_id=os.getenv('AWS_ACCESS_KEY_ID'),
aws_secret_key=os.getenv('AWS_SECRET_ACCESS_KEY'),
number_of_files=number_of_files,
number_of_lines=number_of_lines))
try:
with conn_cnx(user=db_parameters['user'],
account=db_parameters['account'],
password=db_parameters['password']) as cnx:
c = cnx.cursor()
try:
for rec in c.execute(
"rm @{stage_name}".format(stage_name=stage_name)):
logger.info('rec=%s', rec)
finally:
c.close()
success_cnt = 0
skipped_cnt = 0
c = cnx.cursor()
c.execute(
"alter session set disable_put_and_get_on_external_stage = false"
)
try:
for rec in c.execute("put file://{file} @{stage_name}".format(
file=files, stage_name=stage_name)):
logger.info('rec=%s', rec)
if rec[6] == 'UPLOADED':
success_cnt += 1
elif rec[6] == 'SKIPPED':
skipped_cnt += 1
finally:
c.close()
assert success_cnt == number_of_files, 'uploaded files'
assert skipped_cnt == 0, 'skipped files'
logger.info(
'deleting files in {stage_name}'.format(stage_name=stage_name))
deleted_cnt = 0
cnx.cursor().execute(
"rm @{stage_name}/file0".format(stage_name=stage_name))
deleted_cnt += 1
cnx.cursor().execute(
"rm @{stage_name}/file1".format(stage_name=stage_name))
deleted_cnt += 1
cnx.cursor().execute(
"rm @{stage_name}/file2".format(stage_name=stage_name))
deleted_cnt += 1
success_cnt = 0
skipped_cnt = 0
c = cnx.cursor()
try:
for rec in c.execute("put file://{file} @{stage_name}".format(
file=files, stage_name=stage_name),
_raise_put_get_error=False):
logger.info('rec=%s', rec)
if rec[6] == 'UPLOADED':
success_cnt += 1
elif rec[6] == 'SKIPPED':
skipped_cnt += 1
assert success_cnt == deleted_cnt, \
'uploaded files in the second time'
assert skipped_cnt == number_of_files - deleted_cnt, \
'skipped files in the second time'
finally:
c.close()
time.sleep(5)
cnx.cursor().execute("copy into {name} from @{stage_name}".format(
name=db_parameters['name'], stage_name=stage_name))
c = cnx.cursor()
try:
c.execute("select count(*) from {name}".format(
name=db_parameters['name']))
rows = 0
for rec in c:
rows += rec[0]
assert rows == number_of_files * number_of_lines, 'Number of rows'
finally:
c.close()
cnx.cursor().execute(
"rm @{stage_name}".format(stage_name=stage_name))
cnx.cursor().execute("copy into @{stage_name} from {name}".format(
name=db_parameters['name'], stage_name=stage_name))
tmp_dir_user = str(tmpdir.mkdir('stage2'))
cnx.cursor().execute(
"get @{stage_name}/ file://{tmp_dir_user}/".format(
stage_name=stage_name, tmp_dir_user=tmp_dir_user))
for _, _, files in os.walk(tmp_dir_user):
for file in files:
mimetypes.init()
_, encoding = mimetypes.guess_type(file)
assert encoding == 'gzip', "exported file type"
finally:
with conn_cnx(user=db_parameters['user'],
account=db_parameters['account'],
password=db_parameters['password']) as cnx:
cnx.cursor().execute("drop stage if exists {stage_name}".format(
stage_name=stage_name))
cnx.cursor().execute("drop table if exists {name}".format(
name=db_parameters['name']))
class MediaRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
server_version = "MediaTest/" + __version__
def do_GET(self):
if VERBOSE: print "GET", self.path
f = self.send_head()
if f:
self.copyfile(f, self.wfile)
f.close()
def do_HEAD(self):
f = self.send_head()
if f:
f.close()
def send_head(self):
path = self.translate_path(self.path)
f = None
if os.path.isdir(path): return None
ctype = self.guess_type(path)
try:
f = open(path, 'rb')
except IOError:
self.send_error(404, "File not found")
return None
self.send_response(200)
self.send_header("Content-type", ctype)
fs = os.fstat(f.fileno())
self.send_header("Content-Length", str(fs[6]))
self.send_header("Last-Modified", self.date_time_string(fs.st_mtime))
self.end_headers()
return f
def translate_path(self, path):
path = urlparse.urlparse(path)[2]
path = posixpath.normpath(urllib.unquote(path))
words = path.split('/')
words = filter(None, words)
path = os.getcwd()
for word in words:
drive, word = os.path.splitdrive(word)
head, word = os.path.split(word)
if word in (os.curdir, os.pardir): continue
path = os.path.join(path, word)
return path
def copyfile(self, source, outputfile):
global NEXT_TIME
start = time.time()
#print "START", start
total=0
elapsed = 0
while True:
chunk = source.read(4096)
#print "READ", len(chunk)
if len(chunk) == 0: break
now = time.time()
if NEXT_TIME == -1:
NEXT_TIME = now
#print 'NOW =',now, ' NEXT = ', NEXT_TIME
delay = 0
if now < NEXT_TIME:
delay = NEXT_TIME-now
NEXT_TIME += float(len(chunk)*8)/BANDWIDTH
if delay:
time.sleep(delay)
try:
outputfile.write(chunk)
except:
break
total += len(chunk)
now = time.time()
elapsed = now-start
rate = total/elapsed
#print "STAT", total, elapsed, rate
#print "END", time.time()
#print "ELAPSED", elapsed
print "BPS =", (8*total)/elapsed
def guess_type(self, path):
base, ext = posixpath.splitext(path)
if ext in self.extensions_map:
return self.extensions_map[ext]
ext = ext.lower()
if ext in self.extensions_map:
return self.extensions_map[ext]
else:
return self.extensions_map['']
if not mimetypes.inited:
mimetypes.init() # try to read system mime.types
extensions_map = mimetypes.types_map.copy()
extensions_map.update({
'': 'application/octet-stream', # Default
'.py': 'text/plain',
'.c': 'text/plain',
'.h': 'text/plain',
'.mp4': 'video/mp4',
'.mp3': 'audio/mpeg'
})
