def loadURLFast(url=None, cache_dir=CACHE_DIR, aged=AGED, timeout=TIMEOUT):
req_dig = request_digest(url)
req_dig_key = cache_dir + req_dig
req_dig_md_key = file_storage.metadata(req_dig_key)
aged_in_cache = False
in_cache = False
now = datetime.datetime.now()
try:
md = file_storage.getItem(req_dig_md_key)
in_cache = True
if aged is not None and \
now - md["datetime"] > datetime.timedelta(seconds=aged):
aged_in_cache = True
except:
pass
if in_cache and (config.offline_mode or not aged_in_cache):
try:
try:
headers = dict(file_storage.getItem(req_dig_md_key)["headers"])
except:
headers = {'content-type': "application/octet-stream"}
return headers, file_storage.getItem(req_dig_key, mode="file")
except:
file_storage.delItem(req_dig_md_key)
in_cache = False
headers, fh = doLoadURLFast(url, cache_dir=cache_dir, aged=aged, timeout=timeout)
try:
headers = dict(file_storage.getItem(req_dig_md_key)["headers"])
except:
headers = {'content-type': "application/octet-stream"}
return headers, fh
def downloadPreview(file_key, max_x=100, max_y=100):
if Image is None:
return
try:
metadata = file_storage.getItem(file_storage.metadata(file_key))
input_file = file_storage.getItem(file_key, mode="file")
output_file = tempfile.TemporaryFile()
ratio = 1
angle = 0
ct = metadata['Content-Type']
pre_ct = "image/jpeg"
if ct.startswith("image/"):
try:
_preview(input_file, output_file, max_x, max_y, ratio, angle)
except:
pre_ct, output_file = preview_icon(ct)
else:
pre_ct, output_file = preview_icon(ct)
last_modified = metadata["Last-Modified"]
except:
pre_ct, output_file = preview_icon('broken')
last_modified = rfc822.formatdate(time.time()) #!!!?
output_file.seek(0L, 2)
preview_length = output_file.tell()
output_file.seek(0L)
fc = output_file.read()
web.header('Content-Type', pre_ct)
web.header('Content-Length', str(preview_length))
web.header('Last-Modified', last_modified)
return fc
def download_captcha(file_key):
if not goodkey_re.match(file_key):
return
metadata = file_storage.getItem(file_storage.metadata(file_key))
web.header('Content-Type', metadata["Content-Type"])
web.header('Content-Length', str(metadata["Content-Length"]))
web.header('Last-Modified', metadata["Last-Modified"])
return file_storage.getItem(file_key, mode="str")
def downloadMediaFile(file_key):
try:
metadata = file_storage.getItem(file_storage.metadata(file_key))
web.header('Content-Type', metadata["Content-Type"])
web.header('Content-Length', str(metadata["Content-Length"]))
web.header('Last-Modified', metadata["Last-Modified"])
return file_storage.getItem(file_key, mode="str")
except KeyError:
return downloadPreview(file_key) #??? !!!
def doLoadURLFast(url, cache_dir=CACHE_DIR, aged=AGED, timeout=TIMEOUT):
headers, fh = _doRead(url)
req_dig_key = cache_dir + request_digest(url)
req_dig_md_key = file_storage.metadata(req_dig_key)
file_storage.setItem(req_dig_key, fh)
file_storage.setItem(req_dig_md_key, {"datetime": datetime.datetime.now(),
"headers": headers})
return headers, file_storage.getItem(req_dig_key, mode="file")
def loadParsedURL(url):
req_dig_key = CACHE_DIR + request_digest(url)
req_dig_md_key = file_storage.metadata(req_dig_key)
try:
md = file_storage.getItem(req_dig_md_key)
if datetime.datetime.now() - md["datetime"] \
<= datetime.timedelta(seconds=AGED):
return md["parsed"]
except StandardError: # misc errors
pass
return None
def needToReloadURL(url, cache_dir=CACHE_DIR, aged=AGED):
req_dig_key = cache_dir + request_digest(url)
req_dig_md_key = file_storage.metadata(req_dig_key)
try:
md = file_storage.getItem(req_dig_md_key)
if datetime.datetime.now() - md["datetime"] \
<= datetime.timedelta(seconds=aged):
return False
except StandardError: # misc errors
pass
return True
def check_captcha(file_key, user_word):
if not goodkey_re.match(file_key):
return "Invalid key format"
try:
metadata = file_storage.getItem(file_storage.metadata(file_key))
except:
return "No such thing"
try:
if time.time() - metadata["added"] > 180: # captcha is too old
return "Too old"
if user_word.strip() == metadata["secret"]:
return ""
finally:
drop_captcha(file_key)
return "Failed"
def storeParsedURL(url, parsed_data):
req_dig_key = CACHE_DIR + request_digest(url)
req_dig_md_key = file_storage.metadata(req_dig_key)
try:
file_storage.setItem(req_dig_md_key,
{"datetime": datetime.datetime.now(),
"parsed": parsed_data}) #!!! is pickle fails?
return True
except:
try:
file_storage.setItem(req_dig_md_key,
{"datetime": datetime.datetime.now(),}
)
except:
pass
return False
def _upload_captcha(file_contents, secret_word, filename="NotKnown.jpg"):
file_key = CAPTCHA_PREFIX + md5.new(file_contents).hexdigest()
file_storage.setItem(file_key, file_contents)
content_type = mimetypes.guess_type(filename)[0]
# mostly technical metadata used to show file quickly
file_storage.setItem(
file_storage.metadata(file_key),
{
'Owner': 'NotimplementedYet',
'secret': secret_word,
'added': time.time(),
'OriginalFilename': filename, #!!!
'Last-Modified': rfc822.formatdate(time.time()),
'Content-Type': content_type, #!!!
'Content-Length': len(file_contents),
})
return file_key
def uploadMediaFile(file_contents="",
filename="NotKnown.jpg",
content_type=None):
if file_contents:
file_key = storeMediaFile(file_contents)
content_type = content_type or guess_content_type(filename)
# mostly technical metadata used to show file quickly
file_storage.setItem(file_storage.metadata(file_key),
{
'Owner': 'NotimplementedYet',
'OriginalFilename': filename, #!!!
'Last-Modified': rfc822.formatdate(time.time()),
'Content-Type': content_type, #!!!
'Content-Length': str(len(file_contents)),
}
)
return "/media?file_key=" + file_key
# print "FILE SUCCESSFULLY UPLOADED", file_key, len(file_contents)
else:
return ""
def getHeaders(url, cache_dir=CACHE_DIR):
req_dig_key = cache_dir + request_digest(url)
req_dig_md_key = file_storage.metadata(req_dig_key)
md = file_storage.getItem(req_dig_md_key)
return md["headers"]