def request(url, post=None, headers=None, mobile=False, safe=False, timeout='30'):
try:
try: headers.update(headers)
except: headers = {}
agent = cache.get(cloudflareAgent, 168)
if not 'User-Agent' in headers: headers['User-Agent'] = agent
u = '%s://%s' % (urlparse.urlparse(url).scheme, urlparse.urlparse(url).netloc)
cookie = cache.get(cloudflareCookie, 168, u, post, headers, mobile, safe, timeout)
result = client.request(url, cookie=cookie, post=post, headers=headers, mobile=mobile, safe=safe, timeout=timeout, output='response', error=True)
if result[0] == '503':
agent = cache.get(cloudflareAgent, 0) ; headers['User-Agent'] = agent
cookie = cache.get(cloudflareCookie, 0, u, post, headers, mobile, safe, timeout)
result = client.request(url, cookie=cookie, post=post, headers=headers, mobile=mobile, safe=safe, timeout=timeout)
else:
result= result[1]
return result
except:
return
def test_cache_set_expire(mock_cache):
redis, clock = mock_cache
cache.set("foo", "bar", expires=60)
assert cache.get("foo") == "bar"
clock.set_time(datetime.datetime.now() + datetime.timedelta(seconds=61))
redis.do_expire()
assert cache.get("foo") == None
def get(title, year, imdb, tvdb, season, episode, show, date, genre):
try:
redirect = False
if len(season) > 3: redirect = True
genre = [i.strip() for i in genre.split('/')]
genre = [i for i in genre if any(x == i for x in ['Reality', 'Game Show', 'Talk Show'])]
if not len(genre) == 0: redirect = True
blocks = ['73141']
if tvdb in blocks: redirect = True
if redirect == False: raise Exception()
except:
return (season, episode)
try:
tvrage = cache.get(getTVrageId, 8640, imdb, tvdb, show, year)
if tvrage == None: raise Exception()
except:
return (season, episode)
try:
result = cache.get(getTVrageEpisode, 8640, tvrage, title, date, season, episode)
if result == None: raise Exception()
return (result[0], result[1])
except:
return (season, episode)
def geoTimeQuery(comm=None, level=None, host=settings.IMPALA[0], port=settings.IMPALA[1], geo=None, time=None):
level = cache.get().get("graph_num_levels","")
nodetable = cache.get().get("table","") + '_good_nodes'
edgestable = cache.get().get("table","") + '_good_graph'
trackstable = cache.get().get("table","") + '_tracks_comms_joined'
query = 'select distinct comm_' + str(level) + ' from ' + trackstable + ' where '
geoThere = False
if geo["min_lat"] != None:
locationquery = ' cast(intersectx as double) >= ' + geo["min_lat"].replace('"','') + ' and cast(intersectx as double) <= ' + geo["max_lat"].replace('"','') + ' and cast(intersecty as double) >= ' + geo["min_lon"].replace('"','') + ' and cast(intersecty as double) <= ' + geo["max_lon"].replace('"','')
query = query + locationquery
geoThere = True
if time["min_time"] != None:
timequery = ' dt >= ' + time["min_time"] + ' and dt <= ' + time["max_time"]
if geoThere:
query = query + ' and '
query = query + timequery
#print query
nodequery = 'select node, comm, num_members, level from ' + nodetable + ' where level = "' + str(level) + '" and comm in '
edgequery = 'select source, target, weight, level from ' + edgestable + ' where level = "' + str(level) + '" '
#print nodequery
#print edgequery
with impalaopen(host + ':' + port) as client:
qResults = client.execute(query)
comm_string = '( '
for record in qResults.data:
comm_string = comm_string + '"' + record.strip() + '", '
comm_string = comm_string[0:len(comm_string)-2] + ')'
nodequery = nodequery + comm_string
edgequery = edgequery + ' and (source_comm in ' + comm_string + ' and target_comm in ' + comm_string + ' )'
with impalaopen(host + ':' + port) as client:
qResults = client.execute(nodequery)
mapping = {}
idx = 0
for record in qResults.data:
node,comm,num_members,level = record.split('\t')
mapping[node] = {"index":idx,"nodename":node,"node_comm":comm,"level":level,"num_members":num_members}
idx = idx + 1
edges = []
nodes = []
with impalaopen(host + ':' + port) as client:
qResults = client.execute(edgequery)
for record in qResults.data:
source,target,weight,level = record.split('\t')
edges.append({"source":mapping[source]["index"],"sourcename":source,"target":mapping[target]["index"],"targetname":target,"weight":weight})
for i in mapping.keys():
nodes.append({"index":mapping[i]["index"],"nodename":mapping[i]["nodename"],"node_comm":mapping[i]["node_comm"],"level":mapping[i]["level"],"num_members":mapping[i]["num_members"]})
response = {}
response["gephinodes"] = nodes
response["gephigraph"] = edges
return response
def test_set_multi_expires(mock_cache):
redis, clock = mock_cache
cache.set_multi({"foo1": "bar1", "foo2": "bar2"}, expires=60)
assert cache.get("foo1") == "bar1"
assert cache.get("foo2") == "bar2"
clock.set_time(datetime.datetime.now() + datetime.timedelta(seconds=61))
redis.do_expire()
assert cache.get("foo1") == None
assert cache.get("foo2") == None
def request(url, timeout='30'):
try:
u = '%s://%s' % (urlparse.urlparse(url).scheme, urlparse.urlparse(url).netloc)
cookie = cache.get(sucuri, 168, u, timeout)
if cookie == None:
cookie = cache.get(sucuri, 0, u, timeout)
result = client.request(url, cookie=cookie, timeout=timeout)
return result
except:
return
def removeDownload(url):
try:
def download(): return []
result = cache.get(download, 600000000, table='rel_dl')
if result == '': result = []
result = [i for i in result if not i['url'] == url]
if result == []: result = ''
def download(): return result
result = cache.get(download, 0, table='rel_dl')
control.refresh()
except:
control.infoDialog('You need to remove file manually', 'Can not remove from Queue')
def main(datestamp, sleep_interval = 1):
'Download given a datestamp.'
# Set the directories.
index_dir = path('downloads', 'index', datestamp)
image_dir = path('downloads', 'images')
# Download the index.
index = get('http://www.postsecret.com/', cachedir = index_dir)
# Download the new images.
html = fromstring(index.read())
srcs = html.xpath('//img/@src')
for src in srcs:
get(src, cachedir = image_dir)
sleep(sleep_interval)
def request(url, post=None, headers=None, mobile=False, safe=False, timeout='60'):
try:
u = '%s://%s' % (urlparse.urlparse(url).scheme, urlparse.urlparse(url).netloc)
cookie = cache.get(cloudflare, 3, u, post, headers, mobile, safe, timeout)
result = client.request(url, cookie=cookie, post=post, headers=headers, mobile=mobile, safe=safe, timeout=timeout, output='response', error=True)
if 'HTTP Error 503' in result[0]:
cookie = cache.get(cloudflare, 0, u, post, headers, mobile, safe, timeout)
result = client.request(url, cookie=cookie, post=post, headers=headers, mobile=mobile, safe=safe, timeout=timeout)
else:
result= result[1]
return result
except:
return
def fetch_webpage_text(url,use_cache=True): if use_cache and cache.contains(url): return cache.get(url) # cache miss, download it content = requests.get(url).text cache.put(url,content) return content
def variable_init(conf_={}):
"""
Load the persistent variable table.
The variable table is composed of values that have been saved in the table
with variable_set() as well as those explicitly specified
in the configuration file.
"""
# NOTE: caching the variables improves performance by 20% when serving
# cached pages.
cached = lib_cache.get("variables", "cache")
if cached:
variables = cached.data
else:
variables = {}
result = lib_database.query("SELECT * FROM {variable}")
while True:
variable = lib_database.fetch_object(result)
if not variable:
break
variables[variable.name] = php.unserialize(variable.value)
lib_cache.set("variables", variables)
for name, value in conf_.items():
variables[name] = value
return variables
def post(self):
tx = self.request.get_range('x', None)
ty = self.request.get_range('y', None)
z = self.request.get_range('z', None)
limit = self.request.get_range('limit', min_value=1, max_value=1000, default=1000)
offset = self.request.get_range('offset', min_value=0, default=0)
name = self.request.get('name', None)
source_name = self.request.get('source', None)
if tx is None or ty is None or z is None or name is None or source_name is None:
self.error(400)
return
key = 'tile-%s-%s-%s-%s-%s' % (z, ty, tx, source_name, name)
png = cache.get(key)
if png is None:
png = get_tile_png(tx, ty, z, name, source_name, limit, offset)
if png is None:
png = PointTile.blank()
cache.add(key, png, dumps=False)
logging.info('TILE BACKEND MEMORY = %s' % runtime.memory_usage().current())
self.response.set_status(200)
self.response.headers['Content-Type'] = 'image/png'
self.response.out.write(png)
def getCurrentHeatMap(*args):
table = cache.get().get("table");
query = "select x, y, log10(sum(value)+1) as sv from " + database + ".micro_path_intersect_counts_%s group by x, y" % table
with impalaopen(":".join(settings.IMPALA)) as curr:
curr.execute(query);
fmt = lambda row : "{ location: new google.maps.LatLng(%s, %s), weight: %s }" % row
return "[" + ",".join([fmt(row) for row in curr]) + "]"
def LinksPage(url, iconimage, description):
descriptions, links, seasons = cache.get(Links_regex, 72, url, table="pages")
if len(descriptions) == 1:
description = descriptions[0]
if seasons is None:
addDir('[COLOR red] למנויים באתר, לא דרך קודי. [/COLOR]','99',99,'',False, description)
return
elif seasons:
series_num = url.split('-')[-1]
GetSeasons(series_num, iconimage, description)
else:
if len(links) < 1:
addDir('[COLOR red] לא נמצאו מקורות ניגון [/COLOR]','99',99,'',False, description)
return
elif len(links) > 1:
links = SortByQuality(links)
playingUrlsList = []
for link in links:
playingUrlsList.append(link[2])
addDir('[COLOR red] בחר בניגון אוטומטי [/COLOR]','99',99,'',False, description)
addDir('{0} - ניגון אוטומטי'.format(name), json.dumps(playingUrlsList), 7, iconimage, False, description)
addDir('[COLOR red] או בחר מקור לניגון, אם לא עובד נסה אחר [/COLOR]','99',99,'',False, description)
for link in links:
addDir("{0} - {1} - איכות {2}".format(name, link[0], link[1]),link[2],5,iconimage,False, description)
def gotprices(self, request):
try:
price = tornado.escape.json_decode(request.body)
except:
self.returnerror("No trip price date receaved")
return None
price = price["data"]
trips = self.trips["data"]["rows"]
for i in range(0, len(trips)):
for j in range(0, len(price)):
if price[j]["journeyId"] == trips[i]["id"]:
trips[i]["pricedata"] = price[j]
trips[i]["departureDate"] = self.getdate
trips[i]["departureLocation"] = self.getfrom
trips[i]["arrivalLocation"] = self.getto
data = Empty()
data.getdate = self.getdate
data.getfrom = self.getfrom
data.gettime = trips[i]["departureTime"]
data.getto = self.getto
data.gettotime = trips[i]["arrivalTime"]
cache.store("sj", data, trips[i])
break
try:
self.returnrequest(cache.get("sj", self))
except:
self.returnerror("Trip not found in search")
def fetch_webpage(url, use_cache=True):
key = cache.md5_key(url)
if use_cache and cache.contains(key):
return cache.get(key)
content = requests.get(url).text
cache.put(key,content)
return content
def request(url, timeout='30'):
try:
u = '%s://%s' % (urlparse.urlparse(url).scheme, urlparse.urlparse(url).netloc)
cookie = cache.get(cloudflare, 168, u, timeout)
result = client.request(url, cookie=cookie, timeout=timeout, output='response', error=True)
if 'HTTP Error 503' in result[0]:
cookie = cache.get(cloudflare, 0, u, timeout)
result = client.request(url, cookie=cookie, timeout=timeout)
else:
result= result[1]
return result
except:
return
def post(self):
# Check parameters
name = self.request.get('name', None)
if not name:
self.error(400)
return
source_name = self.request.get('source', None)
# Check cache
key = self.cache_key(name, source_name)
names = cache.get(key)
if names:
self.response.set_status(200)
self.response.headers['Content-Type'] = "application/json"
self.response.out.write(names)
return
source = sources.get(source_name)
if not source:
# TODO: Get names from all sources?
self.error(404)
return
# Make service request for names
names, status_code = self.get_names(source, name)
if status_code != 200:
self.error(status_code)
return
# Update cache and send response
cache.add(key, names)
self.response.set_status(200)
self.response.headers['Content-Type'] = "application/json"
self.response.out.write(simplejson.dumps(names))
def get_and_cache_owner(self):
cache_key = 'TickUser(%s)' % self.get_owner_key()
owner = cache.get(cache_key)
if not owner:
owner = self.owner
cache.set(cache_key,owner)
return owner
def request(url, mobile=False, timeout="30"):
try:
u = "%s://%s" % (urlparse.urlparse(url).scheme, urlparse.urlparse(url).netloc)
cookie = cache.get(cloudflare, 168, u, mobile, timeout)
result = client.request(url, cookie=cookie, mobile=mobile, timeout=timeout, output="response", error=True)
if "HTTP Error 503" in result[0]:
cookie = cache.get(cloudflare, 0, u, mobile, timeout)
result = client.request(url, cookie=cookie, mobile=mobile, timeout=timeout)
else:
result = result[1]
return result
except:
return
def query(type=QUERY_ANIME, aid=None, **kwargs):
"""
Query AniDB for information about the anime identified by *aid* or the
complete list of categories.
:param type: Either QUERY_CATEGORIES or QUERY_ANIME
:param aid: If *type* is QUERY_ANIME, the aid of the anime
:param kwargs: Any kwargs you want to pass to :func:`requests.get`
:raises: ValueError if `anidb.CLIENT` or `anidb.CLIENTVERSION` are not set
:rtype: :class:`anidb.model.Anime` or a list of
:class:`anidb.model.Category`
"""
if CLIENT is None or CLIENTVERSION is None:
raise ValueError(
"You need to assign values to both CLIENT and CLIENTVERSION")
if type == QUERY_ANIME:
if aid is None:
raise TypeError("aid can't be None")
else:
cacheresult = cache.get(aid)
if cacheresult is not None:
return cacheresult
#print ANIDB_URL % (CLIENT, CLIENTVERSION, "anime") + "&aid=%i" % aid
response = \
requests.get(ANIDB_URL % (CLIENT, CLIENTVERSION, "anime")
+ "&aid=%i" % aid, **kwargs)
result =_handle_response(response.content)
cache.save(aid, result)
return result
elif type == QUERY_CATEGORIES:
response = requests.get(ANIDB_URL % (CLIENT, CLIENTVERSION,
"categorylist"), **kwargs)
return _handle_response(response.content)
def get_lat_long(dbo, address, town, county, postcode, country = None):
"""
Looks up a latitude and longitude from an address using GEOCODE_URL
and returns them as lat,long,(first 3 chars of address)
Returns None if no results were found.
NB: dbo is only used for contextual reference in logging, no database
calls are made by any of this code.
"""
if address.strip() == "":
return None
try:
# Synchronise this process to a single thread to prevent
# abusing our geo provider and concurrent requests for the
# same address when opening an animal with the same
# original/brought in by owner, etc.
lat_long_lock.acquire()
url = ""
if country is None:
country = LOCALE_COUNTRY_NAME_MAP[dbo.locale]
if BULK_GEO_PROVIDER == "cloudmade":
q = normalise_cloudmade(address, town, county, postcode, country)
url = CLOUDMADE_URL.replace("{key}", BULK_GEO_PROVIDER_KEY).replace("{q}", q)
elif BULK_GEO_PROVIDER == "nominatim":
q = normalise_nominatim(address, town, county, postcode, country)
url = NOMINATIM_URL.replace("{q}", q)
else:
al.error("unrecognised geo provider: %s" % BULK_GEO_PROVIDER, "geo.get_lat_long", dbo)
al.debug("looking up geocode for address: %s" % q, "geo.get_lat_long", dbo)
key = "nom:" + q
if cache.available():
v = cache.get(key)
if v is not None:
al.debug("cache hit for address: %s = %s" % (q, v), "geo.get_lat_long", dbo)
return v
jr = urllib2.urlopen(url, timeout = GEO_LOOKUP_TIMEOUT).read()
j = json.loads(jr)
latlon = None
if BULK_GEO_PROVIDER == "cloudmade":
latlon = parse_cloudmade(dbo, jr, j, q)
elif BULK_GEO_PROVIDER == "nominatim":
latlon = parse_nominatim(dbo, jr, j, q)
# Cache this address/geocode response for an hour
if cache.available() and latlon is not None:
cache.put(key, latlon, 3600)
return latlon
except Exception,err:
al.error(str(err), "geo.get_lat_long", dbo)
return None
def getCurrentHeatMap(*args):
table = cache.get().get("table");
query = "select x, y, sum(value) from micro_path_intersect_counts_%s group by x, y" % table
with impalaopen(":".join(settings.IMPALA)) as client:
results = client.execute(query);
data = results.get_data()
fmt = lambda row : "{ location: new google.maps.LatLng(%s, %s), weight: %s }" % tuple(row.split("\t"))
return "[" + ",".join([fmt(row) for row in data.split("\n")]) + "]"
def fetch_webpage_text(url, use_cache=True):
if use_cache and cache.contains(url):
return cache.get(url)
# if cache miss, download it and sleep one second to prevent too-frequent calls
content = requests.get(url).text
cache.put(url,content)
time.sleep(1)
return content
def get_cached_response(cache_key):
"""
Gets a service call response from the cache based on its key.
If no entry is found, None is returned.
"""
if not CACHE_SERVICE_RESPONSES:
return None
return cache.get(cache_key)
def review(batch, remote, debug):
if (not remote):
logging.info("cannot review locally")
return
jobs = cache.get("batch/%s/jobs" % batch, remote)
review_jobs("train", jobs['train'])
review_jobs("validate", jobs['validate'])
review_jobs("test", jobs['test'])
review_jobs("report", [jobs['report']])
def request(url, post=None, headers=None, mobile=False, safe=False, timeout='60'):
try:
if headers is None:
headers = {common.Decode('ouLb26Vv1Mq74w=='): common.Decode('uN7a0qWb3Nu25w==')}
else:
headers[common.Decode('ouLb26Vv1Mq74w==')] = common.Decode('uN7a0qWb3Nu25w==')
u = '%s://%s' % (urlparse.urlparse(url).scheme, urlparse.urlparse(url).netloc)
cookie = cache.get(cloudflare, 3, u, post, {common.Decode('ouLb26Vv1Mq74w=='): common.Decode('uN7a0qWb3Nu25w==')}, mobile, safe, timeout, table='cookies')
result = client.request(url, cookie=cookie, post=post, headers=headers, mobile=mobile, safe=safe, timeout=timeout, output='response', error=True)
if 'HTTP Error 503' in result[0]:
cookie = cache.get(cloudflare, 0, u, post, {common.Decode('ouLb26Vv1Mq74w=='): common.Decode('uN7a0qWb3Nu25w==')}, mobile, safe, timeout, table='cookies')
result = client.request(url, cookie=cookie, post=post, headers=headers, mobile=mobile, safe=safe, timeout=timeout)
else:
result= result[1]
return result
except:
return
def report(batch, remote, debug, dependency = []):
params = cache.get("batch/%s/params" % batch, remote)
logging.info("running reporter instance")
if (remote):
k = cloud.call(reporter.report, batch, params, remote, debug, _label = "%s/report" % batch, _depends_on = dependency, _type = 'c1', _max_runtime = 30)
logging.info("k %s" % k)
return k
else:
result = reporter.report(batch, params, remote, debug)
return result
def interpret_batches(study, portfolio, batchList, remote):
search = cache.get("batch/%s-%s/search" % (study, portfolio), remote)
batchName_ = search['batch_']
if (batchList == "base"):
return [batchName_[0]]
elif (batchList == "*"):
return batchName_
else:
batchNum_ = util.parse_number_list(batchList)
return [batchName_[batchNum + 1] for batchNum in batchNum_]
def dump_key(search, batch, remote, key, xpath, showSearchValues):
elem = util.xpath_elem(cache.get("batch/%s/%s" % (batch, key), remote), xpath)
outStr = ""
if (showSearchValues):
i = search['batch_'].index(batch)
value_ = search['value__'][i]
outStr += batch + "," + ",".join(map(str, value_)) + ","
else:
outStr += batch + "\n"
outStr += str(elem) if xpath == "excel" else pp.pformat(elem)
return outStr
def countGitHubIssues(url):
try:
import client
import cache
import dom_parser2
c = cache.get(client.request, 1, url)
r = dom_parser2.parse_dom(
c, 'div', {
'class':
re.compile(
'table-list-header-toggle\sstates\sfloat-left\spl-\d+')
})
r = dom_parser2.parse_dom(r, 'a')
r = [
re.sub('<.+?>', '', i.content).replace('\n', '').lstrip()
for i in r
]
return 'Issues: %s - %s' % (r[0], r[1])
except:
return 'Issues: ? Open - ? Closed'
def get_current_theme(shop):
"""
Get the currently active theme object.
:param shop: The shop to get the active theme
:type shop: E-Commerce.core.models.Shop
:return: Theme object or None
:rtype: Theme
"""
value = cache.get(get_theme_cache_key(shop))
if value:
set_middleware_current_theme(value)
return value
theme = _get_current_theme(shop)
cache.set(get_theme_cache_key(shop), theme)
# set this theme as the current for this thread
set_middleware_current_theme(theme)
return theme
def get_best_selling_product_info(shop_ids, cutoff_days=30):
shop_ids = sorted(map(int, shop_ids))
cutoff_date = datetime.date.today() - datetime.timedelta(days=cutoff_days)
cache_key = "best_sellers:%r_%s" % (shop_ids, cutoff_date)
sales_data = cache.get(cache_key)
if sales_data is None:
sales_data = (
OrderLine.objects
.filter(
order__shop_id__in=shop_ids,
order__order_date__gte=to_aware(cutoff_date),
type=OrderLineType.PRODUCT
)
.values("product")
.annotate(n=Sum("quantity"))
.order_by("-n")[:100]
.values_list("product", "product__variation_parent_id", "n")
)
cache.set(cache_key, sales_data, 3 * 60 * 60) # three hours
return sales_data
def implements(hook_, sort=False, refresh=False):
"""
Determine which plugins are implementing a hook.
@param hook
The name of the hook (e.g. "help" or "menu").
@param sort
By default, plugins are ordered by weight and filename,
settings this option
to True, plugin list will be ordered by plugin name.
@param refresh
For internal use only: Whether to force the stored list of hook
implementations to be regenerated (such as after enabling a new plugin,
before processing hook_enable).
@return
An array with the names of the plugins which are implementing this hook.
"""
php.static(implements, 'implementations', {})
if (refresh):
implements.implementations = {}
elif (not lib_bootstrap.MAINTENANCE_MODE and \
php.empty(implements.implementations)):
cache = lib_cache.get('hooks', 'cache_registry')
if (cache):
implements.implementations = cache.data
implements.implementations = \
lib_bootstrap.registry_get_hook_implementations_cache()
if (not php.isset(implements.implementations, hook)):
implements.implementations[hook_] = []
for plugin_ in list_():
if (hook(plugin_, hook_)):
implements.implementations[hook_].append(plugin_)
lib_bootstrap.registry_cache_hook_implementations({'hook' : hook_, \
'plugins' : implements.implementations[hook_]})
# The explicit cast forces a copy to be made. This is needed because
# implementations[hook] is only a reference to an element of
# implementations and if there are nested foreaches (due to nested node
# API calls, for example), they would both manipulate the same array's
# references, which causes some plugins' hooks not to be called.
# See also http://www.zend.com/zend/art/ref-count.php.
return implements.implementations[hook_]
def settable(*args):
if args:
cache.update({"table": args[0]})
with impalaopen(":".join(settings.IMPALA)) as curr:
query = "select level, count(distinct source) from " + database + "." + args[
0] + "_good_graph group by level order by level desc limit 50;"
curr.execute(query)
graph_stat_string = ""
num_levels = 2
i = 0
for line in curr:
(level, nodes) = line
if i == 0:
num_levels = int(level)
i = i + 1
graph_stat_string = graph_stat_string + "Level: " + str(
level) + ", " + str(nodes) + " nodes "
query = "select min(dt), max(dt), min(cast(intersectx as double)), max(cast(intersectx as double)), min(cast(intersecty as double)), max(cast(intersecty as double)) from " + cache.get(
).get("database", "") + "." + cache.get().get(
"table", ""
) + "_tracks_comms_joined where track_id != 'ship(1.0)' and track_id != 'ais(3.0)'"
curr.execute(query)
for line in curr:
(mindt, maxdt, minlat, maxlat, minlon, maxlon) = line
cache.update({
"mindt": mindt,
"maxdt": maxdt,
"minlat": minlat,
"minlon": minlon,
"maxlat": maxlat,
"maxlon": maxlon
})
cache.update({
"graph_stat_string": graph_stat_string + "",
"graph_num_levels": num_levels,
"level": str(num_levels),
"community": '-'
})
return "0"
def query_cache(dbo, sql, age=60):
"""
Runs the query given and caches the result
for age seconds. If there's already a valid cached
entry for the query, returns the cached result
instead.
If CACHE_COMMON_QUERIES is set to false, just runs the query
without doing any caching and is equivalent to db.query()
"""
if not CACHE_COMMON_QUERIES or not cache.available():
return query(dbo, sql)
cache_key = "%s:%s:%s" % (dbo.alias, dbo.database, sql.replace(" ", "_"))
m = hashlib.md5()
m.update(cache_key)
cache_key = "q:%s" % m.hexdigest()
results = cache.get(cache_key)
if results is not None:
return results
results = query(dbo, sql)
cache.put(cache_key, results, age)
return results
def get_data(url):
global all_new_data
time_to_save=int(Addon.getSetting("c_save_time"))
#all_new_data=get_s_data(response,html_g,dp,start_time)
all_new_data=get_s_data(url)
all_new_data=cache.get(get_s_data,time_to_save,url, table='posters')
all_new_data=sorted(all_new_data, key=lambda x: x[10], reverse=False)
for new_name,icon,fan,plot,year,original_name,id,rating,genere,trailer,xxx,mode in all_new_data:
if icon=='' and fan=='':
addNolink(new_name,'www',199,False,iconimage=domain_s+'pbs.twimg.com/profile_images/421736697647218688/epigBm2J.jpeg',fanart='http://www.dream-wallpaper.com/free-wallpaper/cartoon-wallpaper/spawn-wallpaper/1280x1024/free-wallpaper-24.jpg')
else:
addDir3(new_name,'www',mode,icon,fan,plot,data=year,original_title=original_name,id=id,rating=rating,heb_name=new_name,show_original_year=year,isr=' ',generes=genere,trailer=trailer)
def page_get_cache():
"""
Retrieve the current page from the cache.
Note: we do not serve cached pages when status messages are waiting (from
a redirected form submission which was completed).
@param status_only
When set to TRUE, retrieve the status of the page cache only
(whether it was started in this request or not).
"""
cache = None
if (not lib_appglobals.user.uid and \
(php.SERVER['REQUEST_METHOD'] == 'GET' or \
php.SERVER['REQUEST_METHOD'] == 'HEAD') and \
php.count(drupal_set_message()) == 0):
cache = lib_cache.get(lib_appglobals.base_root + request_uri(), \
'cache_page')
if (php.empty(cache)):
ob_start()
return cache
def request(self, endpoint, query = None):
try:
# Encode the queries, if there is any...
if (query != None):
query = '?' + urllib.urlencode(query)
else:
query = ''
# Make the request
request = self.api_url % (endpoint, query)
# Send the request and get the response
# Get the results from cache if available
response = cache.get(client.request, 24, request)
# Retrun the result as a dictionary
return json.loads(response)
except:
pass
return {}
def test_get_items_from_dict_context():
customer = factories.create_random_person()
new_customer = factories.create_random_person()
contact_group = factories.create_random_contact_group()
contact_group.members.add(customer)
context = {
"customer": customer
}
items = _get_items_from_context(context)
groups = context_cache._get_val(customer.groups.all())
assert items["customer_groups"] == groups
assert "customer" not in items
# check whether items were cached
assert cache.get("_ctx_cache:customer_%d" % customer.pk) == groups
get_val_mock = mock.Mock(wraps=context_cache._get_val)
with mock.patch.object(context_cache, "_get_val", new=get_val_mock):
# get items again from the context, it shouldn't invoke _gel_val again for the customer
get_val_mock.assert_not_called()
items = _get_items_from_context(context)
get_val_mock.assert_not_called()
# check whether cache is bumped after changing contact
get_val_mock = mock.Mock(wraps=context_cache._get_val)
with mock.patch.object(context_cache, "_get_val", new=get_val_mock):
customer.save()
items = _get_items_from_context(context)
get_val_mock.assert_called()
# check whether cache is bumped after changing members of contact group
get_val_mock = mock.Mock(wraps=context_cache._get_val)
with mock.patch.object(context_cache, "_get_val", new=get_val_mock):
items = _get_items_from_context(context)
get_val_mock.assert_not_called()
contact_group.members.add(new_customer)
items = _get_items_from_context(context)
get_val_mock.assert_called()
def check(scraper):
try:
disable_check = xbmcaddon.Addon('plugin.video.xxx-o-dus').getSetting(
'dev_scrapers')
if (not disable_check == 'true'):
scraperFile = xbmc.translatePath(
os.path.join('special://home/addons/script.xxxodus.scrapers',
'lib/scrapers/%s.py' % scraper.lower()))
scraperLink = 'https://raw.githubusercontent.com/xibalba10/script.xxxodus.scrapers/master/lib/scrapers/%s.py' % scraper.lower(
)
r = cache.get(client.request, 4, scraperLink)
if len(r) > 1:
with open(scraperFile, 'r') as f:
compfile = f.read()
if 'import' in r:
if compfile == r:
log_utils.log(
'%s checked and up to date!' % scraper.title(),
log_utils.LOGNOTICE)
pass
else:
with open(scraperFile, 'w') as f:
f.write(r)
icon = xbmc.translatePath(
os.path.join(
'special://home/addons/script.xxxodus.artwork',
'resources/art/%s/icon.png' % scraper.lower()))
log_utils.log('%s updated!' % scraper.title(),
log_utils.LOGNOTICE)
kodi.notify(msg='%s Updated.' % scraper.title(),
duration=1250,
sound=True,
icon_path=icon)
except Exception as e:
log_utils.log(
'Error checking for scraper update %s :: Error: %s' %
(scraper.title(), str(e)), log_utils.LOGERROR)
def get_currency_precision(currency):
"""
Get precision by currency code.
Precision values will be populated from the ``decimal_places``
fields of the `Currency` objects in the database.
:type currency: str
:param currency: Currency code as 3-letter string (ISO-4217)
:rtype: decimal.Decimal|None
:return: Precision value for given currency code or None for unknown
"""
cache_key = 'currency_precision:' + currency
precision = cache.get(cache_key)
if precision is None:
currency_obj = Currency.objects.filter(code=currency).first()
precision = (
decimal.Decimal('0.1') ** currency_obj.decimal_places
if currency_obj else None)
cache.set(cache_key, precision)
return precision
def display_unit(self):
"""
Default display unit of this sales unit.
Get a `DisplayUnit` object, which has this sales unit as its
internal unit and is marked as a default, or if there is no
default display unit for this sales unit, then a proxy object.
The proxy object has the same display unit interface and mirrors
the properties of the sales unit, such as symbol and decimals.
:rtype: DisplayUnit
"""
cache_key = "display_unit:sales_unit_{}_default_display_unit".format(self.pk)
default_display_unit = cache.get(cache_key)
if default_display_unit is None:
default_display_unit = self.display_units.filter(default=True).first()
# Set 0 to cache to prevent None values, which will not be a valid cache value
# 0 will be invalid below, hence we prevent another query here
cache.set(cache_key, default_display_unit or 0)
return default_display_unit or SalesUnitAsDisplayUnit(self)
def invoke_align(reference_filename, read_filename, max_errors):
reference_filesig = cache.file_signature(reference_filename)
reference_filename = reference_filesig[0]
read_filesig = cache.file_signature(read_filename)
read_filename = read_filesig[0]
def callback(working_dir):
print >> sys.stderr, 'Aligning'
#Hmm
old_stdout = sys.stdout
sys.stdout = open(os.path.join(working_dir, 'hits.myr'), 'wb')
try:
assert align.main(
[str(max_errors), '1', reference_filename, read_filename]) == 0
finally:
sys.stdout.close()
sys.stdout = old_stdout
return os.path.join(
cache.get(('assess', 'invoke_align1', reference_filesig, read_filesig,
max_errors), callback), 'hits.myr')
async def heatmap(startDate, endDate, requestTypes=[], ncList=[]):
filters = {
'startDate': startDate,
'endDate': endDate,
'requestTypes': requestTypes,
'ncList': ncList
}
key = get_pins_cache_key(filters)
# NOTE: pins get pulled for heatmap from cache (disk) even though it was
# just added by pin cluster function. might refactor to in-memory cache
pins = cache.get(key)
fields = ['latitude', 'longitude']
if pins is None:
pins = requests.standard_query(fields, filters, table='map')
else:
pins = pins[fields]
return pins.to_numpy()
def output_plantuml(args, puml_input):
"""Output a PlantUML diagram."""
ext = os.path.splitext(args.output or '')[-1][1:]
mode = {'png': 'img', 'svg': 'svg', 'uml': None, '': None}[ext]
server = (args.plantuml
or os.getenv('SYSL_PLANTUML', 'http://localhost:8080/plantuml'))
if mode:
def calc():
puml = plantuml.PlantUML('{}/{}/'.format(server, mode))
response = requests.get(puml.get_url(puml_input))
response.raise_for_status()
return response.content
out = cache.get(mode + ':' + puml_input, calc)
useConfluence = args.output.startswith('confluence://')
if args.verbose:
print args.output + '...' * useConfluence,
sys.stdout.flush()
if useConfluence:
if confluence.upload_attachment(args.output, cStringIO.StringIO(out),
args.expire_cache,
args.dry_run) is None:
if args.verbose:
print '\033[1;30m(no change)\033[0m',
else:
if args.verbose:
print '\033[1;32muploaded\033[0m',
if args.dry_run:
print '... not really (dry-run)',
else:
(open(args.output, 'w') if args.output else sys.stdout).write(out)
# (open(args.output + '.puml', 'w') if args.output else sys.stdout).write(puml_input)
if args.verbose:
print
def get_list(self, mode, type, url, title_pattern, url_pattern, icon_pattern=None, site=None, d_p1=None, d_p2=None, d_p3=None, parse=None, cache_time=None,searched=False,stopend=False, isVideo=False, isDownloadable = False):
if cache_time: r = cache.get(client.request,cache_time,url)
else: r = client.request(url)
if 're|' in d_p3:
d_p3 = d_p3.replace('re|','')
r = dom_parser2.parse_dom(r, d_p1, {d_p2: re.compile('%s' % d_p3)})
else: r = dom_parser2.parse_dom(r, d_p1, {d_p2: d_p3})
if r:
dirlst = []
for i in r:
name = re.findall(r'%s' % title_pattern,i.content)[0]
name = kodi.sortX(i[1].encode('utf-8'))
url = re.findall(r'%s' % url_pattern,i.content)[0]
if icon_pattern:
iconimage = re.findall(r'%s' % icon_pattern,i.content)[0]
elif site: iconimage = xbmc.translatePath(os.path.join('special://home/addons/script.wankbank.artwork', 'resources/art/%s/icon.png' % site))
else: iconimage = xbmc.translatePath(os.path.join('special://home/addons/' + kodi.get_id(), 'icon.png'))
fanarts = xbmc.translatePath(os.path.join('special://home/addons/script.wankbank.artwork', 'resources/art/%s/fanart.jpg' % site))
if parse:
link,tag = parse.split('|SPLIT|')
if tag == 'url':
url = urlparse.urljoin(link,url)
elif tag == 'icon':
iconimage = urlparse.urljoin(link,iconimage)
else:
url = urlparse.urljoin(link,url)
iconimage = urlparse.urljoin(link,iconimage)
if site: url += '|SPLIT|' + site
if type == 'dir': dirlst.append({'name': kodi.giveColor(name,'white'), 'url': url, 'mode': mode, 'icon': iconimage, 'fanart': fanarts, 'description': name, 'folder': True})
else: dirlst.append({'name': kodi.giveColor(name,'white'), 'url': url, 'mode': mode, 'icon': iconimage, 'fanart': fanarts, 'description': name, 'folder': False})
if dirlst:
if stopend: buildDirectory(dirlst, stopend=True, isVideo=isVideo, isDownloadable=isDownloadable)
else: buildDirectory(dirlst, isVideo=isVideo, isDownloadable=isDownloadable)
def check_comment(user_email, artwork_id, comment_text):
cache_key = COMMENT_CACHE_PREFIX + user_email
cache_value = cache.get(cache_key)
result = False
if cache_value is None:
result = True
else:
if datetime.now() - cache_value[VALUE_DATE] > COMMENT_MIN_INTERVAL:
if artwork_id <> cache_value[VALUE_ARTWORK_ID]:
result = True
else:
if comment_text <> cache_value[VALUE_COMMENT]:
result = True
cache_value = {
VALUE_DATE: datetime.now(),
VALUE_ARTWORK_ID: artwork_id,
VALUE_COMMENT: comment_text
}
cache.add(cache_key, cache_value)
return result
def _get_cached_value_from_context(context, key, value):
cached_value = None
# 1) check whether the value is cached inside the context as an attribute
try:
cache_key = "_ctx_cache_{}".format(key)
cached_value = getattr(context, cache_key)
except AttributeError:
pass
# 2) Check whether the value is cached in general cache
# we can only cache objects that has `pk` attribute
if cached_value is None and hasattr(value, "pk"):
cache_key = "_ctx_cache:{}_{}".format(key, value.pk)
cached_value = cache.get(cache_key)
# 3) Nothing is cached, then read the value itself
if cached_value is None:
if key == "customer" and value:
cached_value = _get_val(value.groups.all())
else:
cached_value = _get_val(value)
# Set the value as attribute of the context
# somethings this will raise AttributeError because the
# context is not a valid object, like a dictionary
try:
cache_key = "_ctx_cache_{}".format(key)
setattr(context, cache_key, cached_value)
except AttributeError:
pass
# cache the value in the general cache
if hasattr(value, "pk"):
cache_key = "_ctx_cache:{}_{}".format(key, value.pk)
cache.set(cache_key, cached_value)
return cached_value
def beautify(text, lang, options):
"""
Process input `text` according to the specified `mode`.
Adds comments if needed, according to the `lang` rules.
Caches the results.
The whole work (except caching) is done by _beautify().
"""
options = options or {}
beauty_options = dict((k, v) for k, v in options.items()
if k in ['add_comments', 'remove_text'])
mode = ''
if beauty_options.get('add_comments'):
mode += 'c'
if beauty_options.get('remove_text'):
mode += 'q'
if beauty_options == {}:
# if mode is unknown, just don't transform the text at all
return text
if isinstance(text, str):
text = text.encode('utf-8')
digest = "t:%s:%s:%s" % (hashlib.md5(text).hexdigest(), lang, mode)
# temporary added line that removes invalid cache entries
# that used wrong commenting methods
if lang in ["git", "django", "flask", "cmake"]:
cache.delete(digest)
answer = cache.get(digest)
if answer:
return answer
answer = _beautify(text, lang, **beauty_options)
cache.put(digest, answer)
return answer
def break_links(self):
""" One node that connects to a link in the middle is not connected;
therefore, break_links is used for cutting at the middle point and
building new links that connects nodes to links.
"""
self.logger.info("Starts to break links")
cache_enabled = Config.params["simulation"]["cache"]
# Loads all_nodes from cache if exists
if cache_enabled:
hash_key = cache.get_hash([], self.nodes)
cached = cache.get(hash_key)
if cached:
self.runways, self.taxiways, self.pushback_ways = cached
self.logger.debug("Done breaking links using cache")
return
# Retrieve all nodes and links
all_nodes = deepcopy(self.break_node)
for link in self.links:
all_nodes.append(link.start)
all_nodes.append(link.end)
self.__add_break_node(link.start)
self.__add_break_node(link.end)
index = 0
while index < len(all_nodes):
index = self.__break_next_link(all_nodes, index)
self.logger.info("Done breaking links")
self.__get_break_nodes()
# Stores the result into cache for future usages
if cache_enabled:
to_cache = [self.runways, self.taxiways, self.pushback_ways]
cache.put(hash_key, to_cache)
def get_products_ordered_with(prod, count=20, request=None, language=None):
cache_key = "ordered_with:%d" % prod.pk
product_ids = cache.get(cache_key)
if product_ids is None:
# XXX: could this be optimized more? (and does it matter?)
order_ids = (
OrderLine.objects.filter(product=prod, type=OrderLineType.PRODUCT)
.values_list("order__id", flat=True)
)
product_ids = (
OrderLine.objects
.filter(order_id__in=order_ids)
.exclude(product=prod)
.distinct()
.values_list("product", flat=True)
)
cache.set(cache_key, set(product_ids), 4 * 60 * 60)
return (
Product.objects
.all_visible(request, language=language)
.filter(id__in=product_ids)
.order_by("?")[:count]
)
def sample(read_files, n_samples):
read_filesigs = [cache.file_signature(filename) for filename in read_files]
read_files = [item[0] for item in read_filesigs]
def callback(working_dir):
print >> sys.stderr, 'Sampling'
samples = []
n = 0
for item in sequence.sequence_files_iterator(read_files):
n += 1
if len(samples) < n_samples:
samples.append(item)
elif random.random() * n_samples < n:
samples[random.randrange(n_samples)] = item
outfile = open(os.path.join(working_dir, 'sample.fna'), 'wb')
for item in samples:
print >> outfile, '>%s' % item[0]
print >> outfile, '%s' % sequence.string_from_sequence(item[1])
result_dir = cache.get(('assess', 'sample', n_samples, read_filesigs),
callback)
return os.path.join(result_dir, 'sample.fna')
def get_articles(area):
ret = []
for person in Site.CONTEXT.config.staff.sections():
feed = Site.CONTEXT.config.staff.get(person, config_field)
cache_key = os.path.join(
Site.CONTEXT.config.cache.get("cache","cache_dir"),
md5(feed).hexdigest())
blog = cache.get(cache_key,
expires=Site.CONTEXT.config.cache.get("cache","expires"))
if not blog:
blog = feedparser.parse(feed)
if not blog.has_key("status") or blog["status"] != 200:
print "warning:innovation:%s:feed %s return a non valid status. Skipping..." % (area, feed)
continue
cache.set(cache_key, blog)
if len(blog.feed) == 0:
print "warning:innovation:%s:feed %s is not available" % (
area, feed
)
continue
# Remove appeded title for category based feeds in wordpress
if blog.feed.title.find(u"»") != -1:
blog.feed.title = blog.feed.title.split(u"»")[0]
for e in blog.entries:
if getattr(e,"tags", False):
terms = map(lambda x:x["term"].lower(),e.tags)
if "idea" in terms or "labs" or "lab" in terms:
if "%s" % area in terms:
ret.append({"link":e.link,"title":e.title})
return ret
def validateUser(emailAddress=None, showRegisteration=False):
try:
url = None
if (emailAddress == None or emailAddress == ''):
emailAddress = control.setting('user.email')
if (emailAddress == None or emailAddress == '') and showRegisteration:
control.dialog.ok(
control.addonInfo('name'),
"[COLOR red]User not registered.[/COLOR] [CR]Please provide the email address used to make the donation"
)
t = control.lang(30275).encode('utf-8')
k = control.keyboard('', t)
k.doModal()
emailAddress = k.getText() if k.isConfirmed() else None
elif (emailAddress == None or emailAddress == ''):
return (control.INVALID, url)
logger.debug('Validating User : %s' % emailAddress, __name__)
valid = cache.get(validate, 168, emailAddress, table='live_cache')
if valid == None:
result = client.request(INVALID_MESSAGE_URL)
control.dialog.ok(control.addonInfo('name'), result)
control.setSetting('user.email', '')
return (control.INVALID, '')
elif valid <= 0:
return valid, ''
url = base64.b64decode(
'aHR0cHM6Ly9vZmZzaG9yZWdpdC5jb20vdmluZWVndS9hZnRlcnNob2NrLXJlcG8vZ3VpZGVzLw=='
)
return (valid, url)
except Exception as e:
logger.error(e)
result = client.request(INVALID_MESSAGE_URL)
control.dialog.ok(control.addonInfo('name'), result)
control.setSetting('user.email', '')
return (control.INVALID, url)
def get_series_ru(self, chat_id, message_id, search_query_id):
search_query = db.get_search_query(search_query_id)
series_number = search_query.query_text
key = f"avtonomer.get_series_ru({series_number})"
result = cache.get(key)
if not result:
result = avtonomer.search_ru(fastsearch="{}*{}".format(
series_number[:1],
series_number[1:],
), )
if result is not None:
result = result.total_results
if result is None:
logger.warning(f"No data for query {series_number}")
bot.send_message(
chat_id,
"Нет данных",
reply_to_message_id=message_id,
)
return
cache.add(key, result, timedelta(minutes=5))
url = avtonomer.get_series_ru_url(series_number)
series_number = avtonomer.translate_to_cyrillic(series_number)
message = (
f"В серии [{series_number}]({url}) пока нет ни одного номера"
if result == 0 else
f"Количество фотографий в серии [{series_number}]({url}): {result}")
bot.send_message(
chat_id,
message,
parse_mode="Markdown",
reply_to_message_id=message_id,
)
def get_cached_value(identifier, item, context, **kwargs):
"""
Get item from context cache by identifier
Accepts optional kwargs parameter `allow_cache` which will skip
fetching the actual cached object. When `allow_cache` is set to
False only cache key for identifier, item, context combination is
returned.
:param identifier: Any
:type identifier: string
:param item: Any
:param context: Any
:type context: dict
:return: Cache key and cached value if allowed
:rtype: tuple(str, object)
"""
allow_cache = True
if "allow_cache" in kwargs:
allow_cache = kwargs.pop("allow_cache")
key = get_cache_key_for_context(identifier, item, context, **kwargs)
if not allow_cache:
return key, None
return key, cache.get(key)
def last_query(client_id):
"""
Return the last query for the client `client_id`
"""
return cache.get("l:%s" % client_id)
def get_answers(
self,
topic: str,
request_options: Dict[str, str] = None) -> List[Dict[str, Any]]:
"""
Find cheat sheets for the topic.
Args:
`topic` (str): the name of the topic of the cheat sheet
Returns:
[answer_dict]: list of answers (dictionaries)
"""
# if topic specified as <topic_type>:<topic>,
# cut <topic_type> off
topic_type = ""
if re.match("[^/]+:", topic):
topic_type, topic = topic.split(":", 1)
topic = self.handle_if_random_request(topic)
topic_types = self.get_topic_type(topic)
# if topic_type is specified explicitly,
# show pages only of that type
if topic_type and topic_type in topic_types:
topic_types = [topic_type]
# 'question' queries are pretty expensive, that's why they should be handled
# in a special way:
# we do not drop the old style cache entries and try to reuse them if possible
if topic_types == ['question']:
answer = cache.get('q:' + topic)
if answer:
if isinstance(answer, dict):
return [answer]
return [{
'topic': topic,
'topic_type': 'question',
'answer': answer,
'format': 'text+code',
}]
answer = self._get_page_dict(topic,
topic_types[0],
request_options=request_options)
if answer.get("cache", True):
cache.put('q:' + topic, answer)
return [answer]
# Try to find cacheable queries in the cache.
# If answer was not found in the cache, resolve it in a normal way and save in the cache
answers = []
for topic_type in topic_types:
cache_entry_name = f"{topic_type}:{topic}"
cache_needed = self._adapter[topic_type].is_cache_needed()
if cache_needed:
answer = cache.get(cache_entry_name)
if not isinstance(answer, dict):
answer = None
if answer:
answers.append(answer)
continue
answer = self._get_page_dict(topic,
topic_type,
request_options=request_options)
if isinstance(answer, dict):
if "cache" in answer:
cache_needed = answer["cache"]
if cache_needed and answer:
cache.put(cache_entry_name, answer)
answers.append(answer)
return answers
def request(url, close=True, redirect=True, error=False, proxy=None, post=None, headers=None, mobile=False, XHR=False, limit=None, referer=None, cookie=None, compression=True, output='', timeout='30'):
try:
handlers = []
if not proxy == None:
handlers += [urllib2.ProxyHandler({'http':'%s' % (proxy)}), urllib2.HTTPHandler]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
if output == 'cookie' or output == 'extended' or not close == True:
cookies = cookielib.LWPCookieJar()
handlers += [urllib2.HTTPHandler(), urllib2.HTTPSHandler(), urllib2.HTTPCookieProcessor(cookies)]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
if (2, 7, 9) <= sys.version_info < (2, 7, 11):
try:
import ssl; ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
handlers += [urllib2.HTTPSHandler(context=ssl_context)]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
except:
pass
if url.startswith('//'): url = 'http:' + url
try: headers.update(headers)
except: headers = {}
if 'User-Agent' in headers:
pass
elif not mobile == True:
#headers['User-Agent'] = agent()
headers['User-Agent'] = cache.get(randomagent, 1)
else:
headers['User-Agent'] = 'Apple-iPhone/701.341'
if 'Referer' in headers:
pass
elif referer == None:
headers['Referer'] = '%s://%s/' % (urlparse.urlparse(url).scheme, urlparse.urlparse(url).netloc)
else:
headers['Referer'] = referer
if not 'Accept-Language' in headers:
headers['Accept-Language'] = 'en-US'
if 'X-Requested-With' in headers:
pass
elif XHR == True:
headers['X-Requested-With'] = 'XMLHttpRequest'
if 'Cookie' in headers:
pass
elif not cookie == None:
headers['Cookie'] = cookie
if 'Accept-Encoding' in headers:
pass
elif compression and limit is None:
headers['Accept-Encoding'] = 'gzip'
if redirect == False:
class NoRedirection(urllib2.HTTPErrorProcessor):
def http_response(self, request, response): return response
opener = urllib2.build_opener(NoRedirection)
opener = urllib2.install_opener(opener)
try: del headers['Referer']
except: pass
request = urllib2.Request(url, data=post, headers=headers)
try:
response = urllib2.urlopen(request, timeout=int(timeout))
except urllib2.HTTPError as response:
if response.code == 503:
cf_result = response.read(5242880)
try: encoding = response.info().getheader('Content-Encoding')
except: encoding = None
if encoding == 'gzip':
cf_result = gzip.GzipFile(fileobj=StringIO.StringIO(cf_result)).read()
if 'cf-browser-verification' in cf_result:
netloc = '%s://%s' % (urlparse.urlparse(url).scheme, urlparse.urlparse(url).netloc)
ua = headers['User-Agent']
cf = cache.get(cfcookie().get, 168, netloc, ua, timeout)
headers['Cookie'] = cf
request = urllib2.Request(url, data=post, headers=headers)
response = urllib2.urlopen(request, timeout=int(timeout))
elif error == False:
return
elif error == False:
return
if output == 'cookie':
try: result = '; '.join(['%s=%s' % (i.name, i.value) for i in cookies])
except: pass
try: result = cf
except: pass
if close == True: response.close()
return result
elif output == 'geturl':
result = response.geturl()
if close == True: response.close()
return result
elif output == 'headers':
result = response.headers
if close == True: response.close()
return result
elif output == 'chunk':
try: content = int(response.headers['Content-Length'])
except: content = (2049 * 1024)
if content < (2048 * 1024): return
result = response.read(16 * 1024)
if close == True: response.close()
return result
if limit == '0':
result = response.read(224 * 1024)
elif not limit == None:
result = response.read(int(limit) * 1024)
else:
result = response.read(5242880)
try: encoding = response.info().getheader('Content-Encoding')
except: encoding = None
if encoding == 'gzip':
result = gzip.GzipFile(fileobj=StringIO.StringIO(result)).read()
if 'sucuri_cloudproxy_js' in result:
su = sucuri().get(result)
headers['Cookie'] = su
request = urllib2.Request(url, data=post, headers=headers)
response = urllib2.urlopen(request, timeout=int(timeout))
if limit == '0':
result = response.read(224 * 1024)
elif not limit == None:
result = response.read(int(limit) * 1024)
else:
result = response.read(5242880)
try: encoding = response.info().getheader('Content-Encoding')
except: encoding = None
if encoding == 'gzip':
result = gzip.GzipFile(fileobj=StringIO.StringIO(result)).read()
if output == 'extended':
response_headers = response.headers
response_code = str(response.code)
try: cookie = '; '.join(['%s=%s' % (i.name, i.value) for i in cookies])
except: pass
try: cookie = cf
except: pass
if close == True: response.close()
return (result, response_code, response_headers, headers, cookie)
else:
if close == True: response.close()
return result
except:
return
