def reportStateMessage(vmHostname, message, serverHostname=None):
"""Sends a state message to the Server
Keyword arguments:
vmHostname -- Hostname of the virtual machine
message -- message to be sent
"""
if serverHostname == None:
serverHostname = vnConfigNode.getServerHostname()
message = message.replace(" ", "%20")
logging.info("sending a state message " + message + " to " + serverHostname)
logging.info(
proto
+ serverHostname
+ path
+ "?vmAction=setstatemessage&vmHostname="
+ vmHostname
+ "&stateMessage="
+ message
)
urllib2.urlopen(
proto
+ serverHostname
+ path
+ "?vmAction=setstatemessage&vmHostname="
+ vmHostname
+ "&stateMessage="
+ message
).close()
def on_drag_data_received(widget, context, x, y, selection, target_type, timestamp):
if target_type == TARGET_TYPE_URI_LIST:
uri = selection.get_data().strip("\r\n\x00")
print "uri", uri
uri_splitted = uri.split() # we may have more than one file dropped
for uri in uri_splitted:
path = get_file_path_from_dnd_dropped_uri(uri)
print "path to open", path
if os.path.isfile(path): # is it file?
data = file(path).read()
# print data
thread.start_new_thread(serve, (path, 1))
filename = os.path.splitext(os.path.split(path)[1])[0]
extension = os.path.splitext(path)[1]
encoded_string = urllib.quote_plus("http://" + localip + ":8080/" + base64.b32encode(filename) + extension)
full_url = "http://" + w.entry.get_text() + ":2020/stream?url="
if os.path.isfile(os.path.splitext(path)[0] + ".srt"):
urllib2.urlopen(
full_url
+ encoded_string
+ "&subtitles="
+ urllib.quote_plus(
"http://" + localip + ":8080/" + base64.b32encode(os.path.splitext(os.path.split(path)[1])[0])
)
+ ".srt"
).read()
else:
urllib2.urlopen(full_url + encoded_string).read()
def get_feed(max_feed=5):
"""
This function is responsible to get RSS feeds.
"""
try:
f = urllib2.urlopen("http://feeds.feedburner.com/XSSPosed")
except urllib2.URLError:
raise Exception("Internet connection problems")
tree = xml_parser.fromstring(f.read())
channel = tree.find("channel")
items = channel.findall("item")
for item in reversed(list(items)[0:max_feed]):
link = list(item.iter("link"))[0].text
if link not in CONTAINER:
CONTAINER.add(link)
site_response = urllib2.urlopen(link).read()
details = get_details(site_response)
yield {
"title": list(item.iter("title"))[0].text,
"description": details["description"],
"link": link,
"status": str(details["status"]),
"exploit": get_exploit(site_response),
"post_data": details["post_data"],
}
def renren_set(email, password, text):
import urllib2, cookielib, urllib
from BeautifulSoup import BeautifulSoup
cj = cookielib.CookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
urllib2.install_opener(opener)
postdata = urllib.urlencode(
{
"email": email,
"password": password,
"captcha_type": "web_login",
"icode": "",
"origURL": "http://www.renren.com/indexcon",
"domain": "renren.com",
"key_id": "1",
}
)
req = urllib2.Request(url="http://www.renren.com/PLogin.do", data=postdata)
response = urllib2.urlopen(req)
content = response.read()
j = content[content.find("script") :]
rtk = j[j.find("get_check_x:") + 13 : j.find("get_check_x:") + 21] # 寻找rtk，rtk是人人的一个校验码，每次登陆不同。
newdata = urllib.urlencode({"_rtk": rtk, "channel": "renren", "content": text, "hostid": "729595072"})
newreq = urllib2.Request(url="http://shell.renren.com/729595072/status", data=newdata)
newcontent = urllib2.urlopen(newreq)
def getTop100Info(resultsPageUrl, pageIndex, destinationFolder):
try:
response = urllib2.urlopen(resultsPageUrl + str(pageIndex))
html = response.read()
fromRank = (pageIndex - 1) * 20 + 1
toRank = fromRank + 19
localFile = open(os.path.join(destinationFolder, str(fromRank) + "-" + str(toRank) + ".html"), "w")
localFile.write(html)
localFile.close()
ebooksDestinationFolder = os.path.join(destinationFolder, "ebooks")
if not os.path.exists(ebooksDestinationFolder):
os.makedirs(ebooksDestinationFolder)
# Must use the lxml HTML r even though it has external dependecies, because the default python one is not good enough to the Kijiji pages
# lxml Windows distribution was downloaded from http://www.lfd.uci.edu/~gohlke/pythonlibs/#lxml
soup = BeautifulSoup(html, "lxml")
# Extract details of each book in the top 100
for bookDiv in soup.body.find_all("div", {"class", "zg_itemImmersion"}):
itemWrapper = bookDiv.find("div", {"class", "zg_itemWrapper"})
ebookUrl = itemWrapper.find("div", {"class", "zg_itemImageImmersion"}).a["href"].strip()
asin = ebookUrl[-10:]
response = urllib2.urlopen(ebookUrl)
ebookHtml = response.read()
ebookFileName = os.path.join(ebooksDestinationFolder, asin + ".html")
ebookFile = open(ebookFileName, "w")
ebookFile.write(ebookHtml)
ebookFile.close()
print(asin)
except urllib2.URLError as e:
print("Failed to download from " + resultsPageUrl + ":" + e.reason)
def dif_html(a, b, lower_criteria=0.5, upper_criteria=0.7, link_criteria=0.4, img_criteria=0.2, script_criteria=0.2):
from urllib2 import urlopen
from difflib import SequenceMatcher
from my_stats import jaccard as jac
# 詳細にページをみる
try:
html_a, html_b = urlopen(a).read(), urlopen(b).read()
matcher = SequenceMatcher()
matcher.set_seq1(html_a)
matcher.set_seq2(html_b)
if matcher.ratio() >= lower_criteria:
# lower_criteria以上だけどupper_criteria以下の場合にはリンク構造など詳しく調査する
if matcher.ratio() >= upper_criteria:
print "white", matcher.ratio(), upper_criteria, lower_criteria, a, b
return True
else:
# ここにnon-negative-matrix-factorizationを入れてより精緻に分析する予定だったけど、まだあまりうまく行かないのでペンディング(T.B.D.)
print "grey", matcher.ratio(), upper_criteria, lower_criteria, a, b
links_a, titles_a, imgs_a, scripts_a = analyze_html(html_a)
links_b, titles_b, imgs_b, scripts_b = analyze_html(html_b)
j_links, j_imgs, j_scripts = jac(links_a, links_b), jac(imgs_a, imgs_b), jac(imgs_a, imgs_b)
if j_links >= link_criteria and j_imgs >= img_criteria and j_scripts >= script_criteria:
return True
else:
return False
# lower_criteria以上に似てない部分がある場合には似てないと判断する
else:
print "black", matcher.ratio(), upper_criteria, lower_criteria, a, b
return False
except:
# クローリングできない場合には異なるページと見なす
return False
def webservice(url, retries=RETRIES, postdata=False):
"""
A wrapper around urllib.urlopen to allow for retries
:param url: The url to fetch content from.
:param retries: The number of times to retry the urlopen method.
:return: A dictionary with a boolean at success, and
"""
attempt = 0
data = [False, ""]
while attempt < retries:
try:
if postdata is False:
logging.info("urlopen: GET %s" % url)
data[1] = urllib2.urlopen(url, timeout=TIMEOUT).read().strip()
data[0] = True
else:
logging.info("urlopen: POST %s" % url)
data[1] = urllib2.urlopen(url, postdata, timeout=TIMEOUT).read().strip()
data[0] = True
logging.info("urlopen: RESP received %s" % data[1].replace("\n", r"\n"))
break
except urllib2.HTTPError, e:
# Handle specific error types
code = e.getcode()
logging.error("Received HTTP response code: %d - %s" % (code, data[1]))
except BaseException as e:
logging.error(e.message)
def publish(self, url):
"""Try to publish this descriptor to a given directory URL. Returns
'accept' if the publication was successful, 'reject' if the
server refused to accept the descriptor, and 'error' if
publication failed for some other reason."""
fname = self.getDescriptorFileName()
descriptor = readFile(fname)
fields = urllib.urlencode({"desc": descriptor})
f = None
try:
try:
#############################################
# some python versions verify certificates
# anemone.mooo.com uses a self-signed cert
# this workaround is not a problem because
# the directory information is already signed
# (although as Zax says, it is certainly a
# kludge ;)
if sys.version_info >= (2, 7, 9):
ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
f = urllib2.urlopen(url, fields, context=ctx)
else:
f = urllib2.urlopen(url, fields)
#############################################
# f = urllib2.urlopen(url, fields)
info = f.info()
reply = f.read()
except IOError, e:
LOG.error("Error while publishing server descriptor: %s", e)
return "error"
except:
LOG.error_exc(sys.exc_info(), "Error publishing server descriptor")
return "error"
def getXMLPageCore(headers={}, params={}, config={}):
""" """
# returns a XML containing params['limit'] revisions (or current only), ending in </mediawiki>
# if retrieving params['limit'] revisions fails, returns a current only version
# if all fail, returns the empty string
xml = ""
c = 0
maxseconds = 100 # max seconds to wait in a single sleeping
maxretries = 5 # x retries and skip
increment = 20 # increment every retry
while not re.search(r"</mediawiki>", xml):
if c > 0 and c < maxretries:
wait = increment * c < maxseconds and increment * c or maxseconds # incremental until maxseconds
print ' XML for "%s" is wrong. Waiting %d seconds and reloading...' % (params["pages"], wait)
time.sleep(wait)
if (
params["limit"] > 1
): # reducing server load requesting smallest chunks (if curonly then limit = 1 from mother function)
params["limit"] = params["limit"] / 2 # half
if c >= maxretries:
print " We have retried %d times" % (c)
print ' MediaWiki error for "%s", network error or whatever...' % (params["pages"])
# If it's not already what we tried: our last chance, preserve only the last revision...
# config['curonly'] means that the whole dump is configured to save nonly the last
# params['curonly'] means that we've already tried this fallback, because it's set by the following if and passed to getXMLPageCore
if not config["curonly"] and not params["curonly"]:
print " Trying to save only the last revision for this page..."
params["curonly"] = 1
logerror(
config=config,
text='Error while retrieving the full history of "%s". Trying to save only the last revision for this page'
% (params["pages"]),
)
return getXMLPageCore(headers=headers, params=params, config=config)
else:
print " Saving in the errors log, and skipping..."
logerror(
config=config,
text='Error while retrieving the last revision of "%s". Skipping.' % (params["pages"]),
)
return "" # empty xml
data = urllib.urlencode(params)
req = urllib2.Request(url=config["index"], data=data, headers=headers)
try:
f = urllib2.urlopen(req)
except:
try:
print "Server is slow... Waiting some seconds and retrying..."
time.sleep(15)
f = urllib2.urlopen(req)
except:
print 'An error have occurred while retrieving "%s"' % (params["pages"])
print "Please, resume the dump, --resume"
sys.exit()
# The error is usually temporary, but we exit the dump altogether.
xml = f.read()
c += 1
return xml
def post_to_facebook(app_access_token, user_social_auth, message, link):
url = "https://graph.facebook.com/%s/feed" % user_social_auth.uid
params = {"access_token": app_access_token, "message": message, "link": link}
req = urllib2.Request(url, urllib.urlencode(params))
urllib2.urlopen(req)
def _check_prerequisites(self):
"""Sanity checks before attempting to mount SOFS."""
# config is mandatory
config = CONF.scality_sofs_config
if not config:
msg = _("Value required for 'scality_sofs_config'")
LOG.warn(msg)
raise exception.NovaException(msg)
# config can be a file path or a URL, check it
if urlparse.urlparse(config).scheme == "":
# turn local path into URL
config = "file://%s" % config
try:
urllib2.urlopen(config, timeout=5).close()
except urllib2.URLError as e:
msg = _("Cannot access 'scality_sofs_config': %s") % e
LOG.warn(msg)
raise exception.NovaException(msg)
# mount.sofs must be installed
if not os.access("/sbin/mount.sofs", os.X_OK):
msg = _("Cannot execute /sbin/mount.sofs")
LOG.warn(msg)
raise exception.NovaException(msg)
def download_all_logs(dirname=".", filename=None):
"""
Download H2O Log Files to Disk
:param dirname: (Optional) A character string indicating the directory that the log file should be saved in.
:param filename: (Optional) A string indicating the name that the CSV file should be
:return: path of logs written (as a string)
"""
url = "http://" + H2OConnection.ip() + ":" + str(H2OConnection.port()) + "/Logs/download"
response = urllib2.urlopen(url)
if not os.path.exists(dirname):
os.mkdir(dirname)
if filename == None:
for h in response.headers.headers:
if "filename=" in h:
filename = h.split("filename=")[1].strip()
break
path = os.path.join(dirname, filename)
with open(path, "w") as f:
response = urllib2.urlopen(url)
f.write(response.read())
f.close()
print "Writing H2O logs to " + path
return path
def load(self, filename):
"""
_load_
Unpickle data from file
"""
# TODO: currently support both loading from file path or url
# if there are more things to filter may be separate the load function
# urllib2 needs a scheme - assume local file if none given
if not urlparse(filename)[0]:
filename = "file:" + filename
handle = urlopen(Request(filename, headers={"Accept": "*/*"}))
self.data = pickle.load(handle)
handle.close()
elif filename.startswith("file:"):
handle = urlopen(Request(filename, headers={"Accept": "*/*"}))
self.data = pickle.load(handle)
handle.close()
else:
# use own request class so we get authentication if needed
from WMCore.Services.Requests import Requests
request = Requests(filename)
data = request.makeRequest("", incoming_headers={"Accept": "*/*"})
self.data = pickle.loads(data[0])
# TODO: use different encoding scheme for different extension
# extension = filename.split(".")[-1].lower()
return
def test_webhook_no_ssl(self):
print("Testing Webhook without SSL")
self._setup_updater("", messages=0)
d = self.updater.dispatcher
d.addTelegramMessageHandler(self.telegramHandlerTest)
# Select random port for travis
port = randrange(1024, 49152)
self.updater.start_webhook("127.0.0.1", port)
sleep(0.5)
# Now, we send an update to the server via urlopen
message = Message(1, User(1, "Tester 2"), datetime.now(), Chat(1, "group", title="Test Group 2"))
message.text = "Webhook Test 2"
update = Update(1)
update.message = message
try:
payload = bytes(update.to_json(), encoding="utf-8")
except TypeError:
payload = bytes(update.to_json())
header = {"content-type": "application/json", "content-length": str(len(payload))}
r = Request("http://127.0.0.1:%d/" % port, data=payload, headers=header)
urlopen(r)
sleep(1)
self.assertEqual(self.received_message, "Webhook Test 2")
def _http_request(url, request_timeout=None):
"""
PRIVATE METHOD
Uses json.load to fetch the json results from the solr api.
url : str
a complete url that can be passed to urllib.open
request_timeout : int (None)
The number of seconds before the timeout should fail. Leave blank/None to
use the default. __opts__['solr.request_timeout']
Return: dict<str,obj>::
{'success':boolean, 'data':dict, 'errors':list, 'warnings':list}
"""
try:
request_timeout = __opts__["solr.request_timeout"]
if request_timeout is None:
data = json.load(urllib2.urlopen(url))
else:
data = json.load(urllib2.urlopen(url, timeout=request_timeout))
return _get_return_dict(True, data, [])
except Exception as e:
return _get_return_dict(False, {}, ["{0} : {1}".format(url, e)])
def INDEX(url, name):
name = "" + name
req = urllib2.Request(url)
req.add_header("User-Agent", user_agent)
response = urllib2.urlopen(req)
link = response.read()
response.close()
match = re.compile('class="widget widget_text"><h3>' + name + "</h3>(.+?)</div></div>", re.DOTALL).findall(link)
for items in match:
match1 = re.compile(
'<span style="font-size: small;">(.+?)<a href="(.+?)">(.+?)</a>(.+?)</span>', re.DOTALL
).findall(items)
for jnk1, channelurl, name, spc in match1:
try:
req = urllib2.Request(channelurl)
response = urllib2.urlopen(req)
link = response.read()
response.close()
match = re.compile('<div class="entry">(.+?)</div></div>', re.DOTALL).findall(link)
for rawhtml in match:
playertype = identifyLink(rawhtml, name)
streamLink = getStreamLink(playertype, rawhtml, name)
addLink(name.title() + "-" + playertype, streamLink, "")
except:
pass
def ping_newrelic(ctx):
if settings.NEW_RELIC_API_KEY and settings.NEW_RELIC_APP_ID:
with ctx.lcd(settings.SRC_DIR):
log_cmd = "git log --oneline {0}..{1}".format(OLDREV, NEWREV)
changelog = ctx.local(log_cmd).out.strip()
print "Post deployment to New Relic"
desc = generate_desc(OLDREV, NEWREV, changelog)
if changelog:
github_url = GITHUB_URL.format(oldrev=OLDREV, newrev=NEWREV)
changelog = "{0}\n\n{1}".format(changelog, github_url)
data = urllib.urlencode(
{
"deployment[description]": desc,
"deployment[revision]": NEWREV,
"deployment[app_id]": settings.NEW_RELIC_APP_ID,
"deployment[changelog]": changelog,
}
)
headers = {"x-api-key": settings.NEW_RELIC_API_KEY}
try:
request = urllib2.Request(NEW_RELIC_URL, data, headers)
urllib2.urlopen(request)
except urllib.URLError as exp:
print "Error notifying New Relic: {0}".format(exp)
def test_once_mode_three_times(tmpdir):
testfile = str(tmpdir.join("recordmode.yml"))
with vcr.use_cassette(testfile, record_mode="once"):
# get three of the same file
response1 = urllib2.urlopen("http://httpbin.org/").read()
response2 = urllib2.urlopen("http://httpbin.org/").read()
response2 = urllib2.urlopen("http://httpbin.org/").read()
def load(self):
# load docs from a list
info("loading sample data from http://openlibrary.org/people/anand/lists/OL1815L")
system(INTERP + " ./scripts/copydocs.py --list /people/anand/lists/OL1815L")
# Hack to load borrowable ebooks to store so that they appear in the return cart
urllib2.urlopen("http://0.0.0.0:8080/_dev/process_ebooks").read()
def enterprise_main(dest=None):
"""
Build out the JSON file for each organization
"""
output_dir = dest if dest else default_dest
# Make sure the destination folder exists
if not dest and not os.path.exists(default_dest):
os.makedirs(default_dest)
# Get the contents of the big data.json file
response = None
try:
response = urllib2.urlopen(enterprise_datajson_source)
except urllib2.URLError:
# Fall back on default
response = urllib2.urlopen("http://localhost:5000/enterprisedata.json")
datasets = json.load(response)
# group datasets by organization
datasets_grouped_by_org = {}
for dataset in datasets:
org = dataset["organization"]
if org in datasets_grouped_by_org:
datasets_grouped_by_org[org].append(dataset)
else:
datasets_grouped_by_org[org] = [dataset]
# write the organization.json files
for org_name, org_datasets in datasets_grouped_by_org.iteritems():
with open(os.path.join(output_dir, org_name + "_enterprise.json"), "w") as outfile:
json.dump(org_datasets, outfile)
def json_processing_callback(entry):
"""
Send an HTTP post to the registered callback url, if any.
"""
if not entry.processing_metadata:
_log.debug("No processing callback for {0}".format(entry))
return
url = entry.processing_metadata[0].callback_url
_log.debug("Sending processing callback for {0} ({1})".format(entry, url))
headers = {"Content-Type": "application/json"}
data = {"id": entry.id, "state": entry.state}
# Trigger testing mode, no callback will be sent
if url.endswith("secrettestmediagoblinparam"):
TESTS_CALLBACKS.update({url: data})
return True
request = create_post_request(url, data, headers=headers, data_parser=json.dumps)
try:
urlopen(request)
_log.debug("Processing callback for {0} sent".format(entry))
return True
except HTTPError:
_log.error("Failed to send callback: {0}".format(traceback.format_exc()))
return False
def continuity(url):
import md5
format = "%25s: %s"
# first fetch the file with the normal http handler
opener = urllib2.build_opener()
urllib2.install_opener(opener)
fo = urllib2.urlopen(url)
foo = fo.read()
fo.close()
m = md5.new(foo)
print format % ("normal urllib", m.hexdigest())
# now install the keepalive handler and try again
opener = urllib2.build_opener(HTTPHandler())
urllib2.install_opener(opener)
fo = urllib2.urlopen(url)
foo = fo.read()
fo.close()
m = md5.new(foo)
print format % ("keepalive read", m.hexdigest())
fo = urllib2.urlopen(url)
foo = ""
while 1:
f = fo.readline()
if f:
foo = foo + f
else:
break
fo.close()
m = md5.new(foo)
print format % ("keepalive readline", m.hexdigest())
def UploadCodeTallyJson(server, json_file):
"""Reads in the json_file and sends it to the given server."""
with open(json_file, "r") as f:
json_data = f.read()
data = urllib.urlencode({"data": json_data})
req = urllib2.Request(server + UPLOAD_PATH, data)
# If the upload encounters an exception that doesn't seem to be this scripts
# fault, retry a few times.
for attempt in range(5):
try:
urllib2.urlopen(req)
print "Sucessfully uploaded data to server"
return 0
except urllib2.HTTPError as e:
if e.code < 500:
print "Encounter an exception, not retrying.\n%s" % str(e)
return 1
print "Encountered a server error, retrying\n%s" % str(e)
except (httplib.HTTPException, urllib2.URLError) as e:
print "Encountered an exception, retrying.\n%s" % str(e)
# Add an exponential backoff.
duration = random.random() * 3 + math.pow(1.5, (attempt + 1))
duration = min(10, max(0.1, duration))
time.sleep(duration)
print "Unable to upload to the server after 5 attempts, aborting."
return 1
def post_http_request(self, url, data=None, testcase=None, max_attempts=1, json=False):
"""
INPUT
url: the HTTP url to use.
data: a dictionary of data values to encode into the url when posting.
testcase: a testcase object supplied when executing function as part of a testcase step.
max attempts: the maximum number of attempts to perform the operation.
json: whether to post the request using JSON format or not.
OUPUT
successful: whether the function executed successfully or not.
response: the response to the request.
"""
self.log.debug("Posting HTTP request %s %s ..." % (url, str(data)))
result = {"successful": False, "response": None}
attempt = 1
while not result["successful"] and attempt <= max_attempts:
try:
if data is not None:
if json:
# transform the data object into a form data encoded string (using JSON)
s_data = dumps(data) if data is not None else ""
else:
# transform the data object into a form data encoded string
s_data = urlencode(data)
s_data = s_data.replace("+", "%20").replace("%28", "(").replace("%29", ")")
else:
s_data = ""
# post the request
if json:
request = Request(url, s_data, {"Content-Type": "application/json"})
result["response"] = urlopen(request).read().strip()
self.log.trace("Posted JSON request.")
else:
result["response"] = urlopen(url, s_data).read().strip()
self.log.trace("Posted HTTP request.")
self.log.trace("Response: %s" % result["response"])
if result["response"] is not None and result["response"] != "":
result["successful"] = True
break
else:
self.log.trace(
"Failed to post HTTP request %s %s (attempt %s). "
"No response received from server."
"Re-attempting in 5 seconds ..." % (url, str(s_data), attempt)
)
except HTTPError, e:
self.log.trace(str(e))
self.log.trace(
"Failed to post HTTP request %s %s (attempt %s). "
"Re-attempting in 5 seconds ..." % (url, str(s_data), attempt)
)
except BaseException, e:
self.handle_exception(e, operation="post HTTP request %s %s (attempt %s)" % (url, str(s_data), attempt))
self.log.trace("Re-attempting in 5 seconds ...")
def pullPhotos(query):
print "looking for", query
url1 = "https://www.google.com/search?biw=1309&bih=704&sei=bsHjUbvaEILqrQeA-YCYDw&tbs=itp:lineart&tbm=isch&"
query2 = urllib.urlencode({"q": query})
req = urllib2.Request(url1 + query2, headers={"User-Agent": "Chrome"})
response = urllib2.urlopen(req).read()
parser = MyHTMLParser()
parser.feed(response)
print image_lib + "\\buffer\\" + query
if not os.path.exists(image_lib + "\\buffer"):
os.mkdir(image_lib + "\\buffer") # make directory to put them in
if not os.path.exists(image_lib + "\\buffer\\" + query):
os.mkdir(image_lib + "\\buffer\\" + query) # make directory to put them in
for i in xrange(5):
req_cat = urllib2.Request(cat_urls[i], headers={"User-Agent": "Chrome"})
response_cat = urllib2.urlopen(req_cat).read()
name = query + os.sep + query + str(i) + ".jpg"
fd = open(image_lib + "\\buffer\\" + name, "wb")
fd.write(response_cat)
fd.close()
print name, "written", "complexity is ", countComponents(image_lib + "\\buffer\\" + name)
print "done"
def _canUpdate(self):
addon_list_page = urllib2.urlopen(self.url[self.addon_type]["file_list"] % self.addon_id)
# <a href="/addons/recount/files/1478-v4-2-0e-release/">v4.2.0e release</a>
m = re.search(r'href="(/addons/' + self.addon_id + '/files/([a-zA-Z0-9-]+?)/)">(.+?)<', addon_list_page.read())
if m:
self.last_version = m.group(3)
if self.last_version != self.current_version:
download_page = urllib2.urlopen(self.url[self.addon_type]["base"] + m.group(1))
# <a href="http://static.wowace.com/content/files/529/668/Recount-v4.2.0e_release.zip">
# <a href="http://www.curseforge.com/media/files/527/306/TradeSkillMaster-v0.2.4Beta.zip">
# <a href="http://www.curseforge.com/media/files/530/116/TradeSkillMaster_Crafting-r438.zip">
m = re.search(
r'href="(' + self.url[self.addon_type]["static"] + '/media/files/(.+?).zip)"', download_page.read()
)
if m:
self.file = m.group(1)
return True
else:
if self.logging:
self.logging.error(
"CurseAddon: Regex didn't match, probably download page has been changed. (%s)",
self.addon_name,
)
return False
else:
return False
else:
if self.logging:
self.logging.error(
"CurseAddon: Regex didn't match, probably file list page has been changed. (%s)", self.addon_name
)
return False
def Downloadcode(cookie, savefile="code.gif", url=URL):
"""
download code image from url
"""
import urllib2
# url = url + '?ticks=' + str(int(time.time()*1000))
# URL = url
# print url
global ASPXAUTH
global ASPNET_SESSIONID
# print ASPXAUTH,'\n', ASPNET_SESSIONID
# print cookie
cookie += "; %s=%s" % (".ASPXAUTH", ASPXAUTH)
cookie += "; %s=%s" % ("ASP.NET_SessionId", ASPNET_SESSIONID)
# print cookie
req = urllib2.Request(url)
req.add_header("Cookie", cookie)
req.add_header(
"User-Agent",
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; Zune 4.7; MS-RTC LM 8; InfoPath.3; BRI/2)",
)
try:
res_img = urllib2.urlopen(req)
f = open(savefile, "wb")
f.write(res_img.read())
f.close()
except:
res_img = urllib2.urlopen(req)
f = open(savefile, "wb")
f.write(res_img.read())
f.close()
def _send_notify(msg, api_key=None, id=None):
"""
Sends a Telegram notification
:param msg: The message string to send
:param id: The Telegram user/group id to send the message to
:param api_key: Your Telegram bot API token
:returns: True if the message succeeded, False otherwise
"""
id = autosub.TELEGRAMID if id is None else id
api_key = autosub.TELEGRAMAPI if api_key is None else api_key
log.debug("telegram: Notification send to ID: %s" % id)
message = "{0} : {1}".format("Auto-Sub", msg)
payload = urllib.urlencode({"chat_id": id, "text": message})
telegram_api = "https://api.telegram.org/bot%s/%s"
req = urllib2.Request(telegram_api % (api_key, "sendMessage"), payload)
try:
urllib2.urlopen(req)
log.debug("telegram : message sent successfully.")
except Exception as error:
log.error("telegram: %s" % error)
return False
return True
def scrap_items():
for itemlist in ITEMLIST:
soup = BS(urllib2.urlopen("".join([LOLWIKI, itemlist])).read())
item_table = soup.find("table", class_="stdt sortable")
for tr in item_table.find_all("tr"):
tds = tr.find_all("td")
if len(tds) < 1:
continue
if tr.find("p") == None:
continue
item_name = tr.find("p").text.strip()
item_url = tr.find("img")["src"]
if item_url.split(":")[0] == "data":
item_url = tr.find("img")["data-src"]
if not HOOKED:
continue
# store item in database
d_item = Item()
d_item.name = item_name
t_img = NamedTemporaryFile(delete=True)
t_img.write(urllib2.urlopen(item_url).read())
t_img.flush()
t_img.name = ".".join([item_name, "jpg"])
d_item.picture = File(t_img)
d_item.save()
def get_dbusage_info(self, rest_api_port):
"""Collects database usage information from all db nodes
Returns:
A dictionary with db node name as key and db usage in % as value
"""
to_return = {}
try:
uve_url = (
"http://127.0.0.1:" + str(rest_api_port) + "/analytics/uves/database-nodes?cfilt=DatabaseUsageInfo"
)
node_dburls = json.loads(urllib2.urlopen(uve_url).read())
for node_dburl in node_dburls:
# calculate disk usage percentage for analytics in each cassandra node
db_uve_state = json.loads(urllib2.urlopen(node_dburl["href"]).read())
db_usage_in_perc = (
100
* float(db_uve_state["DatabaseUsageInfo"]["database_usage"][0]["analytics_db_size_1k"])
/ float(
db_uve_state["DatabaseUsageInfo"]["database_usage"][0]["disk_space_available_1k"]
+ db_uve_state["DatabaseUsageInfo"]["database_usage"][0]["disk_space_used_1k"]
)
)
to_return[node_dburl["name"]] = db_usage_in_perc
except Exception as inst:
self._logger.error(type(inst)) # the exception instance
self._logger.error(inst.args) # arguments stored in .args
self._logger.error(inst) # __str__ allows args to be printed directly
self._logger.error("Could not retrieve db usage information")
self._logger.info("db usage:" + str(to_return))
return to_return