def __load_config(file):
from os.path import exists, isfile, dirname, join, abspath, normpath
global config
#
default = False
if file is None:
file = __DEFAULT_CONF_FILE
default = True
if not exists(file) or not isfile(file):
file = join(join(dirname(__file__), "../"), file)
file = normpath(abspath(file))
if not exists(file) or not isfile(file):
if default:
log.warn("Config file '%s' is not found." % file)
file = "/tmp/m3highload-test.conf.tmp"
__generate_config(file)
else:
log.error("Config file '%s' is not found." % file)
config = file
log.debug("Configuration file '%s' loaded.\n\n%s" % (config, open(config, "r").read()))
def send_post(self, command):
data = command.data
if not data:
data = {}
data["token"] = Connection.get_message_token(data,
Connection.SECRET_TOKEN)
log.debug("Server request\n" \
"Command: %s\n" \
"Data: %s\n" % (command.name, json.dumps(data)))
self.__connection.request("POST", command.name, json.dumps(data),
{"Content-Type": "application/json"})
response = self.__connection.getresponse()
if response.status != 200:
raise RuntimeError("Invalid server response status %d: %s" %
(response.status, response.reason))
data = response.read()
log.debug("Server response\n" \
"Data: %s\n" % data)
command.process(utils.sdict(json.loads(data.decode("utf-8"))))
return command
def send_get(self, command):
data = command.data
if data:
query = "&".join("%s=%s" % (key, str(data[key])) for key in data)
log.debug("Server request\n" \
"Command: %s\n" \
"Data: %s\n" % (command.name, query))
self.__connection.request("GET", "%s?%s" % (command.name, query))
else:
log.debug("Server request:\n" \
"Command: %s\n" \
"Data: None\n" % command.name)
self.__connection.request("GET", command.name)
response = self.__connection.getresponse()
if response.status != 200:
log.error("Invalid server response status %d: %s" % (response.status, response.reason))
data = response.read()
log.debug("Server response\n" \
"Data: %s\n" % data)
command.process(utils.sdict(json.loads(data.decode("utf-8"))))
return command
def send_get(self, command):
data = command.data
if data:
query = "&".join("%s=%s" % (key, str(data[key])) for key in data)
log.debug("Server request\n" \
"Command: %s\n" \
"Data: %s\n" % (command.name, query))
self.__connection.request("GET", "%s?%s" % (command.name, query))
else:
log.debug("Server request:\n" \
"Command: %s\n" \
"Data: None\n" % command.name)
self.__connection.request("GET", command.name)
response = self.__connection.getresponse()
if response.status != 200:
log.error("Invalid server response status %d: %s" %
(response.status, response.reason))
data = response.read()
log.debug("Server response\n" \
"Data: %s\n" % data)
command.process(utils.sdict(json.loads(data.decode("utf-8"))))
return command
def send_post(self, command):
data = command.data
if not data:
data = {}
data["token"] = Connection.get_message_token(data, Connection.SECRET_TOKEN)
log.debug("Server request\n" \
"Command: %s\n" \
"Data: %s\n" % (command.name, json.dumps(data)))
self.__connection.request("POST", command.name, json.dumps(data), {"Content-Type": "application/json"})
response = self.__connection.getresponse()
if response.status != 200:
raise RuntimeError("Invalid server response status %d: %s" % (response.status, response.reason))
data = response.read()
log.debug("Server response\n" \
"Data: %s\n" % data)
command.process(utils.sdict(json.loads(data.decode("utf-8"))))
return command
def login(self):
# Get login form
log.debug("Facebook: Fetching login form...")
response = FacebookRequest(self, FacebookContext.FACEBOOK_HOST, FacebookContext.FACEBOOK_LOGIN_URL).evaluate()
log.debug("Facebook: Parsing login form...")
# Find JS cookies
js_datr = re.search("\"_js_datr\",\\s*\"(.*?)\"", response.response)
js_gate = re.search("\"_js_reg_fb_gate\",\\s*\"(.*?)\"", response.response)
js_ref = re.search("\"_js_reg_fb_ref\",\\s*\"(.*?)\"", response.response)
if js_datr is not None:
self.set_cookie("_js_datr", js_datr.group(1))
if js_gate is not None:
self.set_cookie("_js_reg_fb_gate", js_gate.group(1))
if js_ref is not None:
self.set_cookie("_js_reg_fb_ref", js_ref.group(1))
# Parse login form
form = utils.html.get_tag(response, "form", {"id": "login_form"})
data = _FBLoginFormParser.parse(form)
data["email"] = self.user
data["pass"] = self.password
url = utils.html.get_tag_attr(form, "action")
# Try to sign in
log.debug("Facebook: Trying to sign in as \"%s\" via %s..." % (data["email"], FacebookContext.FACEBOOK_HOST + url))
response = FacebookRequest(self, FacebookContext.FACEBOOK_HOST, url, data).evaluate()
# If redirect - OK
if response.status == 302:
log.ok("Facebook: Signed in as \"%s\"" % data["email"])
self.logged = True
else:
raise RuntimeError("Facebook sign-in failed.")
payment_name = payment_names[i]
# Get payment config
payment = defs[payment_name]
if "store" in payment:
payment = payment["store"]
else:
continue
if "FB" in payment:
payment = payment["FB"]
else:
continue
log.debug("[%d%%] Checking tier %s..." % (round(100.0 * i / len(payment_names)), payment_name))
# Load the Scrawler's response
try:
result = debugger.check_url(urlparse.urljoin("http://%s" % Net.get_host(), payment["id"]))
except:
log_error("Error occurred for %s during attempt to get scraped data." % payment_name)
continue
if result["status"]["response-code"] != 200:
log_error("The Scrawler's response code is invalid: %d." % result["status"]["response-code"])
continue
# Parse prices
temp_prices = None
def evaluate(self):
# Resolve URL
url = "/" if self.url is None else self.url
# Form default headers
headers = {
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Encoding": "gzip, deflate",
"Accept-Language": "ru-RU,ru;q=0.8,en-US;q=0.5,en;q=0.3",
"Connection": "keep-alive",
"Host": self.host,
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:40.0) Gecko/20100101 Firefox/40.0"
}
# Set request referer
if self.context.referer is not None:
headers["Referer"] = self.context.referer
# Set request content
data = None
if self.data is not None:
if isinstance(self.data, dict):
headers["Content-Type"] = "application/x-www-form-urlencoded"
data = urllib.urlencode(self.data)
else:
headers["Content-Type"] = "application/octet-stream"
data = self.data
# Set request cookies
if self.context.cookies is not None:
cookies = []
for cookie in self.context.cookies:
tmp = {}
tmp[cookie[0]] = cookie[1]
cookies.append(urllib.urlencode(tmp))
if len(cookies) > 0:
headers["Cookie"] = "; ".join(cookies)
# Send request
log.debug("Facebook: Sending request to %s..." % (self.host + url))
connection = httplib.HTTPSConnection(host=self.host)
connection.request(
"GET" if self.data is None else "POST",
url,
data,
headers
)
response = connection.getresponse()
# Parse result
result = FacebookResponse(self.host, url, response.status, response.read(), response.msg.headers)
# Update cookies
for header in result.headers:
if header[0] == "set-cookie":
# Doesn't matter what is a cookie's lifetime and etc., just retrieve its value
cookie = header[1]
cookie = cookie.split(";", 1)
cookie = cookie[0].split("=")
self.context.set_cookie(cookie[0].strip(), cookie[1].strip())
# Check response status
if response.status != 200:
# Redirect
if response.status == 302:
log.warn("Facebook: %s requested redirect to %s." % (str(self.host + url), response.getheader("Location")))
if self.follow_redirects:
# Update referer
self.context.referer = "https://" + self.host + url
# Follow the redirect location
location = urlparse.urlparse(response.getheader("Location"))
return FacebookRequest(self.context, self.host if location[1] == "" else location[1], location[2], self.data, True).evaluate()
# Possibly, some error occurred
else:
raise RuntimeError("Invalid server response status %d: %s" % (response.status, response.reason))
# Update referer
log.debug("Facebook: Successful request to %s." % str(self.host + url))
self.context.referer = "https://" + self.host + url
# Return result
return result