def checkMantisBug(bugID):
source = utils.getURL("https://bugs.mtasa.com/print_bug_page.php?bug_id=" +
str(bugID))
if source is None:
return None, None, None
if source.find(
'<td class="form-title">APPLICATION ERROR #') != -1 or source.find(
"<center><p>Access Denied.</p>") != -1 or source.find(
'<center><h1>502 Bad Gateway</h1></center>') != -1:
return None, None, None
title_startPos = source.find("<title>") + 7
title_startPos = source.find(":", title_startPos) + 2
title_endPos = source.find("</title>", title_startPos) - 11
name = source[title_startPos:title_endPos]
severity_startPos = source.find(
'<td class="print-category">Severity</td><td class="print">') + 58
severity_endPos = source.find('</td>', severity_startPos)
severity = source[severity_startPos:severity_endPos]
status_startPos = source.find(
'<td class="print-category">Status</td><td class="print">') + 56
status_endPos = source.find('</td>', status_startPos)
status = source[status_startPos:status_endPos]
return name, severity, status
def cmd_gtao(chat):
data = utils.getURL(
'https://support.rockstargames.com/hc/en-us/articles/200426246-GTA-Online-Server-Status-Latest-Updates'
)
if data is None:
skype.sendMessageToChat(chat, "Strona R* leży.")
return
pcWarn_start = data.find('<div id="pcWarn">') + 17
pcWarn_end = data.find('</div>', pcWarn_start)
pcWarn = data[pcWarn_start:pcWarn_end]
rsgsUpOrDown_start = data.find(
'<div id="rsgsUpOrDown" data-rsgsupordown="', pcWarn_end) + 42
rsgsUpOrDown_end = data.find('"></div>', rsgsUpOrDown_start)
rsgsUpOrDown = data[rsgsUpOrDown_start:rsgsUpOrDown_end]
pcUpOrDown_start = data.find('<div id="pcUpOrDown" data-upordown="',
rsgsUpOrDown_end) + 36
pcUpOrDown_end = data.find('"></div>', pcUpOrDown_start)
pcUpOrDown = data[pcUpOrDown_start:pcUpOrDown_end]
txt = "Social Club: " + rsgsUpOrDown + "\nPC: " + pcUpOrDown
if pcWarn != "" and pcWarn != "no content":
txt = txt + "\nKomunikat: " + pcWarn
skype.sendMessageToChat(chat, txt)
def findEntityID ( name, chatName ):
with open("vehicles-sa.json") as f:
try:
data = json.loads( f.read() )
except Exception:
return
last = { "ratio": 0.0, "type": None, "name": None, "id": None }
processEntityGroup( data["catalog"]["groups"], name, last )
print("found " + str(last["name"]) + " (" + str(last["ratio"]) + ")")
if last["ratio"] < 0.4:
skype.sendMessageToChat( chatName, "Nic nie znalazłem." )
return
if last["type"] == "element":
txt = last["name"] + " - " + last["id"]
data = utils.getURL( "https://gta.wikia.com/api.php?action=query&list=search&format=json&prop=revisions&rvprop=content&srlimit=1&srwhat=nearmatch&srsearch=" + last["name"] )
if data:
data = json.loads(data)
if len(data["query"]["search"]) > 0:
title = data["query"]["search"][0]["title"].replace(" ", "_")
txt = txt + " | https://gta.wikia.com/" + title
skype.sendMessageToChat( chatName, txt )
elif last["type"] == "group":
printEntitiesInGroup( data["catalog"]["groups"], last["name"], chatName )
else:
skype.sendMessageToChat( chatName, "Nic nie znalazłem." )
def nocountRoute():
""" Return the count for a url """
url = utils.getURL(request)
if url is None:
return config.CANNOT_FIND_URL_MESSAGE, 404
connection = db_connection.get_connection()
count = db_connection.getCount(connection, url)
return makeTextRequest(count, url, False)
def nocountTagRoute():
""" Return svg of count """
url = utils.getURL(request)
if url is None:
return config.CANNOT_FIND_URL_MESSAGE, 404
connection = db_connection.get_connection()
count = db_connection.getCount(connection, url)
return makeSVGRequest(count, url, False, True)
def crawl(self, domain, limit=100):
visited = set() # initialise visited URLs
queue = deque([getURL(domain)]) # initialise queue
trie = Trie() # initialise trie
while(queue and len(visited) < limit):
link = queue.popleft() # next link
if link not in visited and self.inDomain(link, domain):
visited.add(link) # mark as seen
queue.extend(getLinksOnPage(link)) # visit new links later
trie.insert(self.formatURL(link)) # put in trie hierarchy
return trie
def parseJobPosting(url):
"""
return: jobkey[string], position[string], company[string],
location[string], words[list of strings]
params:
url: string | url for the job posting to parse
"""
# extract raw data from URL and remove returns
raw = utils.getURL(url)
raw = raw.replace('\n',' ')
# retrieve the jobkey from the url
jobkey = re.search(r'jk=\w+&', url).group().replace('jk=','').replace('&','')
# extract job position
try:
position = re.search(r'<title>.*</title>', raw).group()
# position = re.search(r'>\w.*\-.*\-.*\|', position).group()
position = re.search(r'>\w.*\-.*\|', position).group()
position = position.replace('>','').replace('|','').split('-')[0].replace('job','').strip()
except AttributeError:
position = 'Unknown'
# retrieve job location
try:
location = re.search(r'<span class="location">.*?<span class="summary">', raw).group()
location = re.search(r'<span class="location">.*?</span>', location).group()
location = location.replace('<span class="location">','').replace('</span>','')
except AttributeError:
location = 'Unknown'
# receive job's company
try:
company = re.search(r'<span class="company">.*?<span class="summary">', raw).group()
company = company.split('</span>')[0].split('>')[-1]
except AttributeError:
company = 'Unknown'
# retrieve the job description section
try:
start = 'span class="summary"'
end = 'days ago'
end = '<span class="sdn">'#+company
#jd = re.search(r''+start+'.*'+end+'.*'+'days ago', raw,re.IGNORECASE).group()
jd = re.search(r''+start+'.*?'+end, raw).group()
jd = jd.replace('span class="summary"','').replace('<span class="sdn">','')
jd = jd.replace('<span class="date">',' ').replace('days ago',' ')
except AttributeError:
jd = ''
# more advanced processing/cleaning of the job description
words = jdClean(jd).split() # list of words
return jobkey, position, company, location, words
def countTagRoute():
""" Return svg of count and add 1 to url """
url = utils.getURL(request)
if url is None:
return config.CANNOT_FIND_URL_MESSAGE, 404
valid_cookie = utils.checkValidCookie(request, url)
connection = db_connection.get_connection()
if not valid_cookie:
db_connection.addView(connection, url)
count = db_connection.getCount(connection, url)
return makeSVGRequest(count, url, not valid_cookie, False)
def nocountTagRoute():
""" Return svg of count """
url = utils.getURL(request)
if url is None:
return config.CANNOT_FIND_URL_MESSAGE, 404
if not utils.checkURLWhitelist(url):
return config.FORBIDDEN_URL_MESSAGE, 403
connection = db_connection.get_connection()
count = db_connection.getCount(connection, url)
return makeSVGRequest(count, url, False)
def countRoute():
""" Return the count for a url and add 1 to it """
# Attempt to find any sign of a url, return 404 if we can't find anything
url = utils.getURL(request)
if url is None:
return config.CANNOT_FIND_URL_MESSAGE, 404
# Get/generate cookie, cleanup views, add a view, get the count and commit changes
valid_cookie = utils.checkValidCookie(request, url)
connection = db_connection.get_connection()
if not valid_cookie:
db_connection.addView(connection, url)
count = db_connection.getCount(connection, url)
return makeTextRequest(count, url, not valid_cookie)
def nocountRoute():
""" Return the count for a url and add 1 to it """
# Attempt to find any sign of a url, return 404 if we can't find anything
url = utils.getURL(request)
if not url.startswith("www.jamesleighton.com"):
return config.CANNOT_FIND_URL_MESSAGE, 404
if url is None:
return config.CANNOT_FIND_URL_MESSAGE, 404
connection = db_connection.get_connection()
count = db_connection.getCount(connection, url)
return makeTextRequest(count, url, False)
def cmd_ets(chat):
# servers status
data = utils.getJSON('https://api.ets2mp.com/servers/')
if data is None:
return
if data["error"] != "false":
return
txt = "Serwery TruckersMP:"
total_players = 0
total_slots = 0
for server in data["response"]:
txt = txt + "\n" + server["game"] + " " + server["name"]
if server["online"]:
txt = txt + " (" + str(server["players"]) + "/" + str(
server["maxplayers"]) + ")"
total_players = total_players + server["players"]
total_slots = total_slots + server["maxplayers"]
else:
txt = txt + " (offline)"
txt = txt + "\nŁącznie " + str(total_players) + "/" + str(
total_slots) + " graczy."
# game time
data = utils.getJSON('https://api.ets2mp.com/game_time/')
if data is not None:
if data["error"]:
return
gameTime = datetime.datetime(
2015, 10, 25) + datetime.timedelta(minutes=data["game_time"])
txt = txt + "\nCzas w grze: " + gameTime.strftime('%H:%M')
# song at TruckersFM
data = utils.getURL('http://truckers.fm/')
if data is not None:
song_start = data.find(
'<span id="currently_playing"> <span id="song"><span class="song-details">'
) + 73
if song_start != 72:
song_end = data.find('</span>', song_start)
txt = txt + "\nTruckersFM: " + data[song_start:song_end]
skype.sendMessageToChat(chat, txt)
def countTagRoute():
""" Return the count for a url and add 1 to it """
# Attempt to find any sign of a url, return 404 if we can't find anything
url = utils.getURL(request)
if not url.startswith("www.jamesleighton.com"):
return config.CANNOT_FIND_URL_MESSAGE, 404
if url is None:
return config.CANNOT_FIND_URL_MESSAGE, 404
valid_cookie = utils.checkValidCookie(request, url)
connection = db_connection.get_connection()
if not valid_cookie:
db_connection.addView(connection, url)
count = db_connection.getCount(connection, url)
return makeSVGRequest(count, url, not valid_cookie)
def getJobURLs(jobQuery, nURLs=1, start=0):
"""
return: list of strings (each string is a URL to an
Indeed.com job posting for "jobQuery")
params:
jobQuery: string | search terms for Indeed.com
(preprocessed for +'s rather than spaces, etc.)
nURLs: int | number of job posting URL's to return
start: int | beginning index for api job search
"""
# pre-process job query
jobQuery = jobQuery.strip().lower().replace(' ','+')
# list for all URL's to be stored
allurls = []
# loop through each page of 20 job postings for jobQuery
for i in range(nURLs//10+1):
print 'url retrieval =', len(allurls)/float(nURLs)*100.0, 'percent complete'
# api link for 10 postings for jobQuery at a time
api = 'http://api.indeed.com/ads/apisearch?publisher=6973678184764538&v=2'
api += '&q=\"'+jobQuery+'\"&start='+str(start + i*10)
# get the content from api URL
raw = utils.getURL(api)
# parse the raw data for individual job URL's
urls = re.findall(r'<url>.*</url>', raw)
# remove the <link> and </link> tags from URL's
urls = [u.replace('<url>','').replace('</url>','') for u in urls]
# append current page of URL's to list of "all" URL's
allurls += urls
# return only up to nURLs number of URL's for job postings
return allurls[:nURLs]
def OnMessageStatus(message, status):
if status == 'RECEIVED' or status == 'SENT':
if (message.Chat.Name == skype.g_chats["mta"] or message.Chat.Name
== skype.g_chats["test"]) and message.Body.find('#') != -1:
bugID_start = message.Body.find('#') + 1
bugID_end = message.Body.find(' ', bugID_start)
if bugID_end == -1:
bugID_end = len(message.Body)
bugID = message.Body[bugID_start:bugID_end]
name, severity, status = notifications.checkMantisBug(bugID)
if name is None:
return
skype.sendMessageToChat(
message.Chat, "[" + severity + "/" + status + "] " + name +
"\nhttps://bugs.mtasa.com/view.php?id=" + bugID)
return
if status == 'RECEIVED':
for command in g_commands:
#if message.Body.find(command.name) == 0:
if message.Body.split()[0] == command.name:
command.process(message.Body, message.Chat)
return
# other stuff
if message.Body.find('v=') >= 0 or message.Body.find('youtu.be/') >= 0:
link_start = message.Body.find('youtu.be/') + 9
if link_start - 9 == -1:
link_start = message.Body.find('v=') + 2
link_end = message.Body.find('&', link_start)
if link_end == -1:
link_end = message.Body.find(' ', link_start)
if link_end == -1:
link_end = len(message.Body)
vidID = message.Body[link_start:link_end]
data = utils.getJSON(
"https://www.googleapis.com/youtube/v3/videos?part=snippet,contentDetails&key=AIzaSyDViQNqCB7CxTiqS5YiBogXVBykLUtrUmY&id="
+ vidID)
if data is None:
return
if len(data["items"]) > 0:
title = data["items"][0]["snippet"]["title"]
skype.sendMessageToChat(message.Chat, 'YT: ' + title)
return
if message.Body.find('lenny') != -1:
skype.sendMessageToChat(message.Chat, "( ͡° ͜ʖ ͡°)")
return
if message.Body.find(
'community.mtasa.com/index.php?p=resources&s=details&id='
) >= 0:
link_start = message.Body.find(
'community.mtasa.com/index.php?set_lang=eng&p=resources&s=details&id='
) + 55
link_end = message.Body.find(' ', link_start)
if link_end == -1:
link_end = len(message.Body)
if link_start >= 0:
source = utils.getURL(
"http://community.mtasa.com/index.php?p=resources&s=details&id="
+ message.Body[link_start:link_end])
if source is None:
return
title_start = source.find('">rss</a></span>') + 16
title_end = source.find('</h2>', title_start) - 3
title = source[title_start:title_end]
if title_start - 16 == -1:
return
author_start = source.find(
'<tr><th>Author:</th><td><a href="?p=profile&id=') + 51
author_start = source.find('">', author_start) + 2
author_end = source.find('</a>', author_start)
author = source[author_start:author_end]
downloads_start = source.find(
'<tr><th>Downloads:</th><td>') + 27
downloads_end = source.find('</td>', downloads_start)
downloads = source[downloads_start:downloads_end]
skype.sendMessageToChat(
message.Chat, 'Community: ' + title + " @ " + author +
" (" + downloads + " pobrań)")
return
if message.Body.find('store.steampowered.com/app/') >= 0:
link_start = message.Body.find('store.steampowered.com/app/') + 27
link_end1 = message.Body.find('/', link_start)
link_end2 = message.Body.find(' ', link_start)
link_end = link_end1
if (link_end2 < link_end and link_end2 != -1) or link_end == -1:
link_end = link_end2
if link_end == -1:
link_end = len(message.Body)
appID = message.Body[link_start:link_end]
data = utils.getJSON(
'http://store.steampowered.com/api/appdetails?appids=' + appID)
if data is None:
return
if not data[appID]["success"]:
return
data = data[appID]["data"]
txt = "Steam: " + data["name"]
if data["is_free"]:
txt = txt + " (darmowe)"
else:
price1 = str(data["price_overview"]["final"])[:-2]
if price1 == "":
price1 = "0"
price2 = str(data["price_overview"]["final"])[-2:]
if data["price_overview"]["currency"] == "EUR":
currency = "€"
else:
currency = "$"
txt = txt + " " + price1 + "." + price2 + currency
if data["price_overview"]["discount_percent"] > 0:
txt = txt + " [-" + str(
data["price_overview"]["discount_percent"]) + "%]"
message.Chat.SendMessage(txt)
return
def testGetURLreturnsNoneForSyntaticallyIncorrectURL():
url = getURL('not_a_url')
assert url is None
def testGetURLreturnsNoneForNonExistantWebsite():
url = getURL('https://useacard.com')
assert url is None
def testGetURLreturnsNoneForURLnotPointingToPage():
url = getURL('mailto:[email protected]')
assert url is None
def testGetURLreturnsValidURLforValidURL():
GO_CARDLESS = 'https://gocardless.com'
url = getURL(GO_CARDLESS)
assert url == GO_CARDLESS
