def start_requests(self):
self.urls_to_crawl = self.generate_queries_urls()
from random import shuffle
# So that we do not always query in the same order, making it less likely for us to be banned
shuffle(self.urls_to_crawl)
self.set_of_urls_should_be_crawled = set(self.urls_to_crawl)
print "----------------------------------------------------------------------------------"
print "There are", len(self.urls_to_crawl), " SERP URLs to be crawled."
print "----------------------------------------------------------------------------------"
if not self.use_proxies:
print "NOT USING PROXIES"
if self.tor_port is not None:
print "USING TOR ON PORT", self.tor_port
# No use of proxies, fallback to parent's implementation and forward result
self.start_urls = self.urls_to_crawl[:]
self.urls_to_crawl = [] # all urls have been transferred to start_urls and will thus be in queue, this starting queue should thus be empty
for req in super(SERPSpider, self).start_requests():
if self.tor_port is not None:
req.meta['proxy'] = self.tor_proxy_host
yield req
else:
print "USING PROXIES"
l("Proxies list is", self.PROXIES)
# Let us shuffle the proxy list so that we do not always do the same queries on the same proxies at start
proxy_list_tmp = list(self.PROXIES)
shuffle(proxy_list_tmp)
self.PROXIES = set(proxy_list_tmp)
self.schedule_requests_injection()
# Generate a first request for every proxy (subsequent requests will be generated when parsing responses)
for p in self.PROXIES:
proxy = 'http://' + p
self.confirmed_proxies[proxy] = False
yield self.build_request_for_proxy(proxy, time()) # Directly executing time() here so that if there is
async def _skip(self, ctx):
if not ctx.player.current:
return await ctx.send(l(ctx, 'errors.notPlaying', {"author": ctx.author.name,
"emoji": self.disco.emoji["false"]}))
track = ctx.player.current
if track.requester.id == ctx.author.id:
await ctx.player.send(l(ctx, 'commands.skip.skippedByRequester', {"track": track,
"emoji": self.disco.emoji["alert"], "author": ctx.author.name}))
return await ctx.player.stop()
elif ctx.author.id in track.skip_votes:
return await ctx.send(l(ctx, 'commands.skip.alreadyVoted', {
"emoji": self.disco.emoji["false"], "author": ctx.author.name}))
track.skip_votes.add(ctx.author.id)
await ctx.player.send(l(ctx, 'commands.skip.voteAdded', {"author": ctx.author.name,
"emoji": self.disco.emoji["alert"], "votes": len(track.skip_votes)}))
if len(track.skip_votes) == 3:
await ctx.player.send(l(ctx, 'commands.skip.skipped', {"track": track,
"emoji": self.disco.emoji["alert"]}))
await ctx.player.stop()
def inject_not_crawled_urls(self):
print "inject_not_crawled_urls()..."
not_crawled = self.get_not_crawled_urls()
if not_crawled:
# Some urls have not been crawled
l("FORCE_COMPLETENESS=True and there are urls that have not been crawled, pushing them back \
into self.urls_to_crawl of urls to be crawled.\
")
l("There are currently", len(not_crawled), "not crawled urls.")
not_crawled_and_not_scheduled = not_crawled - set(self.urls_to_crawl)
self.urls_to_crawl.extend(not_crawled_and_not_scheduled)
i = 0
if self.use_proxies and len(self.pending_requests) != 0:
# There are still pending request, do not deal with anything, it is handled by the query scheduling of
# the proxy mode
return
elif self.use_proxies:
usable_proxies = [p for p, c in self.confirmed_proxies.items() if c]
now = time()
# Let us schedule a new query for every usable proxy, the rest of the queries
# will be scheduled when we received responses from those queries
for proxy in usable_proxies:
r = self.next_request(proxy, now)
self.inject_request(r)
else:
# When not using proxy, the delay is respected directly by Scrapy
# so we just schedule everything and it will take care of the rest
for _ in xrange(len(self.urls_to_crawl)):
if self.tor_proxy_host is not None:
proxy = self.tor_proxy_host
else:
proxy = None
r = self.build_request_for_proxy(proxy, time())
self.inject_request(r)
i += 1
async def _reverse(self, ctx):
if not ctx.player.queue:
return await ctx.send(l(ctx, 'errors.emptyQueue', {"author": ctx.author.name,
"emoji": self.disco.emoji["false"]}))
ctx.player.queue.reverse()
await ctx.player.send(l(ctx, 'commands.reverse.success', {"author": ctx.author.name,
"emoji": self.disco.emoji["shuffle"]}))
def bann_player(self, pid):
l("Player", pid, "is being banned.")
self.banned_players.append(pid)
while self.current_player_thread.is_alive():
d("Killing it!")
terminate_thread(self.current_player_thread)
self.current_player_thread.join(0.01) # Wait ten ms between each kill attempt
l("Banned players:", self.banned_players)
async def _clear(self, ctx):
if not ctx.player.queue:
return await ctx.send(l(ctx, 'commands.clear.alreadyEmpty', {
"emoji": self.disco.emoji["false"], "author": ctx.author.name}))
ctx.player.queue.clear()
await ctx.player.send(l(ctx, 'commands.clear.cleaned', {"author": ctx.author.name,
"emoji": self.disco.emoji["alert"]}))
async def _volume(self, ctx, vol: int):
if not 0 < vol < 151:
return await ctx.send(l(ctx, 'commands.volume.invalidValue', {
"emoji": self.disco.emoji["false"], "author": ctx.author.name}))
await ctx.player.set_volume(vol)
await ctx.player.send(l(ctx, 'commands.volume.changed', {"author": ctx.author.name,
"emoji": self.disco.emoji["volume"], "value": vol}))
async def _force_skip(self, ctx):
if not ctx.player.current:
return await ctx.send(l(ctx, 'errors.notPlaying', {"author": ctx.author.name,
"emoji": self.disco.emoji["false"]}))
await ctx.player.send(l(ctx, 'commands.forceskip.skipped', {"track": ctx.player.current,
"emoji": self.disco.emoji["alert"], "author": ctx.author.name}))
await ctx.player.stop()
def schedule_requests_injection(self):
print "schedule_requests_injection()..."
if not self.closed: # Another check won't harm anyone
l("Scheduling requests injection")
self.requests_injection_timer = threading.Timer(
self.REQUESTS_INJECTION_TIMER_INTERVAL,
self.inject_requests
)
self.requests_injection_timer.start()
def retry_on_confirmed(self, active_proxies_count):
# There are too few active proxies, let us retry on all the confirmed proxies
now = time()
if active_proxies_count <= self.ACTIVE_PROXIES_THRESHOLD_FOR_RETRIES_ON_CONFIRMED \
and self.last_retry_on_confirmed <= (now - self.RETRY_ON_CONFIRMED_MIN_INTERVAL_IN_SEC):
for p, c in self.confirmed_proxies.items():
if c is True and p: # Yes, no check on inactive_proxies_retry_in_progress, we want to retry them all!
req = self.build_request_for_proxy(p, now)
l("Injecting URL", req.url, "to retry on deeply inactive proxy", p)
self.inject_request(req)
self.last_retry_on_confirmed = now
async def _bot_channel(self, ctx, *, channel: discord.TextChannel = None):
if not channel:
if not ctx._guild.data['options']['botChannel']:
raise commands.UserInputError()
ctx._guild.update({"options.botChannel": None})
return await ctx.send(l(ctx, 'commands.botchannel.reset', {"author": ctx.author.name,
"emoji": self.disco.emoji["true"]}))
ctx._guild.update({"options.botChannel": channel.id})
await ctx.send(l(ctx, 'commands.botchannel.set', {"author": ctx.author.name,
"emoji": self.disco.emoji["true"], "channel": channel.mention}))
async def _dj_role(self, ctx, *, role: discord.Role = None):
if not role:
if not ctx._guild.data['options']['djRole']:
raise commands.UserInputError()
ctx._guild.update({"options.djRole": None})
return await ctx.send(l(ctx, 'commands.djrole.reset', {"author": ctx.author.name,
"emoji": self.disco.emoji["true"]}))
ctx._guild.update({"options.djRole": role.id})
await ctx.send(l(ctx, 'commands.djrole.update', {"author": ctx.author.name,
"emoji": self.disco.emoji["true"], "role": role}))
async def _disable_channel(self, ctx, *, channel: discord.TextChannel = None):
if not channel:
channel = ctx.channel
if channel.id in ctx._guild.data['options']['disabledChannels']:
ctx._guild.remove({"options.disabledChannels": channel.id})
await ctx.send(l(ctx, 'commands.disablechannel.allow', {"author": ctx.author.name,
"emoji": self.disco.emoji["true"], "channel": channel.mention}))
else:
ctx._guild.insert({"options.disabledChannels": channel.id})
await ctx.send(l(ctx, 'commands.disablechannel.disallow', {"author": ctx.author.name,
"emoji": self.disco.emoji["true"], "channel": channel.mention}))
async def _pause(self, ctx):
if not ctx.player.current:
return await ctx.send(l(ctx, 'errors.notPlaying', {"author": ctx.author.name,
"emoji": self.disco.emoji["false"]}))
if ctx.player.paused:
await ctx.player.send(l(ctx, 'commands.pause.unpause', {"author": ctx.author.name,
"emoji": self.disco.emoji["pause"]}))
else:
await ctx.player.send(l(ctx, 'commands.pause.pause', {"author": ctx.author.name,
"emoji": self.disco.emoji["pause"]}))
await ctx.player.set_pause(not ctx.player.paused)
async def _remove(self, ctx, index: int):
if not ctx.player.queue:
return await ctx.send(l(ctx, 'errors.emptyQueue', {"author": ctx.author.name,
"emoji": self.disco.emoji["false"]}))
try:
track = ctx.player.queue.pop(index - 1)
except IndexError:
return await ctx.send(l(ctx, 'errors.invalidValue', {"author": ctx.author.name,
"emoji": self.disco.emoji["false"]}))
await ctx.send(l(ctx, 'commands.remove.removed', {"author": ctx.author.name,
"emoji": self.disco.emoji["true"], "track": track}))
async def _disable_role(self, ctx, *, role: discord.Role):
if role >= ctx.author.top_role:
return await ctx.send(l(ctx, 'commands.disablerole.higherRole', {
"author": ctx.author.name, "emoji": self.disco.emoji["false"]}))
if role.id in ctx._guild.data['options']['disabledRoles']:
ctx._guild.remove({"options.disabledRoles": role.id})
await ctx.send(l(ctx, 'commands.disablerole.enabled', {"role": role,
"emoji": self.disco.emoji["true"], "author": ctx.author.name}))
else:
ctx._guild.insert({"options.disabledRoles": role.id})
await ctx.send(l(ctx, 'commands.disablerole.disabled', {"role": role,
"emoji": self.disco.emoji["true"], "author": ctx.author.name}))
async def _repeat(self, ctx):
if not ctx.player.current:
return await ctx.send(l(ctx, 'errors.notPlaying', {"author": ctx.author.name,
"emoji": self.disco.emoji["false"]}))
if ctx.player.repeat:
ctx.player.repeat = None
await ctx.player.send(l(ctx, 'commands.repeat.disable', {"author": ctx.author.name,
"emoji": self.disco.emoji["repeatOne"], "track": ctx.player.current}))
else:
ctx.player.repeat = ctx.player.current
await ctx.player.send(l(ctx, 'commands.repeat.enable', {"author": ctx.author.name,
"emoji": self.disco.emoji["repeatOne"], "track": ctx.player.current}))
async def _local_ban(self, ctx, *, member: discord.Member):
if member.top_role >= ctx.author.top_role:
return await ctx.send(l(ctx, 'commands.localban.higherMember', {
"emoji": self.disco.emoji["false"], "author": ctx.author.name}))
if member.id in ctx._guild.data['options']['bannedMembers']:
ctx._guild.remove({"options.bannedMembers": member.id})
await ctx.send(l(ctx, 'commands.localban.unban', {"emoji": self.disco.emoji["true"],
"author": ctx.author.name, "member": member}))
else:
ctx._guild.insert({"options.bannedMembers": member.id})
await ctx.send(l(ctx, 'commands.localban.ban', {"emoji": self.disco.emoji["true"],
"author": ctx.author.name, "member": member}))
async def _now_playing(self, ctx):
player = self.get_player(ctx.guild.id)
if not player.current:
return await ctx.send(l(ctx, 'errors.notPlaying', {"author": ctx.author.name,
"emoji": self.disco.emoji["false"]}))
track = player.current
em = discord.Embed(
colour=self.disco.color[0],
description=l(ctx, 'commands.nowplaying.text', {"track": track,
"length": get_length(track.length)})
)
await ctx.send(embed=em, delete_after=15)
def spider_idle(self, spider):
l("spider_idle signal received")
if spider is self:
dont_close = False
if self.FORCE_COMPLETENESS:
self.inject_not_crawled_urls()
dont_close = True
if len(self.pending_requests) is not 0 or len(self.urls_to_crawl) is not 0:
l("Hey, do not close me, I still have things to do! (", len(self.pending_requests), "pending requests)",
"and still ", len(self.urls_to_crawl), "urls to be crawled.")
dont_close = True
if dont_close:
raise DontCloseSpider()
async def _bass_boost(self, ctx):
if not ctx.player.current:
return await ctx.send(l(ctx, 'errors.notPlaying', {"author": ctx.author.name,
"emoji": self.disco.emoji["false"]}))
if ctx.player.bass_boost:
await ctx.player.set_preq('flat')
await ctx.player.send(l(ctx, 'commands.bassboost.disabled', {
"emoji": self.disco.emoji["volume"], "author": ctx.author.name}))
else:
await ctx.player.set_preq('boost')
await ctx.player.send(l(ctx, 'commands.bassboost.enabled', {
"emoji": self.disco.emoji["volume"], "author": ctx.author.name}))
ctx.player.bass_boost = not ctx.player.bass_boost
async def _play_at(self, ctx, index: int):
player = ctx.player
if not player.queue:
return await ctx.send(l(ctx, 'errors.emptyQueue', {"author": ctx.author.name,
"emoji": self.disco.emoji["false"]}))
try:
player.queue = player.queue[index - 1:]
track = player.queue.pop(0)
except IndexError:
return await ctx.send(l(ctx, 'errors.invalidValue', {"author": ctx.author.name,
"emoji": self.disco.emoji["false"]}))
player.queue.insert(0, track)
await player.stop()
async def _donate(self, ctx):
await ctx.send(
l(
ctx, 'commands.donate.text', {
"emoji": self.disco.emoji["featured"],
"link": "https://patreon.com/discobot"
}))
async def _default_volume(self, ctx, vol: int = None):
if not vol:
if not ctx._guild.data['options']['defaultVolume']:
raise commands.UserInputError
ctx._guild.update({"options.defaultVolume": None})
return await ctx.send(l(ctx, 'commands.defaultvolume.reset', {"author": ctx.author.name,
"emoji": self.disco.emoji["true"]}))
if not 0 < vol < 151:
return await ctx.send(l(ctx, 'commands.defaultvolume.invalid', {"author": ctx.author.name,
"emoji": self.disco.emoji["false"]}))
ctx._guild.update({"options.defaultVolume": vol})
await ctx.send(l(ctx, 'commands.defaultvolume.success', {"author": ctx.author.name,
"emoji": self.disco.emoji["true"], "volume": vol}))
async def _disable_command(self, ctx, command):
command = self.disco.get_command(command)
if not command:
raise commands.UserInputError()
if command.cog_name in ['Admin', 'Owner']:
return await ctx.send(l(ctx, 'commands.disablecommand.cantDisable', {
"emoji": self.disco.emoji["false"], "author": ctx.author.name}))
if command.name in ctx._guild.data['options']['disabledCommands']:
ctx._guild.remove({"options.disabledCommands": command.name})
await ctx.send(l(ctx, 'commands.disablecommand.enabled', {"command": command.name,
"emoji": self.disco.emoji["true"], "author": ctx.author.name}))
else:
ctx._guild.insert({"options.disabledCommands": command.name})
await ctx.send(l(ctx, 'commands.disablecommand.disabled', {"command": command.name,
"emoji": self.disco.emoji["true"], "author": ctx.author.name}))
async def _invite(self, ctx):
em = discord.Embed(
colour=self.disco.color[1],
description=l(ctx, 'commands.invite.text',
{"link": "https://discobot.site"})).set_author(
name=ctx.me.name, icon_url=ctx.me.avatar_url)
await ctx.send(content=ctx.author.mention, embed=em)
async def on_command_completion(self, ctx):
if ctx.command.name != 'donate' and randint(1, 7) == 1:
await ctx.send(
l(
ctx.locale, 'commands.donate.text', {
"emoji": self.disco.emoji["featured"],
"link": "https://patreon.com/discobot"
}))
async def on_track_event(self, event):
player = event.player
if player.repeat:
track = player.repeat
elif player.size:
track = player.queue.pop(0)
else:
player.current = None
return await player.send(l(player, 'events.queueEnd', {
"emoji": self.disco.emoji["alert"]}))
await player.play(track)
self.disco.played_tracks += 1
if not player.repeat:
await player.send(l(player, 'events.trackStart', {"track": track,
"emoji": self.disco.emoji["download"], "length": get_length(track.length)}))
async def _whats_my_prefix(self, ctx):
command = l(ctx, 'commons.command')
custom_prefix = ctx._guild.data['options']['prefix']
prefixes = [
*([custom_prefix] if custom_prefix else self.disco.prefixes)
]
await ctx.send(
l(
ctx, 'commands.whatsmyprefix.message', {
"author":
ctx.author.name,
"prefixes":
' | '.join(f'`{prefix}<{command}>`'
for prefix in prefixes),
"emoji":
self.disco.emoji["alert"]
}))
async def _locale(self, ctx, locale = None):
locales = [l for l in listdir('./locales') if LOCALE.match(l)]
if not locale or locale not in locales:
available = ', '.join([f"**`{l}`**" for l in locales])
return await ctx.send(l(ctx, 'commands.locale.invalid', {"available": available,
"emoji": self.disco.emoji["false"], "author": ctx.author.name}))
ctx.locale = locale
ctx._guild.update({"options.locale": locale})
await ctx.send(l(locale, 'commands.locale.success', {"locale": locale,
"emoji": self.disco.emoji["true"], "author": ctx.author.name}))
try:
self.disco.wavelink.players[ctx.guild.id].locale = locale
except KeyError:
pass
def count_active_proxies_and_retry_on_inactive(self):
now = time()
five_minutes_ago = now - 5 * 60
inactive_allowance_ago = now - self.PROXIES_INACTIVE_ALLOWANCE_FOR_RETRY_IN_MINUTES * 60
active_proxies_count = 0
# Note: As we are looping through the "alived proxies" we will not go through the unconfirmed proxies
# as an alive proxy implies the proxy has been confirmed, too
for p, t in self.alived_proxies.items():
if t >= five_minutes_ago:
active_proxies_count += 1
# For all proxies that have been alive during the last hour but inactive for >= five minutes
# it is probably that the proxy was not working properly at this time but we can retry it now, who knows?
if p not in self.inactive_proxies_retry_in_progress and t <= five_minutes_ago and t >= inactive_allowance_ago:
req = self.build_request_for_proxy(p, now)
l("Injecting URL", req.url, "to retry on inactive proxy", p)
self.inactive_proxies_retry_in_progress.add(p)
self.inject_request(req)
return active_proxies_count
async def _set_prefix(self, ctx, prefix = None):
if not prefix:
if not ctx._guild.data['options']['prefix']:
raise commands.UserInputError
ctx._guild.update({"options.prefix": None})
self.disco._prefixes[ctx.guild.id] = None
return await ctx.send(l(ctx, 'commands.prefix.reset', {"author": ctx.author.name,
"emoji": self.disco.emoji["true"]}))
if len(prefix) > 7:
return await ctx.send(l(ctx, 'commands.prefix.invalid', {"author": ctx.author.name,
"emoji": self.disco.emoji["false"]}))
ctx._guild.update({"options.prefix": prefix})
self.disco._prefixes[ctx.guild.id] = prefix
await ctx.send(l(ctx, 'commands.prefix.success', {"author": ctx.author.name,
"emoji": self.disco.emoji["true"], "prefix": prefix}))
async def _bot_info(self, ctx):
shard_ping = int(self.disco.shards[ctx.guild.shard_id].ws.latency *
1000)
uptime = get_length(
(datetime.utcnow() - self.disco.started_at).total_seconds() * 1000,
True)
em = discord.Embed(
colour=self.disco.color[0],
title=l(ctx, 'commands.botinfo.statistics'),
description=l(
ctx, 'commands.botinfo.links', {
"support": "https://discord.gg/qN5886E",
"invite": "https://discobot.site",
"donate": "https://patreon.com/discobot",
"vote": "https://botsparadiscord.xyz/bots/disco",
"github": "https://github.com/Naegin/Disco"
})
).set_author(name=ctx.me.name, icon_url=ctx.me.avatar_url).set_footer(
text=l(ctx, 'commons.createdBy', {"creator": "Naegin#0049"}),
icon_url='https://cdn.naeg.in/i/naegin-avatar.gif').set_thumbnail(
url=ctx.me.avatar_url).add_field(
name=l(ctx, 'commands.botinfo.generalInfoTitle'),
value=l(
ctx, 'commands.botinfo.generalInfoDescLeft', {
"shard": ctx.guild.shard_id + 1,
"shards": len(self.disco.shards),
"ping": shard_ping,
"servers": len(self.disco.guilds),
"members": len(set(self.disco.get_all_members())),
"players": len(self.disco.wavelink.players),
"nodes": len(self.disco.wavelink.nodes)
})).add_field(
name='\u200b',
value=l(
ctx, 'commands.botinfo.generalInfoDescRight', {
"uptime": uptime,
"messages":
f'{self.disco.read_messages:,}',
"commands":
f'{self.disco.invoked_commands:,}',
"played": f'{self.disco.played_tracks:,}'
}))
for identifier, node in self.disco.wavelink.nodes.items():
stats = node.stats
em.add_field(
name=f'**LAVALINK NODE {identifier}**',
value=l(
ctx, 'commands.botinfo.nodeInfo', {
"region": node.region.title().replace("_", " "),
"uptime": get_length(stats.uptime, True),
"stats": stats,
"memory": naturalsize(stats.memory_used)
}))
await ctx.send(content=ctx.author.mention, embed=em)
def inject_requests(self):
print "inject_requests()"
if self.closed:
print "But spider is closed!"
return
try:
now = time()
# Look for pending requests that would now have the right to be processed:
for url, r in self.pending_requests.items():
if r.meta['exec_time'] <= now:
l("Request to URL=", r.url, "can now be processed")
r.meta['exec_time'] = now
self.pending_requests.pop(url) # Remove it from the pending requests
self.inject_request(r)
active_proxies_count = self.count_active_proxies_and_retry_on_inactive()
self.retry_on_confirmed(active_proxies_count)
finally: # No matter what happens, we have to schedule ourselves again else the crawler will hang forever
print "Scheduling ourselves again..."
self.schedule_requests_injection()
async def _queue(self, ctx, page: int = 1):
player = self.get_player(ctx.guild.id)
if not player.queue:
return await ctx.send(l(ctx, 'errors.emptyQueue', {"author": ctx.author.name,
"emoji": self.disco.emoji["false"]}))
length = get_length(sum(track.length for track in player.queue), True)
per_page = 10
pages = ceil(player.size / per_page)
if not 0 < page <= pages:
page = 1
skip = (page - 1) * per_page
current = player.current
tracks = player.queue[skip:skip+per_page]
txt = l(ctx, 'commands.queue.currentTrack', {"track": current,
"length": get_length(current.length)})
for i, t in enumerate(tracks, skip+1):
txt += f'**`»`** `{i}` [**{t}**]({t.uri}) `[{get_length(t.length)}]` - {t.requester.mention}\n'
em = discord.Embed(
colour=self.disco.color[1],
title=l(ctx, 'commands.queue.name'),
description=txt
).set_author(
name=ctx.guild.name,
icon_url=ctx.guild.icon_url
).set_thumbnail(
url=ctx.me.avatar_url
).set_footer(
text=l(ctx, 'commands.queue.details', {"length": length,
"page": page, "pages": pages})
)
await ctx.send(content=ctx.author.mention, embed=em)
def k_fold_cross_validation(self):
lbda1, alpha1, lbda2, alpha2 = self.param_
resu = []
for i in range(self.n_splits):
sgl_i = SGL(self.y_train[i], self.X_train[i], self.sgl.knots,
self.sgl.u, lbda1, alpha1, lbda2, alpha2)
K, p = self.sgl.knots.shape
cf = np.ones((K - 1) * p + 1)
init_coefs = np.concatenate((-cf, cf))
sgl_i.initialize_coefficient(init_coefs)
gamma_i, sigma_i = sgl_i.fit_predict(self.X_test[i])
resu.append(np.mean(l(self.y_test[i], gamma_i, sigma_i)))
resu = np.asarray(resu)
return np.mean(resu)
def next_request(self, proxy_param, exec_time_param):
now = time()
if self.use_proxies:
# If there are still some URLs to be loaded with the same proxy
if len(self.urls_to_crawl) > 0:
# We load the next URL that has been assigned to the same proxy as the one used for the request
# that triggered this reply
# But first, let us compute whether this next request should be executed right now or not (in case
# the proxy would have been exceptionally faster than the between-requests delay)
delta = now - exec_time_param
l("delta=", delta)
if delta < PROXY_SPECIFIC_REQUEST_DELAY:
l("Adding a delay to respect between-requests delay for proxy", proxy_param)
# The difference between the required delta and the current delta
# Note: Randomness is in order not to be detected as bot by having a predictable behaviour
r = randu(0.5, 1.5)
l("Randomness factor r=", r)
exec_time_delay = r * (PROXY_SPECIFIC_REQUEST_DELAY - delta)
exec_time = now + exec_time_delay # Approximation of execution time
else:
exec_time = now
exec_time_delay = 0
# Registering the fact that this proxy is working
# (we are building a new request for it = it returned some request = it's working)
self.alive_proxy(proxy_param)
next_req = self.build_request_for_proxy(proxy_param, exec_time)
if exec_time_delay is 0:
l("Returning request ", next_req.url, "without a delay")
yield next_req
else:
l("Storing request", next_req.url, "for later")
self.pending_requests[next_req.url] = next_req
elif self.SERPS_PER_QUERY > 1:
yield self.build_request_for_proxy(proxy_param, now)
else:
yield None
async def _disconnect_player(self, player):
await sleep(60)
try:
player = self.disco.wavelink.players[player.guild_id]
except KeyError:
return
guild = self.disco.get_guild(player.guild_id)
if not guild or not guild.me.voice:
await player.node._send(op='destroy', guildId=str(player.guild_id))
del player.node.players[player.guild_id]
return
elif ((player.current or player.queue) and self.has_listeners(guild)):
return
self.disco.log.info(
f'Desconectando de {guild} {guild.id} devido a inatividade')
await player.destroy()
await player.send(
l(player.locale, 'events.disconnectPlayer',
{"emoji": self.disco.emoji["alert"]}))
def make_transaction(self, **kwargs):
"""
Submits a transaction to the judging party. The judging party will go through standard checks to see if the
calling player has the right to submit a transaction, then it will try to validate the transaction and
potentially refuse it. For a complete description of what happens when a transaction is submitted to the
judging party, please refer to the paper, located in the `paper/` directory.
:param{AbstractPlayer} calling_player: The player that is calling the method (typically, `self`)
:param{string} type: The type of transaction to be performed
:param{dict} args: Arguments to be passed to the transaction
"""
try:
d("make_transaction()")
d("Banned players", self.banned_players)
if self.current_pid in self.banned_players:
return # Just return until it times out
# The calling player has to be the current player, only the current player is allowed to submit transactions
# during her own turn. When calling make_transaction the player has to pass itself as a parameter to prove who
# she claims she is. In case this does not match the comparison with the current player, it means the calling
# player is trying to submit transaction while it is not her turn and thus she is cheating, so we ban her.
if kwargs['calling_player'] is not self.players[self.current_pid]:
# bann_player takes the player id as a parameter but all we have is the player object reference itself
# so we need to first retrieve its id, which is nothing else than the index in the players' list
# @TODO stop using the index of the list as the id, move to a dict {id: player_reference}
self.bann_player(self.players.index(kwargs['calling_player']))
raise PlayerBannedException("Attempted to submit a transaction while it was not her turn.")
if self.current_player_transaction_attempts >= ALLOWED_TRANSACTIONS_ATTEMPTS_PER_ROUND \
or self.current_player_transactions >= ALLOWED_TRANSACTIONS_PER_ROUND:
self.bann_player(self.current_pid)
raise PlayerBannedException("Attempted too many transactions.")
self.current_player_transaction_attempts += 1
d("Current player already attempted", self.current_player_transaction_attempts, "during current turn.")
try:
type_str = kwargs['type']
args = kwargs['args']
except KeyError as ex:
d(ex)
return False
d("make_transaction() with:", str(type_str), str('args'))
try:
transaction = self.transaction_types[type_str](*args)
except Exception as e:
d(e)
return False
d("Instantiated transaction", transaction)
if not isinstance(transaction, AbstractTransaction):
d("Not an instance of AbstractTransaction")
return False
else:
d("Is a rightful instance")
if not self._check_player_agreements(transaction):
d("A player refused the transaction")
return False
d("Was agreed by all players")
d("Validating transaction...")
valid_transaction = transaction.is_valid(self)
d("Transaction validation ended.")
if valid_transaction is True:
# Note that there will not be any concurrent modification between the check of the transaction and
d("WATWATWAT")
l("WATWATWAT")
l("Transaction is valid, applying it")
transaction.apply(self)
self.current_player_transactions += 1
d("Current player has already done", self.current_player_transactions, "in its current turn.")
# Transaction is applied, tick the clock
self.clock.tick()
return True
d("WUTWUTWUT")
l("WUTWUTWUT")
# The transaction was not valid
d("Transaction is invalid")
return False
except Exception as e:
# If anything failed, the transaction should not be accepted
d("Something went wrong:", e)
d("As a consequence, we refuse the transaction.")
return False
async def on_command_error(self, ctx, e):
if isinstance(e, MusicError):
await ctx.send(e)
elif isinstance(e, CommandOnCooldown):
if ctx.command.name == 'whatsmyprefix' and ctx.prefix == ctx.me.mention + ' ':
return
_, s = divmod(e.retry_after, 60)
await ctx.send(l(
ctx, 'errors.onCooldown', {
"emoji": self.disco.emoji["false"],
"author": ctx.author.name,
"cooldown": int(s)
}),
delete_after=s + 6)
elif isinstance(e, MissingRole):
await ctx.send(
l(
ctx, 'errors.missingRole', {
"emoji": self.disco.emoji["false"],
"role": e.missing_role[0] or "DJ",
"author": ctx.author.name
}))
elif isinstance(e, (ConversionError, UserInputError)):
if ctx.prefix == f'<@{ctx.me.id}> ':
ctx.prefix = f'@{ctx.me.name} '
usage = l(ctx, f'commands.{ctx.command.name}.cmdUsage')
await ctx.send(
l(
ctx, 'errors.inputError', {
"emoji": self.disco.emoji["false"],
"usage":
f'{ctx.prefix}{ctx.invoked_with} {usage if usage else ""}',
"author": ctx.author.name
}))
elif isinstance(e, MissingPermissions):
perms = '\n'.join([
f'{self.disco.emoji["idle"]} **`{l(ctx, "permissions." + p).upper()}`**'
for p in e.missing_perms
])
await ctx.send(
l(
ctx, 'errors.missingPermissions', {
"emoji": self.disco.emoji["false"],
"permissions": perms,
"author": ctx.author.name
}))
elif isinstance(e, BotMissingPermissions):
perms = '\n'.join([
f'{self.disco.emoji["idle"]} **`{l(ctx, "permissions." + p).upper()}`**'
for p in e.missing_perms
])
await ctx.send(
l(
ctx, 'errors.botMissingPermissions', {
"emoji": self.disco.emoji["false"],
"permissions": perms,
"author": ctx.author.name
}))
elif ctx.command.name == 'play' and ctx.author.id in ctx.cog.waiting:
ctx.cog.waiting.remove(ctx.author.id)
else:
em = discord.Embed(
colour=0xFF0000,
timestamp=ctx.message.created_at,
description=
f'> {ctx.message.content}\n```py\n{e.__class__.__name__}: {e}```'
).set_author(name=f'{ctx.author} ({ctx.author.id})',
icon_url=ctx.author.avatar_url).set_footer(
text=f'ID: {ctx.message.id}')
await self.error_logs.send(
content=
f'Comando executado no canal {ctx.channel} ({ctx.channel.id})'
f' do servidor {ctx.guild} ({ctx.guild.id}).',
embed=em)
def process_exception(self, response, exception, spider):
l("Exception:", response, exception)
def player_thread(player, round_number):
try:
player.play_round(round_number)
except SystemExit:
l("Thread exited by force!")
return
def parse(self, response):
l("\n\n--- PARSING ITEMLIST PAGE", response.url, "---\n\n")
self.crawled_urls.add(response.url)
hxs = HtmlXPathSelector(response)
page_items = hxs.select('//li[@about="null"]')
l("Found", len(page_items), "items on this page")
try:
pattern_position = response.url.find(self.PAGE_URL_PATTERN)
is_first_page = pattern_position == -1
serp_url_without_page_pattern = response.url if is_first_page else response.url[:pattern_position]
kw = self.log_processor.serp_to_kw[serp_url_without_page_pattern]
if not is_first_page:
page_number = int(response.url[pattern_position+len(self.PAGE_URL_PATTERN):].strip())
else:
page_number = 1
# self.urls_done.add(response.url)
serp_logs_entries = self.log_processor.entries[kw]
l("Domain of this serp, in the logs, is:", len(serp_logs_entries))
serp_i = SERPItem()
serp_i['kw'] = kw
serp_i['page'] = page_number
serp_i['results'] = []
found = self.GRAB_ALL_THE_SERPS
pos = 0 # Result position in the SERP
for item in page_items:# Note: We assume that hxs.select() returns result in the order of the HTML
url = item.select('.//a[@rel="f:url"]/@href').extract()[0]
# Register this SERP result in the SERPItem (serp result= position, url)
serp_i['results'].append((pos, url))
l("Found url", url)
domain = self.extract_domain(url)
l("Domain is", domain)
pos += 1
for entry in serp_logs_entries:
if domain == entry[3]:
found = True
l("Found a result that is the same as before!")
# We store a "LogItem" as an item that is coming from the 2006 logs and that has been found
# still here in the current 2014 results
i = LogItem()
i['kw'] = kw
i['url'] = url
i['page'] = page_number
i['user_id'] = entry[0]
i['date'] = entry[1]
i['orig_pos'] = entry[2]
i['pos'] = (page_number-1)*10 + pos
l("Orig pos:", i['orig_pos'], "curr_pos:", i['pos'])
# Finally we are not going to need those for now, keeping them as a matter of reference on how to do it:
# i['title'] = BeautifulSoup(item.select('.//a[@rel="f:url"]/text()').extract()[0]).string
# i['desc'] = BeautifulSoup(item.select('.//p[@property="f:desc"]/text()').extract()[0]).string
yield i
break
# In the case we found some interesting data for this SERP, also register the SERP item (that includes the
# full SERP information, basically)
if found or not is_first_page:
# Was that the first SERP for this query?
# If yes, as this SERP indeed has a common ranking link/we are keeping it, we will add the other
# SERP pages that we want to crawl, if any
if is_first_page:
print "This was the first page and we found common links, adding the other SERPs URLs..."
for i in xrange(2, self.SERPS_PER_QUERY + 1):
serp_url = response.url + '&page=%d' % i
print "Adding", serp_url
self.urls_to_crawl.append(serp_url)
self.set_of_urls_should_be_crawled.add(serp_url)
yield serp_i
else:
print "There was no common domain between the logs and the returned SERP"
except KeyError as err:
print "KeyError? ->", err
print "The serp URL", serp_url_without_page_pattern, "was not found in the inverted index (full=", response.url, ")"
print self.log_processor.serp_to_kw
pass # Keyword not found? Well, ok, just don't return any item then, so, pass
finally:
for x in self.next_request_from_response(response):
yield x
def play_round(self):
d("Players ids:", self.players_ids)
for pid in self.players_ids:
d("=" * 30, "Changing player")
# This is a quick and dirty fix, @TODO
d("Banned players", self.banned_players)
if pid in self.banned_players:
d("Player", pid, "has been banned, passing this turn.")
continue
d("Current player id:", pid)
self.current_player_transaction_attempts = 0
self.current_player_transactions = 0
p = self.players[pid]
t = None
self.current_pid = pid
try:
t = time()
import threading
self.current_player_thread = threading.Thread(
target=player_thread,
args=(p, self.clock.current_turn_number())
)
self.current_player_thread.start()
self.current_player_thread.join(PLAYER_TIMEOUT)
if self.current_player_thread.is_alive():
l("#"*100)
l("Player", self.current_pid, "timed out! After", time() - t)
l("#"*100)
self.bann_player(pid)
except PlayerBannedException:
# This is a quick and dirty fix, @TODO
self.bann_player(pid)
except SystemExit as ex:
l("Player", self.current_pid, "timed out (exception)! After", time() - t)
self.bann_player(pid)
d(ex)
return False # @TODO
# This displays the state of the game
l(self.game)
self.clock.tick()
if self.clock.is_over():
l("Game Over")
return False
return True
def inject_request(self, r):
if r is None:
l("Asked to inject a None request")
l("Injecting new URL to crawl in the engine")
# import scrapy.project
return self.crawler.engine.crawl(r, self)
