def __init__(self, pcontext: ListenerContext):
Process.__init__(self)
self.p_context = pcontext
self.driverManager = DriverManager(pcontext.console,
pcontext.shutdownEvent,
startService=False)
self.client = client('sqs')
self.totalMessageReceived = 0
self.proxyManager = ProxyManager(proxyFile=PROXY_FILE_LISTENER)
self.lockClient = Lock()
self.lockDriver = Lock()
def __init__(self, pcontext: RegisterContext):
super().__init__()
self.p_context = pcontext
self.driverManager = DriverManager(pcontext.console,
pcontext.shutdownEvent,
startService=False)
self.client = client('sqs')
self.totalAccountCreated = 0
self.proxyRegisterManager = ProxyManager(proxyFile=PROXY_FILE_REGISTER)
self.proxyListenerManager = ProxyManager(proxyFile=PROXY_FILE_LISTENER)
self.userManager = UserManager(self.p_context.console)
self.userAgentManager = UserAgentManager()
self.lockClient = Lock()
self.lockDriver = Lock()
self.lockAccountCount = Lock()
def __init__(self, workerId: int, tasks: TasksQueue, shutdownEvent: multiprocessing.Event, console: Console, states, stats, dryRun: bool):
super(Worker, self).__init__()
self.id = workerId
self.tasks = tasks
self.shutdownEvent = shutdownEvent
self.lock = lock
self.console = console
self.states = states
self.setState(Worker.RUNNING, 'Started')
self.dryRun = dryRun
self.stats = stats
self.taskCount = 0
self.taskDuration = 0
self.timeoutCount = 0
self.errorCount = 0
self.driverManager = DriverManager()
self.locks = {
'tasks': Lock(),
'stats': Lock(),
'states': Lock(),
'accounts': Lock(),
}
from src.services.config import Config
from src.services.console import Console
from src.application.spotify.listener import Listener, ListenerContext, TaskContext
from src.services.drivers import DriverManager
from src.services.proxies import ProxyManager, PROXY_FILE_LISTENER
from time import sleep
from sp-listener import runner
if __name__ == '__main__':
threadsCount = Array('i', 1)
messagesCount = Array('i', 1)
console = Console()
config = Config()
dm = DriverManager(console, Event())
lock = Lock()
config.LISTENER_MAX_PROCESS = 1
config.LISTENER_MAX_THREAD = 1
config.LISTENER_SPAWN_INTERVAL = 1
shutdownEvent = Event()
lpc = ListenerContext(
messagesCount=messagesCount,
threadsCount=threadsCount,
channel=0,
config = config,
maxThread=1,
shutdownEvent=shutdownEvent,
console=console,
lock=lock,
def runner(console: Console, shutdownEvent: Event, headless: bool, user: dict,
proxy: dict, playlist: str, vnc: bool, sqsEndpoint: str,
screenshotDir: str, runnerStats: Array, processStates: Array):
tid = current_process().pid
console.log('#%d Start' % tid)
driver = None
vdisplay = None
x11vnc = None
userDataDir = None
spotify = None
try:
if headless == False:
width = 1280
height = 1024
if 'windowSize' in user:
[width, height] = user['windowSize'].split(',')
vdisplay = Xvfb(width=width,
height=height,
colordepth=24,
tempdir=None,
noreset='+render')
vdisplay.start()
if vnc:
x11vnc = X11vnc(vdisplay)
x11vnc.start()
driverManager = DriverManager(console, shutdownEvent)
driverData = driverManager.getDriver(type='chrome',
uid=tid,
user=user,
proxy=proxy,
headless=headless)
del driverManager
collect()
if not driverData or not driverData['driver']:
if vdisplay:
vdisplay.stop()
if x11vnc:
x11vnc.stop()
raise Exception('No driver was returned from adapter')
driver = driverData['driver']
userDataDir = driverData['userDataDir']
except Exception as e:
runnerStats[STAT_DRIVER_NONE] += 1
console.exception('Driver unavailable')
else:
try:
spotify = Adapter(driver, console, shutdownEvent)
console.log('#%d Start create account for %s' %
(tid, user['email']))
spotify.register(user)
try:
boto3.client('sqs').send_message(
QueueUrl=sqsEndpoint,
MessageBody=dumps({
'user': user,
'playlist': playlist
}),
DelaySeconds=1,
)
except:
console.exception('#%d Failed to send message to the queue' %
tid)
else:
console.log('#%d Account created for %s' %
(tid, user['email']))
runnerStats[STAT_ACCOUNT_CREATED] += 1
except Exception as e:
runnerStats[STAT_ERROR] += 1
try:
id = randint(10000, 99999)
with open(screenshotDir + ('%d.log' % id), 'w') as f:
f.write(str(e))
driver.save_screenshot(screenshotDir + ('%d.png' % id))
except:
console.exception()
if driver:
try:
driver.quit()
del driver
except:
pass
if spotify:
try:
del spotify
except:
pass
if userDataDir:
try:
rmtree(path=userDataDir, ignore_errors=True)
except:
pass
if x11vnc: #Terminate vnc server if any
try:
x11vnc.stop()
del x11vnc
except:
pass
if vdisplay:
try:
vdisplay.stop()
del vdisplay
except:
pass
console.log('#%d Stop' % tid)
collect()
class Register(Process):
def __init__(self, pcontext: RegisterContext):
super().__init__()
self.p_context = pcontext
self.driverManager = DriverManager(pcontext.console,
pcontext.shutdownEvent,
startService=False)
self.client = client('sqs')
self.totalAccountCreated = 0
self.proxyRegisterManager = ProxyManager(proxyFile=PROXY_FILE_REGISTER)
self.proxyListenerManager = ProxyManager(proxyFile=PROXY_FILE_LISTENER)
self.userManager = UserManager(self.p_context.console)
self.userAgentManager = UserAgentManager()
self.lockClient = Lock()
self.lockDriver = Lock()
self.lockAccountCount = Lock()
def run(self):
self.p_context.console.log('Start process %d' % current_process().pid)
threads = []
messages = []
for x in range(self.p_context.accountToCreate):
messages.append({
'Body':
dumps({
'user':
self.userManager.createRandomUser(
proxy=self.proxyListenerManager.getRandomProxy(),
userAgent=self.userAgentManager.getRandomUserAgent(),
application='spotify',
),
'playlist':
self.p_context.playlist
})
})
while True:
try:
sleep(self.p_context.config.REGISTER_SPAWN_INTERVAL)
freeSlot = self.p_context.maxThread - len(threads)
if freeSlot and len(messages) and (
not self.p_context.shutdownEvent.is_set()):
message = messages.pop()
body = loads(message['Body'])
t_context = TaskContext(
batchId=self.p_context.batchId,
playlist=body['playlist'],
proxy=self.proxyRegisterManager.getRandomProxy(),
user=body['user'],
headless=self.p_context.headless,
vnc=self.p_context.vnc)
t = Thread(target=self.runner, args=(t_context, ))
t.start()
threads.append(t)
leftThread = []
for thread in threads:
if thread.is_alive():
leftThread.append(thread)
else:
del thread
collect()
pass
threads = leftThread
self.p_context.threadsCount[self.p_context.channel] = len(
threads)
if len(threads) == 0 and len(messages) == 0:
break
except:
self.p_context.console.exception(
'Register process #%d exception' % self.p_context.channel)
self.driverManager.purge()
self.p_context.console.log('Stop process %d' % current_process().pid)
del self.driverManager
del self.p_context
del self.userAgentManager
del self.userManager
del self.proxyListenerManager
del self.proxyRegisterManager
collect()
def runner(self, t_context: TaskContext):
tid = current_thread().native_id
self.p_context.console.log('Start thread %d' % tid)
driver = None
vdisplay = None
x11vnc = None
userDataDir = None
spotify = None
try:
if self.p_context.shutdownEvent.is_set():
return
if t_context.headless == False:
vdisplay = Xvfb(width=1280,
height=1024,
colordepth=24,
tempdir=None,
noreset='+render')
vdisplay.start()
if t_context.vnc:
x11vnc = X11vnc(vdisplay)
x11vnc.start()
with self.lockDriver:
driverData = self.driverManager.getDriver(
type='chrome',
uid=tid,
user=t_context.user,
proxy=t_context.proxy,
headless=t_context.headless)
if not driverData:
raise Exception('No driverData was returned from adapter')
driver = driverData['driver']
userDataDir = driverData['userDataDir']
if not driver:
raise Exception('No driver was returned from adapter')
except:
self.p_context.console.error('Unavailale webdriver: %s' %
format_exc())
else:
try:
spotify = Spotify.Adapter(driver, self.p_context.console,
self.p_context.shutdownEvent,
t_context.batchId)
self.p_context.console.log('#%d Start create account for %s' %
(tid, t_context.user['email']))
if spotify.register(t_context.user):
self.p_context.console.log('#%d Account created for %s' %
(tid, t_context.user['email']))
message = {
'user': t_context.user,
'playlist': t_context.playlist
}
with self.lockClient:
try:
self.client.send_message(
QueueUrl=self.p_context.config.SQS_ENDPOINT,
MessageBody=dumps(message),
DelaySeconds=1,
)
except:
self.p_context.console.exception(
'T#%d Failed to send message to the queue %s' %
(tid, self.p_context.config.SQS_ENDPOINT))
else:
self.p_context.console.log(
'#%d Message sent for %s' %
(tid, t_context.user['email']))
with self.lockAccountCount:
self.totalAccountCreated += 1
self.p_context.accountsCount[
self.p_context.
channel] = self.totalAccountCreated
else:
if not self.p_context.shutdownEvent.is_set():
self.p_context.console.error(
'#%d Failed to create account for %s' %
(tid, t_context.user['email']))
except:
self.p_context.console.exception()
if driver:
try:
driver.quit()
del driver
except:
pass
if spotify:
try:
del spotify
except:
pass
if userDataDir:
try:
rmtree(path=userDataDir, ignore_errors=True)
except:
pass
if x11vnc: #Terminate vnc server if any
try:
x11vnc.stop()
del x11vnc
except:
pass
if vdisplay:
try:
vdisplay.stop()
del vdisplay
except:
pass
self.p_context.console.log('Stop thread %d' % tid)
collect()
class Listener(Process):
def __init__(self, pcontext: ListenerContext):
Process.__init__(self)
self.p_context = pcontext
self.driverManager = DriverManager(pcontext.console,
pcontext.shutdownEvent,
startService=False)
self.client = client('sqs')
self.totalMessageReceived = 0
self.proxyManager = ProxyManager(proxyFile=PROXY_FILE_LISTENER)
self.lockClient = Lock()
self.lockDriver = Lock()
def run(self):
tid = current_thread().native_id
self.p_context.console.log('#%d Start' % tid)
driver = None
try:
if self.p_context.shutdownEvent.is_set():
return
vdisplay = None
x11vnc = None
if t_context.headless == False:
width = 1280
height = 1024
if 'windowSize' in t_context.user:
[width, height] = t_context.user['windowSize'].split(',')
vdisplay = Xvfb(width=width,
height=height,
colordepth=24,
tempdir=None,
noreset='+render')
vdisplay.start()
if t_context.vnc:
x11vnc = X11vnc(vdisplay)
x11vnc.start()
with self.lockDriver:
driverData = self.driverManager.getDriver(
type='chrome',
uid=tid,
user=t_context.user,
proxy=t_context.proxy,
headless=t_context.headless)
if not driverData:
return
driver = driverData['driver']
userDataDir = driverData['userDataDir']
if not driver:
return
except:
self.p_context.console.error('Unavailale webdriver: %s' %
format_exc())
else:
try:
spotify = Spotify.Adapter(driver, self.p_context.console,
self.p_context.shutdownEvent)
if spotify.login(t_context.user['email'],
t_context.user['password']):
self.p_context.console.log('#%d Logged In' % tid)
if not self.p_context.shutdownEvent.is_set():
self.p_context.console.log('#%d Start listening...' %
tid)
spotify.playPlaylist(t_context.playlist,
self.p_context.shutdownEvent, 90,
110)
self.p_context.console.log('#%d Played' % tid)
with self.lockClient:
self.client.delete_message(
QueueUrl=self.p_context.config.SQS_ENDPOINT,
ReceiptHandle=t_context.receiptHandle)
self.p_context.console.log('#%d Message deleted' % tid)
except:
self.p_context.console.exception()
spotify.saveScreenshot()
if driver:
try:
driver.quit()
del driver
except:
pass
if userDataDir:
try:
rmtree(path=userDataDir, ignore_errors=True)
except:
pass
if x11vnc: #Terminate vnc server if any
try:
x11vnc.stop()
del x11vnc
except:
pass
if vdisplay:
try:
vdisplay.stop()
del vdisplay
except:
pass
collect()
def run(console: Console, shutdownEvent: Event, headless: bool, user: dict,
proxy: dict, playlist: str, vnc: bool, screenshotDir, stats: Array,
states: Array):
tid = current_process().pid
console.log('#%d Start' % tid)
driver = None
try:
if shutdownEvent.is_set():
return
vdisplay = None
x11vnc = None
if headless == False:
width = 1280
height = 1024
if 'windowSize' in user:
[width, height] = user['windowSize'].split(',')
vdisplay = Xvfb(width=width,
height=height,
colordepth=24,
tempdir=None,
noreset='+render')
vdisplay.start()
if vnc:
x11vnc = X11vnc(vdisplay)
x11vnc.start()
driverManager = DriverManager(console, shutdownEvent)
driverData = driverManager.getDriver(type='chrome',
uid=tid,
user=user,
proxy=proxy,
headless=headless)
if not driverData:
raise Exception('No driverData was returned from adapter')
driver = driverData['driver']
userDataDir = driverData['userDataDir']
if not driver:
raise Exception('No driver was returned from adapter')
except:
stats[STAT_DRIVER_NONE] += 1
console.exception('Driver unavailable')
else:
try:
spotify = Adapter(driver, console, shutdownEvent)
spotify.login(user['email'], user['password'])
stats[STAT_LOGGED_IN] += 1
console.log('#%d Logged In' % tid)
spotify.playPlaylist(playlist, shutdownEvent, 80, 100)
console.log('#%d Played' % tid)
stats[STAT_PLAYED] += 1
except Exception as e:
stats[STAT_ERROR] += 1
try:
id = randint(10000, 99999)
with open(screenshotDir + ('%d.log' % id), 'w') as f:
f.write(str(e))
driver.save_screenshot(screenshotDir + ('%d.png' % id))
except:
console.exception()
if driver:
try:
driver.quit()
del driver
except:
pass
if userDataDir:
try:
rmtree(path=userDataDir, ignore_errors=True)
except:
pass
if x11vnc: #Terminate vnc server if any
try:
x11vnc.stop()
del x11vnc
except:
pass
if vdisplay:
try:
vdisplay.stop()
del vdisplay
except:
pass
collect()
vnc = False
for arg in argv:
if arg == '--info':
showInfo = True
if arg == '--nooutput':
noOutput = True
if arg == '--headless':
headless = True
if arg == '--vnc':
vnc = True
shutdownEvent = Event()
config = Config()
processes = []
console = Console(ouput=not noOutput)
driverManager = DriverManager(console, shutdownEvent)
driverVersion = driverManager.getDriverVersion('chrome')
browserVersion = driverManager.getBrowserVersion('chrome')
client = boto3.client('sqs')
totalMessageReceived = 0
stats = Stats()
messages = []
lastProcessStart = 0
lockThreadsCount = Lock()
def _showStats():
showStats(
{
'totalProcess': len(processes),
'totalMessageReceived': totalMessageReceived,
class Listener(Process):
def __init__(self, pcontext: ListenerContext):
super().__init__()
self.p_context = pcontext
self.driverManager = DriverManager(pcontext.console,
pcontext.shutdownEvent,
startService=False)
self.client = client('sqs')
self.totalMessageReceived = 0
self.proxyManager = ProxyManager(proxyFile=PROXY_FILE_LISTENER)
self.lockClient = Lock()
self.lockDriver = Lock()
def run(self):
threads = []
messages = []
while True:
try:
sleep(self.p_context.config.LISTENER_SPAWN_INTERVAL)
freeSlot = self.p_context.maxThread - len(threads)
if len(messages) < 1 and freeSlot and (
not self.p_context.shutdownEvent.is_set()):
try:
with self.lockClient:
response = self.client.receive_message(
QueueUrl=self.p_context.config.SQS_ENDPOINT,
MaxNumberOfMessages=freeSlot,
VisibilityTimeout=600,
WaitTimeSeconds=2,
)
if 'Messages' in response:
newMessages = response['Messages']
messages = newMessages + messages
self.totalMessageReceived += len(newMessages)
self.p_context.messagesCount[
self.p_context.
channel] = self.totalMessageReceived
except:
self.p_context.console.exception()
if freeSlot and len(messages):
message = messages.pop()
body = loads(message['Body'])
proxy = None
if self.p_context.config.LISTENER_OVERIDE_PLAYLIST:
body[
'playlist'] = self.p_context.config.LISTENER_OVERIDE_PLAYLIST
if self.p_context.config.LISTENER_OVERIDE_PROXY:
proxy = self.proxyManager.getRandomProxy()
t_context = TaskContext(
batchId=self.p_context.batchId,
playlist=body['playlist'],
proxy=proxy,
user=body['user'],
receiptHandle=message['ReceiptHandle'],
vnc=self.p_context.vnc,
headless=self.p_context.headless)
t = Thread(target=self.runner, args=(t_context, ))
t.start()
threads.append(t)
leftThread = []
for thread in threads:
if thread.is_alive():
leftThread.append(thread)
else:
del thread
collect()
threads = leftThread
self.p_context.threadsCount[self.p_context.channel] = len(
threads)
if len(threads) == 0:
break
except:
self.p_context.console.exception()
del self.driverManager
del self.p_context
collect()
def runner(self, t_context: TaskContext):
tid = current_thread().native_id
self.p_context.console.log('#%d Start' % tid)
driver = None
try:
if self.p_context.shutdownEvent.is_set():
return
vdisplay = None
x11vnc = None
if t_context.headless == False:
width = 1280
height = 1024
if 'windowSize' in t_context.user:
[width, height] = t_context.user['windowSize'].split(',')
vdisplay = Xvfb(width=width,
height=height,
colordepth=24,
tempdir=None,
noreset='+render')
vdisplay.start()
if t_context.vnc:
x11vnc = X11vnc(vdisplay)
x11vnc.start()
with self.lockDriver:
driverData = self.driverManager.getDriver(
type='chrome',
uid=tid,
user=t_context.user,
proxy=t_context.proxy,
headless=t_context.headless)
if not driverData:
return
driver = driverData['driver']
userDataDir = driverData['userDataDir']
if not driver:
return
except:
self.p_context.console.error('Unavailale webdriver: %s' %
format_exc())
else:
try:
spotify = Spotify.Adapter(driver, self.p_context.console,
self.p_context.shutdownEvent)
if spotify.login(t_context.user['email'],
t_context.user['password']):
self.p_context.console.log('#%d Logged In' % tid)
if not self.p_context.shutdownEvent.is_set():
self.p_context.console.log('#%d Start listening...' %
tid)
spotify.playPlaylist(t_context.playlist,
self.p_context.shutdownEvent, 90,
110)
self.p_context.console.log('#%d Played' % tid)
with self.lockClient:
self.client.delete_message(
QueueUrl=self.p_context.config.SQS_ENDPOINT,
ReceiptHandle=t_context.receiptHandle)
self.p_context.console.log('#%d Message deleted' % tid)
except:
self.p_context.console.exception()
spotify.saveScreenshot()
if driver:
try:
driver.quit()
del driver
except:
pass
if userDataDir:
try:
rmtree(path=userDataDir, ignore_errors=True)
except:
pass
if x11vnc: #Terminate vnc server if any
try:
x11vnc.stop()
del x11vnc
except:
pass
if vdisplay:
try:
vdisplay.stop()
del vdisplay
except:
pass
collect()
print('Shutdown, please wait...')
for p in processes:
if p.is_alive():
try:
p.terminate()
except:
pass
driverManager.purge()
if __name__ == '__main__':
startTime = time()
config = Config()
processes = []
console = Console()
driverManager = DriverManager(console)
driverVersion = driverManager.getDriverVersion('chrome')
browserVersion = driverManager.getBrowserVersion('chrome')
client = boto3.client('sqs')
userManager = UserManager(console)
userAgentManager = UserAgentManager()
proxyManagerListener = ProxyManager(proxyFile=PROXY_FILE_LISTENER)
proxyManagerRegister = ProxyManager(proxyFile=PROXY_FILE_REGISTER)
lock = Lock()
stats = Stats()
users = []
#Start proxy server
proxy.start([],
plugins=[
import time
from xvfbwrapper import Xvfb
from src.services.x11vncwrapper import X11vnc
from src.services.drivers import DriverManager
import fcntl
import os
import selectors
if __name__ == '__main__':
console = Console()
runTest = True
if runTest:
pm = ProxyManager(PROXY_FILE_LISTENER)
proxy = pm.getRandomProxy()
dm = DriverManager(Console(), Event())
vdisplay = Xvfb()
vdisplay.start()
x11vnc = X11vnc(vdisplay)
x11vnc.start()
driverData = dm.getDriver('chrome', 1, {}, proxy, False)
driver = driverData['driver']
spotify = Spotify.Adapter(driver, console, Event())
ip = spotify.getMyIp()
console.success('%s' % ip)
#info = spotify.getClientInfo('http://ec2-35-180-119-212.eu-west-3.compute.amazonaws.com')
#info = spotify.getClientInfo('http://35.180.119.212')
#if 'server' in info:
# console.success('Browser identity: %s, %s' % (info['server']['REMOTE_ADDR'], info['server']['HTTP_USER_AGENT']))