def __init__(self, stream, server):
super(EncodedStreamer, self).__init__()
self.lock = BoundedSemaphore(value=1)
self.mainloop_thread = MainLoop()
self.mainloop_thread.start()
self.stream = stream
self.servers = {}
self.tailbins = []
self.playing = False
self.streamname = stream["name"]
self.description = stream["description"]
self.genre = ", ".join(stream["genres"])
# self.default_quality = default_quality
logging.debug("Streamer::__init__(): Constructing pipeline")
pipe = gst.Pipeline()
# threads
queue = gst.element_factory_make("queue")
# tee
# self.oggdemux = gst.element_factory_make('oggdemux')
self.vorbisparse = gst.element_factory_make('vorbisparse')
# self.oggdemux.connect("pad-added", partial(self.__on_dynamic_pad, link = self.vorbisparse))
tee = gst.element_factory_make('tee')
# self.typefind = gst.element_factory_make('typefind')
pipe.add(self.vorbisparse, tee)
gst.element_link_many(self.vorbisparse, tee)
# fakesink = gst.element_factory_make("fakesink")
# fakesink.set_property("sync", 1)
# pipe.add(fakesink)
# gst.element_link_many(tee, fakesink)
self.tee = tee
self.pipe = pipe
# only one quality, because this is encoded (no reencoding is done)
# self.tees = {default_quality:tee}
logging.debug("Streamer::__init__(): Running distribute")
self.scale(server, sync=1, init=True)
self.queue = queue
logging.debug("Streamer::__init__(): Running EncodedChannel")
self.bus = pipe.get_bus()
self.bus.add_signal_watch()
self.bus.connect('message::state-changed',
self.on_message_state_changed)
self.bus.connect('message::eos', self.on_eos)
def __init__(self, medias, observers):
self.artists = OrderedDict({})
self.history = None
self.lock = BoundedSemaphore(value=1)
for media in medias:
self.__add(media)
# self.show()
self.shuffle()
def __init__(self, maxsize = 0):
if maxsize <= 0:
maxsize = _multiprocessing.SemLock.SEM_VALUE_MAX
self._maxsize = maxsize
self._reader, self._writer = Pipe(duplex=False)
self._rlock = Lock()
self._opid = os.getpid()
if sys.platform == 'win32':
self._wlock = None
else:
self._wlock = Lock()
self._sem = BoundedSemaphore(maxsize)
self._after_fork()
if sys.platform != 'win32':
register_after_fork(self, Queue._after_fork)
return
def __init__(self, stream_id, history, playlist, observers):
super(Buffer, self).__init__()
self.stream_id = stream_id
self.observers = observers
self.history = history
self.playlist = playlist
self.queue = OrderedDict({})
self.lock = BoundedSemaphore(value=1)
while not self.is_full():
self.lock.acquire()
t = self.playlist.get_track()
self.__add_track(t)
self.lock.release()
def BoundedSemaphore(value=1):
"""
Returns a bounded semaphore object
"""
from multiprocessing.synchronize import BoundedSemaphore
return BoundedSemaphore(value)
def copy_s3_bucket(SOURCE_BUCKET, DEST_BUCKET, prefix=None, threads=10):
"""
Example usage: copy_s3_bucket(SOURCE_BUCKET='my-source-bucket', DEST_BUCKET='my-destination-bucket', prefix='parent/child/dir/', threads=20)
"""
# Init s3
conn = S3Connection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
bucket = conn.get_bucket(SOURCE_BUCKET)
dest_bucket = conn.get_bucket(DEST_BUCKET)
# Filter by prefix
rs = bucket.list()
if prefix: rs = bucket.list(prefix)
class CopyKey(Thread):
def __init__(self, key_name):
Thread.__init__(self)
self.key_name = key_name
self.status = False
def run(self):
# We must create new bucket instances for each thread, passing the key is not threadsafe
thread_conn = S3Connection(AWS_ACCESS_KEY_ID,
AWS_SECRET_ACCESS_KEY)
thread_bucket = conn.get_bucket(SOURCE_BUCKET)
thread_dest_bucket = conn.get_bucket(DEST_BUCKET)
thread_key = thread_bucket.get_key(self.key_name)
# Only copy if not exists on dest bucket
if not thread_dest_bucket.get_key(self.key_name):
pool_sema.acquire()
self.status = "%s : Sempahore Acquired, Copy Next" % datetime.datetime.now(
)
try:
thread_key.copy(DEST_BUCKET, self.key_name, None, False,
True)
self.status = "%s : Copy Success : %s" % (
datetime.datetime.now(), self.key_name)
except:
self.status = "%s : Copy Error : %s" % (
datetime.datetime.now(), sys.exc_info())
finally:
pool_sema.release()
else:
self.status = "%s : Key Already Exists, will not overwrite." % datetime.datetime.now(
)
key_copy_thread_list = []
pool_sema = BoundedSemaphore(value=threads)
total_keys = 0
# Request threads
for key in rs:
total_keys += 1
print "%s : Requesting copy thread for key %s" % (
datetime.datetime.now(), key.name)
current = CopyKey(key.name)
key_copy_thread_list.append(current)
current.start()
# Pause if max threads reached - note that enumerate returns all threads, including this parent thread
if len(threading.enumerate()) >= threads:
print "%s : Max Threads (%s) Reached: Pausing until threadcount reduces." % (
datetime.datetime.now(), threads)
while 1:
if len(threading.enumerate()) < threads:
print "%s : Continuing thread creation." % datetime.datetime.now(
)
break
time.sleep(1)
for key_copy_thread in key_copy_thread_list:
key_copy_thread.join(
30
) # Bring this particular thread to this current "parent" thread, blocks parent until joined or 30s timeout
if key_copy_thread.isAlive():
print "%s : TIMEOUT on key %s" % (datetime.datetime.now(),
key_copy_thread.key_name)
continue
print "%s : Status Output: %s" % (datetime.datetime.now(),
key_copy_thread.status)
print "%s : Complete : %s Total Keys Requested" % (datetime.datetime.now(),
total_keys)
def __init__(self, stream):
super(Streamer, self).__init__()
logging.debug("Streamer::__init__(): Constructing pipeline")
pipe = gst.Pipeline("pipeline")
self.lock = BoundedSemaphore(value=1)
self.mainloop_thread = MainLoop()
self.mainloop_thread.start()
self.stream = stream
self.servers = {}
self.tees = {}
self.encodebins = {}
self.quality_tee_srcs = {}
self.tailbins = []
self.playing = False
self.streamname = stream["name"]
self.description = stream["description"]
self.genre = ", ".join(stream["genres"])
# mix audio
adder = gst.element_factory_make("adder")
self.quality_tee = gst.element_factory_make("tee")
# queue = gst.element_factory_make("queue")
pipe.add(adder, self.quality_tee)
gst.element_link_many(adder, self.quality_tee)
self.pipe = pipe
self.bus = pipe.get_bus()
self.bus.add_signal_watch()
self.bus.connect('message::state-changed',
self.on_message_state_changed)
self.bus.connect('message::eos', self.on_eos)
self.bus.connect('message::error', self.on_error)
self.bus.connect("message::tag", self.on_tag)
# self.bus.connect('message', self.on_message)
self.adder = adder
quietbin = QuietBin(self)
sink = self.adder.get_request_pad("sink%d")
quietbin.get_pad("src").link(sink)
for quality in stream["quality"]:
# this has to be to ensure double from database work as expected and key is found
quality = float(quality)
# if quality == min(stream["quality"]):
eb = EncodeBin(self, quality)
quality_tee_src = self.quality_tee.get_request_pad("src%d")
quality_tee_src.link(eb.get_pad("sink"))
self.quality_tee_srcs[quality] = quality_tee_src
tee = gst.element_factory_make("tee")
pipe.add(tee)
sink = tee.get_pad("sink")
src = eb.get_pad("src")
src.link(sink)
self.tees[quality] = tee
self.encodebins[quality] = eb
if quality == min(stream["quality"]):
self.tee = tee
self.scale(None, quality, sync=1, init=True)
def __init__(self, port):
super(StreamManagement, self).__init__()
self.streams = {}
# one command at the time
self.lock = BoundedSemaphore(value=1)
local_ip = Helpers.ip.local_ip()
public_ip = Helpers.ip.public_ip()
self.db.servers.update(
{
"type": "pipeline",
"local_ip": local_ip,
"port": port
}, {
"level": float(0),
"type": "pipeline",
"local_ip": local_ip,
"public_ip": public_ip,
"port": port,
"down": False
},
upsert=True)
server = self.db.servers.find_one(
{
"type": "pipeline",
"local_ip": local_ip,
"port": port
}, {
"_id": 1,
})
# db = Db()
# server = db.conn.Server.find_one({"type": "pipeline", "local_ip": local_ip, "port": port})
# if server == None:
# server = db.conn.Server()
#
# server["level"] = float(0)
# server["type"] = "pipeline"
# server["local_ip"] = local_ip
# server["public_ip"] = public_ip
# server["port"] = port
# server["down"] = False
# server.save()
self.pipeline_server_id = unicode(server["_id"])
PipelineLoadBalancer(self.pipeline_server_id).start()
# self.db = db
Helpers.globals.set_id_pipeline(server["_id"])
self.connect(handler=self.dump_dot_file, signal="dump_dot_file")
self.connect(handler=self.playlist_update, signal="playlist_update")
self.connect(handler=self.change_selection, signal="change_selection")
self.connect(handler=self.next, signal="next")
self.connect(handler=self.is_alive, signal="is_alive")
self.connect(handler=self.scale_streaming, signal="scale")
self.connect(handler=self.start_streaming, signal="start")
self.connect(handler=self.print_playlist, signal="print_playlist")
self.connect(handler=self.register_updates_observer,
signal="register_updates_observer")
self.connect(handler=self.notify_current_track,
signal="notify_current_track")
self.connect(handler=self.unregister_updates_observer,
signal="unregister_updates_observer")
self.connect(handler=self.update_buffer, signal="update_buffer")
self.connect(handler=self.start_live, signal="start_live")
self.connect(handler=self.stop_streaming, signal="stop")
self.connect(handler=self.rescale_streaming, signal="rescale")
self.connect(handler=self.__streamer_initialized,
signal="streamer_initialized")
def BoundedSemaphore(value=1):
from multiprocessing.synchronize import BoundedSemaphore
return BoundedSemaphore(value)
def init(self, player, respond=None):
super(BaseScheduler, self).__init__()
self.playing = True
self.player = player
self.last_artist = None
self.last_stop_unused = time.time()
self.observers = UpdateObservers()
# db = Db()
# self.db = db
self.tracks_selector = TracksSelector(self.player.stream["_id"])
self.history = History(observers=self.observers)
self.program_id = unicode(self.player.stream["default_program_id"])
self.groups = yield task(self.call,
self.get_default_group_ids,
program_id=self.program_id)
self.fade = yield task(self.call,
self.get_fade_length,
program_id=self.program_id)
self.playlist_lock = BoundedSemaphore(value=1)
self.playlist_lock.acquire()
selection = yield task(self.call,
self.get_selection,
program_id=self.program_id)
if selection == "shuffle":
tracks = yield task(self.call,
self.tracks_selector.select_by_groups,
groups=self.groups,
sorted=False)
print tracks
self.playlist = RandomPlaylist(tracks, observers=self.observers)
self.playlist.apply_history(self.history)
else:
tracks = yield task(self.call,
self.tracks_selector.select_by_groups,
groups=self.groups,
sorted=True)
self.playlist = Playlist(tracks, observers=self.observers)
self.buffer = Buffer(self.player.stream["_id"],
self.history,
self.playlist,
observers=self.observers)
self.playlist_lock.release()
track = self.buffer.get_track()
file_id = track["file_id"]
filesrc = GridFSSource(file_id)
if self.__class__.__name__ == "EncodedScheduler":
self.stream_bin = EncodedStreamBin(self.player, filesrc, self)
else:
self.stream_bin = StreamBin(self.player, filesrc, self)
self.stream_bin.link_and_unblock()
respond("bitch, yo!")
os.path.join(os.path.dirname(__file__), "static"),
"ui_methods":
ui_methods,
"cookie_secret":
str("__TODO:_GENERATE_YOUR_OWN_RANDOM_VALUE_HERE__"),
"login_url":
"/admin/login.html",
"debug":
True,
"db":
motor.MotorClient(
"mongodb://*****:*****@127.0.0.1:27017/pipeline"
).pipeline,
"upload_processor": {
"connection": parent_conn,
"lock": BoundedSemaphore(value=1)
},
"facebook_api_key":
"..",
"facebook_secret":
"..",
"executor":
ThreadPoolExecutor(max_workers=4),
# websockets connection cache for receiving metadata and live
"ws_cache": {}
}
application = tornado.web.Application([
(r"/image.json", JSONImageHandler),
(r"/image.jpg", JPGImageHandler),
(r"/", MainHandler),
