def start_torrent(self, torrent, output_dir='.'):
"""
Download (leech or seed) a file via BitTorrent.
The code is adapted from Next-Share's 'BaseLib/Tools/cmdlinedl.py'.
@param torrent .torrent file or URL
"""
# setup and start download
dscfg = DownloadStartupConfig()
dscfg.set_dest_dir(output_dir)
if torrent.startswith("http") or torrent.startswith(P2PURL_SCHEME):
tdef = TorrentDef.load_from_url(torrent)
else:
tdef = TorrentDef.load(torrent)
if tdef.get_live():
raise ValueError("CIS does not support live torrents")
new_download = True
try:
d = self.session.start_download(tdef, dscfg)
except DuplicateDownloadException:
new_download = False
#d.set_state_callback(state_callback, getpeerlist=False)
if new_download:
logger.log_msg('download of torrent "%s" started' % torrent)
def seed(self, transcode_configs):
"""
Creates torrents from the videos passed and then stats seeding them.
@param transcode_configs a list of dictionaries with format settings
"""
logger.log_msg('#%s: creating torrents and starting seeding...' \
% self.job_id)
for transcode_config in transcode_configs:
# * CREATE TORRENTS FOR EACH TRANSCODED VIDEO
# Create torrent file.
bt.create_torrent(transcode_config['output_file'])
output_file = transcode_config['output_file'] + '.tstream'
output_file = output_file[(output_file.rindex('/') + 1):]
# The torrent file is created in the same directory with the
# source file. Move it to the torrents directory.
#if not os.path.exists(
# os.path.join(output_file, config.TORRENTS_PATH)):
try:
shutil.move(transcode_config['output_file'] + '.tstream', \
config.TORRENTS_PATH)
except:
pass
# * SEED TORRENTS
Server.bit_torrent.start_torrent( \
os.path.join(config.TORRENTS_PATH, output_file),
config.MEDIA_PATH)
def start_torrent(self, torrent, output_dir='.'):
"""
Download (leech or seed) a file via BitTorrent.
The code is adapted from Next-Share's 'BaseLib/Tools/cmdlinedl.py'.
@param torrent .torrent file or URL
"""
# setup and start download
dscfg = DownloadStartupConfig()
dscfg.set_dest_dir(output_dir)
if torrent.startswith("http") or torrent.startswith(P2PURL_SCHEME):
tdef = TorrentDef.load_from_url(torrent)
else:
tdef = TorrentDef.load(torrent)
if tdef.get_live():
raise ValueError("CIS does not support live torrents")
new_download = True
try:
d = self.session.start_download(tdef, dscfg)
except DuplicateDownloadException:
new_download = False
#d.set_state_callback(state_callback, getpeerlist=False)
if new_download:
logger.log_msg('download of torrent "%s" started' % torrent)
def seed(self, transcode_configs):
"""
Creates torrents from the videos passed and then stats seeding them.
@param transcode_configs a list of dictionaries with format settings
"""
logger.log_msg('#%s: creating torrents and starting seeding...' \
% self.job_id)
for transcode_config in transcode_configs:
# * CREATE TORRENTS FOR EACH TRANSCODED VIDEO
# Create torrent file.
bt.create_torrent(transcode_config['output_file'])
output_file = transcode_config['output_file'] + '.tstream'
output_file = output_file[(output_file.rindex('/') + 1):]
# The torrent file is created in the same directory with the
# source file. Move it to the torrents directory.
#if not os.path.exists(
# os.path.join(output_file, config.TORRENTS_PATH)):
try:
shutil.move(transcode_config['output_file'] + '.tstream', \
config.TORRENTS_PATH)
except:
pass
# * SEED TORRENTS
Server.bit_torrent.start_torrent( \
os.path.join(config.TORRENTS_PATH, output_file),
config.MEDIA_PATH)
def log(self, msg, show_hdr=False):
hdr = ""
if show_hdr and self.file != None:
hdr = self.file.filename
if self.file.nr_line != None:
hdr += "(%d)" % (self.file.nr_line)
hdr += ": "
log_msg(hdr + msg)
def remove_files(self, files, path):
"""
Deletes files from a specified path.
"""
logger.log_msg('#%s: cleaning up...' % self.job_id)
for f in files:
os.unlink(os.path.join(path, f))
def add_node(self, *nodes):
'''Add a node to the tree.'''
for node in nodes:
node_id = str(node)
selected_nodes = self.all_nodes[node.node_name]
if node_id in selected_nodes:
log_msg("Overwriting node '%s' in '%s'" %
(node_id, node.node_name))
selected_nodes[node_id] = node
def remove_files(self, files, path):
"""
Deletes files from a specified path.
"""
logger.log_msg('#%s: cleaning up...' % self.job_id)
for f in files:
os.unlink(os.path.join(path, f))
def notify_completion(self, code):
logger.log_msg('#%s: notifying web server about the job completion...'\
% self.job_id)
if config.WS_COMPLETION[len(config.WS_COMPLETION) - 1] == '/':
url = config.WS_COMPLETION + code
else:
url = config.WS_COMPLETION + '/' + code
f = urllib.urlopen(url)
f.read()
async def on_message(self, message):
if self.bot.user in message.mentions:
await message.channel.send(
embed=Embed(
"Hey there!", footer="Use .help to see a list of commands."
).get_embed()
)
if message.author == self.bot.user:
log("Bot sent response")
else:
log_msg(message)
def notify_completion(self, code):
logger.log_msg('#%s: notifying web server about the job completion...'\
% self.job_id)
if config.WS_COMPLETION[len(config.WS_COMPLETION) - 1] == '/':
url = config.WS_COMPLETION + code
else:
url = config.WS_COMPLETION + '/' + code
f = urllib.urlopen(url)
f.read()
def _build_links(self):
for proc in self.procs.values():
missing = proc.link_types(self.types)
if missing:
names_missing = map(str, missing)
log_msg("Cannot find %s for procedure %s" %
(" and ".join(names_missing), proc))
# Add missing types
self.add_node(*missing)
def parse_quality(self, file_name):
"""Returns what I hope is the resolution from 'file_name' else returns boolean False.
"""
quality = re.search(r"(\d{3,4}p(?=\.)|\d{1}k(?=\.))", file_name)
if quality is not None:
return quality.group(0)
else:
log_msg(
"warn",
"NO QUALITY: Unable to obtain quality info for '{}'".format(
self.file_name))
return False
def notify_error(self, code):
logger.log_msg('#%s: notifying web server about the error...'\
% code)
if config.WS_ERROR[len(config.WS_ERROR) - 1] == '/':
url = config.WS_ERROR + code
else:
url = config.WS_ERROR + '/' + code
url = url + '/' + 'unreachable'
f = urllib.urlopen(url)
f.read()
def notify_error(self, code):
logger.log_msg('#%s: notifying web server about the error...'\
% self.job_id)
if config.WS_ERROR[len(config.WS_ERROR) - 1] == '/':
url = config.WS_ERROR + code
else:
url = config.WS_ERROR + '/' + code
url = url + '/' + 'internal_error'
f = urllib.urlopen(url)
f.read()
def transfer_in(self, raw_video):
"""
Transfers a raw video file from the Web Server.
@param raw_video raw video file name
"""
logger.log_msg('#%s: transfering in...' % self.job_id)
file_transfer = config.FILE_TRANSFERER_CLASS( \
config.RAW_VIDEOS_PATH, config.WS_UPLOAD_PATH)
file_transfer.get([raw_video])
file_transfer.close()
def transfer_in(self, raw_video):
"""
Transfers a raw video file from the Web Server.
@param raw_video raw video file name
"""
logger.log_msg('#%s: transfering in...' % self.job_id)
file_transfer = config.FILE_TRANSFERER_CLASS( \
config.RAW_VIDEOS_PATH, config.WS_UPLOAD_PATH)
file_transfer.get([raw_video])
file_transfer.close()
def transfer_out(self, local_files, local_path, remote_path):
"""
Transfers some local files to a remote path of the Web Server.
@param local_files list local files to transfer
@param remote_path destination path on the Web Server
"""
logger.log_msg('#%s: transfering out...' % self.job_id)
file_transfer = config.FILE_TRANSFERER_CLASS( \
local_path, remote_path)
file_transfer.put(local_files)
file_transfer.close()
def transfer_out(self, local_files, local_path, remote_path):
"""
Transfers some local files to a remote path of the Web Server.
@param local_files list local files to transfer
@param remote_path destination path on the Web Server
"""
logger.log_msg('#%s: transfering out...' % self.job_id)
file_transfer = config.FILE_TRANSFERER_CLASS( \
local_path, remote_path)
file_transfer.put(local_files)
file_transfer.close()
def parse_season(self, file_name):
# The regex looks for 1 to 2 digits preceeded by 's' or 'season' OR
# followed by 'x'
regex = re.compile(r"s\d{1,2}|season\s*\d{1,2}|\d{1,2}x\d{,2}", re.I)
match = regex.search(file_name)
if match is not None:
self.set_title(self.title.replace(match.group(), "").strip())
return "{:02d}".format(int(re.sub('[^0-9]', '', match.group())))
else:
log_msg(
"warn",
"NO SEASON: Unable to obtain season info for '{}'".format(
self.file_name))
return False
def parse_episode(self, file_name):
regex = re.compile(r"(?:x|e|episode)\d{1,2}", re.I)
match = regex.search(file_name)
if match is not None:
start_ndex = file_name.index(match.group())
self.set_title(
" ".join(self.file_name[:start_ndex - 1].split("."))
) # This will only work if 'episode|e|x' is in 'file_name'
return "{:02d}".format(int(re.sub('[^0-9]', '', match.group())))
else:
log_msg(
"warn",
"NO EPISODE: Unable to obtain episode info for '{}'".format(
self.file_name))
return False
def call_omdb(self):
api = "http://www.omdbapi.com/?apikey={}&s={}&y={}&type={}".format(
environ["OMDBKEY"], self.title.strip(),
self.year.strip() if self.year else "", self.vid_type)
req = requests.get(api)
results = []
if req.status_code == requests.codes.ok:
json = req.json()
if json["Response"] == "True":
results += [result["Title"] for result in json["Search"]]
else:
log_msg(
"error",
"OMDB: '{}' produced error {}".format(self.file_name,
req.status_code))
return results
def stop_torrent(self, torrent, remove_content=False):
"""
Stop leeching or seeding a file via BitTorrent.
!!! Only tested with torrents started with .tstream files. Not tested
for torrents started with URLs.
@param torrent .torrent file or URL
@param remove_content removes downloaded file
"""
downloads = self.session.get_downloads()
for dl in downloads:
tdef = dl.get_def()
if torrent.find(tdef.get_name()) == 0:
self.session.remove_download(dl, remove_content)
logger.log_msg('torrent "%s" stopped' % torrent)
break
def get_filenames(root_dir):
"""
Walks 'root_dir' in order to obtain a list of *.mkv/*.mp4 files.
:param root_dir: Root directory to walk
:param vid_type: Either 'movies' or 'tv'
:return: Either a list of files or an error message
"""
videos = []
for main_dir, sub_dirs, files in oswalk(root_dir):
for file in files:
if ossplitext(file)[1] in (".mkv", ".mp4"):
videos.append(ospathjoin(main_dir, file))
if len(videos) > 0:
return videos
else:
log_msg("error", "No video files found in {}".format(root_dir))
return False
def stop_torrent(self, torrent, remove_content=False):
"""
Stop leeching or seeding a file via BitTorrent.
!!! Only tested with torrents started with .tstream files. Not tested
for torrents started with URLs.
@param torrent .torrent file or URL
@param remove_content removes downloaded file
"""
downloads = self.session.get_downloads()
for dl in downloads:
tdef = dl.get_def()
if torrent.find(tdef.get_name()) == 0:
self.session.remove_download(dl, remove_content)
logger.log_msg('torrent "%s" stopped' % torrent)
break
def cross_check_title(self):
"""
This makes calls to the OMDB and TMDB APIs and stores the results
of those calls the 'omdb' and 'tmdb' variables. If the result is not
an empty list it is added to the 'results' list and passed to 'match_title'.
"""
omdb = self.call_omdb()
tmdb = self.call_tmdb()
results = []
for call_result in [omdb, tmdb]:
if len(call_result) > 0:
results += call_result
if len(results) > 0:
return self.match_title(results)
else:
log_msg(
"warn", "NO MATCH: Found zero possible titles for '{}'".format(
self.file_name))
return self.title
def process_filenames(files, vid_type, output_dir):
"""
Takes a list of files and passes their names to confirmTitle
to try and get the actual movie title before copying the file
to a the 'clean_titles' directory. Creating said directory
if necessary.
:param files: List of files to check
:param vid_type: Either "movies" or "series"
"""
osmakedirs(output_dir, exist_ok=True)
for file in files:
vid_type = file.split("/")[1].lower()
vid_obj = Series(file) if vid_type.lower() == "tv" else Movie(file) # This will have to be changed later to grab the torrent files as they finish
new_path = ospathjoin(output_dir, vid_obj.path)
if not osisdir(osdirname(new_path)):
osmakedirs(osdirname(new_path), exist_ok=True)
copy2(file, new_path)
log_msg("info", "CLEANED: '{}' has been renamed to '{}'.".format(osbasename(file), vid_obj.file_name))
log_msg("info", "NEW_PATH: '{}'".format(new_path))
def transcode(self, input_video, video_name, transcode_configs):
"""
Transcodes a video in each requested formats.
@param input_video input video file name
@param video_name a video name which must be a valid file name
@param transcode_configs a list of dictionaries with format settings
"""
logger.log_msg('#%s: transcoding...' % self.job_id)
transcoder = config.TRANSCODER_CLASS( \
input_file = os.path.join(config.RAW_VIDEOS_PATH, input_video), \
name = video_name, prog_bin = config.TRANSCODER_BIN)
transcoder.dest_path = config.MEDIA_PATH
# Transcode the raw video in each requested format.
# TODO report partial errors
for transcode_config in transcode_configs:
transcode_config['output_file'] = \
transcoder.transcode(**transcode_config)
def transcode(self, input_video, video_name, transcode_configs):
"""
Transcodes a video in each requested formats.
@param input_video input video file name
@param video_name a video name which must be a valid file name
@param transcode_configs a list of dictionaries with format settings
"""
logger.log_msg('#%s: transcoding...' % self.job_id)
transcoder = config.TRANSCODER_CLASS( \
input_file = os.path.join(config.RAW_VIDEOS_PATH, input_video), \
name = video_name, prog_bin = config.TRANSCODER_BIN)
transcoder.dest_path = config.MEDIA_PATH
# Transcode the raw video in each requested format.
# TODO report partial errors
for transcode_config in transcode_configs:
transcode_config['output_file'] = \
transcoder.transcode(**transcode_config)
def run(self):
while True:
url = self.queue_in.get()
try:
f = urllib.urlopen(url + 'get_load')
r = f.read()
parsed_r = json.loads(r)
except IOError:
self.queue_out.put( (url, None) )
logger.log_msg('%s: Failed to request load to %s' \
% (self.name, url), \
logger.LOG_LEVEL_ERROR)
continue
# Put response load to the output queue.
self.queue_out.put( (url, parsed_r['load']) )
logger.log_msg('%s: Received load %s from %s' \
% (self.name, parsed_r['load'], url), \
logger.LOG_LEVEL_INFO)
def run(self):
while True:
url = self.queue_in.get()
try:
f = urllib.urlopen(url + 'get_load')
r = f.read()
parsed_r = json.loads(r)
except IOError:
self.queue_out.put((url, None))
logger.log_msg('%s: Failed to request load to %s' \
% (self.name, url), \
logger.LOG_LEVEL_ERROR)
continue
# Put response load to the output queue.
self.queue_out.put((url, parsed_r['load']))
logger.log_msg('%s: Received load %s from %s' \
% (self.name, parsed_r['load'], url), \
logger.LOG_LEVEL_INFO)
def run(self):
while True:
(request, data) = self.queue.get()
urls = config.CIS_URLS[:]
code = json.loads(data)['code']
success = False
while len(urls) != 0:
cis = self.choose(urls)
# Request is forwarded to the chosen CIS.
try:
urllib.urlopen(cis + request, data)
except IOError:
logger.log_msg('#%s: Failed to forward request to %s' \
% (code, cis), \
logger.LOG_LEVEL_ERROR)
continue
success = True
logger.log_msg('#%s: Request forwarded to %s' \
% (code, cis), \
logger.LOG_LEVEL_INFO)
break
if len(urls) == 0 and not success:
logger.log_msg('#%s: Failed to forward request to any CIS' \
% code, \
logger.LOG_LEVEL_FATAL)
self.notify_error(code)
self.queue.task_done()
def run(self):
while True:
(request, data) = self.queue.get()
urls = config.CIS_URLS[:]
code = json.loads(data)['code']
success = False
while len(urls) != 0:
cis = self.choose(urls)
# Request is forwarded to the chosen CIS.
try:
urllib.urlopen(cis + request, data)
except IOError:
logger.log_msg('#%s: Failed to forward request to %s' \
% (code, cis), \
logger.LOG_LEVEL_ERROR)
continue
success = True
logger.log_msg('#%s: Request forwarded to %s' \
% (code, cis), \
logger.LOG_LEVEL_INFO)
break
if len(urls) == 0 and not success:
logger.log_msg('#%s: Failed to forward request to any CIS' \
% code, \
logger.LOG_LEVEL_FATAL)
self.notify_error(code)
self.queue.task_done()
def choose(self, urls):
self.tasks_queue.queue.clear()
self.loads_queue.queue.clear()
while len(urls) != 0:
# Choose k CIS machines.
k_urls = self.subset(urls)
# Find out their load by giving tasks to HTTPReqWorker-s.
for url in k_urls:
self.tasks_queue.put(url)
# Wait for load answers from HTTPReqWorker-s and choose the least
# loaded CIS machine.
best_url = None
best_load = sys.maxint
for i in range(0, self.k):
(url, load) = self.loads_queue.get()
if load == None:
continue
else:
load = int(load)
if load < best_load:
logger.log_msg('Got %s %s' % (url, load), \
logger.LOG_LEVEL_DEBUG)
best_load = load
best_url = url
if best_url != None:
break
#del( urls[ urls.index(best_url) ] )
logger.log_msg('Returning best_url = "%s"' % best_url, \
logger.LOG_LEVEL_DEBUG)
return best_url
def choose(self, urls):
self.tasks_queue.queue.clear()
self.loads_queue.queue.clear()
while len(urls) != 0:
# Choose k CIS machines.
k_urls = self.subset(urls)
# Find out their load by giving tasks to HTTPReqWorker-s.
for url in k_urls:
self.tasks_queue.put(url)
# Wait for load answers from HTTPReqWorker-s and choose the least
# loaded CIS machine.
best_url = None
best_load = sys.maxint
for i in range(0, self.k):
(url, load) = self.loads_queue.get()
if load == None:
continue
else:
load = int(load)
if load < best_load:
logger.log_msg('Got %s %s' % (url, load), \
logger.LOG_LEVEL_DEBUG)
best_load = load
best_url = url
if best_url != None:
break
#del( urls[ urls.index(best_url) ] )
logger.log_msg('Returning best_url = "%s"' % best_url, \
logger.LOG_LEVEL_DEBUG)
return best_url
def extract_thumbs(self, input_video, video_name, thumbs):
"""
Extracts thumbnail images from a video.
@param input_video input video file name
@param video_name a video name which must be a valid file name
@param thumbs use 'random' to extract a thumbnail image from a random
point of the video or use a positive integer n to extract n summary
thumbnail
"""
logger.log_msg('#%s: extracting image thumbnails...' % self.job_id)
# TODO report partial errors
thumb_extractor = config.THUMB_EXTRACTOR_CLASS( \
input_file = os.path.join(config.RAW_VIDEOS_PATH, input_video), \
name = video_name, \
prog_bin = config.THUMB_EXTRACTOR_BIN)
thumb_extractor.dest_path = config.THUMBS_PATH
if thumbs == 'random':
thumb_extractor.extract_random_thumb()
elif type(thumbs) is int and thumbs > 0:
thumb_extractor.extract_summary_thumbs(thumbs)
def extract_thumbs(self, input_video, video_name, thumbs):
"""
Extracts thumbnail images from a video.
@param input_video input video file name
@param video_name a video name which must be a valid file name
@param thumbs use 'random' to extract a thumbnail image from a random
point of the video or use a positive integer n to extract n summary
thumbnail
"""
logger.log_msg('#%s: extracting image thumbnails...' % self.job_id)
# TODO report partial errors
thumb_extractor = config.THUMB_EXTRACTOR_CLASS( \
input_file = os.path.join(config.RAW_VIDEOS_PATH, input_video), \
name = video_name, \
prog_bin = config.THUMB_EXTRACTOR_BIN)
thumb_extractor.dest_path = config.THUMBS_PATH
if thumbs == 'random':
thumb_extractor.extract_random_thumb()
elif type(thumbs) is int and thumbs > 0:
thumb_extractor.extract_summary_thumbs(thumbs)
def add_node(self, tree):
'''See DoxBlock.add_node'''
node = DoxSourceBlock.add_node(self, tree)
if node == None:
return None
# We first try to find the first non-source slice that precedes ours.
cs = self.classified_slice
while cs != None:
cs = cs.prev_slice
if cs.type == SLICE_BLOCK:
if cs.block and cs.block.target:
break
else:
log_msg("Cannot find target for 'Same' block")
return None
# We have the previous target: we now steal stuff from it.
source_node = cs.block.target
for block in source_node.get_blocks():
node.add_block(copy.copy(block))
return node
def create_blocks_from_classified_slices(classified_slices):
'''Create documentation block objects from the classified text slices.
In this phase, text slices are parsed and mapped to DoxBlock objects of
different kinds (``Intro'' blocks are mapped to ``DoxIntro'' blocks, etc).'''
# Get the text from the slices
if len(classified_slices) > 0:
text = classified_slices[0].text_slice.text
for cs in classified_slices:
if cs.text_slice.text != text:
raise ValueError("Slices are referring to different texts")
# For each block construct an appropriate DoxBlock object
doxblocks = []
for cs in classified_slices:
if cs.type == SLICE_BLOCK:
lines = get_block_lines(str(cs.text_slice))
if not lines:
log_msg('Invalid documentation block')
continue
paragraph = " ".join(lines).strip()
block_type, content = split_block(paragraph)
if not (block_type and content):
log_msg('Missing documentation identifier')
continue
block_type = block_type.lower()
target_on_left = block_type.startswith('<')
if target_on_left:
block_type = block_type[1:]
constructor = known_block_types.get(block_type)
if constructor != None:
# Call the constructor for the block
doxblock = constructor(cs, content, target_on_left=target_on_left)
cs.set_block(doxblock)
doxblocks.append(doxblock)
else:
log_msg("Unrecognized block '%s'" % block_type)
return doxblocks
def run(self):
while True:
job = Server.queue.get()
self.job_id = job['code']
# * TRANSFER RAW VIDEO IN
try:
self.transfer_in(job['raw_video'])
except cis_exceptions.FileAlreadyExistsException as e:
logger.log_msg('#%s: %s' \
% (job['code'], repr(e)), logger.LOG_LEVEL_ERROR)
self.notify_error(job['code'])
continue
except Exception as e:
logger.log_msg('#%s: error while transferring in: %s' \
% (job['code'], repr(e)), logger.LOG_LEVEL_FATAL)
self.notify_error(job['code'])
continue
# * TRANSCODE RAW VIDEO
try:
self.transcode(job['raw_video'], job['name'], \
job['transcode_configs'])
except cis_exceptions.FileAlreadyExistsException as e:
logger.log_msg('#%s: %s' \
% (job['code'], repr(e)), logger.LOG_LEVEL_ERROR)
self.notify_error(job['code'])
continue
except Exception as e:
logger.log_msg('#%s: error while transcoding: %s' \
% (job['code'], repr(e)), logger.LOG_LEVEL_FATAL)
self.notify_error(job['code'])
continue
# * EXTRACT THUMBNAIL IMAGES
if job['thumbs'] != 0:
try:
self.extract_thumbs(job['raw_video'], job['name'], \
job['thumbs'])
except cis_exceptions.FileAlreadyExistsException as e:
logger.log_msg('#%s: %s' \
% (job['code'], repr(e)), logger.LOG_LEVEL_ERROR)
self.notify_error(job['code'])
continue
except Exception as e:
logger.log_msg( \
'#%s: error while extracting thumbnail images: %s' \
% (job['code'], repr(e)), logger.LOG_LEVEL_FATAL)
self.notify_error(job['code'])
continue
# * CREATE TORRENTS AND START SEEDING OF TRANSCODED VIDEOS
self.seed(job['transcode_configs'])
# Torrent files.
files = [f for f in os.listdir(config.TORRENTS_PATH) \
if os.path.isfile(os.path.join( \
config.TORRENTS_PATH, f))]
torrent_files = fnmatch.filter(files, job['name'] + "_*")
# Thumbnail images files.
files = [f for f in os.listdir(config.THUMBS_PATH) \
if os.path.isfile(os.path.join( \
config.THUMBS_PATH, f))]
thumb_files = fnmatch.filter(files, job['name'] + "_*")
# * TRANSFER TORRENTS AND THUMBNAIL IMAGES OUT
try:
self.transfer_out(torrent_files, config.TORRENTS_PATH, \
config.WS_TORRENTS_PATH)
self.transfer_out(thumb_files, config.THUMBS_PATH, \
config.WS_THUMBS_PATH)
except Exception as e:
logger.log_msg('#%s: error while transferring out: %s' \
% (job['code'], repr(e)), logger.LOG_LEVEL_FATAL)
self.notify_error(job['code'])
continue
# * NOTIFY WEB SERVER ABOUT CONTENT INGESTION COMPLETION
try:
self.notify_completion(job['code'])
except Exception as e:
logger.log_msg(
'#%s: error while notifying web server about the job completion: %s' \
% (job['code'], repr(e)), logger.LOG_LEVEL_FATAL)
self.notify_error(job['code'])
continue
# * CLEANUP RAW VIDEOS AND THUMBNAIL IMAGES
self.remove_files([ job['raw_video'] ], config.RAW_VIDEOS_PATH)
self.remove_files(thumb_files, config.THUMBS_PATH)
# * JOB FINISHED
Server.queue.task_done()
Server.load -= job['weight']
logger.log_msg('#%s: finished' \
% job['code'], logger.LOG_LEVEL_INFO)
def run(self):
while True:
job = Server.queue.get()
self.job_id = job['code']
# * TRANSFER RAW VIDEO IN
try:
self.transfer_in(job['raw_video'])
except cis_exceptions.FileAlreadyExistsException as e:
logger.log_msg('#%s: %s' \
% (job['code'], repr(e)), logger.LOG_LEVEL_ERROR)
self.notify_error(job['code'])
continue
except Exception as e:
logger.log_msg('#%s: error while transferring in: %s' \
% (job['code'], repr(e)), logger.LOG_LEVEL_FATAL)
self.notify_error(job['code'])
continue
# * TRANSCODE RAW VIDEO
try:
self.transcode(job['raw_video'], job['name'], \
job['transcode_configs'])
except cis_exceptions.FileAlreadyExistsException as e:
logger.log_msg('#%s: %s' \
% (job['code'], repr(e)), logger.LOG_LEVEL_ERROR)
self.notify_error(job['code'])
continue
except Exception as e:
logger.log_msg('#%s: error while transcoding: %s' \
% (job['code'], repr(e)), logger.LOG_LEVEL_FATAL)
self.notify_error(job['code'])
continue
# * EXTRACT THUMBNAIL IMAGES
if job['thumbs'] != 0:
try:
self.extract_thumbs(job['raw_video'], job['name'], \
job['thumbs'])
except cis_exceptions.FileAlreadyExistsException as e:
logger.log_msg('#%s: %s' \
% (job['code'], repr(e)), logger.LOG_LEVEL_ERROR)
self.notify_error(job['code'])
continue
except Exception as e:
logger.log_msg( \
'#%s: error while extracting thumbnail images: %s' \
% (job['code'], repr(e)), logger.LOG_LEVEL_FATAL)
self.notify_error(job['code'])
continue
# * CREATE TORRENTS AND START SEEDING OF TRANSCODED VIDEOS
self.seed(job['transcode_configs'])
# Torrent files.
files = [f for f in os.listdir(config.TORRENTS_PATH) \
if os.path.isfile(os.path.join( \
config.TORRENTS_PATH, f))]
torrent_files = fnmatch.filter(files, job['name'] + "_*")
# Thumbnail images files.
files = [f for f in os.listdir(config.THUMBS_PATH) \
if os.path.isfile(os.path.join( \
config.THUMBS_PATH, f))]
thumb_files = fnmatch.filter(files, job['name'] + "_*")
# * TRANSFER TORRENTS AND THUMBNAIL IMAGES OUT
try:
self.transfer_out(torrent_files, config.TORRENTS_PATH, \
config.WS_TORRENTS_PATH)
self.transfer_out(thumb_files, config.THUMBS_PATH, \
config.WS_THUMBS_PATH)
except Exception as e:
logger.log_msg('#%s: error while transferring out: %s' \
% (job['code'], repr(e)), logger.LOG_LEVEL_FATAL)
self.notify_error(job['code'])
continue
# * NOTIFY WEB SERVER ABOUT CONTENT INGESTION COMPLETION
try:
self.notify_completion(job['code'])
except Exception as e:
logger.log_msg(
'#%s: error while notifying web server about the job completion: %s' \
% (job['code'], repr(e)), logger.LOG_LEVEL_FATAL)
self.notify_error(job['code'])
continue
# * CLEANUP RAW VIDEOS AND THUMBNAIL IMAGES
self.remove_files([job['raw_video']], config.RAW_VIDEOS_PATH)
self.remove_files(thumb_files, config.THUMBS_PATH)
# * JOB FINISHED
Server.queue.task_done()
Server.load -= job['weight']
logger.log_msg('#%s: finished' \
% job['code'], logger.LOG_LEVEL_INFO)
def set_section(self, section):
'''Set the section this node belongs to.'''
if self.section != None and section != self.section:
log_msg("Contraddictory section assignment for node '%s': "
"got '%s', was '%s'" % (self, section, self.section))
self.section = section
