def ngramize(self, document, minSize, maxSize, tagTokens, filters, stemmer):
"""
common tagTokens decomposition method
returns a dict of filtered NGram instances
@tagTokens == [[word1 tokens], [word2 tokens], etc]
"""
# content is the list of words from tagTokens
content = self.getContent(tagTokens)
stemmedcontent = []
for word in content:
stemmedcontent += [stemmer.stem(word)]
# tags is the list of tags from tagTokens
tags = self.getTag(tagTokens)
#logging.debug(tags)
for i in range(len(content)):
for n in range(minSize, maxSize + 1):
if len(content) >= i+n:
# id made from the stemmedcontent and label made from the real tokens
sha256ngid = getNodeId(stemmedcontent[i:n+i])
# ngram is already attached to this cable
if sha256ngid in document['edges']['NGram']:
document = addEdge(document, 'NGram', sha256ngid, 1)
else:
# else if ngram is not already in the corpus
#savedngram = self.mongodb.ngrams.find_one({'_id': sha256ngid})
if self.mongodb.ngrams.find_one({'_id': sha256ngid}) is None:
# create NGram object to pass it throug the filters
label = getNodeLabel(content[i:n+i])
ngram = {
'_id': sha256ngid,
'label': label,
'content': content[i:n+i],
'edges': {
'postag' : { label : tags[i:n+i] },
'label': { label : 1 }
},
'postag' : tags[i:n+i],
'category': "NGram",
'occs': 1,
}
# application defined filtering
if filtering.apply_filters(ngram, filters) is True:
# create the node
ngram['postag'] = ",".join(ngram['postag'])
# increment document occurrences
document = addEdge(document, 'NGram', sha256ngid, 1)
# save the new NGram
self.mongodb.ngrams.save(ngram)
print "saving ngram %s"%ngram['postag']
else:
# was already in the corpus and not in this document
#savedngram['occs'] += 1
# increments ngram total occurrences
self.mongodb.ngrams.update({'_id': sha256ngid}, {"$inc":{"occs":1}})
#self.mongodb.ngrams.save(savedngram)
document = addEdge(document, 'NGram', sha256ngid, 1)
def search(payload, method="general"):
""" Main search entrypoint
Args:
payload (dict): Search payload from Elementum.
method (str): Type of search, can be ``general``, ``movie``, ``show``, ``season`` or ``anime``
Returns:
list: All filtered results in the format Elementum expects
"""
log.debug("Searching with payload (%s): %s" % (method, repr(payload)))
if method == 'general':
if 'query' in payload:
payload['title'] = payload['query']
payload['titles'] = {'source': payload['query']}
else:
payload = {
'title': payload,
'titles': {
'source': payload
},
}
payload['titles'] = dict(
(k.lower(), v) for k, v in payload['titles'].iteritems())
# If titles[] exists in payload and there are special chars in titles[source]
# then we set a flag to possibly modify the search query
payload['has_special'] = 'titles' in payload and \
bool(payload['titles']) and \
'source' in payload['titles'] and \
any(c in payload['titles']['source'] for c in special_chars)
if payload['has_special']:
log.debug(
"Query title contains special chars, so removing any quotes in the search query"
)
if 'proxy_url' not in payload:
payload['proxy_url'] = ''
if 'internal_proxy_url' not in payload:
payload['internal_proxy_url'] = ''
if 'elementum_url' not in payload:
payload['elementum_url'] = ''
if 'silent' not in payload:
payload['silent'] = False
if 'skip_auth' not in payload:
payload['skip_auth'] = False
global request_time
global provider_names
global provider_results
global available_providers
provider_names = []
provider_results = []
available_providers = 0
request_time = time.time()
providers = get_enabled_providers(method)
if len(providers) == 0:
if not payload['silent']:
notify(translation(32060), image=get_icon_path())
log.error("No providers enabled")
return []
log.info(
"Burstin' with %s" %
", ".join([definitions[provider]['name'] for provider in providers]))
if get_setting('kodi_language', bool):
kodi_language = xbmc.getLanguage(xbmc.ISO_639_1)
if not kodi_language:
log.warning("Kodi returned empty language code...")
elif 'titles' not in payload or not payload['titles']:
log.info("No translations available...")
elif payload['titles'] and kodi_language not in payload['titles']:
log.info("No '%s' translation available..." % kodi_language)
p_dialog = xbmcgui.DialogProgressBG()
if not payload['silent']:
p_dialog.create('Elementum [COLOR FFFF6B00]Burst[/COLOR]',
translation(32061))
for provider in providers:
available_providers += 1
provider_names.append(definitions[provider]['name'])
task = Thread(target=run_provider, args=(provider, payload, method))
task.start()
providers_time = time.time()
total = float(available_providers)
# Exit if all providers have returned results or timeout reached, check every 100ms
while time.time() - providers_time < timeout and available_providers > 0:
timer = time.time() - providers_time
log.debug("Timer: %ds / %ds" % (timer, timeout))
if timer > timeout:
break
message = translation(
32062
) % available_providers if available_providers > 1 else translation(
32063)
if not payload['silent']:
p_dialog.update(int((total - available_providers) / total * 100),
message=message)
time.sleep(0.25)
if not payload['silent']:
p_dialog.close()
del p_dialog
if available_providers > 0:
message = u', '.join(provider_names)
message = message + translation(32064)
log.warning(message.encode('utf-8'))
if not payload['silent']:
notify(message, ADDON_ICON)
log.debug("all provider_results: %s" % repr(provider_results))
filtered_results = apply_filters(provider_results)
log.debug("all filtered_results: %s" % repr(filtered_results))
log.info("Providers returned %d results in %s seconds" %
(len(filtered_results), round(time.time() - request_time, 2)))
return filtered_results
def search(payload, method="general"):
""" Main search entrypoint
Args:
payload (dict): Search payload from Quasar.
method (str): Type of search, can be ``general``, ``movie``, ``show``, ``season`` or ``anime``
Returns:
list: All filtered results in the format Quasar expects
"""
log.debug("Searching with payload (%s): %s" % (method, repr(payload)))
if method == 'general':
payload = {'title': payload}
global request_time
global provider_names
global provider_results
global available_providers
provider_names = []
provider_results = []
available_providers = 0
request_time = time.time()
providers = get_enabled_providers()
if len(providers) == 0:
notify(translation(32060), image=get_icon_path())
log.error("No providers enabled")
return []
log.info(
"Burstin' with %s" %
", ".join([definitions[provider]['name'] for provider in providers]))
if get_setting("use_cloudhole", bool):
clearance, user_agent = get_cloudhole_clearance(get_cloudhole_key())
set_setting('clearance', clearance)
set_setting('user_agent', user_agent)
if get_setting('kodi_language', bool):
kodi_language = xbmc.getLanguage(xbmc.ISO_639_1)
if not kodi_language:
log.warning("Kodi returned empty language code...")
elif 'titles' not in payload or not payload['titles']:
log.info("No translations available...")
elif payload['titles'] and kodi_language not in payload['titles']:
log.info("No '%s' translation available..." % kodi_language)
p_dialog = xbmcgui.DialogProgressBG()
p_dialog.create('Quasar [COLOR FFFF6B00]Burst[/COLOR]', translation(32061))
for provider in providers:
available_providers += 1
provider_names.append(definitions[provider]['name'])
task = Thread(target=run_provider, args=(provider, payload, method))
task.start()
providers_time = time.time()
total = float(available_providers)
# Exit if all providers have returned results or timeout reached, check every 100ms
while time.time() - providers_time < timeout and available_providers > 0:
timer = time.time() - providers_time
log.debug("Timer: %ds / %ds" % (timer, timeout))
if timer > timeout:
break
message = translation(
32062
) % available_providers if available_providers > 1 else translation(
32063)
p_dialog.update(int((total - available_providers) / total * 100),
message=message)
time.sleep(0.25)
p_dialog.close()
del p_dialog
if available_providers > 0:
message = u', '.join(provider_names)
message = message + translation(32064)
log.warning(message.encode('utf-8'))
notify(message, ADDON_ICON)
log.debug("all provider_results: %s" % repr(provider_results))
filtered_results = apply_filters(provider_results)
log.debug("all filtered_results: %s" % repr(filtered_results))
log.info("Providers returned %d results in %s seconds" %
(len(filtered_results), round(time.time() - request_time, 2)))
return filtered_results
def search(method, payload_json, provider=""):
"""
Search for torrents - call providers
:param method: method of search
:type method: str
:param payload_json: parload in json format
:type payload_json: str
:param provider: name of provider, if the search is individually
:type provider: str
:return list of results
"""
global provider_results
global available_providers
global request_time
global provider_name
# reset global variables
provider_results = []
provider_name = []
available_providers = 0
request_time = time.clock()
# collect data
if len(provider) == 0:
addons = get_list_providers_enabled()
else:
addons = [provider]
if len(addons) == 0:
# return empty list
notify(string(32060), image=get_icon_path())
logger.log.info("No providers installed")
return {'results': 0, 'duration': "0 seconds", 'magnets': []}
p_dialog = xbmcgui.DialogProgressBG()
p_dialog.create('Magnetic Manager', string(32061))
for addon in addons:
available_providers += 1
provider_name.append(addon)
task = Thread(target=run_provider, args=(addon, method, payload_json))
task.start()
providers_time = time.clock()
total = float(available_providers)
# while all providers have not returned results or timeout not reached
time_out = min(get_setting("timeout", int), 60)
# if all providers have returned results exit
# check every 100ms
while time.clock() - providers_time < time_out and available_providers > 0:
xbmc.sleep(100)
message = string(
32062
) % available_providers if available_providers > 1 else string(32063)
p_dialog.update(int((total - available_providers) / total * 100),
message=message)
# time-out provider
if len(provider_name) > 0:
message = ', '.join(provider_name)
message = message.replace('script.magnetic.',
'').title() + string(32064)
logger.log.info(message)
notify(message, ADDON_ICON)
# filter magnets and append to results
filtered_results = dict(magnets=filtering.apply_filters(provider_results))
# append number and time on payload
filtered_results['results'] = len(filtered_results['magnets'])
filtered_results['duration'] = str(
"%.1f" % round(time.clock() - request_time, 2)) + " seconds"
logger.log.info("Providers search returned: %s results in %s" %
(str(len(provider_results)), filtered_results['duration']))
# destroy notification object
p_dialog.close()
del p_dialog
return filtered_results
def ngramize(self, document, minSize, maxSize, tagTokens, filters,
stemmer):
"""
common tagTokens decomposition method
returns a dict of filtered NGram instances
@tagTokens == [[word1 tokens], [word2 tokens], etc]
"""
# content is the list of words from tagTokens
content = self.getContent(tagTokens)
stemmedcontent = []
for word in content:
stemmedcontent += [stemmer.stem(word)]
# tags is the list of tags from tagTokens
tags = self.getTag(tagTokens)
#logging.debug(tags)
for i in range(len(content)):
for n in range(minSize, maxSize + 1):
if len(content) >= i + n:
# id made from the stemmedcontent and label made from the real tokens
sha256ngid = getNodeId(stemmedcontent[i:n + i])
# ngram is already attached to this cable
if sha256ngid in document['edges']['NGram']:
document = addEdge(document, 'NGram', sha256ngid, 1)
else:
# else if ngram is not already in the corpus
#savedngram = self.mongodb.ngrams.find_one({'_id': sha256ngid})
if self.mongodb.ngrams.find_one({'_id': sha256ngid
}) is None:
# create NGram object to pass it throug the filters
label = getNodeLabel(content[i:n + i])
ngram = {
'_id': sha256ngid,
'label': label,
'content': content[i:n + i],
'edges': {
'postag': {
label: tags[i:n + i]
},
'label': {
label: 1
}
},
'postag': tags[i:n + i],
'category': "NGram",
'occs': 1,
}
# application defined filtering
if filtering.apply_filters(ngram, filters) is True:
# create the node
ngram['postag'] = ",".join(ngram['postag'])
# increment document occurrences
document = addEdge(document, 'NGram',
sha256ngid, 1)
# save the new NGram
self.mongodb.ngrams.save(ngram)
print "saving ngram %s" % ngram['postag']
else:
# was already in the corpus and not in this document
#savedngram['occs'] += 1
# increments ngram total occurrences
self.mongodb.ngrams.update({'_id': sha256ngid},
{"$inc": {
"occs": 1
}})
#self.mongodb.ngrams.save(savedngram)
document = addEdge(document, 'NGram', sha256ngid,
1)
def search(payload, method="general"):
""" Main search entrypoint
Args:
payload (dict): Search payload from Quasar.
method (str): Type of search, can be ``general``, ``movie``, ``show``, ``season`` or ``anime``
Returns:
list: All filtered results in the format Quasar expects
"""
log.debug("Searching with payload (%s): %s" % (method, repr(payload)))
if method == 'general':
payload = {
'title': payload
}
global request_time
global provider_names
global provider_results
global available_providers
provider_names = []
provider_results = []
available_providers = 0
request_time = time.time()
providers = get_enabled_providers()
if len(providers) == 0:
notify(translation(32060), image=get_icon_path())
log.error("No providers enabled")
return []
log.info("Burstin' with %s" % ", ".join([definitions[provider]['name'] for provider in providers]))
if get_setting("use_cloudhole", bool):
clearance, user_agent = get_cloudhole_clearance(get_cloudhole_key())
set_setting('clearance', clearance)
set_setting('user_agent', user_agent)
if get_setting('kodi_language', bool):
kodi_language = xbmc.getLanguage(xbmc.ISO_639_1)
if not kodi_language:
log.warning("Kodi returned empty language code...")
elif 'titles' not in payload or not payload['titles']:
log.info("No translations available...")
elif payload['titles'] and kodi_language not in payload['titles']:
log.info("No '%s' translation available..." % kodi_language)
p_dialog = xbmcgui.DialogProgressBG()
p_dialog.create('Quasar [COLOR FFFF6B00]Burst[/COLOR]', translation(32061))
for provider in providers:
available_providers += 1
provider_names.append(definitions[provider]['name'])
task = Thread(target=run_provider, args=(provider, payload, method))
task.start()
providers_time = time.time()
total = float(available_providers)
# Exit if all providers have returned results or timeout reached, check every 100ms
while time.time() - providers_time < timeout and available_providers > 0:
timer = time.time() - providers_time
log.debug("Timer: %ds / %ds" % (timer, timeout))
if timer > timeout:
break
message = translation(32062) % available_providers if available_providers > 1 else translation(32063)
p_dialog.update(int((total - available_providers) / total * 100), message=message)
time.sleep(0.25)
p_dialog.close()
del p_dialog
if available_providers > 0:
message = u', '.join(provider_names)
message = message + translation(32064)
log.warning(message.encode('utf-8'))
notify(message, ADDON_ICON)
log.debug("all provider_results: %s" % repr(provider_results))
filtered_results = apply_filters(provider_results)
log.debug("all filtered_results: %s" % repr(filtered_results))
log.info("Providers returned %d results in %s seconds" % (len(filtered_results), round(time.time() - request_time, 2)))
return filtered_results