def generate_music_nabresponse(self):
dstring = []
if(self.args.has_key('artist')):
dstring.append(SearchModule.sanitize_strings(self.args['artist']))
if(self.args.has_key('album')):
dstring.append(SearchModule.sanitize_strings(self.args['album']))
if(self.args.has_key('track')):
dstring.append(SearchModule.sanitize_strings(self.args['track']))
if(self.args.has_key('year')):
dstring.append(SearchModule.sanitize_strings(self.args['year']))
music_search_str = ''
for i in xrange(len(dstring)):
if(len(dstring[i]) and i<len(dstring)-1):
music_search_str = music_search_str + dstring[i]
print music_search_str
#~ print movie_search_str
self.searchstring = music_search_str
self.typesearch = 0
#~ compile results
#~ results = SearchModule.performSearch(movie_search_str, self.cfg )
#~ flatten and summarize them
#~ cleaned_results = megasearch.summary_results(results,movie_search_str)
#~ render XML
#~ return self.cleanUpResultsXML(cleaned_results)
return 'm'
def dosearch(self, args):
#~ restore originals
self.cfg = copy.deepcopy(self.cfg_cpy)
if('q' not in args):
self.results = []
return self.results
nuqry = args['q'] + ' ' + self.cgen['searchaddontxt']
self.logic_items = self.logic_expr.findall(nuqry)
self.qry_nologic = self.logic_expr.sub(" ",nuqry)
if('selcat' in args):
if(args['selcat'] != ""):
self.qry_nologic += " " + args['selcat']
#~ speed class
speed_class_sel = 1
if('tm' in args):
speed_class_sel = int(args['tm'])
#~ speed class deepsearch
self.ds.set_timeout_speedclass(speed_class_sel)
#~ speed class Nabbased
for conf in self.cfg :
if ( (conf['speed_class'] <= speed_class_sel) and (conf['valid'])):
conf['timeout'] = self.cgen['timeout_class'][ speed_class_sel ]
#~ print conf['type'] + " " + str(conf['timeout'] ) + ' ' + str(speed_class_sel )
else:
conf['valid'] = 0
if( len(args['q']) == 0 ):
if('selcat' in args):
if(len(args['selcat'])==0):
self.results = []
return self.results
else:
self.results = []
return self.results
if(self.qry_nologic.replace(" ", "") == ""):
self.results = []
return self.results
self.cleancache()
#~ cache hit, no server report
self.returncode_fine['code'] = 2
self.resultsraw = self.chkforcache(self.wrp.chash64_encode(SearchModule.sanitize_strings(self.qry_nologic)), speed_class_sel)
if( self.resultsraw is None):
self.resultsraw = SearchModule.performSearch(self.qry_nologic, self.cfg, self.ds )
self.prepareretcode();
if( self.cgen['smartsearch'] == 1):
#~ smartsearch
self.results = summary_results(self.resultsraw, self.qry_nologic, self.logic_items)
else:
#~ no cleaning just flatten in one array
self.results = []
for provid in xrange(len(self.resultsraw)):
for z in xrange(len(self.resultsraw[provid])):
if (self.resultsraw[provid][z]['title'] != None):
self.results.append(self.resultsraw[provid][z])
def generate_music_nabresponse(self):
dstring = []
if self.args.has_key("artist"):
dstring.append(SearchModule.sanitize_strings(self.args["artist"]))
if self.args.has_key("album"):
dstring.append(SearchModule.sanitize_strings(self.args["album"]))
if self.args.has_key("track"):
dstring.append(SearchModule.sanitize_strings(self.args["track"]))
if self.args.has_key("year"):
dstring.append(SearchModule.sanitize_strings(self.args["year"]))
music_search_str = ""
for i in xrange(len(dstring)):
if len(dstring[i]) and i < len(dstring) - 1:
music_search_str = music_search_str + dstring[i]
print music_search_str
# ~ print movie_search_str
self.searchstring = music_search_str
self.typesearch = 0
# ~ compile results
# ~ results = SearchModule.performSearch(movie_search_str, self.cfg )
# ~ flatten and summarize them
# ~ cleaned_results = megasearch.summary_results(results,movie_search_str)
# ~ render XML
# ~ return self.cleanUpResultsXML(cleaned_results)
return "m"
def generate_tvserie_nabresponse(self,tvrage_show ):
#~ compile string
season_num = self.args.get('season',-1, type=int)
relaxed_seasonmatch = 0
serie_search_str = SearchModule.sanitize_strings(tvrage_show['showtitle'])
if(self.args.has_key('ep')):
ep_num = self.args.get('ep',-1, type=int)
serie_search_str = serie_search_str + '.s%02d' % season_num + 'e%02d' % ep_num
else:
serie_search_str = serie_search_str + '.s%02d' % season_num
relaxed_seasonmatch = 1
self.typesearch = 1
self.searchstring = serie_search_str
#~ compile results
results = SearchModule.performSearch(serie_search_str, self.cfg , self.cfg_ds )
cleaned_results = []
if(relaxed_seasonmatch):
#~ no cleaning just flatten in one array
for provid in xrange(len(results)):
if(results[provid] is not None):
for z in xrange(len(results[provid])):
cleaned_results.append(results[provid][z])
else:
#~ flatten and summarize them
cleaned_results = megasearch.summary_results(results,serie_search_str)
#~ render XML
return self.cleanUpResultsXML(cleaned_results)
def generate_tsearch_nabresponse(self):
if(self.args.has_key('q')):
freesearch_str = SearchModule.sanitize_strings(self.args['q'])
self.searchstring = freesearch_str
self.typesearch = 2
#~ compile results
results = SearchModule.performSearch(freesearch_str, self.cfg, self.cfg_ds )
#~ flatten and summarize them
cleaned_results = megasearch.summary_results(results, freesearch_str)
#~ render XML
return self.cleanUpResultsXML(cleaned_results)
def __init__(self):
import urlparse
cfgsets = config_settings.CfgSettings()
self.cgen = cfgsets.cgen
self.logsdir = SearchModule.resource_path('logs/nzbmegasearch.log')
self.scriptsdir = SearchModule.resource_path('get_stats.sh')
self.cfg_urlidx = []
self.excludeurls= ['http://ftdworld.net', 'https://nzbx.co']
if(cfgsets.cfg is not None):
self.config = cfgsets.cfg
for i in xrange(len(self.config)):
if(self.config[i]['builtin'] == 0):
self.cfg_urlidx.append(i)
def search():
data = request.query.data
str_lim = request.query.limit
columns = request.query.getall("selected[]")
logging.debug("search(). columns=" + str(columns))
if(str_lim == ''):
limit = 0
else:
limit = int(str_lim)
callback_name = cgi.escape(request.query.callback)
logging.debug("search(). callback=" + str(callback_name))
res = SearchModule.search_by_id(data, limit, columns, True)
add_list_to_process_queue(res[0:10])
# to only show a few columns (uggly stuff)
if(len(columns) == 0):
show = ["sha1", "description", "size"]
else:
show = ["sha1"]
for col in columns:
dic = tree_menu.ids[int(col)]
path = str(dic["path"]).split('.')[-1]
show.append(path)
responsex = {}
responsex["normal"] = res
responsex["show"] = show
return jsonp(clean_tree(responsex), callback_name)
def search():
data = request.query.data
str_lim = request.query.limit
columns = request.query.getall("selected[]")
#print(request.query.keys())
print(columns)
if (str_lim == ''):
limit = 0
else:
limit = int(str_lim)
callback_name = cgi.escape(request.query.callback)
print "callback=" + str(callback_name)
res = SearchModule.search_by_id(data, limit, columns)
add_list_to_process_queue(res[0:10])
#para que muestre solo algunas columnas (gronchada)
if (len(columns) == 0):
show = ["sha1", "description", "size"]
else:
show = ["sha1"]
for col in columns:
dic = tree_menu.ids[int(col)]
path = str(dic["path"]).split('.')[-1]
show.append(path)
responsex = {}
responsex["normal"] = res
responsex["show"] = show
return jsonp(responsex, callback_name)
def api_batch_process_file():
logging.debug("api_batch_process_file(): Running Batch process")
file_hashes = request.forms.get('file_hash')
# transform file_hashes in a list of hashes
if file_hashes is None:
return jsonize({"Error: file_hash parameter is missing."})
not_found = []
added_to_queue = 0
downloaded_from_vt = 0
for hash_id in file_hashes.split("\n"):
hash_id = clean_hash(hash_id)
if hash_id is None:
continue
data = "1=" + str(hash_id)
res = SearchModule.search_by_id(data, 1, [], True)
if (len(res) == 0):
not_found.append(hash_id)
continue
else:
sha1 = res[0]["sha1"]
added_to_queue += 1
logging.debug(str(hash_id) + " added to queue")
add_hash_to_process_queue(sha1)
responsex = str(added_to_queue) + " files added to the process queue.\n"
if (downloaded_from_vt > 0):
responsex += str(downloaded_from_vt) + " new hashes.\n"
if (len(not_found) != 0):
responsex += str(len(not_found)) + " hashes not found.\n"
responsex += "Not Found:\n"
for aux in not_found:
responsex = responsex + str(aux) + "\n"
return jsonize({"message": responsex})
def __init__(self, conf, cgen):
self.trendsdir = SearchModule.resource_path('logs/')
self.timeout = cgen['default_timeout']
self.movie_trend = []
self.movie_trend_ts = 0
self.show_trend = []
self.show_trend_ts = 0
self.sugg_info = []
self.active_trend = 1
self.trends_refreshrate = cgen['trends_refreshrate']
self.detached_trendpolling = cgen['large_server']
self.best_k = cgen['trends_qty']
self.cgen = cgen
self.logic_expr = re.compile("(?:^|\s)([-+])(\w+)")
self.predb_info = []
self.tvrage_rqheaders = {
'Connection': 'keep-alive;',
'Cache-Control': 'max-age=0',
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'User-Agent':
'Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.52 Safari/537.17',
'Referer': 'http://services.tvrage.com/info.php?page=main',
'Accept-Encoding': 'gzip,deflate,sdch',
'Accept-Language': 'en-US,en;q=0.8',
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3'
}
if (int(cgen['general_trend']) == 0):
self.active_trend = 0
def search():
data = request.query.data
str_lim = request.query.limit
columns = request.query.getall("selected[]")
logging.debug("search(). columns=" + str(columns))
if (str_lim == ''):
limit = 0
else:
limit = int(str_lim)
callback_name = cgi.escape(request.query.callback)
logging.debug("search(). callback=" + str(callback_name))
res = SearchModule.search_by_id(data, limit, columns, True)
add_list_to_process_queue(res[0:10])
# to only show a few columns (uggly stuff)
if (len(columns) == 0):
show = ["sha1", "description", "size"]
else:
show = ["sha1"]
for col in columns:
dic = tree_menu.ids[int(col)]
path = str(dic["path"]).split('.')[-1]
show.append(path)
responsex = {}
responsex["normal"] = res
responsex["show"] = show
return jsonp(clean_tree(responsex), callback_name)
def __init__(self, conf, cgen):
self.trendsdir = SearchModule.resource_path('logs/')
self.timeout = cgen['default_timeout']
self.movie_trend = []
self.movie_trend_ts = 0
self.show_trend = []
self.show_trend_ts = 0
self.sugg_info = []
self.active_trend = 1
self.trends_refreshrate = cgen['trends_refreshrate']
self.detached_trendpolling = cgen['large_server']
self.best_k = cgen['trends_qty']
self.cgen = cgen
self.logic_expr = re.compile("(?:^|\s)([-+])(\w+)")
self.predb_info = []
self.tvrage_rqheaders = {
'Connection': 'keep-alive;' ,
'Cache-Control': 'max-age=0',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'User-Agent': 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.52 Safari/537.17',
'Referer': 'http://services.tvrage.com/info.php?page=main',
'Accept-Encoding': 'gzip,deflate,sdch',
'Accept-Language': 'en-US,en;q=0.8',
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3'
}
if(int(cgen['general_trend']) == 0):
self.active_trend = 0
def search():
data=request.query.data
str_lim=request.query.limit
columns=request.query.getall("selected[]")
#print(request.query.keys())
print(columns)
if(str_lim==''):
limit=0
else:
limit=int(str_lim)
callback_name = cgi.escape(request.query.callback)
print "callback="+str(callback_name)
res=SearchModule.search_by_id(data,limit,columns)
add_list_to_process_queue(res[0:10])
#para que muestre solo algunas columnas (gronchada)
if(len(columns)==0):
show=["sha1","description","size"]
else:
show=["sha1"]
for col in columns:
dic=tree_menu.ids[int(col)]
path=str(dic["path"]).split('.')[-1]
show.append(path)
responsex={}
responsex["normal"]=res
responsex["show"]=show
return jsonp(responsex,callback_name)
def filter_obsolete_providers(self): #~ avoids obsolete modules to appear in the search routine #~ this is an additional safety measure if 'loadedModules' not in globals(): SearchModule.loadSearchModules() saved_cfg = [] for index in xrange(len(self.cfg)): index_found = False for module in SearchModule.loadedModules: if( module.typesrch == self.cfg[index]['type']): index_found = True if(index_found is True): saved_cfg.append(self.cfg[index]) self.cfg = copy.deepcopy(saved_cfg)
def api_batch_process_file():
logging.debug("api_batch_process_file(): Running Batch process")
file_hashes = request.forms.get('file_hash')
# transform file_hashes in a list of hashes
if file_hashes is None:
return jsonize({"Error: file_hash parameter is missing."})
not_found = []
added_to_queue = 0
downloaded_from_vt = 0
for hash_id in file_hashes.split("\n"):
hash_id = clean_hash(hash_id)
if hash_id is None:
continue
data = "1=" + str(hash_id)
res = SearchModule.search_by_id(data, 1, [], True)
if(len(res) == 0):
not_found.append(hash_id)
continue
else:
sha1 = res[0]["sha1"]
added_to_queue += 1
logging.debug(str(hash_id) + " added to queue")
add_hash_to_process_queue(sha1)
responsex = str(added_to_queue) + " files added to the process queue.\n"
if(downloaded_from_vt > 0):
responsex += str(downloaded_from_vt) + " new hashes.\n"
if(len(not_found) != 0):
responsex += str(len(not_found)) + " hashes not found.\n"
responsex += "Not Found:\n"
for aux in not_found:
responsex = responsex + str(aux) + "\n"
return jsonize({"message": responsex})
def get_result_from_av():
hash_id=request.query.file_hash
if len(hash_id) == 0:
response.code = 400
return jsonize({'error': 4, 'error_message':'file_hash parameter is missing.'})
hash_id=clean_hash(hash_id)
if not valid_hash(hash_id):
return jsonize({'error': 5, 'error_message':'Invalid hash format.'})
if(len(hash_id)!=40):
data="1="+str(hash_id)
res=SearchModule.search_by_id(data,1,[],True)
if(len(res)==0):
response.code = 400
return jsonize({'error': 6, 'error_message':'File not found'})
else:
sha1=res[0]["sha1"]
else:
sha1=hash_id
if(vt_key()):
av_result=get_av_result(sha1)
else:
return jsonize({'error': 7, "error_message": "Error: VirusTotal API key missing from secrets.py file"})
if(av_result==None):
return jsonize({"error": 8, "error_message": "Cannot get analysis (hash not found in VT? out of credits?)"})
return jsonize({"message": "AV scans downloaded."})
def ask(self, arguments):
self.args = arguments
self.search_str = SearchModule.sanitize_strings(
self.args['q']).replace(".", " ")
movieinfo = self.imdb_titlemovieinfo()
sugg_info_raw = self.movie_bestmatch(movieinfo)
self.sugg_info = self.prepareforquery(sugg_info_raw)
def dosearch(self, args):
#~ restore originals
self.cfg = copy.deepcopy(self.cfg_cpy)
if ('q' not in args):
self.results = []
return self.results
nuqry = args['q'] + ' ' + self.cgen['searchaddontxt']
self.logic_items = self.logic_expr.findall(nuqry)
self.qry_nologic = self.logic_expr.sub(" ", nuqry)
if ('selcat' in args):
if (args['selcat'] != ""):
self.qry_nologic += " " + args['selcat']
#~ speed class
speed_class_sel = 1
if ('tm' in args):
speed_class_sel = int(args['tm'])
#~ speed class deepsearch
self.ds.set_timeout_speedclass(speed_class_sel)
#~ speed class Nabbased
for conf in self.cfg:
if ((conf['speed_class'] <= speed_class_sel) and (conf['valid'])):
conf['timeout'] = self.cgen['timeout_class'][speed_class_sel]
#~ print conf['type'] + " " + str(conf['timeout'] ) + ' ' + str(speed_class_sel )
else:
conf['valid'] = 0
if (len(args['q']) == 0):
if ('selcat' in args):
if (len(args['selcat']) == 0):
self.results = []
return self.results
else:
self.results = []
return self.results
if (self.qry_nologic.replace(" ", "") == ""):
self.results = []
return self.results
self.cleancache()
self.resultsraw = self.chkforcache(
self.wrp.chash64_encode(self.qry_nologic), speed_class_sel)
if (self.resultsraw is None):
self.resultsraw = SearchModule.performSearch(
self.qry_nologic, self.cfg, self.ds)
if (self.cgen['smartsearch'] == 1):
#~ smartsearch
self.results = summary_results(self.resultsraw, self.qry_nologic,
self.logic_items)
else:
#~ no cleaning just flatten in one array
self.results = []
for provid in xrange(len(self.resultsraw)):
for z in xrange(len(self.resultsraw[provid])):
if (self.resultsraw[provid][z]['title'] != None):
self.results.append(self.resultsraw[provid][z])
def html_builtin_output(cffile, genopt):
count = 0
if 'SearchModule.loadedModules' not in globals():
SearchModule.loadSearchModules()
cffileb = []
for module in SearchModule.loadedModules:
if(module.builtin):
option='checked=yes'
flogin=0
login_name = ''
login_pwd = ''
if(module.active == 0):
option=''
for i in xrange(len(cffile)):
if(cffile[i]['type'] == module.typesrch):
if(cffile[i]['valid'] == '0'):
option=''
else:
option='checked=yes'
login_name=cffile[i]['login']
login_pwd=cffile[i]['pwd']
if(module.login == 1):
flogin = 1
tmpcfg= {'stchk' : option,
'humanname' : module.name,
'idx' : count,
'type' : module.typesrch,
'flogin': flogin,
'loginname': login_name,
'loginpwd': login_pwd,
}
cffileb.append(tmpcfg)
count = count + 1
count = 0
for i in xrange(len(cffile)):
if(cffile[i]['builtin'] == 0):
cffile[i]['idx'] = count
count = count + 1
return render_template('config.html', cfg=cffile, cnt=count, genopt = genopt, cnt_max=MAX_PROVIDER_NUMBER, cfg_bi=cffileb)
def generate_tvserie_nabresponse(self,tvrage_show ):
#~ compile string
season_num = self.args.get('season',-1, type=int)
serie_search_str = SearchModule.sanitize_strings(tvrage_show['showtitle'])
if(self.args.has_key('ep')):
ep_num = self.args.get('ep',-1, type=int)
serie_search_str = serie_search_str + '.s%02d' % season_num + 'e%02d' % ep_num
else:
serie_search_str = serie_search_str + '.s%02d' % season_num
self.typesearch = 1
self.searchstring = serie_search_str
#~ compile results
results = SearchModule.performSearch(serie_search_str, self.cfg )
#~ flatten and summarize them
cleaned_results = megasearch.summary_results(results,serie_search_str)
#~ render XML
return self.cleanUpResultsXML(cleaned_results)
def generate_tvserie_nabresponse(self,tvrage_show ):
#~ compile string
season_num = self.args.get('season',-1, type=int)
serie_search_str = SearchModule.sanitize_strings(tvrage_show['showtitle'])
if(self.args.has_key('ep')):
ep_num = self.args.get('ep',-1, type=int)
serie_search_str = serie_search_str + '.s%02d' % season_num + 'e%02d' % ep_num
else:
serie_search_str = serie_search_str + '.s%02d' % season_num
self.typesearch = 1
self.searchstring = serie_search_str
#~ compile results
results = SearchModule.performSearch(serie_search_str, self.cfg )
#~ flatten and summarize them
cleaned_results = megasearch.summary_results(results,serie_search_str)
#~ render XML
return self.cleanUpResultsXML(cleaned_results)
def filter_obsolete_providers(self):
#~ avoids obsolete modules to appear in the search routine
#~ this is an additional safety measure
if 'loadedModules' not in globals():
SearchModule.loadSearchModules()
saved_cfg = []
if (self.cfg is not None):
for index in xrange(len(self.cfg)):
index_found = False
for module in SearchModule.loadedModules:
if (module.typesrch == self.cfg[index]['type']):
index_found = True
if (index_found is True):
saved_cfg.append(self.cfg[index])
self.cfg = copy.deepcopy(saved_cfg)
def api_batch_process_debug_file():
yield "<html><body><pre>"
yield "Running Batch process\n"
file_hashes = request.forms.get('file_hash')
#print(dir(request.forms))
#print(request.forms.keys())
#transformar file_hashes a una lista de hashes
not_found = []
added_to_queue = 0
downloaded_from_vt = 0
for hash_id in file_hashes.split("\n"):
hash_id = clean_hash(hash_id)
if hash_id is None:
continue
data = "1=" + hash_id
res = SearchModule.search_by_id(data, 1, [], False)
if (len(res) == 0):
print "downloading " + str(hash_id) + " from vt"
sha1 = SearchModule.add_file_from_vt(hash_id)
if (sha1 == None):
not_found.append(hash_id)
continue
else:
downloaded_from_vt += 1
else:
sha1 = res[0]["sha1"]
added_to_queue += 1
add_hash_to_process_queue(sha1)
if (env['auto_get_av_result']):
get_av_result(sha1)
yield str(sha1) + "\n"
responsex = str(added_to_queue) + " files added to the process queue.\n"
if (downloaded_from_vt > 0):
responsex += str(downloaded_from_vt) + " new hashes.\n"
if (len(not_found) != 0):
responsex += str(len(not_found)) + " hashes not found.\n"
responsex += "Not Found:\n"
for aux in not_found:
responsex = responsex + str(aux) + "\n"
yield responsex
yield "END"
def api_batch_process_debug_file():
yield "<html><body><pre>"
yield "Running Batch process\n"
file_hashes=request.forms.get('file_hash')
#print(dir(request.forms))
#print(request.forms.keys())
#transformar file_hashes a una lista de hashes
not_found=[]
added_to_queue=0
downloaded_from_vt=0
for hash_id in file_hashes.split("\n"):
hash_id=clean_hash(hash_id)
if hash_id is None:
continue
data="1="+hash_id
res=SearchModule.search_by_id(data,1,[],False)
if(len(res)==0):
print "downloading "+str(hash_id)+" from vt"
sha1=SearchModule.add_file_from_vt(hash_id)
if(sha1==None):
not_found.append(hash_id)
continue
else:
downloaded_from_vt+=1
else:
sha1=res[0]["sha1"]
added_to_queue+=1
add_hash_to_process_queue(sha1)
yield str(sha1)+"\n"
responsex=str(added_to_queue)+" files added to the process queue.\n"
if(downloaded_from_vt > 0):
responsex+=str(downloaded_from_vt)+" new hashes.\n"
if(len(not_found)!=0):
responsex+=str(len(not_found))+ " hashes not found.\n"
responsex+="Not Found:\n"
for aux in not_found:
responsex=responsex+str(aux)+"\n"
yield responsex
yield "END"
def prepareforquery_show(self, sugg_info_raw, lastepisode, sugg_info):
for i in xrange(len(lastepisode)):
si = {'searchstr': SearchModule.sanitize_strings(sugg_info_raw['title'])
+ '.S%02d' % int(lastepisode[i]['season'])
+ 'E%02d' % int(lastepisode[i]['ep']),
'prettytxt': sugg_info_raw['title'] + ' S%02d ' % int(lastepisode[i]['season'])
+ 'E%02d' % int(lastepisode[i]['ep']),
'imdb_url': sugg_info_raw['tvdb_url']}
sugg_info.append(si)
return sugg_info
def prepareforquery_show(self, sugg_info_raw, lastepisode, sugg_info):
for i in xrange(len(lastepisode)):
si = {'searchstr': SearchModule.sanitize_strings(sugg_info_raw['title'])
+ '.S%02d' % int(lastepisode[i]['season'])
+ 'E%02d' % int(lastepisode[i]['ep']),
'prettytxt': sugg_info_raw['title'] + ' S%02d ' % int(lastepisode[i]['season'])
+ 'E%02d' % int(lastepisode[i]['ep']),
'imdb_url': sugg_info_raw['tvdb_url']}
sugg_info.append(si)
return sugg_info
def get_package_file():
tmp_folder="/tmp/mass_download"
subprocess.call(["mkdir","-p",tmp_folder])
hashes = request.forms.dict.get("file_hash[]")
if hashes is None:
hashes = request.forms.get("file_hash").split("\n")
if hashes is not None:
if len(hashes) == 1:
random_id=hashes[0]
else:
random_id = id_generator()
else:
return jsonize({'message':'Error. no file selected'})
folder_path=os.path.join(tmp_folder,random_id)
subprocess.call(["mkdir","-p",folder_path])
zip_name=os.path.join(tmp_folder,random_id+".zip")
pc=PackageController()
for file_hash in hashes:
file_hash = clean_hash(file_hash.replace('\r',''))
data="1="+file_hash
res=SearchModule.search_by_id(data,1)
if(len(res)==0):
pass
else:
file_hash=res[0]["sha1"]
res=pc.searchFile(file_hash)
if res != 1 and res is not None:
res=pc.getFile(file_hash)
file_name=os.path.join(folder_path,str(file_hash)+".codex")
fd=open(file_name,"wb")
fd.write(res)
fd.close()
elif res == 1:
fd=open(os.path.join(folder_path,'readme.txt'),'a+')
fd.write(str(file_hash)+" is not available to download.\n")
fd.close()
elif res is None:
fd=open(os.path.join(folder_path,'readme.txt'),'a+')
fd.write(str(file_hash)+" not found.")
fd.close()
else:
print "Unknown res:"+str(res)
subprocess.call(["zip","-P","codex","-jr", zip_name,folder_path])
resp = static_file(str(random_id)+".zip",root=tmp_folder,download=True)
resp.set_cookie('fileDownload','true');
# http://johnculviner.com/jquery-file-download-plugin-for-ajax-like-feature-rich-file-downloads/
return resp
def get_package_file():
tmp_folder = "/tmp/mass_download"
subprocess.call(["mkdir", "-p", tmp_folder])
hashes = request.forms.dict.get("file_hash[]")
if hashes is None:
hashes = request.forms.get("file_hash").split("\n")
if hashes is not None:
if len(hashes) == 1:
random_id = hashes[0]
else:
random_id = id_generator()
else:
return jsonize({'message': 'Error. no file selected'})
folder_path = os.path.join(tmp_folder, random_id)
subprocess.call(["mkdir", "-p", folder_path])
zip_name = os.path.join(tmp_folder, random_id + ".zip")
pc = PackageController()
for file_hash in hashes:
file_hash = clean_hash(file_hash.replace('\r', ''))
data = "1=" + file_hash
res = SearchModule.search_by_id(data, 1)
if (len(res) == 0):
pass
else:
file_hash = res[0]["sha1"]
res = pc.searchFile(file_hash)
if res != 1 and res is not None:
res = pc.getFile(file_hash)
file_name = os.path.join(folder_path, str(file_hash) + ".codex")
fd = open(file_name, "wb")
fd.write(res)
fd.close()
elif res == 1:
fd = open(os.path.join(folder_path, 'readme.txt'), 'a+')
fd.write(str(file_hash) + " is not available to download.\n")
fd.close()
elif res is None:
fd = open(os.path.join(folder_path, 'readme.txt'), 'a+')
fd.write(str(file_hash) + " not found.")
fd.close()
else:
print "Unknown res:" + str(res)
subprocess.call(["zip", "-P", "codex", "-jr", zip_name, folder_path])
resp = static_file(str(random_id) + ".zip", root=tmp_folder, download=True)
resp.set_cookie('fileDownload', 'true')
# http://johnculviner.com/jquery-file-download-plugin-for-ajax-like-feature-rich-file-downloads/
return resp
def generate_movie_nabresponse(self,imdb_show ):
movie_search_str = imdb_show['movietitle'].lower().replace("'", "").replace("-", " ").replace(":", " ")
movie_search_str = " ".join(movie_search_str.split()).replace(" ", ".") + '.' +imdb_show['year']
#~ print movie_search_str
self.searchstring = movie_search_str
self.typesearch = 0
#~ compile results
results = SearchModule.performSearch(movie_search_str, self.cfg , self.cfg_ds )
#~ flatten and summarize them
cleaned_results = megasearch.summary_results(results,movie_search_str)
#~ render XML
return self.cleanUpResultsXML(cleaned_results)
def generate_movie_nabresponse(self,imdb_show ):
movie_search_str = imdb_show['movietitle'].lower().replace("'", "").replace("-", " ").replace(":", " ")
movie_search_str = " ".join(movie_search_str.split()).replace(" ", ".") + '.' +imdb_show['year']
#~ print movie_search_str
self.searchstring = movie_search_str
self.typesearch = 0
#~ compile results
results = SearchModule.performSearch(movie_search_str, self.cfg , self.cfg_ds )
#~ flatten and summarize them
cleaned_results = megasearch.summary_results(results,movie_search_str)
#~ render XML
return self.cleanUpResultsXML(cleaned_results)
def get_result_from_av():
hash_id = request.query.file_hash
if len(hash_id) == 0:
response.status = 400
return jsonize({
'error': 4,
'error_message': 'file_hash parameter is missing.'
})
hash_id = clean_hash(hash_id)
if not valid_hash(hash_id):
return jsonize({'error': 5, 'error_message': 'Invalid hash format.'})
if (len(hash_id) != 40):
data = "1=" + str(hash_id)
res = SearchModule.search_by_id(data, 1, [], True)
if (len(res) == 0):
response.status = 400
return jsonize({'error': 6, 'error_message': 'File not found'})
else:
sha1 = res[0]["sha1"]
else:
sha1 = hash_id
key_manager = KeyManager()
if (key_manager.check_keys_in_secrets()):
av_result = get_av_result(sha1, 'high')
else:
return jsonize({
'error':
7,
"error_message":
"Error: VirusTotal API key missing from secrets.py file"
})
if (av_result.get('status') == "added"):
return jsonize({"message": "AV scans downloaded."})
elif (av_result.get('status') == "already_had_it"):
return jsonize({"message": "File already have AV scans."})
elif (av_result.get('status') == "not_found"):
return jsonize({"error": 10, "error_message": "Not found on VT."})
elif (av_result.get('status') == "no_key_available"):
return jsonize({
"error":
11,
"error_message":
"No key available right now. Please try again later."
})
else:
logging.error("av_result for hash=" + str(sha1))
logging.error("av_result=" + str(av_result))
return jsonize({"error": 9, "error_message": "Cannot get analysis."})
def generate_tvserie_nabresponse_broadcast(self):
addparams = dict(age="1500", t="tvsearch", cat="5040,5030")
rawResults = SearchModule.performSearch("", self.cfg, self.cfg_ds, addparams)
# ~ rawResults = SearchModule.performSearch('', self.cfg, None, addparams)
results = []
# ~ no cleaning just flatten in one array
for provid in xrange(len(rawResults)):
for z in xrange(len(rawResults[provid])):
results.append(rawResults[provid][z])
self.searchstring = ""
self.typesearch = 1
return self.cleanUpResultsXML(results)
def prepareforquery(self, sugg_info_raw):
sugg_info = []
for i in xrange(len(sugg_info_raw)):
shorinfo = sugg_info_raw[i]['title']
if (len(shorinfo) > MAX_CHAR_LEN):
shorinfo = shorinfo[0:MAX_CHAR_LEN-2] + '..'
si = {'searchstr': SearchModule.sanitize_strings(sugg_info_raw[i]['title']) + '.' + sugg_info_raw[i]['year'] ,
'prettytxt': shorinfo + '('+ sugg_info_raw[i]['year'] + ')',
'imdb_url': sugg_info_raw[i]['imdb_url']}
sugg_info.append(si)
#~ print si
#~ print 'dcdddddddddddddddd'
return sugg_info
def prepareforquery(self, sugg_info_raw):
sugg_info = []
for i in xrange(len(sugg_info_raw)):
shorinfo = sugg_info_raw[i]['title']
if (len(shorinfo) > MAX_CHAR_LEN):
shorinfo = shorinfo[0:MAX_CHAR_LEN-2] + '..'
si = {'searchstr': SearchModule.sanitize_strings(sugg_info_raw[i]['title']) + '.' + sugg_info_raw[i]['year'] ,
'prettytxt': shorinfo + '('+ sugg_info_raw[i]['year'] + ')',
'imdb_url': sugg_info_raw[i]['imdb_url']}
sugg_info.append(si)
#~ print si
#~ print 'dcdddddddddddddddd'
return sugg_info
def generate_tvserie_nabresponse_broadcast(self):
addparams = dict(age='1500', t='tvsearch', cat='5040,5030')
rawResults = SearchModule.performSearch('', self.cfg, self.cfg_ds,
addparams)
#~ rawResults = SearchModule.performSearch('', self.cfg, None, addparams)
results = []
#~ no cleaning just flatten in one array
for provid in xrange(len(rawResults)):
for z in xrange(len(rawResults[provid])):
results.append(rawResults[provid][z])
self.searchstring = ''
self.typesearch = 1
return self.cleanUpResultsXML(results)
def generate_tvserie_nabresponse_broadcast(self):
addparams = dict(
age= '1500',
t='tvsearch',
cat='5040,5030')
rawResults = SearchModule.performSearch('', self.cfg, None, addparams)
results = []
#~ no cleaning just flatten in one array
for provid in xrange(len(rawResults)):
for z in xrange(len(rawResults[provid])):
results.append(rawResults[provid][z])
self.searchstring = ''
return self.cleanUpResultsXML(results)
def api_batch_process_file():
print("Running Batch process")
file_hashes = request.forms.get('file_hash')
#print(dir(request.forms))
#print(request.forms.keys())
#transformar file_hashes a una lista de hashes
not_found = []
added_to_queue = 0
downloaded_from_vt = 0
for hash_id in file_hashes.split("\n"):
hash_id = clean_hash(hash_id)
if hash_id is None:
continue
data = "1=" + str(hash_id)
res = SearchModule.search_by_id(data, 1, [], True)
if (len(res) == 0):
not_found.append(hash_id)
continue
"""
print "downloading "+str(hash_id)+" from vt"
sha1=SearchModule.add_file_from_vt(hash_id)
if(sha1==None):
print "not found on vt: "+str(hash_id)
not_found.append(hash_id)
continue
else:
downloaded_from_vt+=1
"""
else:
sha1 = res[0]["sha1"]
added_to_queue += 1
print str(hash_id) + " added to queue"
add_hash_to_process_queue(sha1)
if (env['auto_get_av_result']):
get_av_result(sha1)
responsex = str(added_to_queue) + " files added to the process queue.\n"
if (downloaded_from_vt > 0):
responsex += str(downloaded_from_vt) + " new hashes.\n"
if (len(not_found) != 0):
responsex += str(len(not_found)) + " hashes not found.\n"
responsex += "Not Found:\n"
for aux in not_found:
responsex = responsex + str(aux) + "\n"
return jsonize({"message": responsex})
def dosearch(self, args):
#~ restore originals
self.cfg = copy.deepcopy(self.cfg_cpy)
if ('q' not in args):
self.results = []
return self.results
self.logic_items = self.logic_expr.findall(args['q'])
self.qry_nologic = self.logic_expr.sub(" ", args['q'])
if ('selcat' in args):
self.qry_nologic += " " + args['selcat']
#~ speed class
speed_class_sel = 1
if ('tm' in args):
speed_class_sel = int(args['tm'])
#~ speed class deepsearch
self.ds.set_timeout_speedclass(speed_class_sel)
#~ speed class Nabbased
for conf in self.cfg:
if ((conf['speed_class'] <= speed_class_sel) and (conf['valid'])):
conf['timeout'] = self.cgen['timeout_class'][speed_class_sel]
#~ print conf['type'] + " " + str(conf['timeout'] ) + ' ' + str(speed_class_sel )
else:
conf['valid'] = 0
if (len(args['q']) == 0):
if ('selcat' in args):
if (len(args['selcat']) == 0):
self.results = []
return self.results
else:
self.results = []
return self.results
if (self.qry_nologic.replace(" ", "") == ""):
self.results = []
return self.results
self.logic_items = self.logic_expr.findall(args['q'])
results = SearchModule.performSearch(self.qry_nologic, self.cfg,
self.ds)
self.results = summary_results(results, self.qry_nologic,
self.logic_items)
def api_batch_process_file():
print("Running Batch process")
file_hashes=request.forms.get('file_hash')
#print(dir(request.forms))
#print(request.forms.keys())
#transformar file_hashes a una lista de hashes
not_found=[]
added_to_queue=0
downloaded_from_vt=0
for hash_id in file_hashes.split("\n"):
hash_id=clean_hash(hash_id)
if hash_id is None:
continue
data="1="+str(hash_id)
res=SearchModule.search_by_id(data,1,[],False)
if(len(res)==0):
not_found.append(hash_id)
continue
"""
print "downloading "+str(hash_id)+" from vt"
sha1=SearchModule.add_file_from_vt(hash_id)
if(sha1==None):
print "not found on vt: "+str(hash_id)
not_found.append(hash_id)
continue
else:
downloaded_from_vt+=1
"""
else:
sha1=res[0]["sha1"]
added_to_queue+=1
print str(hash_id)+" added to queue"
add_hash_to_process_queue(sha1)
responsex=str(added_to_queue)+" files added to the process queue.\n"
if(downloaded_from_vt > 0):
responsex+=str(downloaded_from_vt)+" new hashes.\n"
if(len(not_found)!=0):
responsex+=str(len(not_found))+ " hashes not found.\n"
responsex+="Not Found:\n"
for aux in not_found:
responsex=responsex+str(aux)+"\n"
return jsonize({"message":responsex})
def dosearch(self, args):
# ~ restore originals
self.cfg = copy.deepcopy(self.cfg_cpy)
if "q" not in args:
self.results = []
return self.results
self.logic_items = self.logic_expr.findall(args["q"])
self.qry_nologic = self.logic_expr.sub(" ", args["q"])
if "selcat" in args:
self.qry_nologic += " " + args["selcat"]
# ~ speed class
speed_class_sel = 1
if "tm" in args:
speed_class_sel = int(args["tm"])
# ~ speed class deepsearch
self.ds.set_timeout_speedclass(speed_class_sel)
# ~ speed class Nabbased
for conf in self.cfg:
if (conf["speed_class"] <= speed_class_sel) and (conf["valid"]):
conf["timeout"] = self.cgen["timeout_class"][speed_class_sel]
# ~ print conf['type'] + " " + str(conf['timeout'] ) + ' ' + str(speed_class_sel )
else:
conf["valid"] = 0
if len(args["q"]) == 0:
if "selcat" in args:
if len(args["selcat"]) == 0:
self.results = []
return self.results
else:
self.results = []
return self.results
if self.qry_nologic.replace(" ", "") == "":
self.results = []
return self.results
self.logic_items = self.logic_expr.findall(args["q"])
results = SearchModule.performSearch(self.qry_nologic, self.cfg, self.ds)
self.results = summary_results(results, self.qry_nologic, self.logic_items)
def dosearch_rss(self, arguments, hname):
self.args = arguments
self.rqurl = hname.scheme + "://" + hname.netloc
addparams = dict(age="1500", limit="20000", t="search", cat="1000,2000,3000,4000,5000,6000,7000")
if "cat" in self.args:
addparams["cat"] = self.args["cat"]
rawResults = SearchModule.performSearch("", self.cfg, self.cfg_ds, addparams)
results = []
# ~ no cleaning just flatten in one array
for provid in xrange(len(rawResults)):
if rawResults[provid] is not None:
for z in xrange(len(rawResults[provid])):
results.append(rawResults[provid][z])
self.searchstring = ""
self.typesearch = 3
return self.cleanUpResultsXML(results)
def get_result_from_av():
hash_id = request.query.file_hash
if len(hash_id) == 0:
response.status = 400
return jsonize({'error': 4, 'error_message': 'file_hash parameter is missing.'})
hash_id = clean_hash(hash_id)
if not valid_hash(hash_id):
return jsonize({'error': 5, 'error_message': 'Invalid hash format.'})
if(len(hash_id) != 40):
data = "1=" + str(hash_id)
res = SearchModule.search_by_id(data, 1, [], True)
if(len(res) == 0):
response.status = 400
return jsonize({'error': 6, 'error_message': 'File not found'})
else:
sha1 = res[0]["sha1"]
else:
sha1 = hash_id
key_manager = KeyManager()
if(key_manager.check_keys_in_secrets()):
av_result = get_av_result(sha1, 'high')
else:
return jsonize({'error': 7, "error_message": "Error: VirusTotal API key missing from secrets.py file"})
if(av_result.get('status') == "added"):
return jsonize({"message": "AV scans downloaded."})
elif(av_result.get('status') == "already_had_it"):
return jsonize({"message": "File already have AV scans."})
elif(av_result.get('status') == "not_found"):
return jsonize({"error": 10, "error_message": "Not found on VT."})
elif(av_result.get('status') == "no_key_available"):
return jsonize({"error": 11, "error_message": "No key available right now. Please try again later."})
else:
logging.error("av_result for hash=" + str(sha1))
logging.error("av_result=" + str(av_result))
return jsonize({"error": 9, "error_message": "Cannot get analysis."})
def dosearch_rss(self, arguments, hname):
self.args = arguments
self.rqurl = hname.scheme+'://'+hname.netloc
addparams = dict(
age= '1500',
limit='20000',
t='search',
cat='1000,2000,3000,4000,5000,6000,7000')
if('cat' in self.args):
addparams['cat'] = self.args['cat']
rawResults = SearchModule.performSearch('', self.cfg, self.cfg_ds, addparams)
results = []
#~ no cleaning just flatten in one array
for provid in xrange(len(rawResults)):
if(rawResults[provid] is not None):
for z in xrange(len(rawResults[provid])):
results.append(rawResults[provid][z])
self.searchstring = ''
self.typesearch = 3
return self.cleanUpResultsXML(results)
def dosearch_rss(self, arguments, hname):
self.args = arguments
self.rqurl = hname.scheme+'://'+hname.netloc
addparams = dict(
age= '1500',
limit='20000',
t='search',
cat='1000,2000,3000,4000,5000,6000,7000')
if('cat' in self.args):
addparams['cat'] = self.args['cat']
rawResults = SearchModule.performSearch('', self.cfg, self.cfg_ds, addparams)
results = []
#~ no cleaning just flatten in one array
for provid in xrange(len(rawResults)):
if(rawResults[provid] is not None):
for z in xrange(len(rawResults[provid])):
results.append(rawResults[provid][z])
self.searchstring = ''
self.typesearch = 3
return self.cleanUpResultsXML(results)
def cleanUpResults(self, params):
sugg_list = params['sugg']
results = self.results
svalid = self.svalid
args = params['args']
ver_notify = params['ver']
niceResults = []
existduplicates = 0
#~ tries to match predb entries
self.matchpredb(results, params['predb'])
#~ avoids GMT problems
for i in xrange(len(results)):
totdays = int(
(time.time() - results[i]['posting_date_timestamp']) /
(3600 * 24))
if (totdays == 0):
totdays = float(
(time.time() - results[i]['posting_date_timestamp']) /
(3600))
if (totdays < 0):
totdays = -totdays
totdays = totdays / 100
results[i]['posting_date_timestamp_refined'] = float(totdays)
#~ sorting
if 'order' not in args:
results = sorted(results,
key=itemgetter('posting_date_timestamp_refined'),
reverse=False)
else:
if (args['order'] == 't'):
results = sorted(results, key=itemgetter('title'))
if (args['order'] == 's'):
results = sorted(results, key=itemgetter('size'), reverse=True)
if (args['order'] == 'p'):
results = sorted(results, key=itemgetter('providertitle'))
if (args['order'] == 'd'):
results = sorted(
results,
key=itemgetter('posting_date_timestamp_refined'),
reverse=False)
if (args['order'] == 'x'):
results = sorted(results,
key=itemgetter('predb'),
reverse=True)
if (args['order'] == 'c'):
results = sorted(results,
key=itemgetter('categ'),
reverse=True)
#~ do nice
for i in xrange(len(results)):
if (results[i]['ignore'] == 2):
continue
if (results[i]['ignore'] == 1):
existduplicates = 1
# Convert sized to smallest SI unit (note that these are powers of 10, not powers of 2, i.e. OS X file sizes rather than Windows/Linux file sizes)
szf = float(results[i]['size'] / 1000000.0)
mgsz = ' MB '
if (szf > 1000.0):
szf = szf / 1000
mgsz = ' GB '
fsze1 = str(round(szf, 1)) + mgsz
if (results[i]['size'] == -1):
fsze1 = 'N/A'
totdays = results[i]['posting_date_timestamp_refined']
if (totdays < 1):
totdays = str(int(totdays * 100)) + "h"
else:
totdays = str(int(totdays)) + "d"
category_str = ''
keynum = len(results[i]['categ'])
keycount = 0
for key in results[i]['categ'].keys():
category_str = category_str + key
keycount = keycount + 1
if (keycount < keynum):
category_str = category_str + ' - '
if (results[i]['url'] is None):
results[i]['url'] = ""
qryforwarp = self.wrp.chash64_encode(results[i]['url'])
if ('req_pwd' in results[i]):
qryforwarp += '&m=' + results[i]['req_pwd']
niceResults.append({
'id':
i,
'url':
results[i]['url'],
'url_encr':
'warp?x=' + qryforwarp,
'title':
results[i]['title'],
'filesize':
fsze1,
'cat':
category_str.upper(),
'age':
totdays,
'details':
results[i]['release_comments'],
'details_deref':
'http://www.derefer.me/?' + results[i]['release_comments'],
'providerurl':
results[i]['provider'],
'providertitle':
results[i]['providertitle'],
'ignore':
results[i]['ignore'],
'predb':
results[i]['predb'],
'predb_lnk':
results[i]['predb_lnk']
})
send2nzbget_exist = None
if ('nzbget_url' in self.cgen):
if (len(self.cgen['nzbget_url'])):
send2nzbget_exist = self.sckname
send2sab_exist = None
if ('sabnzbd_url' in self.cgen):
if (len(self.cgen['sabnzbd_url'])):
send2sab_exist = self.sckname
speed_class_sel = 1
if ('tm' in args):
speed_class_sel = int(args['tm'])
#~ save for caching
if (self.resultsraw is not None):
if (self.cgen['cache_active'] == 1 and len(self.resultsraw) > 0):
if (len(self.collect_info) < self.cgen['max_cache_qty']):
if (self.chkforcache(
self.wrp.chash64_encode(
SearchModule.sanitize_strings(
self.qry_nologic)), speed_class_sel) is
None):
collect_all = {}
collect_all['searchstr'] = self.wrp.chash64_encode(
SearchModule.sanitize_strings(self.qry_nologic))
collect_all['tstamp'] = time.time()
collect_all['resultsraw'] = self.resultsraw
collect_all['speedclass'] = speed_class_sel
self.collect_info.append(collect_all)
#~ print 'Result added to the cache list'
#~ ~ ~ ~ ~ ~ ~ ~ ~
scat = ''
if ('selcat' in params['args']):
scat = params['args']['selcat']
return render_template('main_page.html',
results=niceResults,
exist=existduplicates,
vr=ver_notify,
args=args,
nc=svalid,
sugg=sugg_list,
speed_class_sel=speed_class_sel,
send2sab_exist=send2sab_exist,
send2nzbget_exist=send2nzbget_exist,
cgen=self.cgen,
trend_show=params['trend_show'],
trend_movie=params['trend_movie'],
debug_flag=params['debugflag'],
sstring=params['args']['q'],
scat=scat,
selectable_opt=params['selectable_opt'],
search_opt=params['search_opt'],
sid=params['sid'],
servercode_return=self.returncode_fine,
large_server=self.cgen['large_server'],
motd=params['motd'])
import threading
from SuggestionModule import SuggestionResponses
import config_settings
import SearchModule
import logging
import logging.handlers
import time
DEBUGFLAG = True
motd = '\n\n~*~ ~*~ NZBMegasearcH detached trend ~*~ ~*~'
print motd
#~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~
cfgsets = config_settings.CfgSettings()
logsdir = SearchModule.resource_path('logs/')
logging.basicConfig(
filename=logsdir + 'nzbmegasearch_detachedtrend.log',
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
log = logging.getLogger(__name__)
handler = logging.handlers.RotatingFileHandler(
logsdir + 'nzbmegasearch_detachedtrend.log',
maxBytes=cfgsets.cgen['log_size'],
backupCount=cfgsets.cgen['log_backupcount'])
log.addHandler(handler)
log.info(motd)
cfgsets.cgen['trends'] = 1
cfgsets.cgen['search_suggestions'] = 1
cfgsets.cgen['large_server'] = False
sugg = SuggestionResponses(cfgsets.cfg, cfgsets.cgen)
def api_batch_process_debug_file():
yield "<html><body><pre>"
yield "Running Batch process\n"
file_hashes = request.forms.get('file_hash')
if file_hashes is None:
response.status = 422
logging.debug("api_batch_process_debug_file(): file_hash is missing")
yield "file_hash parameter is missing"
# transform file_hashes in a list of hashes.
not_found = []
added_to_queue = 0
downloaded_from_vt = 0
for hash_id in file_hashes.split("\n"):
hash_id = clean_hash(hash_id)
if hash_id is None:
continue
data = "1=" + hash_id
if (len(hash_id) == 40 or len(hash_id) == 32):
pc = PackageController()
res = pc.getFile(hash_id)
if res is not None and len(
SearchModule.search_by_id(data, 1, [], False)) == 0:
logging.debug("Processing right now: " + str(hash_id))
process_file(hash_id)
if (env['auto_get_av_result']):
add_task_to_download_av_result(hash_id)
continue
res = SearchModule.search_by_id(data, 1, [], False)
if (len(res) == 0):
legging.debug("process_debug(): metadata of " + str(hash_id) +
" was not found. We will look in Pc. hash length: " +
str(len(hash_id)))
if (len(hash_id) == 40 or len(hash_id) == 32):
pc = PackageController()
res = pc.getFile(hash_id)
if res is not None:
logging.debug("process_debug(): hash was found (" +
str(hash_id) + ")")
else:
logging.debug("process_debug(): hash was not found(" +
str(hash_id) + ")")
logging.debug("process_debug():")
logging.debug("process_debug(): going to search " + str(hash_id) +
" in vt")
add_response = SearchModule.add_file_from_vt(hash_id)
sha1 = add_response.get('hash')
if (sha1 == None):
logging.debug("process_debug(): sha1 is None: " + str(hash_id))
not_found.append(hash_id)
continue
else:
downloaded_from_vt += 1
else:
sha1 = res[0]["sha1"]
added_to_queue += 1
add_hash_to_process_queue(sha1)
if (env['auto_get_av_result']):
add_task_to_download_av_result(sha1)
yield str(sha1) + "\n"
responsex = str(added_to_queue) + " files added to the process queue.\n"
if (downloaded_from_vt > 0):
responsex += str(downloaded_from_vt) + " new hashes.\n"
if (len(not_found) != 0):
responsex += str(len(not_found)) + " hashes not found.\n"
responsex += "Not Found:\n"
for aux in not_found:
responsex = responsex + str(aux) + "\n"
yield responsex
yield "END"
def check(self, args):
ret = 0
if (('hostname' in args) and ('type' in args)):
# Perform the search using every module
global globalResults
if 'loadedModules' not in globals():
SearchModule.loadSearchModules()
# ~ specials
if (args['type'] == 'OMG'):
ret = 1
cfg_tmp = {
'valid': 1,
'type': 'OMG',
'speed_class': 2,
'extra_class': 0,
'login': args['user'],
'pwd': args['pwd'],
'timeout': self.cgen['timeout_class'][2],
'builtin': 1
}
for module in SearchModule.loadedModules:
if (module.typesrch == 'OMG'):
module.search('Ubuntu', cfg_tmp)
print cfg_tmp['retcode']
if (cfg_tmp['retcode'][0] != 200):
ret = 0
# ~ server based API
if (args['type'] == 'NAB'):
ret = 1
cfg_tmp = {
'url': args['hostname'],
'type': 'NAB',
'api': args['api'],
'speed_class': 2,
'extra_class': 0,
'valid': 1,
'timeout': self.cgen['timeout_class'][2],
'builtin': 0
}
for module in SearchModule.loadedModules:
if (module.typesrch == 'NAB'):
module.search('Ubuntu', cfg_tmp)
print cfg_tmp['retcode']
if (cfg_tmp['retcode'][0] != 200):
ret = 0
# ~ server based WEB
if (args['type'] == 'DSN' or args['type'] == 'DS_GNG'):
cfg_deep_tmp = [{
'url': args['hostname'],
'user': args['user'],
'pwd': args['pwd'],
'type': args['type'],
'speed_class': 2,
'extra_class': 0,
'valid': 1,
}]
ds_tmp = DeepsearchModule.DeepSearch(cfg_deep_tmp, self.cgen)
ret_bool = ds_tmp.ds[0].search('Ubuntu')
if (ret_bool):
ret = 1
else:
ret = 0
return ret
def get_sample_count():
count = SearchModule.count_documents()
res = {"count": count}
return jsonize(res)
def cleanUpResults(self, params):
sugg_list = params['sugg']
results = self.results
svalid = self.svalid
args = params['args']
ver_notify = params['ver']
niceResults = []
existduplicates = 0
# ~ tries to match predb entries
self.matchpredb(results, params['predb'])
# ~ avoids GMT problems
for i in xrange(len(results)):
totdays = int((time.time() - results[i]['posting_date_timestamp']) / (3600 * 24))
if (totdays == 0):
totdays = float((time.time() - results[i]['posting_date_timestamp']) / (3600))
if (totdays < 0):
totdays = -totdays
totdays = totdays / 100
results[i]['posting_date_timestamp_refined'] = float(totdays)
# ~ sorting
if 'order' not in args:
results = sorted(results, key=itemgetter('posting_date_timestamp_refined'), reverse=False)
else:
if (args['order'] == 't'):
results = sorted(results, key=itemgetter('title'))
if (args['order'] == 's'):
results = sorted(results, key=itemgetter('size'), reverse=True)
if (args['order'] == 'p'):
results = sorted(results, key=itemgetter('providertitle'))
if (args['order'] == 'd'):
results = sorted(results, key=itemgetter('posting_date_timestamp_refined'), reverse=False)
if (args['order'] == 'x'):
results = sorted(results, key=itemgetter('predb'), reverse=True)
if (args['order'] == 'c'):
results = sorted(results, key=itemgetter('categ'), reverse=True)
# ~ do nice
for i in xrange(len(results)):
if (results[i]['posting_date_timestamp_refined'] > self.cgen['daysretention']):
continue
if (results[i]['ignore'] == 2):
continue
if (results[i]['ignore'] == 1):
existduplicates = 1
# Convert sized to smallest SI unit (note that these are powers of 10, not powers of 2, i.e. OS X file sizes rather than Windows/Linux file sizes)
szf = float(results[i]['size'] / 1000000.0)
mgsz = ' MB '
if (szf > 1000.0):
szf = szf / 1000
mgsz = ' GB '
fsze1 = str(round(szf, 1)) + mgsz
if (results[i]['size'] == -1):
fsze1 = 'N/A'
totdays = results[i]['posting_date_timestamp_refined']
if (totdays < 1):
totdays = str(int(totdays * 100)) + "h"
else:
totdays = str(int(totdays)) + "d"
category_str = ''
keynum = len(results[i]['categ'])
keycount = 0
for key in results[i]['categ'].keys():
category_str = category_str + key
keycount = keycount + 1
if (keycount < keynum):
category_str = category_str + ' - '
if (results[i]['url'] is None):
results[i]['url'] = ""
qryforwarp = self.wrp.chash64_encode(results[i]['url'])
if ('req_pwd' in results[i]):
qryforwarp += '&m=' + results[i]['req_pwd']
niceResults.append({
'id': i,
'url': results[i]['url'],
'url_encr': 'warp?x=' + qryforwarp,
'title': results[i]['title'],
'filesize': fsze1,
'cat': category_str.upper(),
'age': totdays,
'details': results[i]['release_comments'],
'details_deref': 'http://www.derefer.me/?' + results[i]['release_comments'],
'providerurl': results[i]['provider'],
'providertitle': results[i]['providertitle'],
'ignore': results[i]['ignore'],
'predb': results[i]['predb'],
'predb_lnk': results[i]['predb_lnk']
})
send2nzbget_exist = None
if ('nzbget_url' in self.cgen):
if (len(self.cgen['nzbget_url'])):
send2nzbget_exist = self.sckname
send2sab_exist = None
if ('sabnzbd_url' in self.cgen):
if (len(self.cgen['sabnzbd_url'])):
send2sab_exist = self.sckname
speed_class_sel = 1
if ('tm' in args):
speed_class_sel = int(args['tm'])
# ~ save for caching
if (self.resultsraw is not None):
if (self.cgen['cache_active'] == 1 and len(self.resultsraw) > 0):
if (len(self.collect_info) < self.cgen['max_cache_qty']):
if (self.chkforcache(self.wrp.chash64_encode(SearchModule.sanitize_strings(self.qry_nologic)),
speed_class_sel) is None):
collect_all = {}
collect_all['searchstr'] = self.wrp.chash64_encode(
SearchModule.sanitize_strings(self.qry_nologic))
collect_all['tstamp'] = time.time()
collect_all['resultsraw'] = self.resultsraw
collect_all['speedclass'] = speed_class_sel
self.collect_info.append(collect_all)
# ~ print 'Result added to the cache list'
# ~ ~ ~ ~ ~ ~ ~ ~ ~
scat = ''
if ('selcat' in params['args']):
scat = params['args']['selcat']
return render_template('main_page.html', results=niceResults, exist=existduplicates,
vr=ver_notify, args=args, nc=svalid, sugg=sugg_list,
speed_class_sel=speed_class_sel,
send2sab_exist=send2sab_exist,
send2nzbget_exist=send2nzbget_exist,
cgen=self.cgen,
trend_show=params['trend_show'],
trend_movie=params['trend_movie'],
debug_flag=params['debugflag'],
sstring=params['args']['q'],
scat=scat,
selectable_opt=params['selectable_opt'],
search_opt=params['search_opt'],
sid=params['sid'],
servercode_return=self.returncode_fine,
large_server=self.cgen['large_server'],
motd=params['motd'])
def summary_results(rawResults, strsearch, logic_items=[]):
results = []
titles = []
sptitle_collection = []
#~ all in one array
for provid in xrange(len(rawResults)):
for z in xrange(len(rawResults[provid])):
rawResults[provid][z]['title'] = SearchModule.sanitize_html(
rawResults[provid][z]['title'])
title = SearchModule.sanitize_strings(
rawResults[provid][z]['title'])
titles.append(title)
sptitle_collection.append(Set(title.split(".")))
results.append(rawResults[provid][z])
strsearch1 = SearchModule.sanitize_strings(strsearch)
strsearch1_collection = Set(strsearch1.split("."))
rcount = [0] * 3
for z in xrange(len(results)):
findone = 0
results[z]['ignore'] = 0
intrs = strsearch1_collection.intersection(sptitle_collection[z])
if (len(intrs) == len(strsearch1_collection)):
findone = 1
else:
results[z]['ignore'] = 2
#~ print strsearch1_collection
#~ print intrs
#~ print findone
#~ print '------------------'
if (findone and results[z]['ignore'] == 0):
#~ print titles[z]
for v in xrange(z + 1, len(results)):
if (titles[z] == titles[v]):
sz1 = float(results[z]['size'])
sz2 = float(results[v]['size'])
if (abs(sz1 - sz2) < 5000000):
results[z]['ignore'] = 1
#~ stats
rcount[results[z]['ignore']] += 1
#~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~
#~ logic params
exclude_coll = Set([])
include_coll = Set([])
#~ print '*'+logic_items[0][1]+'*'
for i in xrange(len(logic_items)):
if (logic_items[i][0] == '-'):
exclude_coll.add(logic_items[i][1])
if (logic_items[i][0] == '+'):
include_coll.add(logic_items[i][1])
if (len(include_coll)):
for z in xrange(len(results)):
if (results[z]['ignore'] < 2):
intrs_i = include_coll.intersection(sptitle_collection[z])
if (len(intrs_i) == 0):
results[z]['ignore'] = 2
if (len(exclude_coll)):
for z in xrange(len(results)):
if (results[z]['ignore'] < 2):
intrs_e = exclude_coll.intersection(sptitle_collection[z])
if (len(intrs_e) > 0):
results[z]['ignore'] = 2
#~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~
mssg = '[' + strsearch1 + ']' + ' [' + strsearch + '] ' + str(
rcount[0]) + ' ' + str(rcount[1]) + ' ' + str(rcount[2])
print mssg
log.info(mssg)
return results
def html_editpage(self):
count = 0
if 'SearchModule.loadedModules' not in globals():
SearchModule.loadSearchModules()
dsearchsupport = DeepsearchModule.supportedengines()
cffileb = []
cffile = copy.deepcopy(self.cfg)
cdsfile = self.cfg_deep
genopt = self.cgen
if (cffile is None):
cffile = []
if (cdsfile is None):
cdsfile = []
cdomainname = megasearch.getdomainext()
for module in SearchModule.loadedModules:
if (module.builtin):
option = 'checked=yes'
flogin = 0
flogin_caption_user = '******'
flogin_caption_pwd = 'pwd'
login_name = ''
login_pwd = ''
speed_cl = 1
if (module.active == 0):
option = ''
for i in xrange(len(cffile)):
if (cffile[i]['type'] == module.typesrch):
if (cffile[i]['valid'] == 0):
option = ''
else:
option = 'checked=yes'
login_name = cffile[i]['login']
login_pwd = cffile[i]['pwd']
speed_cl = cffile[i]['speed_class']
if (module.login == 1):
flogin = 1
#~ if('caption_login_user' in module):
if (hasattr(module, 'caption_login_user')):
flogin_caption_user = module.caption_login_user
flogin_caption_pwd = module.caption_login_pwd
tmpcfg = {
'stchk': option,
'humanname': module.name,
'url': '',
'idx': count,
'speed_class': speed_cl,
'type': module.typesrch,
'flogin': flogin,
'flogin_caption_user': flogin_caption_user,
'flogin_caption_pwd': flogin_caption_pwd,
'loginname': login_name,
'loginpwd': login_pwd,
}
cffileb.append(tmpcfg)
count = count + 1
#~ scrapers with web login
for dsearchmodule in dsearchsupport:
if (dsearchmodule['opts']['builtin']):
option = 'checked=yes'
flogin = 0
login_name = ''
login_pwd = ''
speed_cl = dsearchmodule['opts']['speed_cl']
if (dsearchmodule['opts']['active'] == 0):
option = ''
for i in xrange(len(cdsfile)):
if (cdsfile[i]['type'] == dsearchmodule['opts']['typesrch']
):
if (cdsfile[i]['valid'] == 0):
option = ''
else:
option = 'checked=yes'
#~ speed_cl = cdsfile[i]['speed_cl']
login_name = cdsfile[i]['user']
login_pwd = cdsfile[i]['pwd']
if (dsearchmodule['opts']['login'] == 1):
flogin = 1
tmpcfg = {
'stchk': option,
'humanname': dsearchmodule['name'],
'url': dsearchmodule['opts']['url'],
'idx': count,
'flogin_caption_user': '******',
'flogin_caption_pwd': 'pwd',
'speed_class': speed_cl,
'type': dsearchmodule['opts']['typesrch'],
'flogin': flogin,
'loginname': login_name,
'loginpwd': login_pwd,
}
cffileb.append(tmpcfg)
count = count + 1
count = 0
for i in xrange(len(cffile)):
if (cffile[i]['builtin'] == 0):
cffile[i]['idx'] = count
cffile[i]['valid_verbose'] = ''
if (cffile[i]['valid'] == 1):
cffile[i]['valid_verbose'] = 'checked=yes'
count = count + 1
sel_speedopt_tmp = copy.deepcopy(self.selectable_speedopt)
sel_speedopt_tmp[cffile[i]['speed_class'] - 1][2] = 'selected'
cffile[i]['selspeed_sel'] = sel_speedopt_tmp
sel_speedopt_basic = copy.deepcopy(self.selectable_speedopt)
sel_speedopt_basic[0][2] = 'selected'
count_ds = 0
cdsfile_toshow1 = []
for i in xrange(len(cdsfile)):
if (cdsfile[i]['type'] == 'DSN'):
cdsfile_toshow = copy.deepcopy(cdsfile[i])
cdsfile_toshow['idx'] = count_ds
cdsfile_toshow['valid_verbose'] = ''
if (cdsfile_toshow['valid'] == 1):
cdsfile_toshow['valid_verbose'] = 'checked=yes'
count_ds = count_ds + 1
sel_speedopt_tmp = copy.deepcopy(self.selectable_speedopt)
sel_speedopt_tmp[cdsfile[i]['speed_class'] - 1][2] = 'selected'
cdsfile_toshow['selspeed_sel'] = sel_speedopt_tmp
cdsfile_toshow1.append(cdsfile_toshow)
possibleopt = megasearch.listpossiblesearchoptions()
for slctg in possibleopt:
if (slctg[0] == genopt['search_default']):
slctg[2] = 'selected'
tnarray = []
for ntn in xrange(1, 50):
if (genopt['trends_qty'] == ntn):
tnarray.append([ntn, ntn, 'selected'])
else:
tnarray.append([ntn, ntn, ''])
genopt['general_https_verbose'] = ''
genopt['general_trend_verbose'] = ''
genopt['general_suggestion_verbose'] = ''
genopt['smartsearch_verbose'] = ''
genopt['max_cache_verbose'] = ''
genopt['predb_active_verbose'] = ''
if (genopt['predb_active'] == 1):
genopt['predb_active_verbose'] = 'checked=yes'
if (genopt['general_https'] == 1):
genopt['general_https_verbose'] = 'checked=yes'
if (genopt['general_suggestion'] == 1):
genopt['general_suggestion_verbose'] = 'checked=yes'
if (genopt['general_trend'] == 1):
genopt['general_trend_verbose'] = 'checked=yes'
if (genopt['smartsearch'] == 1):
genopt['smartsearch_verbose'] = 'checked=yes'
if (genopt['cache_active'] == 1):
genopt['cache_active_verbose'] = 'checked=yes'
genopt['general_ipaddress_verbose'] = 'AUTO'
if (genopt['general_ipaddress'] != ''):
genopt['general_ipaddress_verbose'] = genopt['general_ipaddress']
openshift_install = False
if (len(self.dirconf)):
openshift_install = True
return render_template('config.html',
cfg=cffile,
cfg_dp=cdsfile_toshow1,
cnt=count,
cnt_ds=count_ds,
genopt=genopt,
selectable_opt=possibleopt,
sel_speedopt_basic=sel_speedopt_basic,
openshift_install=openshift_install,
tnarray=tnarray,
cdomainname=cdomainname,
cnt_max=MAX_PROVIDER_NUMBER,
cfg_bi=cffileb)
import threading
from SuggestionModule import SuggestionResponses
import config_settings
import SearchModule
import logging
import logging.handlers
import time
DEBUGFLAG = True
motd = '\n\n~*~ ~*~ NZBMegasearcH detached trend ~*~ ~*~'
print motd
#~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~
cfgsets = config_settings.CfgSettings()
logsdir = SearchModule.resource_path('logs/')
logging.basicConfig(filename=logsdir+'nzbmegasearch_detachedtrend.log',level=logging.INFO,format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
log = logging.getLogger(__name__)
handler = logging.handlers.RotatingFileHandler(logsdir+'nzbmegasearch_detachedtrend.log', maxBytes=cfgsets.cgen['log_size'], backupCount=cfgsets.cgen['log_backupcount'])
log.addHandler(handler)
log.info(motd)
cfgsets.cgen['trends'] = 1
cfgsets.cgen['search_suggestions'] = 1
cfgsets.cgen['large_server'] = False
sugg = SuggestionResponses(cfgsets.cfg, cfgsets.cgen)
#~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~
def pollsuggestions():
sugg.asktrend_allparallel()
sugg.asktrend_saveondisk()
def html_editpage(self):
count = 0
if 'SearchModule.loadedModules' not in globals():
SearchModule.loadSearchModules()
dsearchsupport = DeepsearchModule.supportedengines()
cffileb = []
cffile = copy.deepcopy(self.cfg)
cdsfile = self.cfg_deep
genopt = self.cgen
if (cffile is None):
cffile = []
if (cdsfile is None):
cdsfile = []
cdomainname = megasearch.getdomainext()
for module in SearchModule.loadedModules:
if (module.builtin):
option = 'checked=yes'
flogin = 0
flogin_caption_user = '******'
flogin_caption_pwd = 'pwd'
login_name = ''
login_pwd = ''
speed_cl = 1
extra_cl = 0
if (module.active == 0):
option = ''
for i in xrange(len(cffile)):
if (cffile[i]['type'] == module.typesrch):
if (cffile[i]['valid'] == 0):
option = ''
else:
option = 'checked=yes'
login_name = cffile[i]['login']
login_pwd = cffile[i]['pwd']
speed_cl = cffile[i]['speed_class']
extra_cl = cffile[i]['extra_class']
if (module.login == 1):
flogin = 1
# ~ if('caption_login_user' in module):
if (hasattr(module, 'caption_login_user')):
flogin_caption_user = module.caption_login_user
flogin_caption_pwd = module.caption_login_pwd
tmpcfg = {'stchk': option,
'humanname': module.name,
'url': '',
'idx': count,
'speed_class': speed_cl,
'extra_class': extra_cl,
'type': module.typesrch,
'flogin': flogin,
'flogin_caption_user': flogin_caption_user,
'flogin_caption_pwd': flogin_caption_pwd,
'loginname': login_name,
'loginpwd': login_pwd,
}
cffileb.append(tmpcfg)
count = count + 1
# ~ scrapers with web login
for dsearchmodule in dsearchsupport:
if (dsearchmodule['opts']['builtin']):
option = 'checked=yes'
flogin = 0
login_name = ''
login_pwd = ''
speed_cl = dsearchmodule['opts']['speed_cl']
extra_cl = dsearchmodule['opts']['extra_cl']
if (dsearchmodule['opts']['active'] == 0):
option = ''
for i in xrange(len(cdsfile)):
if (cdsfile[i]['type'] == dsearchmodule['opts']['typesrch']):
if (cdsfile[i]['valid'] == 0):
option = ''
else:
option = 'checked=yes'
# ~ speed_cl = cdsfile[i]['speed_cl']
login_name = cdsfile[i]['user']
login_pwd = cdsfile[i]['pwd']
if (dsearchmodule['opts']['login'] == 1):
flogin = 1
tmpcfg = {'stchk': option,
'humanname': dsearchmodule['name'],
'url': dsearchmodule['opts']['url'],
'idx': count,
'flogin_caption_user': '******',
'flogin_caption_pwd': 'pwd',
'speed_class': speed_cl,
'extra_class': extra_cl,
'type': dsearchmodule['opts']['typesrch'],
'flogin': flogin,
'loginname': login_name,
'loginpwd': login_pwd,
}
cffileb.append(tmpcfg)
count = count + 1
count = 0
for i in xrange(len(cffile)):
if (cffile[i]['builtin'] == 0):
cffile[i]['idx'] = count
cffile[i]['valid_verbose'] = ''
if (cffile[i]['valid'] == 1):
cffile[i]['valid_verbose'] = 'checked=yes'
count = count + 1
sel_speedopt_tmp = copy.deepcopy(self.selectable_speedopt)
sel_speedopt_tmp[cffile[i]['speed_class'] - 1][2] = 'selected'
sel_extraopt_tmp = copy.deepcopy(self.selectable_extraopt)
sel_extraopt_tmp[cffile[i]['extra_class']][2] = 'selected'
cffile[i]['selspeed_sel'] = sel_speedopt_tmp
cffile[i]['selextra_sel'] = sel_extraopt_tmp
sel_speedopt_basic = copy.deepcopy(self.selectable_speedopt)
sel_speedopt_basic[0][2] = 'selected'
sel_extraopt_basic = copy.deepcopy(self.selectable_extraopt)
sel_extraopt_basic[0][2] = 'selected'
count_ds = 0
cdsfile_toshow1 = []
for i in xrange(len(cdsfile)):
if (cdsfile[i]['type'] == 'DSN'):
cdsfile_toshow = copy.deepcopy(cdsfile[i])
cdsfile_toshow['idx'] = count_ds
cdsfile_toshow['valid_verbose'] = ''
if (cdsfile_toshow['valid'] == 1):
cdsfile_toshow['valid_verbose'] = 'checked=yes'
count_ds = count_ds + 1
sel_speedopt_tmp = copy.deepcopy(self.selectable_speedopt)
sel_speedopt_tmp[cdsfile[i]['speed_class'] - 1][2] = 'selected'
sel_extraopt_tmp = copy.deepcopy(self.selectable_extraopt)
sel_extraopt_tmp[cdsfile[i]['extra_class']][2] = 'selected'
cdsfile_toshow['selspeed_sel'] = sel_speedopt_tmp
cdsfile_toshow['selextra_sel'] = sel_extraopt_tmp
cdsfile_toshow1.append(cdsfile_toshow)
possibleopt = megasearch.listpossiblesearchoptions()
for slctg in possibleopt:
if (slctg[0] == genopt['search_default']):
slctg[2] = 'selected'
tnarray = []
for ntn in xrange(1, 50):
if (genopt['trends_qty'] == ntn):
tnarray.append([ntn, ntn, 'selected'])
else:
tnarray.append([ntn, ntn, ''])
genopt['general_https_verbose'] = ''
genopt['general_trend_verbose'] = ''
genopt['general_suggestion_verbose'] = ''
genopt['smartsearch_verbose'] = ''
genopt['max_cache_verbose'] = ''
genopt['predb_active_verbose'] = ''
genopt['general_restrictopt1_verbose'] = ''
genopt['general_dereferer_verbose'] = ''
if (genopt['predb_active'] == 1):
genopt['predb_active_verbose'] = 'checked=yes'
if (genopt['general_https'] == 1):
genopt['general_https_verbose'] = 'checked=yes'
if (genopt['general_restrictopt1'] == 1):
genopt['general_restrictopt1_verbose'] = 'checked=yes'
if (genopt['general_dereferer'] == 1):
genopt['general_dereferer_verbose'] = 'checked=yes'
if (genopt['general_suggestion'] == 1):
genopt['general_suggestion_verbose'] = 'checked=yes'
if (genopt['general_trend'] == 1):
genopt['general_trend_verbose'] = 'checked=yes'
if (genopt['smartsearch'] == 1):
genopt['smartsearch_verbose'] = 'checked=yes'
if (genopt['use_warp'] == 1):
genopt['use_warp_verbose'] = 'checked=yes'
if (genopt['cache_active'] == 1):
genopt['cache_active_verbose'] = 'checked=yes'
genopt['general_ipaddress_verbose'] = 'AUTO'
if (genopt['general_ipaddress'] != ''):
genopt['general_ipaddress_verbose'] = genopt['general_ipaddress']
openshift_install = False
if (len(self.dirconf_oshift)):
openshift_install = True
return render_template('config.html', cfg=cffile, cfg_dp=cdsfile_toshow1, cnt=count, cnt_ds=count_ds,
genopt=genopt,
selectable_opt=possibleopt,
sel_speedopt_basic=sel_speedopt_basic,
sel_extraopt_basic=sel_extraopt_basic,
openshift_install=openshift_install,
tnarray=tnarray,
cdomainname=cdomainname,
cnt_max=MAX_PROVIDER_NUMBER, cfg_bi=cffileb)
def summary_results(rawResults, strsearch, logic_items=[], results_stats={}):
results = []
titles = []
sptitle_collection = []
#~ stats for each provider
for provid in xrange(len(rawResults)):
if (len(rawResults[provid])):
results_stats[str(rawResults[provid][0]['providertitle'])] = [
len(rawResults[provid]), 0
]
#~ all in one array
for provid in xrange(len(rawResults)):
for z in xrange(len(rawResults[provid])):
if (rawResults[provid][z]['title'] != None):
rawResults[provid][z]['title'] = SearchModule.sanitize_html(
rawResults[provid][z]['title'])
rawResults[provid][z]['provid'] = provid
title = SearchModule.sanitize_strings(
rawResults[provid][z]['title'])
titles.append(title)
sptitle_collection.append(Set(title.split(".")))
results.append(rawResults[provid][z])
strsearch1 = SearchModule.sanitize_strings(strsearch)
strsearch1_collection = Set(strsearch1.split("."))
rcount = [0] * 3
for z in xrange(len(results)):
findone = 0
results[z]['ignore'] = 0
intrs = strsearch1_collection.intersection(sptitle_collection[z])
if (len(intrs) == len(strsearch1_collection)):
findone = 1
else:
results[z]['ignore'] = 2
#~ relax the search ~ 0.45
unmatched_terms_search = strsearch1_collection.difference(intrs)
unmatched_count = 0
for mst in unmatched_terms_search:
my_list = [
i for i in sptitle_collection[z] if i.find(mst) == 0
]
if (len(my_list)):
unmatched_count = unmatched_count + 1
if (unmatched_count == len(unmatched_terms_search)):
findone = 1
results[z]['ignore'] = 0
#~ print unmatched_terms_search
#~ print unmatched_count
#~ print unmatched_terms_search
#~ print strsearch1_collection
#~ print intrs
#~ print findone
#~ print '------------------'
if (findone and results[z]['ignore'] == 0):
#~ print titles[z]
for v in xrange(z + 1, len(results)):
if (titles[z] == titles[v]):
sz1 = float(results[z]['size'])
sz2 = float(results[v]['size'])
if (abs(sz1 - sz2) < 5000000):
results[z]['ignore'] = 1
#~ stats
rcount[results[z]['ignore']] += 1
#~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~
#~ logic params
exclude_coll = Set([])
include_coll = Set([])
#~ print '*'+logic_items[0][1]+'*'
for i in xrange(len(logic_items)):
if (logic_items[i][0] == '-'):
exclude_coll.add(logic_items[i][1])
if (logic_items[i][0] == '+'):
include_coll.add(logic_items[i][1])
if (len(include_coll)):
for z in xrange(len(results)):
if (results[z]['ignore'] < 2):
intrs_i = include_coll.intersection(sptitle_collection[z])
if (len(intrs_i) == 0):
results[z]['ignore'] = 2
if (len(exclude_coll)):
for z in xrange(len(results)):
if (results[z]['ignore'] < 2):
intrs_e = exclude_coll.intersection(sptitle_collection[z])
if (len(intrs_e) > 0):
results[z]['ignore'] = 2
#~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~
mssg = 'Overall search stats: [' + strsearch1 + ']' + ' [' + strsearch + '] ' + str(
rcount[0]) + ' ' + str(rcount[1]) + ' ' + str(rcount[2])
log.info(mssg)
for z in xrange(len(results)):
if (results[z]['ignore'] != 2):
results_stats[str(
results[z]['providertitle'])][1] = results_stats[str(
results[z]['providertitle'])][1] + 1
return results
mega_parall = megasearch.DoParallelSearch(cfgsets.cfg, cfgsets.cgen, ds,
wrp)
apiresp = ApiResponses(cfgsets.cfg, cfgsets.cgen, wrp, ds)
getsmartinfo = nzbsanity.GetNZBInfo(cfgsets.cfg, cfgsets.cgen, ds, app)
auth = miscdefs.Auth(cfgsets)
testserver = miscdefs.ChkServer(cfgsets.cgen)
# ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~
motd = '\n\n~*~ ~*~ NZBMegasearcH ~*~ ~*~'
print motd
DEBUGFLAG = False
LARGESERVER = False
logsdir = SearchModule.resource_path('logs/')
if (len(os.getenv('OPENSHIFT_DATA_DIR', ''))):
logsdir = os.environ.get('OPENSHIFT_DATA_DIR')
if (len(sys.argv) > 1):
for argv in sys.argv:
if (argv == 'help'):
print ''
print '`debug`: start in debug mode'
print '`large`: modality for GUNICORN + NGINX large server'
print '`daemon`: start in daemon mode, detached from terminal'
print ''
exit()
if (argv == 'debug'):
print '====== DEBUGMODE DEBUGMODE DEBUGMODE DEBUGMODE ======'
def dosearch(self, args):
# ~ restore originals
self.cfg = copy.deepcopy(self.cfg_cpy)
if ('q' not in args):
self.results = []
return self.results
nuqry = args['q'] + ' ' + self.cgen['searchaddontxt']
self.logic_items = self.logic_expr.findall(nuqry)
self.qry_nologic = self.logic_expr.sub(" ", nuqry)
if ('selcat' in args):
if (args['selcat'] != ""):
self.qry_nologic += " " + args['selcat']
# ~ speed class
speed_class_sel = 1
if ('tm' in args):
speed_class_sel = int(args['tm'])
# ~ speed class deepsearch
self.ds.set_extraopt(speed_class_sel, 'manual')
# ~ speed class Nabbased
self.set_timeout_speedclass(speed_class_sel)
# ~ manual search Nabbased
self.set_extraopt()
if (len(args['q']) == 0):
if ('selcat' in args):
if (len(args['selcat']) == 0):
self.results = []
return self.results
else:
self.results = []
return self.results
if (self.qry_nologic.replace(" ", "") == ""):
self.results = []
return self.results
log.info('TYPE OF SEARCH: ' + str(speed_class_sel))
self.cleancache()
# ~ cache hit, no server report
cachehit = True
self.returncode_fine['code'] = 2
self.resultsraw = self.chkforcache(self.wrp.chash64_encode(SearchModule.sanitize_strings(self.qry_nologic)),
speed_class_sel)
if (self.resultsraw is None):
self.resultsraw = SearchModule.performSearch(self.qry_nologic, self.cfg, self.ds)
cachehit = False
if (self.cgen['smartsearch'] == 1):
# ~ smartsearch
self.res_results = {}
self.results = summary_results(self.resultsraw, self.qry_nologic, self.logic_items, self.res_results)
else:
# ~ no cleaning just flatten in one array
self.results = []
self.res_results = {}
for provid in xrange(len(self.resultsraw)):
if (len(self.resultsraw[provid])):
self.res_results[str(self.resultsraw[provid][0]['providertitle'])] = [len(self.resultsraw[provid]),
0]
for provid in xrange(len(self.resultsraw)):
for z in xrange(len(self.resultsraw[provid])):
if (self.resultsraw[provid][z]['title'] != None):
self.results.append(self.resultsraw[provid][z])
# ~ server status output
if (cachehit == False):
self.prepareretcode();
def dosearch(self, args):
#~ restore originals
self.cfg = copy.deepcopy(self.cfg_cpy)
if ('q' not in args):
self.results = []
return self.results
nuqry = args['q'] + ' ' + self.cgen['searchaddontxt']
self.logic_items = self.logic_expr.findall(nuqry)
self.qry_nologic = self.logic_expr.sub(" ", nuqry)
if ('selcat' in args):
if (args['selcat'] != ""):
self.qry_nologic += " " + args['selcat']
#~ speed class
speed_class_sel = 1
if ('tm' in args):
speed_class_sel = int(args['tm'])
#~ speed class deepsearch
self.ds.set_extraopt(speed_class_sel, 'manual')
#~ speed class Nabbased
self.set_timeout_speedclass(speed_class_sel)
#~ manual search Nabbased
self.set_extraopt()
if (len(args['q']) == 0):
if ('selcat' in args):
if (len(args['selcat']) == 0):
self.results = []
return self.results
else:
self.results = []
return self.results
if (self.qry_nologic.replace(" ", "") == ""):
self.results = []
return self.results
log.info('TYPE OF SEARCH: ' + str(speed_class_sel))
self.cleancache()
#~ cache hit, no server report
cachehit = True
self.returncode_fine['code'] = 2
self.resultsraw = self.chkforcache(
self.wrp.chash64_encode(
SearchModule.sanitize_strings(self.qry_nologic)),
speed_class_sel)
if (self.resultsraw is None):
self.resultsraw = SearchModule.performSearch(
self.qry_nologic, self.cfg, self.ds)
cachehit = False
if (self.cgen['smartsearch'] == 1):
#~ smartsearch
self.res_results = {}
self.results = summary_results(self.resultsraw, self.qry_nologic,
self.logic_items, self.res_results)
else:
#~ no cleaning just flatten in one array
self.results = []
self.res_results = {}
for provid in xrange(len(self.resultsraw)):
if (len(self.resultsraw[provid])):
self.res_results[str(
self.resultsraw[provid][0]['providertitle'])] = [
len(self.resultsraw[provid]), 0
]
for provid in xrange(len(self.resultsraw)):
for z in xrange(len(self.resultsraw[provid])):
if (self.resultsraw[provid][z]['title'] != None):
self.results.append(self.resultsraw[provid][z])
#~ server status output
if (cachehit == False):
self.prepareretcode()
def summary_results(rawResults, strsearch, logic_items=[], results_stats={}):
results = []
titles = []
sptitle_collection = []
# ~ stats for each provider
for provid in xrange(len(rawResults)):
if (len(rawResults[provid])):
results_stats[str(rawResults[provid][0]['providertitle'])] = [len(rawResults[provid]), 0]
# ~ all in one array
for provid in xrange(len(rawResults)):
for z in xrange(len(rawResults[provid])):
if (rawResults[provid][z]['title'] != None):
rawResults[provid][z]['title'] = SearchModule.sanitize_html(rawResults[provid][z]['title'])
rawResults[provid][z]['provid'] = provid
title = SearchModule.sanitize_strings(rawResults[provid][z]['title'])
titles.append(title)
sptitle_collection.append(Set(title.split(".")))
results.append(rawResults[provid][z])
strsearch1 = SearchModule.sanitize_strings(strsearch)
strsearch1_collection = Set(strsearch1.split("."))
rcount = [0] * 3
for z in xrange(len(results)):
findone = 0
results[z]['ignore'] = 0
intrs = strsearch1_collection.intersection(sptitle_collection[z])
if (len(intrs) == len(strsearch1_collection)):
findone = 1
else:
results[z]['ignore'] = 2
# ~ relax the search ~ 0.45
unmatched_terms_search = strsearch1_collection.difference(intrs)
unmatched_count = 0
for mst in unmatched_terms_search:
my_list = [i for i in sptitle_collection[z] if i.find(mst) == 0]
if (len(my_list)):
unmatched_count = unmatched_count + 1
if (unmatched_count == len(unmatched_terms_search)):
findone = 1
results[z]['ignore'] = 0
# ~ print unmatched_terms_search
# ~ print unmatched_count
# ~ print unmatched_terms_search
# ~ print strsearch1_collection
# ~ print intrs
# ~ print findone
# ~ print '------------------'
if (findone and results[z]['ignore'] == 0):
# ~ print titles[z]
for v in xrange(z + 1, len(results)):
if (titles[z] == titles[v]):
sz1 = float(results[z]['size'])
sz2 = float(results[v]['size'])
if (abs(sz1 - sz2) < 5000000):
results[z]['ignore'] = 1
# ~ stats
rcount[results[z]['ignore']] += 1
# ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~
# ~ logic params
exclude_coll = Set([])
include_coll = Set([])
# ~ print '*'+logic_items[0][1]+'*'
for i in xrange(len(logic_items)):
if (logic_items[i][0] == '-'):
exclude_coll.add(logic_items[i][1])
if (logic_items[i][0] == '+'):
include_coll.add(logic_items[i][1])
if (len(include_coll)):
for z in xrange(len(results)):
if (results[z]['ignore'] < 2):
intrs_i = include_coll.intersection(sptitle_collection[z])
if (len(intrs_i) == 0):
results[z]['ignore'] = 2
if (len(exclude_coll)):
for z in xrange(len(results)):
if (results[z]['ignore'] < 2):
intrs_e = exclude_coll.intersection(sptitle_collection[z])
if (len(intrs_e) > 0):
results[z]['ignore'] = 2
# ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~
mssg = 'Overall search stats: [' + strsearch1 + ']' + ' [' + strsearch + '] ' + str(rcount[0]) + ' ' + str(
rcount[1]) + ' ' + str(rcount[2])
log.info(mssg)
for z in xrange(len(results)):
if (results[z]['ignore'] != 2):
results_stats[str(results[z]['providertitle'])][1] = results_stats[str(results[z]['providertitle'])][1] + 1
return results
def beam_cookie(self, urltouse, args):
retfail = -1
global globalResults
if 'loadedModules' not in globals():
SearchModule.loadSearchModules()
cookie = {}
dwntype = 0
index = 0
selcfg_idx = 0
#~ take the right password
for i in xrange(len(self.cfg)):
if (self.cfg[i]['type'] == args['m']):
selcfg_idx = i
#~ standard search
for module in SearchModule.loadedModules:
if (module.typesrch == args['m']):
dwntype = 0
if (module.dologin(self.cfg[selcfg_idx]) == True):
cookie = module.cookie
else:
return retfail
#~ deep search
deepidx = 0
for index in xrange(len(self.dsearch.ds)):
if (self.dsearch.ds[index].typesrch == args['m']):
dwntype = 1
deepidx = index
f_name = ''
if (dwntype == 0):
log.info('WARPNGN Cookie FTD found')
try:
opener = urllib2.build_opener()
opener.addheaders.append(
('Cookie', 'FTDWSESSID=' + cookie['FTDWSESSID']))
response = opener.open(urltouse)
except Exception as e:
return retfail
if (dwntype == 1):
log.info('WARPNGN Cookie deep found')
response = self.dsearch.ds[deepidx].download(urltouse)
if (response == ''):
return -1
fcontent = response.read()
#~ print response.info()
f = tempfile.NamedTemporaryFile(delete=False)
f.write(fcontent)
f.close()
fresponse = send_file(f.name,
mimetype='application/x-nzb;',
as_attachment=True,
attachment_filename='yourmovie.nzb',
add_etags=False,
cache_timeout=None,
conditional=False)
try:
os.remove(f.name)
except Exception as e:
print 'Cannot remove temporary NZB file'
for i in xrange(len(response.info().headers)):
if (response.info().headers[i].find('Content-Encoding') != -1):
fresponse.headers["Content-Encoding"] = 'gzip'
riff = response.info().headers[i].find('Content-Disposition')
if (riff != -1):
fresponse.headers["Content-Disposition"] = response.info(
).headers[i][riff +
21:len(response.info().headers[i])].strip()
return fresponse