def searchit(self): chk_id = None #logger.info('searchterm: %s' % self.searchterm) #self.searchterm is a tuple containing series name, issue number, volume and publisher. series_search = self.searchterm['series'] issue_search = self.searchterm['issue'] volume_search = self.searchterm['volume'] if series_search.startswith('0-Day Comics Pack'): #issue = '21' = WED, #volume='2' = 2nd month torrentid = 22247 #2018 publisher_search = None #'2' #2nd month comic_id = None elif all([ self.searchterm['torrentid_32p'] is not None, self.searchterm['torrentid_32p'] != 'None' ]): torrentid = self.searchterm['torrentid_32p'] comic_id = self.searchterm['id'] publisher_search = self.searchterm['publisher'] else: torrentid = None comic_id = self.searchterm['id'] annualize = False if 'annual' in series_search.lower(): series_search = re.sub(' annual', '', series_search.lower()).strip() annualize = True publisher_search = self.searchterm['publisher'] spl = [x for x in self.publisher_list if x in publisher_search] for x in spl: publisher_search = re.sub(x, '', publisher_search).strip() #logger.info('publisher search set to : %s' % publisher_search) # lookup the ComicID in the 32p sqlite3 table to pull the series_id to use. if comic_id: chk_id = helpers.checkthe_id(comic_id) if any([chk_id is None, mylar.CONFIG.DEEP_SEARCH_32P is True]): #generate the dynamic name of the series here so we can match it up as_d = filechecker.FileChecker() as_dinfo = as_d.dynamic_replace(series_search) mod_series = re.sub('\|', '', as_dinfo['mod_seriesname']).strip() as_puinfo = as_d.dynamic_replace(publisher_search) pub_series = as_puinfo['mod_seriesname'] logger.fdebug('series_search: %s' % series_search) if '/' in series_search: series_search = series_search[:series_search.find('/')] if ':' in series_search: series_search = series_search[:series_search.find(':')] if ',' in series_search: series_search = series_search[:series_search.find(',')] logger.fdebug('config.search_32p: %s' % mylar.CONFIG.SEARCH_32P) if mylar.CONFIG.SEARCH_32P is False: url = 'https://walksoftly.itsaninja.party/serieslist.php' params = { 'series': re.sub('\|', '', mod_series.lower()).strip() } #series_search} logger.fdebug('search query: %s' % re.sub('\|', '', mod_series.lower()).strip()) try: t = requests.get( url, params=params, verify=True, headers={ 'USER-AGENT': mylar.USER_AGENT[:mylar.USER_AGENT.find('/') + 7] + mylar.USER_AGENT[mylar.USER_AGENT.find('(') + 1] }) except requests.exceptions.RequestException as e: logger.warn(e) return "no results" if t.status_code == '619': logger.warn('[%s] Unable to retrieve data from site.' % t.status_code) return "no results" elif t.status_code == '999': logger.warn( '[%s] No series title was provided to the search query.' % t.status_code) return "no results" try: results = t.json() except: results = t.text if len(results) == 0: logger.warn('No results found for search on 32P.') return "no results" # with cfscrape.create_scraper(delay=15) as s: # s.headers = self.headers # cj = LWPCookieJar(os.path.join(mylar.CONFIG.SECURE_DIR, ".32p_cookies.dat")) # cj.load() # s.cookies = cj data = [] pdata = [] pubmatch = False if any([ series_search.startswith('0-Day Comics Pack'), torrentid is not None ]): data.append({"id": torrentid, "series": series_search}) else: if any([not chk_id, mylar.CONFIG.DEEP_SEARCH_32P is True]): if mylar.CONFIG.SEARCH_32P is True: url = 'https://32pag.es/torrents.php' #?action=serieslist&filter=' + series_search #&filter=F params = {'action': 'serieslist', 'filter': series_search} time.sleep( 1) #just to make sure we don't hammer, 1s pause. t = self.session.get(url, params=params, verify=True, allow_redirects=True) soup = BeautifulSoup(t.content, "html.parser") results = soup.find_all("a", {"class": "object-qtip"}, {"data-type": "torrentgroup"}) for r in results: if mylar.CONFIG.SEARCH_32P is True: torrentid = r['data-id'] torrentname = r.findNext(text=True) torrentname = torrentname.strip() else: torrentid = r['id'] torrentname = r['series'] as_d = filechecker.FileChecker() as_dinfo = as_d.dynamic_replace(torrentname) seriesresult = re.sub('\|', '', as_dinfo['mod_seriesname']).strip() logger.fdebug('searchresult: %s --- %s [%s]' % (seriesresult, mod_series, publisher_search)) if seriesresult.lower() == mod_series.lower(): logger.fdebug('[MATCH] %s [%s]' % (torrentname, torrentid)) data.append({"id": torrentid, "series": torrentname}) elif publisher_search.lower() in seriesresult.lower(): logger.fdebug('[MATCH] Publisher match.') tmp_torrentname = re.sub(publisher_search.lower(), '', seriesresult.lower()).strip() as_t = filechecker.FileChecker() as_tinfo = as_t.dynamic_replace(tmp_torrentname) if re.sub('\|', '', as_tinfo['mod_seriesname']).strip( ) == mod_series.lower(): logger.fdebug('[MATCH] %s [%s]' % (torrentname, torrentid)) pdata.append({ "id": torrentid, "series": torrentname }) pubmatch = True logger.fdebug('%s series listed for searching that match.' % len(data)) else: logger.fdebug( 'Exact series ID already discovered previously. Setting to : %s [%s]' % (chk_id['series'], chk_id['id'])) pdata.append({"id": chk_id['id'], "series": chk_id['series']}) pubmatch = True if all([len(data) == 0, len(pdata) == 0]): return "no results" else: dataset = [] if len(data) > 0: dataset += data if len(pdata) > 0: dataset += pdata logger.fdebug( str(len(dataset)) + ' series match the tile being searched for on 32P...') if all([ chk_id is None, not series_search.startswith('0-Day Comics Pack'), self.searchterm['torrentid_32p'] is not None, self.searchterm['torrentid_32p'] != 'None' ]) and any([len(data) == 1, len(pdata) == 1]): #update the 32p_reference so we avoid doing a url lookup next time helpers.checkthe_id(comic_id, dataset) else: if all([ not series_search.startswith('0-Day Comics Pack'), self.searchterm['torrentid_32p'] is not None, self.searchterm['torrentid_32p'] != 'None' ]): pass else: logger.debug( 'Unable to properly verify reference on 32P - will update the 32P reference point once the issue has been successfully matched against.' ) results32p = [] resultlist = {} for x in dataset: #for 0-day packs, issue=week#, volume=month, id=0-day year pack (ie.issue=21&volume=2 for feb.21st) payload = { "action": "groupsearch", "id": x['id'], #searchid, "issue": issue_search } #in order to match up against 0-day stuff, volume has to be none at this point #when doing other searches tho, this should be allowed to go through #if all([volume_search != 'None', volume_search is not None]): # payload.update({'volume': re.sub('v', '', volume_search).strip()}) if series_search.startswith('0-Day Comics Pack'): payload.update({"volume": volume_search}) payload = json.dumps(payload) payload = json.loads(payload) logger.fdebug('payload: %s' % payload) url = 'https://32pag.es/ajax.php' time.sleep(1) #just to make sure we don't hammer, 1s pause. try: d = self.session.get(url, params=payload, verify=True, allow_redirects=True) except Exception as e: logger.error('%s [%s] Could not POST URL %s' % (self.module, e, url)) try: searchResults = d.json() except Exception as e: searchResults = d.text logger.debug( '[%s] %s Search Result did not return valid JSON, falling back on text: %s' % (e, self.module, searchResults.text)) return False if searchResults[ 'status'] == 'success' and searchResults['count'] > 0: logger.fdebug('successfully retrieved %s search results' % searchResults['count']) for a in searchResults['details']: if series_search.startswith('0-Day Comics Pack'): title = series_search else: title = self.searchterm['series'] + ' v' + a[ 'volume'] + ' #' + a['issues'] results32p.append({ 'link': a['id'], 'title': title, 'filesize': a['size'], 'issues': a['issues'], 'pack': a['pack'], 'format': a['format'], 'language': a['language'], 'seeders': a['seeders'], 'leechers': a['leechers'], 'scanner': a['scanner'], 'chkit': { 'id': x['id'], 'series': x['series'] }, 'pubdate': datetime.datetime.fromtimestamp(float( a['upload_time'])).strftime( '%a, %d %b %Y %H:%M:%S'), 'int_pubdate': float(a['upload_time']) }) else: logger.fdebug('32P did not return any valid search results.') if len(results32p) > 0: resultlist['entries'] = sorted(results32p, key=itemgetter('pack', 'title'), reverse=False) logger.debug('%s Resultslist: %s' % (self.module, resultlist)) else: resultlist = 'no results' return resultlist
def searchit(self): #self.searchterm is a tuple containing series name, issue number, volume and publisher. series_search = self.searchterm['series'] comic_id = self.searchterm['id'] if comic_id: chk_id = helpers.checkthe_id(comic_id) annualize = False if 'Annual' in series_search: series_search = re.sub(' Annual', '', series_search).strip() annualize = True issue_search = self.searchterm['issue'] volume_search = self.searchterm['volume'] publisher_search = self.searchterm['publisher'] spl = [x for x in self.publisher_list if x in publisher_search] for x in spl: publisher_search = re.sub(x, '', publisher_search).strip() logger.info('publisher search set to : ' + publisher_search) chk_id = None # lookup the ComicID in the 32p sqlite3 table to pull the series_id to use. if comic_id: chk_id = helpers.checkthe_id(comic_id) if not chk_id: #generate the dynamic name of the series here so we can match it up as_d = filechecker.FileChecker() as_dinfo = as_d.dynamic_replace(series_search) mod_series = re.sub('\|','', as_dinfo['mod_seriesname']).strip() as_puinfo = as_d.dynamic_replace(publisher_search) pub_series = as_puinfo['mod_seriesname'] logger.info('series_search: ' + series_search) if '/' in series_search: series_search = series_search[:series_search.find('/')] if ':' in series_search: series_search = series_search[:series_search.find(':')] if ',' in series_search: series_search = series_search[:series_search.find(',')] if not mylar.SEARCH_32P: url = 'https://walksoftly.itsaninja.party/serieslist.php' params = {'series': re.sub('\|','', mod_series.lower()).strip()} #series_search} try: t = requests.get(url, params=params, verify=True, headers={'USER-AGENT': mylar.USER_AGENT[:mylar.USER_AGENT.find('/')+7] + mylar.USER_AGENT[mylar.USER_AGENT.find('(')+1]}) except requests.exceptions.RequestException as e: logger.warn(e) return "no results" if t.status_code == '619': logger.warn('[' + str(t.status_code) + '] Unable to retrieve data from site.') return "no results" elif t.status_code == '999': logger.warn('[' + str(t.status_code) + '] No series title was provided to the search query.') return "no results" try: results = t.json() except: results = t.text if len(results) == 0: logger.warn('No results found for search on 32P.') return "no results" with cfscrape.create_scraper() as s: s.headers = self.headers cj = LWPCookieJar(os.path.join(mylar.CACHE_DIR, ".32p_cookies.dat")) cj.load() s.cookies = cj data = [] pdata = [] pubmatch = False if not chk_id: if mylar.SEARCH_32P: url = 'https://32pag.es/torrents.php' #?action=serieslist&filter=' + series_search #&filter=F params = {'action': 'serieslist', 'filter': series_search} time.sleep(1) #just to make sure we don't hammer, 1s pause. t = s.get(url, params=params, verify=True, allow_redirects=True) soup = BeautifulSoup(t.content, "html.parser") results = soup.find_all("a", {"class":"object-qtip"},{"data-type":"torrentgroup"}) for r in results: if mylar.SEARCH_32P: torrentid = r['data-id'] torrentname = r.findNext(text=True) torrentname = torrentname.strip() else: torrentid = r['id'] torrentname = r['series'] as_d = filechecker.FileChecker() as_dinfo = as_d.dynamic_replace(torrentname) seriesresult = re.sub('\|','', as_dinfo['mod_seriesname']).strip() #seriesresult = as_dinfo['mod_seriesname'] logger.info('searchresult: ' + seriesresult + ' --- ' + mod_series + '[' + publisher_search + ']') if seriesresult == mod_series: logger.info('[MATCH] ' + torrentname + ' [' + str(torrentid) + ']') data.append({"id": torrentid, "series": torrentname}) elif publisher_search in seriesresult: logger.info('publisher match.') tmp_torrentname = re.sub(publisher_search, '', seriesresult).strip() as_t = filechecker.FileChecker() as_tinfo = as_t.dynamic_replace(tmp_torrentname) logger.info('tmp_torrentname:' + tmp_torrentname) logger.info('as_tinfo:' + as_tinfo['mod_seriesname']) if re.sub('\|', '', as_tinfo['mod_seriesname']).strip() == mod_series: logger.info('[MATCH] ' + torrentname + ' [' + str(torrentid) + ']') pdata.append({"id": torrentid, "series": torrentname}) pubmatch = True logger.info(str(len(data)) + ' series listed for searching that match.') else: logger.info('Exact series ID already discovered previously. Setting to :' + chk_id['series'] + '[' + str(chk_id['id']) + ']') pdata.append({"id": chk_id['id'], "series": chk_id['series']}) pubmatch = True if all([len(data) == 0, len(pdata) == 0]): return "no results" if len(pdata) == 1: logger.info(str(len(pdata)) + ' series match the title being search for') dataset = pdata searchid = pdata[0]['id'] elif len(data) == 1: logger.info(str(len(data)) + ' series match the title being search for') dataset = data searchid = data[0]['id'] else: dataset = [] if len(data) > 0: dataset += data if len(pdata) > 0: dataset += pdata if chk_id is None and any([len(data) == 1, len(pdata) == 1]): #update the 32p_reference so we avoid doing a url lookup next time helpers.checkthe_id(comic_id, dataset) else: logger.warn('More than one result - will update the 32p reference point once the issue has been successfully matched against.') results32p = [] resultlist = {} for x in dataset: payload = {'action': 'groupsearch', 'id': x['id'], #searchid, 'issue': issue_search} #in order to match up against 0-day stuff, volume has to be none at this point #when doing other searches tho, this should be allowed to go through #if all([volume_search != 'None', volume_search is not None]): # payload.update({'volume': re.sub('v', '', volume_search).strip()}) logger.info('payload: ' + str(payload)) url = 'https://32pag.es/ajax.php' time.sleep(1) #just to make sure we don't hammer, 1s pause. try: d = s.post(url, params=payload, verify=True, allow_redirects=True) #logger.debug(self.module + ' Reply from AJAX: \n %s', d.text) except Exception as e: logger.info(self.module + ' Could not POST URL %s', url) try: searchResults = d.json() except: searchResults = d.text logger.debug(self.module + ' Search Result did not return valid JSON, falling back on text: %s', searchResults.text) return False #logger.debug(self.module + " Search Result: %s", searchResults) if searchResults['status'] == 'success' and searchResults['count'] > 0: logger.info('successfully retrieved ' + str(searchResults['count']) + ' search results.') for a in searchResults['details']: results32p.append({'link': a['id'], 'title': self.searchterm['series'] + ' v' + a['volume'] + ' #' + a['issues'], 'filesize': a['size'], 'issues': a['issues'], 'pack': a['pack'], 'format': a['format'], 'language': a['language'], 'seeders': a['seeders'], 'leechers': a['leechers'], 'scanner': a['scanner'], 'chkit': {'id': x['id'], 'series': x['series']}, 'pubdate': datetime.datetime.fromtimestamp(float(a['upload_time'])).strftime('%c')}) if len(results32p) > 0: resultlist['entries'] = sorted(results32p, key=itemgetter('pack','title'), reverse=False) else: resultlist = 'no results' return resultlist
def searchit(self): #self.searchterm is a tuple containing series name, issue number, volume and publisher. series_search = self.searchterm['series'] comic_id = self.searchterm['id'] annualize = False if 'Annual' in series_search: series_search = re.sub(' Annual', '', series_search).strip() annualize = True issue_search = self.searchterm['issue'] volume_search = self.searchterm['volume'] publisher_search = self.searchterm['publisher'] spl = [x for x in self.publisher_list if x in publisher_search] for x in spl: publisher_search = re.sub(x, '', publisher_search).strip() logger.info('publisher search set to : ' + publisher_search) chk_id = None # lookup the ComicID in the 32p sqlite3 table to pull the series_id to use. if comic_id: chk_id = helpers.checkthe_id(comic_id) if any([not chk_id, mylar.DEEP_SEARCH_32P is True]): #generate the dynamic name of the series here so we can match it up as_d = filechecker.FileChecker() as_dinfo = as_d.dynamic_replace(series_search) mod_series = re.sub('\|','', as_dinfo['mod_seriesname']).strip() as_puinfo = as_d.dynamic_replace(publisher_search) pub_series = as_puinfo['mod_seriesname'] logger.info('series_search: ' + series_search) if '/' in series_search: series_search = series_search[:series_search.find('/')] if ':' in series_search: series_search = series_search[:series_search.find(':')] if ',' in series_search: series_search = series_search[:series_search.find(',')] if not mylar.SEARCH_32P: url = 'https://walksoftly.itsaninja.party/serieslist.php' params = {'series': re.sub('\|','', mod_series.lower()).strip()} #series_search} try: t = requests.get(url, params=params, verify=True, headers={'USER-AGENT': mylar.USER_AGENT[:mylar.USER_AGENT.find('/')+7] + mylar.USER_AGENT[mylar.USER_AGENT.find('(')+1]}) except requests.exceptions.RequestException as e: logger.warn(e) return "no results" if t.status_code == '619': logger.warn('[' + str(t.status_code) + '] Unable to retrieve data from site.') return "no results" elif t.status_code == '999': logger.warn('[' + str(t.status_code) + '] No series title was provided to the search query.') return "no results" try: results = t.json() except: results = t.text if len(results) == 0: logger.warn('No results found for search on 32P.') return "no results" with cfscrape.create_scraper() as s: s.headers = self.headers cj = LWPCookieJar(os.path.join(mylar.CACHE_DIR, ".32p_cookies.dat")) cj.load() s.cookies = cj data = [] pdata = [] pubmatch = False if any([not chk_id, mylar.DEEP_SEARCH_32P is True]): if mylar.SEARCH_32P: url = 'https://32pag.es/torrents.php' #?action=serieslist&filter=' + series_search #&filter=F params = {'action': 'serieslist', 'filter': series_search} time.sleep(1) #just to make sure we don't hammer, 1s pause. t = s.get(url, params=params, verify=True, allow_redirects=True) soup = BeautifulSoup(t.content, "html.parser") results = soup.find_all("a", {"class":"object-qtip"},{"data-type":"torrentgroup"}) for r in results: if mylar.SEARCH_32P: torrentid = r['data-id'] torrentname = r.findNext(text=True) torrentname = torrentname.strip() else: torrentid = r['id'] torrentname = r['series'] as_d = filechecker.FileChecker() as_dinfo = as_d.dynamic_replace(torrentname) seriesresult = re.sub('\|','', as_dinfo['mod_seriesname']).strip() #seriesresult = as_dinfo['mod_seriesname'] logger.info('searchresult: ' + seriesresult + ' --- ' + mod_series + '[' + publisher_search + ']') if seriesresult == mod_series: logger.info('[MATCH] ' + torrentname + ' [' + str(torrentid) + ']') data.append({"id": torrentid, "series": torrentname}) elif publisher_search in seriesresult: logger.info('publisher match.') tmp_torrentname = re.sub(publisher_search, '', seriesresult).strip() as_t = filechecker.FileChecker() as_tinfo = as_t.dynamic_replace(tmp_torrentname) logger.info('tmp_torrentname:' + tmp_torrentname) logger.info('as_tinfo:' + as_tinfo['mod_seriesname']) if re.sub('\|', '', as_tinfo['mod_seriesname']).strip() == mod_series: logger.info('[MATCH] ' + torrentname + ' [' + str(torrentid) + ']') pdata.append({"id": torrentid, "series": torrentname}) pubmatch = True logger.info(str(len(data)) + ' series listed for searching that match.') else: logger.info('Exact series ID already discovered previously. Setting to :' + chk_id['series'] + '[' + str(chk_id['id']) + ']') pdata.append({"id": chk_id['id'], "series": chk_id['series']}) pubmatch = True if all([len(data) == 0, len(pdata) == 0]): return "no results" else: dataset = [] if len(data) > 0: dataset += data if len(pdata) > 0: dataset += pdata logger.info('dataset: %s' % dataset) logger.info(str(len(dataset)) + ' series match the tile being searched for on 32P...') if chk_id is None and any([len(data) == 1, len(pdata) == 1]): #update the 32p_reference so we avoid doing a url lookup next time helpers.checkthe_id(comic_id, dataset) else: logger.debug('Unable to properly verify reference on 32P - will update the 32P reference point once the issue has been successfully matched against.') results32p = [] resultlist = {} for x in dataset: #for 0-day packs, issue=week#, volume=month, id=0-day year pack payload = {'action': 'groupsearch', 'id': x['id'], #searchid, 'issue': issue_search} #in order to match up against 0-day stuff, volume has to be none at this point #when doing other searches tho, this should be allowed to go through #if all([volume_search != 'None', volume_search is not None]): # payload.update({'volume': re.sub('v', '', volume_search).strip()}) logger.info('payload: ' + str(payload)) url = 'https://32pag.es/ajax.php' time.sleep(1) #just to make sure we don't hammer, 1s pause. try: d = s.post(url, params=payload, verify=True, allow_redirects=True) #logger.debug(self.module + ' Reply from AJAX: \n %s', d.text) except Exception as e: logger.info(self.module + ' Could not POST URL %s', url) try: searchResults = d.json() except: searchResults = d.text logger.debug(self.module + ' Search Result did not return valid JSON, falling back on text: %s', searchResults.text) return False #logger.debug(self.module + " Search Result: %s", searchResults) if searchResults['status'] == 'success' and searchResults['count'] > 0: logger.info('successfully retrieved ' + str(searchResults['count']) + ' search results.') for a in searchResults['details']: results32p.append({'link': a['id'], 'title': self.searchterm['series'] + ' v' + a['volume'] + ' #' + a['issues'], 'filesize': a['size'], 'issues': a['issues'], 'pack': a['pack'], 'format': a['format'], 'language': a['language'], 'seeders': a['seeders'], 'leechers': a['leechers'], 'scanner': a['scanner'], 'chkit': {'id': x['id'], 'series': x['series']}, 'pubdate': datetime.datetime.fromtimestamp(float(a['upload_time'])).strftime('%a, %d %b %Y %H:%M:%S'), 'int_pubdate': float(a['upload_time'])}) if len(results32p) > 0: resultlist['entries'] = sorted(results32p, key=itemgetter('pack','title'), reverse=False) else: resultlist = 'no results' return resultlist
def searchit(self): chk_id = None #logger.info('searchterm: %s' % self.searchterm) #self.searchterm is a tuple containing series name, issue number, volume and publisher. series_search = self.searchterm['series'] issue_search = self.searchterm['issue'] volume_search = self.searchterm['volume'] if series_search.startswith('0-Day Comics Pack'): #issue = '21' = WED, #volume='2' = 2nd month torrentid = 22247 #2018 publisher_search = None #'2' #2nd month comic_id = None elif all([self.searchterm['torrentid_32p'] is not None, self.searchterm['torrentid_32p'] != 'None']): torrentid = self.searchterm['torrentid_32p'] comic_id = self.searchterm['id'] publisher_search = self.searchterm['publisher'] else: torrentid = None comic_id = self.searchterm['id'] annualize = False if 'annual' in series_search.lower(): series_search = re.sub(' annual', '', series_search.lower()).strip() annualize = True publisher_search = self.searchterm['publisher'] spl = [x for x in self.publisher_list if x in publisher_search] for x in spl: publisher_search = re.sub(x, '', publisher_search).strip() #logger.info('publisher search set to : %s' % publisher_search) # lookup the ComicID in the 32p sqlite3 table to pull the series_id to use. if comic_id: chk_id = helpers.checkthe_id(comic_id) if any([chk_id is None, mylar.CONFIG.DEEP_SEARCH_32P is True]): #generate the dynamic name of the series here so we can match it up as_d = filechecker.FileChecker() as_dinfo = as_d.dynamic_replace(series_search) mod_series = re.sub('\|','', as_dinfo['mod_seriesname']).strip() as_puinfo = as_d.dynamic_replace(publisher_search) pub_series = as_puinfo['mod_seriesname'] logger.fdebug('series_search: %s' % series_search) if '/' in series_search: series_search = series_search[:series_search.find('/')] if ':' in series_search: series_search = series_search[:series_search.find(':')] if ',' in series_search: series_search = series_search[:series_search.find(',')] logger.fdebug('config.search_32p: %s' % mylar.CONFIG.SEARCH_32P) if mylar.CONFIG.SEARCH_32P is False: url = 'https://walksoftly.itsaninja.party/serieslist.php' params = {'series': re.sub('\|','', mod_series.lower()).strip()} #series_search} logger.fdebug('search query: %s' % re.sub('\|', '', mod_series.lower()).strip()) try: t = requests.get(url, params=params, verify=True, headers={'USER-AGENT': mylar.USER_AGENT[:mylar.USER_AGENT.find('/')+7] + mylar.USER_AGENT[mylar.USER_AGENT.find('(')+1]}) except requests.exceptions.RequestException as e: logger.warn(e) return "no results" if t.status_code == '619': logger.warn('[%s] Unable to retrieve data from site.' % t.status_code) return "no results" elif t.status_code == '999': logger.warn('[%s] No series title was provided to the search query.' % t.status_code) return "no results" try: results = t.json() except: results = t.text if len(results) == 0: logger.warn('No results found for search on 32P.') return "no results" # with cfscrape.create_scraper(delay=15) as s: # s.headers = self.headers # cj = LWPCookieJar(os.path.join(mylar.CONFIG.SECURE_DIR, ".32p_cookies.dat")) # cj.load() # s.cookies = cj data = [] pdata = [] pubmatch = False if any([series_search.startswith('0-Day Comics Pack'), torrentid is not None]): data.append({"id": torrentid, "series": series_search}) else: if any([not chk_id, mylar.CONFIG.DEEP_SEARCH_32P is True]): if mylar.CONFIG.SEARCH_32P is True: url = 'https://32pag.es/torrents.php' #?action=serieslist&filter=' + series_search #&filter=F params = {'action': 'serieslist', 'filter': series_search} time.sleep(1) #just to make sure we don't hammer, 1s pause. t = self.session.get(url, params=params, verify=True, allow_redirects=True) soup = BeautifulSoup(t.content, "html.parser") results = soup.find_all("a", {"class":"object-qtip"},{"data-type":"torrentgroup"}) for r in results: if mylar.CONFIG.SEARCH_32P is True: torrentid = r['data-id'] torrentname = r.findNext(text=True) torrentname = torrentname.strip() else: torrentid = r['id'] torrentname = r['series'] as_d = filechecker.FileChecker() as_dinfo = as_d.dynamic_replace(torrentname) seriesresult = re.sub('\|','', as_dinfo['mod_seriesname']).strip() logger.fdebug('searchresult: %s --- %s [%s]' % (seriesresult, mod_series, publisher_search)) if seriesresult.lower() == mod_series.lower(): logger.fdebug('[MATCH] %s [%s]' % (torrentname, torrentid)) data.append({"id": torrentid, "series": torrentname}) elif publisher_search.lower() in seriesresult.lower(): logger.fdebug('[MATCH] Publisher match.') tmp_torrentname = re.sub(publisher_search.lower(), '', seriesresult.lower()).strip() as_t = filechecker.FileChecker() as_tinfo = as_t.dynamic_replace(tmp_torrentname) if re.sub('\|', '', as_tinfo['mod_seriesname']).strip() == mod_series.lower(): logger.fdebug('[MATCH] %s [%s]' % (torrentname, torrentid)) pdata.append({"id": torrentid, "series": torrentname}) pubmatch = True logger.fdebug('%s series listed for searching that match.' % len(data)) else: logger.fdebug('Exact series ID already discovered previously. Setting to : %s [%s]' % (chk_id['series'], chk_id['id'])) pdata.append({"id": chk_id['id'], "series": chk_id['series']}) pubmatch = True if all([len(data) == 0, len(pdata) == 0]): return "no results" else: dataset = [] if len(data) > 0: dataset += data if len(pdata) > 0: dataset += pdata logger.fdebug(str(len(dataset)) + ' series match the tile being searched for on 32P...') if all([chk_id is None, not series_search.startswith('0-Day Comics Pack'), self.searchterm['torrentid_32p'] is not None, self.searchterm['torrentid_32p'] != 'None']) and any([len(data) == 1, len(pdata) == 1]): #update the 32p_reference so we avoid doing a url lookup next time helpers.checkthe_id(comic_id, dataset) else: if all([not series_search.startswith('0-Day Comics Pack'), self.searchterm['torrentid_32p'] is not None, self.searchterm['torrentid_32p'] != 'None']): pass else: logger.debug('Unable to properly verify reference on 32P - will update the 32P reference point once the issue has been successfully matched against.') results32p = [] resultlist = {} for x in dataset: #for 0-day packs, issue=week#, volume=month, id=0-day year pack (ie.issue=21&volume=2 for feb.21st) payload = {"action": "groupsearch", "id": x['id'], #searchid, "issue": issue_search} #in order to match up against 0-day stuff, volume has to be none at this point #when doing other searches tho, this should be allowed to go through #if all([volume_search != 'None', volume_search is not None]): # payload.update({'volume': re.sub('v', '', volume_search).strip()}) if series_search.startswith('0-Day Comics Pack'): payload.update({"volume": volume_search}) payload = json.dumps(payload) payload = json.loads(payload) logger.fdebug('payload: %s' % payload) url = 'https://32pag.es/ajax.php' time.sleep(1) #just to make sure we don't hammer, 1s pause. try: d = self.session.get(url, params=payload, verify=True, allow_redirects=True) except Exception as e: logger.error('%s [%s] Could not POST URL %s' % (self.module, e, url)) try: searchResults = d.json() except Exception as e: searchResults = d.text logger.debug('[%s] %s Search Result did not return valid JSON, falling back on text: %s' % (e, self.module, searchResults.text)) return False if searchResults['status'] == 'success' and searchResults['count'] > 0: logger.fdebug('successfully retrieved %s search results' % searchResults['count']) for a in searchResults['details']: if series_search.startswith('0-Day Comics Pack'): title = series_search else: title = self.searchterm['series'] + ' v' + a['volume'] + ' #' + a['issues'] results32p.append({'link': a['id'], 'title': title, 'filesize': a['size'], 'issues': a['issues'], 'pack': a['pack'], 'format': a['format'], 'language': a['language'], 'seeders': a['seeders'], 'leechers': a['leechers'], 'scanner': a['scanner'], 'chkit': {'id': x['id'], 'series': x['series']}, 'pubdate': datetime.datetime.fromtimestamp(float(a['upload_time'])).strftime('%a, %d %b %Y %H:%M:%S'), 'int_pubdate': float(a['upload_time'])}) else: logger.fdebug('32P did not return any valid search results.') if len(results32p) > 0: resultlist['entries'] = sorted(results32p, key=itemgetter('pack','title'), reverse=False) logger.debug('%s Resultslist: %s' % (self.module, resultlist)) else: resultlist = 'no results' return resultlist