def loadConfigs(pt, model): if DEBUG_Mario: print("\n--- load configs ---") print("in pt:", pt) print("in model:", model) dcdic = dcfgs.dcdic print("\nCurrent pruning configuration (loaded from lib/cfgs.c.dcdic)") for key, value in dcdic.items(): print(key, value) changeConfigs = checkOperation( question='\nModify the prunig configuration?', default='no') if changeConfigs: net = Net(pt, model=model, phase=caffe.TEST) convs = net.convs for conv in convs: print( 'input number remaining filters of %s after pruning [original number of filters %d ]' % (conv, net.conv_param_num_output(conv))) dc = input("[leave blank to skip]: ") if dc == '': dcdic[conv] = dcdic[conv] else: dcdic[conv] = int(dc) return dcdic
def test_03_model(self): n = Net(Net.ALL, training_epochs=25, epochs_per_save=5, epochs_per_archve=10, save_m=True, gen_steps=60, kwargs={'hidden_size': 32}) # Should not load - no files self.assertFalse(n.load()) # Should train and gen output n.train() y = n.gen() filepath = os.path.join(test_output_dir, 'test1.wav') utils.write_output(filepath, y) # Build new model and load weights m = Net(Net.ALL, build=False, gen_steps=60, kwargs={'hidden_size': 32}) self.assertTrue(m.load()) y = m.gen() filepath = os.path.join(test_output_dir, 'test2.wav') utils.write_output(filepath, y) # Let us confirm our files were saved files = filter(lambda e: e[0] != '.', os.listdir(test_output_dir)) self.assertEqual(len(files), 2)
def main(files=files): """ Generates music and saves to filenames set in command line args, or in config.output.default_file. Runs automatically from command line. Arguments: list:files -- List of files, taken from args when run on command line """ # Build net print('Loading net...') net = Net(Net.GEN) print('Loading model...') net.load() # Generate for filepath in filepaths: y = net.gen() print('Finished generating "{0}."'.format(filepath)) sclog('Generated "{0}."'.format(filepath)) utils.write_output(filepath, y) if (cf.output.save_raw): utils.save_array(os.path.splitext(filepath)[0], y)
def __init__(self): self.net = Net() url = str(sys.argv[1]) host = self.get_host_and_id(url)[0] media_id = self.get_host_and_id(url)[1] return self.get_media_url(host, media_id)
class AllmyvideosResolver(object): name = "allmyvideos" domains = ["allmyvideos.net"] pattern = '(?://|\.)(allmyvideos\.net)/(?:embed-)?([0-9a-zA-Z]+)' def __init__(self): self.net = Net() url = str(sys.argv[1]) host = self.get_host_and_id(url)[0] media_id = self.get_host_and_id(url)[1] return self.get_media_url(host, media_id) def get_host_and_id(self, url): r = re.search(self.pattern, url, re.I) if r: return r.groups() else: return False def get_media_url(self, host, media_id): url = self.get_url1st(host, media_id) headers = {'User-Agent': common.IE_USER_AGENT, 'Referer': url} html = self.net.http_GET(url, headers=headers).content stream_url = self.__get_best_source(html) if stream_url: # xbmc.sleep(2000) print stream_url url = self.get_url(host, media_id) headers = {'User-Agent': common.IE_USER_AGENT, 'Referer': url} html = self.net.http_GET(url, headers=headers).content data = helpers.get_hidden(html) html = self.net.http_POST(url, data, headers=headers).content stream_url = self.__get_best_source(html) if stream_url: # xbmc.sleep(2000) print stream_url def __get_best_source(self, html): r = re.search('"sources"\s*:\s*(\[.*?\])', html, re.DOTALL) if r: sources = json.loads(r.group(1)) max_label = 0 stream_url = '' for source in sources: if 'label' in source and int(re.sub('[^0-9]', '', source['label'])) > max_label: stream_url = source['file'] max_label = int(re.sub('[^0-9]', '', source['label'])) if stream_url: stream_url = '%s?%s&direct=false&ua=false' % (stream_url.split('?')[0], urlparse.urlparse(stream_url).query) return stream_url + '|' + urllib.urlencode({'User-Agent': common.IE_USER_AGENT}) def get_url(self, host, media_id): return 'http://allmyvideos.net/%s' % media_id def get_url1st(self, host, media_id): return 'http://allmyvideos.net/embed-%s.html' % media_id
class Resolver(object): name = "novamov" domains = ['novamov.com', 'auroravid.to'] pattern = '(?://|\.)(novamov.com|auroravid.to)/(?:video/|embed/\?v=|embed\.php\?v=)([A-Za-z0-9]+)' def __init__(self): self.net = Net() url = str(sys.argv[1]) host = self.get_host_and_id(url)[0] media_id = self.get_host_and_id(url)[1] return self.get_media_url(host, media_id) def get_host_and_id(self, url): r = re.search(self.pattern, url, re.I) if r: return r.groups() else: return False def get_media_url(self, host, media_id): web_url = self.get_url(host, media_id) headers = {'User-Agent': common.FF_USER_AGENT} stream_url = '' html = self.net.http_GET(web_url, headers=headers).content try: r = re.search('flashvars.filekey=(.+?);', html) if r == None: raise Exception() r = r.group(1) try: filekey = re.compile('\s+%s="(.+?)"' % r).findall(html)[-1] except: filekey = r player_url = 'http://www.auroravid.to/api/player.api.php?key=%s&file=%s' % ( filekey, media_id) html = self.net.http_GET(player_url).content r = re.search('url=(.+?)&', html) except: r = re.search('source src="(.+?)"', html) if r: stream_url = r.group(1) headers.update({ 'Referer': web_url, }) print stream_url + helpers.append_headers(headers) else: print 'File Not Found or removed' def get_url(self, host, media_id): return 'http://www.auroravid.to/embed/?v=%s' % media_id
def __init__(self): self.net = Net() self.headers = {'User-Agent': common.ANDROID_USER_AGENT} url = str(sys.argv[1]) host = self.get_host_and_id(url)[0] media_id = self.get_host_and_id(url)[1] return self.get_media_url(host, media_id)
def __init__(self): self.net = Net(cookie_file='/mnt/network/cookies', cloudflare=True) self.headers = {'User-Agent': common.ANDROID_USER_AGENT} url = str(sys.argv[1]) host = self.get_host_and_id(url)[0] media_id = self.get_host_and_id(url)[1] return self.get_media_url(host, media_id)
def __init__(self): # self.net = Net() # self.net = Net(cookie_file='/mnt/network/cookies', http_debug = True) self.net = Net(cookie_file='/mnt/network/cookies', http_debug=False) url = str(sys.argv[1]) host = self.get_host_and_id(url)[0] media_id = self.get_host_and_id(url)[1] return self.get_media_url(host, media_id)
class MovshareResolver(object): name = "movshare" domains = ["movshare.net", 'wholecloud.net', 'vidgg.to'] pattern = '(?://|\.)(movshare.net|wholecloud.net|vidgg.to)/(?:video/|embed(?:/|\.php)\?(?:v|id)=)([A-Za-z0-9]+)' def __init__(self): self.net = Net() url = str(sys.argv[1]) host = self.get_host_and_id(url)[0] media_id = self.get_host_and_id(url)[1] return self.get_media_url(host, media_id) def get_host_and_id(self, url): r = re.search(self.pattern, url, re.I) if r: return r.groups() else: return False def get_media_url(self, host, media_id): web_url = self.get_url(host, media_id) headers = {'User-Agent': common.FF_USER_AGENT} html = self.net.http_GET(web_url, headers=headers).content stream_url = '' match = re.search('<video.*?</video>', html, re.DOTALL) if match: links = re.findall('<source[^>]+src="([^"]+)', match.group(0), re.DOTALL) if links: stream_url = random.choice(links) if not stream_url: match = re.search('fkzd="([^"]+)', html) if match: query = {'pass': '******', 'key': match.group(1), 'cid3': 'undefined', 'cid': 0, 'numOfErrors': 0, 'file': media_id, 'cid2': 'undefined', 'user': '******'} api_url = 'http://www.wholecloud.net//api/player.api.php?' + urllib.urlencode(query) html = self.net.http_GET(api_url, headers=headers).content match = re.search('url=([^&]+)', html) if match: stream_url = match.group(1) if stream_url: headers.update({'Referer': web_url, }) print stream_url + helpers.append_headers(headers) # else: # raise ResolverError('File Not Found or removed') def get_url(self, host, media_id): if 'vidgg' in host: return 'http://%s/embed/?id=%s' % (host, media_id) # template = 'http://{host}/embed/?id={media_id}' else: return 'http://%s/embed/?v=%s' % (host, media_id) # template = 'http://{host}/embed/?v={media_id}' print self._default_get_url(host, media_id, template)
def flop(model, weights, orig=15346630656): setting = getattr(cfgs, args.setting) if model == '': model = setting.model if weights == '': weights = setting.weights orig = setting.flop print('orig', orig) net = Net(model, model=weights, noTF=1) after = net.computation() print(after * 100 / orig)
class VideoweedResolver(object): name = 'videoweed.es' domains = ['bitvid.sx', 'videoweed.es', 'videoweed.com'] pattern = '(?://|\.)(bitvid.sx|videoweed.es|videoweed.com)/(?:mobile/video\.php\?id=|video/|embed/\?v=|embed\.php\?v=|file/)([0-9a-z]+)' def __init__(self): self.net = Net() url = str(sys.argv[1]) host = self.get_host_and_id(url)[0] media_id = self.get_host_and_id(url)[1] return self.get_media_url(host, media_id) def get_host_and_id(self, url): r = re.search(self.pattern, url, re.I) if r: return r.groups() else: return False def get_media_url(self, host, media_id): web_url = self.get_url(host, media_id) html = self.net.http_GET(web_url).content stream_url = '' r = re.search('flashvars.filekey=(.+?);', html) if r: r = r.group(1) try: filekey = re.compile('\s+%s="(.+?)"' % r).findall(html)[-1] except: filekey = r player_url = 'http://www.bitvid.sx/api/player.api.php?key=%s&file=%s' % ( filekey, media_id) html = self.net.http_GET(player_url).content r = re.search('url=(.+?)&', html) if r: stream_url = r.group(1) # else: # raise ResolverError('File Not Found or removed') print stream_url def get_url(self, host, media_id): return 'http://www.bitvid.sx/embed/?v=%s' % media_id
def step0(pt, model): net = Net(pt, model=model, noTF=1) # lib/net.Net instantiate the NetBuilder -by Mario print("\tthe big model is initialized.") WPQ, pt, model = net.preprocess_resnet( ) # WPQ stores pruned values, which will be later saved to the caffemodel -by Mario redprint( "\tstep0 end! and ...(this step mainly change the pt:split the inpalce relu)" ) print("\tWPQ : ", WPQ) print("\tpt : ", pt) print("\tmodel : ", model) return {"WPQ": WPQ, "pt": pt, "model": model}
class CloudflareResolver(object): def __init__(self): # self.net = Net(cookie_file='/mnt/network/cookies', http_debug=True, cloudflare=True) self.net = Net(cookie_file='/mnt/network/cookies', cloudflare=True) url = str(sys.argv[1]) return self.get_answer_code(url) def get_answer_code(self, web_url): headers = {'User-Agent': common.FF_USER_AGENT} html = self.net.http_GET(web_url, headers=headers).content #html = self.request(web_url, cookie_file=/mnt/network/cookies, cloudflare=True) ret = self.net.save_cookies('/mnt/network/cookies') print "html", html.encode('utf8')
def step1(pt, model, WPQ, check_exist=False): print(pt) net = Net(pt, model, noTF=1) model = net.finalmodel(WPQ) if 1: convs = net.convs else: convs = net.convs[:-1] redprint("ignoring last conv!") if dcfgs.dic.option == 1: sums = net.type2names('Eltwise')[:-1] newsums = [] for i in sums: if not i.endswith('block8_sum'): newsums.append(i) newconvs = [] for i in convs: if i.endswith('_proj'): newconvs.insert(0,i) else: newconvs.append(i) convs = newsums + newconvs else: convs += net.type2names('Eltwise')[:-1] if dcfgs.dic.fitfc: convs += net.type2names('InnerProduct') if dcfgs.model in [cfgs.Models.xception,cfgs.Models.resnet]: for i in net.bns: if 'branch1' in i: convs += [i] net.freeze_images(check_exist=check_exist, convs=convs) return {"model":model}
def step1(pt, model, WPQ, check_exist=False): print(pt) net = Net(pt, model, noTF=1) model = net.finalmodel(WPQ) # loads weights into the caffemodel - by Mario if 1:#TODO: Consider adding a configuration paramter to cfgs.py in order to control whether or not to prune the last conv layer -by Mario convs = net.convs redprint("including last conv layer!") else: convs = net.convs[:-1] redprint("ignoring last conv layer!") if dcfgs.dic.option == 1: if DEBUG: redprint("This line executed because dcfgs.dic.option is set to 1 [train.step1()]") sums = net.type2names('Eltwise')[:-1] newsums = [] for i in sums: if not i.endswith('block8_sum'): newsums.append(i) newconvs = [] for i in convs: if i.endswith('_proj'): newconvs.insert(0,i) else: newconvs.append(i) convs = newsums + newconvs else: convs += net.type2names('Eltwise')[:-1] # I guess Element-wise operations are included in ResNet or Xception -by Mario if dcfgs.dic.fitfc: convs += net.type2names('InnerProduct') if dcfgs.model in [cfgs.Models.xception,cfgs.Models.resnet]: for i in net.bns: if 'branch1' in i: convs += [i] net.freeze_images(check_exist=check_exist, convs=convs) return {"model":model}
def __init__(self): # print "1111111111111" self.net = Net() # print "2222222222222" url = str(sys.argv[1]) # print "3333333333333" host = self.get_host_and_id(url)[0] # print "4444444444444" media_id = self.get_host_and_id(url)[1] # print "5555555555555" return self.get_media_url(host, media_id)
class BriskfileResolver(object): name = "briskfile" domains = ["briskfile.com"] pattern = '(?://|\.)(briskfile\.com)/(?:l|e)/([0-9A-Za-z\-]+)' def __init__(self): self.net = Net() url = str(sys.argv[1]) host = self.get_host_and_id(url)[0] media_id = self.get_host_and_id(url)[1] return self.get_media_url(host, media_id) def get_host_and_id(self, url): r = re.search(self.pattern, url, re.I) if r: return r.groups() else: return False def get_media_url(self, host, media_id): web_url = self.get_url(host, media_id) headers = {'User-Agent': common.FF_USER_AGENT} html = self.net.http_GET(web_url, headers=headers).content match = re.search('''val\(\)\s*\+\s*['"]([^"']+)''', html) suffix = match.group(1) if match else '' data = helpers.get_hidden(html) for name in data: data[name] = data[name] + suffix headers['Referer'] = web_url html = self.net.http_POST(web_url, form_data=data, headers=headers).content html = re.compile( r'clip\s*:\s*\{.*?(?:url|src)\s*:\s*[\"\'](.+?)[\"\']', re.DOTALL).search(html) if not html: print 'File Not Found or removed' stream_url = html.group(1) req = urllib2.Request(stream_url) for key in headers: req.add_header(key, headers[key]) stream_url = urllib2.urlopen(req).geturl() print stream_url + '|User-Agent=%s&Referer=%s' % (common.FF_USER_AGENT, web_url) def get_url(self, host, media_id): return 'http://www.briskfile.com/l/%s' % (media_id)
class VodlockerResolver(object): name = "vodlocker.com" domains = ["vodlocker.com"] pattern = '(?://|\.)(vodlocker\.com)/(?:embed-)?([0-9a-zA-Z]+)' def __init__(self): self.net = Net() url = str(sys.argv[1]) host = self.get_host_and_id(url)[0] media_id = self.get_host_and_id(url)[1] return self.get_media_url(host, media_id) def get_host_and_id(self, url): r = re.search(self.pattern, url, re.I) if r: return r.groups() else: return False def get_media_url(self, host, media_id): web_url = self.get_url(host, media_id) link = self.net.http_GET(web_url).content if 'FILE WAS DELETED' in link: print 'File deleted.' else: video_link = str(re.compile('file[: ]*"(.+?)"').findall(link)[0]) if len(video_link) > 0: print video_link else: print 'No playable video found.' def get_url(self, host, media_id): return 'http://vodlocker.com/embed-%s-640x400.html' % media_id
def stepend(new_pt, model, WPQ): net = Net(new_pt, model=model) net.WPQ = WPQ net.finalmodel(save=False) # load weights into the caffemodel -by Mario net.dis_memory() #final = net.finalmodel(WPQ, prefix='3r') new_pt, new_model = net.save(prefix='3c') print('caffe test -model',new_pt, '-weights',new_model) return {"final": None}
class NosvideoResolver(object): name = "nosvideo" domains = ["nosvideo.com", "noslocker.com"] pattern = '(?://|\.)(nosvideo.com|noslocker.com)/(?:\?v\=|embed/|.+?\u=)?([0-9a-zA-Z]+)' def __init__(self): self.net = Net() url = str(sys.argv[1]) host = self.get_host_and_id(url)[0] media_id = self.get_host_and_id(url)[1] return self.get_media_url(host, media_id) def get_host_and_id(self, url): r = re.search(self.pattern, url, re.I) if r: return r.groups() else: return False def get_media_url(self, host, media_id): web_url = self.get_url(host, media_id) html = self.net.http_GET(web_url).content if 'File Not Found' in html: raise ResolverError('File Not Found') web_url = 'http://nosvideo.com/vj/video.php?u=%s&w=&h=530' % media_id html = self.net.http_GET(web_url).content smil_url = re.compile('\':\'(.+?)\'').findall(html) smil_url = [i for i in smil_url if '.smil' in i][0] html = self.net.http_GET(smil_url).content streamer = re.findall('base\s*=\s*"(.+?)"', html)[0] playpath = re.findall('src\s*=\s*"(.+?)"', html)[0] stream_url = '%s playpath=%s' % (streamer, playpath) print stream_url def get_url(self, host, media_id): return 'http://nosvideo.com/%s' % media_id
def _config_network(self, conf): from lib.net import Net if CONFIG_HOSTNAME not in conf: conf[CONFIG_HOSTNAME] = self._name + '.local' conf = self._get_config_key(self._config, CONFIG_NET) nets = self._get_config_key(conf, CONFIG_WIFI, suppress=True) self._comps[KEY_NET] = Net(self._tidx, self._proxy, nets, **conf) self._tidx += 1
def test_darknet(): num_classes = 21 net3 = Net(model_def, model_weights, num_classes, deploy=True) image_file = 'dog2.jpg' input_shape = net3._net.blobs['data'].data.shape transformered_im = preprocess(image_file, input_shape) output = net3._net.forward(data=np.asarray([transformered_im])) # all_start_spatial_position = [(4, 7), (7, 7)] # all_anchor_idx = [1, 3] # all_class_idx = [14, 12] all_start_spatial_position = [(7, 7)] all_anchor_idx = [3] all_class_idx = [11] # print net3.get_receptive_field('conv4') ############################ # ###### test the first ##### ############################ start_layer_name = 'conv9' idx = 0 kernel_idx = all_anchor_idx[idx] * 25 + 5 + all_class_idx[idx] start_spatial_position = all_start_spatial_position[idx] start_kernel_idx_weight_pairs = ([kernel_idx], [1]) start_part_object_scores = Part_Object_Scores(start_layer_name, start_spatial_position, net3) start_part_object_scores.init(start_kernel_idx_weight_pairs) print start_part_object_scores.get_score_map() # stop() queue, bias_sum = top_down(start_part_object_scores, net3, threshold_policy='float', min_receptive_field=30, threshold=None) # sum of all scores should be equal to start score print(get_scores_sum(queue) + bias_sum) # TODO we should exchange the order of two process # and we should note that for different task we have concentrate on different points # import pdb # pdb.set_trace() filtered_queue = threshold_system(queue, input_shape=input_shape, reserve_num=300, reserve_ratio=0.1, display=True) # filtered_queue = filter_repeat_part_object_scores(queue, input_shape[-2:]) # print 'filtered_queue: {:d}'.format(len(filtered_queue)) # filtered_queue = filter_ratio(filtered_queue, 1, reserve_num=30) all_raw_spatial_position = get_all_raw_spatial_position(filtered_queue) # filtered_raw_spatial_position = filter_repeat( # all_raw_spatial_position, input_shape[-2:]) # print 'remain activated points: {:d}'.format(len(all_raw_spatial_position)) # visualization all points vis_square(net3._net.blobs['data'].data[net3.img_idx].transpose( 1, 2, 0)[np.newaxis, ...]) vis_activated_point(net3._net, all_raw_spatial_position, net3.img_idx, [1, 0, 0])
class YouWatchResolver(object): name = "youwatch" domains = ["youwatch.org", "chouhaa.info", "ay8ou8ohth.com"] pattern = '(?://|\.)(youwatch\.org|chouhaa\.info|ay8ou8ohth\.com)/(?:embed-)?([A-Za-z0-9]+)' def __init__(self): self.net = Net() url = str(sys.argv[1]) host = self.get_host_and_id(url)[0] media_id = self.get_host_and_id(url)[1] return self.get_media_url(host, media_id) def get_host_and_id(self, url): r = re.search(self.pattern, url, re.I) if r: return r.groups() else: return False def get_media_url(self, host, media_id): web_url = self.get_url(host, media_id) web_url = self.net.http_GET(web_url)._response.url headers = {'Referer': web_url} tries = 0 while tries < MAX_TRIES: html = self.net.http_GET(web_url).content html = html.replace('\n', '') r = re.search('<iframe\s+src\s*=\s*"([^"]+)', html) if r: web_url = r.group(1) else: break tries += 1 html = self.net.http_GET(web_url, headers=headers).content r = re.search('file\s*:\s*"([^"]+)', html) if r: print r.group(1) + '|' + urllib.urlencode({'Referer': web_url}) def get_url(self, host, media_id): return 'http://youwatch.org/embed-%s.html' % media_id
def test_all(): model_def = 'model/nin_max_big/train_val.prototxt' model_weights = 'model/nin_max_big/cifar10_nin_iter_120000.caffemodel' num_classes = 10 net3 = Net(model_def, model_weights, num_classes) net3.img_idx = 0 assert net3.get_label() == net3.get_prediction(), 'prediction is error' cls_idx = net3.get_label() # print net3._net.blobs['data'].data[net3.img_idx] # global config net3.display() input_shape = (32, 32) ############################### # ########## first level######### ############################### layer_name = 'cccp6' center = [] center.append(cls_idx) top_blob_name = net3._net.top_names[layer_name][0] center += list( get_max_xy(net3._net.blobs[top_blob_name].data[net3.img_idx, cls_idx])) spatial_position = center[1:] start_prior_score = 0 part_object_scores = Part_Object_Scores(layer_name, start_prior_score, spatial_position, net3) last_filtered_kernel_idx_weight_pairs = ([cls_idx], [1]) part_object_scores.init(last_filtered_kernel_idx_weight_pairs) print part_object_scores.get_score() # print part_object_scores.get_score() # start from part_object_scores queue = top_down(part_object_scores, net3, threshold_policy='float', min_receptive_field=0, threshold=0) # print queue[0].get_raw_spatial_position() print(get_scores_sum(queue)) filtered_queue = threshold_system(queue, input_shape=input_shape, reserve_num=200, reserve_ratio=1, display=True) all_raw_spatial_position = get_all_raw_spatial_position(filtered_queue) # visualization all points vis_square(net3._net.blobs['data'].data[net3.img_idx].transpose( 1, 2, 0)[np.newaxis, ...]) vis_activated_point(net3._net, all_raw_spatial_position, net3.img_idx, 0.5)
def saveModel(new_pt, model, WPQ): """ This functions writes to the protobuf to generate the new .caffemodel """ if DEBUG_Mario: print("\n--- saveModel ---") print("in pt (new_pt): ", new_pt) print("in model: ", model) print("in WPQ: ", WPQ.keys()) net = Net(new_pt, model=model) net.WPQ = WPQ net.finalmodel(save=False) # model new_pt, new_model = net.save(prefix='pruned') if DEBUG_Mario: print("\nout pt (new_pt): ", new_pt) print("out model (new_model): ", new_model) print('\nFinal model ready. For testing you can use:') print('\t$CAFFE_ROOT/build/tools/caffe test -model', new_pt, '-weights', new_model) return new_pt, new_model
class StreamcloudResolver(object): name = "streamcloud" domains = ["streamcloud.eu"] pattern = '(?://|\.)(streamcloud\.eu)/([0-9a-zA-Z]+)' def __init__(self): self.net = Net() url = str(sys.argv[1]) host = self.get_host_and_id(url)[0] media_id = self.get_host_and_id(url)[1] return self.get_media_url(host, media_id) def get_host_and_id(self, url): r = re.search(self.pattern, url, re.I) if r: return r.groups() else: return False def get_media_url(self, host, media_id): web_url = self.get_url(host, media_id) resp = self.net.http_GET(web_url) html = resp.content post_url = resp.get_url() if re.search('>(File Not Found)<', html): print 'File Not Found or removed' form_values = {} for i in re.finditer('<input.*?name="(.*?)".*?value="(.*?)">', html): form_values[i.group(1)] = i.group(2).replace( "download1", "download2") html = self.net.http_POST(post_url, form_data=form_values).content r = re.search('file: "(.+?)",', html) if r: print r.group(1) def get_url(self, host, media_id): return 'http://streamcloud.eu/%s' % (media_id)
def main(): # We set autosave to true even if it is False in config net = Net(Net.TRAIN, autosave=True) # Simplified parameters for setting epochs for k in cf.flags.keys(): if utils.is_str_int(k): net.set(training_epochs=int(k)) break # Load weights unless new flag provided if not cf.flags.get('new'): net.load() print('Completed {0} epochs. Training next {1}...'.format( net.epochs, net.training_epochs)) net.train() sclog('Finished training {0} epochs.'.format(net.training_epochs))
class VidUpMeResolver(object): name = "vidup.me" domains = ["vidup.me", "beta.vidup.me"] pattern = '(?://|\.)(vidup\.me)/(?:embed-)?([0-9a-zA-Z]+)' def __init__(self): self.net = Net() url = str(sys.argv[1]) host = self.get_host_and_id(url)[0] media_id = self.get_host_and_id(url)[1] return self.get_media_url(host, media_id) def get_host_and_id(self, url): r = re.search(self.pattern, url, re.I) if r: return r.groups() else: return False def get_media_url(self, host, media_id): web_url = self.get_url(host, media_id) html = self.net.http_GET(web_url).content js_data = re.findall('(eval\(function.*?)</script>', html.replace('\n', '')) for i in js_data: try: html += jsunpack.unpack(i) except: pass match = re.findall('''["']?sources['"]?\s*:\s*\[(.*?)\]''', html) if match: stream_url = re.findall('''['"]?file['"]?\s*:\s*['"]?([^'"]+)''', match[0]) if stream_url: print stream_url[-1] def get_url(self, host, media_id): return 'http://beta.vidup.me/embed-%s.html' % media_id
def __init__(self): # print "1111111111111" self.net = Net() # print "2222222222222" url = str(sys.argv[1]) # print "3333333333333" # host = self.get_host_and_id(url)[0] # print "4444444444444" # media_id = self.get_host_and_id(url)[1] # print "5555555555555" # mod_request(request, urlPart) # mod_request("request", url) # curl -L -H "BS-Token:JwdWJsaWNfa2V5IjogIlVQUEthT2UxVm9zcU1iY0ZEUWVNZnN4MDI1QUEzQnVjIiwgImhtYWMiOiAiNDA4NzU2ZTMyNzhkZGUwZDVhZGQyMzMxMzNkZTYwM2RhZGEwN2I5MDU5MmM0MDM0OGQ2ZDQwYzdkYjNjZDI4ZSIsICJ0aW1lc3RhbXAiOiAxNTA0OTc4MjcyfQ==" https://www.bs.to/api/watch/2977318 return mod_request("request", url)