def eye_fetch_1(): urls = [ 'https://www.iana.org/domains/reserved', 'https://cdn.pornpics.com/pics1/2018-03-29/512309_05big.jpg', 'https://cdn.pornpics.com/pics1/2018-03-29/512309_06big.jpg', 'https://cdn.pornpics.com/pics1/2018-03-29/512309_11big.jpg', 'https://cdn.pornpics.com/pics1/2018-03-29/512309_10big.jpg', 'https://cdn.pornpics.com/pics1/2018-03-29/512309_12big.jpg', 'https://cdn.pornpics.com/pics1/2018-03-29/512309_13big.jpg', 'https://cdn.pornpics.com/pics1/2017-01-12/393496_01big.jpg', 'https://www.pornpics.com/galleries/asian-solo-girl-saya-song-pulls-down-her-jean-shorts-on-her-way-to-posing-nude/', ] input = Fs('/home/boril/Desktop/Tests/input') output = Fs('/home/boril/Desktop/Tests/output') fetch_dir = str(output + 'fetched') archive_dir = str(output + 'archive') config = Config(fetch_dir=fetch_dir, archive_dir=archive_dir, chunk_size=4096) fetcher = Fetcher(config) for url in urls: resource = fetch(url, fetcher) print('resource:', str(resource)) if resource.is_text: print('resource.as_text:', resource.as_text) else: print('resource.is_text:', False)
def eye_Worldsex_01(): def source(): #yield 'https://www.worldsex.com/p**n-pics/small-titted-asian-femdom-ready-for-some-facesitting-8454/' #yield 'https://www.worldsex.com/p**n-pics/' #yield 'https://www.worldsex.com/p**n-pics/galleries/' #yield 'https://www.worldsex.com/p**n-pics/indian-amateur-shows-her-sweet-pussy-and-ass-8651/' yield 'bahaha! PRONZ IS THE BST TARGET DDE PHAHAA FAPPYNESS SLAPPED ME ALREADY FUKK REDDIT DUDE... LET REDDIT BURRN IN HIPSTER TRASH I HATE REDDIT ... I R EMO F****T .. . FUKK REDDIT LONG LIVE DIGG FUKK REDDIT LONG LIVE DIGG THE ORIGINAL GANGSTAs LONG LIVE DOWN WITH SYMBOLIK POLLUTTED MINDS! -= SIMPLODIK POLLUTION? DAMMAGE IS REAL | THE WORLD IS SYMBOLICALLY POLLUTED WITH; (#TRASH; SIMBOLIK); SOMEONE; HAS; TO PULL THE PLUGG GG ' input = Fs('/home/boril/Desktop/Tests/input') output = Fs('/home/boril/Desktop/Tests/output') fetch_dir = str(output + 'fetched') archive_dir = str(output + 'archive') config = Config(fetch_dir=fetch_dir, archive_dir=archive_dir, chunk_size=4096) fetcher = Fetcher(config) worldsex = Worldsex(fetcher) visited = [] collected_urls = [] forbidden_urls = [] def new_image(uri, client, result): print('new_image:', uri) def new_page(uri, client, result): for img in result['img']: #if img[0:2] == '//': # img = 'http:' + img #if not img in collected_urls: print('new_page[img]:', img) collected_urls.append(img) success = worldsex.begin(img, new_image) if not success: print('unsuccessful:', img) else: print('success:', img) for a in result['a']: #if worldsex.valid_page(a): #print('new_page[a]:', a) collected_urls.append(a) success = worldsex.begin(a, new_page) if not success: print('unsuccessful:', a) else: print('success:', img) for url in source(): worldsex.begin(url, new_page)
def eye_snapshot_1(): input = Fs('/home/boril/Desktop/Tests/input') output = Fs('/home/boril/Desktop/Tests/output') fetch_dir = str(output + 'fetched') archive_dir = str(output + 'archive') config = Config(fetch_dir=fetch_dir, archive_dir=archive_dir, chunk_size=4096) fetcher = Fetcher(config) resource = snapshot(uri='https://api.random.org/json-rpc/1/invoke', fetcher=fetcher) print('resource:', resource.as_text)
def snapshot(uri='http://random.org/', generate_section=date_path_string, fetcher=Fetcher): """ snapshot? Use this function to further; [snapshot]; a web resource into an Archival space on the disk. An already [fetched] resource will be moved into the archival directory under a generated name. Here the [dir] is to be treated as an :Archival ::Space. """ resource = fetch(uri, fetcher) generated_str = generate_section(resource) archive_path = Fs(fetcher.archive_dir) + generated_str + resource.local_dir if View(archive_path).exists(): Make(archive_path).remove() Make(archive_path).dirs() Make(resource.path).move(towards=archive_path) resource.path = Fs(archive_path) + resource.filename return resource
def fetch(uri='http://random.org/', fetcher=Fetcher): """ fetch? Use this function to :download and [fetch]; a web resource into a specified directory. Part of the [Interface] """ resource = download(uri, fetcher) #resource.path = os.path.join(str(fetcher.fetch_dir), str(resource.path)) Make(os.path.dirname(str(resource.path))).dirs() Make(resource.path).touch(exist_ok=True) if not Fs(str(resource.path)).is_dir: with open(str(resource.path), 'wb') as f: for chunk in resource.request.iter_content( chunk_size=fetcher.chunk_size): if chunk: # filter out keep-alive new chunks f.write(chunk) return resource
def eye_Daemon_01(): def initial_program_setup(): pass def do_main_program(): pass def program_cleanup(): pass def reload_program_config(): pass context_fetcher = { 'config': Config(fetch_dir=Fs('/home/boril/Desktop/Tests/output/fetch')), } context = daemon.DaemonContext( working_directory=context_fetcher['config']['fetch_dir'], umask=0o002, pidfile=lockfile.FileLock('/var/run/spam.pid'), ) context.signal_map = { signal.SIGTERM: program_cleanup, signal.SIGHUP: 'terminate', signal.SIGUSR1: reload_program_config, } def _step0(): mail_gid = grp.getgrnam('mail').gr_gid context.gid = mail_gid important_file = open('spam.data', 'w') interesting_file = open('eggs.data', 'w') context.files_preserve = [important_file, interesting_file] _step0() initial_program_setup() with context: do_main_program()
def download(Uri, fetcher, allow_redirects=True): """ download? Use this function to; [download]; a single web resource. The locality doesn't matter in this case, so we don't use a directory as :part of ::the {Interface}""" parsed_uri = urlparse(Uri) h = requests.head(Uri, allow_redirects=True) request = requests.get(Uri, stream=True, allow_redirects=allow_redirects) resource = Resource(request=request, head=h) # # Please use the New Identify class to mark a resource as Identified # (that is, all the details of the download should be held in that # object class and a resource instance should be marked by it # Further down, the fetch() and the snapshot() functions would use # :that instance {For Good} # resource.parsed_uri = parsed_uri resource.downloadable_file = is_downloadable(parsed_uri, h) # #: Re-Write this section, please. # Use Identify class to Aggregate the Locality Details into It. Thanx! # Let resource.identity Live in it. Do NOT choose Id or id for the :Name # Please, let it be a whole word please, it is IMPORTANT. # DO NOT Change the Name please, IT is IDENTITY of the LINK/URI/URL so the # Name is Chosen Carefully and After A Serious Work and So on... # Be Normal from time to time, please... What do We Want? An interface that speaks for Itself and Less MANNNINNGGGGGъепаГотенЕеМанингМаЩоСтанаТраверсБНалимумаматанезнамбрать. if resource.downloadable_file: resource.filename = Fs(os.path.basename(parsed_uri.path)) resource.local_dir = Fs( os.path.dirname(str(parsed_uri.netloc + parsed_uri.path))) resource.origin_dir = Fs(fetcher.fetch_dir) else: resource.filename = Fs('index.html') resource.local_dir = Fs(parsed_uri.netloc + parsed_uri.path) resource.origin_dir = Fs(fetcher.fetch_dir) resource.path = Fs( os.path.join(str(resource.origin_dir), str(resource.local_dir), str(resource.filename))) #print('resource:', resource) return resource
important_file = open('spam.data', 'w') interesting_file = open('eggs.data', 'w') context.files_preserve = [important_file, interesting_file] _step0() initial_program_setup() with context: do_main_program() ################################################################################ def source(): yield 'http://random.org/' input = Fs('/home/boril/Desktop/Tests/input') # Let the DAEMON occupy this directory output = Fs('/home/boril/Desktop/Tests/output') # Let the DAEMON occupy this output directory fetch_dir = str(output + 'fetched') archive_dir = str(output + 'archive') for path in View(input).listing(): content = [] with open(str(path), 'r') as f: content = [ line.strip() for line in f.readlines() if len(line.strip()) > 0 ] # #: Triok Zwer. s Tri Opashleta ahahahah kak da stane ya... Triabwa da se pulniat edno po edno da ima propagation na :eve? no. # #: Trqq za Vseki Zwer, po edin Vid Client.py (klasse), 4 threads per Zwer = 12 threads (4*3 == pochti 4*4 = 16; Zwer3 << Quatro) #
def source(): yield 'https://www.worldsex.com/p**n-pics/galleries/' for path in View(input).listing(): content = [] with open(str(path), 'r') as f: content = [ line.strip() for line in f.readlines() if len(line.strip()) > 0 ] #uris.extend(content) #for uri in uris: # yield uri input = Fs('/home/boril/Desktop/Tests/input') output = Fs('/home/boril/Desktop/Tests/output') fetch_dir = str(output + 'fetched') archive_dir = str(output + 'archive') q_pages = queue.Queue() q_files = queue.Queue() def error_callback(err): pass from worldsex_com import Worldsex config = Config(fetch_dir=fetch_dir, archive_dir=archive_dir, chunk_size=4096)