def main(): ts = time() download_dir = setup_download_dir() links = [l for l in get_links() if l.endswith('.csv')] for link in links: download_link(download_dir, link) print('Took {}s'.format(time() - ts))
def run(self): while True: dir, link = self.queue.get() try: download_link(dir, link) finally: self.queue.task_done()
def run(self): while True: # Get work from the queue and expand the tuple directory, link = self.queue.get() try: download_link(directory, link) finally: self.queue.task_done()
def main(): print("In Main") ts = time() download_dir = setup_download_dir() links = [l for l in get_links('c53645e1e12ad62') if l.endswith('.jpg')] for link in links: download_link(download_dir, link) print('Took {}s'.format(time() -ts))
def run(self): while True: # Get the work from the queue and expand the tuple item = self.queue.get() if item is None: break directory, link = item download_link(directory, link) self.queue.task_done()
def main(): ts = time() client_id = os.getenv('IMGUR_CLIENT_ID') if not client_id: raise Exception("Couldn't find IMGUR_CLIENT_ID environment variable!") download_dir = setup_download_dir() links = [l for l in get_links(client_id) if l.endswith('.jpg')] for link in links: download_link(download_dir, link) print('Took {}s'.format(time() - ts))
def main(): ts = time() client_id = os.getenv('IMGUR_CLIENT_ID') if not client_id: raise Exception('Couldn\'t find IMGUR_CLIENT_ID environment variable!') download_dir = setup_download_dir() links = [l for l in get_links(client_id) if l.endswith('.jpg')] for link in links: download_link(download_dir, link) print('Took {}s'.format(time() - ts))
def main(): ts = time() client_id = os.getenv('IMGUR_CLIENT_ID') if not client_id: raise Exception("Couldn't find IMGUR_CLIENT_ID environment variable!") download_dir = setup_download_dir() links = get_links(client_id) for link in links: download_link(download_dir, link) logging.info('Took %s seconds', time() - ts)
def main(): ts = time() client_id = os.getenv('IMGUR_CLIENT_ID') if not client_id: raise Exception("Couldn't find IMGUR_CLIENT_ID environment variable!") download_dir = setup_download_dir() links = (l for l in get_links(client_id) if l.endswith('.jpg')) for link in links: download_link(download_dir, link) logging.info('Took %s seconds', time() - ts)
def main(): ts = time() client_id = 'bef2d9292d6bcbd' if not client_id: raise Exception("Couldn't find IMGUR_CLIENT_ID environment variable!") download_dir = setup_download_dir() links = get_links(client_id) for link in links: download_link(download_dir, link) logging.info('Took %s seconds', time() - ts)
def main(): ts = time() download_dir = setup_download_dir() links = ['http://img3.6comic.com:99/2/103/861/001_mdh.jpg', 'http://img3.6comic.com:99/2/103/861/002_9uj.jpg', 'http://img3.6comic.com:99/2/103/861/003_c8x.jpg', 'http://img3.6comic.com:99/2/103/861/004_y3b.jpg', 'http://img3.6comic.com:99/2/103/861/005_hu6.jpg'] for link in links: download_link(download_dir, link) print('Took {}s'.format(time() - ts))
def main(): ts = time() try: client_id = key.client_id except ImportError: logger.error('Cannot Import client_id') download_dir = setup_download_dir() links = get_links(client_id) for link in links: download_link(download_dir, link) logger.info(f"Took {time() - ts} seconds")
def main(): ts = time() hello() client_id = os.getenv('IMGUR_CLIENT_ID') if not client_id: raise Exception("Couldn't find IMGUR_CLIENT_ID environment variable!") download_dir = setup_download_dir() links = [l for l in get_links(client_id) if l.endswith('.jpg')] print(download_dir) testurl = u'http://i.imgur.com/i5QjTPA.jpg' download_link(download_dir, testurl) for link in links: download_link(download_dir, link) print('Took {}s'.format(time() - ts))
def main(): ts = time() # To set the environment variable in the terminal: # export IMGUR_CLIENT_ID='my-client-id' client_id = os.getenv('IMGUR_CLIENT_ID') if not client_id: raise Exception("Couldn't find IMGUR_CLIENT_ID environment variable!") if not os.path.isdir('images'): os.makedirs('images') links = get_links(client_id) for link in links: download_link(Path('images'), link) logging.info('Took %s seconds', time() - ts)
def main(): ts = time() url1 = 'http://www.toutiao.com/a6333981316853907714' url2 = 'http://www.toutiao.com/a6334459308533350658' url3 = 'http://www.toutiao.com/a6313664289211924737' url4 = 'http://www.toutiao.com/a6334337170774458625' url5 = 'http://www.toutiao.com/a6334486705982996738' download_dir = setup_download_dir('single_imgs') links = list( chain( get_links(url1), get_links(url2), get_links(url3), get_links(url4), get_links(url5), )) for link in links: download_link(download_dir, link) print('一共下载了 {} 张图片'.format(len(links))) print('Took {}s'.format(time() - ts))
def run(self): while True: # Get the work from the queue and expand the tuple link, s, state = self.queue.get() try: brewdict, beerslist = download_link(link,s) # logger.info(beerslist[0]) try: brewid = db.add_brewery(brewdict, state, self.cur) except Exception as e: self.conn.rollback() continue raise e else: self.conn.commit() logger.info(brewid) # if brewid == 0: # continue for beer in beerslist: try: db.add_beer(beer, state, brewid, self.cur) except Exception as e: logger.info('beeer: %s',e) self.conn.rollback() raise e else: self.conn.commit() except Exception as e: logger.info('breeew: %s',e) # self.conn.rollback() finally: # self.conn.commit() self.queue.task_done() self.cur.close() self.conn.close()
def run(self): while True: directory, link = self.queue.get() # get a task from the queue download_link(directory, link) # execute the task self.queue.task_done() # signal completion
import logging import os from time import time from download import setup_download_dir, get_links, download_link logging.basicConfig( level=logging.INFO, format='%(asctime)s|%(levelname)s|%(threadName)s|%(message)s') log = logging.getLogger(__name__) if __name__ == '__main__': ts = time() client_id = os.getenv('IMGUR_CLIENT_ID') if not client_id: raise Exception("Couldn't find IMGUR_CLIENT_ID env variable") download_dir = setup_download_dir() links = get_links(client_id) for link in links: download_link(download_dir, link) log.info('Took %s seconds', time() - ts)
def run(self): while True: # Get the work from the queue and expand the tuple directory, link = self.queue.get() download_link(directory, link) self.queue.task_done()