Ejemplo n.º 1
0
def main():
	ts = time()
	download_dir = setup_download_dir()
	links = [l for l in get_links() if l.endswith('.csv')]
	for link in links:
		download_link(download_dir, link)
	print('Took {}s'.format(time() - ts))
Ejemplo n.º 2
0
 def run(self):
     while True:
         dir, link = self.queue.get()
         try:
             download_link(dir, link)
         finally:
             self.queue.task_done()
Ejemplo n.º 3
0
 def run(self):
     while True:
         # Get work from the queue and expand the tuple
         directory, link = self.queue.get()
         try:
             download_link(directory, link)
         finally:
             self.queue.task_done()
Ejemplo n.º 4
0
def main():
    print("In Main")
    ts = time()
    download_dir = setup_download_dir()
    links = [l for l in get_links('c53645e1e12ad62') if l.endswith('.jpg')]
    for link in links:
        download_link(download_dir, link)
    print('Took {}s'.format(time() -ts))
Ejemplo n.º 5
0
 def run(self):
     while True:
         # Get the work from the queue and expand the tuple
         item = self.queue.get()
         if item is None:
             break
         directory, link = item
         download_link(directory, link)
         self.queue.task_done()
Ejemplo n.º 6
0
def main():
    ts = time()
    client_id = os.getenv('IMGUR_CLIENT_ID')
    if not client_id:
        raise Exception("Couldn't find IMGUR_CLIENT_ID environment variable!")
    download_dir = setup_download_dir()
    links = [l for l in get_links(client_id) if l.endswith('.jpg')]
    for link in links:
        download_link(download_dir, link)
    print('Took {}s'.format(time() - ts))
Ejemplo n.º 7
0
def main():
    ts = time()
    client_id = os.getenv('IMGUR_CLIENT_ID')
    if not client_id:
        raise Exception('Couldn\'t find IMGUR_CLIENT_ID environment variable!')
    download_dir = setup_download_dir()
    links = [l for l in get_links(client_id) if l.endswith('.jpg')]
    for link in links:
        download_link(download_dir, link)
    print('Took {}s'.format(time() - ts))
Ejemplo n.º 8
0
def main():
    ts = time()
    client_id = os.getenv('IMGUR_CLIENT_ID')
    if not client_id:
        raise Exception("Couldn't find IMGUR_CLIENT_ID environment variable!")
    download_dir = setup_download_dir()
    links = get_links(client_id)
    for link in links:
        download_link(download_dir, link)
    logging.info('Took %s seconds', time() - ts)
Ejemplo n.º 9
0
def main():
    ts = time()
    client_id = os.getenv('IMGUR_CLIENT_ID')
    if not client_id:
        raise Exception("Couldn't find IMGUR_CLIENT_ID environment variable!")
    download_dir = setup_download_dir()
    links = (l for l in get_links(client_id) if l.endswith('.jpg'))
    for link in links:
        download_link(download_dir, link)
    logging.info('Took %s seconds', time() - ts)
Ejemplo n.º 10
0
def main():
    ts = time()
    client_id = 'bef2d9292d6bcbd'
    if not client_id:
        raise Exception("Couldn't find IMGUR_CLIENT_ID environment variable!")
    download_dir = setup_download_dir()
    links = get_links(client_id)
    for link in links:
        download_link(download_dir, link)
    logging.info('Took %s seconds', time() - ts)
Ejemplo n.º 11
0
def main():
    ts = time()
    download_dir = setup_download_dir()
    links = ['http://img3.6comic.com:99/2/103/861/001_mdh.jpg',
            'http://img3.6comic.com:99/2/103/861/002_9uj.jpg',
            'http://img3.6comic.com:99/2/103/861/003_c8x.jpg',
            'http://img3.6comic.com:99/2/103/861/004_y3b.jpg',
            'http://img3.6comic.com:99/2/103/861/005_hu6.jpg']
    for link in links:
        download_link(download_dir, link)
    print('Took {}s'.format(time() - ts))
Ejemplo n.º 12
0
def main():
    ts = time()
    try:
        client_id = key.client_id
    except ImportError:
        logger.error('Cannot Import client_id')
    download_dir = setup_download_dir()
    links = get_links(client_id)
    for link in links:
        download_link(download_dir, link)
    logger.info(f"Took {time() - ts} seconds")
Ejemplo n.º 13
0
def main():
   ts = time()
   hello()
   client_id = os.getenv('IMGUR_CLIENT_ID')
   if not client_id:
       raise Exception("Couldn't find IMGUR_CLIENT_ID environment variable!")
   download_dir = setup_download_dir()
   links = [l for l in get_links(client_id) if l.endswith('.jpg')]
   print(download_dir)
   testurl = u'http://i.imgur.com/i5QjTPA.jpg'
   download_link(download_dir, testurl)
   for link in links:
       download_link(download_dir, link)
   print('Took {}s'.format(time() - ts))
Ejemplo n.º 14
0
def main():
    ts = time()
    # To set the environment variable in the terminal:
    # export IMGUR_CLIENT_ID='my-client-id'
    client_id = os.getenv('IMGUR_CLIENT_ID')

    if not client_id:
        raise Exception("Couldn't find IMGUR_CLIENT_ID environment variable!")

    if not os.path.isdir('images'):
        os.makedirs('images')

    links = get_links(client_id)
    for link in links:
        download_link(Path('images'), link)

    logging.info('Took %s seconds', time() - ts)
Ejemplo n.º 15
0
def main():
    ts = time()

    url1 = 'http://www.toutiao.com/a6333981316853907714'
    url2 = 'http://www.toutiao.com/a6334459308533350658'
    url3 = 'http://www.toutiao.com/a6313664289211924737'
    url4 = 'http://www.toutiao.com/a6334337170774458625'
    url5 = 'http://www.toutiao.com/a6334486705982996738'
    download_dir = setup_download_dir('single_imgs')
    links = list(
        chain(
            get_links(url1),
            get_links(url2),
            get_links(url3),
            get_links(url4),
            get_links(url5),
        ))
    for link in links:
        download_link(download_dir, link)
    print('一共下载了 {} 张图片'.format(len(links)))
    print('Took {}s'.format(time() - ts))
Ejemplo n.º 16
0
    def run(self):
        while True:
            # Get the work from the queue and expand the tuple
            link, s, state = self.queue.get()
            try:
                brewdict, beerslist = download_link(link,s)
                # logger.info(beerslist[0])
                try:
                    brewid = db.add_brewery(brewdict, state, self.cur)
                except Exception as e:
                    self.conn.rollback()
                    continue
                    raise e
                else:
                    self.conn.commit()

                logger.info(brewid)
                # if brewid == 0:
                #      continue

                for beer in beerslist:
                    try:
                        db.add_beer(beer, state, brewid, self.cur)
                    except Exception as e:
                        logger.info('beeer: %s',e)
                        self.conn.rollback()
                        raise e
                    else:
                        self.conn.commit()

            except Exception as e:
                logger.info('breeew: %s',e)
                # self.conn.rollback()
            finally:
                # self.conn.commit()
                self.queue.task_done()
        self.cur.close()
        self.conn.close()
Ejemplo n.º 17
0
 def run(self):
     while True:
         directory, link = self.queue.get() # get a task from the queue
         download_link(directory, link)     # execute the task
         self.queue.task_done()             # signal completion
Ejemplo n.º 18
0
import logging
import os
from time import time

from download import setup_download_dir, get_links, download_link

logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s|%(levelname)s|%(threadName)s|%(message)s')
log = logging.getLogger(__name__)

if __name__ == '__main__':
    ts = time()

    client_id = os.getenv('IMGUR_CLIENT_ID')
    if not client_id:
        raise Exception("Couldn't find IMGUR_CLIENT_ID env variable")

    download_dir = setup_download_dir()
    links = get_links(client_id)
    for link in links:
        download_link(download_dir, link)

    log.info('Took %s seconds', time() - ts)
Ejemplo n.º 19
0
 def run(self):
     while True:
         # Get the work from the queue and expand the tuple
         directory, link = self.queue.get()
         download_link(directory, link)
         self.queue.task_done()