def update_network(accounts, depth):
for account_id in accounts.keys():
print '\tupdating user:'******'\t\tupdating', acc_type, 'account:', acc_username
if acc_username == '': continue
if acc_type == DELICIOUS:
crawl = DeliciousCrawler.factory(acc_username, depth=depth)
elif acc_type == FLICKR:
crawl = FlickrCrawler.factory(acc_username, depth=depth)
elif acc_type == SLIDESHARE:
crawl = SlideShareCrawler.factory(acc_username, depth=depth)
elif acc_type == TWITTER:
crawl = TwitterCrawler.factory(acc_username, depth=depth)
elif acc_type == YOUTUBE:
crawl = YouTubeCrawler.factory(acc_username, depth=depth)
personUris.append(crawl.getUserUri(acc_username))
crawlers.append(crawl)
# create crawlers using user network accounts data
t1 = time.clock()
CrawlNetworks(crawlers).crawl(start_time=sdate)
t2 = time.clock()
# link resources to each other
Resource.unify_all(personUris)
print 'Finished in %d seconds' % (t2 - t1)
cthreads = [CommunityCrawlerThread(c, start_time=start_time)
for c in self.crawlers]
# run all threads
for cth in cthreads:
cth.start()
# wait for all threads to finish
for cth in cthreads:
cth.join()
if __name__ == '__main__':
# create crawlers using user network accounts data
# Delicious crawler
from sna.crawler.deliciouscrawler import DeliciousCrawler
delicrawl = DeliciousCrawler()
delicrawl.setStartUserId('anamaria0509')
delicrawl.setMaxLevel(2)
# Flickr crawler
from sna.crawler.flickrcrawler import FlickrCrawler
params = {'api_key': 'ac91a445a4223af2ceafb06ae50f9a25'}
fcrawl = FlickrCrawler(params)
fcrawl.setStartUserId('anamaria stoica')
fcrawl.setMaxLevel(2)
# YouTube crawler
from sna.crawler.youtubecrawler import YouTubeCrawler
ytcrawl = YouTubeCrawler()
ytcrawl.setStartUserId('anamaria0509')
ytcrawl.setMaxLevel(2)
