import mongodb_client import zillow_api_client import zillow_web_scraper_client from cloudAMQP_client import CloudAMQPClient # Automatically feed zpids into queue ### REPLACE CLOUD_AMQP_URL WITH YOUR OWN ### CLOUD_AMQP_URL = '''amqp://*****:*****@hyena.rmq.cloudamqp.com/htwgtamk''' DATA_FETCHER_QUEUE_NAME = 'dataFetcherTaskQueue' ZIPCODE_FILE = 'bay_area_zipcode_list.txt' WAITING_TIME = 3 cloudAMQP_client = CloudAMQPClient(CLOUD_AMQP_URL, DATA_FETCHER_QUEUE_NAME) zipcode_list = [] with open(ZIPCODE_FILE, 'r') as zipcode_file: for zipcode in zipcode_file: zipcode_list.append(str(zipcode)) for zipcode in zipcode_list: zpids = zillow_web_scraper_client.search_zillow_by_zip(zipcode) time.sleep(WAITING_TIME) for zpid in zpids: cloudAMQP_client.sendDataFetcherTask({'zpid': zpid})
from cloudAMQP_client import CloudAMQPClient CLOUDAMQP_URL = 'amqp://*****:*****@hyena.rmq.cloudamqp.com/jnqrsjwd' QUEUE_NAME = 'dataFetcherTaskQueue' # Initialize a client client = CloudAMQPClient(CLOUDAMQP_URL, QUEUE_NAME) # Send a message client.sendDataFetcherTask({'name': 'test message'}) # Receive a message # client.getDataFetcherTask()
from cloudAMQP_client import CloudAMQPClient # REPLACE URL WITH YOUR OWN CLOUDAMQP_URL = 'amqp://*****:*****@rhino.rmq.cloudamqp.com/aznxmzpt' QUEUE_NAME = 'dataFetcherTaskQueue' # Initialize a client client = CloudAMQPClient(CLOUDAMQP_URL, QUEUE_NAME) # Send a message client.sendDataFetcherTask({'zpid' : '83154148'}) # Receive a message #client.getDataFetcherTask()
import zillow_web_scraper_client from cloudAMQP_client import CloudAMQPClient # RabbitMQ config CLOUD_AMQP_URL = '''amqp://*****:*****@hyena.rmq.cloudamqp.com/kdflangt''' DATA_FETCHER_QUEUE_NAME = 'dataFetcherTaskQueue' ZIPCODE_FILE = 'san_diego_zipcode_list.txt' SHUFFLE_ZIPCODES = True WAITING_TIME = 3 cloudAMQP_client = CloudAMQPClient(CLOUD_AMQP_URL, DATA_FETCHER_QUEUE_NAME) zipcode_list = [] with open(ZIPCODE_FILE, 'r') as zipcode_file: for zipcode in zipcode_file: zipcode_list.append(str(zipcode)) if SHUFFLE_ZIPCODES: print "shuffle zipcodes!" random.shuffle(zipcode_list) for zipcode in zipcode_list: zpids = zillow_web_scraper_client.get_zpid_by_zipcode(zipcode) time.sleep(WAITING_TIME) for zpid in zpids: cloudAMQP_client.sendDataFetcherTask({"zpid": zpid})