#! /usr/bin/env python

# command_line_scrape_and_extract_apps.py

import sys

reload(sys)

sys.path.append('/Users/herdrick/appbackr/appbackr_data_science/')

sys.setdefaultencoding('utf-8')

if __name__ == "__main__":

import sys

arg_pos=0

arg_pos+=1

working_dir=sys.argv[arg_pos]

arg_pos+=1

category=sys.argv[arg_pos]

arg_pos+=1

paid_or_free=sys.argv[arg_pos].lower()

arg_pos+=1

offline_or_online=sys.argv[arg_pos].lower()

arg_pos+=1

html_cache_path=sys.argv[arg_pos]

arg_pos+=1

resolved_urls_cache_path=sys.argv[arg_pos]

arg_pos+=1

scrape_timestamp_s=sys.argv[arg_pos]

arg_pos+=1

starting_page=sys.argv[arg_pos]

arg_pos+=1

max_ending_page=sys.argv[arg_pos]

arg_pos+=1

database=sys.argv[arg_pos]

#arg_pos+=1

#profiler_web_server_port=sys.argv[arg_pos]

print 'working_dir: '+working_dir

print 'category: ' +category

print 'paid_or_free: ' +paid_or_free

print 'offline_or_online: ' +offline_or_online

print 'html_cache_path: ' +html_cache_path

print 'resolved_urls_cache_path: ' +resolved_urls_cache_path

print 'starting_page: ' +starting_page

print 'max_ending_page: '+max_ending_page

print 'scrape_timestamp_s: '+scrape_timestamp_s

print 'database: '+database

#print 'profiler_web_server_port: '+profiler_web_server_port

print

# for memory profiling:

#import cherrypy

#import dowser

#cherrypy.config.update({'server.socket_port': int(profiler_web_server_port)})

#cherrypy.config.update({'server.socket_host': '0.0.0.0'})

#cherrypy.tree.mount(dowser.Root())

#cherrypy.engine.autoreload.unsubscribe()

### Windows only

###cherrypy._console_control_handler.unsubscribe()

#cherrypy.engine.start()

print 'now starting scrape'

import os

os.chdir(working_dir)

sys.path.append(working_dir)

reload(sys)

import datetime,time

from common import utilities

reload(utilities)

from android_market_data_public import scrape_and_extract_apps

reload(scrape_and_extract_apps)

from android import analysis

from android import db

offline=offline_or_online.lower()=='offline'

extraction_date=datetime.datetime.now()

start_time=datetime.datetime.now()

print 'time is: '+str(start_time)

print 'extraction_date: '+str(extraction_date)

print 'extraction_date converted to a timestamp: '+str(int(time.mktime(extraction_date.timetuple())*1000)) # int(time.mktime((datetime.datetime(2011,6,27,0,0,1)).timetuple())*1000)

print

if offline:

print '--OFFLINE SCRAPE-- '

print

scrape_timestamp=int(scrape_timestamp_s)

scrape_date=datetime.datetime.fromtimestamp(scrape_timestamp/1000)

if not html_cache_path.endswith('/'):

html_cache_path+='/'

if not resolved_urls_cache_path.endswith('/'):

resolved_urls_cache_path+='/'

else: #online scape - generate paths and times for timestamps

print '--online scrape--'

print

if scrape_timestamp_s=='new':

scrape_date=extraction_date

scrape_timestamp=(int(time.mktime(scrape_date.timetuple())*1000))

else:

print 'PROBLEM: this is an online scrape yet the scrape_timestamp was specified. Bad.'

raise Exception('PROBLEM: this is an online scrape yet the scrape_timestamp was specified. Bad.')

if html_cache_path=='new':

html_cache_path=utilities.make_new_dated_path('../cache/html_','_'+paid_or_free+'/',scrape_date)

print "html cache in:"+html_cache_path

os.mkdir(html_cache_path)

else:

print 'PROBLEM: this is an online scrape yet the cache path for html was specified. Bad.'

raise Exception('PROBLEM: this is an online scrape yet the cache path for html was specified. Bad.')

if resolved_urls_cache_path=='new':

resolved_urls_cache_path=utilities.make_new_dated_path('../cache/resolved_urls_','_'+paid_or_free+'/',scrape_date)

print "resolved_urls cache in:"+resolved_urls_cache_path

os.mkdir(resolved_urls_cache_path)

else:

print 'PROBLEM: this is an online scrape yet the cache path for resolved urls was specified. Bad.'

raise Exception('PROBLEM: this is an online scrape yet the cache path for resolved urls was specified. Bad.')

print

print 'scrape_date converted to a timestamp: '+str(int(time.mktime(scrape_date.timetuple())*1000)) # int(time.mktime((datetime.datetime(2011,6,27,0,0,1)).timetuple())*1000)

print 'html_cache_path: ' +html_cache_path

print 'resolved_urls_cache_path: ' +resolved_urls_cache_path

try:

log_path=utilities.make_new_dated_path('../cache/scrape_'+paid_or_free+'_','.log',extraction_date)

log_file=open(log_path,'w')

print "log file: "+str(log_file)

print "now logging to that file"

sys.stdout=log_file

sys.stderr=log_file

print "now logging to file"

print 'making stdout and stderr unbuffered (actually buffered only per call to write())'

sys.stdout=utilities.Unbuffered(sys.stdout)

sys.stderr =utilities.Unbuffered(sys.stderr)

#need to call set_globals here only because the call to get_categories needs them. it's clumsy doing this here, and later on in inhale_market_data too.

#set_globals(html_cache_path,resolved_urls_cache_path,offline)

paid_or_free=paid_or_free.lower()

print 'paid_or_free :'+paid_or_free

if category.lower()=='all':

cats=scrape_and_extract_apps.get_categories()

else:

cats=[category]

print 'categories count: '+str(len(cats))+' '+str(cats)

for cat in cats:

scrape_and_extract_apps.inhale_market_data(category=cat,paid=(paid_or_free=='paid'), html_cache_path=html_cache_path, resolved_urls_cache_path=resolved_urls_cache_path, scrape_date=scrape_date, extraction_date=extraction_date, database=database, offline=(offline_or_online.lower()=='offline'),starting_page=int(starting_page),max_ending_page=int(max_ending_page))

print 'finished with scrape.'

print 'now filling in calculated values in rows'

analysis.fill_in_calculated_values(scrape_timestamp,database=database)

print 'finished with filling in calculated values.'

#print 'there are now '+count+' apps with this scrape_timestamp in the db.'

print 'now declaring this was a good scrape.'

db.record_scrape(scrape_timestamp,paid_or_free=paid_or_free,database=database)

print 'scrape and fill in is finished'

finally:

try:

print 'closing things out'

#cherrypy.engine.exit()

print 'log_file is a :'

print type(log_file)

print log_file

sys.stdout=sys.__stdout__

sys.stderr=sys.__stderr__

if type(log_file)==file:

print 'closing log file'

log_file.flush()

log_file.close()

except:

print 'it appears we were logging to stdout. no need to close any log file.'

print 'finished with scrape and closed log file, if any.'

end_time=datetime.datetime.now()

print 'scrape time:'+str(start_time)

print 'scrape end time: '+str(end_time)

print 'elapsed time: '+str(end_time-start_time)