def get(self):
try:
dt =datetime.datetime.now()
logThis(AEL_LEVEL_INFO, 'Master Task added to its Q at[%s]' %(dt))
task_name = GAET_MASTERTASK_NAME + str(datetime.datetime.now())
task_name = re.sub('[^a-zA-Z0-9_-]', '_', task_name)
taskqueue.add(queue_name=GAEQ_FOR_MASTERTASK,name=task_name)
logThis(AEL_LEVEL_INFO, "OK-MASTER TASK ADD")
except taskqueue.DuplicateTaskNameError:
logThis(AEL_LEVEL_CRITICAL, "EXCEPTION on QueueAMasterTask-" + traceback.format_exc())
except:
logThis(AEL_LEVEL_CRITICAL, "EXP on QueueAMasterTask-" + traceback.format_exc())
def get(self):
try:
self.response.write('''
<ul>
<li><a href="#env">Environment Variables</a></li>
<li><a href="#filesystem">File System</a></li>
<li><a href="#request">Request Data</a></li>
</ul>
<hr noshade><h2 id="env">Environment Variables</h2><table>
''')
keys = os.environ.keys()
keys.sort()
for k in keys:
self.response.write(
'<tr><td valign="top"><code>%s</code></td>'
'<td valign="top"><code>%s</code></td></tr>' %
(cgi.escape(k), cgi.escape(str(os.environ[k]))))
self.response.write('''
</table>
<hr noshade><h2 id="filesystem">File System</h2>
<pre>''')
for path, dirnames, filenames in os.walk('.'):
for filename in filenames:
self.response.write(
cgi.escape(os.path.join(path, filename)) + '\n')
self.response.write('''
</pre>
<hr noshade><h2 id="request">Request Data</h2>
<pre>''')
self.response.write(cgi.escape(sys.stdin.read()))
self.response.write('</pre>')
self.response.write('<hr noshade><p>The time is: %s</p>' %
str(datetime.datetime.now()))
except:
logThis(AEL_LEVEL_CRITICAL,
"EXP on Maina\Page-" + traceback.format_exc())
def fashionista_register_shutdown_hook():
apiproxy_stub_map.apiproxy.CancelApiCalls()
# We can save state to datastore here or Log some statistics.
logThis(AEL_LEVEL_WARNING, 'SHUTDOWN IN PROGRESS...')
logThis(AEL_LEVEL_INFO,'CPU USAGE: %s' % runtime.cpu_usage())
logThis(AEL_LEVEL_INFO,'MEMORY USAGE: %s' % runtime.memory_usage())
def fashionista_register_shutdown_hook():
apiproxy_stub_map.apiproxy.CancelApiCalls()
# We can save state to datastore here or Log some statistics.
logThis(AEL_LEVEL_WARNING, 'SHUTDOWN IN PROGRESS...')
logThis(AEL_LEVEL_INFO, 'CPU USAGE: %s' % runtime.cpu_usage())
logThis(AEL_LEVEL_INFO, 'MEMORY USAGE: %s' % runtime.memory_usage())
def post(self):
try:
logThis(AEL_LEVEL_INFO, 'SUBTASK:START')
Queue = self.request.get('taskQ')
ac = WindowsCrawler(self.request.get('url'))
logThis(AEL_LEVEL_INFO, str(ac.isdone()))
# Retry if task didn't succeed
if ac.isdone() == False and ac.retry() == True:
taskName = self.request.get('url')
task_name_on_Q = '[%s_%s]' % (taskName, datetime.datetime.now(
).strftime("[%Y-%m-%d %H:%M:%S]"))
task_name_on_Q = re.sub('[^a-zA-Z0-9_-]', '_', task_name_on_Q)
taskqueue.add(queue_name=Queue,
name=task_name_on_Q,
params={
'url': self.request.get('url'),
'taskQ': Queue
})
elif ac.isdone() == True:
print "TASK DONE"
dbA.updateStatus([self.request.get('url')])
#POST EXCEPT
except:
logThis(AEL_LEVEL_CRITICAL,
"EXP on HandlerForWindowsTaskQ-" + traceback.format_exc())
def post(self):
try:
logThis(AEL_LEVEL_INFO, 'SUBTASK:START')
Queue = self.request.get('taskQ')
print "IOS CRAWL STARTING"
ac = IOsCrawler(self.request.get('url'))
logThis(AEL_LEVEL_INFO, str(ac.isdone()) )
# Retry if task didn't succeed
if ac.isdone() == False and ac.retry() == True:
taskName = self.request.get('url')
print "THIS IS QUEUE", Queue
task_name_on_Q = '[%s_%s]' % (taskName,
datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]"))
task_name_on_Q = re.sub('[^a-zA-Z0-9_-]', '_', task_name_on_Q)
taskqueue.add(queue_name=Queue,name=task_name_on_Q,
params={'url': self.request.get('url'), 'taskQ' : Queue} )
elif ac.isdone() == True:
print "TASK DONE"
dbA.updateStatus([self.request.get('url')])
#POST EXCEPT
except:
logThis(AEL_LEVEL_CRITICAL, "EXP on HandlerForAppleTaskQ-" + traceback.format_exc())
class HandlerForMasterTaskQ(webapp2.RequestHandler):
#HAS TO BE POST FOR QUEUE HANDLERS
def post(self):
try:
logThis(AEL_LEVEL_INFO, 'START PROCESSING MASTERTASK FROM MasterTaskQ')
#Do Something Here
dbG.populate('AppStore.GooglePlay_Master_DB')
urls = dbG.toCrawl()
for url in urls:
self._addSubTaskToQueue(url, GAEQ_FOR_GOOGLE)
dbA.populate('AppStore.AppStore_Master_DB')
urls = dbA.toCrawl()
for url in urls:
self._addSubTaskToQueue(url, GAEQ_FOR_APPLE)
logThis(AEL_LEVEL_INFO, 'MASTERTASK: Added SubTasks')
#POST EXCEPT
except:
logThis(AEL_LEVEL_CRITICAL, "EXP on HandlerForMasterTaskQ-" + traceback.format_exc())
def _addSubTaskToQueue(self, taskObj, Queue):
try:
if taskObj != None:
taskName = taskObj
logThis(AEL_LEVEL_INFO, 'ADDING-SUBTASK [%s] TO SUB TASK Q '% taskName)
task_name_on_Q = '[%s_%s]' % (taskName,
datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]"))
task_name_on_Q = re.sub('[^a-zA-Z0-9_-]', '_', task_name_on_Q)
taskqueue.add(queue_name=Queue,name=task_name_on_Q,
params={'url': taskObj, 'taskQ' : Queue} )
logThis(AEL_LEVEL_INFO, "NEW SUB TASK ON Q:" + task_name_on_Q)
except taskqueue.TombstonedTaskError, e:
logThis(AEL_LEVEL_CRITICAL, "EXCEPTION on _addSubTaskToQueue-" + traceback.format_exc())
except taskqueue.DuplicateTaskNameError, e:
logThis(AEL_LEVEL_CRITICAL, "EXCEPTION on _addSubTaskToQueue-" + traceback.format_exc())
def _addSubTaskToQueue(self, taskObj, Queue):
try:
if taskObj != None:
taskName = taskObj
logThis(AEL_LEVEL_INFO, 'ADDING-SUBTASK [%s] TO SUB TASK Q '% taskName)
task_name_on_Q = '[%s_%s]' % (taskName,
datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]"))
task_name_on_Q = re.sub('[^a-zA-Z0-9_-]', '_', task_name_on_Q)
taskqueue.add(queue_name=Queue,name=task_name_on_Q,
params={'url': taskObj, 'taskQ' : Queue} )
logThis(AEL_LEVEL_INFO, "NEW SUB TASK ON Q:" + task_name_on_Q)
except taskqueue.TombstonedTaskError, e:
logThis(AEL_LEVEL_CRITICAL, "EXCEPTION on _addSubTaskToQueue-" + traceback.format_exc())
def post(self):
try:
logThis(AEL_LEVEL_INFO, 'START PROCESSING MASTERTASK FROM MasterTaskQ')
#Do Something Here
dbG.populate('AppStore.GooglePlay_Master_DB')
urls = dbG.toCrawl()
for url in urls:
self._addSubTaskToQueue(url, GAEQ_FOR_GOOGLE)
dbA.populate('AppStore.AppStore_Master_DB')
urls = dbA.toCrawl()
for url in urls:
self._addSubTaskToQueue(url, GAEQ_FOR_APPLE)
logThis(AEL_LEVEL_INFO, 'MASTERTASK: Added SubTasks')
#POST EXCEPT
except:
logThis(AEL_LEVEL_CRITICAL, "EXP on HandlerForMasterTaskQ-" + traceback.format_exc())
logThis(AEL_LEVEL_INFO, 'MASTERTASK: Added SubTasks')
#POST EXCEPT
except:
logThis(AEL_LEVEL_CRITICAL, "EXP on HandlerForMasterTaskQ-" + traceback.format_exc())
def _addSubTaskToQueue(self, taskObj, Queue):
try:
if taskObj != None:
taskName = taskObj
logThis(AEL_LEVEL_INFO, 'ADDING-SUBTASK [%s] TO SUB TASK Q '% taskName)
task_name_on_Q = '[%s_%s]' % (taskName,
datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]"))
task_name_on_Q = re.sub('[^a-zA-Z0-9_-]', '_', task_name_on_Q)
taskqueue.add(queue_name=Queue,name=task_name_on_Q,
params={'url': taskObj, 'taskQ' : Queue} )
logThis(AEL_LEVEL_INFO, "NEW SUB TASK ON Q:" + task_name_on_Q)
except taskqueue.TombstonedTaskError, e:
logThis(AEL_LEVEL_CRITICAL, "EXCEPTION on _addSubTaskToQueue-" + traceback.format_exc())
except taskqueue.DuplicateTaskNameError, e:
logThis(AEL_LEVEL_CRITICAL, "EXCEPTION on _addSubTaskToQueue-" + traceback.format_exc())
except:
logThis(AEL_LEVEL_CRITICAL, "EXCEPTION on _addSubTaskToQueue-" + traceback.format_exc())
app = webapp2.WSGIApplication([('/_ah/queue/masterTaskQ', HandlerForMasterTaskQ)
], debug=True)
