def get(self):
logging.info("Cron starting...")
try:
# authenticate to twitter
client = twitter.Api(consumer_key=Globals.CONSUMER_KEY,
consumer_secret=Globals.CONSUMER_SECRET,
access_token_key=Globals.CLIENT_TOKEN,
access_token_secret=Globals.CLIENT_SECRET,
cache=None)
q_futures = []
for region in Globals.REGIONS:
# make request
response = client.GetTrendsWoeid(id=region)
# get current timestamp in seconds
timestamp = int(math.floor(time.time()))
# put trends to db
entityList = []
for trend in response:
entityList.append(
Trend(name=trend.name,
woeid=region,
timestamp=timestamp,
time=10))
q_futures.extend(ndb.put_multi_async(entityList))
# wait all async put operations to finish.
ndb.Future.wait_all(q_futures)
except Exception, e:
traceback.print_exc()
Error(msg=str(e), timestamp=int(math.floor(time.time()))).put()
def get(self):
logging.info("SummaryTask starting...")
# init class and variables
bucket_name = os.environ.get(
'BUCKET_NAME', app_identity.get_default_gcs_bucket_name())
bucket = '/' + bucket_name
trendManager = TrendManager()
dataModelConverter = DataModelConverter()
csvUtils = CsvUtils()
cloudStorageUtils = CloudStorageUtils()
previous_day_timestamp = int(time.time()) - Globals._1_DAY
q_futures = []
for region in self.getRegions():
try:
date = TimezoneAwareDate(region, self.request.get('date'))
trendsJson = self.getTrends(region, trendManager)
self.saveToCloudStorage(dataModelConverter, csvUtils,
cloudStorageUtils, trendsJson, region,
bucket, date)
self.saveToDatastore(q_futures, trendsJson, region, date)
self.deleteFromDatastore(q_futures, region,
previous_day_timestamp)
except Exception, e:
traceback.print_exc()
Error(msg=str(e), timestamp=int(time.time())).put()
SendEmail().send('Error on SummaryTask', str(e))
self.retry()
def valid_age(line):
"""Everyone ordering must be 21 or older."""
dob = line.o_DOB
if not _is_21(dob):
rule = 'Allowed age'
new_row = Error(e_name=rule, order_key=line.primary_key)
line.errors.append(new_row)
return False
return True
def valid_zip_sum(line):
"""The sum of digits in a zip code may not exceed 20
("90210": 9+0+2+1+0 = 12)."""
zipcode = line.o_zip_code
if not sum(int(x) for x in zipcode) <= 20:
rule = 'Zipcode sum'
new_row = Error(e_name=rule, order_key=line.primary_key)
line.errors.append(new_row)
return False
return True
def valid_email(line):
""" Email address must be valid."""
email = line.o_email
is_valid = validate_email(email)
if not is_valid:
rule = 'Email validation'
new_row = Error(e_name=rule, order_key=line.primary_key)
line.errors.append(new_row)
return False
return True
def valid_zipcode(line):
"""Valid zip codes must be 5 or 9 digits."""
zipcode = line.o_zip_code
invalid_zip = len(zipcode) not in [5, 9] and zipcode.isdigit()
if invalid_zip:
rule = 'Zipcode length'
new_row = Error(e_name=rule, order_key=line.primary_key)
line.errors.append(new_row)
return False
return True
def valid_state(line): # line is an instance of Order class
"""No wine can ship to New Jersey, Connecticut, Pennsylvania, Massachusetts,
Illinois, Idaho or Oregon."""
state = line.o_state
invalid_state = state in app.config.get('ALLOWED_STATES')
if invalid_state:
rule = 'Allowed states'
new_row = Error(e_name=rule, order_key=line.primary_key)
line.errors.append(new_row)
return False
return True
def valid_domain(line):
"""Customers from NY may not have .net email addresses."""
email = line.o_email
ln = len(email)
invalid_domain = (email[ln - 4:] == '.net')
invalid_state = (line.o_state == 'NY')
if invalid_domain and invalid_state:
rule = '.net domain'
new_row = Error(e_name=rule, order_key=line.primary_key)
line.errors.append(new_row)
return False
return True
def readDateInfo(self, retry=True):
try:
result = self.rpc.get_result()
if result.status_code == 200:
data = result.content
jsonData = json.loads(data)
return jsonData['formatted'].split()[0]
else:
if retry:
time.sleep(1)
return self.readDateInfo(False)
else:
SendEmail().send('Error on TimezoneAwareDate',
'Timezonedb api request error.')
except Exception, e:
traceback.print_exc()
Error(msg=str(e), timestamp=int(time.time())).put()
SendEmail().send('Error on TimezoneAwareDate', str(e))
def get(self):
logging.info("GetTrendsTask starting...")
try:
# create twitter client
client = TwitterApi(
consumer_key=Crenditals.CONSUMER_KEY,
consumer_secret=Crenditals.CONSUMER_SECRET,
access_token_key=Crenditals.CLIENT_TOKEN,
access_token_secret=Crenditals.CLIENT_SECRET)
q_futures = []
for region in Globals.REGIONS:
# request trends from twitter
response = client.getTrendsByWoeid(woeid=region)
# get current timestamp in seconds
timestamp = int(time.time())
# put trends to db
entityList = []
for trend in response:
entityList.append(
TrendWindow(
name=trend['name'],
woeid=region,
timestamp=timestamp,
time=10,
volume=trend['tweet_volume']))
q_futures.extend(ndb.put_multi_async(entityList))
self.updateCacheValues(region, entityList)
# wait all async put operations to finish.
ndb.Future.wait_all(q_futures)
except ValueError as v_e:
logging.error(v_e)
# self.retry()
except Exception, e:
traceback.print_exc()
Error(msg=str(e), timestamp=int(time.time())).put()
SendEmail().send('Error on GetTrendsTask', str(e))
def get(p_min: int, p_max: int):
p_max = 100 if p_max == 0 else p_max
return NumberObject(minimum=p_min, maximum=p_max, type="integer", result=random.randint(p_min, p_max)).serialize() \
if p_max >= p_min else (Error(code=400, description="Maximum should be higher than Minimum").serialize(), 400)