def run(self, bucket_name, key, current_date):
# type: (str, str, long) -> GeneratorType
shards = max(2, int(App.all(keys_only=True).count() / 100)) # A maximum of 100 apps will be logged per shard
logging.info('Starting ServiceStatsPipeline with %d shards', shards)
params = {
'mapper_spec': 'rogerthat.bizz.job.service_stats.mapper',
'mapper_params': {
'bucket_name': bucket_name,
'entity_kind': 'rogerthat.models.ServiceIdentity',
'filters': []
},
'combiner_spec': 'rogerthat.bizz.job.service_stats.combiner',
'reducer_spec': 'rogerthat.bizz.job.service_stats.reducer',
'reducer_params': {
'output_writer': {
'bucket_name': bucket_name
}
},
'input_reader_spec': 'mapreduce.input_readers.DatastoreInputReader',
'output_writer_spec': 'mapreduce.output_writers.GoogleCloudStorageConsistentOutputWriter',
'shards': shards
}
output = yield mapreduce_pipeline.MapreducePipeline(key, **params)
process_output_pipeline = yield ProcessOutputPipeline(output, current_date)
with pipeline.After(process_output_pipeline):
yield CleanupGoogleCloudStorageFiles(output)
def test_static_flex_service(self):
self.set_datastore_hr_probability(1)
print 'Test service creation with static modules'
email = u'test1.flex.foo.com'
r = create_flex_service(email,
name="test",
address="Antwerpsesteenweg 19\n9080 Lochristi",
phone_number="+32 9 324 25 64",
languages=["en", "nl"],
currency=u"€",
modules=list(SolutionModule.STATIC_MODULES),
broadcast_types=['test1', 'test2', 'test3'],
apps=[a.app_id for a in App.all()],
allow_redeploy=False,
organization_type=random.choice([
x for x in OrganizationType.all() if x > 0
]))
service_user = users.User(r.login)
set_current_user(service_user)
print 'Test provisioning of static modules'
common_provision(service_user)
print 'Test rendering the home page'
FlexHomeHandler({}, webapp2.Response()).get()
def test_association_creation(self):
self.set_datastore_hr_probability(1)
apps = [a.app_id for a in App.all()]
r = create_flex_service(email=u'*****@*****.**',
name=u"test",
address=u"Antwerpsesteenweg 19\n9080 Lochristi",
phone_number=u"+32 9 324 25 64",
languages=[u"en", u"nl"],
currency=u"€",
modules=[SolutionModule.CITY_APP, SolutionModule.BROADCAST, SolutionModule.ASK_QUESTION,
SolutionModule.WHEN_WHERE],
broadcast_types=['News', 'test'],
apps=apps,
allow_redeploy=False,
organization_type=OrganizationType.CITY)
# Create city customer
shop_user = users.User(u'*****@*****.**')
customer_id = None
vat = u''
name = u'Town Lochristi'
address1 = u'Dorp - West 52'
address2 = u''
zip_code = u'9080'
city = u'Lochristi'
country = u'BE'
language = u'nl'
organization_type = OrganizationType.CITY
prospect_id = None
city_customer = create_or_update_customer(shop_user, customer_id, vat, name, address1, address2, zip_code, city,
country, language, organization_type, prospect_id,
team_id=RegioManagerTeam.all().get().id)
city_customer.service_email = city_customer.user_email = r.login
city_customer.default_app_id = apps[0]
city_customer.app_ids = apps
city_customer.put()
# Create city contact
first_name = u'Firstname'
last_name = u'Lastname'
email_address = u'*****@*****.**'
phone_number = u'+3293268806'
create_contact(city_customer, first_name, last_name, email_address, phone_number)
name = u'test-Test association'
address1 = u'Antwerpsesteenweg 19'
address2 = u''
zip_code = u'9080'
city = u'Lochristi'
user_email = u'*****@*****.**'
telephone = u'+32 9 324 25 64'
language = u'nl'
modules = [u'agenda', u'bulk_invite', u'static_content', u'ask_question', u'broadcast']
broadcast_types = [u'Evenementen', u'Nieuws']
set_current_user(users.User(r.login), set_google_user=False)
output = rest_put_service(name, address1, address2, zip_code, city, user_email, telephone, language, modules,
broadcast_types)
self.assertTrue(output.success, output.errormsg)
self.assertFalse(output.errormsg)
def migrate():
for app in App.all():
settings_key = AppSettings.create_key(app.app_id)
new_settings = AppSettings(key=settings_key,
background_fetch_timestamps=[21600] if app.type == App.APP_TYPE_CITY_APP else [])
settings = AppSettings.get(settings_key) or new_settings
settings.put()
get_app_settings.invalidate_cache(app.app_id) # @UndefinedVariable
def generate_all_stats():
apps = App.all().fetch(None)
app_ids = [a.app_id for a in apps]
d = datetime.now()
date = datetime(year=d.year, month=d.month, day=d.day)
results = [
generate_created_app_stats(apps),
generate_released_app_stats(apps),
generate_users_per_app_stats(app_ids)
]
for type_, result in results:
log_offload.create_log(None, type_, result, None, timestamp=time.mktime(date.timetuple()))
def _test_dynamic_flex_service(self, language):
self.set_datastore_hr_probability(1)
print 'Test %s service creation with all modules' % language
email = u'*****@*****.**'
r = create_flex_service(email,
name="test",
address="Antwerpsesteenweg 19\n9080 Lochristi",
phone_number="+32 9 324 25 64",
languages=[language],
currency=u"€",
modules=SolutionModule.visible_modules(),
broadcast_types=['test1', 'test2', 'test3'],
apps=[a.app_id for a in App.all()],
allow_redeploy=False,
organization_type=random.choice([
x for x in OrganizationType.all() if x > 0
]))
service_user = users.User(r.login)
set_current_user(service_user)
print 'Setting order type to advanced'
_put_default_menu(service_user)
sln_settings = get_solution_settings(service_user)
sln_order_settings = SolutionOrderSettings(
key=SolutionOrderSettings.create_key(service_user))
sln_order_settings.text_1 = 'text_1'
sln_order_settings.order_type = SolutionOrderSettings.TYPE_ADVANCED
sln_order_settings.leap_time = 1
sln_order_settings.leap_time_type = 86400
sln_order_settings.put()
SolutionOrderWeekdayTimeframe.create_default_timeframes_if_nessecary(
service_user, sln_settings.solution)
print 'Test provisioning of all modules'
common_provision(service_user)
print 'Test rendering the home page'
FlexHomeHandler({}, webapp2.Response()).get()
print 'Test deletion of all modules'
solution_settings = get_solution_settings(service_user)
solution_settings.modules = []
solution_settings.put()
common_provision(service_user)
print 'Test rendering the home page'
FlexHomeHandler({}, webapp2.Response()).get()
def get(self):
expired_apps = [(app.app_id, time.ctime(app.apple_push_cert_valid_until))
for app in App.all() \
.filter('apple_push_cert_valid_until !=', None) \
.filter('apple_push_cert_valid_until <', now() + 60 * 86400) \
.order('apple_push_cert_valid_until')]
if expired_apps:
expired_apps_str = pprint.pformat(expired_apps)
settings = get_server_settings()
send_mail(settings.dashboardEmail,
settings.supportWorkers,
"There are APN certs that are about to expire",
"The following APN certs are about to expire:\n%s" % expired_apps_str)
logging.error("The following APN certs are about to expire:\n%s" % expired_apps_str)
def generate_app_stats_history(log_type):
if log_type == 'rogerthat.created_apps':
stats_func = _generate_created_app_stats
elif log_type == 'rogerthat.released_apps':
stats_func = _generate_released_app_stats
else:
raise Exception('Invalid log type')
d = datetime.utcfromtimestamp(1410963082)
date = datetime(year=d.year, month=d.month, day=d.day)
all_apps = App.all().fetch(None)
now = datetime.now()
while date < now:
ts = time.mktime(date.timetuple())
result = stats_func([app for app in all_apps if app.creation_time <= ts])
log_offload.create_log(None, log_type, result, None, timestamp=ts)
date = date + relativedelta(days=1)
def delete_app(app_id):
app = get_app(app_id)
validate_can_delete_app(app)
to_delete = [
AppSettings.create_key(app_id),
app.key()
]
to_put = []
for profile_key in UserProfile.all(keys_only=True).filter('app_id', app_id):
delete_account(users.User(profile_key.parent().name()))
for other_app in App.all():
if app_id in other_app.orderable_app_ids:
other_app.orderable_app_ids.remove(app_id)
to_put.append(other_app)
db.delete(to_delete)
if to_put:
put_and_invalidate_cache(to_put)
def _validate_job(from_service_user, to_service_user):
# FROM != TO
bizz_check(from_service_user.email() != to_service_user.email(),
'FROM and TO should not be equal')
# FROM should exist and be a ServiceProfile
from_service_profile = _get_profile_not_cached(from_service_user)
bizz_check(from_service_profile,
'ServiceProfile %s not found' % from_service_user)
bizz_check(isinstance(from_service_profile, ServiceProfile),
'Profile %s is not of expected type ServiceProfile, but of type %s' % (from_service_user,
from_service_profile.kind()))
# TO should not exist
to_profile = _get_profile_not_cached(to_service_user)
if to_profile:
raise BusinessException('%s %s already exists' % (to_profile.kind(), to_service_user))
# FROM should not be an auto_connected_service in an App
for app in App.all():
if from_service_user.email() in app.admin_services:
raise BusinessException('FROM should not be an app admin service (found in app %s)' % app.app_id)
for acs in app.auto_connected_services:
acs_si_user = users.User(acs.service_identity_email)
if get_service_user_from_service_identity_user(acs_si_user).email() == from_service_user.email():
raise BusinessException('FROM should not be an AutoConnectedService (found %s in app %s)'
% (acs_si_user, app.app_id))
# Validate that there are no running jobs for <from_service> or <to_service>
def get_base_query():
return MigrateServiceJob.all().filter('phase !=', MigrateServiceJob.PHASE_DONE)
for service_user in (from_service_user, to_service_user):
for prop in ('from_service_user', 'to_service_user'):
job = get_base_query().filter(prop, service_user).get()
if job:
raise BusinessException('There already is a MigrateServiceJob with %s=%s\n%s'
% (prop, service_user, db.to_dict(job)))
return from_service_profile
def _get_all_apps():
return App.all(keys_only=True)
def prepare_app_statistics_cache():
for app_key in list(App.all(keys_only=True)):
app_id = app_key.name()
invalidate_cache(get_user_count_in_app, app_id)
get_user_count_in_app(app_id)
def f():
return App.all().filter('is_default =', True).get()
def get_app_ids_without_custom_asset(asset_type):
app_ids = set([key.name() for key in App.all(keys_only=True)])
for asset in AppAsset.list_by_type(asset_type):
for app_id in asset.app_ids:
app_ids.remove(app_id)
return app_ids
def get_visible_apps():
return App.all().filter("visible =", True)
def get_all_apps():
return App.all()
def get_extra_apps():
return App.all().filter("demo =", False).filter("visible =", True)
def get_apps(app_types, only_visible=True):
# type: (list[int], bool) -> list[App]
qry = App.all().filter('type IN', app_types)
if only_visible:
qry.filter("visible =", True)
return qry
def f():
return App.all(keys_only=True).filter('is_default =', True).get()
def _get_all_apps_keys_query():
return App.all(keys_only=True)
def get_apps_by_type(app_type):
# type: (list[int]) -> list[App]
qry = App.all(keys_only=True).filter('type', app_type)
return get_apps_by_keys(qry.fetch(None))
def run():
apps = list(App.all())
for app in apps:
app.demo = 'demo' in app.app_id
put_and_invalidate_cache(*apps)
def re_index_all_vouchers_all_apps(queue=HIGH_LOAD_WORKER_QUEUE):
for app_key in App.all(keys_only=True):
_re_index_all_vouchers(app_key.name(), queue)
def get_apps_that_use_embedded_app(name):
app_ids = []
for k in App.all(keys_only=True).filter('embedded_apps =', name):
app_ids.append(k.name())
return app_ids
def get_apps(app_types, only_visible=True):
qry = App.all().filter('type IN', app_types)
if only_visible:
qry.filter("visible =", True)
return qry