def create_fact(cls, **data):
event_time = data['date']
# using geoip we assert what our country code is based upon the ip address chain
country_code = ''
for ip_address in data['ip_address_chain']:
country_code = GEO_IP_LOOKUP.get_country_code(ip_address)
if country_code:
break
# lets now fill in our dimensions
territory = DimensionTerritory.insert(code=country_code)
client = DimensionClient.insert(client_id=data['client_id'])
item = DimensionItem.insert(item_id=data['item_id'])
device = DimensionDevice.insert(item_id=data['device_id'])
status = DimensionPlayStatus.insert(status=data['status'])
user = DimensionUser.insert(internal_user_id=data['internal_user_id'],
external_user_id=data['external_user_id'],
client=client)
# we create our fact play object
fshp = FactServicesHeartbeatPlay.objects.create(
item_id=item,
user=user,
device_id=device,
status=status,
event_time=event_time,
country_code=territory,
event_utc_date=DimensionUTCDate.date_from_datetime(event_time),
position=get(data, 'position', disallow=['']), # Preserve 0
bit_rate=get(data, 'bitrate', disallow=['']), # Preserve 0
)
# for any occuring buffers we create a play buffer, we can then add this to our many to many relation
for buffer_duration in get(data,
'buffer_events',
disallow=[None, ''],
default=[]):
fshpb = FactServicesHeartbeatPlayBuffer.objects.create(
duration=buffer_duration,
event_utc_date=DimensionUTCDate.date_from_datetime(event_time),
event_time=event_time)
fshp.play_buffer.add(fshpb)
# if we have an init in our event - we can add this and add the foreign key to our play event
# this will make for good and easy lookup of full fact play data
init_duration = get(data, 'init_length', disallow=[''])
if init_duration is not None:
init_licensed = get(data,
'init_licensed',
disallow=['', None],
default=False)
FactServicesHeartbeatPlayInit.objects.create(
heartbeat=fshp,
duration=init_duration,
licensed=init_licensed,
event_utc_date=DimensionUTCDate.date_from_datetime(event_time),
event_time=event_time)
def create_fact(cls, **data):
event_time = data['date']
# using geoip we assert what our country code is based upon the ip address chain
country_code = ''
for ip_address in data['ip_address_chain']:
country_code = GEO_IP_LOOKUP.get_country_code(ip_address)
if country_code:
break
# lets now fill in our dimensions
territory = DimensionTerritory.insert(code=country_code)
client = DimensionClient.insert(client_id=data['client_id'])
item = DimensionItem.insert(item_id=data['item_id'])
device = DimensionDevice.insert(item_id=data['device_id'])
status = DimensionPlayStatus.insert(status=data['status'])
user = DimensionUser.insert(internal_user_id=data['internal_user_id'],
external_user_id=data['external_user_id'], client=client)
# we create our fact play object
fshp = FactServicesHeartbeatPlay.objects.create(
item_id=item,
user=user,
device_id=device,
status=status,
event_time=event_time,
country_code=territory,
event_utc_date=DimensionUTCDate.date_from_datetime(event_time),
position=get(data, 'position', disallow=['']), # Preserve 0
bit_rate=get(data, 'bitrate', disallow=['']), # Preserve 0
)
# for any occuring buffers we create a play buffer, we can then add this to our many to many relation
for buffer_duration in get(data, 'buffer_events', disallow=[None, ''], default=[]):
fshpb = FactServicesHeartbeatPlayBuffer.objects.create(
duration=buffer_duration,
event_utc_date=DimensionUTCDate.date_from_datetime(event_time),
event_time=event_time)
fshp.play_buffer.add(fshpb)
# if we have an init in our event - we can add this and add the foreign key to our play event
# this will make for good and easy lookup of full fact play data
init_duration = get(data, 'init_length', disallow=[''])
if init_duration is not None:
init_licensed = get(data, 'init_licensed', disallow=['', None], default=False)
FactServicesHeartbeatPlayInit.objects.create(
heartbeat=fshp,
duration=init_duration,
licensed=init_licensed,
event_utc_date=DimensionUTCDate.date_from_datetime(event_time),
event_time=event_time)
def create_fact(cls, logs, **data):
platform = get_platform(local_data=data, model=FactServicesStorefrontRegistration)
territory = DimensionTerritory.insert(code=data['territory'])
client = DimensionClient.insert(client_id=data['client_id'])
try:
device = DimensionDevice.insert(
device_id=data.get('device_id', ''),
make=data.get('make', ''),
model=data.get('model', ''),
os=data.get('os', ''),
os_version=data.get('version', '')
)
except:
device = None
try:
user_info = data.get('attributes', None)
user = DimensionUser.insert(
internal_user_id=data.get('internal_user_id', data.get('user_id', '')),
external_user_id=data.get('external_user_id', ''),
territory=territory,
client=client,
email=data.get('email', None),
first_name=user_info.get('first_name', '') if user_info else '',
last_name=user_info.get('last_name', '') if user_info else '',
marketing_preference=user_info.get('marketing_preference', '') if user_info else '',
country_of_residence=user_info.get('country_of_residence', '') if user_info else None
)
except Exception:
user = None
try:
account = DimensionAccount.insert(
account_id=data.get('external_user_id', data.get('internal_user_id', '')),
account_created=data.get('account_created', None)
)
except:
account = None
fact = FactServicesStorefrontRegistration.objects.create(
account=account,
user=user,
client=client,
platform=platform,
device=device,
event_utc_date=DimensionUTCDate.date_from_datetime(datetime=data['event_time']),
event_utc_datetime=data['event_time'],
mnc=data.get('MNC', None),
mcc=data.get('MCC', None)
)
try:
FactServicesStorefrontRegistration.objects.get(pk=fact.pk)
exist = True
except FactServicesStorefrontRegistration.DoesNotExist:
exist = False
logs.completed = exist
logs.save()
def create_fact(cls, **data):
item = DimensionItem.insert(item_id=data['item_id'])
device, _ = DimensionDevice.objects.get_or_create(device_id=data['device_id'])
client = DimensionClient.insert(client_id=data['client_id'])
external_user_id = get(data, 'external_user_id', default='')
internal_user_id = get(data, 'internal_user_id', default='')
user = DimensionUser.insert(
external_user_id=external_user_id,
internal_user_id=internal_user_id,
client=client
)
# store the fact
cls.objects.create(
item=item,
user=user,
device=device,
product_id=data['product_id'],
asset_id=data['asset_id'],
status=data['status'],
network=data['network'],
expiry_date=data['expiry_date'],
event_utc_datetime=data['event_time'],
event_utc_date=DimensionUTCDate.date_from_datetime(data['event_time']),
)
def create_fact(cls, **data):
client = DimensionClient.insert(client_id=data['client_id'])
item = DimensionItem.insert(item_id=data['item_id'])
data_role = DimensionDataRole.insert(role=data['data_role'])
definition = DimensionDefinition.insert(definition=data['definition'])
processing_state = DimensionProcessingState.insert(processing_state=data['processing_state'])
territory = DimensionTerritory.insert(code=data['territory'])
asset = DimensionAssets.insert(
asset_id=data['asset_id'],
type=data['asset_type']
)
provider = DimensionItemProvider.insert(
True,
provider_id=data['provider_item_id'],
provider_name=data['provider'],
)
delivery_date = datetime.datetime.strptime(str(data['delivery_date']), settings.DATETIME_FORMAT)
event_date = datetime.datetime.strptime(data['event_time'], settings.DATETIME_FORMAT)
event_utc_datetime = datetime.datetime.strptime(data['event_time'], settings.DATETIME_FORMAT)
fact = FactServicesBackstageAssetMatch.objects.create(
client=client,
item=item,
asset=asset,
data_role=data_role,
processing_state=processing_state,
provider=provider,
territory=territory,
spec_name=data['spec_name'],
delivery_date=delivery_date,
definition=definition,
file_size=data['file_size'],
duration=data['duration'],
event_date=event_date.date(),
event_utc_date=DimensionUTCDate.date_from_datetime(datetime=data['event_time']),
event_utc_datetime=event_utc_datetime
)
fact.save()
if data['used_asset_ids']:
for asset_id in data['used_asset_ids']:
asset_dim = DimensionAssets.objects.get_or_create(asset_id=asset_id)
fact.used_asset_ids.add(asset_dim[0].id)
else:
fact.used_asset_ids = ''
if data['languages']:
for language in data['languages']:
lang_dim = DimensionLanguage.objects.get_or_create(iso_code=language)
fact.languages.add(lang_dim[0].id)
else:
fact.languages = ''
return fact
def create_flag(cls, **kwargs):
# This code only happens if the objects is
# not in the database yet. Otherwise it would
# have pk
# we are going to set some datetime params - we may need to create a report
# on this data later and this will be useful information to correlate against
kwargs['event_utc_datetime'] = datetime.datetime.utcnow()
kwargs['event_utc_date'] = DimensionUTCDate.date_from_datetime(kwargs['event_utc_datetime'])
cls.objects.create(**kwargs)
def create_flag(cls, event_name, packet, **kwargs):
# This code only happens if the objects is
# not in the database yet. Otherwise it would
# have pk
# we are going to set some datetime params - we may need to create a report
# on this data later and this will be useful information to correlate against
kwargs['id'] = str(uuid4())
kwargs['event'] = event_name
kwargs['packet'] = json.dumps(packet)
kwargs['event_utc_datetime'] = datetime.datetime.utcnow()
kwargs['event_utc_date'] = DimensionUTCDate.date_from_datetime(
kwargs['event_utc_datetime'])
cls.objects.create(**kwargs)
def create_fact(cls, **data):
dip = DimensionItemProvider.insert(
True,
provider_id=data['provider_id'],
provider_name=data['provider_name'],
)
pl = DimensionLanguage.insert(iso_code=data['primary_language'])
vendor = DimensionVendor.insert(name=data['vendor'])
country = DimensionCountryCode.objects.get(
iso_code=data['country_of_origin_iso'])
meta_type = DimensionMetadataType.insert(type=data['type'])
item = DimensionItem.insert(item_id=data['item_id'])
f = FactServicesBackstageItemMetadata.objects.create(
metadata_id=data['metadata_id'],
version=data['version'],
country_of_origin=country,
title=data['title'],
copyright_cline=data['copyright_cline'],
provider=dip,
type=meta_type,
isan=data['isan'],
eidr=data['eidr'],
release_date=data['release_date'],
production_company=data['production_company'],
release_year=data['release_year'],
primary_language=pl,
short_synopsis=data['short_synopsis'],
medium_synopsis=data['medium_synopsis'],
long_synopsis=data['long_synopsis'],
runtime=data['runtime'],
vendor=vendor,
episode_number=data['episode_number'] or None,
season=data['season'] or None,
show_title=data['show_title'] or None,
created=data['created'],
modified=data['modified'] or data['created'],
original_url=data['original_url'],
ultraviolet=data['ultraviolet'],
event_utc_date=DimensionUTCDate.date_from_datetime(
datetime=data['event_time']),
event_utc_datetime=data['event_time'])
f.item_meta.add(item)
return f
def create_fact(cls, **data):
dip = DimensionItemProvider.insert(
True,
provider_id=data['provider_id'],
provider_name=data['provider_name'],
)
pl = DimensionLanguage.insert(iso_code=data['primary_language'])
vendor = DimensionVendor.insert(name=data['vendor'])
country = DimensionCountryCode.objects.get(iso_code=data['country_of_origin_iso'])
meta_type = DimensionMetadataType.insert(type=data['type'])
item = DimensionItem.insert(item_id=data['item_id'])
f = FactServicesBackstageItemMetadata.objects.create(
metadata_id=data['metadata_id'],
version=data['version'],
country_of_origin=country,
title=data['title'],
copyright_cline=data['copyright_cline'],
provider=dip,
type=meta_type,
isan=data['isan'],
eidr=data['eidr'],
release_date=data['release_date'],
production_company=data['production_company'],
release_year=data['release_year'],
primary_language=pl,
short_synopsis=data['short_synopsis'],
medium_synopsis=data['medium_synopsis'],
long_synopsis=data['long_synopsis'],
runtime=data['runtime'],
vendor=vendor,
episode_number=data['episode_number'] or None,
season=data['season'] or None,
show_title=data['show_title'] or None,
created=data['created'],
modified=data['modified'] or data['created'],
original_url=data['original_url'],
ultraviolet=data['ultraviolet'],
event_utc_date=DimensionUTCDate.date_from_datetime(datetime=data['event_time']),
event_utc_datetime=data['event_time']
)
f.item_meta.add(item)
return f
def create_fact(cls, data):
# This is a DRM Device Id, not a Device Id.
device, _ = DimensionDevice.objects.get_or_create(device_id=data['device_id'])
client = DimensionClient.insert(client_id=data['client_id'])
external_user_id = get(data, 'external_user_id', default=None)
if external_user_id:
user = DimensionUser.insert(external_user_id=external_user_id, client=client)
else:
user = DimensionUser.objects.get(id=-1)
cls.objects.create(
device=device,
drm_type=data['drm_type'],
user=user,
transaction_id=data.get('transaction_id', None),
event_utc_datetime=data['event_time'],
event_utc_date=DimensionUTCDate.date_from_datetime(data['event_time']),
)
def create_fact(cls, **data):
job_manager = DimensionJobManager.insert(job_manager_id=data['job_manager_id'])
item = DimensionItem.objects.get(item_id=data['item_id'])
fact = FactServicesEncoderEncode.objects.create(
job_manager_id=job_manager,
item_id=item,
start_time=data['start_time'],
end_time=data['end_time'],
file_size=data['file_size'],
event_utc_date=DimensionUTCDate.date_from_datetime(datetime=data['event_time']),
event_utc_datetime=data['event_time']
)
for asset in data['asset_ids']:
asset_dim = DimensionAssets.objects.create(asset_id=asset)
fact.asset_ids.add(asset_dim)
fact.save()
return fact
def create_fact(cls, **data):
job_manager = DimensionJobManager.insert(job_manager_id=data['job_manager_id'])
item = DimensionItem.objects.get(item_id=data['item_id'])
fact = FactServicesPackagerPackage.objects.create(
job_manager_id=job_manager,
item_id=item,
start_time=data['start_time'],
end_time=data['end_time'],
profile=data['profile'],
event_utc_date=DimensionUTCDate.date_from_datetime(datetime=data['event_time']),
event_utc_datetime=data['event_time']
)
for asset in data['asset_ids']:
asset_dim = DimensionAssets.objects.create(asset_id=asset)
fact.asset_ids.add(asset_dim)
fact.save()
return fact
def create_fact(cls, data):
# This is a DRM Device Id, not a Device Id.
device, _ = DimensionDevice.objects.get_or_create(
device_id=data['device_id'])
client = DimensionClient.insert(client_id=data['client_id'])
external_user_id = get(data, 'external_user_id', default=None)
if external_user_id:
user = DimensionUser.insert(external_user_id=external_user_id,
client=client)
else:
user = DimensionUser.objects.get(id=-1)
cls.objects.create(
device=device,
drm_type=data['drm_type'],
user=user,
transaction_id=data.get('transaction_id', None),
event_utc_datetime=data['event_time'],
event_utc_date=DimensionUTCDate.date_from_datetime(
data['event_time']),
)
def create_fact(cls, **data):
asset = DimensionAssets.insert(
asset_id=data['asset_id'],
type=data['asset_type']
)
unique_studio_item = DimensionStudioItem.insert(unique_studio_item_id=data['unique_studio_item_id'])
studio = DimensionStudio.insert(name=data['studio'])
data_role = DimensionDataRole.insert(role=data['data_role'])
definition = DimensionDefinition.insert(definition=data['definition'])
language = DimensionLanguage.insert(iso_code=data['language'])
audio_channel = DimensionAudioChannels.insert(channel=data['audio_channel'])
client = DimensionClient.insert(client_id=data['client_id'])
vendor = DimensionVendor.insert(name=data['vendor'])
fact = FactServicesAggregatorAggregation.objects.create(
asset=asset,
unique_studio_item=unique_studio_item,
studio=studio,
file_name=data['file_name'],
start_time=data['start_time'],
end_time=data['end_time'],
file_size=data['file_size'],
checksum=data['checksum'],
data_role=data_role,
content_duration=data['content_duration'],
definition=definition,
language=language,
audio_channel=audio_channel,
client=client,
vendor=vendor,
event_utc_date=DimensionUTCDate.date_from_datetime(datetime=data['event_time']),
event_utc_datetime=data['event_time']
)
return fact
def create_fact(cls, **data):
asset = DimensionAssets.insert(asset_id=data['asset_id'],
type=data['asset_type'])
unique_studio_item = DimensionStudioItem.insert(
unique_studio_item_id=data['unique_studio_item_id'])
studio = DimensionStudio.insert(name=data['studio'])
data_role = DimensionDataRole.insert(role=data['data_role'])
definition = DimensionDefinition.insert(definition=data['definition'])
language = DimensionLanguage.insert(iso_code=data['language'])
audio_channel = DimensionAudioChannels.insert(
channel=data['audio_channel'])
client = DimensionClient.insert(client_id=data['client_id'])
vendor = DimensionVendor.insert(name=data['vendor'])
fact = FactServicesAggregatorAggregation.objects.create(
asset=asset,
unique_studio_item=unique_studio_item,
studio=studio,
file_name=data['file_name'],
start_time=data['start_time'],
end_time=data['end_time'],
file_size=data['file_size'],
checksum=data['checksum'],
data_role=data_role,
content_duration=data['content_duration'],
definition=definition,
language=language,
audio_channel=audio_channel,
client=client,
vendor=vendor,
event_utc_date=DimensionUTCDate.date_from_datetime(
datetime=data['event_time']),
event_utc_datetime=data['event_time'])
return fact
def create_fact(cls, logs, **data):
# dims
platform = save_platform(data, event='Storefront Subscription')
territory = DimensionTerritory.insert(code=data['territory']) if exists('territory', data) else None
client = DimensionClient.insert(client_id=data['client_id']) if exists('client_id', data) else None
du = save_user(data, territory, client, event='Storefront Subscription')
dst = DST.insert(
period=data['subscription_period'] if exists('subscription_period', data) else None,
recurrence=data['subscription_recurrence'] if exists('subscription_recurrence', data) else None
) if any([d in data for d in ['subscription_period', 'subscription_recurrence']]) else None
dss_status, _ = DSS.insert(event=int(data['subscription_status'])) if exists('subscription_status', data) else None
state, err = get_rule(
status=data.get('subscription_status', None),
current_state=FactServicesStorefrontSubscription.get_last_event(
subscription_id=data.get('subscription_id', None)
),
window_ongoing=bool(data.get('window_on_going', False))
)
dsse, _ = DSSe.objects.get_or_create(state=state)
item = DimensionItem.insert(item_id=data.get('item_id')) if exists('item_id', data) else None
product = DimensionProduct.insert(product_id=data.get('product_id')) if exists('product_id', data) else None
usage_right = DimensionRight.insert(
right_id=data.get('window_usage_right_id')
) if exists('window_usage_right_id', data) else None
window = DimensionWindow.insert(
window_id=data.get('window_id', None),
item=item,
product=product,
start_date=data['window_start_date'],
end_date=data['window_end_date'],
pricing_id=data['window_pricing_id'],
usage_right_id=usage_right,
tier=data['window_tier'],
deleted=data['window_deleted_date'],
window_type=data['window_type'],
on_going=data['window_on_going'],
repeat_count=data['window_repeat_count'],
auto_upgrade=data['window_auto_upgrade'],
allow_repurchase=data['window_allow_repurchase'],
apple_product_id=data['apple_product_id'],
)if any([i in data for i in [
'window_id',
'item_id',
'product_id',
'window_start_date',
'window_end_date',
'window_pricing_id',
'window_usage_right_id',
'window_tier',
'window_deleted_date',
'window_type',
'window_on_going',
'window_repeat_count',
'window_auto_upgrade',
'window_allow_repurchase',
'apple_product_id',
]]) else None
# fact create
fact = FactServicesStorefrontSubscription.objects.create(
user=du,
subscription_id=data.get('subscription_id', None),
subscription_type=dst,
subscription_status=dss_status,
subscription_state=dsse,
subscription_state_error=err,
platform=platform,
window=window,
transaction_id=data.get('transaction_id', None),
event_utc_date=DimensionUTCDate.date_from_datetime(datetime=data['event_time']),
event_utc_datetime=data.get('event_time', None)
)
try:
FactServicesStorefrontSubscription.objects.get(pk=fact.pk)
exist = True
except FactServicesStorefrontSubscription.DoesNotExist:
exist = False
logs.completed = exist
logs.save()
def create_fact(cls, logs, **data):
# insert dims
item = DimensionItem.insert(item_id=data.get('item_id')) if exists('item_id', data) else None
product = DimensionProduct.insert(product_id=data.get('product_id')) if exists('product_id', data) else None
right = DimensionRight.insert(right_id=data.get('right_id')) if exists('right_id', data) else None
currency = DimensionCurrency.insert(code=data.get('currency')) if exists('currency', data) else None
retail_model = DimensionRetailModel.insert(
model=data.get('retail_model')
) if exists('retail_model', data) else None
definition = DimensionDefinition.insert(
definition=data.get('definition')
) if exists('definition', data) else None
window = DimensionWindow.insert(
window_id=data.get('window_id', None),
product=product,
usage_right=right,
item=item
)
transaction_status = DimensionStoreTransactionStatus.insert(
status=data.get('transaction_status')
) if exists('transaction_status', data) else None
platform = save_platform(data, event='Storefront Transaction')
territory = DimensionTerritory.insert(code=data.get('territory')) if exists('territory', data) else None
client = DimensionClient.insert(client_id=data.get('client_id')) if exists('client_id', data) else None
user = save_user(data, territory, client, event='Storefront Transaction')
account = DimensionAccount.insert(
account_id=data.get(
'account_id', data.get(
'external_user_id', data.get(
'internal_user_id', None
)
)
)
)
try:
device = DimensionDevice.insert(
make=data.get('make', ''),
model=data.get('model', ''),
os=data.get('os', ''),
os_version=data.get('version', ''),
)if any([
d in data for d in ['make', 'model', 'os', 'version']
]) else None
except:
log.exception('')
try:
trans = cls.objects.get(transaction_id=data.get('transaction_id', ''))
trans.last_4_digits = data.get('last_4_digits', None)
trans.event_utc_datetime = data.get('event_time', timezone.now().isoformat())
trans.event_utc_date = DimensionUTCDate.date_from_datetime(data.get('event_time', timezone.now().isoformat()))
trans.save()
except cls.DoesNotExist as e:
trans = cls.objects.create(
item=item,
product=product,
window=window,
transaction_id=data.get('transaction_id', None),
transaction_status=transaction_status,
user=user,
account=account,
right=right,
price=data.get('price', 0.00),
currency=currency,
retail_model=retail_model,
definition=definition,
client=client,
territory=territory,
platform=platform,
device=device,
mnc=data.get('MNC', None),
mcc=data.get('MCC', None),
last_4_digits=data.get('last_4_digits', None),
event_utc_datetime=data.get('event_time', timezone.now().isoformat()),
event_utc_date=DimensionUTCDate.date_from_datetime(data.get('event_time', timezone.now().isoformat())),
)
try:
cls.objects.get(pk=trans.pk)
exist = True
except cls.DoesNotExist:
exist = False
logs.completed = exist
logs.save()
def create_fact(cls, logs, **data):
# dims
platform = save_platform(data, event='Storefront Subscription')
territory = DimensionTerritory.insert(
code=data['territory']) if exists('territory', data) else None
client = DimensionClient.insert(
client_id=data['client_id']) if exists('client_id', data) else None
du = save_user(data,
territory,
client,
event='Storefront Subscription')
dst = DST.insert(
period=data['subscription_period'] if exists(
'subscription_period', data) else None,
recurrence=data['subscription_recurrence']
if exists('subscription_recurrence', data) else None) if any([
d in data
for d in ['subscription_period', 'subscription_recurrence']
]) else None
dss_status, _ = DSS.insert(
event=int(data['subscription_status'])) if exists(
'subscription_status', data) else None
state, err = get_rule(
status=data.get('subscription_status', None),
current_state=FactServicesStorefrontSubscription.get_last_event(
subscription_id=data.get('subscription_id', None)),
window_ongoing=bool(data.get('window_on_going', False)))
dsse, _ = DSSe.objects.get_or_create(state=state)
item = DimensionItem.insert(
item_id=data.get('item_id')) if exists('item_id', data) else None
product = DimensionProduct.insert(
product_id=data.get('product_id')) if exists('product_id',
data) else None
usage_right = DimensionRight.insert(
right_id=data.get('window_usage_right_id')) if exists(
'window_usage_right_id', data) else None
window = DimensionWindow.insert(
window_id=data.get('window_id', None),
item=item,
product=product,
start_date=data['window_start_date'],
end_date=data['window_end_date'],
pricing_id=data['window_pricing_id'],
usage_right_id=usage_right,
tier=data['window_tier'],
deleted=data['window_deleted_date'],
window_type=data['window_type'],
on_going=data['window_on_going'],
repeat_count=data['window_repeat_count'],
auto_upgrade=data['window_auto_upgrade'],
allow_repurchase=data['window_allow_repurchase'],
apple_product_id=data['apple_product_id'],
) if any([
i in data for i in [
'window_id',
'item_id',
'product_id',
'window_start_date',
'window_end_date',
'window_pricing_id',
'window_usage_right_id',
'window_tier',
'window_deleted_date',
'window_type',
'window_on_going',
'window_repeat_count',
'window_auto_upgrade',
'window_allow_repurchase',
'apple_product_id',
]
]) else None
# fact create
fact = FactServicesStorefrontSubscription.objects.create(
user=du,
subscription_id=data.get('subscription_id', None),
subscription_type=dst,
subscription_status=dss_status,
subscription_state=dsse,
subscription_state_error=err,
platform=platform,
window=window,
transaction_id=data.get('transaction_id', None),
event_utc_date=DimensionUTCDate.date_from_datetime(
datetime=data['event_time']),
event_utc_datetime=data.get('event_time', None))
try:
FactServicesStorefrontSubscription.objects.get(pk=fact.pk)
exist = True
except FactServicesStorefrontSubscription.DoesNotExist:
exist = False
logs.completed = exist
logs.save()
def create_fact(cls, logs, **data):
platform = get_platform(local_data=data,
model=FactServicesStorefrontRegistration)
territory = DimensionTerritory.insert(code=data['territory'])
client = DimensionClient.insert(client_id=data['client_id'])
try:
device = DimensionDevice.insert(device_id=data.get(
'device_id', ''),
make=data.get('make', ''),
model=data.get('model', ''),
os=data.get('os', ''),
os_version=data.get('version', ''))
except:
device = None
try:
user_info = data.get('attributes', None)
user = DimensionUser.insert(
internal_user_id=data.get('internal_user_id',
data.get('user_id', '')),
external_user_id=data.get('external_user_id', ''),
territory=territory,
client=client,
email=data.get('email', None),
first_name=user_info.get('first_name', '')
if user_info else '',
last_name=user_info.get('last_name', '') if user_info else '',
marketing_preference=user_info.get('marketing_preference', '')
if user_info else '',
country_of_residence=user_info.get('country_of_residence', '')
if user_info else None)
except Exception:
user = None
try:
account = DimensionAccount.insert(
account_id=data.get('external_user_id',
data.get('internal_user_id', '')),
account_created=data.get('account_created', None))
except:
account = None
fact = FactServicesStorefrontRegistration.objects.create(
account=account,
user=user,
client=client,
platform=platform,
device=device,
event_utc_date=DimensionUTCDate.date_from_datetime(
datetime=data['event_time']),
event_utc_datetime=data['event_time'],
mnc=data.get('MNC', None),
mcc=data.get('MCC', None))
try:
FactServicesStorefrontRegistration.objects.get(pk=fact.pk)
exist = True
except FactServicesStorefrontRegistration.DoesNotExist:
exist = False
logs.completed = exist
logs.save()
def create_fact(cls, logs, **data):
# insert dims
item = DimensionItem.insert(
item_id=data.get('item_id')) if exists('item_id', data) else None
product = DimensionProduct.insert(
product_id=data.get('product_id')) if exists('product_id',
data) else None
right = DimensionRight.insert(right_id=data.get('right_id')) if exists(
'right_id', data) else None
currency = DimensionCurrency.insert(
code=data.get('currency')) if exists('currency', data) else None
retail_model = DimensionRetailModel.insert(
model=data.get('retail_model')) if exists('retail_model',
data) else None
definition = DimensionDefinition.insert(
definition=data.get('definition')) if exists('definition',
data) else None
window = DimensionWindow.insert(window_id=data.get('window_id', None),
product=product,
usage_right=right,
item=item)
transaction_status = DimensionStoreTransactionStatus.insert(
status=data.get('transaction_status')) if exists(
'transaction_status', data) else None
platform = save_platform(data, event='Storefront Transaction')
territory = DimensionTerritory.insert(
code=data.get('territory')) if exists('territory', data) else None
client = DimensionClient.insert(
client_id=data.get('client_id')) if exists('client_id',
data) else None
user = save_user(data,
territory,
client,
event='Storefront Transaction')
account = DimensionAccount.insert(account_id=data.get(
'account_id',
data.get('external_user_id', data.get('internal_user_id', None))))
try:
device = DimensionDevice.insert(
make=data.get('make', ''),
model=data.get('model', ''),
os=data.get('os', ''),
os_version=data.get('version', ''),
) if any([d in data
for d in ['make', 'model', 'os', 'version']]) else None
except:
log.exception('')
try:
trans = cls.objects.get(
transaction_id=data.get('transaction_id', ''))
trans.last_4_digits = data.get('last_4_digits', None)
trans.event_utc_datetime = data.get('event_time',
timezone.now().isoformat())
trans.event_utc_date = DimensionUTCDate.date_from_datetime(
data.get('event_time',
timezone.now().isoformat()))
trans.save()
except cls.DoesNotExist as e:
trans = cls.objects.create(
item=item,
product=product,
window=window,
transaction_id=data.get('transaction_id', None),
transaction_status=transaction_status,
user=user,
account=account,
right=right,
price=data.get('price', 0.00),
currency=currency,
retail_model=retail_model,
definition=definition,
client=client,
territory=territory,
platform=platform,
device=device,
mnc=data.get('MNC', None),
mcc=data.get('MCC', None),
last_4_digits=data.get('last_4_digits', None),
event_utc_datetime=data.get('event_time',
timezone.now().isoformat()),
event_utc_date=DimensionUTCDate.date_from_datetime(
data.get('event_time',
timezone.now().isoformat())),
)
try:
cls.objects.get(pk=trans.pk)
exist = True
except cls.DoesNotExist:
exist = False
logs.completed = exist
logs.save()
def test_initial_ingestion(self):
# clear out
FactServicesStorefrontSubscription.objects.all().delete()
Flags.objects.all().delete()
# sample uuids
i_uid = str(uuid4())
e_uid = str(uuid4())
c_id = str(uuid4())
sub_id = str(uuid4())
win_id = str(uuid4())
item_id = str(uuid4())
prod_id = str(uuid4())
win_price_id = str(uuid4())
win_usage_right_id = str(uuid4())
apple_prod_id = str(uuid4())
tran_id = str(uuid4())
win_start = datetime.datetime(year=2013, month=1, day=1).strftime("%Y-%m-%d")
win_end = datetime.datetime(year=2030, month=1, day=1).strftime("%Y-%m-%d")
# dict structure for out mock event
data = {
'window_id': win_id,
'item_id': item_id,
'product_id': prod_id,
'window_start_date': win_start,
'window_end_date': win_end,
'window_pricing_id': win_price_id,
'window_usage_right_id': win_usage_right_id,
'window_tier': 'TEST',
'window_deleted_date': '', # we wont necessarily have this more likely not
'window_type': 'master',
'window_on_going': 1,
'window_repeat_count': 1,
'window_auto_upgrade': 1,
'window_allow_repurchase': 1,
'apple_product_id': apple_prod_id,
'transaction_id': tran_id,
'internal_user_id': i_uid,
'external_user_id': e_uid,
'territory': 'GB',
'client_id': c_id,
'subscription_id': sub_id,
'subscription_period': 'P1M',
'subscription_recurrence': '1',
'subscription_status': 0, # inactive
'platform_os': 'iOS',
'platform_name': 'test-ios-platform',
'platform_version': '7.1.0',
'event_time': str(when),
}
# json encode our data string to replicate ingestion of event
data_string = json.dumps(data)
# push to event method
user_subscription(data_string)
# have we injected?
subscriptions = FactServicesStorefrontSubscription.objects.all()
client = DimensionClient.objects.get(client_id=c_id)
platform = DimensionPlatform.objects.get(
os='iOS',
name='test-ios-platform',
version='7.1.0'
)
state = DimensionSubscriptionState.objects.get(state=0)
status = DimensionSubscriptionStatus.objects.get(id=0)
sub_type = DimensionSubscriptionType.objects.get(period='P1M', recurrence=1)
territory = DimensionTerritory.objects.get(code='GB')
utc_date = DimensionUTCDate.date_from_datetime(datetime=str(when))
user = DimensionUser.objects.get(
internal_user_id=i_uid,
external_user_id=e_uid,
territory=territory,
client=client
)
# lets check to see if we have our subscription
assert_that(subscriptions, has_length(1))
# is it.. our subscription?
assert_that(
subscriptions[0],
has_properties(
subscription_id=sub_id,
user_id=user.id,
platform=platform,
subscription_type=sub_type,
subscription_status=status,
subscription_state=state,
event_utc_date=utc_date,
event_utc_datetime=when
)
)
# as a bonus feature...
# we created a non-registered client.. we should have flagged this error...
# we also created a non-registered usage right.. we should have flagged this error too...
# and lastly we also created a non-registered product.. we will also flag this.
# fetch the flags
flags = Flags.objects.all()
# do we have them? we should have 3 flags
assert_that(flags, has_length(3))
def test_initial_injestion(self):
# clear out
FactServicesStorefrontSubscription.objects.all().delete()
Flags.objects.all().delete()
# sample uuids
i_uid = str(uuid4())
e_uid = str(uuid4())
c_id = str(uuid4())
sub_id = str(uuid4())
win_id = str(uuid4())
item_id = str(uuid4())
prod_id = str(uuid4())
win_price_id = str(uuid4())
win_usage_right_id = str(uuid4())
apple_prod_id = str(uuid4())
tran_id = str(uuid4())
win_start = datetime.datetime(year=2013, month=1,
day=1).strftime("%Y-%m-%d")
win_end = datetime.datetime(year=2030, month=1,
day=1).strftime("%Y-%m-%d")
# dict structure for out mock event
data = {
'window_id': win_id,
'item_id': item_id,
'product_id': prod_id,
'window_start_date': win_start,
'window_end_date': win_end,
'window_pricing_id': win_price_id,
'window_usage_right_id': win_usage_right_id,
'window_tier': 'TEST',
'window_deleted_date':
'', # we wont necessarily have this more likely not
'window_type': 'master',
'window_on_going': 1,
'window_repeat_count': 1,
'window_auto_upgrade': 1,
'window_allow_repurchase': 1,
'apple_product_id': apple_prod_id,
'transaction_id': tran_id,
'internal_user_id': i_uid,
'external_user_id': e_uid,
'territory': 'GB',
'client_id': c_id,
'subscription_id': sub_id,
'subscription_period': 'P1M',
'subscription_recurrence': '1',
'subscription_status': 0, # inactive
'platform_os': 'iOS',
'platform_name': 'test-ios-platform',
'platform_version': '7.1.0',
'event_time': str(when),
}
# json encode our data string to replicate ingestion of event
data_string = json.dumps(data)
# push to event method
user_subscription(data_string)
# have we injected?
subscriptions = FactServicesStorefrontSubscription.objects.all()
client = DimensionClient.objects.get(client_id=c_id)
platform = DimensionPlatform.objects.get(os='iOS',
name='test-ios-platform',
version='7.1.0')
state = DimensionSubscriptionState.objects.get(state=0)
status = DimensionSubscriptionStatus.objects.get(id=0)
sub_type = DimensionSubscriptionType.objects.get(period='P1M',
recurrence=1)
territory = DimensionTerritory.objects.get(code='GB')
utc_date = DimensionUTCDate.date_from_datetime(datetime=str(when))
user = DimensionUser.objects.get(internal_user_id=i_uid,
external_user_id=e_uid,
territory=territory,
client=client)
# lets check to see if we have our subscription
assert_that(subscriptions, has_length(1))
# is it.. our subscription?
assert_that(
subscriptions[0],
has_properties(subscription_id=sub_id,
user_id=user.id,
platform=platform,
subscription_type=sub_type,
subscription_status=status,
subscription_state=state,
event_utc_date=utc_date,
event_utc_datetime=when))
# as a bonus feature...
# we created a non-registered client.. we should have flagged this error...
# we also created a non-registered usage right.. we should have flagged this error too...
# and lastly we also created a non-registered product.. we will also flag this.
# fetch the flags
flags = Flags.objects.all()
# do we have them? we should have 3 flags
assert_that(flags, has_length(3))
