def load_product_partition_data(
session: Session, adgroup_ids: Tuple = ()):
"""Load ad group data from account db."""
q = ProductPartition.select(
ProductPartition.criterion_id,
ProductPartition.adgroup_id,
ProductPartition.dimension_type,
ProductPartition.dimension_value,
ProductPartition.partition_type,
ProductPartition.parent_id)
if adgroup_ids:
q = q.where(ProductPartition.adgroup_id.in_(adgroup_ids))
inserts = ['insert']
tpl = ('$pp isa ProductPartition, '
'has criterion-id {}, '
'has adgroup-id {}, '
'has partition-type {} '
'has parent-id {};')
ids = []
for r in q.dicts():
dt = r.pop('dimension_type')
dv = r.pop('dimension_value')
q = tpl.format(*r.values())
inserts.append(q)
with session.transaction().write() as tx:
pp = tx.get_schema_concept('ProductPartition').create()
for k, v in r.items():
attr = tx.get_schema_concept(k.replace('_', '-'))
pp.has(attr.create(v))
ids.append(pp.id)
tx.commit()
if dt:
with session.transaction().write() as tx:
it = tx.query(
'match $x isa ProductDimension, '
f'has dimension-type "{dt}"; get;')
pd = next(iter(it.collect_concepts()), None)
if not pd:
pd = tx.get_schema_concept('ProductDimension').create()
pd.has(tx.get_schema_concept('dimension-type').create(dt))
dv = tx.get_schema_concept('dimension-value').create(dv)
cv = tx.get_schema_concept('case-value').create()
cv.has(dv)
cv.assign(tx.put_role('product-dimension'), pd)
cv.assign(tx.put_role(
'product-partition'), tx.get_concept(ids[-1]))
tx.commit()
else:
query = '\n'.join(inserts)
print(query)
log.info('Inserted {} Product Partitions.'.format(len(ids)))
def define_node_heirarchy_relation(session: Session):
"""A basic heirarchical relationship.
define
parent-child sub relationship,
relates parent,
relates child,
plays ancestor,
plays descendent;
ancestorship sub relationship,
relates ancestor,
relates descedent;
"""
with session.transaction().write() as tx:
rel = tx.put_relation_type('node-heirarchy')
id = rel.id
rel.relates(tx.put_role('parent-node'))
rel.relates(tx.put_role('child-node'))
rel.plays(tx.put_role('ancestor'))
rel.plays(tx.put_role('descedent'))
tx.commit()
return id
def define_case_value_relation(session: Session):
"""Relate Product Partitions to Product Dimensions.
define
dimension-value sub attribute, datatype string;
case-value sub relationship,
relates product-dimension,
relates product-partition,
has dimension-value;
"""
with session.transaction().write() as tx:
rel = tx.put_relation_type('case-value')
id = rel.id
rel.has(tx.put_attribute_type('dimension-value', DataType.STRING))
rel.relates(tx.put_role('product-dimension'))
rel.relates(tx.put_role('product-partition'))
tx.commit()
return id
def define_product_partition_entity(session: Session):
"""ProductPartition entity
define
parent-id sub criterion-id, datatype long;
partition-type sub attribute, datatype string;
ProductPartition sub criterion,
has adgroup-id,
has parent-id,
has partition-type,
plays product-partition,
plays parent;
"""
with session.transaction().write() as tx:
ent = tx.put_entity_type('ProductPartition')
ent.sup(tx.get_schema_concept('Criterion'))
ent.has(tx.put_attribute_type('adgroup-id', DataType.LONG))
ent.has(tx.put_attribute_type('parent-id', DataType.LONG))
ent.has(tx.put_attribute_type('partition-type', DataType.STRING))
ent.plays(tx.put_role('product-partition'))
ent.plays(tx.put_role('parent-node'))
ent.plays(tx.put_role('child-node'))
id = ent.id
tx.commit()
return id
def define_campaign_entity(session: Session):
"""Campaign entity.
define
campaign-id sub attribute, datatype long;
campaign-name sub attribute, datatype string;
aw-campaign-type sub attribute, datatype string;
Campaign sub entity,
has campaign-id,
has campaign-name,
has aw-campaign-type,
has status,
plays campaign;
"""
with session.transaction().write() as tx:
entity = tx.put_entity_type('Campaign')
# attributes
entity.has(tx.put_attribute_type('campaign-id', DataType.LONG))
entity.has(tx.put_attribute_type('campaign-name', DataType.STRING))
entity.has(tx.put_attribute_type('status', DataType.STRING))
entity.has(tx.put_attribute_type('aw-campaign-type', DataType.STRING))
# roles
entity.plays(tx.put_role('campaign'))
id = entity.id
tx.commit()
return id
def define_entity_product(session: Session):
"""Product entity.
define
item-id sub attribute, datatype string;
title sub attribute, datatype string;
Product sub entity,
has item-id,
has title,
plays product;
"""
with session.transaction().write() as tx:
entity = tx.put_entity_type('Product')
entity.has(tx.put_attribute_type('item-id', DataType.STRING))
entity.has(tx.put_attribute_type('title', DataType.STRING))
# entity.plays(tx.put_role('product'))
id = entity.id
tx.commit()
return id
def define_sibling_relation(session: Session):
with session.transaction().write() as tx:
rel = tx.put_relation_type('siblings')
id = rel.id
rel.relates(tx.put_role('child-node'))
tx.commit()
return id
def define_ancestorship_relation(session: Session):
"""Relate two node heirarchies."""
with session.transaction().write() as tx:
rel = tx.put_relation_type('ancestorship')
id = rel.id
rel.relates(tx.put_role('ancestor'))
rel.relates(tx.put_role('descedent'))
tx.commit()
return id
def put_common_attributes(session: Session):
"""Colection of often used attributes.
define
status sub attribute, datatype string;
"""
with session.transaction().write() as tx:
tx.put_attribute_type('name', DataType.STRING)
tx.put_attribute_type('status', DataType.STRING)
tx.put_attribute_type('value', DataType.STRING)
tx.commit()
def describe_concept_type(session: Session, concept_id):
with session.transaction().read() as tx:
concept = tx.get_concept(concept_id)
if concept.is_thing():
concept = concept.type()
label = concept.label()
if concept.is_entity_type():
thing_type = 'an entity'
elif concept.is_attribute_type():
thing_type = 'an attribute'
elif concept.is_relation_type():
thing_type = 'a relation'
elif concept.is_rule():
thing_type = 'an inference rule'
else:
thing_type = 'Unknown'
attrs = keys = roles = []
if hasattr(concept, 'keys'):
keys = [k.label() for k in concept.keys()]
if hasattr(concept, 'attributes'):
attrs = [a.label() for a in concept.attributes()]
if hasattr(concept, 'playing'):
roles = [
r.label() for r in concept.playing()
if not r.label().startswith('@')
]
print(f'\n{label} with id {concept_id} is {thing_type}')
if concept.is_rule():
print('\nRule definition:')
print(f'\twhen: {concept.get_when()}')
print(f'\tthen: {concept.get_then()}\n\n')
else:
if keys:
print('\nKeys: ', end='')
pprint.pprint(keys)
print('\nAttributes: ', end='')
fattrs = '\n' + pprint.pformat(attrs) if attrs else 'None'
print(fattrs)
print('Roles: ', end='')
froles = '\n' + pprint.pformat(roles) if roles else 'None'
print(froles, end='\n' + str('-' * 79) + '\n')
tx.close()
def define_offer_relationship(session: Session):
q = """
define
product-offer sub relation,
relates product,
relates ad-group;
"""
with session.transaction().write() as tx:
tx.query(q)
id = tx.get_schema_concept('product-offer').id
tx.commit()
return id
def define_subdivision_relation(session: Session):
with session.transaction().write() as tx:
log.info('adding relationship "subdivision"')
rel = tx.put_relation_type('subdivision')
rel.relates(tx.put_role('parent'))
rel.relates(tx.put_role('dimension-value'))
# rel.relates(tx.put_role('product-partition'))
rel.has(tx.put_attribute_type('dimension-type', DataType.STRING))
tx.get_schema_concept('value').plays(tx.put_role('dimension-value'))
id = rel.id
tx.commit()
return id
def define_node_adjacency_rule(session: Session):
q = """define
node-adjacency sub rule,
when {
(parent-node: $p, $x) isa node-heirarchy;
(parent-node: $p, $y) isa node-heirarchy;
$x != $y;
}, then {
($x, $y) isa siblings;
};"""
with session.transaction().write() as tx:
tx.query(q)
id = tx.get_schema_concept('node-adjacency').id
tx.commit()
return id
def define_transitive_ancestorship_rule(session: Session):
q = """define
transitive-ancestorship sub rule,
when {
$r1 (parent-node: $a, child-node: $p) isa node-heirarchy;
$r2 (parent-node: $p, child-node: $c) isa node-heirarchy;
$a isa ProductPartition;
$p isa ProductPartition;
$c isa ProductPartition;
}, then {
(ancestor: $r1, descedent: $r2) isa ancestorship;
};"""
with session.transaction().write() as tx:
tx.query(q)
id = tx.get_schema_concept('transitive-ancestorship').id
tx.commit()
return id
def define_infer_node_hierarchy_rule(session: Session):
q = """define
infer-node-heirarchy sub rule,
when {
$parent isa ProductPartition, has criterion-id $x, has adgroup-id $a-id;
$child isa ProductPartition, has parent-id $y, has adgroup-id $a-id;
$x == $y;
$parent != $child;
}, then {
(parent-node: $parent, child-node: $child) isa node-heirarchy;
};
"""
with session.transaction().write() as tx:
tx.query(q)
id = tx.get_schema_concept('infer-node-heirarchy').id
tx.commit()
return id
def define_campaign_adgroup_relation(session: Session):
"""AdGroup in Campaign.
define
campaign-adgroup sub relation,
relates campaign,
relates adgroup;
"""
with session.transaction().write() as tx:
rel = tx.put_relation_type('campaign-adgroup')
id = rel.id
rel.relates(tx.put_role('campaign'))
rel.relates(tx.put_role('adgroup'))
tx.commit()
return id
def define_adgroup_criterion_relation(session: Session):
"""Criterion in AdGroup.
define
adgroup-criterion sub relation,
relates adgroup,
relates biddable-criterion;
"""
with session.transaction().write() as tx:
rel = tx.put_relation_type('adgroup-criterion')
id = rel.id
rel.relates(tx.put_role('adgroup'))
rel.relates(tx.put_role('biddable-criterion'))
tx.commit()
return id
def define_adgroup_entity(session: Session):
"""AdGroup entity.
define
adgroup-id sub attribute, datatype long;
adgroup-name sub attribute, datatype string;
aw-adgroup-type sub attribute, datatype long;
AdGroup sub entity,
has adgroup-id,
has campaign-id,
has adgroup-name,
has status,
has aw-adgroup-type,
plays parent
plays child
plays adgroup;
"""
with session.transaction().write() as tx:
entity = tx.put_entity_type('AdGroup')
# attributes
entity.has(tx.put_attribute_type('adgroup-id', DataType.LONG))
entity.has(tx.put_attribute_type('campaign-id', DataType.LONG))
entity.has(tx.put_attribute_type('adgroup-name', DataType.STRING))
entity.has(tx.put_attribute_type('status', DataType.STRING))
entity.has(tx.put_attribute_type('aw-adgroup-type', DataType.STRING))
# roles
entity.plays(tx.put_role('adgroup'))
id = entity.id
tx.commit()
return id
def load_product_data(session: Session, adgroup_ids: Tuple = ()):
"""Product Data."""
q = AdwordsOffer.select(
AdwordsOffer.adgroup_id,
AdwordsOffer.item_id,
ProductDimension.dimension_type,
ProductDimension.dimension_value)
q = q.join(
ProductDimension, JOIN.LEFT_OUTER,
on=(AdwordsOffer.item_id == ProductDimension.item_id))
if adgroup_ids:
q = q.where(AdwordsOffer.adgroup_id.in_(adgroup_ids))
ids = []
for row in q.dicts():
with session.transaction().write() as tx:
attr = tx.put_attribute_type('item-id', DataType.STRING)
e = tx.put_entity_type('Product').create()
e.has(attr.create(row['item_id']))
ids.append(e.id)
tx.commit()
def define_entity_product_dimension(session: Session):
"""Product Dimension entity.
define
dimension-type sub attribute, datatype string;
ProductDimension sub entity,
has dimension-type,
plays product-dimension;
"""
with session.transaction().write() as tx:
entity = tx.put_entity_type('ProductDimension')
id = entity.id
entity.has(tx.put_attribute_type('dimension-type', DataType.STRING))
entity.plays(tx.put_role('product-dimension'))
tx.commit()
return id
def load_adgroup_data(
session: Session,
limit: int = 0,
adgroup_types: Tuple = (),
include_paused: bool = False):
"""Load ad group data from account db."""
q = AdGroup.select(
AdGroup.adgroup_id,
AdGroup.adgroup_name,
AdGroup.campaign_id,
AdGroup.status,
AdGroup.aw_adgroup_type)
q = q.where(AdGroup.status == 'Active')
if adgroup_types:
q = q.where(AdGroup.aw_adgroup_type.in_(adgroup_types))
if limit:
q = q.limit(limit)
ids = []
for r in q.dicts():
# keep entity references consistent
r['campaign_id'] = r['campaign']
del r['campaign']
with session.transaction().write() as tx:
adgroup = tx.get_schema_concept('AdGroup').create()
ids.append(adgroup.id)
for k, v in r.items():
try:
attr = tx.get_schema_concept(k.replace('_', '-'))
adgroup.has(attr.create(v))
except TypeError:
pass
tx.commit()
log.info('Inserted {} Ad Groups.'.format(len(ids)))
def define_abstract_criterion_entity(session: Session):
"""Criterion entity.
define
criterion-id sub attribute, datatype long;
criterion-name sub attribute, datatype long;
crit-key sub attribute, datatype long;
Criterion sub entity is-abstract,
has adgroup-id,
has criterion-id,
has criterion-name,
has crit-key,
has status,
plays biddable-criterion;
"""
with session.transaction().write() as tx:
criterion = tx.put_entity_type('Criterion')
criterion.is_abstract(True)
id = criterion.id
# attributes
criterion.has(tx.put_attribute_type('adgroup-id', DataType.LONG))
criterion.has(tx.put_attribute_type('criterion-id', DataType.LONG))
criterion.has(tx.put_attribute_type('crit-key', DataType.LONG))
criterion.has(tx.put_attribute_type('criterion-name', DataType.STRING))
criterion.has(tx.put_attribute_type('status', DataType.STRING))
# roles
criterion.plays(tx.put_role('biddable-criterion'))
tx.commit()
return id
def define_adgroup_in_campaign_rule(session: Session):
"""Infer campaign-adgroup relations via matching campaign-id."""
q = """define
adgroup-in-campaign sub rule,
when {
$c isa Campaign, has campaign-id $c-id;
$a isa AdGroup, has campaign-id $c-id;
}, then {
(campaign: $c, adgroup: $a) isa campaign-adgroup;
};"""
with session.transaction().write() as tx:
when = ('$c isa Campaign, has campaign-id $c-id; '
'$a isa AdGroup, has campaign-id $c-id;')
then = '(campaign: $c, adgroup: $a) isa campaign-adgroup;'
tx.query(q)
rule = tx.get_schema_concept('adgroup-in-campaign')
id = rule.id
tx.commit()
return id
def define_criterion_in_adgroup_rule(session: Session):
"""Infer adgroup-criterion relations via matching adgroup-id."""
q = """define
criterion-in-adgroup sub rule,
when {
$a isa AdGroup, has adgroup-id $a-id;
$c isa Criterion, has adgroup-id $a-id;
}, then {
(adgroup: $a, biddable-criterion: $c) isa adgroup-criterion;
};"""
with session.transaction().write() as tx:
when = ('$adgroup isa AdGroup, has adgroup-id $adgroup-id; '
'$criterion isa Criterion, has adgroup-id $adgroup-id;')
then = ('(adgroup: $adgroup, biddable-criterion: $criterion) '
'isa campaign-adgroup;')
tx.query(q)
rule = tx.get_schema_concept('adgroup-in-campaign')
id = rule.id
tx.commit()
return id