def _update_company_record(self, domain, _id):
start_time = time.time()
print "UPDATE COMPANY RECORD"
#conn = r.connect(host="localhost", port=28015, db="triggeriq")
conn = rethink_conn.conn()
print domain
#company= [i for i in r.table('companies').filter({"domain":domain}).run(conn)]
company = []
print "COMPANY FOUND"
# TODO - wtf is result and why is it included
if not company:
company = clearbit.Company.find(domain=domain, stream=True)
company = company if company else {}
print company
r.table('companies').insert(company).run(conn)
result = "found"
else:
result = "not found"
data = {"company_domain_research_completed":r.now(),
"company_domain_research_result": result}
r.table('triggers').get(_id).update(data).run(conn)
bitmapist.mark_event("function:time:clearbit_search_company_record",
int((time.time() - start_time)*10**6))
rd.zadd("function:time:clearbit_search_company_record",
str((time.time() - start_time)*10**6),
arrow.now().timestamp)
def mark_user_event(user_event, user_id, event_date):
# to abstract out naming convention, easily prevent future events, and debug
if event_date > now.date():
raise TimeTravelError('date has not yet come to pass (%s)' % event_date)
event_name = 'user:%s' % user_event
mark_event(event_name, user_id, now=event_date)
def _signal(self, qry, locale, profile, country=None):
page = 1
start_time = time.time()
print "Simply Hired"
html = self._html(qry, page, locale, country)
listings = self._listings(html)
#print listings
if listings.empty: return "none found"
while 'day' not in listings.date.tolist()[-1]:
page = page + 1
html = self._html(qry, page, locale, country)
listings = listings.append(self._listings(html))
print page
listings = listings[~listings.date.str.contains('day')]
listings["keyword"] = qry
listings = listings.drop_duplicates('company_name')
listings['source'] = 'Simply Hired'
listings["profile"] = profile
#print listings
companies = listings
keys = [row.company_name.lower().replace(" ","")+"_"+profile for i, row in companies.iterrows()]
companies["company_key"] = keys
companies["createdAt"] = arrow.now().timestamp
conn = rethink_conn.conn()
#r.table("hiring_signals").insert(companies.to_dict('r')).run(conn)
r.table("triggers").insert(companies.to_dict('r')).run(conn)
bitmapist.mark_event("function:time:simplyhired_job_scrape",
int((time.time() - start_time)*10**6))
rd.zadd("function:time:simplyhired_job_scrape",
str((time.time() - start_time)*10**6),
arrow.now().timestamp)
def test_mark_with_diff_days():
delete_all_events()
mark_event('active', 123, track_hourly=True)
now = datetime.utcnow()
# Month
assert 123 in MonthEvents('active', now.year, now.month)
assert 124 not in MonthEvents('active', now.year, now.month)
# Week
assert 123 in WeekEvents('active', now.year, now.isocalendar()[1])
assert 124 not in WeekEvents('active', now.year, now.isocalendar()[1])
# Day
assert 123 in DayEvents('active', now.year, now.month, now.day)
assert 124 not in DayEvents('active', now.year, now.month, now.day)
# Hour
assert 123 in HourEvents('active', now.year, now.month, now.day, now.hour)
assert 124 not in HourEvents('active', now.year, now.month, now.day,
now.hour)
assert 124 not in HourEvents('active', now.year, now.month, now.day,
now.hour - 1)
def _signal(self, qry, locale, profile, country=None):
start_time = time.time()
html = self._html(qry, 1, locale, country)
listings = self._listings(html)
last_page = html.find('ul',{'class':'paginationNumbers'})
last_page = last_page.find_all('li') if last_page else None
last_page = int(last_page[-1].text.strip()) if last_page else 1
conn = rethink_conn.conn()
for page in range(last_page):
html = self._html(qry, page, locale, country)
#listings = listings.append(self._listings(html))
listings = self._listings(html)
listings['source'] = 'Zip Recruiter'
listings["keyword"] = qry
listings["profile"] = profile
companies = listings
keys = [row.company_name.lower().replace(" ","")+"_"+profile for i, row in companies.iterrows()]
companies["company_key"] = keys
companies["createdAt"] = arrow.now().timestamp
#r.table("hiring_signals").insert(companies.to_dict('r')).run(conn)
r.table("triggers").insert(companies.to_dict('r')).run(conn)
bitmapist.mark_event("function:time:ziprecruiter_job_scrape",
int((time.time() - start_time)*10**6))
#HiringSignal()._persist(listings, profile, report)
rd.zadd("function:time:ziprecruiter_job_scrape",
str((time.time() - start_time)*10**6),
arrow.now().timestamp)
def test_mark_unmark():
now = datetime.utcnow()
mark_event('active', 125)
assert 125 in MonthEvents('active', now.year, now.month)
unmark_event('active', 125)
assert 125 not in MonthEvents('active', now.year, now.month)
def test_mark_unmark():
now = datetime.utcnow()
mark_event('active', 125)
assert 125 in MonthEvents('active', now.year, now.month)
unmark_event('active', 125)
assert 125 not in MonthEvents('active', now.year, now.month)
def test_different_buckets():
now = datetime.utcnow()
mark_event('active', 123)
mark_event('tasks:completed', 23232)
assert MonthEvents('active', now.year, now.month).get_count() == 1
assert MonthEvents('tasks:completed', now.year, now.month).get_count() == 1
def test_different_buckets():
now = datetime.utcnow()
mark_event('active', 123)
mark_event('tasks:completed', 23232)
assert MonthEvents('active', now.year, now.month).get_count() == 1
assert MonthEvents('tasks:completed', now.year, now.month).get_count() == 1
def test_mark_counts():
now = datetime.utcnow()
assert MonthEvents('active', now.year, now.month).get_count() == 0
mark_event('active', 123)
mark_event('active', 23232)
assert len(MonthEvents('active', now.year, now.month)) == 2
def test_mark_counts():
now = datetime.utcnow()
assert MonthEvents('active', now.year, now.month).get_count() == 0
mark_event('active', 123)
mark_event('active', 23232)
assert len(MonthEvents('active', now.year, now.month)) == 2
def test_different_buckets():
delete_all_events()
now = datetime.utcnow()
mark_event("active", 123)
mark_event("tasks:completed", 23232)
assert MonthEvents("active", now.year, now.month).get_count() == 1
assert MonthEvents("tasks:completed", now.year, now.month).get_count() == 1
def test_events_marked():
now = datetime.utcnow()
assert MonthEvents('active', now.year, now.month).get_count() == 0
assert MonthEvents('active', now.year, now.month).has_events_marked() == False
mark_event('active', 123, now=now)
assert MonthEvents('active', now.year, now.month).get_count() == 1
assert MonthEvents('active', now.year, now.month).has_events_marked() == True
def test_different_dates():
now = datetime.utcnow()
yesterday = now - timedelta(days=1)
mark_event('active', 123, now=now)
mark_event('active', 23232, now=yesterday)
assert DayEvents('active', now.year, now.month, now.day).get_count() == 1
assert DayEvents('active', yesterday.year, yesterday.month,
yesterday.day).get_count() == 1
def mark_event(self, event_name, uuid, system='default', now=None, track_hourly=None):
"""
"""
if event_name not in self.story_dict:
self.story_dict[event_name]=True
# mark event in bitmapist
mark_event(event_name, uuid, system=system, now=now, track_hourly=track_hourly)
# update most popular event
self.update_most_popular_event(event_name,system)
def test_mark_counts():
delete_all_events()
now = datetime.utcnow()
assert MonthEvents("active", now.year, now.month).get_count() == 0
mark_event("active", 123)
mark_event("active", 23232)
assert len(MonthEvents("active", now.year, now.month)) == 2
def test_events_marked():
delete_all_events()
now = datetime.utcnow()
assert MonthEvents('active', now.year, now.month).get_count() == 0
assert MonthEvents('active', now.year, now.month).has_events_marked() == False
mark_event('active', 123, now=now)
assert MonthEvents('active', now.year, now.month).get_count() == 1
assert MonthEvents('active', now.year, now.month).has_events_marked() == True
def test_different_dates():
delete_all_events()
now = datetime.utcnow()
yesterday = now - timedelta(days=1)
mark_event("active", 123, now=now)
mark_event("active", 23232, now=yesterday)
assert DayEvents("active", now.year, now.month, now.day).get_count() == 1
assert DayEvents("active", yesterday.year, yesterday.month, yesterday.day).get_count() == 1
def test_events_marked():
delete_all_events()
now = datetime.utcnow()
assert MonthEvents("active", now.year, now.month).get_count() == 0
assert MonthEvents("active", now.year, now.month).has_events_marked() == False
mark_event("active", 123, now=now)
assert MonthEvents("active", now.year, now.month).get_count() == 1
assert MonthEvents("active", now.year, now.month).has_events_marked() == True
def test_is_user_active(self):
userid = '{F07B3878-CD6F-4B96-B52F-95C4D23077E0}'
id = get_id(userid)
self.assertTrue(is_user_active(ACTIVE_USERS_DICT_CHOICES['all'], userid))
self.assertFalse(is_user_active(ACTIVE_USERS_DICT_CHOICES['week'], userid))
self.assertFalse(is_user_active(ACTIVE_USERS_DICT_CHOICES['month'], userid))
mark_event('request', id)
self.assertTrue(is_user_active(ACTIVE_USERS_DICT_CHOICES['all'], userid))
self.assertTrue(is_user_active(ACTIVE_USERS_DICT_CHOICES['week'], userid))
self.assertTrue(is_user_active(ACTIVE_USERS_DICT_CHOICES['month'], userid))
def test_is_user_active(self):
userid = "{F07B3878-CD6F-4B96-B52F-95C4D23077E0}"
id = get_id(userid)
self.assertTrue(is_user_active(ACTIVE_USERS_DICT_CHOICES["all"], userid))
self.assertFalse(is_user_active(ACTIVE_USERS_DICT_CHOICES["week"], userid))
self.assertFalse(is_user_active(ACTIVE_USERS_DICT_CHOICES["month"], userid))
mark_event("request", id)
self.assertTrue(is_user_active(ACTIVE_USERS_DICT_CHOICES["all"], userid))
self.assertTrue(is_user_active(ACTIVE_USERS_DICT_CHOICES["week"], userid))
self.assertTrue(is_user_active(ACTIVE_USERS_DICT_CHOICES["month"], userid))
def test_is_user_active(self):
userid = '{F07B3878-CD6F-4B96-B52F-95C4D23077E0}'
id = get_id(userid)
self.assertTrue(is_user_active(ACTIVE_USERS_DICT_CHOICES['all'], userid))
self.assertFalse(is_user_active(ACTIVE_USERS_DICT_CHOICES['week'], userid))
self.assertFalse(is_user_active(ACTIVE_USERS_DICT_CHOICES['month'], userid))
mark_event('request', id)
self.assertTrue(is_user_active(ACTIVE_USERS_DICT_CHOICES['all'], userid))
self.assertTrue(is_user_active(ACTIVE_USERS_DICT_CHOICES['week'], userid))
self.assertTrue(is_user_active(ACTIVE_USERS_DICT_CHOICES['month'], userid))
def collect_statistics(request, appid, channel, now=None):
deviceID = request.GET.get('deviceID')
version = request.GET.get('appVersionShort')
if not deviceID or not version:
return
userid = get_id(deviceID)
app = dict(appid=appid,
version=version,
tag=channel)
mark_event('request', userid, now=now)
add_app_statistics(userid, 'mac', app)
update_live_statistics(userid, appid, version)
def save(self, *args, **kwargs):
created = self.pk is None
super(Account, self).save(*args, **kwargs)
setup_redis(
'engine_redis',
settings.REDIS_HOSTNAME,
port=settings.REDIS_PORT,
password=settings.REDIS_PASSWORD)
if created:
mark_event('account:created', self.id, system='engine_redis')
else:
mark_event('account:updated', self.id, system='engine_redis')
def test_ops_with_dates():
bitmapist.mark_event('active', 1)
bitmapist.mark_event('active', 2)
bitmapist.mark_unique('foo', 2)
bitmapist.mark_unique('foo', 3)
foo = bitmapist.UniqueEvents('foo')
active = bitmapist.DayEvents('active')
assert list(foo & active) == [2]
assert list(foo | active) == [1, 2, 3]
assert list(foo & active.prev()) == []
assert list(foo | active.prev()) == [2, 3]
def _bulk_update_employee_record(self, _id, pattern, domain):
start_time = time.time()
conn = r.connect(host="localhost", port=28015, db="triggeriq")
#_id = "eab41007-6b8c-11e5-b7e1-7831c1d137aa"
employees = list(r.table("company_employees").filter({"company_id":_id}).run(conn))
print employees
for person in employees:
#pattern = change["new_val"]["email_pattern"]
_data = {"first":person["name"].split(" ")[0].lower(),
"last":person["name"].split(" ")[-1].lower()}
email = pattern.format(**_data)+"@"+domain
print email, person["id"]
hq.enqueue(ClearbitSearch()._update_person_record, email, person["id"])
bitmapist.mark_event("function:time:bulk_update_employee_record",
int((time.time() - start_time)*10**6))
rd.zadd("function:time:bulk_update_employee_record",
str((time.time() - start_time)*10**6),
arrow.now().timestamp)
def _get_employee_record(self, company_name, _id, keyword=None, profile_id=None):
start_time = time.time()
#conn = r.connect(host="localhost", port=28015, db="triggeriq")
conn = rethink_conn.conn()
res = self._employees(company_name, keyword)
res["company_id"] = _id
res["profile_id"] = profile_id
print "EMPLOYEES FOUND", company_name, res.shape
r.table('company_employees').insert(res.to_dict("r")).run(conn)
epsc = "employee_search_completed"
r.table("triggers").get(_id).update({epsc: r.now()}).run(conn)
bitmapist.mark_event("function:time:company_employee_search",
int((time.time() - start_time)*10**6))
rd.zadd("function:time:company_employee_search",
str((time.time() - start_time)*10**6),
arrow.now().timestamp)
def _generate_fake_statistics(self):
# now = datetime(2016, 2, 13)
date = datetime(2016, 2, 13, 0)
for i in range(self.n_hours):
for id in range(0, i):
mark_event('online:app:win:2.0.0.0', id, now=date, track_hourly=True)
mark_event('online:app:mac:4.0.0.1', id, now=date, track_hourly=True)
for id in range(i, self.n_hours):
mark_event('online:app:win:1.0.0.0', id, now=date, track_hourly=True)
mark_event('online:app:mac:3.0.0.0', id, now=date, track_hourly=True)
date += timedelta(hours=1)
def test_bit_operations_magic():
mark_event('foo', 1)
mark_event('foo', 2)
mark_event('bar', 2)
mark_event('bar', 3)
foo = DayEvents('foo')
bar = DayEvents('bar')
assert list(foo & bar) == [2]
assert list(foo | bar) == [1, 2, 3]
assert list(foo ^ bar) == [1, 3]
assert list(~foo & bar) == [3]
def test_bit_operations_magic():
mark_event('foo', 1)
mark_event('foo', 2)
mark_event('bar', 2)
mark_event('bar', 3)
foo = DayEvents('foo')
bar = DayEvents('bar')
assert list(foo & bar) == [2]
assert list(foo | bar) == [1, 2, 3]
assert list(foo ^ bar) == [1, 3]
assert list(~foo & bar) == [3]
def test_mark_iter():
now = datetime.utcnow()
ev = MonthEvents('active', now.year, now.month)
assert list(ev) == []
mark_event('active', 5)
mark_event('active', 55)
mark_event('active', 555)
mark_event('active', 5555)
assert list(ev) == [5, 55, 555, 5555]
def _generate_fake_statistics(self):
now = datetime(2016, 2, 13)
for i in range(self.n_hours):
date = datetime(now.year, now.month, now.day, i)
for id in range(0, i):
mark_event('online:app:win:2.0.0.0', id, now=date, track_hourly=True)
mark_event('online:app:mac:4.0.0.1', id, now=date, track_hourly=True)
for id in range(i, self.n_hours):
mark_event('online:app:win:1.0.0.0', id, now=date, track_hourly=True)
mark_event('online:app:mac:3.0.0.0', id, now=date, track_hourly=True)
def test_mark_iter():
now = datetime.utcnow()
ev = MonthEvents('active', now.year, now.month)
assert list(ev) == []
mark_event('active', 5)
mark_event('active', 55)
mark_event('active', 555)
mark_event('active', 5555)
assert list(ev) == [5, 55, 555, 5555]
def test_mark_with_diff_days():
mark_event('active', 123, track_hourly=True)
now = datetime.utcnow()
# Month
assert 123 in MonthEvents('active', now.year, now.month)
assert 124 not in MonthEvents('active', now.year, now.month)
# Week
assert 123 in WeekEvents('active', now.year, now.isocalendar()[1])
assert 124 not in WeekEvents('active', now.year, now.isocalendar()[1])
# Day
assert 123 in DayEvents('active', now.year, now.month, now.day)
assert 124 not in DayEvents('active', now.year, now.month, now.day)
# Hour
assert 123 in HourEvents('active', now.year, now.month, now.day, now.hour)
assert 124 not in HourEvents('active', now.year, now.month, now.day, now.hour)
assert 124 not in HourEvents('active', now.year, now.month, now.day, now.hour-1)
def test_get_app_version_active_users(self):
userid = "{%s}" % UUID(int=1)
userid_beta = "{%s}" % UUID(int=40)
userid_beta_not_active = "{%s}" % UUID(int=60)
version = VersionFactory.create(file=SimpleUploadedFile('./chrome_installer.exe', b''))
version_beta = Version.objects.create(
file=SimpleUploadedFile('./chrome_installer.exe', b''),
app=version.app,
platform=version.platform,
channel=version.channel,
version='39.0.0.0',
)
id = get_id(userid_beta)
mark_event('request', id)
PartialUpdate.objects.create(version=version_beta,
percent=5,
start_date=datetime.now(),
end_date=datetime.now())
self.assertEqual(version, get_version(version.app.pk,
version.platform.name,
version.channel.name,
'36.0.2062.124',
userid))
self.assertEqual(version_beta, get_version(version.app.pk,
version.platform.name,
version.channel.name,
'36.0.2062.124',
userid_beta))
self.assertEqual(version, get_version(version.app.pk,
version.platform.name,
version.channel.name,
'36.0.2062.124',
userid_beta_not_active))
def _get_employee_record(self,
company_name,
_id,
keyword=None,
profile_id=None):
start_time = time.time()
#conn = r.connect(host="localhost", port=28015, db="triggeriq")
conn = rethink_conn.conn()
res = self._employees(company_name, keyword)
res["company_id"] = _id
res["profile_id"] = profile_id
print "EMPLOYEES FOUND", company_name, res.shape
r.table('company_employees').insert(res.to_dict("r")).run(conn)
epsc = "employee_search_completed"
r.table("triggers").get(_id).update({epsc: r.now()}).run(conn)
bitmapist.mark_event("function:time:company_employee_search",
int((time.time() - start_time) * 10**6))
rd.zadd("function:time:company_employee_search",
str((time.time() - start_time) * 10**6),
arrow.now().timestamp)
def test_mark_with_diff_days():
delete_all_events()
mark_event("active", 123)
now = datetime.utcnow()
# Month
assert 123 in MonthEvents("active", now.year, now.month)
assert 124 not in MonthEvents("active", now.year, now.month)
# Week
assert 123 in WeekEvents("active", now.year, now.isocalendar()[1])
assert 124 not in WeekEvents("active", now.year, now.isocalendar()[1])
# Day
assert 123 in DayEvents("active", now.year, now.month, now.day)
assert 124 not in DayEvents("active", now.year, now.month, now.day)
# Hour
assert 123 in HourEvents("active", now.year, now.month, now.day, now.hour)
assert 124 not in HourEvents("active", now.year, now.month, now.day, now.hour)
assert 124 not in HourEvents("active", now.year, now.month, now.day, now.hour - 1)
def _signal(self, qry, locale, profile, country=None):
page = 1
start_time = time.time()
print "Simply Hired"
html = self._html(qry, page, locale, country)
listings = self._listings(html)
#print listings
if listings.empty: return "none found"
while 'day' not in listings.date.tolist()[-1]:
page = page + 1
html = self._html(qry, page, locale, country)
listings = listings.append(self._listings(html))
print page
listings = listings[~listings.date.str.contains('day')]
listings["keyword"] = qry
listings = listings.drop_duplicates('company_name')
listings['source'] = 'Simply Hired'
listings["profile"] = profile
#print listings
companies = listings
keys = [
row.company_name.lower().replace(" ", "") + "_" + profile
for i, row in companies.iterrows()
]
companies["company_key"] = keys
companies["createdAt"] = arrow.now().timestamp
conn = rethink_conn.conn()
#r.table("hiring_signals").insert(companies.to_dict('r')).run(conn)
r.table("triggers").insert(companies.to_dict('r')).run(conn)
bitmapist.mark_event("function:time:simplyhired_job_scrape",
int((time.time() - start_time) * 10**6))
rd.zadd("function:time:simplyhired_job_scrape",
str((time.time() - start_time) * 10**6),
arrow.now().timestamp)
def test_bit_operations():
delete_all_events()
now = datetime.utcnow()
last_month = datetime.utcnow() - timedelta(days=30)
# 123 has been active for two months
mark_event('active', 123, now=now)
mark_event('active', 123, now=last_month)
# 224 has only been active last_month
mark_event('active', 224, now=last_month)
# Assert basic premises
assert MonthEvents('active', last_month.year, last_month.month).get_count() == 2
assert MonthEvents('active', now.year, now.month).get_count() == 1
# Try out with bit AND operation
active_2_months = BitOpAnd(
MonthEvents('active', last_month.year, last_month.month),
MonthEvents('active', now.year, now.month)
)
assert active_2_months.get_count() == 1
assert 123 in active_2_months
assert 224 not in active_2_months
active_2_months.delete()
# Try out with bit OR operation
assert BitOpOr(
MonthEvents('active', last_month.year, last_month.month),
MonthEvents('active', now.year, now.month)
).get_count() == 2
# Try out with a different system
active_2_months = BitOpAnd(
'default_copy',
MonthEvents('active', last_month.year, last_month.month),
MonthEvents('active', now.year, now.month),
)
assert active_2_months.get_count() == 1
assert active_2_months.system == 'default_copy'
active_2_months.delete()
# Try nested operations
active_2_months = BitOpAnd(
BitOpAnd(
MonthEvents('active', last_month.year, last_month.month),
MonthEvents('active', now.year, now.month)
),
MonthEvents('active', now.year, now.month)
)
assert 123 in active_2_months
assert 224 not in active_2_months
active_2_months.delete()
def test_bit_operations():
delete_all_events()
now = datetime.utcnow()
last_month = datetime.utcnow() - timedelta(days=30)
# 123 has been active for two months
mark_event('active', 123, now=now)
mark_event('active', 123, now=last_month)
# 224 has only been active last_month
mark_event('active', 224, now=last_month)
# Assert basic premises
assert MonthEvents('active', last_month.year, last_month.month).get_count() == 2
assert MonthEvents('active', now.year, now.month).get_count() == 1
# Try out with bit AND operation
active_2_months = BitOpAnd(
MonthEvents('active', last_month.year, last_month.month),
MonthEvents('active', now.year, now.month)
)
assert active_2_months.get_count() == 1
assert 123 in active_2_months
assert 224 not in active_2_months
# Try out with bit OR operation
assert BitOpOr(
MonthEvents('active', last_month.year, last_month.month),
MonthEvents('active', now.year, now.month)
).get_count() == 2
# Try out with a different system
active_2_months = BitOpAnd(
'default_copy',
MonthEvents('active', last_month.year, last_month.month),
MonthEvents('active', now.year, now.month),
)
assert active_2_months.get_count() == 1
assert active_2_months.system == 'default_copy'
# Try nested operations
active_2_months = BitOpAnd(
BitOpAnd(
MonthEvents('active', last_month.year, last_month.month),
MonthEvents('active', now.year, now.month)
),
MonthEvents('active', now.year, now.month)
)
assert 123 in active_2_months
assert 224 not in active_2_months
def test_bitop_key_sharing():
today = datetime.utcnow()
mark_event('task1', 111, now=today)
mark_event('task2', 111, now=today)
mark_event('task1', 222, now=today)
mark_event('task2', 222, now=today)
ev1_task1 = DayEvents('task1', today.year, today.month, today.day)
ev1_task2 = DayEvents('task2', today.year, today.month, today.day)
ev1_both = BitOpAnd(ev1_task1, ev1_task2)
ev2_task1 = DayEvents('task1', today.year, today.month, today.day)
ev2_task2 = DayEvents('task2', today.year, today.month, today.day)
ev2_both = BitOpAnd(ev2_task1, ev2_task2)
assert ev1_both.redis_key == ev2_both.redis_key
assert len(ev1_both) == len(ev1_both) == 2
ev1_both.delete()
assert len(ev1_both) == len(ev1_both) == 0
def test_bitop_key_sharing():
today = datetime.utcnow()
mark_event('task1', 111, now=today)
mark_event('task2', 111, now=today)
mark_event('task1', 222, now=today)
mark_event('task2', 222, now=today)
ev1_task1 = DayEvents('task1', today.year, today.month, today.day)
ev1_task2 = DayEvents('task2', today.year, today.month, today.day)
ev1_both = BitOpAnd(ev1_task1, ev1_task2)
ev2_task1 = DayEvents('task1', today.year, today.month, today.day)
ev2_task2 = DayEvents('task2', today.year, today.month, today.day)
ev2_both = BitOpAnd(ev2_task1, ev2_task2)
assert ev1_both.redis_key == ev2_both.redis_key
assert len(ev1_both) == len(ev1_both) == 2
ev1_both.delete()
assert len(ev1_both) == len(ev1_both) == 0
def save(self, *args, **kwargs):
created = self.pk is None
super(Issue, self).save(*args, **kwargs)
setup_redis(
'engine_redis',
settings.REDIS_HOSTNAME,
port=settings.REDIS_PORT,
password=settings.REDIS_PASSWORD)
if created:
mark_event('issue:created', self.id, system='engine_redis')
else:
mark_event('issue:updated', self.id, system='engine_redis')
if self.status != 0:
mark_event('issue:solved', self.id, system='engine_redis')
def userid_counting(userid, apps_list, platform, now=None):
id = get_id(userid)
mark_event('request', id, now=now)
list(
map(partial(add_app_statistics, id, platform, now=now), apps_list
or []))
def test_track_unique():
bitmapist.mark_event('foo', 1, track_unique=True)
bitmapist.mark_event('foo', 2, track_unique=False)
assert list(bitmapist.DayEvents('foo')) == [1, 2]
assert list(bitmapist.UniqueEvents('foo')) == [1]
def test_year_events():
mark_event('foo', 1, system='db1')
assert 1 in YearEvents('foo', system='db1')
def test_get_event_names_prefix():
event_names = {'foo', 'bar', 'baz', 'spam', 'egg'}
for e in event_names:
mark_event(e, 1)
BitOpAnd(DayEvents('foo'), DayEvents('bar'))
assert set(get_event_names(prefix='b', batch=2)) == {'bar', 'baz'}
def events():
today = datetime.utcnow()
tomorrow = today + timedelta(days=1)
mark_event('signup', 111, now=today)
mark_event('active', 111, now=today)
mark_event('active', 111, now=tomorrow)
mark_event('signup', 222, now=today)
mark_event('active', 222, now=today)
mark_event('task1', 111, now=today)
mark_event('task1', 111, now=tomorrow)
mark_event('task2', 111, now=today)
mark_event('task2', 111, now=tomorrow)
mark_event('task1', 222, now=today)
mark_event('task1', 222, now=tomorrow)
mark_event('task2', 222, now=today)
mark_event('task2', 222, now=tomorrow)
def mark_login(sender, user, **extra):
mark_event('user:logged_in', user.id)
def mark_logout(sender, user, **extra):
mark_event('user:logged_out', user.id)
def test_bit_operations_complex():
now = datetime.utcnow()
tom = now + timedelta(days=1)
mark_event('task1', 111, now=now)
mark_event('task1', 111, now=tom)
mark_event('task2', 111, now=now)
mark_event('task2', 111, now=tom)
mark_event('task1', 222, now=now)
mark_event('task1', 222, now=tom)
mark_event('task2', 222, now=now)
mark_event('task2', 222, now=tom)
now_events = BitOpAnd(DayEvents('task1', now.year, now.month, now.day),
DayEvents('task2', now.year, now.month, now.day))
tom_events = BitOpAnd(DayEvents('task1', tom.year, tom.month, tom.day),
DayEvents('task2', tom.year, tom.month, tom.day))
both_events = BitOpAnd(now_events, tom_events)
assert len(now_events) == len(tom_events)
assert len(now_events) == len(both_events)
def userid_counting(userid, app, platform, now=None):
id = get_id(userid)
mark_event('request', id, now=now)
add_app_statistics(id, platform, app, now=now)
def _generate_fake_statistics(self):
date = datetime(2016, 2, 13, 0)
for i in range(self.n_hours):
for id in range(0, i):
mark_event('request:app:win:2.0.0.0',
id,
now=date,
track_hourly=True)
mark_event('request:app:mac:4.0.0.1',
id,
now=date,
track_hourly=True)
mark_event('request:app:win:%s:2.0.0.0' % self.channel2.name,
id,
now=date,
track_hourly=True)
mark_event('request:app:mac:%s:4.0.0.1' % self.channel2.name,
id,
now=date,
track_hourly=True)
for id in range(i, self.n_hours):
mark_event('request:app:win:1.0.0.0',
id,
now=date,
track_hourly=True)
mark_event('request:app:mac:3.0.0.0',
id,
now=date,
track_hourly=True)
mark_event('request:app:win:%s:1.0.0.0' % self.channel.name,
id,
now=date,
track_hourly=True)
mark_event('request:app:mac:%s:3.0.0.0' % self.channel.name,
id,
now=date,
track_hourly=True)
date += timedelta(hours=1)
def bitmapist_before_delete(mapper, connection, target):
mark_event('%s:deleted' % target.__class__.__name__.lower(), target.id)
def wrapper(*args, **kwargs):
mark_event(event_name, uuid, system, now, track_hourly,
use_pipeline)
return fn(*args, **kwargs)
