def transformOldRecents(uid, recents): from dateutil import parser from sefaria.system.exceptions import InputError import pytz default_epoch_time = epoch_time( datetime(2017, 12, 1)) # the Sefaria epoch. approx time since we added time stamps to recent items def xformer(recent): try: return { "uid": uid, "ref": recent[0], "he_ref": recent[1], "book": Ref(recent[0]).index.title, "last_place": True, "time_stamp": epoch_time(parser.parse(recent[2]).replace(tzinfo=None)) if recent[2] is not None else default_epoch_time, "server_time_stamp": epoch_time(parser.parse(recent[2]).replace(tzinfo=None)) if recent[2] is not None else default_epoch_time, "num_times_read": (recent[3] if recent[3] and isinstance(recent[3], int) else 1), # we dont really know how long they've read this book. it's probably correlated with the number of times they opened the book "versions": { "en": recent[4], "he": recent[5] } } except InputError: return None except ValueError: return None except IndexError: return None except AttributeError: return None return filter(None, [xformer(r) for r in recents])
def transformOldRecents(uid, recents): from dateutil import parser from sefaria.system.exceptions import InputError import pytz default_epoch_time = epoch_time( datetime(2017, 12, 1)) # the Sefaria epoch. approx time since we added time stamps to recent items def xformer(recent): try: return { "uid": uid, "ref": recent[0], "he_ref": recent[1], "book": Ref(recent[0]).index.title, "last_place": True, "time_stamp": epoch_time(parser.parse(recent[2]).replace(tzinfo=None)) if recent[2] is not None else default_epoch_time, "server_time_stamp": epoch_time(parser.parse(recent[2]).replace(tzinfo=None)) if recent[2] is not None else default_epoch_time, "num_times_read": (recent[3] if recent[3] and isinstance(recent[3], int) else 1), # we dont really know how long they've read this book. it's probably correlated with the number of times they opened the book "versions": { "en": recent[4], "he": recent[5] } } except InputError: return None except ValueError: return None except IndexError: return None except AttributeError: return None return filter(None, [xformer(r) for r in recents])
def update_attr_time_stamps(self, obj): if "settings" in obj: settings_changed = False for k, v in obj["settings"].items(): if k not in self.settings: settings_changed = True elif v != self.settings[k]: settings_changed = True if settings_changed: obj["attr_time_stamps"] = obj.get("attr_time_stamps", {}) obj["attr_time_stamps"]["settings"] = epoch_time()
def update_attr_time_stamps(self, obj): if "settings" in obj: settings_changed = False for k, v in obj["settings"].items(): if k not in self.settings: settings_changed = True elif v != self.settings[k]: settings_changed = True if settings_changed: obj["attr_time_stamps"] = obj.get("attr_time_stamps", {}) obj["attr_time_stamps"]["settings"] = epoch_time()
def xformer(recent): try: return { "uid": uid, "ref": recent[0], "he_ref": recent[1], "book": Ref(recent[0]).index.title, "last_place": True, "time_stamp": epoch_time(parser.parse(recent[2]).replace(tzinfo=None)) if recent[2] is not None else default_epoch_time, "server_time_stamp": epoch_time(parser.parse(recent[2]).replace(tzinfo=None)) if recent[2] is not None else default_epoch_time, "num_times_read": (recent[3] if recent[3] and isinstance(recent[3], int) else 1), # we dont really know how long they've read this book. it's probably correlated with the number of times they opened the book "versions": { "en": recent[4], "he": recent[5] } } except InputError: return None except ValueError: return None except IndexError: return None except AttributeError: return None
def xformer(recent): try: return { "uid": uid, "ref": recent[0], "he_ref": recent[1], "book": Ref(recent[0]).index.title, "last_place": True, "time_stamp": epoch_time(parser.parse(recent[2]).replace(tzinfo=None)) if recent[2] is not None else default_epoch_time, "server_time_stamp": epoch_time(parser.parse(recent[2]).replace(tzinfo=None)) if recent[2] is not None else default_epoch_time, "num_times_read": (recent[3] if recent[3] and isinstance(recent[3], int) else 1), # we dont really know how long they've read this book. it's probably correlated with the number of times they opened the book "versions": { "en": recent[4], "he": recent[5] } } except InputError: return None except ValueError: return None except IndexError: return None except AttributeError: return None
def process_register_form(request, auth_method='session'): from sefaria.utils.util import epoch_time from sefaria.helper.file import get_resized_file import hashlib import urllib.parse, urllib.request from google.cloud.exceptions import GoogleCloudError from PIL import Image form = SefariaNewUserForm(request.POST) if auth_method == 'session' else SefariaNewUserFormAPI(request.POST) token_dict = None if form.is_valid(): with transaction.atomic(): new_user = form.save() user = authenticate(email=form.cleaned_data['email'], password=form.cleaned_data['password1']) p = UserProfile(id=user.id) p.assign_slug() p.join_invited_collections() if hasattr(request, "interfaceLang"): p.settings["interface_language"] = request.interfaceLang # auto-add profile pic from gravatar if exists email_hash = hashlib.md5(p.email.lower().encode('utf-8')).hexdigest() gravatar_url = "https://www.gravatar.com/avatar/" + email_hash + "?d=404&s=250" try: with urllib.request.urlopen(gravatar_url) as r: bucket_name = GoogleStorageManager.PROFILES_BUCKET with Image.open(r) as image: now = epoch_time() big_pic_url = GoogleStorageManager.upload_file(get_resized_file(image, (250, 250)), "{}-{}.png".format(p.slug, now), bucket_name, None) small_pic_url = GoogleStorageManager.upload_file(get_resized_file(image, (80, 80)), "{}-{}-small.png".format(p.slug, now), bucket_name, None) p.profile_pic_url = big_pic_url p.profile_pic_url_small = small_pic_url except urllib.error.HTTPError as e: logger.info("The Gravatar server couldn't fulfill the request. Error Code {}".format(e.code)) except urllib.error.URLError as e: logger.info("HTTP Error from Gravatar Server. Reason: {}".format(e.reason)) except GoogleCloudError as e: logger.warning("Error communicating with Google Storage Manager. {}".format(e)) p.save() if auth_method == 'session': auth_login(request, user) elif auth_method == 'jwt': token_dict = TokenObtainPairSerializer().validate({"username": form.cleaned_data['email'], "password": form.cleaned_data['password1']}) return { k: v[0] if len(v) > 0 else str(v) for k, v in list(form.errors.items()) }, token_dict, form
def save_history_item(cls, uid, hist, time_stamp=None): if time_stamp is None: time_stamp = epoch_time() hist["uid"] = uid if "he_ref" not in hist or "book" not in hist: oref = Ref(hist["ref"]) hist["he_ref"] = oref.he_normal() hist["book"] = oref.index.title hist["server_time_stamp"] = time_stamp if "server_time_stamp" not in hist else hist["server_time_stamp"] # DEBUG: helpful to include this field for debugging action = hist.pop("action", None) saved = True if action == "add_saved" else (False if action == "delete_saved" else hist.get("saved", False)) uh = UserHistory(hist, load_existing=(action is not None), update_last_place=(action is None), field_updates={ "saved": saved, "server_time_stamp": hist["server_time_stamp"] }) uh.save() return uh
def save_history_item(cls, uid, hist, time_stamp=None): if time_stamp is None: time_stamp = epoch_time() hist["uid"] = uid if "he_ref" not in hist or "book" not in hist: oref = Ref(hist["ref"]) hist["he_ref"] = oref.he_normal() hist["book"] = oref.index.title hist["server_time_stamp"] = time_stamp if "server_time_stamp" not in hist else hist["server_time_stamp"] # DEBUG: helpful to include this field for debugging action = hist.pop("action", None) saved = True if action == "add_saved" else (False if action == "delete_saved" else hist.get("saved", False)) uh = UserHistory(hist, load_existing=(action is not None), update_last_place=(action is None), field_updates={ "saved": saved, "server_time_stamp": hist["server_time_stamp"] }) uh.save() return uh
start_at = int(sys.argv[1]) except: start_at = 0 cursor = db.profiles.find({"profile_pic_url": {"$not": {"$regex": "{}.*".format(GoogleStorageManager.BASE_URL)}}, "id" : {"$gt": start_at}}, no_cursor_timeout=True).sort("id") try: for profile_mongo in cursor: old_profile_pic_url = recentlyViewed = profile_mongo["profile_pic_url"] profile = UserProfile(id=profile_mongo['id']) email_hash = hashlib.md5(profile.email.lower().encode('utf-8')).hexdigest() gravatar_url = "https://www.gravatar.com/avatar/" + email_hash + "?d=404&s=250" try: r = urllib.request.urlopen(gravatar_url) bucket_name = GoogleStorageManager.PROFILES_BUCKET with Image.open(r) as image: now = epoch_time() big_pic_url = GoogleStorageManager.upload_file(get_resized_file(image, (250, 250)), "{}-{}.png".format(profile.slug, now), bucket_name, None) small_pic_url = GoogleStorageManager.upload_file(get_resized_file(image, (80, 80)), "{}-{}-small.png".format(profile.slug, now), bucket_name, None) profile_mongo["profile_pic_url"] = big_pic_url profile_mongo["profile_pic_url_small"] = small_pic_url except urllib.error.HTTPError as e: if e.code == 404: profile_mongo["profile_pic_url"]="" profile_mongo["profile_pic_url_small"]="" else: print('unexpected ERROR: {}'.format(e)) print(profile_mongo["id"]) except urllib.error.URLError as e: print("HTTP ERROR from Gravatar Server. Reason: {}".format(e.reason)) print(profile_mongo["id"]) except GoogleCloudError as e:
limit).limit(limit)) except AutoReconnect as e: tries += 1 if tries >= 200: print "Tried: {} times".format(tries) raise e time.sleep(5) continue has_more = False for s in temp_sheets: has_more = True yield s page += 1 now = epoch_time() default_epoch_time = epoch_time( datetime(2017, 12, 1) ) # the Sefaria epoch. approx time since we added time stamps to recent items for sheet in get_all_sheets(): likes = sheet.get("likes", []) if "id" not in sheet: print sheet continue tref = "Sheet {}".format(sheet["id"]) if len(likes) > 0: oref = Ref(tref) for l in likes: d = { "uid": l, "ref": tref,
try: temp_sheets = list(db.sheets.find().skip(page*limit).limit(limit)) except AutoReconnect as e: tries += 1 if tries >= 200: print "Tried: {} times".format(tries) raise e time.sleep(5) continue has_more = False for s in temp_sheets: has_more = True yield s page += 1 now = epoch_time() default_epoch_time = epoch_time(datetime(2017, 12, 1)) # the Sefaria epoch. approx time since we added time stamps to recent items for sheet in get_all_sheets(): likes = sheet.get("likes", []) if "id" not in sheet: print sheet continue tref = "Sheet {}".format(sheet["id"]) if len(likes) > 0: oref = Ref(tref) for l in likes: d = { "uid": l, "ref": tref, "he_ref": oref.he_normal(), "book": oref.book,