def get_siblings(member_id): parents = get_parents(member_id) if not parents: return [] db, VIS_NEVER = inject('db', 'VIS_NEVER') pa, ma = parents.pa, parents.ma q = (db.TblMembers.id != member_id) & (db.TblMembers.visibility != VIS_NEVER) & (db.TblMembers.deleted == False) if pa: lst1 = db(q & (db.TblMembers.father_id == pa.id)).select( orderby=db.TblMembers.date_of_birth) if pa else [] lst1 = [r.id for r in lst1] else: lst1 = [] if ma: lst2 = db(q & (db.TblMembers.mother_id == ma.id)).select( orderby=db.TblMembers.date_of_birth) if ma else [] lst2 = [r.id for r in lst2] else: lst2 = [] lst = list(set(lst1 + lst2)) #make it unique lst = [get_member_rec(id, prepend_path=True) for id in lst] for rec in lst: if not rec.date_of_birth: rec.date_of_birth = datetime.date( year=1, month=1, day=1) #should not happen but it did... lst = sorted(lst, key=lambda rec: rec.date_of_birth) return lst
def get_header_info(self, msg, result): comment = inject('comment') if result.sender: return #already done though in the wrong place result.sender = msg.get('from') if not result.sender: return lst = decode_header(result.sender) if len(lst) == 2: result.sender_name = lst[0][0] result.sender_email = lst[1][0][1:-1] #get rid of <> else: s = lst[0][0] comment("sender email {}", s) parts = s.split('<') if len(parts) == 1: result.sender_email = s elif parts: result.sender_name = parts[0].strip() result.sender_email = parts[1][:-1] result.to = msg.get('to') result.what = result.to.split('@')[0] subject = msg.get('subject') lst = decode_header(subject) lst = [itm[0] for itm in lst] result.subject = ' '.join(lst) result.date = msg.get('date')
def find_or_insert_word(wrd): from injections import inject db = inject('db') rec = db(db.TblWords.word == wrd).select().first() if rec: return rec.id, False else: return db.TblWords.insert(word=wrd), True
def _get_all_story_previews(): db = inject('db') result = [] for rec in db(db.TblStories).select(orderby=~db.TblStories.story_len): html = rec.story preview = get_reisha(html, 30) result.append(dict(name=rec.name, id=rec.id, prview=preview)) return result
def messaging_group(user=None, group=None): if not group: if not user: auth = inject('auth') user = str(auth.user_id) group = 'USER' + str(user) group = Cache.fix_key(group) #to distinguish between dev, test and www messages return group
def get_story(self, story_id, from_story_version=None, to_story_version=None): db = inject('db') rec = db(db.TblStories.id == story_id).select().first() if not rec: return None if self.language: if rec.language != self.language: rec1 = self.find_translation(story_id, language) if rec1: rec = rec1 # else we have to read in the original language rec = Storage(rec) story = rec.story q = (db.TblStoryVersions.story_id == story_id) story_versions = db(q).select(db.TblStoryVersions.story_id, db.TblStoryVersions.creation_date, db.TblStoryVersions.author_id) story_versions = [Storage(sv) for sv in story_versions] current_version = len(story_versions) display_version = 'The original Story' for i, sv in enumerate(story_versions): if not sv: continue tmp = db(db.auth_user.id == sv.author_id).select( db.auth_user.email).first() sv.email = tmp.email if tmp else '' sv.display_version = get_display_version(i, sv) if to_story_version is None: story_text = story to_story = current_version else: from_story_version = from_story_version or current_version reverse = to_story_version < from_story_version if reverse: dif_recs = story_versions[to_story_version:from_story_version] else: dif_recs = story_versions[from_story_version:to_story_version] diffs = [r.delta for r in dif_recs] merger = mim.Merger() story_text = merger.diff_apply_bulk(story, diffs, reverse=reverse) display_version = story_versions[to_story_version].display_version story_info = Storage( story_text=story_text, story_preview=get_reisha(story_text), name=rec.name, topic=rec.topic, story_id=story_id, #we always access via the master source=rec.source, used_for=rec.used_for, author_id=rec.author_id, language=rec.language, story_curr_version=to_story_version, story_versions=story_versions, display_version=display_version) return story_info
def save_uploaded_photo(file_name, blob, user_id, sub_folder=None): auth, comment, log_exception, db = inject('auth', 'comment', 'log_exception', 'db') user_id = user_id or auth.current_user() crc = zlib.crc32(blob) cnt = db(db.TblPhotos.crc==crc).count() if cnt > 0: return 'duplicate' original_file_name, ext = os.path.splitext(file_name) file_name = '{crc:x}{ext}'.format(crc=crc & 0xffffffff, ext=ext) comment('start saving {}', original_file_name) today = datetime.date.today() month = str(today)[:-3] if not sub_folder: sub_folder = sub_folder or 'uploads/' + month + '/' path = local_photos_folder() + sub_folder dir_util.mkpath(path) try: stream = StringIO(blob) img = Image.open(stream) photo_date = None exif = None try: exif = img._getexif() except: pass if exif: for key in [36867, 306]: if key in exif: photo_date = exif[key] width, height = img.size square_img = crop_to_square(img, width, height, 256) if square_img: path = local_photos_folder("squares") + sub_folder dir_util.mkpath(path) square_img.save(path + file_name) got_square = True else: got_square = False oversize = False if height > MAX_HEIGHT or width > MAX_WIDTH: oversize = True path = local_photos_folder("oversize") + sub_folder dir_util.mkpath(path) img.save(path + file_name) width, height = resized(width, height) img = img.resize((width, height), Image.LANCZOS) path = local_photos_folder() + sub_folder if photo_date: photo_date = datetime_from_str(photo_date, date_only=True) elif os.path.isfile(path + file_name): photo_date = modification_date(path + file_name) else: photo_date = None img.save(path + file_name) except Exception, e: log_exception("saving photo {} failed".format(original_file_name)) return 'failed'
def __init__(self): db = inject('db') self.user_fields = [ db.auth_user.id, db.auth_user.email, db.auth_user.first_name, db.auth_user.last_name, db.auth_user.registration_key, ]
def find_story(self, used_for, topic): db = inject('db') q = (db.TblStories.used_for == used_for) & (db.TblStories.topic == topic) rec = db(q).select(db.TblStories.id).first() if rec: return self.get_story(rec.id) else: return None
def tally_all_stories(): from injections import inject db = inject('db') dic = dict() N = 0 for rec in db(db.TblStories).select(): html = rec.story if tally_words(html, dic, rec.id, rec.name): N += 1 return dic
def retrieve_story_words(story_id): from injections import inject db = inject('db') q = (db.TblWordStories.story_id == story_id) & (db.TblWords.id == db.TblWordStories.word_id) lst = db(q).select() dic = dict() for rec in lst: w = rec.TblWords.word dic[w] = rec.TblWordStories.word_count return dic
def on_update_task_status(self, task_id, data): #comment('on update task status', log_name='dbg_scheduler') comment = inject('comment') try: ###comment("task {task_id} status changed {data}", task_id=task_id, data=data) ws_messaging.send_message(key='task_status_changed', group='TASK_MONITOR', task_id=task_id, data=data) except Exception, e: log_exception('failed on update task status')
def get_slides_from_photo_list(q): db = inject('db') q &= (db.TblPhotos.width > 0) db = inject('db') lst = db(q).select() if not lst: return [] if 'TblPhotos' in lst[0]: lst = [rec.TblPhotos for rec in lst] lst1 = [] visited = set([]) for rec in lst: if rec.id in visited: continue visited |= set([rec.id]) lst1.append(rec) lst = lst1 folder = photos_folder() slides = [dict(photo_id=rec.id, src=folder + rec.photo_path, width=rec.width, height=rec.height, title=rec.Description or rec.Name) for rec in lst] return slides
def _verify_tasks_started(): if db(db.auth_user).count() < 2: return comment = inject('comment') comment("verify tasks started") for task_name in permanent_tasks: comment("start {}", task_name) if db(db.scheduler_task.task_name == task_name).isempty(): permanent_tasks[task_name]() comment('tasks verified.') return True
def verify_email(self, key): """ Action used to verify the registration email """ EDITOR, PHOTO_UPLOADER = inject('EDITOR', 'PHOTO_UPLOADER') table_user = self.table_user() user = table_user(registration_key=key) if not user: return False user.update_record(registration_key='') self.add_membership(user_id=user.id, group_id=EDITOR) self.add_membership(user_id=user.id, group_id=PHOTO_UPLOADER) return True
def one_time_tasks(self): db = inject('db') lst = db(db.scheduler_task.task_name.like('Execute%')).select( db.scheduler_task.id, db.scheduler_task.task_name, db.scheduler_task.status, db.scheduler_task.start_time, db.scheduler_task.last_run_time, db.scheduler_task.next_run_time, db.scheduler_task.times_run, db.scheduler_task.times_failed, orderby=~db.scheduler_task.last_run_time) return lst
def get_users_data(self): db, auth = inject('db', 'auth') if auth.user: by_developer = auth.user.email in ['*****@*****.**'] #name = (auth.user.first_name + ' ' + auth.user.last_name).lower() #by_developer = name in ('haim avni', 'barak shohat') else: by_developer = False result = [] lst = db(db.auth_user).select(*self.user_fields) for usr in lst: data = self.user_data(usr, by_developer) result.append(data) return result
def user_data(self, usr, by_developer=False): auth = inject('auth') result = usr roles = [] for grp in self.get_groups(by_developer): role_title = ' '.join( [z.capitalize() for z in grp.role.split('_')]) roles.append( dict(role=grp.role, role_title=role_title, active=auth.has_membership(grp.id, usr.id))) result.roles = roles result.status = 'Unconfirmed' if result.registration_key else '' return result
def update_key_list(self, delete=False): #maintain list of all keys - did not find mc method that does it... k = Cache.fix_key('all_cached_keys') key_list = self.mc.get(k) or set([]) if delete: if self.key in key_list: key_list -= set([self.key]) elif self.key not in key_list: key_list |= set([self.key]) self.mc.set(k, key_list) check_list = self.mc.get(k) if check_list != key_list: comment = inject('comment') comment("update key list failed!")
def create_word_index(): import datetime t0 = datetime.datetime.now() db = inject('db') db.TblWords.truncate('RESTART IDENTITY CASCADE') dic = tally_all_stories() for wrd in dic: word_id = db.TblWords.insert(word=wrd) for story_id in dic[wrd]: db.TblWordStories.insert(word_id=word_id, story_id=story_id, word_count=dic[wrd][story_id]) elapsed = datetime.datetime.now() - t0 db.commit() print elapsed
def fix_key(key): #prefix with server id and convert to hex if key contains control characters request = inject('request') s = request.env.http_host if '.' in s: prefix = s.split('.')[0] + '-' if prefix == '127-': #just for the development PC prefix = '' else: prefix = '' prefix += request.application + '-' key = prefix + key if not valid_key_chars_re.match(key): key = key.encode('hex') return key
def register_new_user(email, password, first_name, last_name, registration_key=''): db = inject('db') cpassword = encrypt_password(password) if not db(db.auth_user.email == email).isempty(): raise UserError('User "{em}" already exists!'.format(em=email)) fields = dict( email=email, first_name=first_name, last_name=last_name, password=cpassword, registration_key=registration_key, ) return db.auth_user.insert(**fields)
def collect_mail(): comment = inject('comment') email_photos_collector = EmailCollector() results = [] for msg in email_photos_collector.collect(): user_id = get_user_id_of_sender(msg.sender_email, msg.sender_name) user_id = user_id or 1 #todo: if we decide not to create new user text = msg.html_content or msg.plain_content comment( 'New email: subject: {subject}, emages: {image_names} sent by {sender}', subject=msg.subject, image_names=msg.images.keys(), sender=msg.sender_email) if msg.images: result = save_uploaded_photo_collection(msg.images, user_id) comment("upload result {result}", result=result) return results
def get_children(member_id, hidden_too=False): member_rec = get_member_rec(member_id) db, VIS_NEVER = inject('db', 'VIS_NEVER') if member_rec.gender == 'F': q = db.TblMembers.mother_id == member_id elif member_rec.gender == 'M': q = db.TblMembers.father_id == member_id else: return [] #error! if not hidden_too: q &= (db.TblMembers.visibility != VIS_NEVER) q &= (db.TblMembers.deleted == False) lst = db(q).select(db.TblMembers.id, db.TblMembers.date_of_birth, orderby=db.TblMembers.date_of_birth) lst = [get_member_rec(rec.id, prepend_path=True) for rec in lst] return lst
def fit_size(rec): db, log_exception = inject('db', 'log_exception') fname = local_photos_folder() + rec.photo_path try: img = Image.open(fname) oversize_file_name = local_photos_folder("oversize") + rec.photo_path oversize_path, f = os.path.split(oversize_file_name) dir_util.mkpath(oversize_path) img.save(oversize_file_name) width, height = resized(rec.width, rec.height) img = img.resize((width, height), Image.LANCZOS) img.save(fname) rec.update_record(width=width, height=height) except Exception, e: log_exception("resizing file {} failed.".format(rec.photo_path)) rec.update_record(photo_missing=True) return False
def fit_all_sizes(): db, request, comment = inject('db', 'request', 'comment') comment('started fitting size for chunk of photos') q = (db.TblPhotos.width > MAX_WIDTH) | (db.TblPhotos.height > MAX_HEIGHT) q &= (db.TblPhotos.photo_missing != True) chunk = 100 num_failed = 0 while True: n = db(q).count() comment('started fitting size for chunk of photos. {} more to go.'.format(n)) if db(q).count() == 0: break lst = db(q).select(limitby=(0, chunk)) for rec in lst: if not fit_size(rec): num_failed += 1 db.commit() return dict(num_failed=num_failed)
def extract_story_words(story_id): from injections import inject db = inject('db') rec = db(db.TblStories.id == story_id).select().first() if not rec: return None story_name = rec.name html = rec.story s = story_name.decode('utf8') + ' ' + remove_all_tags(html) lst = extract_words(s) if not lst: return None dic = dict() for w in lst: if w not in dic: dic[w] = 0 dic[w] += 1 return dic
def _calc_used_languages(used_for): db = inject('db') dic = {} q = (db.TblStories.id > 0) if used_for: q &= (db.TblStories.used_for == used_for) for rec in db(q).select(): lang = rec.language if lang not in dic: dic[lang] = 0 dic[lang] += 1 lst = [] for lang in sorted(dic): item = dict(id=lang, name='stories.' + language_name(lang), count=dic[lang]) lst.append(item) return dict(used_languages=lst)
def get_member_rec(member_id, member_rec=None, prepend_path=False): db = inject('db') if member_rec: rec = member_rec #used when initially all members are loaded into the cache elif not member_id: return None else: rec = db(db.TblMembers.id == member_id).select().first() if not rec: return None if rec.deleted: return None rec = Storage(rec.as_dict()) rec.full_name = member_display_name(rec, full=True) rec.name = member_display_name(rec, full=False) if prepend_path: rec.facePhotoURL = photos_folder('profile_photos') + ( rec.facePhotoURL or 'dummy_face.png') return rec
def notify_registration(self, user_info): db, mail = inject('db', 'mail') ui = user_info user_name = ui.first_name + ' ' + ui.last_name email = ui.email lst = db((db.auth_membership.group_id == ADMIN) & (db.auth_user.id == db.auth_membership.user_id)).select( db.auth_user.email) receivers = [r.email for r in lst] message = ('', ''' {uname} has just registered to <b>GB Stories</b>. Email adddress is {uemail}. Click <a href="http://gbstories.org/gbs__www/stories">here</a> for access manager. '''.format(uname=user_name, uemail=email).replace('\n', '<br>')) mail.send(to=receivers, subject='New GB Stories registration', message=message)
def project(network, freqs, h, theta, phi, psi, data, figtuple=None, dh=None, units="$1/\sqrt{\mathrm{Hz}}$"): """ projects the strain "h" onto the network and overlays it with "data" at each detector if supplied, "dh" is the upper/lower bounds for h and will be shaded on the plots """ n_ifo = len(network) n_freqs = len(freqs) n_f, n_pol = np.shape(h) if n_f != (n_freqs): raise ValueError, "bad shape for h" if (dh != None) and (np.shape(dh) != (n_freqs, n_pol, 2)): raise ValueError, "bad shape for dh" if np.shape(data) != (n_freqs, n_ifo): raise ValueError, "bad shape for data" n_axs = n_ifo + n_pol if figtuple: fig, axs = figtuple if len(axs) != n_axs: raise ValueError, "inconsistent number of ifos between network and figtuples" else: fig = plt.figure() axs = [plt.subplot(n_axs, 1, i + 1) for i in xrange(n_axs)] figtuple = (fig, axs) iax = 0 ### geocenter h for pol in xrange(n_pol): ax = axs[iax] ax.plot(freqs, h[:, pol].real, color="b", label="$\mathrm{Real}\{h_%d\}$" % pol) if np.any(h.imag): ax.plot(freqs, h[:, pol].imag, color="r", label="$\mathrm{Imag}\{h_%d\}$" % pol) if dh != None: ax.fill_between(freqs, dh[:, pol, 0].real, dh[:, pol, 1].real, color="b", alpha=0.25) if np.any(dh.imag): ax.fill_between(freqs, dh[:, pol, 0].imag, dh[:, pol, 1].imag, color="r", alpha=0.25) ax.set_ylabel("$h_%d$ [%s]" % (pol, units)) iax += 1 ### inject h into the network inj = injections.inject(network, h, theta, phi, psi=psi) if dh != None: dinj0 = np.transpose(injections.inject(network, dh[:, 0], theta, phi, psi=psi)) dinj1 = np.transpose(injections.inject(network, dh[:, 1], theta, phi, psi=psi)) else: dinj0 = dinj1 = [None] * n_ifo ### iterate and plot for (name, i, di0, di1, d) in zip( network.detector_names_list(), np.transpose(inj), dinj0, dinj1, np.transpose(data) ): ax = axs[iax] ax.plot(freqs, d.real, color="c", label="$\mathrm{Real}{d_{\mathrm{%s}}}$" % name) if np.any(d.imag): ax.plot(freqs, d.imag, color="m", label="$\mathrm{Imag}{d_{\mathrm{%s}}}$" % name) ax.plot(freqs, i.real, color="b", label="$\mathrm{Real}\{F*h\}$") if np.any(i.imag): ax.plot(freqs, i.imag, color="r", label="$\mathrm{Imag}\{F*h\}$") if di0 != None: ax.fill_between(freqs, di0, di1, color="b", alpha=0.25) if np.any(di0.imag): ax.fill_between(freqs, di0, di1, color="r", alpha=0.25) iax += 1 return figtuple
print """injecting SineGaussian with parameters fo = %f to = %f phio = %f tau = %f hrss = %fe-23 alpha = %f theta = %f phi = %f psi = %f"""%(fo, t, phio, tau, hrss*1e23, alpha, theta, phi, psi) if opts.time: to = time.time() h = injections.sinegaussian_f(freqs, to=t, phio=phio, fo=fo, tau=tau, hrss=hrss, alpha=alpha) inj = injections.inject( network, h, theta, phi, psi) if opts.time: print "\t", time.time()-to elif config.has_option("injection","cache"): ### read injection frame inj = np.empty((n_freqs, n_ifo), complex) inj_cache = eval(config.get("injection","cache")) inj_chans = eval(config.get("injection","channels")) ### pull out parameters for PSD estimation dseg = 0.5*seglen for ifo_ind, ifo in enumerate(ifos):
to=time.time() ### antenna pattern jazz theta = theta_inj[inj_id] phi = phi_inj[inj_id] psi = psi_inj[inj_id] ### singal amplitude hrss = hrss_inj[inj_id] snrs = snrs_inj[inj_id] snr_net = np.sum(snrs**2)**0.5 print "\t\t", time.time()-to ### generate data stream print "\tgenerating data" to=time.time() data_inj = injections.inject(network, waveform_func(freqs, hrss=hrss, **waveform_args), theta, phi, psi=psi) print "\t\t", time.time()-to injang = (theta, phi) if opts.zero_noise: noise = np.zeros((n_freqs, n_ifo), complex) else: print "\tdrawing noise" to=time.time() noise = network.draw_noise() print "\t\t", time.time()-to data = data_inj + noise print "\tset_data"