def get_siblings(member_id):
parents = get_parents(member_id)
if not parents:
return []
db, VIS_NEVER = inject('db', 'VIS_NEVER')
pa, ma = parents.pa, parents.ma
q = (db.TblMembers.id != member_id) & (db.TblMembers.visibility !=
VIS_NEVER) & (db.TblMembers.deleted
== False)
if pa:
lst1 = db(q & (db.TblMembers.father_id == pa.id)).select(
orderby=db.TblMembers.date_of_birth) if pa else []
lst1 = [r.id for r in lst1]
else:
lst1 = []
if ma:
lst2 = db(q & (db.TblMembers.mother_id == ma.id)).select(
orderby=db.TblMembers.date_of_birth) if ma else []
lst2 = [r.id for r in lst2]
else:
lst2 = []
lst = list(set(lst1 + lst2)) #make it unique
lst = [get_member_rec(id, prepend_path=True) for id in lst]
for rec in lst:
if not rec.date_of_birth:
rec.date_of_birth = datetime.date(
year=1, month=1, day=1) #should not happen but it did...
lst = sorted(lst, key=lambda rec: rec.date_of_birth)
return lst
def get_header_info(self, msg, result):
comment = inject('comment')
if result.sender:
return #already done though in the wrong place
result.sender = msg.get('from')
if not result.sender:
return
lst = decode_header(result.sender)
if len(lst) == 2:
result.sender_name = lst[0][0]
result.sender_email = lst[1][0][1:-1] #get rid of <>
else:
s = lst[0][0]
comment("sender email {}", s)
parts = s.split('<')
if len(parts) == 1:
result.sender_email = s
elif parts:
result.sender_name = parts[0].strip()
result.sender_email = parts[1][:-1]
result.to = msg.get('to')
result.what = result.to.split('@')[0]
subject = msg.get('subject')
lst = decode_header(subject)
lst = [itm[0] for itm in lst]
result.subject = ' '.join(lst)
result.date = msg.get('date')
def find_or_insert_word(wrd):
from injections import inject
db = inject('db')
rec = db(db.TblWords.word == wrd).select().first()
if rec:
return rec.id, False
else:
return db.TblWords.insert(word=wrd), True
def _get_all_story_previews():
db = inject('db')
result = []
for rec in db(db.TblStories).select(orderby=~db.TblStories.story_len):
html = rec.story
preview = get_reisha(html, 30)
result.append(dict(name=rec.name, id=rec.id, prview=preview))
return result
def messaging_group(user=None, group=None):
if not group:
if not user:
auth = inject('auth')
user = str(auth.user_id)
group = 'USER' + str(user)
group = Cache.fix_key(group) #to distinguish between dev, test and www messages
return group
def get_story(self,
story_id,
from_story_version=None,
to_story_version=None):
db = inject('db')
rec = db(db.TblStories.id == story_id).select().first()
if not rec:
return None
if self.language:
if rec.language != self.language:
rec1 = self.find_translation(story_id, language)
if rec1:
rec = rec1
# else we have to read in the original language
rec = Storage(rec)
story = rec.story
q = (db.TblStoryVersions.story_id == story_id)
story_versions = db(q).select(db.TblStoryVersions.story_id,
db.TblStoryVersions.creation_date,
db.TblStoryVersions.author_id)
story_versions = [Storage(sv) for sv in story_versions]
current_version = len(story_versions)
display_version = 'The original Story'
for i, sv in enumerate(story_versions):
if not sv:
continue
tmp = db(db.auth_user.id == sv.author_id).select(
db.auth_user.email).first()
sv.email = tmp.email if tmp else ''
sv.display_version = get_display_version(i, sv)
if to_story_version is None:
story_text = story
to_story = current_version
else:
from_story_version = from_story_version or current_version
reverse = to_story_version < from_story_version
if reverse:
dif_recs = story_versions[to_story_version:from_story_version]
else:
dif_recs = story_versions[from_story_version:to_story_version]
diffs = [r.delta for r in dif_recs]
merger = mim.Merger()
story_text = merger.diff_apply_bulk(story, diffs, reverse=reverse)
display_version = story_versions[to_story_version].display_version
story_info = Storage(
story_text=story_text,
story_preview=get_reisha(story_text),
name=rec.name,
topic=rec.topic,
story_id=story_id, #we always access via the master
source=rec.source,
used_for=rec.used_for,
author_id=rec.author_id,
language=rec.language,
story_curr_version=to_story_version,
story_versions=story_versions,
display_version=display_version)
return story_info
def save_uploaded_photo(file_name, blob, user_id, sub_folder=None):
auth, comment, log_exception, db = inject('auth', 'comment', 'log_exception', 'db')
user_id = user_id or auth.current_user()
crc = zlib.crc32(blob)
cnt = db(db.TblPhotos.crc==crc).count()
if cnt > 0:
return 'duplicate'
original_file_name, ext = os.path.splitext(file_name)
file_name = '{crc:x}{ext}'.format(crc=crc & 0xffffffff, ext=ext)
comment('start saving {}', original_file_name)
today = datetime.date.today()
month = str(today)[:-3]
if not sub_folder:
sub_folder = sub_folder or 'uploads/' + month + '/'
path = local_photos_folder() + sub_folder
dir_util.mkpath(path)
try:
stream = StringIO(blob)
img = Image.open(stream)
photo_date = None
exif = None
try:
exif = img._getexif()
except:
pass
if exif:
for key in [36867, 306]:
if key in exif:
photo_date = exif[key]
width, height = img.size
square_img = crop_to_square(img, width, height, 256)
if square_img:
path = local_photos_folder("squares") + sub_folder
dir_util.mkpath(path)
square_img.save(path + file_name)
got_square = True
else:
got_square = False
oversize = False
if height > MAX_HEIGHT or width > MAX_WIDTH:
oversize = True
path = local_photos_folder("oversize") + sub_folder
dir_util.mkpath(path)
img.save(path + file_name)
width, height = resized(width, height)
img = img.resize((width, height), Image.LANCZOS)
path = local_photos_folder() + sub_folder
if photo_date:
photo_date = datetime_from_str(photo_date, date_only=True)
elif os.path.isfile(path + file_name):
photo_date = modification_date(path + file_name)
else:
photo_date = None
img.save(path + file_name)
except Exception, e:
log_exception("saving photo {} failed".format(original_file_name))
return 'failed'
def __init__(self):
db = inject('db')
self.user_fields = [
db.auth_user.id,
db.auth_user.email,
db.auth_user.first_name,
db.auth_user.last_name,
db.auth_user.registration_key,
]
def find_story(self, used_for, topic):
db = inject('db')
q = (db.TblStories.used_for == used_for) & (db.TblStories.topic
== topic)
rec = db(q).select(db.TblStories.id).first()
if rec:
return self.get_story(rec.id)
else:
return None
def tally_all_stories():
from injections import inject
db = inject('db')
dic = dict()
N = 0
for rec in db(db.TblStories).select():
html = rec.story
if tally_words(html, dic, rec.id, rec.name):
N += 1
return dic
def retrieve_story_words(story_id):
from injections import inject
db = inject('db')
q = (db.TblWordStories.story_id
== story_id) & (db.TblWords.id == db.TblWordStories.word_id)
lst = db(q).select()
dic = dict()
for rec in lst:
w = rec.TblWords.word
dic[w] = rec.TblWordStories.word_count
return dic
def on_update_task_status(self, task_id, data):
#comment('on update task status', log_name='dbg_scheduler')
comment = inject('comment')
try:
###comment("task {task_id} status changed {data}", task_id=task_id, data=data)
ws_messaging.send_message(key='task_status_changed',
group='TASK_MONITOR',
task_id=task_id,
data=data)
except Exception, e:
log_exception('failed on update task status')
def get_slides_from_photo_list(q):
db = inject('db')
q &= (db.TblPhotos.width > 0)
db = inject('db')
lst = db(q).select()
if not lst:
return []
if 'TblPhotos' in lst[0]:
lst = [rec.TblPhotos for rec in lst]
lst1 = []
visited = set([])
for rec in lst:
if rec.id in visited:
continue
visited |= set([rec.id])
lst1.append(rec)
lst = lst1
folder = photos_folder()
slides = [dict(photo_id=rec.id, src=folder + rec.photo_path, width=rec.width, height=rec.height, title=rec.Description or rec.Name) for rec in lst]
return slides
def _verify_tasks_started():
if db(db.auth_user).count() < 2:
return
comment = inject('comment')
comment("verify tasks started")
for task_name in permanent_tasks:
comment("start {}", task_name)
if db(db.scheduler_task.task_name == task_name).isempty():
permanent_tasks[task_name]()
comment('tasks verified.')
return True
def verify_email(self, key):
"""
Action used to verify the registration email
"""
EDITOR, PHOTO_UPLOADER = inject('EDITOR', 'PHOTO_UPLOADER')
table_user = self.table_user()
user = table_user(registration_key=key)
if not user:
return False
user.update_record(registration_key='')
self.add_membership(user_id=user.id, group_id=EDITOR)
self.add_membership(user_id=user.id, group_id=PHOTO_UPLOADER)
return True
def one_time_tasks(self):
db = inject('db')
lst = db(db.scheduler_task.task_name.like('Execute%')).select(
db.scheduler_task.id,
db.scheduler_task.task_name,
db.scheduler_task.status,
db.scheduler_task.start_time,
db.scheduler_task.last_run_time,
db.scheduler_task.next_run_time,
db.scheduler_task.times_run,
db.scheduler_task.times_failed,
orderby=~db.scheduler_task.last_run_time)
return lst
def get_users_data(self):
db, auth = inject('db', 'auth')
if auth.user:
by_developer = auth.user.email in ['*****@*****.**']
#name = (auth.user.first_name + ' ' + auth.user.last_name).lower()
#by_developer = name in ('haim avni', 'barak shohat')
else:
by_developer = False
result = []
lst = db(db.auth_user).select(*self.user_fields)
for usr in lst:
data = self.user_data(usr, by_developer)
result.append(data)
return result
def user_data(self, usr, by_developer=False):
auth = inject('auth')
result = usr
roles = []
for grp in self.get_groups(by_developer):
role_title = ' '.join(
[z.capitalize() for z in grp.role.split('_')])
roles.append(
dict(role=grp.role,
role_title=role_title,
active=auth.has_membership(grp.id, usr.id)))
result.roles = roles
result.status = 'Unconfirmed' if result.registration_key else ''
return result
def update_key_list(self, delete=False):
#maintain list of all keys - did not find mc method that does it...
k = Cache.fix_key('all_cached_keys')
key_list = self.mc.get(k) or set([])
if delete:
if self.key in key_list:
key_list -= set([self.key])
elif self.key not in key_list:
key_list |= set([self.key])
self.mc.set(k, key_list)
check_list = self.mc.get(k)
if check_list != key_list:
comment = inject('comment')
comment("update key list failed!")
def create_word_index():
import datetime
t0 = datetime.datetime.now()
db = inject('db')
db.TblWords.truncate('RESTART IDENTITY CASCADE')
dic = tally_all_stories()
for wrd in dic:
word_id = db.TblWords.insert(word=wrd)
for story_id in dic[wrd]:
db.TblWordStories.insert(word_id=word_id,
story_id=story_id,
word_count=dic[wrd][story_id])
elapsed = datetime.datetime.now() - t0
db.commit()
print elapsed
def fix_key(key):
#prefix with server id and convert to hex if key contains control characters
request = inject('request')
s = request.env.http_host
if '.' in s:
prefix = s.split('.')[0] + '-'
if prefix == '127-': #just for the development PC
prefix = ''
else:
prefix = ''
prefix += request.application + '-'
key = prefix + key
if not valid_key_chars_re.match(key):
key = key.encode('hex')
return key
def register_new_user(email,
password,
first_name,
last_name,
registration_key=''):
db = inject('db')
cpassword = encrypt_password(password)
if not db(db.auth_user.email == email).isempty():
raise UserError('User "{em}" already exists!'.format(em=email))
fields = dict(
email=email,
first_name=first_name,
last_name=last_name,
password=cpassword,
registration_key=registration_key,
)
return db.auth_user.insert(**fields)
def collect_mail():
comment = inject('comment')
email_photos_collector = EmailCollector()
results = []
for msg in email_photos_collector.collect():
user_id = get_user_id_of_sender(msg.sender_email, msg.sender_name)
user_id = user_id or 1 #todo: if we decide not to create new user
text = msg.html_content or msg.plain_content
comment(
'New email: subject: {subject}, emages: {image_names} sent by {sender}',
subject=msg.subject,
image_names=msg.images.keys(),
sender=msg.sender_email)
if msg.images:
result = save_uploaded_photo_collection(msg.images, user_id)
comment("upload result {result}", result=result)
return results
def get_children(member_id, hidden_too=False):
member_rec = get_member_rec(member_id)
db, VIS_NEVER = inject('db', 'VIS_NEVER')
if member_rec.gender == 'F':
q = db.TblMembers.mother_id == member_id
elif member_rec.gender == 'M':
q = db.TblMembers.father_id == member_id
else:
return [] #error!
if not hidden_too:
q &= (db.TblMembers.visibility != VIS_NEVER)
q &= (db.TblMembers.deleted == False)
lst = db(q).select(db.TblMembers.id,
db.TblMembers.date_of_birth,
orderby=db.TblMembers.date_of_birth)
lst = [get_member_rec(rec.id, prepend_path=True) for rec in lst]
return lst
def fit_size(rec):
db, log_exception = inject('db', 'log_exception')
fname = local_photos_folder() + rec.photo_path
try:
img = Image.open(fname)
oversize_file_name = local_photos_folder("oversize") + rec.photo_path
oversize_path, f = os.path.split(oversize_file_name)
dir_util.mkpath(oversize_path)
img.save(oversize_file_name)
width, height = resized(rec.width, rec.height)
img = img.resize((width, height), Image.LANCZOS)
img.save(fname)
rec.update_record(width=width, height=height)
except Exception, e:
log_exception("resizing file {} failed.".format(rec.photo_path))
rec.update_record(photo_missing=True)
return False
def fit_all_sizes():
db, request, comment = inject('db', 'request', 'comment')
comment('started fitting size for chunk of photos')
q = (db.TblPhotos.width > MAX_WIDTH) | (db.TblPhotos.height > MAX_HEIGHT)
q &= (db.TblPhotos.photo_missing != True)
chunk = 100
num_failed = 0
while True:
n = db(q).count()
comment('started fitting size for chunk of photos. {} more to go.'.format(n))
if db(q).count() == 0:
break
lst = db(q).select(limitby=(0, chunk))
for rec in lst:
if not fit_size(rec):
num_failed += 1
db.commit()
return dict(num_failed=num_failed)
def extract_story_words(story_id):
from injections import inject
db = inject('db')
rec = db(db.TblStories.id == story_id).select().first()
if not rec:
return None
story_name = rec.name
html = rec.story
s = story_name.decode('utf8') + ' ' + remove_all_tags(html)
lst = extract_words(s)
if not lst:
return None
dic = dict()
for w in lst:
if w not in dic:
dic[w] = 0
dic[w] += 1
return dic
def _calc_used_languages(used_for):
db = inject('db')
dic = {}
q = (db.TblStories.id > 0)
if used_for:
q &= (db.TblStories.used_for == used_for)
for rec in db(q).select():
lang = rec.language
if lang not in dic:
dic[lang] = 0
dic[lang] += 1
lst = []
for lang in sorted(dic):
item = dict(id=lang,
name='stories.' + language_name(lang),
count=dic[lang])
lst.append(item)
return dict(used_languages=lst)
def get_member_rec(member_id, member_rec=None, prepend_path=False):
db = inject('db')
if member_rec:
rec = member_rec #used when initially all members are loaded into the cache
elif not member_id:
return None
else:
rec = db(db.TblMembers.id == member_id).select().first()
if not rec:
return None
if rec.deleted:
return None
rec = Storage(rec.as_dict())
rec.full_name = member_display_name(rec, full=True)
rec.name = member_display_name(rec, full=False)
if prepend_path:
rec.facePhotoURL = photos_folder('profile_photos') + (
rec.facePhotoURL or 'dummy_face.png')
return rec
def notify_registration(self, user_info):
db, mail = inject('db', 'mail')
ui = user_info
user_name = ui.first_name + ' ' + ui.last_name
email = ui.email
lst = db((db.auth_membership.group_id == ADMIN)
& (db.auth_user.id == db.auth_membership.user_id)).select(
db.auth_user.email)
receivers = [r.email for r in lst]
message = ('', '''
{uname} has just registered to <b>GB Stories</b>.
Email adddress is {uemail}.
Click <a href="http://gbstories.org/gbs__www/stories">here</a> for access manager.
'''.format(uname=user_name, uemail=email).replace('\n', '<br>'))
mail.send(to=receivers,
subject='New GB Stories registration',
message=message)
def project(network, freqs, h, theta, phi, psi, data, figtuple=None, dh=None, units="$1/\sqrt{\mathrm{Hz}}$"):
"""
projects the strain "h" onto the network and overlays it with "data" at each detector
if supplied, "dh" is the upper/lower bounds for h and will be shaded on the plots
"""
n_ifo = len(network)
n_freqs = len(freqs)
n_f, n_pol = np.shape(h)
if n_f != (n_freqs):
raise ValueError, "bad shape for h"
if (dh != None) and (np.shape(dh) != (n_freqs, n_pol, 2)):
raise ValueError, "bad shape for dh"
if np.shape(data) != (n_freqs, n_ifo):
raise ValueError, "bad shape for data"
n_axs = n_ifo + n_pol
if figtuple:
fig, axs = figtuple
if len(axs) != n_axs:
raise ValueError, "inconsistent number of ifos between network and figtuples"
else:
fig = plt.figure()
axs = [plt.subplot(n_axs, 1, i + 1) for i in xrange(n_axs)]
figtuple = (fig, axs)
iax = 0
### geocenter h
for pol in xrange(n_pol):
ax = axs[iax]
ax.plot(freqs, h[:, pol].real, color="b", label="$\mathrm{Real}\{h_%d\}$" % pol)
if np.any(h.imag):
ax.plot(freqs, h[:, pol].imag, color="r", label="$\mathrm{Imag}\{h_%d\}$" % pol)
if dh != None:
ax.fill_between(freqs, dh[:, pol, 0].real, dh[:, pol, 1].real, color="b", alpha=0.25)
if np.any(dh.imag):
ax.fill_between(freqs, dh[:, pol, 0].imag, dh[:, pol, 1].imag, color="r", alpha=0.25)
ax.set_ylabel("$h_%d$ [%s]" % (pol, units))
iax += 1
### inject h into the network
inj = injections.inject(network, h, theta, phi, psi=psi)
if dh != None:
dinj0 = np.transpose(injections.inject(network, dh[:, 0], theta, phi, psi=psi))
dinj1 = np.transpose(injections.inject(network, dh[:, 1], theta, phi, psi=psi))
else:
dinj0 = dinj1 = [None] * n_ifo
### iterate and plot
for (name, i, di0, di1, d) in zip(
network.detector_names_list(), np.transpose(inj), dinj0, dinj1, np.transpose(data)
):
ax = axs[iax]
ax.plot(freqs, d.real, color="c", label="$\mathrm{Real}{d_{\mathrm{%s}}}$" % name)
if np.any(d.imag):
ax.plot(freqs, d.imag, color="m", label="$\mathrm{Imag}{d_{\mathrm{%s}}}$" % name)
ax.plot(freqs, i.real, color="b", label="$\mathrm{Real}\{F*h\}$")
if np.any(i.imag):
ax.plot(freqs, i.imag, color="r", label="$\mathrm{Imag}\{F*h\}$")
if di0 != None:
ax.fill_between(freqs, di0, di1, color="b", alpha=0.25)
if np.any(di0.imag):
ax.fill_between(freqs, di0, di1, color="r", alpha=0.25)
iax += 1
return figtuple
print """injecting SineGaussian with parameters
fo = %f
to = %f
phio = %f
tau = %f
hrss = %fe-23
alpha = %f
theta = %f
phi = %f
psi = %f"""%(fo, t, phio, tau, hrss*1e23, alpha, theta, phi, psi)
if opts.time:
to = time.time()
h = injections.sinegaussian_f(freqs, to=t, phio=phio, fo=fo, tau=tau, hrss=hrss, alpha=alpha)
inj = injections.inject( network, h, theta, phi, psi)
if opts.time:
print "\t", time.time()-to
elif config.has_option("injection","cache"): ### read injection frame
inj = np.empty((n_freqs, n_ifo), complex)
inj_cache = eval(config.get("injection","cache"))
inj_chans = eval(config.get("injection","channels"))
### pull out parameters for PSD estimation
dseg = 0.5*seglen
for ifo_ind, ifo in enumerate(ifos):
to=time.time() ### antenna pattern jazz theta = theta_inj[inj_id] phi = phi_inj[inj_id] psi = psi_inj[inj_id] ### singal amplitude hrss = hrss_inj[inj_id] snrs = snrs_inj[inj_id] snr_net = np.sum(snrs**2)**0.5 print "\t\t", time.time()-to ### generate data stream print "\tgenerating data" to=time.time() data_inj = injections.inject(network, waveform_func(freqs, hrss=hrss, **waveform_args), theta, phi, psi=psi) print "\t\t", time.time()-to injang = (theta, phi) if opts.zero_noise: noise = np.zeros((n_freqs, n_ifo), complex) else: print "\tdrawing noise" to=time.time() noise = network.draw_noise() print "\t\t", time.time()-to data = data_inj + noise print "\tset_data"
