def post(self):
if login.isAdmin():
isActive = self.request.get('active') == 'True'
activeProperty = Configuration.all().filter('name', 'checkin').get()
if (activeProperty):
activeProperty.value = isActive
else:
activeProperty = Configuration(name='checkin', value=isActive)
activeProperty.put()
def createConfiguration(request):
if not admin():
return HttpResponseRedirect(users.create_login_Configuration('/blogs'))
form = ConfigurationForm(request.POST)
if form.is_valid():
configuration = Configuration()
configuration.title = form.cleaned_data['title']
configuration.motto = form.cleaned_data['motto']
configuration.put()
return HttpResponseRedirect('/blogs')
else:
return newConfiguration(request)
def test_set_kmer_freq_promoter(self):
k = 7
exp_setting = Configuration()
exp_setting.set_kmer_size(kmer_size=k)
conn = Pgsql.Common.connect(settings.conn_string_test)
#gnid = 58737
gnid = 76
seq_type = 'm1'
gs_pep = GeneSequence(gnid=gnid, seq_type=seq_type, is_max_seq_len=True, conn=conn, k=k)
print(gs_pep.get_seq_str())
kf = gs_pep.get_kmer_freq(k=k)
kf.print(sort_type=4, limit=10)
def register_new_computer(name, uid, distribution, site, configuration):
"""Register a new computer with the admin system - after registration, the
computer will be submitted for approval."""
try:
new_pc = PC.objects.get(uid=uid)
package_list = new_pc.package_list
custom_packages = new_pc.custom_packages
except PC.DoesNotExist:
new_pc = PC(name=name, uid=uid)
new_pc.site = Site.objects.get(uid=site)
# TODO: Better to enforce existence of package list in AfterSave
# signal.
package_list = PackageList(name=name)
custom_packages = CustomPackages(name=name)
new_pc.distribution = Distribution.objects.get(uid=distribution)
new_pc.is_active = False
# Create new configuration, populate with data from computer's config.
# If a configuration with the same ID is hanging, reuse.
config_name = '_'.join([site, name, uid])
try:
my_config = Configuration.objects.get(name=config_name)
except Configuration.DoesNotExist:
my_config = Configuration()
my_config.name = config_name
finally:
# Delete pre-existing entries
entries = ConfigurationEntry.objects.filter(
owner_configuration=my_config)
for e in entries:
e.delete()
my_config.save()
# And load configuration
for k, v in configuration.items():
entry = ConfigurationEntry(key=k, value=v,
owner_configuration=my_config)
entry.save()
# Tell us about yourself
new_pc.do_send_package_info = True
# Set and save PmC
new_pc.configuration = my_config
package_list.save()
new_pc.package_list = package_list
custom_packages.save()
new_pc.custom_packages = custom_packages
new_pc.save()
return 0
def popular(section_id = None):
results = Configuration.query(Configuration.key == "OPHAN_API_KEY")
if not results.iter().has_next():
return None
ophan_api_key = results.iter().next().value
logging.info(ophan_api_key)
most_read_url = "http://api.ophan.co.uk/api/mostread"
if section_id:
most_read_url = most_read_url + "/" + section_id
params = {'age' : FIFTEEN_MINUTES,
'api-key' : ophan_api_key}
most_read_url = most_read_url + "?" + urllib.urlencode(params)
logging.info(most_read_url)
result = fetch(most_read_url)
if result.status_code == 200:
return result.content
logging.error("Ophan read failed with status code %d" % result.status_code)
return None
def get(self):
config = memcache.get('config') or Configuration.all().get()
notify_descriptions = (memcache.get('ndesc') or
NotificationTypes.all())
if config:
template_values = {
'key_id': config.key_id,
'vcode': config.vcode,
'rcpt_char': config.rcpt_char,
'rcpt_org': config.rcpt_org,
'rcpt_org2': config.rcpt_org2 or '',
'dest_email': config.dest_email,
'notify_types': config.notify_types,
'notify_descriptions': notify_descriptions,
}
else:
template_values = {
'key_id': '',
'vcode': '',
'rcpt_char': '',
'rcpt_org': '',
'rcpt_org2': '',
'dest_email': '',
'notify_types': [],
'notify_descriptions': notify_descriptions,
}
template = jinja_environment.get_template('index.html')
self.response.out.write(template.render(template_values))
async def get_result(index: Index = Path(..., title="The name of the Index")):
config = Configuration(index=index)
try:
result = await workflow_runner.run(config)
return JSONResponse(status_code=200, content=result)
except Exception as e:
raise MyException(e)
def popular_by_country(country_code="gb", hours="1", section_id=None, referrer=None):
results = Configuration.query(Configuration.key == "OPHAN_API_KEY")
if not results.iter().has_next():
return None
ophan_api_key = results.iter().next().value
#logging.info(ophan_api_key)
most_read_url = "http://api.ophan.co.uk/api/mostread"
if section_id:
most_read_url = most_read_url + "/" + section_id
params = {'hours' : hours,
'country' : country_code,
'api-key' : ophan_api_key,
}
if referrer:
params['referrer'] = referrer
most_read_url = most_read_url + "?" + urllib.urlencode(params)
#logging.info(most_read_url)
result = fetch(most_read_url)
if result.status_code == 200:
return result.content
logging.error("Ophan read failed with status code %d" % result.status_code)
return None
def sendInitialEmail(order):
status_base_url = Configuration().getValue('espa.status.url')
status_url = ('%s/%s') % (status_base_url, order.email)
header = (
"""Thank you for your order ( %s ). Your order has been received and is currently being processed.
You will receive an email notification when all units on this order have been completed.
You can check the status of your order and download already completed scenes directly from %s
Requested scenes:\n""") % (order.orderid, status_url)
scenes = Scene.objects.filter(order__id=order.id)
ordered = header
if scenes:
for s in scenes:
ordered = ordered + s.name + '\n'
#configure all these values
msg = MIMEText(ordered)
msg['Subject'] = 'Processing order received.'
msg['To'] = order.email
msg['From'] = '*****@*****.**'
s = SMTP(host='gssdsflh01.cr.usgs.gov')
s.sendmail('*****@*****.**', order.email, msg.as_string())
s.quit()
def instagram_push_update():
data = request.stream.read()
instagram_client_secret = Configuration.value_for_name('instagram_client_secret')
server_signature = request.headers.get('X-Hub-Signature', None)
signature = hmac.new(str(instagram_client_secret), data, digestmod=hashlib.sha1).hexdigest()
if server_signature != signature:
logger.warn('Got PuSH subscribe POST from instagram w/o valid signature: sent=%s != expected=%s',
server_signature, signature)
raise ndb.Return('')
logger.info('Got PuSH body: %s', data)
logger.info('Got PuSH headers: %s', request.headers)
parsed_feed = json.loads(data)
user_ids = [int(x.get('object_id')) for x in parsed_feed]
feeds = InstagramFeed.query(InstagramFeed.user_id.IN(user_ids))
cursor = None
more = True
keys = []
while more:
feed_keys, cursor, more = feeds.fetch_page(100, keys_only=True, start_cursor=cursor)
keys += feed_keys
keys = ','.join([x.urlsafe() for x in keys])
if keys:
yield Queue('poll').add_async(Task(url=url_for('tq_feed_poll'), method='POST', params={'keys': keys}))
raise ndb.Return('ok')
def get(self):
template = jinja_environment.get_template('admin/configuration.html')
template_values = {}
template_values['configuration'] = Configuration.query()
self.response.out.write(template.render(template_values))
def get(self):
template = jinja_environment.get_template('admin/configuration.html')
template_values = {}
template_values['configuration'] = Configuration.query()
self.response.out.write(template.render(template_values))
def lookup(key): results = Configuration.query(Configuration.key == key) if not results.iter().has_next(): return None key_value = results.iter().next().value return key_value
def configuration_upload(request):
if request.method == 'POST':
# script= request.POST.get('script')
# print type(script)
print request
myFile = request.FILES.get('script', None)
name = request.POST.get('name')
describe = request.POST.get('describe')
print name, describe
if name:
save_name = name
else:
save_name = myFile
# print(myFile._size) # 文件大小字节数
if Configuration.objects.filter(configuration_name=myFile).exists():
return render_to_response('400.html', {'info': '脚本已经存在'})
data = myFile.read()
job_obj = Configuration()
job_obj.configuration_name = save_name
job_obj.info = data
job_obj.describe = describe
job_obj.save()
return render_to_response('configuration_upload.html',
{'username': request.user.username})
else:
return render_to_response('configuration_upload.html',
{'username': request.user.username})
def post(self, request, format=None):
try:
tkn = request.META["HTTP_AUTHORIZATION"]
except:
return Response({'status': 'error','errorCode':'101', 'errorMessage':'Invalid Token'})
logger.debug("Token = " + tkn)
ret,user = get_user_by_token(tkn)
if ret == False:
return Response({'status': 'error','errorCode':'102', 'errorMessage':'Invalid Token'})
serializer = ConfigurationSerializer(data=request.data)
if serializer.is_valid():
config = json.dumps(serializer.data['construct_list'])
name = serializer.data['name']
config_obj = Configuration()
config_obj.name = name
config_obj.construct_list = config
config_obj.last_modified_by = user
config_obj.submit = serializer.data['submit']
config_obj.save()
logger.debug("Configuration created successfully")
serializer = ConfigurationGetSerializer(config_obj)
config_details = serializer.data
config_details['construct_list'] = json.loads(config_details['construct_list'])
return Response(config_details, status=status.HTTP_201_CREATED)
logger.error("Invalid Json ")
return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
def lookup(key, default=None): results = Configuration.query(Configuration.key == key) if not results.iter().has_next(): return default key_value = results.iter().next().value logging.info(key_value) return key_value
def lookup(key, default):
results = Configuration.query(Configuration.key == key)
if not results.iter().has_next():
return default
key_value = results.iter().next().value
return key_value
class TestPredictionResults(unittest.TestCase):
gnids = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
assigned_gnids = [1, 2, 3, 4, 5]
corresp_tissue = 1
genes = Genes(gnids)
class_size = 2
fold_size = 2
kmer_size = 3
target_features = '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23'
exp_setting = Configuration()
exp_setting.set_target_features(target_features=target_features)
#exp_setting.set_neg_class_mode(settings.NEG_CLASS_MODE_RND_S)
exp_setting.set_neg_class_mode(settings.NEG_CLASS_MODE_NOT_P)
def test_prediction_results(self):
all_prediction_results = list()
for corresp_tissue in range(1, 24):
cv = CrossValidation(genes=self.genes,
all_gnids=self.gnids,
class_size=self.class_size,
fold_size=self.fold_size,
kmer_size=self.kmer_size,
exp_setting=self.exp_setting)
cv.build_datasets(assigned_genes=self.assigned_gnids,
neg_class_mode=self.exp_setting.get_neg_class_mode(),
corresp_tissue=corresp_tissue)
prediction_results = cv.validation()
all_prediction_results.append(prediction_results)
feature_vector = dict()
for gnid in self.gnids:
gnid_vector = list()
for prediction_results in all_prediction_results:
if prediction_results:
gnid_pr = prediction_results.get(gnid)
if gnid_pr is None:
gnid_vector.append('?')
else:
gnid_vector.append(gnid_pr.get_predicted_class())
feature_vector[gnid] = gnid_vector
# show feature_vector
for tissue in range(1, 24):
print('Tissue#:', tissue)
for gnid, vector in feature_vector.items():
line = ",".join(str(value) for value in vector)
p_results = all_prediction_results[tissue - 1].get(gnid)
if p_results is None:
data_label = '?'
else:
data_label = 'data_label:{}'.format(p_results.get_assigned_class())
print("%s,%s,%s\n" % (gnid, line, data_label))
def instagram_subscribe():
mode = request.args['hub.mode']
challenge = request.args['hub.challenge']
verify_token = request.args.get('hub.verify_token')
if mode == 'subscribe':
instagram_verify_token = Configuration.value_for_name('instagram_verify_token')
if verify_token and verify_token != instagram_verify_token:
logger.info('Failed verification feed.verify_token:%s GET verify_token:%s', instagram_verify_token, verify_token)
return "Failed Verification", 400
logger.info('Responding to instagram challange: %s', challenge)
return challenge
def new_configuration():
"""
Add a new album
"""
form = ConfigurationForm(request.form)
if request.method == 'POST':
# save the album
configuration = Configuration()
save_changes_configuration(configuration, form, new=True)
return redirect('/configurations')
return render_template('new_configuration.html', form=form)
def updateConfiguration(request, key):
if not admin():
return HttpResponseRedirect(users.create_login_Configuration("/blogs"))
form = ConfigurationForm(request.POST)
if form.is_valid():
configuration = Configuration.get(key)
configuration.title = form.cleaned_data["title"]
configuration.motto = form.cleaned_data["motto"]
configuration.pagesize = form.cleaned_data["pagesize"]
configuration.put()
return HttpResponseRedirect("/blogs")
else:
return editConfiguration(request, key)
def get(self):
config = memcache.get('config') or Configuration.all().get()
if not config:
# We haven't set up our configuration yet, so don't try to do anything
return
elink_api = elink_appengine.AppEngineAPI(api_key=(config.key_id, config.vcode))
elink_char = evelink.char.Char(config.rcpt_char, api=elink_api)
elink_eve = evelink.eve.EVE(api=elink_api)
recips = set([config.rcpt_org])
if config.rcpt_org2:
recips.add(config.rcpt_org2)
headers = elink_char.messages()
message_ids = set(h['id'] for h in headers if h['to']['org_id'] in recips)
headers = dict((h['id'], h) for h in headers)
message_ids_to_relay = set()
sender_ids = set()
for m_id in message_ids:
seen = memcache.get('seen-%s' % m_id) or SeenMail.gql("WHERE mail_id = :1", m_id).get()
if not seen:
message_ids_to_relay.add(m_id)
sender_ids.add(headers[m_id]['sender_id'])
else:
memcache.set('seen-%s' % m_id, True)
if not message_ids_to_relay:
self.response.out.write("No pending messages.")
return
bodies = elink_char.message_bodies(message_ids_to_relay)
senders = elink_eve.character_names_from_ids(sender_ids)
e = EmailMessage()
e.to = config.dest_email
e.sender = '*****@*****.**'
for m_id in message_ids_to_relay:
sender = senders[headers[m_id]['sender_id']]
timestamp = headers[m_id]['timestamp']
e.subject = '[EVEMail] %s' % headers[m_id]['title']
e.html = self.format_message(bodies[m_id] or '', timestamp, sender)
e.send()
SeenMail(mail_id=m_id).put()
memcache.set('seen-%s' % m_id, True)
self.response.out.write("Processed message ID %s.<br/>\n" % m_id)
return
def get_activities_per_city(city: City = Path(
..., title="The name of the Ccity you want to scrap the activities")):
url_scraping = 'http://www.tripadvisor.com'
config = Configuration(
city=city #I'm passing this argument from the path that the user use
)
try:
result = workflow_runner.run(config, url_scraping)
return JSONResponse(status_code=200, content=result)
except Exception as e:
raise MyException(e)
def editConfiguration(request, key):
context = base_context()
if not admin():
return HttpResponseRedirect(users.create_login_Configuration('/blogs'))
configuration = Configuration.get(key)
if request.method == 'POST':
form = ConfigurationForm(request.POST)
else:
form = ConfigurationForm()
form.fields['title'].initial = configuration.title
form.fields['motto'].initial = configuration.motto
context.configuration = configuration
context.form = form
template = loader.get_template('blogs/editConfiguration.html')
return HttpResponse(template.render(context))
def get(self):
config = memcache.get('config') or Configuration.all().get()
notify_descriptions = (memcache.get('ndesc') or
NotificationTypes.all())
if not config:
# We haven't set up our configuration yet, so don't try to do anything
return
elink_api = elink_appengine.AppEngineAPI(api_key=(config.key_id, config.vcode))
elink_char = evelink.char.Char(config.rcpt_char, api=elink_api)
elink_eve = evelink.eve.EVE(api=elink_api)
self.send_emails(config, elink_api, elink_char, elink_eve)
self.send_notifications(config, elink_api, elink_char, elink_eve,
notify_descriptions)
def base_context():
context = Context({
'current_user':current_user(),
'admin':admin(),
'login_url':users.create_login_url('/blogs'),
'logout_url':users.create_logout_url('/blogs'),
'recent_comments':Comment.all().order('-date').fetch(5),
'categories':Category.all(),
'blogs_count':Blog.all().count(),
'archives':Archive.all().order('-year').order('-month'),
'friendlyURLs':FriendlyURL.all()
})
configuration=Configuration.all().fetch(1)
if configuration:
context.configuration=configuration[0]
return context
def editConfiguration(request, key):
context = base_context()
if not admin():
return HttpResponseRedirect(users.create_login_Configuration("/blogs"))
configuration = Configuration.get(key)
if request.method == "POST":
form = ConfigurationForm(request.POST)
else:
form = ConfigurationForm()
form.fields["title"].initial = configuration.title
form.fields["motto"].initial = configuration.motto
form.fields["pagesize"].initial = configuration.pagesize
context.configuration = configuration
context.form = form
template = loader.get_template("blogs/editConfiguration.html")
return HttpResponse(template.render(context))
def get_result(country: Country = Path(
...,
title="The name of the country you want to scrap the best cities to explore"
)):
url_scraping = 'http://www.tripadvisor.com'
config = Configuration(
country=
country #I'm passing this argument from the path that the user use
)
try:
result = workflow_runner.get_cities(config, url_scraping)
return JSONResponse(status_code=200, content=result)
except Exception as e:
raise MyException(e)
def start (self):
self.config = Configuration.get_dictionary()
if self.config.__len__() < 1: raise Exception, "Need configuration fixture"
for conf in self.config:
try:
x = eval(self.config[conf])
if x.__class__ == bool:
self.config[conf] = x
except:
pass
# import carrier class from simpleoperator
self.carrier = eval("%s()" % self.config['carrier'])
# storing backend for USSD interaction
self.backend = self._router.backends[-1]
# set alias for logs
self.me = self.config['local_alias']
self.keywords = []
self.keywords.append({'id': self.BALANCE_ID, 'keyw': self.config['keyword_balance']})
self.keywords.append({'id': self.TOPUP_ID, 'keyw': self.config['keyword_topup']})
# triggers initialization
try:
self.balance_allow = import_function(self.config['balance_allow_helper'])
except:
self.balance_allow = None
try:
self.balance_followup = import_function(self.config['balance_followup'])
except:
self.balance_followup = None
try:
self.topup_allow = import_function(self.config['topup_allow_helper'])
except:
self.topup_allow = None
try:
self.topup_followup = import_function(self.config['topup_followup'])
except:
self.topup_followup = None
# Registering loops
self.router.call_at(to_seconds(self.config['balance_check_interval']), self.period_balance_check)
self.log = self._router.log
pass
def markSceneComplete(name,
orderid,
processing_loc,
completed_file_location,
destination_cksum_file=None,
log_file_contents=""):
print("Marking scene:%s complete for order:%s" % (name, orderid))
o = Order.objects.get(orderid=orderid)
s = Scene.objects.get(name=name, order__id=o.id)
if s:
s.status = 'complete'
s.processing_location = processing_loc
s.product_distro_location = completed_file_location
s.completion_date = datetime.datetime.now()
s.cksum_distro_location = destination_cksum_file
#if source_l1t_location is not None:
#s.source_distro_location = source_l1t_location
s.log_file_contents = log_file_contents
#Need to modify this as soon as we're going to start
#providing more than 1 product
base_url = Configuration().getValue('distribution.cache.home.url')
product_file_parts = completed_file_location.split('/')
product_file = product_file_parts[len(product_file_parts) - 1]
cksum_file_parts = destination_cksum_file.split('/')
cksum_file = cksum_file_parts[len(cksum_file_parts) - 1]
s.product_dload_url = ('%s/orders/%s/%s') % (base_url, orderid,
product_file)
s.cksum_download_url = ('%s/orders/%s/%s') % (base_url, orderid,
cksum_file)
s.save()
if o.order_source == 'ee':
#update ee
lta_service = lta.LtaServices()
lta_service.update_order(o.ee_order_id, s.ee_unit_id, 'C')
update_order_if_complete(o.orderid, s)
return True
else:
print("MarkSceneComplete:No scene was found with the name:%s" % name)
return False
def create_conf():
form = ConfigurationForm(request.form)
if request.method == 'POST' and form.validate():
try:
configuration = Configuration()
form.populate_obj(configuration)
db.session.add(configuration)
db.session.commit()
return redirect(url_for('conf', id=configuration.id))
except Exception as error:
flash("Error creating configuration.", category="danger")
app.logger.error("Error creating configuration {}\n{}".format(
error, traceback.format_exc()))
return render_template('forms/model.jinja',
form=form,
action=url_for('create_conf'),
section='other')
def base_context():
context = Context(
{
"current_user": current_user(),
"admin": admin(),
"login_url": users.create_login_url("/blogs"),
"logout_url": users.create_logout_url("/blogs"),
"recent_comments": Comment.all().order("-date").fetch(5),
"categories": Category.all(),
"blogs_count": Blog.all().count(),
"archives": Archive.all().order("-year").order("-month"),
"friendlyURLs": FriendlyURL.all(),
}
)
configuration = Configuration.all().fetch(1)
if configuration:
context.configuration = configuration[0]
else:
context.configuration = False
return context
def put(self, id):
isActive = Configuration.all().filter('name', 'checkin').get()
isAdmin = login.isAdmin()
isPlayer = login.isPlayer() and not isAdmin
user = users.get_current_user()
if not isAdmin and not isPlayer:
self.response.out.write('not_logged_in')
else:
request = json.loads(self.request.body)
status = request['status']
n_presents = Player.all().filter('status =', 'present').count()
if status != 'present' or n_presents < 10:
player = None
if isAdmin:
id = long(id)
player = Player.get_by_id(id)
logging.info('Admin set status to \'' + status + '\' for ' +
str(player.email))
elif isPlayer and (isActive is None or isActive.value):
player = Player.all()
player = player.filter('email =', user.email()).get()
logging.info(str(player.email) + ' set status to \'' +
status + '\'')
if player:
player.status = status
player.put()
#self.response.out.write('ok')
else:
logging.info(user.nickname() + ' (' + user.email() +
') tried to set status to \'' +
status + '\' but it was inactive')
self.response.set_status(401, 'La lista è bloccata o \
non sei più loggato')
else:
logging.info(user.nickname() + ' (' + user.email() +
') tried to set status to \'' + status +
'\' but it was full')
self.response.set_status(500, 'La lista è piena')
def load_configuration() -> None:
global configuration
global mock_configuration
global mock_configuration_file_observer
print('Loading configuration...')
path = f'{dirname(realpath(__file__))}/..'
raw_config = yaml.safe_load(open(f'{path}/{CONFIG_FILE_NAME}', 'r'))
configuration = Configuration(
path=path,
active_mock=raw_config.get('active_mock', None),
record_session=bool(raw_config.get('record', False)))
if configuration.active_mock:
load_mock_configuration()
if mock_configuration is not None:
print(f'Active mock configuration: {configuration.active_mock}')
mock_configuration_file_observer = observe_file_modifications(
path=mock_configuration.path, update=load_mock_configuration)
else:
print('Failed to load mock configuration...')
else:
if mock_configuration is not None:
mock_configuration = None
print('Mock configuration disabled')
else:
pass
if mock_configuration_file_observer is not None:
mock_configuration_file_observer.stop()
mock_configuration_file_observer.join()
mock_configuration_file_observer = None
else:
pass
return None
class TestCrossValidation(unittest.TestCase):
gnids = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
assigned_gnids = [1, 2, 3, 4, 5]
corresp_tissue = 1
genes = Genes(gnids)
class_size = 2
fold_size = 2
kmer_size = 3
target_features = '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23'
exp_setting = Configuration()
exp_setting.set_target_features(target_features=target_features)
#exp_setting.set_neg_class_mode(settings.NEG_CLASS_MODE_RND_S)
exp_setting.set_neg_class_mode(settings.NEG_CLASS_MODE_NOT_P)
def test_cross_validation(self):
all_prediction_results = list()
for corresp_tissue in range(1, 24):
cv = CrossValidation(genes=self.genes,
all_gnids=self.gnids,
class_size=self.class_size,
fold_size=self.fold_size,
kmer_size=self.kmer_size,
exp_setting=self.exp_setting)
cv.build_datasets(assigned_genes=self.assigned_gnids,
neg_class_mode=self.exp_setting.get_neg_class_mode(),
corresp_tissue=corresp_tissue)
prediction_results = cv.validation()
all_prediction_results.append(prediction_results)
feature_vector = dict()
for gnid in self.gnids:
gnid_vector = list()
for prediction_results in all_prediction_results:
if prediction_results:
gnid_pr = prediction_results.get(gnid)
if gnid_pr is None:
gnid_vector.append('?')
else:
gnid_vector.append(gnid_pr.get_predicted_class())
feature_vector[gnid] = gnid_vector
def post(self):
#email = users.get_current_user().email()
isActive = Configuration.all().filter('name', 'checkin').get()
isAdmin = login.isAdmin()
isPlayer = login.isPlayer() and not isAdmin
user = users.get_current_user()
if not isAdmin and not isPlayer:
self.response.out.write('not_logged_in')
else:
n_presents = Player.all().filter('status =', 'present').count()
status = self.request.get('status')
if status != 'present' or n_presents < 10:
player = None
id = self.request.get('id')
if isAdmin and id:
id = long(id)
player = Player.get_by_id(id)
logging.info('Admin set status to \'' +
status + '\' for ' + str(player.email))
elif isActive == None or isActive.value:
player = Player.all()
player = player.filter('email =', user.email()).get()
logging.info(str(player.email) + ' set status to \'' +
status + '\'')
if player:
player.status = status
player.put()
self.response.out.write('ok')
else:
logging.info(user.nickname() + ' (' + user.email() +
') tried to set status to \'' + status +
'\' but it was inactive')
self.response.out.write('inactive')
else:
logging.info(user.nickname() + ' (' + user.email() +
') tried to set status to \'' + status +
'\' but it was full')
self.response.out.write('full')
def sendCompletionEmail(email, ordernum, readyscenes=[]):
status_base_url = Configuration().getValue('espa.status.url')
status_url = ('%s/%s') % (status_base_url, email)
msg = ("""Your order is now complete and can be downloaded from %s
This order will remain available for 14 days. Any data not downloaded will need to be reordered after this time.
Please contact Customer Services at 1-800-252-4547 or email [email protected] with any questions.
Your scenes
-------------------------------------------\n""") % (status_url)
for r in readyscenes:
msg = msg + r + '\n'
#configure these values
msg = MIMEText(msg)
msg['Subject'] = 'Processing for %s complete.' % (ordernum)
msg['To'] = email
msg['From'] = '*****@*****.**'
s = SMTP(host='gssdsflh01.cr.usgs.gov')
s.sendmail('*****@*****.**', email, msg.as_string())
s.quit()
def get(self):
config = memcache.get('config') or Configuration.all().get()
if config:
template_values = {
'key_id': config.key_id,
'vcode': config.vcode,
'rcpt_char': config.rcpt_char,
'rcpt_org': config.rcpt_org,
'rcpt_org2': config.rcpt_org2 or '',
'dest_email': config.dest_email,
}
else:
template_values = {
'key_id': '',
'vcode': '',
'rcpt_char': '',
'rcpt_org': '',
'rcpt_org2': '',
'dest_email': '',
}
template = jinja_environment.get_template('index.html')
self.response.out.write(template.render(template_values))
def upload_configuration(self, request):
if request.GET.get('source') == 'master':
data = request.DATA
myFile = request.FILES.get('script', None)
name = data.get('name')
describe = data.get('describe')
if name:
save_name = name
else:
save_name = myFile.name
# print(myFile._size) # 文件大小字节数
if Configuration.objects.filter(
configuration_name=save_name).exists():
return Response('脚本已经存在', status=status.HTTP_400_BAD_REQUEST)
data = myFile.read()
job_obj = Configuration()
job_obj.configuration_name = save_name
job_obj.info = data
job_obj.describe = describe
job_obj.save()
return Response("success", status=status.HTTP_200_OK)
subject="MtrHub: Remember to take a meter reading.",
body=body)
meter.set_next_reminder()
meter.put()
self.return_ok({})
routes = [
(r'/', Home), (r'/view_meter', ViewMeter), (r'/view_read', ViewRead),
(r'/edit_read', EditRead), (r'/send_read', SendRead), (r'/upload', Upload),
(r'/chart', Chart), (r'/meter_settings', MeterSettings),
(r'/view_reader', ViewReader), (r'/reader_settings', ReaderSettings),
(r'/welcome', Welcome), (r'/export_reads', ExportReads),
(r'/add_meter', AddMeter), (r'/cron', Cron),
(r'/cron/reminders', Reminders), (r'/add_reader', AddReader)]
template_names = dict(
[(cls.__name__, rt[1:] + '.html') for rt, cls in routes if rt != '/'])
template_names['Home'] = 'home.html'
conf = Configuration.all().get()
if conf is None:
session_key = ''.join(
random.choice(
string.ascii_uppercase + string.digits) for x in range(10)
).encode('ascii', 'ignore')
conf = Configuration(session_key=session_key)
conf.put()
config = {
'webapp2_extras.sessions': {
'secret_key': conf.session_key.encode('ascii', 'ignore')}}
app = webapp2.WSGIApplication(routes, debug=True, config=config)
def post(self):
key_id = self.request.get('key_id')
vcode = self.request.get('vcode')
rcpt_char = self.request.get('rcpt_char')
rcpt_org = self.request.get('rcpt_org')
rcpt_org2 = self.request.get('rcpt_org2')
dest_email = self.request.get('dest_email')
if not (key_id and vcode and rcpt_org and dest_email):
self.response.out.write("Missing one or more fields.")
return
try:
key_id = int(key_id)
except (ValueError, TypeError):
self.response.out.write("Key ID must be an integer.")
return
try:
rcpt_char = int(rcpt_char)
except (ValueError, TypeError):
rcpt_char = self.get_entity_id(rcpt_char)
try:
rcpt_org = int(rcpt_org)
except (ValueError, TypeError):
rcpt_org = self.get_entity_id(rcpt_org)
if not rcpt_org:
self.response.out.write("Invalid organization name/id.")
return
if rcpt_org2:
try:
rcpt_org2 = int(rcpt_org2)
except (ValueError, TypeError):
rcpt_org2 = self.get_entity_id(rcpt_org2)
if not rcpt_org2:
self.response.out.write("Invalid organization #2 name/id.")
return
config = Configuration.all().get()
if not config:
config = Configuration(
key_id=key_id,
vcode=vcode,
rcpt_char=rcpt_char,
rcpt_org=rcpt_org,
rcpt_org2=rcpt_org2,
dest_email=dest_email,
)
else:
config.key_id = key_id
config.vcode = vcode
config.rcpt_char = rcpt_char
config.rcpt_org = rcpt_org
if rcpt_org2:
config.rcpt_org2 = rcpt_org2
config.dest_email = dest_email
config.put()
memcache.set('config', config)
self.response.out.write("Configuration saved.")
return
def create(key, value):
config = Configuration(key = key, value = value)
key = config.put()
return key
from email.mime.text import MIMEText
from smtplib import *
from models import Scene
from models import Order
from models import Configuration
from espa.scene_cache import SceneCache
import json
import datetime
import lta
########################################################################################################################
#load configuration values at the module level...
########################################################################################################################
try:
smtp_url = Configuration().getValue('smtp.url')
espa_email_address = Configuration().getValue('espa.email.address')
order_status_base_url = Configuration().getValue('order.status.base.url')
except Exception, err:
print("Could not load configuration values:%s" % err)
########################################################################################################################
# Default product options
########################################################################################################################
def get_default_product_options():
'''returns default options for product selection'''
options = {}
#standard product selection options
options[
'include_sourcefile'] = False #delivers underlying raster product as part of order
def main(argv):
# global exp_setting
exp_setting = Configuration()
debug_mode = list()
reduced_mode = False
test_mode = False
seq_type = None
gene_prot = None
feature_set = list()
percentile_range = list()
args = parser.parse_args(argv[1:])
if len(argv) <= 1:
parser.parse_args(['--help'])
return
# set version
exp_setting.set_version(settings.DEV_VERSION)
# Show Version
version = exp_setting.get_version()
print('Version:', version.get_version())
# Get Cutoffs
cutoffs = Cutoffs()
cutoffs.query_cutoffs('95, 0, -5')
exp_setting.set_cutoffs(cutoffs)
print('Cutoffs data Initialized.')
# enable debugging mode
if args.enable_debug:
if set(args.enable_debug) & enable_debug:
#debug_mode = [1, 100000]
debug_mode = [1, 1000]
reduced_mode = True
if args.use_real_db:
if set(args.use_real_db) & choice_yes:
print('USING TEST DB: NO (USEING REAL/PRODUCTION DB)')
else:
settings.conn_string = settings.conn_string_test
print('USING TEST DB: YES')
if set(args.test_mode) & choice_yes:
exp_setting.set_test_mode(True)
print('TEST MODE: YES')
else:
exp_setting.set_test_mode(False)
print('TEST MODE: NO')
# ignore zero values
if set(args.ignore_zero) & choice_yes:
exp_setting.set_ignore_null(True)
print('Ignore zero values: YES')
else:
exp_setting.set_ignore_null(False)
print('Ignore zero values: NO')
# Gene info loading mode
if args.gene_load_mode:
if set(args.gene_load_mode) & gene_load_mode_pl:
exp_setting.set_gene_load_mode(settings.GN_LD_MODE_PL)
print('Gene loading mode: pre-load')
elif set(args.gene_load_mode) & gene_load_mode_dl:
exp_setting.set_gene_load_mode(settings.GN_LD_MODE_DL)
print('Gene loading mode: dynamic load')
# sequence type
if args.seq_type:
if set(args.seq_type) & seq_type_pep:
seq_type = 'p'
print('sequence type: amino acid (peptide)')
elif set(args.seq_type) & seq_type_dna:
seq_type = 'd'
print('sequence type: DNA')
elif set(args.seq_type) & seq_type_pmt:
seq_type = 'm1'
print('sequence type: Promoter data')
# set missing gnids in promoter data
exp_setting.set_missing_gnids_in_promoter()
elif set(args.seq_type) & seq_type_rda:
seq_type = 'p'
print('sequence type: Reduced Alphabet')
settings.RA_MODE = True
else: # default
seq_type = 'p'
print('sequence type: amino acid (peptide) - Default')
exp_setting.set_seq_type(seq_type)
# gp_type
gp_type = 'g'
if args.gp_type:
if set(args.gp_type) & gp_type_g:
gp_type = 'g'
print('gp type: g')
elif set(args.gp_type) & gp_type_p:
gp_type = 'p'
print('gp type: p')
elif set(args.gp_type) & gp_type_b:
gp_type = 'b'
print('gp type: b')
else:
gp_type = 'g'
print('gp type: g (default)')
exp_setting.set_gp_type(gp_type)
# assign feature groups
if args.feature_group:
if set(args.feature_group) & feature_group_gl:
print('new feature group: gene low expressed')
Features.gene_low_exp()
if set(args.feature_group) & feature_group_gh:
print('new feature group: gene high expressed, top 5%')
Features.gene_high_exp()
if set(args.feature_group) & feature_group_gh10:
print('new feature group: gene high expressed, top 10%')
Features.gene_high_exp_t10()
if set(args.feature_group) & feature_group_gt:
print('new feature group: gene for each tissue, top 10%')
Features.gene_tissues()
# feature set
if args.feature_set:
feature_set = args.feature_set
print('feature set: {}'.format(feature_set))
# set negative class mode
if args.neg_class_mode:
neg_class_mode = args.neg_class_mode
print('NEG_CLASS_MODE:', neg_class_mode)
if neg_class_mode in (settings.NEG_CLASS_MODE_NOT_P,
settings.NEG_CLASS_MODE_RND_S,
settings.NEG_CLASS_MODE_RND_M):
exp_setting.set_neg_class_mode(neg_class_mode)
else:
error_mesg = 'NEG_CLASS_MODE:', neg_class_mode, 'is UNKNOWN.'
raise ValueError(error_mesg)
# set percentile for new feature set
if args.percentile:
percentile_range = args.percentile.split(',')
percentile_range = [int(x)
for x in percentile_range] # str -> int type
print('Set percnetile range:', args.percentile)
# set gp combo configurations
feature_set_gp_comb = list()
if args.multi_gp:
multi_gp_conf = args.multi_gp
for conf in multi_gp_conf:
print(conf)
conf_list = conf.split(':')
feature_set_gp_comb.append(conf_list)
print(feature_set_gp_comb)
# class assignment for features
if args.features:
if set(args.features) & (settings.FN_GE_N | settings.FN_GE_B
| settings.FN_PA_N | settings.FN_PA_B):
if len(percentile_range) <= 0:
raise ValueError(
'percentile range is empty. Please set percentile range.')
'''
It supports adding multiple features at the same time, so it needs to do independently as belows.
'''
if set(args.features) & settings.FN_GE_N:
print('GE_N')
is_top = True
gp_type = 'g'
feature_set_name = next(iter(settings.FN_GE_N))
for percentile in range(percentile_range[0],
percentile_range[1],
percentile_range[2]):
add_feature_by_percentile(
gp_type=gp_type,
feature_set_name=feature_set_name,
percentile=percentile,
is_top=is_top)
if set(args.features) & settings.FN_GE_B:
print('GE_B')
is_top = False
gp_type = 'g'
feature_set_name = next(iter(settings.FN_GE_B))
for percentile in range(percentile_range[0],
percentile_range[1],
percentile_range[2]):
add_feature_by_percentile(
gp_type=gp_type,
feature_set_name=feature_set_name,
percentile=percentile,
is_top=is_top)
if set(args.features) & settings.FN_PA_N:
print('PA_N')
is_top = True
gp_type = 'p'
feature_set_name = next(iter(settings.FN_PA_N))
for percentile in range(percentile_range[0],
percentile_range[1],
percentile_range[2]):
add_feature_by_percentile(
gp_type=gp_type,
feature_set_name=feature_set_name,
percentile=percentile,
is_top=is_top)
if set(args.features) & settings.FN_PA_B:
print('PA_B')
is_top = False
gp_type = 'p'
feature_set_name = next(iter(settings.FN_PA_B))
for percentile in range(percentile_range[0],
percentile_range[1],
percentile_range[2]):
add_feature_by_percentile(
gp_type=gp_type,
feature_set_name=feature_set_name,
percentile=percentile,
is_top=is_top)
if set(args.features) & settings.FN_GPCB:
print('GE&PA Combination data')
for conf in feature_set_gp_comb:
add_feature_gp_comb(conf, exp_setting)
# build feature vector
if args.feature_vector:
intervals = [1000]
if set(args.feature_vector) & feature_vector:
print('build feature vector')
fs_set_idx = 0
#build_feature_vector()
if reduced_mode:
#for i in range(1,58938, interval):
for interval in intervals:
for i in range(1, 39324, interval):
#for i in range(16001,39324, interval):
debug_mode = [i, interval]
exp_setting.set_debug_mode(debug_mode)
build_feature_vector(exp_setting)
else:
for k in range(3, 8):
exp_setting.set_kmer_size(kmer_size=k)
exp_setting.set_genes_info(genes_info=None)
for fsid in feature_set:
# Version Info
print('Version:', settings.DEV_VERSION)
if fsid == 0:
# set feature info with dummy data for small assigned gene at random
fs_info = FeatureInfo(fsid=0,
fs_name='SM_RND',
gp_type='g',
class_size=2)
# Set assigned genes limit
assigned_genes_limit = [
int((x + 23 * fs_set_idx) * 10)
for x in range(1, 24)
]
exp_setting.set_assigned_genes_limit(
assigned_genes_limit)
fs_set_idx += 1
else:
# get feature set info from DB
res_fs_info = Pgsql.Common.select_data(
sqls.get_feature_set, (fsid))
fs_info = FeatureInfo(
fsid=fsid,
fs_name=res_fs_info[0][0].strip(),
gp_type=res_fs_info[0][1].strip(),
class_size=int(res_fs_info[0][2]))
exp_setting.set_fs_info(fs_info)
# for test
print(
'### MESSAGE ### fsid: {}, fs_name: {}, gp_type: {}, class_size: {}'
.format(exp_setting.get_fsid(),
exp_setting.get_fs_name(),
exp_setting.get_gp_type(),
exp_setting.get_class_size()))
debug_mode = [1, 0]
exp_setting.set_debug_mode(debug_mode)
build_feature_vector(exp_setting)
# single step classification
if args.validation_mode:
if set(args.validation_mode) & validation_mode_rg:
# reduced gene model
intervals = [1000, 2000, 3000, 4000, 5000]
print('validation - reduced genes model mode')
for interval in intervals:
for i in range(1, 39324, interval):
#for i in range(16001,39324, interval):
debug_mode = [i, interval]
exp_setting.set_debug_mode(debug_mode)
build_feature_vector(debug_mode=debug_mode,
gene_prot=gene_prot,
seq_type=seq_type)
class TestFeatureVector(unittest.TestCase):
# set target features
#target_features = '1'
#target_features = '23'
#target_features = '1,2,3,4'
#target_features = '4,7,9,10'
#target_features = '1,2,3,4,5,6,7,8,9,10'
#target_features = '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22'
#target_features = '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20'
target_features = '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23'
# set assigned genes limit
#assigned_genes_limit = [int(10) + x for x in range(1, 24)]
#assigned_genes_limit = [int(50) for x in range(1, 24)]
assigned_genes_limit = [int(10) for x in range(1, 24)]
exp_setting = Configuration()
def test_feature_vector_random(self):
self.exp_setting.set_target_features(
target_features=self.target_features)
self.exp_setting.set_assigned_genes_limit(self.assigned_genes_limit)
# set debug_mode
debug_mode = [1, 0]
self.exp_setting.set_debug_mode(debug_mode)
# set test_mode
# exp_setting.set_test_mode(test_mode=True)
# exp_setting.set_test_mode(test_mode=settings.TEST_MODE_KMER_FREQ)
self.exp_setting.set_test_mode(test_mode=settings.TEST_MODE_TRUE)
# set NEG class mode
self.exp_setting.set_neg_class_mode(settings.NEG_CLASS_MODE_RND_S)
# set class size
self.exp_setting.set_class_size(class_size=2)
# set fold size for cross-validation
self.exp_setting.set_fold_size(fold_size=10)
# set kmer size
self.exp_setting.set_kmer_size(kmer_size=3)
# set seq_type
self.exp_setting.set_seq_type(seq_type='p')
# set fsid
fs_info = FeatureInfo(fsid=0,
fs_name='SM_RND',
gp_type='g',
class_size=2)
self.exp_setting.set_fs_info(fs_info=fs_info)
# Feature Vector
fv = FeatureVector(self.exp_setting)
# fv.test_features_dataset()
fv.cross_validation()
# write prediction results into a file (.csv)
#fv.write_prediction_results()
# Build feature vectors with prediction results
#fv.build_feature_vector()
print('Build Feature Vector')
#gene_dataset = self.exp_setting.get_gene_dataset()
all_gnids_list = self.exp_setting.get_gene_dataset_all_gnids_list()
#for gnid in gene_dataset.get_all_gnids():
for gnid in all_gnids_list:
#for gnid in self.wd_all_gnids:
gnid_vector = list()
for feature in fv.features:
#if feature.prediction_results is not None:
if feature.prediction_results:
predicted_results = feature.prediction_results.get(
gnid, None)
if predicted_results is None:
gnid_vector.append('?')
else:
#gnid_vector.append(feature.prediction_results[gnid].get_predicted_class())
gnid_vector.append(
predicted_results.get_predicted_class())
#print(gnid_vector)
fv.feature_vector[gnid] = gnid_vector
# TEST write feature vector
for feature in fv.features:
print('Feature Name: {}, tissue#: {}'.format(
feature.name, feature.corresp_tissue))
# gnids
gnids = self.exp_setting.get_gene_dataset_gnids_list(
feature_id=feature.corresp_tissue)
#for gnid, vector in fv.feature_vector.items():
for gnid in gnids:
vector = fv.feature_vector.get(gnid, None)
if vector is not None:
line = ",".join(str(value) for value in vector)
predicted_results = feature.prediction_results.get(
gnid, None)
if predicted_results is None:
data_label = '?'
else:
data_label = predicted_results.get_assigned_class()
#f.write("%s,%s,%s\n" % (gnid, line, feature.prediction_results[gnid].get_assigned_class()))
print("%s,%s,%s\n" % (gnid, line, data_label))
#print('gnid:', gnid)
#print('line:', line)
#print('data_label:', data_label)
#fv.write_feature_vector()
#fv.create_arff()
#fv.write_prediction_summary()
def test_feature_vector_by_fsid(self):
self.exp_setting.set_target_features(
target_features=self.target_features)
self.exp_setting.set_assigned_genes_limit(self.assigned_genes_limit)
# set debug_mode
debug_mode = [1, 0]
self.exp_setting.set_debug_mode(debug_mode)
# set test_mode
# exp_setting.set_test_mode(test_mode=True)
# exp_setting.set_test_mode(test_mode=settings.TEST_MODE_KMER_FREQ)
self.exp_setting.set_test_mode(test_mode=settings.TEST_MODE_TRUE)
# set NEG class mode
self.exp_setting.set_neg_class_mode(settings.NEG_CLASS_MODE_RND_S)
# set class size
self.exp_setting.set_class_size(class_size=2)
# set fold size for cross-validation
self.exp_setting.set_fold_size(fold_size=10)
# set kmer size
self.exp_setting.set_kmer_size(kmer_size=3)
# set seq_type
self.exp_setting.set_seq_type(seq_type='p')
# set fsid
fsid = 44
res_fs_info = Pgsql.Common.select_data(sqls.get_feature_set, (fsid))
fs_info = FeatureInfo(fsid=fsid,
fs_name=res_fs_info[0][0].strip(),
gp_type=res_fs_info[0][1].strip(),
class_size=int(res_fs_info[0][2]))
self.exp_setting.set_fs_info(fs_info=fs_info)
# Feature Vector
fv = FeatureVector(self.exp_setting)
# fv.test_features_dataset()
fv.cross_validation()
# write prediction results into a file (.csv)
#fv.write_prediction_results()
# Build feature vectors with prediction results
#fv.build_feature_vector()
print('Build Feature Vector')
#gene_dataset = self.exp_setting.get_gene_dataset()
all_gnids_list = self.exp_setting.get_gene_dataset_all_gnids_list()
#for gnid in gene_dataset.get_all_gnids():
for gnid in all_gnids_list:
#for gnid in self.wd_all_gnids:
gnid_vector = list()
for feature in fv.features:
#if feature.prediction_results is not None:
if feature.prediction_results:
predicted_results = feature.prediction_results.get(
gnid, None)
if predicted_results is None:
gnid_vector.append('?')
else:
#gnid_vector.append(feature.prediction_results[gnid].get_predicted_class())
gnid_vector.append(
predicted_results.get_predicted_class())
#print(gnid_vector)
fv.feature_vector[gnid] = gnid_vector
# TEST write feature vector
for feature in fv.features:
print('Feature Name: {}, tissue#: {}'.format(
feature.name, feature.corresp_tissue))
# gnids
gnids = self.exp_setting.get_gene_dataset_gnids_list(
feature_id=feature.corresp_tissue)
#for gnid, vector in fv.feature_vector.items():
for gnid in gnids:
vector = fv.feature_vector.get(gnid, None)
if vector is not None:
line = ",".join(str(value) for value in vector)
predicted_results = feature.prediction_results.get(
gnid, None)
if predicted_results is None:
data_label = '?'
else:
data_label = predicted_results.get_assigned_class()
#f.write("%s,%s,%s\n" % (gnid, line, feature.prediction_results[gnid].get_assigned_class()))
print("%s,%s,%s\n" % (gnid, line, data_label))
def create(key, value):
config = Configuration(id=key, key=key, value=value)
config.put()
return config
def test_congistore_admin_handles_unknown_keys(self):
Configuration(key='unknown-key', site=Site.objects.get_current()).save()
self.login()
self.client.get(urlresolvers.reverse('admin:configstore_configuration_changelist'))
def run(self):
conf = Configuration()
conf.name = 'Aðalstilling'
db.session.add(conf)
db.session.commit()
def create(key, value): config = Configuration(id=key, key=key, value=value) config.put() return config
def get(self):
if login.isAdmin():
isActive = Configuration.all().filter('name', 'checkin').get()
self.response.out.write(isActive == None or isActive.value)
