class SettingsProxy(object):
def __init__(self):
self.settings = None
def __getattr__(self, name):
namespace = namespace_manager.get_namespace()
now = time.time()
if not self.settings or now >= _last_config_refreshes.get(namespace, now - 300) + 300:
self.settings = ndb.Key(Settings, '_settings').get(use_cache=False, use_memcache=False)
if not self.settings:
self.settings = Settings(id='_settings')
settings_updated = False
for s, v in default_settings.items():
if not hasattr(self.settings, s):
setattr(self.settings, s, v)
settings_updated = True
if settings_updated:
self.settings.put()
logging.info('Settings refreshed.')
_last_config_refreshes[namespace] = now
return getattr(self.settings, name)
class SettingsDictAction(argparse.Action):
"""Handles the settings dict with argparse"""
VALUE_DICT = {
"x264":
Settings(),
"nhevc":
Settings(
width=1920,
height=1080,
extension="_1080p",
parameter=EncodingParameters(
resolution=[
"1920x1080",
"1600x900",
"1280x720",
],
qp=[24, 29, 34, 40, 46],
gop_l=[1, 3, 6, 10, 20, 40, 120],
rate=[30, 15, 3],
),
ffmpeg_config=lambda params:
f"hevc_nvenc -g {params.gop_len} -qp {params.qp}",
),
}
def __call__(self, arg_parser, namespace, values, option_string=None):
setattr(namespace, self.dest, self.VALUE_DICT.get(values))
def read_config(p: Path = config_path):
try:
with open(p.absolute(), "r") as f:
return Settings(**json.load(f))
except Exception as e:
logger.error(f"Issue reading settings.json, {e}")
return Settings(**default_config)
def _load_model():
# These imports are slow - lazy import.
import tensorflow as tf
from data import Vocab
from model import Hps, Settings, SummarizationModel
global _settings, _hps, _vocab, _sess, _model
# Define settings and hyperparameters
_settings = Settings(
embeddings_path='',
log_root='',
trace_path='', # traces/traces_blog',
)
_hps = Hps(
# parameters important for decoding
attn_only_entities=False,
batch_size=_beam_size,
copy_only_entities=False,
emb_dim=128,
enc_hidden_dim=200,
dec_hidden_dim=300,
max_dec_steps=1,
max_enc_steps=400,
mode='decode',
output_vocab_size=20000,
restrictive_embeddings=False,
save_matmul=False,
tied_output=True,
two_layer_lstm=True,
# other parameters
adagrad_init_acc=.1,
adam_optimizer=True,
copy_common_loss_wt=0.,
cov_loss_wt=0.,
high_attn_loss_wt=0.,
lr=.15,
max_grad_norm=2.,
people_loss_wt=0.,
rand_unif_init_mag=.02,
scatter_loss_wt=0.,
sharp_loss_wt=0.,
trunc_norm_init_std=1e-4,
)
# Define model
_vocab = Vocab(_vocab_path, _vocab_size)
_model = SummarizationModel(_settings, _hps, _vocab)
_model.build_graph()
# Load model from disk
saver = tf.train.Saver()
config = tf.ConfigProto(
allow_soft_placement=True,
#intra_op_parallelism_threads=1,
#inter_op_parallelism_threads=1,
)
_sess = tf.Session(config=config)
ckpt_state = tf.train.get_checkpoint_state(_model_dir)
saver.restore(_sess, ckpt_state.model_checkpoint_path)
def get(self):
if self.get_argument('lp',""):
hAnnounce.add_cb(self.on_announce)
return
d=Settings.objects().first()
self.write({'announce':d.announce})
self.finish()
def ensure_global_settings():
current_namespace = namespace_manager.get_namespace()
namespace_manager.set_namespace('global')
settings = ndb.Key(Settings, '_settings').get()
settings_updated = False
if not settings:
logging.debug('Settings not found. Init.')
settings = Settings(id='_settings')
for s, v in default_settings.items():
if not hasattr(settings, s):
setattr(settings, s, v)
settings_updated = True
if settings_updated:
settings.put()
namespace_manager.set_namespace(current_namespace)
logging.info('Running with settings %s', settings)
def wizard():
print 'Welcome to external SPARQL endpoint to Virtuoso dumper.'
if not engine.dialect.has_table(engine.connect(), 'task'):
Base.metadata.create_all(engine)
settings = session.query(Settings).first()
if settings == None:
print 'It seems that is the first time that you use this tool. Pleas fulfill the following config parameters:'
virtuoso_endpoint = raw_input('Virtuoso SPARQL endpoint: ')
settings = Settings()
settings.virtuoso_endpoint = virtuoso_endpoint
session.add(settings)
session.commit()
print ''
exit = False
while not exit:
print 'a) Create a new dump task'
print 'b) Delete all tasks'
print 'c) Show tasks'
print 'd) Resume task'
print 'h) Exit'
option = raw_input('Select your choice: ')
if option == 'a':
create_task()
elif option == 'b':
clear_tasks()
elif option == 'c':
show_tasks()
elif option == 'd':
resume_task()
elif option == 'h':
print 'Bye!'
sys.exit(0)
else:
print 'Wrong option!'
def _get_namespace(domain):
global mapping
if mapping is None:
namespace_manager.set_namespace('global')
settings = ndb.Key(Settings, '_settings').get()
if not settings:
settings = Settings(id='_settings')
settings.set_mapping({})
settings.put()
mapping = settings.get_mapping_dict()
return mapping.get(domain, domain)
def post(self):
password=self.get_argument('password',"")
if password!=self.current_user.password:
return self.write({'err':'密码错误'})
d=Settings.objects().first()
if d.phase==6:
return self.write({'err':'不能再推进阶段'})
if d.phase in [2,4]:
clearSelection()
d.phase+=1
d.save()
self.write({'phase':d.phase})
def post(self):
password = self.request.POST.getone('password')
settings = Settings.get_by_id('1')
if password != settings.password:
self.response.set_status(403)
self.response.write('invalid password')
data = self.request.POST.getone('data')
if not data:
self.response.set_status(400)
self.response.write('no data given')
return
o = json.loads(data)
for badge in o:
logging.info('Updating %s' % badge)
b = self.update_badge_entity(badge)
self.update_user_data(badge, b)
def _get_namespace(domain):
global mapping
if mapping is None:
namespace_manager.set_namespace('global')
settings = ndb.Key(Settings, '_settings').get()
if not settings:
settings = Settings(id='_settings')
settings.set_mapping({})
settings.put()
mapping = settings.get_mapping_dict()
return mapping.get(domain, domain)
def daily(self):
email = self.request.get('email')
days = int(self.request.get('days'))
user = User(email=email)
settings = Settings.singletonForUser(user)
timeZoneDelta = datetime.timedelta(hours=settings.timeZoneOffset)
utcDate = datetime.datetime.today() #fixme might be incorrect
utcDateTrimmed = datetime.datetime.combine(utcDate,
datetime.time(0, 0))
fromDate = utcDateTrimmed - datetime.timedelta(
days=days) - timeZoneDelta
toDate = utcDateTrimmed - datetime.timedelta(days=days -
1) - timeZoneDelta
event_reports.SendEmailDailyReport(user, email, fromDate, toDate)
def __eq__(self,rhs):
return Settings.objects().first().phase==rhs
def __ne__(self,rhs):
return Settings.objects().first().phase!=rhs
def write_config(p: Path = config_path, data: dict = None):
data = data or default_config
with open(p.absolute(), "w") as f:
json.dump(data, f)
return Settings(**data)
def load_settings(path=None):
with open((path or config_path).absolute(), "r") as f:
return Settings(**json.load(f))
def post(self):
d=Settings.objects().first()
d.announce=self.get_argument('announce')
d.save()
hAnnounce.fire_all(d.announce)
self.write({})
async def _instantiate_dialog(self, user: User, peer: Union[User, Bot]):
log.info(f'instantiating the dialog')
conversation = Conversation(
participant1=ConversationPeer(
peer=user, peer_conversation_guid=uuid4().__str__()),
participant2=ConversationPeer(
peer=peer, peer_conversation_guid=uuid4().__str__()))
tags_set: QuerySet = Settings.objects(name='tags')
active_tags = tags_set.first().value if tags_set.count() else []
if active_tags:
profiles: QuerySet = await run_sync_in_executor(
PersonProfile.objects(tags__in=active_tags))
if profiles.count() == 0:
log.warning(f'Not found any profiles with tags: {active_tags}')
profiles: QuerySet = await run_sync_in_executor(
PersonProfile.objects)
else:
profiles: QuerySet = await run_sync_in_executor(
PersonProfile.objects)
first_profile = None
linked_profile_uuid = None
for p in conversation.participants:
if first_profile is None:
p.assigned_profile = first_profile = random.choice(profiles)
linked_profile_uuid = first_profile.link_uuid
else:
# profiles assignment order:
# other profile from the same linked group || profile with unmatching sentences || same profile
second_profile = random.choice(
profiles(id__ne=first_profile.id,
link_uuid=linked_profile_uuid)
or (profiles(persona__ne=first_profile.persona)
or [first_profile]))
p.assigned_profile = second_profile
while True:
conv_id = random.getrandbits(31)
if conv_id not in self._active_dialogs and \
await run_sync_in_executor(lambda: Conversation.objects(conversation_id=conv_id).count()) == 0:
break
conversation.conversation_id = conv_id
conversation.messages_to_switch_topic = self.dialog_options[
'n_messages_to_switch_topic']
conversation.reset_topic_switch_counter()
self._active_dialogs[conv_id] = conversation
topics = [
'Социальные сети', 'Фитнес', 'Уборка и чистота', 'Одиночество',
'Мода', 'Женская внешность', 'Мужская внешность',
'Деньги, богатство', 'Машины', 'Счастье'
]
curr_topic = random.sample(topics, k=1)
str_topic = curr_topic[0]
msg = conversation.add_message(text='Switched to topic ' + str_topic,
sender=peer,
system=True)
for p in conversation.participants:
target_gateway = self._gateway_for_peer(p)
p.assigned_profile.topics[0] = str_topic
await target_gateway.start_conversation(conv_id, p.peer,
p.assigned_profile,
p.peer_conversation_guid)
self._reset_inactivity_timer(conv_id)
def __str__(self):
return str(Settings.objects().first().phase)
def get(self):
d=Settings.objects().first()
self.write({'phase':d.phase})
def main(unused_argv):
if len(unused_argv) != 1: # prints a message if you've entered flags incorrectly
raise Exception("Problem with flags: %s" % unused_argv)
tf.logging.set_verbosity(tf.logging.INFO) # choose what level of logging you want
tf.logging.info('Starting seq2seq_attention in %s mode...', (FLAGS.mode))
# Change log_root to FLAGS.log_root/FLAGS.exp_name and create the dir if necessary
FLAGS.log_root = os.path.join(FLAGS.log_root, FLAGS.exp_name)
if not os.path.exists(FLAGS.log_root):
if FLAGS.mode=="train":
os.makedirs(FLAGS.log_root)
else:
raise Exception(
"Logdir %s doesn't exist. Run in train mode to create it." % (FLAGS.log_root)
)
vocab_size = 50000 if FLAGS.restrictive_embeddings else 20000
if FLAGS.output_vocab_size == 0:
FLAGS.output_vocab_size = vocab_size
assert FLAGS.output_vocab_size <= vocab_size
vocab = Vocab(FLAGS.vocab_path, vocab_size) # create a vocabulary
# If in decode mode, set batch_size = beam_size
# Reason: in decode mode, we decode one example at a time.
# On each step, we have beam_size-many hypotheses in the beam, so we need to make a batch
# of these hypotheses.
if FLAGS.mode == 'decode':
FLAGS.batch_size = FLAGS.beam_size
# If single_pass=True, check we're in decode mode
if FLAGS.single_pass and FLAGS.mode != 'decode':
raise Exception("The single_pass flag should only be True in decode mode")
if FLAGS.restrictive_embeddings and not FLAGS.embeddings_path:
raise Exception("Cannot use restrictive embeddings with no pretrained embeddings")
settings_dict = {}
hps_dict = {}
for key, val in FLAGS.__flags.iteritems(): # for each flag
if key in Settings._fields:
settings_dict[key] = val
elif key in Hps._fields:
hps_dict[key] = val
settings = Settings(**settings_dict)
hps = Hps(**hps_dict)
# Create a batcher object that will create minibatches of data
batcher = Batcher(FLAGS.data_path, vocab, hps, single_pass=FLAGS.single_pass)
tf.set_random_seed(111) # a seed value for randomness
if hps.mode == 'train':
print "creating model..."
model = SummarizationModel(settings, hps, vocab)
setup_training(model, batcher, hps, vocab_size)
elif hps.mode == 'eval':
model = SummarizationModel(settings, hps, vocab)
run_eval(model, batcher, vocab)
elif hps.mode == 'decode':
# The model is configured with max_dec_steps=1 because we only ever run one step of the
# decoder at a time (to do beam search). Note that the batcher is initialized with
# max_dec_steps equal to e.g. 100 because the batches need to contain the full summaries.
decode_model_hps = hps._replace(max_dec_steps=1)
model = SummarizationModel(settings, decode_model_hps, vocab)
decoder = BeamSearchDecoder(model, batcher, vocab)
# decode indefinitely (unless single_pass=True, in which case decode the dataset exactly once)
decoder.decode()
else:
raise ValueError("The 'mode' flag must be one of train/eval/decode")
def SendEmailDailyReport(currentUser, email, fromDate, toDate):
#todo refactor
cursor_inDateRange = None
cursor_outDateRange = None
queryEndInDateRange = None
queryStartInDateRange = None
try:
queryEndInDateRange = Event.query(
Event.actor == currentUser,
(ndb.AND(Event.endTime >= fromDate,
Event.endTime < toDate)),
).order(Event.endTime)
queryStartInDateRange = Event.query(
Event.actor == currentUser,
Event.startTime >= fromDate
).order(-Event.startTime) # the only event could be started and not completed in range with max time
# ndb.AND(Event.endTime < toDate,
# Event.endTime > toDate)
except:
logging.info('error cant be prepared %s', sys.exc_info()[0])
pass
if cursor_inDateRange:
queryEndInDateRange.with_cursor(cursor_inDateRange)
if cursor_outDateRange:
queryStartInDateRange.with_cursor(cursor_outDateRange)
settings = Settings.singletonForUser(currentUser)
inRangeItems = queryEndInDateRange.fetch(BATCH_SIZE)
outRangeItems = queryStartInDateRange.fetch(1) # the only event could be started and not completed in range with max time
if (len(outRangeItems) > 0):
outRangeItem = outRangeItems[0]
if ((outRangeItem.startTime < toDate) and (outRangeItem.endTime > toDate)):
outRangeItem.endTime = toDate
inRangeItems.append(outRangeItem)
preFormatedItems = map(lambda (event): itemToPreFormattedItem(event, settings, fromDate, toDate), inRangeItems)
summary = {}
markedTime = datetime.timedelta(seconds=0)
for item in preFormatedItems:
if item.activityCode in summary:
summary[item.activityCode] += item.timeSpan
else:
summary[item.activityCode] = item.timeSpan
markedTime += item.timeSpan
totalDelta = (toDate - fromDate)
htmlOut = StringIO.StringIO()
htmlOut.write("<h2>Summary:</h2>")
htmlOut.write("<table border=\"1\">")
noneTimeDelta = toDate - fromDate - markedTime
summary['None'] = noneTimeDelta
for (sumKey, sumDelta) in summary.iteritems():
markedTime += sumDelta
percent = math.floor(sumDelta.total_seconds() / totalDelta.total_seconds() * 100)
htmlOut.write("<tr><td>%s</td><td>%d%%</td><td>%s</td></tr>" % (sumKey, percent, formatTimeDelta(sumDelta)))
htmlOut.write("</table>")
htmlOut.write("<br/>")
htmlOut.write("<h2>Details:</h2>")
htmlOut.write("<table border=\"1\">")
html_items = map(activityToHtmlRow, preFormatedItems)
cvs_items = map(activityToCvs, preFormatedItems)
htmlOut.write("\n\r".join(html_items))
cvsOut = StringIO.StringIO()
cvsOut.write("\n\r".join(cvs_items))
htmlOut.write("</table>")
charOut = StringIO.StringIO()
charOut.write(PIE_CHART_TEMPLATE_HEADER)
charDataItems = map( summaryItemToDataItemRow , summary.iteritems())
charOut.write("\n\r,".join(charDataItems))
charOut.write(PIE_CHART_TEMPLATE_FOOTHDER)
mail.send_mail(sender="*****@*****.**",
to=email,
subject="Time report: %s - %s" % (fromDate.strftime("%Y-%m-%d"), toDate.strftime("%Y-%m-%d")),
body='see html email',
html=htmlOut.getvalue(),
attachments=[('report.csv', cvsOut.getvalue()), ('chart.html', charOut.getvalue()) ])
htmlOut.close()
cvsOut.close()
charOut.close()
pass
def SendEmailDailyReport(currentUser, email, fromDate, toDate):
#todo refactor
cursor_inDateRange = None
cursor_outDateRange = None
queryEndInDateRange = None
queryStartInDateRange = None
try:
queryEndInDateRange = Event.query(
Event.actor == currentUser,
(ndb.AND(Event.endTime >= fromDate, Event.endTime < toDate)),
).order(Event.endTime)
queryStartInDateRange = Event.query(
Event.actor == currentUser, Event.startTime >= fromDate
).order(
-Event.startTime
) # the only event could be started and not completed in range with max time
# ndb.AND(Event.endTime < toDate,
# Event.endTime > toDate)
except:
logging.info('error cant be prepared %s', sys.exc_info()[0])
pass
if cursor_inDateRange:
queryEndInDateRange.with_cursor(cursor_inDateRange)
if cursor_outDateRange:
queryStartInDateRange.with_cursor(cursor_outDateRange)
settings = Settings.singletonForUser(currentUser)
inRangeItems = queryEndInDateRange.fetch(BATCH_SIZE)
outRangeItems = queryStartInDateRange.fetch(
1
) # the only event could be started and not completed in range with max time
if (len(outRangeItems) > 0):
outRangeItem = outRangeItems[0]
if ((outRangeItem.startTime < toDate)
and (outRangeItem.endTime > toDate)):
outRangeItem.endTime = toDate
inRangeItems.append(outRangeItem)
preFormatedItems = map(
lambda
(event): itemToPreFormattedItem(event, settings, fromDate, toDate),
inRangeItems)
summary = {}
markedTime = datetime.timedelta(seconds=0)
for item in preFormatedItems:
if item.activityCode in summary:
summary[item.activityCode] += item.timeSpan
else:
summary[item.activityCode] = item.timeSpan
markedTime += item.timeSpan
totalDelta = (toDate - fromDate)
htmlOut = StringIO.StringIO()
htmlOut.write("<h2>Summary:</h2>")
htmlOut.write("<table border=\"1\">")
noneTimeDelta = toDate - fromDate - markedTime
summary['None'] = noneTimeDelta
for (sumKey, sumDelta) in summary.iteritems():
markedTime += sumDelta
percent = math.floor(sumDelta.total_seconds() /
totalDelta.total_seconds() * 100)
htmlOut.write("<tr><td>%s</td><td>%d%%</td><td>%s</td></tr>" %
(sumKey, percent, formatTimeDelta(sumDelta)))
htmlOut.write("</table>")
htmlOut.write("<br/>")
htmlOut.write("<h2>Details:</h2>")
htmlOut.write("<table border=\"1\">")
html_items = map(activityToHtmlRow, preFormatedItems)
cvs_items = map(activityToCvs, preFormatedItems)
htmlOut.write("\n\r".join(html_items))
cvsOut = StringIO.StringIO()
cvsOut.write("\n\r".join(cvs_items))
htmlOut.write("</table>")
charOut = StringIO.StringIO()
charOut.write(PIE_CHART_TEMPLATE_HEADER)
charDataItems = map(summaryItemToDataItemRow, summary.iteritems())
charOut.write("\n\r,".join(charDataItems))
charOut.write(PIE_CHART_TEMPLATE_FOOTHDER)
mail.send_mail(
sender="*****@*****.**",
to=email,
subject="Time report: %s - %s" %
(fromDate.strftime("%Y-%m-%d"), toDate.strftime("%Y-%m-%d")),
body='see html email',
html=htmlOut.getvalue(),
attachments=[('report.csv', cvsOut.getvalue()),
('chart.html', charOut.getvalue())])
htmlOut.close()
cvsOut.close()
charOut.close()
pass
