def compile_resume(output_dir, has_pubs, timeout):
"""
Compile resume files
Args:
output_dir: the resume output directory
has_pubs: the boolean whether there is a publication section
timeout: the timeout value
Returns:
None
"""
log = Logger()
log.notice("Compiling resume files")
curr_dir = os.getcwd()
os.chdir(output_dir)
if run_cmd("xelatex resume.tex", timeout):
if has_pubs and (not run_cmd("biber resume", timeout)
or not run_cmd("xelatex resume.tex", timeout)):
log.warn(
"Failed to compile resume files, please compile them manually")
else:
log.warn(
"Failed to compile resume files, please compile them manually")
os.chdir(curr_dir)
def plot_angle_network(log: Logger,
network: nx.DiGraph,
node_positions: Dict,
sse_list: List[str],
prefix: Union[Path, str],
write: bool = True) -> Tuple[plt.Figure, Path]:
"""Plots the per SSE geometrical distributions from the :ref:`MASTER` search.
:param log: Job Logger.
:param network: Network Graph to use for angle network.
:param node_positions: Positions of the nodes (SSEs).
:param sse_list: The SSEs to be considered.
:param prefix: Prefix for plot name.
:param write: Shall the plot by saved (default: True).
"""
fig = plt.figure(figsize=[25, 25])
ax = plt.subplot2grid((1, 1), (0, 0), fig=fig)
nx.draw(network, pos=node_positions, ax=ax)
ax.set_axis_on()
ax.set_xticks(range(len(sse_list)))
ax.set_xticklabels(sse_list)
ax.set_xlabel('SSE')
ax.set_ylabel('zeta angle')
plt.tight_layout()
imagename = Path(str(prefix) + TBcore.get_option('system', 'image'))
if write:
log.notice(f'Layer angle network image summary at {imagename}')
plt.savefig(imagename, dpi=300)
return fig, imagename
def plot_loop_length_distribution(
log: Logger,
dfloop: pd.DataFrame,
pick: int,
prefix: Union[Path, str],
title: str,
write: bool = True) -> Tuple[plt.Figure, Path]:
"""Plots the loop length count distribution for a jump.
:param log: Job Logger.
:param dfloop: Loop set for a particular jump.
:param pick: The selected length that was picked (for annotation on the plot).
:param prefix: Prefix for plot name.
:param title: Plot title.
:param write: Shall the plot by saved (default: True).
"""
fig = plt.figure()
ax = plt.subplot2grid((1, 1), (0, 0), fig=fig)
sns.countplot(x='loop_length', data=dfloop, ax=ax, palette="PRGn")
cnt = dfloop[dfloop['loop_length'] == pick]['length_count'].values[0]
pick = [int(x.get_text()) for x in ax.get_xticklabels()].index(pick)
ax.plot(pick, cnt, marker=11, color='black')
ax.set_title(title)
plt.tight_layout()
imagename = Path(str(prefix) + TBcore.get_option('system', 'image'))
if write:
log.notice(f'loop image summary at {imagename}')
plt.savefig(imagename, dpi=300)
return fig, imagename
def make_resume_files(profile, output_dir, timeout):
"""
Create resume files
Args:
profile: the dict of the profile
output_dir: the output directory
timeout: the timeout value
Returns:
None
"""
log = Logger()
log.notice("Creating resume files")
output_dir = os.path.join(output_dir, "resume")
make_dir(output_dir)
copy_files(
__name__.split(".")[0], "templates/awesome_cv_files", output_dir)
if PUBLICATIONS in profile:
has_publications = make_publication_section(profile[PUBLICATIONS],
output_dir)
else:
has_publications = False
if SKILLS in profile and LANGUAGES in profile:
make_skill_section(profile[SKILLS], profile[LANGUAGES], output_dir)
for section in RESUME_SECTIONS:
make_resume_section(profile, section, output_dir)
make_resume_main(profile, has_publications, output_dir)
compile_resume(output_dir, has_publications, timeout)
def compile_resume(output_dir, has_pubs):
"""
Compile resume files
Args:
output_dir: The resume output directory
has_pubs: The boolean whether there is a publication section
Returns:
None
"""
log = Logger()
log.notice('Compiling resume files')
curr_dir = os.getcwd()
os.chdir(output_dir)
if run_cmd('xelatex resume.tex'):
if has_pubs and (not run_cmd('biber resume')
or not run_cmd('xelatex resume.tex')):
log.warn(
'Failed to compile resume files, please compile them manually')
else:
log.warn(
'Failed to compile resume files, please compile them manually')
os.chdir(curr_dir)
def make_resume_files(profile, output_dir):
"""
Create resume files
Args:
profile: the dict of the profile
output_dir: the output directory
Returns:
None
"""
log = Logger()
log.notice('Creating resume files')
output_dir = os.path.join(output_dir, 'resume')
make_dir(output_dir)
copy_files(__name__, 'awesome_cv_files', output_dir)
has_publications = make_publication_section(profile[PUBLICATIONS],
output_dir)
make_skill_section(profile[SKILLS], profile[LANGUAGES], output_dir)
for section in RESUME_SECTIONS:
make_resume_section(profile, section, output_dir)
make_resume_main(profile, has_publications, output_dir)
compile_resume(output_dir, has_publications)
def execute(log: Logger, data: Dict, wpaths: Dict) -> Dict:
"""Run Rosetta.
"""
if TBcore.get_option('slurm', 'use'):
slurm_file = wpaths['main'].joinpath('submit_hybridize.sh')
TButil.plugin_filemaker('Submission file at {}'.format(slurm_file))
with slurm_file.open('w') as fd:
fd.write(TButil.slurm_header() + '\n')
for k in ['assembly', 'design']:
cmd = [
'srun',
]
cmd.extend(data['cmd'][k])
fd.write(' '.join([str(x) for x in cmd]) + '\n')
if TBcore.get_option('system', 'verbose'):
sys.stdout.write(
'Submiting jobs to SLURM... this might take a while\n')
TButil.submit_slurm(slurm_file)
else:
for k in ['assembly', 'design']:
log.notice(
f'EXECTUE: {" ".join([str(x) for x in data["cmd"][k]])}')
run([str(x) for x in data['cmd'][k]], stdout=DEVNULL)
return data
def setup_db(log: logbook.Logger):
db_file = os.path.join(
os.path.dirname(__file__),
'db',
'pypi.sqlite')
db_session.global_init(db_file)
log.notice(f'Database initialized.')
def test2():
log = Logger('Logbook-test-2')
log.critical("critical")
log.error("error")
log.warn("warn")
log.notice("notice")
log.info("test")
log.debug("debug")
def setup_markdown(log: logbook.Logger):
store = TemplateDBStorage()
storage.set_storage(store)
log_engine = TemplateLogger(logging.LogLevel.info)
logging.set_log(log_engine)
log.notice(f"Set markdown storage engine to: {type(store).__name__}.")
log.notice(f"Set markdown log engine to: {type(log_engine).__name__}.")
def init_markdown(log: logbook.Logger):
store = SubTemplateDBStorage()
storage.set_storage(store)
log_engine = SubTemplateLogger(LogLevel.info)
logging.set_log(log_engine)
log.notice(f'Markdown storage engine set: {type(store).__name__}.')
log.notice(f'Markdown logging engine set: {type(log_engine).__name__}.')
def percent_reporter(complete):
log = Logger('Pool Dispatcher')
while True:
try:
if not complete.empty():
log.notice('{:.2f}% complete'.format(complete.get()))
except Exception:
pass
finally:
time.sleep(0.01)
def register_blueprints(log: logbook.Logger):
from pypi_org.views import home_views
from pypi_org.views import package_views
from pypi_org.views import account_views
from pypi_org.views import seo_view
app.register_blueprint(package_views.blueprint)
app.register_blueprint(home_views.blueprint)
app.register_blueprint(account_views.blueprint)
app.register_blueprint(seo_view.blueprint)
log.notice("Registered blueprints.")
def init_routing(config, log: logbook.Logger):
config.add_static_view('static', 'static', cache_max_age=3600)
# home controller
config.add_route('home', '/')
config.add_route('about', '/about')
# package controller
config.add_route(
'popular',
'/{num}',
custom_predicates=[
lambda info, _: info['match'].get('num', '').isdigit()
])
config.add_route('package_details', '/project/{package_name}')
config.add_route('package_details/', '/project/{package_name}/')
config.add_route('releases', '/project/{package_name}/releases')
config.add_route('releases/', '/project/{package_name}/releases/')
config.add_route('release_version',
'/project/{package_name}/releases/{release_version}')
# account controller
config.add_route('account_home', '/account')
config.add_route('login', '/account/login')
config.add_route('register', '/account/register')
config.add_route('logout', '/account/logout')
# utils controller
config.add_route('sitemap.xml', '/sitemap.xml')
config.add_route('robots.txt', '/robots.txt')
# admin controller
config.add_route('admin_index', '/admin')
config.add_route('redirects', '/admin/redirects')
config.add_route('add_redirect', '/admin/add_redirect')
config.add_route('edit_redirect', '/admin/edit_redirect/{redirect_id}')
config.add_route('pages', '/admin/pages')
config.add_route('add_page', '/admin/add_page')
config.add_route('edit_page', '/admin/edit_page/{page_id}')
# CMS Route
# /company/history
# sub_path = [company, history]
config.add_route('cms_request', '*sub_path')
config.scan()
log.notice(f'Web routes registered.')
def plot_geometric_distributions(
log: Logger,
df: pd.DataFrame,
prefix: Union[Path, str],
write: bool = True) -> Tuple[plt.Figure, Path]:
"""Plots the per SSE geometrical distributions from the :ref:`MASTER` search.
:param log: Job Logger.
:param df: Calculated geometric statistics from the :ref:`MASTER` search.
:param prefix: Prefix for plot name.
:param write: Shall the plot by saved (default: True).
"""
ordering = sorted(df.sse.unique())
fig = plt.figure(figsize=[15, 15])
grid = (3, 2)
for i, l in enumerate(['layer', 'floor', 'side']):
ax = plt.subplot2grid(grid, (i, 0), fig=fig)
sns.violinplot(x='sse',
y=f'angles_{l}',
hue='bin',
data=df,
palette="Set3",
order=ordering,
ax=ax,
cut=1)
ax.legend().remove()
ax.set_ylabel('angle')
ax.set_title(f'angles_{l}')
ax = plt.subplot2grid(grid, (i, 1), fig=fig)
sns.violinplot(x='sse',
y='points_{}'.format(l),
hue='bin',
data=df,
palette="Set3",
order=ordering,
ax=ax,
cut=0)
ax.set_ylabel('distance')
ax.set_title(f'points_{l}')
if i != 0:
ax.legend().remove()
else:
ax.legend(ncol=len(df.bin.unique()))
plt.tight_layout()
imagename = Path(str(prefix) + TBcore.get_option('system', 'image'))
if write:
log.notice(f'Geometric distributions image summary at {imagename}')
plt.savefig(imagename, dpi=300)
return fig, imagename
def make_references(publications, output_dir):
"""
Create reference bib file
Args:
publications: the list of publications
output_dir: the output directory
Returns:
A list of reference identifiers
"""
log = Logger()
cr = Crossref()
lines = []
references = []
for i, publication in enumerate(publications):
log.notice(
f"Querying and formatting {i + 1} out of {len(publications)} publications"
)
link = publication[LINK]
title = publication[TITLE]
# Check if it is a DOI url
if link and "doi.org" in link:
doi = urlparse(link).path.strip("/")
# Extract the DOI using the title
else:
results = cr.works(query_bibliographic=title, limit=1)
if (results["message"]["total-results"] == 0
or results["message"]["items"][0]["title"][0].lower() !=
title.lower()):
log.warn(f'Could not find the doi for "{title}"')
continue
doi = results["message"]["items"][0]["DOI"]
try:
reference = cn.content_negotiation(doi)
lines.append(reference)
references.append(
re.sub("^@.*{", "",
reference.split("\n")[0]).strip(","))
except HTTPError:
log.warn(f'Could not Create reference for "{title}"')
with open(os.path.join(output_dir, "references.bib"), "w") as f:
f.write("\n\n".join(lines))
return references
def receive_feedback(update, context):
"""
Log the feedback on Slack
Args:
update: the update object
context: the context object
Returns:
The variable indicating the conversation has ended
"""
message = update.effective_message
tele_username = message.chat.username
tele_id = message.chat.id
feedback_msg = message.text
feedback_lang = None
b = TextBlob(feedback_msg)
try:
feedback_lang = b.detect_language()
except TranslatorError:
pass
_ = set_lang(update, context)
if not feedback_lang or feedback_lang.lower() not in VALID_LANGS:
message.reply_text(
_("The feedback is not in English or Chinese, try again"))
return 0
text = "Feedback received from @{} ({}):\n\n{}".format(
tele_username, tele_id, feedback_msg)
success = False
if SLACK_TOKEN is not None:
client = WebClient(token=SLACK_TOKEN)
response = client.chat_postMessage(channel="#pdf-bot-feedback",
text=text)
if response["ok"] and response["message"]["text"] == text:
success = True
if not success:
log = Logger()
log.notice(text)
message.reply_text(
_("Thank you for your feedback, I've already forwarded it to my developer"
))
return ConversationHandler.END
class ILogger(object):
def __init__(self, **kwargs):
self.logger = Logger(self.name)
def _process_kwargs(self, kwargs):
for key, value in kwargs.items():
setattr(self, key, value)
def log(self, msg):
self.logger.notice(msg)
def log_error(self, msg):
self.logger.error(msg)
def plot_fragment_templates(log: Logger,
dfsmall: Union[FragmentFrame, pd.DataFrame],
dflarge: Union[FragmentFrame, pd.DataFrame],
prefix: Union[Path, str],
write: bool = True) -> Tuple[plt.Figure, Path]:
"""Plots the fragment sources across the sequence.
:param log: Job Logger.
:param dfsmall: Small fragment set.
:param dflarge: Large fragment set.
:param prefix: Prefix for plot name.
:param write: Shall the plot by saved (default: True).
"""
fig = plt.figure(figsize=(20, 8))
ax0 = plt.subplot2grid((2, 1), (0, 0), fig=fig)
pp = dfsmall.groupby(['position', 'source'
])['position'].count() / dfsmall['size'].values[0]
pp = pp.unstack().fillna(0)
color = ListedColormap(
sns.diverging_palette(220, 20, n=len(pp.columns)).as_hex())
pp.plot(kind='bar', stacked=True, ax=ax0, colormap=color)
ax0.set_xlim(-0.5, dfsmall.position.max() - dfsmall.position.min() + 1)
ax0.set_ylabel('3mers coverage')
ax0.get_xaxis().set_visible(False)
ax0.get_legend().remove()
ax1 = plt.subplot2grid((2, 1), (1, 0), sharex=ax0, fig=fig)
pp = dflarge.groupby(['position', 'source'
])['position'].count() / dflarge['size'].values[0]
pp = pp.unstack().fillna(0)
pp.plot(kind='bar', stacked=True, ax=ax1, colormap=color)
ax1.set_xlim(-0.5, dflarge.position.max() - dflarge.position.min() + 1)
ax1.set_ylabel('9mers coverage')
ax1.set_xticks(
range(0,
dflarge.position.max() - dflarge.position.min() + 1, 5))
ax1.set_xticklabels(range(dflarge.position.min(),
dflarge.position.max() + 1, 5),
rotation=0)
ax1.legend(loc=9, bbox_to_anchor=(0.5, -0.15), ncol=len(pp.columns))
plt.subplots_adjust(wspace=0, hspace=0.025)
imagename = Path(str(prefix) + TBcore.get_option('system', 'image'))
if write:
log.notice(f'fragment templates image summary at {imagename}')
plt.savefig(imagename, dpi=300)
return fig, imagename
def run(driver, email, password, keep_creds, output_dir, scrape_only, resume_only, website_only, profile_file, timeout,
**kwargs):
# Setup logging
logbook.set_datetime_format('local')
format_string = '[{record.time:%Y-%m-%d %H:%M:%S}] {record.level_name}: {record.message}'
StreamHandler(sys.stdout, format_string=format_string).push_application()
log = Logger()
# Create output directory
make_dir(output_dir)
# Check if user has provided the profile json file
if profile_file is None:
if driver.lower() not in DRIVERS:
raise ValueError(f'Browser driver has to be one of these: {", ".join(DRIVERS)}')
# Check if credentials file exists
credentials_file = os.path.expanduser(CREDENTIALS_FILE)
if os.path.exists(credentials_file):
with open(credentials_file) as f:
credentials = json.load(f)
email = credentials['email']
password = credentials['password']
else:
if email is None:
email = input('Enter your LinkedIn login email: ')
if password is None:
password = getpass('Enter your LinkedIn login password: '******'Scraping LinkedIn profile')
log.notice('Please keep the browser window on top')
profile = scrape(driver.lower(), email, password, output_dir, timeout)
if keep_creds:
store_creds(email, password, credentials_file)
else:
with open(profile_file) as f:
profile = json.load(f)
if not scrape_only:
if resume_only:
make_resume_files(profile, output_dir, timeout)
elif website_only:
make_website_files(profile, output_dir)
else:
make_resume_files(profile, output_dir, timeout)
make_website_files(profile, output_dir)
class MyEventHandler(FileSystemEventHandler):
"""Logs all the events captured."""
def __init__(self, logfile, run_handler):
"""pass logfile to be opened and handler to flush writing the file"""
super(MyEventHandler, self).__init__()
self.run_log = Logger('Runs')
self.fs_log = Logger('Files')
socket_path = environ.get("NVIM_LISTEN_ADDRESS")
self.nvim = attach('socket', path=socket_path)
self.log_file = logfile
self.run_handler = run_handler
def on_moved(self, event):
"""called when a file or folder is moved"""
super(MyEventHandler, self).on_moved(event)
what = 'directory' if event.is_directory else 'file'
log_msg = "Moved {}: from {} to {}".format(what, event.src_path, event.dest_path)
self.fs_log.info(log_msg)
def on_created(self, event):
"""called on creation of a file or folder"""
super(MyEventHandler, self).on_created(event)
what = 'directory' if event.is_directory else 'file'
self.fs_log.info("Created {}: {}".format(what, event.src_path))
def on_deleted(self, event):
"""called on deletion of a file or folder"""
super(MyEventHandler, self).on_deleted(event)
what = 'directory' if event.is_directory else 'file'
self.fs_log.info("Deleted {}: {}".format(what, event.src_path))
def on_modified(self, event):
"""when a file is modified the event is logged and appended to a separate file
Then the script is run through python and the output is (over)written to self.log_file
and appended to the file handled by the info handler"""
super(MyEventHandler, self).on_modified(event)
what = 'directory' if event.is_directory else 'file'
self.fs_log.info("Modified {}: {}".format(what, event.src_path))
event_path = Path(event.src_path) # using plain src_path gives nonexistent path
if event_path.is_file():
in_file = str(event_path)
out_str = python(in_file)
self.run_log.notice('Output:\n{}'.format(out_str))
self.run_handler.close()
self.nvim.command('pedit ' + self.log_file)
def make_references(publications, output_dir):
"""
Create reference bib file
Args:
publications: the list of publications
output_dir: the output directory
Returns:
A list of reference identifiers
"""
log = Logger()
cr = Crossref()
lines = []
references = []
for i, publication in enumerate(publications):
log.notice(f'Querying and formatting {i + 1} out of {len(publications)} publications')
link = publication[LINK]
title = publication[TITLE]
# Check if it is a DOI url
if link and 'doi.org' in link:
doi = urlparse(link).path.strip('/')
# Extract the DOI using the title
else:
results = cr.works(query_title=title, limit=1)
if results['message']['total-results'] == 0 or \
results['message']['items'][0]['title'][0].lower() != title.lower():
log.warn(f'Could not find the doi for "{title}"')
continue
doi = results['message']['items'][0]['DOI']
try:
reference = cn.content_negotiation(doi)
lines.append(reference)
references.append(re.sub('^@.*{', '', reference.split('\n')[0]).strip(','))
except HTTPError:
log.warn(f'Could not Create reference for "{title}"')
with open(os.path.join(output_dir, 'references.bib'), 'w') as f:
f.write('\n\n'.join(lines))
return references
class ILogger(object):
def __init__(self, **kwargs):
self.logger = Logger(self.name)
def _process_kwargs(self, kwargs):
for key, value in kwargs.items():
setattr(self, key, value)
def log(self, msg):
"""Records record at notice level"""
self.logger.notice(msg)
def log_error(self, msg):
"""Records record at error level"""
self.logger.error(msg)
def write(self, msg):
"""Writes to a file"""
raise NotImplementedError
def receive_feedback(update: Update, context: CallbackContext) -> int:
message = update.effective_message
tele_username = message.chat.username
tele_id = message.chat.id
feedback_msg = message.text
feedback_lang = None
b = TextBlob(feedback_msg)
try:
feedback_lang = b.detect_language()
except TranslatorError:
pass
_ = set_lang(update, context)
if feedback_lang is None or feedback_lang.lower() != "en":
message.reply_text(_("The feedback is not in English, try again"))
return 0
text = "Feedback received from @{} ({}):\n\n{}".format(
tele_username, tele_id, feedback_msg)
success = False
if SLACK_TOKEN is not None:
client = WebClient(token=SLACK_TOKEN)
response = client.chat_postMessage(channel="#pdf-bot-feedback",
text=text)
if response["ok"] and response["message"]["text"] == text:
success = True
if SLACK_TOKEN is None:
context.bot.send_message(DEV_TELE_ID, text=text)
if not success:
log = Logger()
log.notice(text)
message.reply_text(
_("Thank you for your feedback, I've already forwarded it to my developer"
))
return ConversationHandler.END
def count_single_master_matches( log: Logger, pdbfile: Path, folder: Path ) -> pd.DataFrame:
"""
"""
createpds = TButil.createPDS(pdbfile)
log.notice(f'EXECUTE: {createpds}')
run(createpds, stdout=DEVNULL)
masters = TButil.master_best_each(pdbfile.with_suffix('.pds'), folder.joinpath('_master'), 5)
unimaster = folder.joinpath('match.master')
if not TBcore.get_option('slurm', 'use'):
no_slurm(cmd, current_case_file, current_sse, unimaster, imaster, unidata.with_suffix(''))
else:
with_slurm(cmd, current_case_file, current_sse, unimaster, imaster, unidata)
return {'matches': unimaster, 'stats': unidata, 'corrections': None,
'layers': list(set([x[0] for x in current_sse.split('.')]))}
data = submit_searches(masters, stepfolder, current_case_file, '.'.join([x['id'] for x in sses]))
def plot_match_bin(log: Logger,
master_match: pd.DataFrame,
prefix: Union[Path, str],
expected: Union[int, float],
groupby: Optional[List] = None,
write: bool = True) -> Tuple[plt.Figure, Path]:
"""Plots the number of matches per defined bin of the :ref:`MASTER` search.
:param log: Job Logger.
:param master_match: Collected :ref:`MASTER` matches.
:param prefix: Prefix for plot name.
:param expected: Limitation of y-axis height.
:param groupby: Groups to order by (e.g. close, mid, far, extreme).
:param write: Shall the plot by saved (default: True).
"""
fig = plt.figure(figsize=[15, 5])
ax = plt.subplot2grid((1, 1), (0, 0), fig=fig)
master_match = master_match.sort_values('rmsd')
df = master_match.groupby(groupby).head(
1) if groupby is not None else master_match
bins = ['close', 'mid', 'far', 'extreme']
sns.countplot(x="bin", data=df, ax=ax, order=bins, palette="Set3")
ax.set_xticklabels(
['close\n[0, 2)', 'mid\n[2, 2.5)', 'far\n[2.5, 3)', 'extreme\n[3, 5)'])
ax.set_ylim(top=expected)
match_count = []
for p in ax.patches:
pp = int(0 if math.isnan(p.get_height()) else p.get_height())
ypos = pp + 1000 if pp + 1000 < expected - 500 else pp - 1000
ax.annotate('{:d}'.format(pp), (p.get_x() + 0.37, ypos))
match_count.append(bool(pp))
plt.tight_layout()
imagename = Path(str(prefix) + TBcore.get_option('system', 'image'))
if write:
log.notice(f'MASTER match image summary at {imagename}')
plt.savefig(imagename, dpi=300)
return fig, imagename, dict(zip(bins, match_count))
def execute(log: Logger, data: Dict, wpaths: Dict) -> Dict:
"""Run Rosetta.
"""
if TBcore.get_option('slurm', 'use'):
slurm_file = wpaths['main'].joinpath('submit_funfoldes.sh')
log.notice(f'Submission file at {slurm_file}')
with slurm_file.open('w') as fd:
fd.write(TButil.slurm_header() + '\n')
for k in ['folding', 'design']:
cmd = [
'srun',
]
cmd.extend(data['cmd'][k])
fd.write(' '.join([str(x) for x in cmd]) + '\n')
log.info('Submiting jobs to SLURM... this might take a while\n')
TButil.submit_slurm(log, slurm_file)
else:
for k in ['folding', 'design']:
log.notice(
f'EXECUTE: {" ".join([str(x) for x in data["cmd"][k]])}')
run([str(x) for x in data['cmd'][k]], stdout=DEVNULL)
return data
class Worker(multiprocessing.Process):
"""
Distributed tf worker class.
Sets up environment, trainer and starts training process in supervised session.
"""
env_list = None
def __init__(self,
env_config,
policy_config,
trainer_config,
cluster_spec,
job_name,
task,
log_dir,
log_level,
max_env_steps,
random_seed=None,
render_last_env=False,
test_mode=False):
"""
Args:
env_config: environment class_config_dict.
policy_config: model policy estimator class_config_dict.
trainer_config: algorithm class_config_dict.
cluster_spec: tf.cluster specification.
job_name: worker or parameter server.
task: integer number, 0 is chief worker.
log_dir: for tb summaries and checkpoints.
log_level: int, logbook.level
max_env_steps: number of environment steps to run training on
random_seed: int or None
render_last_env: bool, if True - render enabled for last environment in a list; first otherwise
test_mode: if True - use Atari mode, BTGym otherwise.
Note:
- Conventional `self.global_step` refers to number of environment steps,
summarized over all environment instances, not to number of policy optimizer train steps.
- Every worker can run several environments in parralell, as specified by `cluster_config'['num_envs'].
If use 4 forkers and num_envs=4 => total number of environments is 16. Every env instance has
it's own ThreadRunner process.
- When using replay memory, keep in mind that every ThreadRunner is keeping it's own replay memory,
If memory_size = 2000, num_workers=4, num_envs=4 => total replay memory size equals 32 000 frames.
"""
super(Worker, self).__init__()
self.env_class = env_config['class_ref']
self.env_kwargs = env_config['kwargs']
self.policy_config = policy_config
self.trainer_class = trainer_config['class_ref']
self.trainer_kwargs = trainer_config['kwargs']
self.cluster_spec = cluster_spec
self.job_name = job_name
self.task = task
self.log_dir = log_dir
self.max_env_steps = max_env_steps
self.log_level = log_level
self.log = None
self.test_mode = test_mode
self.random_seed = random_seed
self.render_last_env = render_last_env
def run(self):
"""Worker runtime body.
"""
# Logging:
StreamHandler(sys.stdout).push_application()
self.log = Logger('Worker_{}'.format(self.task), level=self.log_level)
tf.reset_default_graph()
if self.test_mode:
import gym
# Define cluster:
cluster = tf.train.ClusterSpec(self.cluster_spec).as_cluster_def()
# Start tf.server:
if self.job_name in 'ps':
server = tf.train.Server(
cluster,
job_name=self.job_name,
task_index=self.task,
config=tf.ConfigProto(device_filters=["/job:ps"])
)
self.log.debug('parameters_server started.')
# Just block here:
server.join()
else:
server = tf.train.Server(
cluster,
job_name='worker',
task_index=self.task,
config=tf.ConfigProto(
intra_op_parallelism_threads=1, # original was: 1
inter_op_parallelism_threads=2 # original was: 2
)
)
self.log.debug('tf.server started.')
self.log.debug('making environments:')
# Making as many environments as many entries in env_config `port` list:
# TODO: Hacky-II: only one example over all parallel environments can be data-master [and renderer]
# TODO: measure data_server lags, maybe launch several instances
self.env_list = []
env_kwargs = self.env_kwargs.copy()
env_kwargs['log_level'] = self.log_level
port_list = env_kwargs.pop('port')
data_port_list = env_kwargs.pop('data_port')
data_master = env_kwargs.pop('data_master')
render_enabled = env_kwargs.pop('render_enabled')
render_list = [False for entry in port_list]
if render_enabled:
if self.render_last_env:
render_list[-1] = True
else:
render_list[0] = True
data_master_list = [False for entry in port_list]
if data_master:
data_master_list[0] = True
# Parallel envs. numbering:
if len(port_list) > 1:
task_id = 0.0
else:
task_id = 0
for port, data_port, is_render, is_master in zip(port_list, data_port_list, render_list, data_master_list):
# Get random seed for environments:
env_kwargs['random_seed'] = random.randint(0, 2 ** 30)
if not self.test_mode:
# Assume BTgym env. class:
self.log.debug('setting env at port_{} is data_master: {}'.format(port, data_master))
self.log.debug('env_kwargs:')
for k, v in env_kwargs.items():
self.log.debug('{}: {}'.format(k, v))
try:
self.env_list.append(
self.env_class(
port=port,
data_port=data_port,
data_master=is_master,
render_enabled=is_render,
task=self.task + task_id,
**env_kwargs
)
)
data_master = False
self.log.info('set BTGym environment {} @ port:{}, data_port:{}'.
format(self.task + task_id, port, data_port))
task_id += 0.01
except:
self.log.exception(
'failed to make BTGym environment at port_{}.'.format(port)
)
raise RuntimeError
else:
# Assume atari testing:
try:
self.env_list.append(self.env_class(env_kwargs['gym_id']))
self.log.debug('set Gyn/Atari environment.')
except:
self.log.exception('failed to make Gym/Atari environment')
raise RuntimeError
self.log.debug('Defining trainer...')
# Define trainer:
trainer = self.trainer_class(
env=self.env_list,
task=self.task,
policy_config=self.policy_config,
log_level=self.log_level,
cluster_spec=self.cluster_spec,
random_seed=self.random_seed,
**self.trainer_kwargs,
)
self.log.debug('trainer ok.')
# Saver-related:
variables_to_save = [v for v in tf.global_variables() if not 'local' in v.name]
local_variables = [v for v in tf.global_variables() if 'local' in v.name] + tf.local_variables()
init_op = tf.variables_initializer(variables_to_save)
local_init_op = tf.variables_initializer(local_variables)
init_all_op = tf.global_variables_initializer()
saver = _FastSaver(variables_to_save)
self.log.debug('VARIABLES TO SAVE:')
for v in variables_to_save:
self.log.debug('{}: {}'.format(v.name, v.get_shape()))
def init_fn(ses):
self.log.info("initializing all parameters.")
ses.run(init_all_op)
config = tf.ConfigProto(device_filters=["/job:ps", "/job:worker/task:{}/cpu:0".format(self.task)])
logdir = os.path.join(self.log_dir, 'train')
summary_dir = logdir + "_{}".format(self.task)
summary_writer = tf.summary.FileWriter(summary_dir)
self.log.debug('before tf.train.Supervisor... ')
# TODO: switch to tf.train.MonitoredTrainingSession
sv = tf.train.Supervisor(
is_chief=(self.task == 0),
logdir=logdir,
saver=saver,
summary_op=None,
init_op=init_op,
local_init_op=local_init_op,
init_fn=init_fn,
#ready_op=tf.report_uninitialized_variables(variables_to_save),
ready_op=tf.report_uninitialized_variables(),
global_step=trainer.global_step,
save_model_secs=300,
)
self.log.info("connecting to the parameter server... ")
with sv.managed_session(server.target, config=config) as sess, sess.as_default():
#sess.run(trainer.sync)
trainer.start(sess, summary_writer)
# Note: `self.global_step` refers to number of environment steps
# summarized over all environment instances, not to number of policy optimizer train steps.
global_step = sess.run(trainer.global_step)
self.log.notice("started training at step: {}".format(global_step))
while not sv.should_stop() and global_step < self.max_env_steps:
trainer.process(sess)
global_step = sess.run(trainer.global_step)
# Ask for all the services to stop:
for env in self.env_list:
env.close()
sv.stop()
self.log.notice('reached {} steps, exiting.'.format(global_step))
class Estimator:
def __init__(
self,
data,
model_config=None,
opt_learn_rate=1e-4,
opt_decay_steps=None,
opt_end_learn_rate=1e-4,
max_train_steps=10000,
grad_clip=50,
class_loss_lambda=1,
regress_loss_lambda=1,
entropy_beta=0.01,
dropout_keep_prob=1,
validation_period=100,
summary_period=10,
log_dir='/log',
purge_previous_log=1,
name='Estimator'
):
assert isinstance(data, DataCorpus)
self.data = data
self.opt_learn_rate = opt_learn_rate
self.opt_decay_steps = opt_decay_steps
self.opt_end_learn_rate = opt_end_learn_rate
self.max_train_steps = max_train_steps
self.grad_clip = grad_clip
self.class_loss_lambda = class_loss_lambda
self.regress_loss_lambda = regress_loss_lambda
self.entropy_beta = entropy_beta
self.dropout_keep_prob = dropout_keep_prob
self.validation_period = validation_period
self.summary_period = summary_period
self.log_dir = log_dir
self.purge_previous_log = purge_previous_log
self.name = name
StreamHandler(sys.stdout).push_application()
self.log = Logger(self.name)
# Log_dir:
if os.path.exists(self.log_dir):
# Remove previous log files and saved model if opted:
if self.purge_previous_log > 0:
confirm = 'y'
if self.purge_previous_log < 2:
confirm = input('<{}> already exists. Override[y/n]? '.format(self.log_dir))
if confirm in 'y':
files = glob.glob(self.log_dir + '/*')
p = psutil.Popen(['rm', '-R', ] + files, stdout=PIPE, stderr=PIPE)
self.log.notice('Files in <{}> purged.'.format(self.log_dir))
else:
self.log.notice('Appending to <{}>.'.format(self.log_dir))
else:
os.makedirs(self.log_dir)
self.log.notice('<{}> created.'.format(self.log_dir))
self.keep_prob_pl = tf.placeholder(tf.float32, name='dropout_parameter')
if model_config is None:
self.model_class_ref = Model
self.model_kwargs = {
'activation': tf.nn.elu,
'name': self.name + '/model',
}
else:
self.model_class_ref = model_config['class_ref']
self.model_kwargs = model_config['kwargs']
self.train_model = self.model_class_ref(
data_batch=self.data.next_train_batch_op[0],
labels_batch=self.data.next_train_batch_op[-1],
keep_prob=self.keep_prob_pl,
reuse=False,
**self.model_kwargs
)
self.cv_model = self.model_class_ref(
data_batch=self.data.next_cv_batch_op[0],
labels_batch=self.data.next_cv_batch_op[-1],
keep_prob=self.keep_prob_pl,
reuse=True,
**self.model_kwargs
)
self.test_model = self.model_class_ref(
data_batch=self.data.next_test_batch_op[0],
labels_batch=None,
keep_prob=self.keep_prob_pl,
reuse=True,
**self.model_kwargs
)
self.global_step = tf.get_variable(
"global_step",
[],
tf.int32,
initializer=tf.constant_initializer(
0,
dtype=tf.int32
),
trainable=False
)
self.inc_global_step = self.global_step.assign_add(1)
#self.reset_global_step = self.global_step.assign(0)
self.local_step = tf.get_variable(
"local_step",
[],
tf.int32,
initializer=tf.constant_initializer(
0,
dtype=tf.int32
),
trainable=False
)
self.inc_local_step = self.local_step.assign_add(1)
self.reset_local_step = self.local_step.assign(0)
self.inc_step = tf.group([self.inc_global_step, self.inc_local_step])
# self.inc_step = self.inc_global_step
# Learning rate annealing:
if self.opt_decay_steps is not None:
self.learn_rate_decayed = tf.train.polynomial_decay(
self.opt_learn_rate,
self.local_step + 1,
self.opt_decay_steps,
self.opt_end_learn_rate,
power=1,
cycle=False,
)
else:
self.learn_rate_decayed = self.opt_end_learn_rate
# Loss, train_op:
self.train_loss = self.class_loss_lambda * self.train_model.class_loss \
+ self.regress_loss_lambda * self.train_model.regress_loss \
- self.entropy_beta * self.train_model.class_entropy
try:
self.cv_loss = self.class_loss_lambda * self.cv_model.class_loss \
+ self.regress_loss_lambda * self.cv_model.regress_loss \
- self.entropy_beta * self.cv_model.class_entropy
except TypeError:
self.cv_loss = 0
self.grads, _ = tf.clip_by_global_norm(
tf.gradients(self.train_loss, self.train_model.var_list),
grad_clip
)
self.grads_and_vars = list(zip(self.grads, self.train_model.var_list))
self.optimizer = tf.train.AdamOptimizer(self.learn_rate_decayed, epsilon=1e-5)
self.optimize_op = self.optimizer.apply_gradients(self.grads_and_vars)
self.train_op = tf.group([self.optimize_op, self.inc_step, self.train_model.auc_update_op])
# Summaries:
train_summaries = [
tf.summary.scalar("train_grad_global_norm", tf.global_norm(self.grads)),
tf.summary.scalar("train_total_loss", self.train_loss),
tf.summary.scalar("train_auc", self.train_model.auc),
tf.summary.scalar("train_regression_loss", self.train_model.regress_loss),
tf.summary.scalar("train_class_loss", self.train_model.class_loss),
tf.summary.scalar('train_class_entropy', self.train_model.class_entropy),
]
cv_summaries = [
tf.summary.scalar("cv_total_loss", self.cv_loss),
tf.summary.scalar("cv_class_loss", self.cv_model.class_loss),
tf.summary.scalar("cv_regression_loss", self.cv_model.regress_loss),
tf.summary.scalar("cv_auc", self.cv_model.auc_update_op),
#tf.summary.scalar('cv_class_entropy', self.cv_model.class_entropy),
]
self.train_summaries_op = tf.summary.merge(train_summaries)
self.cv_summaries_op = tf.summary.merge(cv_summaries)
self.summary_writer = tf.summary.FileWriter(self.log_dir)
def initialise(self, sess):
sess.run(
[tf.variables_initializer(self.train_model.var_list), tf.local_variables_initializer()]
)
def train(self, sess, run_cv=True):
self.data.reset()
sess.run(self.reset_local_step)
step = 0
while step <= self.max_train_steps:
try:
step = sess.run(self.local_step)
write_summaries = step % self.summary_period == 0
fetches = [self.train_op, self.train_loss]
if write_summaries:
fetches.append(self.train_summaries_op)
fetched = sess.run(fetches, {self.keep_prob_pl: self.dropout_keep_prob})
loss = fetched[1]
if step % 500 == 0:
self.log.info('step: {}, train_total_loss: {:0.3f}'.format(step, loss))
if write_summaries:
self.summary_writer.add_summary(tf.Summary.FromString(fetched[-1]), sess.run(self.global_step))
self.summary_writer.flush()
if step % self.validation_period == 0 and run_cv:
self.validate(sess)
except tf.errors.OutOfRangeError:
self.log.info('max_train_set_repeats reached.')
break
# Final validation:
if run_cv:
self.validate(sess)
self.log.info('train fold finished at step: {}'.format(sess.run(self.local_step)))
def validate(self, sess):
self.data.reset_cv()
step = sess.run(self.global_step)
class_loss = []
regress_loss = []
while True:
try:
fetches = [
self.cv_summaries_op,
self.cv_model.class_loss,
self.cv_model.regress_loss,
]
fetched = sess.run(fetches, {self.keep_prob_pl: 1.0})
class_loss.append(fetched[1])
regress_loss.append(fetched[2])
self.summary_writer.add_summary(tf.Summary.FromString(fetched[0]), step)
self.summary_writer.flush()
step += 1
except tf.errors.OutOfRangeError:
break
self.log.info(
'c_val. at {} train step, class_loss: {:0.3f}, regress_loss: {:0.3f}'.
format(sess.run(self.local_step), np.asarray(class_loss).mean(), np.asarray(regress_loss).mean())
)
def test(self, sess):
self.data.reset()
cl_id = []
target_flag = []
target_sum = []
while True:
try:
fetches = [
self.test_model.data_batch,
self.test_model.predicted_target_sum,
self.test_model.predicted_flag,
]
fetched = sess.run(fetches, {self.keep_prob_pl: 1.0})
cl_id.append(fetched[0]['cl_id'])
target_sum.append(fetched[1])
target_flag.append(fetched[2])
except tf.errors.OutOfRangeError:
return cl_id, target_sum, target_flag
def test_and_save(self, sess):
self.log.info('Processing test data...')
cl_id, target_sum, target_flag = self.test(sess)
predictions = {
'cl_id': np.concatenate(cl_id),
'target_sum': np.concatenate(target_sum, axis=0)[:, 0],
'target_flag': np.concatenate(target_flag),
}
# Threshold values:
predictions['target_sum'][predictions['target_sum'] < 1000] = 0
task_1_filename = './data/submission_task_1_{}.csv'.format(datetime.datetime.now().strftime("%Y-%m-%d_%H:%M"))
task_2_filename = './data/submission_task_2_{}.csv'.format(datetime.datetime.now().strftime("%Y-%m-%d_%H:%M"))
self.log.info('Saving results in: {}, {}...'.format(task_1_filename, task_2_filename))
self.task_1_df = pd.DataFrame(
np.stack([predictions['cl_id'], predictions['target_flag']], axis=-1),
index = predictions['cl_id'],
columns=['_ID_', '_VAL_']
)
self.task_2_df = pd.DataFrame(
np.stack([predictions['cl_id'], predictions['target_sum']], axis=-1),
index = predictions['cl_id'],
columns=['_ID_', '_VAL_']
)
self.task_1_df.to_csv(task_1_filename, index=False)
self.task_2_df.to_csv(task_2_filename, index=False)
self.log.info('Done.')
import discord
from discord.ext import commands
import json
import datetime
import time
from logbook import Logger, StreamHandler
import sys
import inspect
import sqlite3
import asyncio
from utils import checks
StreamHandler(sys.stdout).push_application()
log = Logger("GAF Bot")
log.notice("Loading Configuration File")
try:
with open("config/config.json") as f:
_config = json.load(f)
except FileNotFoundError:
log.error("Config file not found, loading defaults")
df = open("config/defaults/default.config.json")
_config = json.load(df)
with open("config/config.json", "w") as f:
log.info("Saved new config to file")
f.write(json.dumps(_config))
try:
with open("config/modules.json") as f:
_modules = json.load(f)
with open("config/defaults/default.modules.json") as df:
class BaseSynchroRunner():
"""
Data provider class. Interacts with environment and outputs data in form of rollouts augmented with
relevant summaries and metadata. This runner is `synchronous` in sense that data collection is `in-process'
and every rollout is collected by explicit call to respective `get_data()` method [this is unlike 'async-`
thread-runner version found earlier in this this package which, once being started,
runs on its own and can not be moderated].
Makes precise control on policy being executed possible.
Does not support 'atari' mode.
"""
def __init__(self,
env,
task,
rollout_length,
episode_summary_freq,
env_render_freq,
ep_summary,
test=False,
policy=None,
data_sample_config=None,
memory_config=None,
test_conditions=None,
slowdown_steps=0,
global_step_op=None,
aux_render_modes=None,
_implemented_aux_render_modes=None,
name='synchro',
log_level=WARNING,
**kwargs):
"""
Args:
env: BTgym environment instance
task: int, runner task id
rollout_length: int
episode_summary_freq: int
env_render_freq: int
test: legacy, not used
ep_summary: legacy, not used
policy: policy instance to execute
data_sample_config: dict, data sampling configuration dictionary
memory_config: dict, replay memory configuration
test_conditions: dict or None,
dictionary of single experience conditions to check to mark it as test one.
slowdown_time: time to sleep between steps
aux_render_modes: iterable of str, additional summaries to compute
_implemented_aux_render_modes iterable of str, implemented additional summaries
name: str, name scope
log_level: int, logbook.level
"""
self.env = env
self.task = task
self.name = name
self.rollout_length = rollout_length
self.episode_summary_freq = episode_summary_freq
self.env_render_freq = env_render_freq
self.memory_config = memory_config
self.policy = policy
self.data_sample_config = data_sample_config
self.log_level = log_level
StreamHandler(sys.stdout).push_application()
self.log = Logger('{}_Runner_{}'.format(self.name, self.task),
level=self.log_level)
# Aux rendering setup:
if _implemented_aux_render_modes is None:
self.implemented_aux_render_modes = []
else:
self.implemented_aux_render_modes = _implemented_aux_render_modes
self.aux_render_modes = []
if aux_render_modes is not None:
for mode in aux_render_modes:
if mode in self.implemented_aux_render_modes:
self.aux_render_modes.append(mode)
else:
msg = 'Render mode `{}` is not implemented.'.format(mode)
self.log.error(msg)
raise NotImplementedError(msg)
self.log.debug('self.render modes: {}'.format(self.aux_render_modes))
self.sess = None
self.summary_writer = None
self.global_step_op = global_step_op
if self.task == 0 and slowdown_steps > 0 and self.global_step_op is not None:
self.log.notice(
'is slowed down by {} global_iterations/step'.format(
slowdown_steps))
self.slowdown_steps = slowdown_steps
else:
self.slowdown_steps = 0
if test_conditions is None:
# Default test conditions are: experience comes from test episode, from target domain:
self.test_conditions = {
'state': {
'metadata': {
'type': 1,
'trial_type': 1
}
}
}
else:
self.test_conditions = test_conditions
# Make replay memory:
if self.memory_config is not None:
self.memory = self.memory_config['class_ref'](
**self.memory_config['kwargs'])
else:
self.memory = _DummyMemory()
self.length = 0
self.local_episode = 0
self.reward_sum = 0
self.terminal_end = True
# Summary averages accumulators:
self.total_r = []
self.cpu_time = []
self.final_value = []
self.total_steps = []
self.total_steps_atari = []
self.info = [None]
self.pre_experience = None
self.state = None
self.context = None
self.last_action = None
self.last_reward = None
# Episode accumulators:
self.ep_accum = None
self.log.debug('__init__() done.')
def sleep(self):
if self.slowdown_steps > 0:
start_global_step = self.sess.run(self.global_step_op)
while start_global_step + self.slowdown_steps > self.sess.run(
self.global_step_op):
time.sleep(0.05)
def start_runner(self, sess, summary_writer, **kwargs):
"""
Legacy wrapper.
"""
self.start(sess, summary_writer, **kwargs)
def start(self,
sess,
summary_writer,
init_context=None,
data_sample_config=None):
"""
Executes initial sequence; fills initial replay memory if any.
"""
assert self.policy is not None, 'Initial policy not specified'
self.sess = sess
self.summary_writer = summary_writer
# # Hacky but we need env.renderer methods ready: NOT HERE, went to VerboseRunner
# self.env.renderer.initialize_pyplot()
self.pre_experience, self.state, self.context, self.last_action, self.last_reward = self.get_init_experience(
policy=self.policy,
init_context=init_context,
data_sample_config=data_sample_config)
if self.memory_config is not None:
while not self.memory.is_full():
# collect some rollouts to fill memory:
_ = self.get_data()
self.log.notice('Memory filled')
self.log.notice('started collecting data.')
def get_init_experience(self,
policy,
policy_sync_op=None,
init_context=None,
data_sample_config=None):
"""
Starts new environment episode.
Args:
policy: policy to execute.
policy_sync_op: operation copying local behavioural policy params from global one
init_context: initial policy context for new episode.
data_sample_config: configuration dictionary of type `btgym.datafeed.base.EnvResetConfig`
Returns:
incomplete initial experience of episode as dictionary (misses bootstrapped R value),
next_state,
next, policy RNN context
action_reward
"""
self.length = 0
self.reward_sum = 0
# Increment global and local episode counters:
self.sess.run(self.policy.inc_episode)
self.local_episode += 1
# self.log.warning('get_init_exp() data_sample_config: {}'.format(data_sample_config))
if data_sample_config is None:
data_sample_config = policy.get_sample_config()
# Pass sample config to environment (.get_sample_config() is actually aac framework method):
init_state = self.env.reset(**data_sample_config)
# Master worker always resets context at the episode beginning:
# TODO: !
# if not self.data_sample_config['mode']:
init_context = None
# self.log.warning('init_context_passed: {}'.format(init_context))
# self.log.warning('state_metadata: {}'.format(state['metadata']))
init_action = self.env.action_space.encode(
self.env.get_initial_action())
init_reward = np.asarray(0.0)
# Update policy:
if policy_sync_op is not None:
self.sess.run(policy_sync_op)
init_context = policy.get_initial_features(state=init_state,
context=init_context)
action, logits, value, next_context = policy.act(
init_state,
init_context,
init_action[None, ...],
init_reward[None, ...],
)
next_state, reward, terminal, self.info = self.env.step(
action['environment'])
experience = {
'position': {
'episode': self.local_episode,
'step': self.length
},
'state': init_state,
'action': action['one_hot'],
'reward': reward,
'value': value,
'terminal': terminal,
'context': init_context,
'last_action': init_action,
'last_reward': init_reward,
'r': None, # to be updated
'info': self.info[-1],
}
# Execute user-defined callbacks to policy, if any:
for key, callback in policy.callback.items():
experience[key] = callback(**locals())
# reset per-episode counters and accumulators:
self.ep_accum = {
'logits': [logits],
'value': [value],
'context': [init_context]
}
self.terminal_end = terminal
#self.log.warning('init_experience_context: {}'.format(context))
# Take a nap:
self.sleep()
return experience, next_state, next_context, action['encoded'], reward
def get_experience(self,
policy,
state,
context,
action,
reward,
policy_sync_op=None):
"""
Get single experience (possibly terminal).
Returns:
incomplete experience as dictionary (misses bootstrapped R value),
next_state,
next, policy RNN context
action_reward
"""
# Update policy:
# TODO: for contd. enable for meta-learning
# if policy_sync_op is not None:
# self.sess.run(policy_sync_op)
# TODO: meta-learning related; TEMPORAL, refract:
# if hasattr(policy, 'meta') and self.task == 0:
# wait = 1
# i = 0
# while wait:
# wait = policy.meta.act()
# # self.log.warning('policy_meta_action: {}'.format(wait))
# time.sleep(4)
# # i += 1
# self.sess.run([policy_sync_op, policy.meta.sync_slot_op])
#
# policy.meta.reset()
# policy.meta.global_reset()
#
# # self.log.notice('waited: {}'.format(i))
# Continue adding experiences to rollout:
next_action, logits, value, next_context = policy.act(
state,
context,
action[None, ...],
reward[None, ...],
)
self.ep_accum['logits'].append(logits)
self.ep_accum['value'].append(value)
self.ep_accum['context'].append(next_context)
# self.log.notice('context: {}'.format(context))
next_state, next_reward, terminal, self.info = self.env.step(
next_action['environment'])
# Partially compose experience:
experience = {
'position': {
'episode': self.local_episode,
'step': self.length
},
'state': state,
'action': next_action['one_hot'],
'reward': next_reward,
'value': value,
'terminal': terminal,
'context': context,
'last_action': action,
'last_reward': reward,
'r': None,
'info': self.info[-1],
}
for key, callback in policy.callback.items():
experience[key] = callback(**locals())
# Housekeeping:
self.length += 1
# Take a nap:
# self.sleep()
return experience, next_state, next_context, next_action[
'encoded'], next_reward
def get_train_stat(self, is_test=False):
"""
Updates and computes average statistics for train episodes.
Args:
is_test: bool, current episode type
Returns:
dict of stats
"""
ep_stat = {}
if not is_test:
self.total_r += [self.reward_sum]
episode_stat = self.env.get_stat() # get episode statistic
last_i = self.info[-1] # pull most recent info
self.cpu_time += [episode_stat['runtime'].total_seconds()]
self.final_value += [last_i['broker_value']]
self.total_steps += [episode_stat['length']]
if self.local_episode % self.episode_summary_freq == 0:
ep_stat = dict(total_r=np.average(self.total_r),
cpu_time=np.average(self.cpu_time),
final_value=np.average(self.final_value),
steps=np.average(self.total_steps))
self.total_r = []
self.cpu_time = []
self.final_value = []
self.total_steps = []
self.total_steps_atari = []
return ep_stat
def get_test_stat(self, is_test=False):
"""
Updates and computes statistics for single test episode.
Args:
is_test: bool, current episode type
Returns:
dict of stats
"""
ep_stat = {}
if is_test:
episode_stat = self.env.get_stat() # get episode statistic
last_i = self.info[-1] # pull most recent info
ep_stat = dict(total_r=self.reward_sum,
final_value=last_i['broker_value'],
steps=episode_stat['length'])
return ep_stat
def get_ep_render(self, is_test=False):
"""
Collects environment renderings. Relies on environment renderer class methods,
so it is only valid when environment rendering is enabled (typically it is true for master runner).
Returns:
dictionary of images as rgb arrays
"""
# Only render chief worker and test (slave) environment:
# if self.task < 1 and (
# is_test or(
# self.local_episode % self.env_render_freq == 0 and not self.data_sample_config['mode']
# )
# ):
if self.task < 1 and self.local_episode % self.env_render_freq == 0:
# Render environment (chief worker only):
render_stat = {
mode: self.env.render(mode)[None, :]
for mode in self.env.render_modes
}
else:
render_stat = None
return render_stat
def get_data(self,
policy=None,
policy_sync_op=None,
init_context=None,
data_sample_config=None,
rollout_length=None,
force_new_episode=False):
"""
Collects single trajectory rollout and bunch of summaries using specified policy.
Updates episode statistics and replay memory.
Args:
policy: policy to execute
policy_sync_op: operation copying local behavioural policy params from global one
init_context: if specified, overrides initial episode context provided bu self.context
(valid only if new episode is started within this rollout).
data_sample_config: environment configuration parameters for next episode to sample:
configuration dictionary of type `btgym.datafeed.base.EnvResetConfig
rollout_length: length of rollout to collect, if specified - overrides self.rollout_length attr
force_new_episode: bool, if True - resets the environment
Returns:
data dictionary
"""
if policy is None:
policy = self.policy
if init_context is None:
init_context = self.context
if rollout_length is None:
rollout_length = self.rollout_length
rollout = Rollout()
is_test = False
train_ep_summary = None
test_ep_summary = None
render_ep_summary = None
if self.terminal_end or force_new_episode:
# Start new episode:
self.pre_experience, self.state, self.context, self.last_action, self.last_reward = self.get_init_experience(
policy=policy,
policy_sync_op=policy_sync_op,
init_context=init_context,
data_sample_config=data_sample_config)
# self.log.warning(
# 'started new episode with:\ndata_sample_config: {}\nforce_new_episode: {}'.
# format(data_sample_config, force_new_episode)
# )
# self.log.warning('pre_experience_metadata: {}'.format(self.pre_experience['state']['metadata']))
# NOTE: self.terminal_end is set actual via get_init_experience() method
# Collect single rollout:
while rollout.size < rollout_length - 1 and not self.terminal_end:
if self.pre_experience['terminal']:
# Episode has been just finished,
# need to complete and push last experience and update all episode summaries
self.pre_experience['r'] = np.asarray([0.0])
experience = None
self.state = None
self.context = None
self.last_action = None
self.last_reward = None
self.terminal_end = True
train_ep_summary = self.get_train_stat(is_test)
test_ep_summary = self.get_test_stat(is_test)
render_ep_summary = self.get_ep_render(is_test)
else:
experience, self.state, self.context, self.last_action, self.last_reward = self.get_experience(
policy=policy,
policy_sync_op=policy_sync_op,
state=self.state,
context=self.context,
action=self.last_action,
reward=self.last_reward)
# Complete previous experience by bootstrapping V from next one:
self.pre_experience['r'] = experience['value']
# Push:
rollout.add(self.pre_experience)
# Where are you coming from?
is_test = is_subdict(self.test_conditions, self.pre_experience)
# try:
# # Was it test (i.e. test episode from traget domain)?
# if self.pre_experience['state']['metadata']['type']\
# and self.pre_experience['state']['metadata']['trial_type']:
# is_test = True
#
# except KeyError:
# pass
# Only training rollouts are added to replay memory:
if not is_test:
self.memory.add(self.pre_experience)
self.reward_sum += self.pre_experience['reward']
# Move one step froward:
self.pre_experience = experience
# Done collecting rollout, either got termination of episode or not:
if not self.terminal_end:
# Bootstrap:
self.pre_experience['r'] = np.asarray([
policy.get_value(
self.pre_experience['state'],
self.pre_experience['context'],
self.pre_experience['last_action'][None, ...],
self.pre_experience['last_reward'][None, ...],
)
])
rollout.add(self.pre_experience)
if not is_test:
self.memory.add(self.pre_experience)
# self.log.warning('rollout.terminal: {}'.format(self.terminal_end))
# self.log.warning('rollout.size: {}'.format(rollout.size))
data = dict(
on_policy=rollout,
terminal=self.terminal_end,
off_policy=self.memory.sample_uniform(
sequence_size=rollout_length),
off_policy_rp=self.memory.sample_priority(exact_size=True),
ep_summary=train_ep_summary,
test_ep_summary=test_ep_summary,
render_summary=render_ep_summary,
is_test=is_test,
)
return data
def get_episode(self,
policy=None,
policy_sync_op=None,
init_context=None,
data_sample_config=None):
"""
== WRONG DO NOT USE ===
Collects entire episode trajectory as single rollout and bunch of summaries using specified policy.
Updates episode statistics and replay memory.
Args:
policy: policy to execute
policy_sync_op: operation copying local behavioural policy params from global one
init_context: if specified, overrides initial episode context provided bu self.context
data_sample_config: environment configuration parameters for next episode to sample:
configuration dictionary of type `btgym.datafeed.base.EnvResetConfig
Returns:
data dictionary
"""
if policy is None:
policy = self.policy
if init_context is None:
init_context = self.context
elif init_context == 0: # mmm... TODO: fix this shame
init_context = None
rollout = Rollout()
train_ep_summary = None
test_ep_summary = None
render_ep_summary = None
# Start new episode:
self.pre_experience, self.state, self.context, self.last_action, self.last_reward = self.get_init_experience(
policy=policy,
policy_sync_op=policy_sync_op,
init_context=
init_context, # None (initially) or final context of previous episode
data_sample_config=data_sample_config)
is_test = is_subdict(self.test_conditions, self.pre_experience)
# try:
# # Was it test (`type` in metadata is not zero)?
# # TODO: change to source/target?
# if self.pre_experience['state']['metadata']['type']:
# is_test = True
#
# except KeyError:
# pass
# Collect data until episode is over:
while not self.terminal_end:
experience, self.state, self.context, self.last_action = self.get_experience(
policy=policy,
policy_sync_op=policy_sync_op,
state=self.state,
context=self.context,
action=self.last_action,
reward=self.last_reward,
)
# Complete previous experience by bootstrapping V from next one:
self.pre_experience['r'] = experience['value']
# Push:
rollout.add(self.pre_experience)
if not is_test:
self.memory.add(self.pre_experience)
# Move one step froward:
self.pre_experience = experience
self.reward_sum += experience['reward']
if self.pre_experience['terminal']:
# Episode has been just finished,
# need to complete and push last experience and update all episode summaries:
self.terminal_end = True
# Bootstrap:
# TODO: should be zero here
self.pre_experience['r'] = np.asarray([
policy.get_value(
self.pre_experience['state'],
self.pre_experience['context'],
self.pre_experience['last_reward'][None, ...],
self.pre_experience['last_reward'][None, ...],
)
])
rollout.add(self.pre_experience)
if not is_test:
self.memory.add(self.pre_experience)
train_ep_summary = self.get_train_stat(is_test)
test_ep_summary = self.get_test_stat(is_test)
render_ep_summary = self.get_ep_render(is_test)
# self.log.warning('episodic_rollout.size: {}'.format(rollout.size))
data = dict(
on_policy=rollout,
terminal=self.terminal_end,
off_policy=self.memory.sample_uniform(
sequence_size=self.rollout_length),
off_policy_rp=self.memory.sample_priority(exact_size=True),
ep_summary=train_ep_summary,
test_ep_summary=test_ep_summary,
render_summary=render_ep_summary,
is_test=is_test,
)
return data
def get_batch(self,
size,
policy=None,
policy_sync_op=None,
require_terminal=True,
same_trial=True,
init_context=None,
data_sample_config=None):
"""
Returns batch as list of 'size' or more rollouts collected under specified policy.
Rollouts can be collected from several episodes consequently; there is may be more rollouts than set 'size' if
it is necessary to collect at least one terminal rollout.
Args:
size: int, number of rollouts to collect
policy: policy to use
policy_sync_op: operation copying local behavioural policy params from global one
require_terminal: bool, if True - require at least one terminal rollout to be present.
same_trial: bool, if True - all episodes are sampled from same trial
init_context: if specified, overrides initial episode context provided bu self.context
data_sample_config: environment configuration parameters for all episodes in batch:
configuration dictionary of type `btgym.datafeed.base.EnvResetConfig
Returns:
dict containing:
'data'key holding list of data dictionaries;
'terminal_context' key holding list of terminal output contexts.
If 'require_terminal = True, this list is guarantied to hold at least one element.
"""
batch = []
terminal_context = []
if require_terminal:
got_terminal = False
else:
got_terminal = True
if same_trial:
assert isinstance(data_sample_config, dict),\
'get_batch(same_trial=True) expected `data_sample_config` dict., got: {}'.format(data_sample_config)
# Collect first rollout:
batch = [
self.get_data(policy=policy,
policy_sync_op=policy_sync_op,
init_context=init_context,
data_sample_config=data_sample_config,
force_new_episode=True)
]
if same_trial:
# sample new episodes from same trial only:
data_sample_config['trial_config']['get_new'] = False
collected_size = 1
if batch[0]['terminal']:
terminal_context.append(self.context)
got_terminal = True
# Collect others:
while not (collected_size >= size and got_terminal):
rollout_data = self.get_data(policy=policy,
policy_sync_op=policy_sync_op,
init_context=init_context,
data_sample_config=data_sample_config)
batch.append(rollout_data)
if rollout_data['terminal']:
terminal_context.append(self.context)
got_terminal = True
collected_size += 1
data = dict(
data=batch,
terminal_context=terminal_context,
)
return data
"""
import os
import sys
from logbook import Processor, StreamHandler, DEBUG, Logger, FileHandler
my_handler = FileHandler("test.log", encoding="utf-8", level=DEBUG)
# my_handler = StreamHandler(sys.stdout, level=DEBUG)
def log_other_info(record):
"""
a) 通过 with.processor可以让在其中的日志拥有共同的逻辑，相当于一个切面注入
比如这里的例子是 在每条日志中记录一些额外的信息（额外的信息是通过在日志对象（logRecord）的extra（字典对象）属性中添加
一些其他的信息），这样每条日志都会有这里添加的额外的信息。
b) 有个疑问就是，这些额外的信息怎么运用呢，比如这些信息如何能和日志一块记录在文件中呢
c) 关于日志的属性，见 logrecord.py
"""
record.extra['myname'] = 'kute'
record.extra['mycwd'] = os.getcwd()
# update myname propertiy
record.extra.update(myname="lisa")
print(record.to_dict())
if __name__ == "__main__":
with my_handler.applicationbound():
with Processor(log_other_info).applicationbound():
mylog = Logger("processor")
mylog.notice("notice msg.")
def on_closed(self, resp):
#
pass
def userstream(self,**args):
pass
def filter(self,follow=None, async=False, start_time=None):
if self.running:
pass # *** there should raise ERROR like already connected. *** #
self.url = '/%s/me/home' % API_VERSION
self.fb_api = facebook.GraphAPI(access_token = self.auth)
if start_time:
self.start_time = start_time
else:
self.start_time = int( time.time() )
logging.notice(u"start_time is {0}".format( time.strftime("%Y/%m/%d %H:%M:%S %p",time.localtime(int(start_time)) )))
if self.expired:
self.extend_token()
self._start(async)
def extend_token(self):
try:
new_token = self.fb_api.extend_access_token(self.client_id, self.client_secret)
self.fb_api.access_token = new_token['access_token']
self.expired = int(new_token['expires']) - 86400
self.config.set('facebook-client', 'facebook_token', self.fb_api.access_token)
self.config.write(open('./conf/tweets-client.ini', 'w'))
logging.notice(u'extend_token success, expire in {0}'.format(self.expired))
except Exception as e:
logging.debug(u'extend_token error:{exception}',exception=e)
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.core.management.base import BaseCommand
from notify.models import PreNotification, Notification
from time import sleep
import signal
import www.settings as settings
from django.db import close_connection
from logbook import Logger
log = Logger('Notification')
log.notice('Start deamon')
class Command(BaseCommand):
def handle(self, *args, **options):
for sig in (signal.SIGABRT, signal.SIGILL, signal.SIGINT, signal.SIGSEGV, signal.SIGTERM):
signal.signal(sig, self.terminate)
try:
while True:
self.process_pack(settings.PACK_SIZE)
except KeyboardInterrupt:
print ""
self.terminate(None,None)
def process_pack(self, pack_size):
objects = PreNotification.objects.order_by('created').filter(delivered=False)[:pack_size]
if len(objects) == 0:
sleep(2)
else:
result = {"result": "travel date not provided"}
status_code = status.HTTP_400_BAD_REQUEST
else:
result = {"result": "license plate not provided"}
status_code = status.HTTP_400_BAD_REQUEST
else:
result = {"result": "wrong content type"}
status_code = status.HTTP_415_UNSUPPORTED_MEDIA_TYPE
return Response(
json.dumps(result),
status=status_code,
mimetype="application/json"
)
if __name__ == "__main__":
"""
The main entry point of the application
"""
StreamHandler(sys.stdout).push_application()
log = Logger('Main - Urban Transport System')
log.notice("Flask server started.")
check_thread.main()
log.notice("Control Thread started.")
app.run(host="0.0.0.0", port=5000, debug=False)
