def printList(listItems, title="", showSelector=False,
showByStep=20, showUrl=False):
if title:
separator("=", title)
total = len(listItems)
printLine("Total found: %d" % total)
for key, item in enumerate(listItems):
key += 1
printLine("%s : %s%s%s" % (
str(key).rjust(3, " "),
printDate(item.created).ljust(12, " ") if hasattr(item, 'created') else '',
item.title if hasattr(item, 'title') else item.name,
" " + (">>> " + config.NOTE_URL % item.guid) if showUrl else '',))
if key % showByStep == 0 and key < total:
printLine("-- More --", "\r")
tools.getch()
printLine(" " * 12, "\r")
if showSelector:
printLine(" 0 : -Cancel-")
try:
while True:
num = rawInput(": ")
if tools.checkIsInt(num) and 1 <= int(num) <= total:
return listItems[int(num) - 1]
if num == '0':
exit(1)
failureMessage('Incorrect number "%s", '
'please try again:\n' % num)
except (KeyboardInterrupt, SystemExit):
tools.exit()
def wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except Exception, e:
logging.error("Error: %s : %s", func.__name__, str(e))
if not hasattr(e, 'errorCode'):
out.failureMessage("Sorry, operation has failed!!!.")
tools.exit()
errorCode = int(e.errorCode)
# auth-token error, re-auth
if errorCode == 9:
storage = Storage()
storage.removeUser()
GeekNote()
return func(*args, **kwargs)
elif errorCode == 3:
out.failureMessage("Sorry, you do not have permissions to do this operation.")
else:
return False
tools.exit()
def GetUserAuthCode():
"""Prompts the user for a two factor auth code."""
try:
code = None
if code is None:
code = rawInput("Two-Factor Authentication Code: ")
except (KeyboardInterrupt, SystemExit):
tools.exit()
return code
def login(self):
if self.getEvernote().checkAuth():
out.successMessage("You have already logged in.")
return tools.exit()
if self.getEvernote().auth():
out.successMessage("You have successfully logged in.")
else:
out.failureMessage("Login error.")
return tools.exit()
def allowAccess(self):
response = self.loadPage(self.url['base'],
self.url['access'],
"GET",
{'oauth_token': self.tmpOAuthToken})
logging.debug(response.data)
tree = html.fromstring(response.data);
token = "&" + urlencode({ 'csrfBusterToken': tree.xpath("//input[@name='csrfBusterToken']/@value")[0]}) + "&" + urlencode({ 'csrfBusterToken': tree.xpath("//input[@name='csrfBusterToken']/@value")[1]})
sourcePage = tree.xpath("//input[@name='_sourcePage']/@value")[0]
fp = tree.xpath("//input[@name='__fp']/@value")[0]
targetUrl = tree.xpath("//input[@name='targetUrl']/@value")[0]
logging.debug(token);
if response.status != 200:
logging.error("Unexpected response status "
"on login 200 != %s", response.status)
tools.exitErr()
if 'JSESSIONID' not in self.cookies:
logging.error("Not found value JSESSIONID in the response cookies")
tools.exitErr()
access = self.postData['access']
access['oauth_token'] = self.tmpOAuthToken
access['oauth_callback'] = ""
access['embed'] = 'false'
access['suggestedNotebookName'] = 'Geeknote'
access['supportLinkedSandbox'] = ''
access['analyticsLoginOrigin'] = 'Other'
access['clipperFlow'] = 'false'
access['showSwitchService'] = 'true'
access['_sourcePage'] = sourcePage
access['__fp'] = fp
access['targetUrl'] = targetUrl
response = self.loadPage(self.url['base'],
self.url['access'],
"POST", access, token)
if response.status != 302:
logging.error("Unexpected response status on allowing "
"access 302 != %s", response.status)
logging.error(response.data)
tools.exitErr()
responseData = self.parseResponse(response.location)
if not responseData.has_key('oauth_verifier'):
logging.error("OAuth verifier not found")
tools.exit()
self.verifierToken = responseData['oauth_verifier']
logging.debug("OAuth verifier token take")
def confirm(message):
printLine(message)
try:
while True:
answer = rawInput("Yes/No: ")
if answer.lower() in ["yes", "ye", "y"]:
return True
if answer.lower() in ["no", "n"]:
return False
failureMessage('Incorrect answer "%s", ' "please try again:\n" % answer)
except (KeyboardInterrupt, SystemExit):
tools.exit()
def GetUserCredentials():
"""Prompts the user for a username and password."""
try:
login = None
password = None
if login is None:
login = rawInput("Login: "******"Password: ", True)
except (KeyboardInterrupt, SystemExit):
tools.exit()
return (login, password)
def logout(self, force=None):
if not self.getEvernote().checkAuth():
out.successMessage("You have already logged out.")
return tools.exit()
if not force and not out.confirm('Are you sure you want to logout?'):
return tools.exit()
result = self.getEvernote().removeUser()
if result:
out.successMessage("You have successfully logged out.")
else:
out.failureMessage("Logout error.")
return tools.exit()
def GetUserAuthCode():
"""Prompts the user for a two factor auth code."""
creds = _getCredentialsFromFile()
if creds is not None:
return creds[2]
try:
code = None
if code is None:
code = rawInput("Two-Factor Authentication Code: ")
except (KeyboardInterrupt, SystemExit), e:
if e.message:
tools.exit(e.message)
else:
tools.exit
def getOAuthToken(self):
response = self.loadPage(self.url['base'], self.url['token'], "GET",
self.getTokenRequestData(oauth_token=self.tmpOAuthToken, oauth_verifier=self.verifierToken))
if response.status != 200:
logging.error("Unexpected response status on getting oauth token 200 != %s", response.status)
tools.exit()
responseData = self.parseResponse(response.data)
if not responseData.has_key('oauth_token'):
logging.error("OAuth token not found")
tools.exit()
logging.debug("OAuth token take : %s", responseData['oauth_token'])
self.OAuthToken = responseData['oauth_token']
def checkVersion(self):
versionOK = self.getUserStore().checkVersion("Python EDAMTest",
UserStoreConstants.EDAM_VERSION_MAJOR,
UserStoreConstants.EDAM_VERSION_MINOR)
if not versionOK:
logging.error("Old EDAM version")
return tools.exit()
def textToENML(content, raise_ex=False, format='markdown'):
"""
Create an ENML format of note.
"""
if not isinstance(content, str):
content = ""
try:
content = unicode(content, "utf-8")
# add 2 space before new line in paragraph for creating br tags
content = re.sub(r'([^\r\n])([\r\n])([^\r\n])', r'\1 \n\3', content)
if format=='markdown':
contentHTML = markdown.markdown(content).encode("utf-8")
# Non-Pretty HTML output
contentHTML = str(BeautifulSoup(contentHTML))
else:
contentHTML = Editor.HTMLEscape(content)
return Editor.wrapENML(contentHTML)
except:
if raise_ex:
raise Exception("Error while parsing text to html."
" Content must be an UTF-8 encode.")
logging.error("Error while parsing text to html. "
"Content must be an UTF-8 encode.")
out.failureMessage("Error while parsing text to html. "
"Content must be an UTF-8 encode.")
return tools.exit()
def _searchNote(self, note):
note = tools.strip(note)
# load search result
result = self.getStorage().getSearch()
if result and tools.checkIsInt(note) and 1 <= int(note) <= len(result.notes):
note = result.notes[int(note)-1]
else:
request = self._createSearchRequest(search=note)
logging.debug("Search notes: %s" % request)
result = self.getEvernote().findNotes(request, 20)
logging.debug("Search notes result: %s" % str(result))
if result.totalNotes == 0:
out.successMessage("Notes have not been found.")
return tools.exit()
elif result.totalNotes == 1 or self.selectFirstOnUpdate:
note = result.notes[0]
else:
logging.debug("Choose notes: %s" % str(result.notes))
note = out.SelectSearchResult(result.notes)
logging.debug("Selected note: %s" % str(note))
return note
def textToENML(content, raise_ex=False, format='markdown'):
"""
Create an ENML format of note.
"""
if not isinstance(content, str):
content = ""
try:
content = unicode(content, "utf-8")
# add 2 space before new line in paragraph for cteating br tags
content = re.sub(r'([^\r\n])([\r\n])([^\r\n])', r'\1 \n\3', content)
if format=='markdown':
contentHTML = markdown.markdown(content).encode("utf-8")
# remove all new-lines characters in html
contentHTML = re.sub(r'\n', r'', contentHTML)
else:
contentHTML = HTMLEscape(content)
return wrapENML(contentHTML)
except:
if raise_ex:
raise Exception("Error while parsing text to html."
" Content must be an UTF-8 encode.")
logging.error("Error while parsing text to html. "
"Content must be an UTF-8 encode.")
out.failureMessage("Error while parsing text to html. "
"Content must be an UTF-8 encode.")
return tools.exit()
def _createSearchRequest(self, search=None, tags=None, notebooks=None, date=None, exact_entry=None, content_search=None):
request = ""
if notebooks:
for notebook in tools.strip(notebooks.split(',')):
if notebook.startswith('-'):
request += '-notebook:"%s" ' % tools.strip(notebook[1:])
else:
request += 'notebook:"%s" ' % tools.strip(notebook)
if tags:
for tag in tools.strip(tags.split(',')):
if tag.startswith('-'):
request +='-tag:"%s" ' % tag[1:]
else:
request +='tag:"%s" ' % tag
if date:
date = tools.strip(date.split('-'))
try:
dateStruct = time.strptime(date[0]+" 00:00:00", "%d.%m.%Y %H:%M:%S")
request +='created:%s ' % time.strftime("%Y%m%d", time.localtime(time.mktime(dateStruct)))
if len(date) == 2:
dateStruct = time.strptime(date[1]+" 00:00:00", "%d.%m.%Y %H:%M:%S")
request += '-created:%s ' % time.strftime("%Y%m%d", time.localtime(time.mktime(dateStruct)+60*60*24))
except ValueError, e:
out.failureMessage('Incorrect date format in --date attribute. Format: %s' % time.strftime("%d.%m.%Y", time.strptime('19991231', "%Y%m%d")))
return tools.exit()
def getTmpOAuthToken(self):
response = self.loadPage(self.url['base'], self.url['token'], "GET",
self.getTokenRequestData(oauth_callback="https://"+self.url['base']))
if response.status != 200:
logging.error("Unexpected response status on get temporary oauth_token 200 != %s", response.status)
tools.exit()
responseData = self.parseResponse(response.data)
if not responseData.has_key('oauth_token'):
logging.error("OAuth temporary not found")
tools.exit()
self.tmpOAuthToken = responseData['oauth_token']
logging.debug("Temporary OAuth token : %s", self.tmpOAuthToken)
def loadPage(self, url, uri=None, method="GET", params=""):
if not url:
logging.error("Request URL undefined")
tools.exit()
if not uri:
urlData = urlparse(url)
url = urlData.netloc
uri = urlData.path + '?' + urlData.query
# prepare params, append to uri
if params:
params = urlencode(params)
if method == "GET":
uri += ('?' if uri.find('?') == -1 else '&') + params
params = ""
# insert local cookies in request
headers = {
"Cookie": '; '.join([key + '=' + self.cookies[key] for key in self.cookies.keys()])
}
if method == "POST":
headers["Content-type"] = "application/x-www-form-urlencoded"
logging.debug("Request URL: %s:/%s > %s # %s", url,
uri, unquote(params), headers["Cookie"])
conn = httplib.HTTPSConnection(url)
conn.request(method, uri, params, headers)
response = conn.getresponse()
data = response.read()
conn.close()
logging.debug("Response : %s > %s",
response.status,
response.getheaders())
result = tools.Struct(status=response.status,
location=response.getheader('location', None),
data=data)
# update local cookies
sk = Cookie.SimpleCookie(response.getheader("Set-Cookie", ""))
for key in sk:
self.cookies[key] = sk[key].value
return result
def handleTwoFactor(self):
self.code = out.GetUserAuthCode()
self.postData['tfa']['code'] = self.code
response = self.loadPage(self.url['base'], self.url['tfa']+";jsessionid="+self.cookies['JSESSIONID'], "POST", self.postData['tfa'])
if not response.location and response.status == 200:
if self.incorrectCode < 3:
out.preloader.stop()
out.printLine('Sorry, incorrect two factor code')
out.preloader.setMessage('Authorize...')
self.incorrectCode += 1
return self.handleTwoFactor()
else:
logging.error("Incorrect two factor code")
if not response.location:
logging.error("Target URL was not found in the response on login")
tools.exit()
def allowAccess(self):
self.postData['access']['oauth_token'] = self.tmpOAuthToken
self.postData['access']['oauth_callback'] = "https://"+self.url['base']
response = self.loadPage(self.url['base'], self.url['access'], "POST", self.postData['access'])
if response.status != 302:
logging.error("Unexpected response status on allowing access 302 != %s", response.status)
tools.exit()
responseData = self.parseResponse(response.location)
if not responseData.has_key('oauth_verifier'):
logging.error("OAuth verifier not found")
tools.exit()
self.verifierToken = responseData['oauth_verifier']
logging.debug("OAuth verifier token take")
def modifyArgsByStdinStream():
content = sys.stdin.read()
content = tools.stdinEncode(content)
if not content:
out.failureMessage("Input stream is empty.")
return tools.exit()
title = " ".join(content.split(" ", 5)[:-1])
title = re.sub(r"(\r\n|\r|\n)", r" ", title)
if not title:
out.failureMessage("Error while crating title of note from stream.")
return tools.exit()
elif len(title) > 50:
title = title[0:50] + "..."
ARGS = {"title": title, "content": content}
return ("create", ARGS)
def user(self, full=None):
if not self.getEvernote().checkAuth():
out.failureMessage("You not logged in.")
return tools.exit()
if full:
info = self.getEvernote().getUserInfo()
else:
info = self.getStorage().getUserInfo()
out.showUser(info, full)
def GetUserCredentials():
"""Prompts the user for a username and password."""
creds = _getCredentialsFromFile()
if creds is not None:
return creds[:2]
try:
login = None
password = None
if login is None:
login = rawInput("Login: "******"Password: ", True)
except (KeyboardInterrupt, SystemExit), e:
if e.message:
tools.exit(e.message)
else:
tools.exit
def printList(listItems, title="", showSelector=False,
showByStep=0, showUrl=False, showTags=False,
showNotebook=False, showGUID=False):
if title:
separator("=", title)
total = len(listItems)
printLine("Found %d item%s" % (total, ('s' if total != 1 else '')))
for key, item in enumerate(listItems):
key += 1
printLine("%s : %s%s%s%s%s%s" % (
item.guid if showGUID and hasattr(item, 'guid') else str(key).rjust(3, " "),
printDate(item.created).ljust(11, " ") if hasattr(item, 'created') else '',
printDate(item.updated).ljust(11, " ") if hasattr(item, 'updated') else '',
item.notebookName.ljust(18, " ") if showNotebook and hasattr(item, 'notebookName') else '',
item.title if hasattr(item, 'title') else item.name,
"".join(map(lambda s: " #" + s, item.tagGuids)) if showTags and hasattr(item, 'tagGuids') and item.tagGuids else '',
" " + (">>> " + config.NOTE_WEBCLIENT_URL % item.guid) if showUrl else '',))
if showByStep != 0 and key % showByStep == 0 and key < total:
printLine("-- More --", "\r")
tools.getch()
printLine(" " * 12, "\r")
if showSelector:
printLine(" 0 : -Cancel-")
try:
while True:
num = rawInput(": ")
if tools.checkIsInt(num) and 1 <= int(num) <= total:
return listItems[int(num) - 1]
if num == '0' or num == 'q':
exit(1)
failureMessage('Incorrect number "%s", '
'please try again:\n' % num)
except (KeyboardInterrupt, SystemExit), e:
if e.message:
tools.exit(e.message)
else:
tools.exit
def edit(self, notebook, title):
notebook = self._searchNotebook(notebook)
out.preloader.setMessage("Updating notebook...")
result = self.getEvernote().updateNotebook(guid=notebook.guid, name=title)
if result:
out.successMessage("Notebook has been successfully updated.")
else:
out.failureMessage("Error while the updating the notebook.")
return tools.exit()
def edit(self, tagname, title):
tag = self._searchTag(tagname)
out.preloader.setMessage("Updating tag...")
result = self.getEvernote().updateTag(guid=tag.guid, name=title)
if result:
out.successMessage("Tag has been successfully updated.")
else:
out.failureMessage("Error while the updating the tag.")
return tools.exit()
def create(self, title):
self.connectToEvertone()
out.preloader.setMessage("Creating notebook...")
result = self.getEvernote().createNotebook(name=title)
if result:
out.successMessage("Notebook has been successfully created.")
else:
out.failureMessage("Error while the process of creating the notebook.")
return tools.exit()
return result
def login(self):
response = self.loadPage(self.url['base'], self.url['login'], "GET", {'oauth_token': self.tmpOAuthToken})
if response.status != 200:
logging.error("Unexpected response status on login 200 != %s", response.status)
tools.exit()
if not self.cookies.has_key('JSESSIONID'):
logging.error("Not found value JSESSIONID in the response cookies")
tools.exit()
# get login/password
self.username, self.password = out.GetUserCredentials()
self.postData['login']['username'] = self.username
self.postData['login']['password'] = self.password
self.postData['login']['targetUrl'] = self.url['oauth']%self.tmpOAuthToken
response = self.loadPage(self.url['base'], self.url['login']+";jsessionid="+self.cookies['JSESSIONID'], "POST",
self.postData['login'])
if not response.location and response.status == 200:
if self.incorrectLogin < 3:
out.preloader.stop()
out.printLine('Sorry, incorrect login or password')
out.preloader.setMessage('Authorize...')
self.incorrectLogin += 1
return self.login()
else:
logging.error("Incorrect login or password")
if not response.location:
logging.error("Target URL was not found in the response on login")
tools.exit()
logging.debug("Success authorize, redirect to access page")
def remove(self, notebook, force=None):
notebook = self._searchNotebook(notebook)
if not force and not out.confirm('Are you sure you want to delete this notebook: "%s"?' % notebook.name):
return tools.exit()
out.preloader.setMessage("Deleting notebook...")
result = self.getEvernote().removeNotebook(guid=notebook.guid)
if result:
out.successMessage("Notebook has been successfully removed.")
else:
out.failureMessage("Error while removing the notebook.")
def remove(self, tagname, force=None):
tag = self._searchTag(tagname)
if not force and not out.confirm('Are you sure you want to delete this tag: "%s"?' % tag.name):
return tools.exit()
out.preloader.setMessage("Deleting tag...")
result = self.getEvernote().removeTag(guid=tag.guid)
if result:
out.successMessage("Tag has been successfully removed.")
else:
out.failureMessage("Error while removing the tag.")
def modifyArgsByStdinStream():
content = sys.stdin.read()
content = tools.stdinEncode(content)
if not content:
out.failureMessage("Input stream is empty.")
return tools.exit()
title = ' '.join(content.split(' ', 5)[:-1])
title = re.sub(r'(\r\n|\r|\n)', r' ', title)
if not title:
out.failureMessage("Error while crating title of note from stream.")
return tools.exit()
elif len(title) > 50:
title = title[0:50] + '...'
ARGS = {
'title': title,
'content': content
}
return ('create', ARGS)
if COMMAND == 'notebook-remove':
Notebooks().remove(**ARGS)
# Tags
if COMMAND == 'tag-list':
Tags().list(**ARGS)
if COMMAND == 'tag-create':
Tags().create(**ARGS)
if COMMAND == 'tag-edit':
Tags().edit(**ARGS)
if COMMAND == 'tag-remove':
Tags().remove(**ARGS)
except (KeyboardInterrupt, SystemExit, tools.ExitException), e:
if e.message:
exit_status_code = e.message
except Exception, e:
traceback.print_exc()
logging.error("App error: %s", str(e))
# exit preloader
tools.exit('exit', exit_status_code)
if __name__ == "__main__":
main()
def main(args=None):
try:
exit_status_code = 0
sys_argv = sys.argv[1:]
if isinstance(args, list):
sys_argv = args
sys_argv = tools.decodeArgs(sys_argv)
COMMAND = sys_argv[0] if len(sys_argv) >= 1 else None
aparser = argparser(sys_argv)
ARGS = aparser.parse()
# error or help
if COMMAND is None or ARGS is False:
return tools.exit()
logging.debug("CLI options: %s", str(ARGS))
# Users
if COMMAND == 'user':
User().user(**ARGS)
if COMMAND == 'login':
User().login(**ARGS)
if COMMAND == 'logout':
User().logout(**ARGS)
if COMMAND == 'settings':
User().settings(**ARGS)
# Notes
if COMMAND == 'create':
Notes().create(**ARGS)
if COMMAND == 'edit':
Notes().edit(**ARGS)
if COMMAND == 'remove':
Notes().remove(**ARGS)
if COMMAND == 'show':
Notes().show(**ARGS)
if COMMAND == 'find':
Notes().find(**ARGS)
# Notebooks
if COMMAND == 'notebook-list':
Notebooks().list(**ARGS)
if COMMAND == 'notebook-create':
Notebooks().create(**ARGS)
if COMMAND == 'notebook-edit':
Notebooks().edit(**ARGS)
if COMMAND == 'notebook-remove':
Notebooks().remove(**ARGS)
# Tags
if COMMAND == 'tag-list':
Tags().list(**ARGS)
if COMMAND == 'tag-create':
Tags().create(**ARGS)
if COMMAND == 'tag-edit':
Tags().edit(**ARGS)
if COMMAND == 'tag-remove':
Tags().remove(**ARGS)
except (KeyboardInterrupt, SystemExit, tools.ExitException), e:
if e.message:
exit_status_code = e.message
def main(args=None):
try:
# if terminal
if config.IS_IN_TERMINAL:
sys_argv = sys.argv[1:]
if isinstance(args, list):
sys_argv = args
sys_argv = tools.decodeArgs(sys_argv)
COMMAND = sys_argv[0] if len(sys_argv) >= 1 else None
aparser = argparser(sys_argv)
ARGS = aparser.parse()
# if input stream
else:
COMMAND, ARGS = modifyArgsByStdinStream()
# error or help
if COMMAND is None or ARGS is False:
return tools.exit()
logging.debug("CLI options: %s", str(ARGS))
# Users
if COMMAND == 'user':
User().user(**ARGS)
if COMMAND == 'login':
User().login(**ARGS)
if COMMAND == 'logout':
User().logout(**ARGS)
if COMMAND == 'settings':
User().settings(**ARGS)
# Notes
if COMMAND == 'create':
Notes().create(**ARGS)
if COMMAND == 'edit':
Notes().edit(**ARGS)
if COMMAND == 'remove':
Notes().remove(**ARGS)
if COMMAND == 'show':
Notes().show(**ARGS)
if COMMAND == 'find':
Notes().find(**ARGS)
# Notebooks
if COMMAND == 'notebook-list':
Notebooks().list(**ARGS)
if COMMAND == 'notebook-create':
Notebooks().create(**ARGS)
if COMMAND == 'notebook-edit':
Notebooks().edit(**ARGS)
if COMMAND == 'notebook-remove':
Notebooks().remove(**ARGS)
# Tags
if COMMAND == 'tag-list':
Tags().list(**ARGS)
if COMMAND == 'tag-create':
Tags().create(**ARGS)
if COMMAND == 'tag-edit':
Tags().edit(**ARGS)
if COMMAND == 'tag-remove':
Tags().remove(**ARGS)
except (KeyboardInterrupt, SystemExit, tools.ExitException):
pass
except Exception, e:
traceback.print_exc()
logging.error("App error: %s", str(e))
def train(self,
inputs,
autosave=None,
cont=False,
count=False,
save_graph=True):
"""
Args:
inputs: A python iterable of things that encode to python iterables (if long) or lists (if short) of numbers
"""
if count:
logging.info(
'Counting enabled! This should only be used if inputs can be iterated over multiple times.'
)
logging.info('Counting number of steps...')
# Only use count if inputs and the items in it can be iterated over multiple times
total_round_steps = self.encode_and_count_batches_for_training(
inputs)
logging.info('Number of steps is %s' % total_round_steps)
batches = self.encode_and_make_batches_for_training(inputs)
if cont:
logging.info('Continuing enabled! Starting from round %s' %
self.round_steps)
# Note: round_steps is the number of rounds completed, so
# is also the index of the next round to be done
batches = itertools.islice(batches, self.round_steps, None)
else:
self.round_steps = 0
save_dir = os.path.join(self.saved_summaries_dir, self.name, self.tag)
os.makedirs(save_dir, exist_ok=True)
summaries_dir = save_dir
logging.info('Using summaries dir %s' % summaries_dir)
if save_graph and self.training_steps == 0:
logging.info('Saving graph...')
# Only write graph during the first training attempt
summary_writer = tf.summary.FileWriter(summaries_dir, self.graph)
else:
summary_writer = tf.summary.FileWriter(summaries_dir)
logging.info('Starting training...')
curr_states = None
starting_i = self.training_steps
starting_round = self.round_steps
# Need this i set for if there ends up being no batches due to continuing
i = starting_i
try:
for batch in batches:
with tools.DelayedKeyboardInterrupt():
# Use key instead of i or self.training_steps
# to prevent the unlikely race condition
# of hitting the interrupt after
# the key is updated but before entering the
# DelayedKeyboardInterrupt section
self.training_steps += 1
self.round_steps += 1
i = self.training_steps
# i, training_steps will be steps completed after the run_batch
if i - 1 == starting_i:
loss_max, loss_mean, loss_min = self._run_batch(
[self.loss_max, self.loss_mean, self.loss_min],
batch, curr_states)
logging.info(
'Starting values: steps: %s round: %s loss max: %s mean: %s min: %s'
% (starting_i, starting_round, loss_max, loss_mean,
loss_min))
if count:
time_rem = tools.TimeRemaining(
total_round_steps, starting_round)
if i % 10 != 0:
_ = self._run_batch([self.optimize], batch,
curr_states)
else:
_, summary, loss_max, loss_mean, loss_min = self._run_batch(
[
self.optimize, self.summary, self.loss_max,
self.loss_mean, self.loss_min
], batch, curr_states)
summary_writer.add_summary(summary, i)
# Note: The printed numbers are the numbers from before the optimization update happens
# Note: Step numbers start from 1
if count:
time_rem_str = time_rem.get_str(self.round_steps)
logging.info(
'Step %s round %s/%s: loss max: %s mean: %s min: %s time rem: %s'
%
(i, self.round_steps, total_round_steps,
loss_max, loss_mean, loss_min, time_rem_str))
else:
logging.info(
'Step %s round %s: loss max: %s mean: %s min: %s'
% (i, self.round_steps, loss_max, loss_mean,
loss_min))
#losses, probabilities = self.sess.run([self.losses, self.probabilities], feed_dict={self.inputs: batch})
#logging.info('Step %s: losses: %s probs: %s' % (i, losses, probabilities))
# curr_states = new_states
if autosave is not None and (autosave is not True
and i % autosave == 0):
self.save_to_file()
except KeyboardInterrupt:
logging.info('Cancelling training...')
logging.info('Saved summaries to %s' % summaries_dir)
if autosave is not None and (autosave is True
or i % autosave != 0):
# Save the last one only if it hasn't already been saved
self.save_to_file()
tools.exit(22)
else:
self.round_steps = 0
logging.info('Saved summaries to %s' % summaries_dir)
if autosave is not None:
# Always save the last one because round_steps has changed
self.save_to_file()
logging.info('Done training.')
if COMMAND == 'notebook-remove':
Notebooks().remove(**ARGS)
# Tags
if COMMAND == 'tag-list':
Tags().list(**ARGS)
if COMMAND == 'tag-create':
Tags().create(**ARGS)
if COMMAND == 'tag-edit':
Tags().edit(**ARGS)
if COMMAND == 'tag-remove':
Tags().remove(**ARGS)
except (KeyboardInterrupt, SystemExit, tools.ExitException):
pass
except Exception, e:
traceback.print_exc()
logging.error("App error: %s", str(e))
# exit preloader
tools.exit()
if __name__ == "__main__":
main()
