def login(self):
response = self.loadPage(self.url['base'], self.url['login'], "GET", {'oauth_token': self.tmpOAuthToken})
if response.status != 200:
logging.error("Unexpected response status on login 200 != %s", response.status)
tools.exit()
if not self.cookies.has_key('JSESSIONID'):
logging.error("Not found value JSESSIONID in the response cookies")
tools.exit()
# get login/password
self.username, self.password = out.GetUserCredentials()
self.postData['login']['username'] = self.username
self.postData['login']['password'] = self.password
self.postData['login']['targetUrl'] = self.url['oauth']%self.tmpOAuthToken
response = self.loadPage(self.url['base'], self.url['login']+";jsessionid="+self.cookies['JSESSIONID'], "POST",
self.postData['login'])
if not response.location and response.status == 200:
if self.incorrectLogin < 3:
out.preloader.stop()
out.printLine('Sorry, incorrect login or password')
out.preloader.setMessage('Authorize...')
self.incorrectLogin += 1
return self.login()
else:
logging.error("Incorrect login or password")
if not response.location:
logging.error("Target URL was not found in the response on login")
tools.exit()
logging.debug("Success authorize, redirect to access page")
def find(self, search=None, tags=None, notebooks=None,
date=None, exact_entry=None, content_search=None,
with_url=None, count=None, ignore_completed=None, reminders_only=None,):
request = self._createSearchRequest(search, tags, notebooks,
date, exact_entry,
content_search,
ignore_completed, reminders_only)
if not count:
count = 20
else:
count = int(count)
logging.debug("Search count: %s", count)
createFilter = True if search == "*" else False
result = self.getEvernote().findNotes(request, count, createFilter)
if result.totalNotes == 0:
out.failureMessage("Notes have not been found.")
return tools.exitErr()
# save search result
# print result
self.getStorage().setSearch(result)
out.SearchResult(result.notes, request, showUrl=with_url)
def updateNote(self, guid, title=None, content=None,
tags=None, notebook=None, resources=None):
note = Types.Note()
note.guid = guid
if title:
note.title = title
if content:
note.content = content
if tags:
note.tagNames = tags
if notebook:
note.notebookGuid = notebook
if resources:
""" TODO """
print("Updating a note's resources is not yet supported.")
raise NotImplementedError()
logging.debug("Update note : %s", note)
self.getNoteStore().updateNote(self.authToken, note)
return True
def __init__(self, sys_argv):
self.sys_argv = sys_argv
self.LVL = len(sys_argv)
self.INPUT = sys_argv
self.CMD_LIST = self.COMMANDS.keys()
self.CMD = None if self.LVL == 0 else self.INPUT[0]
self.CMD_ARGS = self.COMMANDS[self.CMD]['arguments'] if self.LVL > 0 and self.COMMANDS.has_key(self.CMD) and self.COMMANDS[self.CMD].has_key('arguments') else {}
self.CMD_FLAGS = self.COMMANDS[self.CMD]['flags'] if self.LVL > 0 and self.COMMANDS.has_key(self.CMD) and self.COMMANDS[self.CMD].has_key('flags') else {}
self.INP = [] if self.LVL <= 1 else self.INPUT[1:]
logging.debug("CMD_LIST : %s", str(self.CMD_LIST))
logging.debug("CMD: %s", str(self.CMD))
logging.debug("CMD_ARGS : %s", str(self.CMD_ARGS))
logging.debug("CMD_FLAGS : %s", str(self.CMD_FLAGS))
logging.debug("INP : %s", str(self.INP))
def edit(self):
"""
Call the system editor, that types as a default in the system.
Editing goes in markdown format, and then the markdown
converts into HTML, before uploading to Evernote.
"""
# Try to find default editor in the system.
storage = Storage()
editor = storage.getUserprop('editor')
if not editor:
editor = os.environ.get("editor")
if not editor:
editor = os.environ.get("EDITOR")
if not editor:
# If default editor is not finded, then use nano as a default.
if sys.platform == 'win32':
editor = config.DEF_WIN_EDITOR
else:
editor = config.DEF_UNIX_EDITOR
# Make a system call to open file for editing.
logging.debug("launch system editor: %s %s" % (editor, self.tempfile))
out.preloader.stop()
os.system(editor + " " + self.tempfile)
out.preloader.launch()
newContent = open(self.tempfile, 'r').read()
return newContent
def get_page_data(self):
"""
Calls API to get page data as dictionary, which includes page token.
"manage_pages" permissions must first be granted.
"""
url = FB_HOST + "/%s/accounts" % self.id
params = {'access_token': self.access_token,
'client_id': FB_APP_ID,
'client_secret': FB_APP_SECRET}
response = requests.get(url, params=params)
logging.debug("get_page_data response: %s" % response.text)
try:
response_dict = response.json()
except ValueError:
raise FacebookAPIError("Unexpected response. No JSON object could be decoded.")
if 'error' in response_dict:
raise FacebookAPIError("Error in response: %r" % response_dict['error'])
page_data = response_dict.get('data')
if not page_data:
logging.warn("No data object in response.")
return page_data
def simulate(self):
'''
This method is used to solve the resulting initial value problem
after the computation of a solution for the input trajectories.
'''
logging.debug("Solving Initial Value Problem")
# calulate simulation time
T = self.dyn_sys.b - self.dyn_sys.a
# get list of start values
start = []
if self.constraints is not None:
sys = self._dyn_sys_orig
else:
sys = self.dyn_sys
x_vars = sys.states
start_dict = dict([(k, v[0]) for k, v in sys.boundary_values.items() if k in x_vars])
ff = sys.f_num
for x in x_vars:
start.append(start_dict[x])
# create simulation object
S = Simulator(ff, T, start, self.eqs.trajectories.u)
logging.debug("start: %s"%str(start))
# start forward simulation
self.sim_data = S.simulate()
def get_access_code(self, redirect_uri=""):
"""
Calls API using long-term access token to get access code.
Takes and passes one parameter to API:
redirect_uri Defaults to empty string.
"""
url = FB_HOST + "/oauth/client_code"
params = {'access_token': self.long_term_access_token,
'client_id': FB_APP_ID,
'client_secret': FB_APP_SECRET,
'redirect_uri': redirect_uri}
response = requests.get(url, params=params)
logging.debug("get_access_code response: %s" % response.text)
try:
response_dict = response.json()
except ValueError:
raise FacebookAPIError("Unexpected response. No JSON object could be decoded.")
if 'error' in response_dict:
raise FacebookAPIError("Error in response: %r" % response_dict['error'])
access_code = self._get_json(response).get('code')
if not access_code:
logging.warn("No access_code in response.")
return access_code
def write_to_json_file():
"""
爬虫数据写入json文件,
而不是直接写入数据库
"""
logging.info("enter write_to_json_file...")
#获得最大页数，确定爬虫范围
last_page_number = get_last_web_page_number()
#打开文件
current_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir))+"/json"
yyyy = time.strftime("%Y")
mm = time.strftime("%m")
dd = time.strftime("%d")
folder = os.path.join(current_dir, yyyy, mm, dd)
if not os.path.exists(folder):
os.makedirs(folder)
os.chdir(folder)
logging.debug("max page is :%s"%last_page_number)
for i in range(1,last_page_number+1):
with open("news_%s.json"%i, "w+") as news:
#获取页面
page = get_web_page_by_url(URL.format(page_number=i))
#解析
for item_json_data in load_and_parse(page):
#写入文件
news.write(item_json_data+"\n")
logging.info("leaving write_to_json_file...")
def updateNote(self, guid, title=None, content=None, tags=None, notebook=None, attributes=None):
# allow us to pass in a note object instead
# due to the way evernote is imported via geeknote.lib
# the isinstance check won't work
if hasattr(guid, "title"):
note = guid
logging.debug("Update note : %s", note)
self.getNoteStore().updateNote(self.authToken, note)
return note
na = Types.NoteAttributes()
if attributes:
for k, v in attributes.items():
setattr(na, k, v)
note = Types.Note()
note.guid = guid
note.attributes = na
if title:
note.title = title
if content:
note.content = content
if tags:
note.tagNames = tags
if notebook:
note.notebookGuid = notebook
logging.debug("Update note : %s", note)
self.getNoteStore().updateNote(self.authToken, note)
return note
def find(self, search=None, tags=None, notebooks=None, date=None, exact_entry=None, content_search=None, with_url=None, count=None, ):
request = self._createSearchRequest(search, tags, notebooks, date, exact_entry, content_search)
if not count:
count = config.DEF_FINDLIMIT
else:
count = int(count)
#print "Search count: %s", count
logging.debug("Search count: %s", count)
createFilter = True if search == "*" else False
result = self.getEvernote().findNotes(request, count, createFilter)
totalNotes = result.totalNotes
notes = result.notes
while len(notes)<totalNotes:
r = self.getEvernote().findNotes(request, count, createFilter, fromindex = len(notes))
notes = notes + r.notes
if result.totalNotes == 0:
out.successMessage("Notes have not been found.")
# save search result
# print result
self.getStorage().setSearch(result)
out.SearchResult(notes, request, showUrl=with_url)
def createNote(self, title, content, tags=None, notebook=None, created=None, resources=None):
def make_resource(filename):
try:
mtype = mimetypes.guess_type(filename)[0]
if mtype.split('/')[0] == "text":
rmode = "r"
else:
rmode = "rb"
with open(filename, rmode) as f:
""" file exists """
resource = Types.Resource()
resource.data = Types.Data()
data = f.read()
md5 = hashlib.md5()
md5.update(data)
resource.data.bodyHash = md5.hexdigest()
resource.data.body = data
resource.data.size = len(data)
resource.mime = mtype
resource.attributes = Types.ResourceAttributes()
resource.attributes.fileName = os.path.basename(filename)
return resource
except IOError:
msg = "The file '%s' does not exist." % filename
out.failureMessage(msg)
raise IOError(msg)
note = Types.Note()
note.title = title
note.content = content
note.created = created
if tags:
note.tagNames = tags
if notebook:
note.notebookGuid = notebook
if resources:
""" make EverNote API resources """
note.resources = map(make_resource, resources)
""" add to content """
resource_nodes = ""
for resource in note.resources:
resource_nodes += '<en-media type="%s" hash="%s" />' % (resource.mime, resource.data.bodyHash)
note.content = note.content.replace("</en-note>", resource_nodes + "</en-note>")
logging.debug("New note : %s", note)
self.getNoteStore().createNote(self.authToken, note)
return True
def createTag(self, name):
tag = Types.Tag()
tag.name = name
logging.debug("New tag : %s", tag)
result = self.getNoteStore().createTag(self.authToken, tag)
return result
def createNotebook(self, name):
notebook = Types.Notebook()
notebook.name = name
logging.debug("New notebook : %s", notebook)
result = self.getNoteStore().createNotebook(self.authToken, notebook)
return result
def createNote(
self, title, content, tags=None, created=None, notebook=None, resources=None, reminder=None, url=None
):
note = Types.Note()
note.title = title
try:
note.content = content.encode("utf-8")
except UnicodeDecodeError:
note.content = content
if tags:
note.tagNames = tags
note.created = created
if notebook:
note.notebookGuid = notebook
if resources:
""" make EverNote API resources """
note.resources = map(make_resource, resources)
""" add to content """
resource_nodes = ""
for resource in note.resources:
resource_nodes += '<en-media type="%s" hash="%s" />' % (resource.mime, resource.data.bodyHash)
note.content = note.content.replace("</en-note>", resource_nodes + "</en-note>")
# Allow creating a completed reminder (for task tracking purposes),
# skip reminder creation steps if we have a DELETE
if reminder and reminder != config.REMINDER_DELETE:
if not note.attributes: # in case no attributes available
note.attributes = Types.NoteAttributes()
now = int(round(time.time() * 1000))
if reminder == config.REMINDER_NONE:
note.attributes.reminderOrder = now
elif reminder == config.REMINDER_DONE:
note.attributes.reminderOrder = now
note.attributes.reminderDoneTime = now
else: # we have an actual reminder time stamp
if reminder > now: # future reminder only
note.attributes.reminderOrder = now
note.attributes.reminderTime = reminder
else:
out.failureMessage("Error: reminder must be in the future.")
tools.exitErr()
if url:
if note.attributes is None:
note.attributes = Types.NoteAttributes()
note.attributes.sourceURL = url
logging.debug("New note : %s", note)
return self.getNoteStore().createNote(self.authToken, note)
def updateTag(self, guid, name):
tag = Types.Tag()
tag.name = name
tag.guid = guid
logging.debug("Update tag : %s", tag)
self.getNoteStore().updateTag(self.authToken, tag)
return True
def updateNotebook(self, guid, name):
notebook = Types.Notebook()
notebook.name = name
notebook.guid = guid
logging.debug("Update notebook : %s", notebook)
self.getNoteStore().updateNotebook(self.authToken, notebook)
return True
def find(
self,
search=None,
tag=None,
notebook=None,
date=None,
exact_entry=None,
content_search=None,
with_url=None,
with_tags=None,
with_notebook=None,
count=None,
ignore_completed=None,
reminders_only=None,
guid=None,
):
request = self._createSearchRequest(
search, tag, notebook, date, exact_entry, content_search, ignore_completed, reminders_only
)
if not count:
count = 20
else:
count = int(count)
logging.debug("Search count: %s", count)
createFilter = True if search == "*" else False
result = self.getEvernote().findNotes(request, count, createFilter)
if result.totalNotes == 0:
out.failureMessage("Notes have not been found.")
return tools.exitErr()
# save search result
# print result
self.getStorage().setSearch(result)
for note in result.notes:
self.getStorage().setNote(note)
if with_notebook:
noteStore = self.getEvernote().getNoteStore()
notebookNameFromGuid = dict()
for note in result.notes:
if note.notebookGuid not in notebookNameFromGuid:
notebookNameFromGuid[note.notebookGuid] = noteStore.getNotebook(
self.getEvernote().authToken, note.notebookGuid
).name
note.notebookName = notebookNameFromGuid[note.notebookGuid]
out.SearchResult(
result.notes, request, showUrl=with_url, showTags=with_tags, showNotebook=with_notebook, showGUID=guid
)
def _searchNotebook(self, notebook):
result = self.getEvernote().findNotebooks()
notebook = [item for item in result if item.name == notebook]
if notebook:
notebook = notebook[0]
else:
notebook = out.SelectSearchResult(result)
logging.debug("Selected notebook: %s" % str(notebook))
return notebook
def _searchTag(self, tag):
result = self.getEvernote().findTags()
tag = [item for item in result if item.name == tag]
if tag:
tag = tag[0]
else:
tag = out.SelectSearchResult(result)
logging.debug("Selected tag: %s" % str(tag))
return tag
def __init__(self, tcpserver=None):
logging.debug("")
if tcpserver is None:
self._ip = Config.O_TCPSERVER
self._port = Config.O_TCPPORT
self._settimeout = Config.O_TCPTIMEOUT
else:
ip, port, timeout = tcpserver
self._ip = ip
self._port = port
self._settimeout = timeout
self._buffersize = 1024
def get_permissions(self):
"""
Calls API using app access token and returns a list of dictionaries containing
user's permissions:
permission
status
"""
url = FB_HOST + "/%s/permissions" % self.id
params = {'access_token': self.app_access_token}
response = requests.get(url, params=params)
logging.debug("get_permissions response: %s" % response.text)
return self._get_json(response).get('data')
def _searchNotebook(self, notebook):
result = self.getEvernote().findNotebooks()
notebook = [item for item in result if item.name == notebook]
if notebook:
notebook = notebook[0]
else:
notebook = out.SelectSearchResult(result)
logging.debug("Selected notebook: %s" % str(notebook))
return notebook
def geoip(host):
logging.debug("")
# TEST IP IN GEOIP IN OS
process = subprocess.run(['geoiplookup', host], capture_output=True)
data = str(process.stdout)
if data.find("IP Address not found") >= 0:
return [True, str(host)]
elif data.find("Germany") >= 0:
return [True, "from Germany"]
return [False, "Not from Germany"]
def login(self):
response = self.loadPage(self.url['base'],
self.url['login'],
"GET",
{'oauth_token': self.tmpOAuthToken})
# parse hpts and hptsh from page content
hpts = re.search('.*\("hpts"\)\.value.*?"(.*?)"', response.data)
hptsh = re.search('.*\("hptsh"\)\.value.*?"(.*?)"', response.data)
if response.status != 200:
logging.error("Unexpected response status "
"on login 200 != %s", response.status)
tools.exitErr()
if 'JSESSIONID' not in self.cookies:
logging.error("Not found value JSESSIONID in the response cookies")
tools.exitErr()
# get login/password
self.username, self.password = out.GetUserCredentials()
self.postData['login']['username'] = self.username
self.postData['login']['password'] = self.password
self.postData['login']['targetUrl'] = self.url['oauth'] % self.tmpOAuthToken
self.postData['login']['hpts'] = hpts and hpts.group(1) or ""
self.postData['login']['hptsh'] = hptsh and hptsh.group(1) or ""
response = self.loadPage(self.url['base'],
self.url['login'] + ";jsessionid=" + self.cookies['JSESSIONID'],
"POST",
self.postData['login'])
if not response.location and response.status == 200:
if self.incorrectLogin < 3:
out.preloader.stop()
out.printLine('Sorry, incorrect login or password')
out.preloader.setMessage('Authorize...')
self.incorrectLogin += 1
return self.login()
else:
logging.error("Incorrect login or password")
if not response.location:
logging.error("Target URL was not found in the response on login")
tools.exitErr()
if response.status == 302:
# the user has enabled two factor auth
return self.handleTwoFactor()
logging.debug("Success authorize, redirect to access page")
def login(self):
response = self.loadPage(self.url['base'],
self.url['login'],
"GET",
{'oauth_token': self.tmpOAuthToken})
# parse hpts and hptsh from page content
hpts = re.search('.*\("hpts"\)\.value.*?"(.*?)"', response.data)
hptsh = re.search('.*\("hptsh"\)\.value.*?"(.*?)"', response.data)
if response.status != 200:
logging.error("Unexpected response status "
"on login 200 != %s", response.status)
tools.exitErr()
if 'JSESSIONID' not in self.cookies:
logging.error("Not found value JSESSIONID in the response cookies")
tools.exitErr()
# get login/password
self.username, self.password = out.GetUserCredentials()
self.postData['login']['username'] = self.username
self.postData['login']['password'] = self.password
self.postData['login']['targetUrl'] = self.url['oauth'] % self.tmpOAuthToken
self.postData['login']['hpts'] = hpts and hpts.group(1) or ""
self.postData['login']['hptsh'] = hptsh and hptsh.group(1) or ""
response = self.loadPage(self.url['base'],
self.url['login'] + ";jsessionid=" + self.cookies['JSESSIONID'],
"POST",
self.postData['login'])
if not response.location and response.status == 200:
if self.incorrectLogin < 3:
out.preloader.stop()
out.printLine('Sorry, incorrect login or password')
out.preloader.setMessage('Authorize...')
self.incorrectLogin += 1
return self.login()
else:
logging.error("Incorrect login or password")
if not response.location:
logging.error("Target URL was not found in the response on login")
tools.exitErr()
# @todo response.status is always 302, regardless of whether TwoFactor is enabled
# if response.status == 302:
# # the user has enabled two factor auth
# return self.handleTwoFactor()
logging.debug("Success authorize, redirect to access page")
def loadPage(self, url, uri=None, method="GET", params=""):
if not url:
logging.error("Request URL undefined")
tools.exitErr()
if not uri:
urlData = urlparse(url)
url = urlData.netloc
uri = urlData.path + '?' + urlData.query
# prepare params, append to uri
if params:
params = urlencode(params)
if method == "GET":
uri += ('?' if uri.find('?') == -1 else '&') + params
params = ""
# insert local cookies in request
headers = {
"Cookie":
'; '.join(
[key + '=' + self.cookies[key] for key in self.cookies.keys()])
}
if method == "POST":
headers["Content-type"] = "application/x-www-form-urlencoded"
logging.debug("Request URL: %s:/%s > %s # %s", url, uri,
unquote(params), headers["Cookie"])
conn = httplib.HTTPSConnection(url)
conn.request(method, uri, params, headers)
response = conn.getresponse()
data = response.read()
conn.close()
logging.debug("Response : %s > %s", response.status,
response.getheaders())
result = tools.Struct(status=response.status,
location=response.getheader('location', None),
data=data)
# update local cookies
sk = Cookie.SimpleCookie(response.getheader("Set-Cookie", ""))
for key in sk:
self.cookies[key] = sk[key].value
# delete cookies whose content is "deleteme"
for key in self.cookies.keys():
if self.cookies[key] == "deleteme":
del self.cookies[key]
return result
def _init_thread(self):
logging.debug ("")
server_address = (self._listen_addr, self._listen_port)
httpd = HTTPServer(server_address, Webserver)
httpd.socket = ssl.wrap_socket (httpd.socket, certfile=self._server_cert ,keyfile=self._server_key, server_side=True)
logging.info ("DOH input start from :"+str( self._listen_addr)+":"+str( self._listen_port))
try:
httpd.serve_forever()
except KeyboardInterrupt:
pass
httpd.server_close()
logging.error('Stopping httpd...\n')
def loadPage(self, url, uri=None, method="GET", params=""):
if not url:
logging.error("Request URL undefined")
tools.exitErr()
if not uri:
urlData = urlparse(url)
url = urlData.netloc
uri = urlData.path + '?' + urlData.query
# prepare params, append to uri
if params:
params = urlencode(params)
if method == "GET":
uri += ('?' if uri.find('?') == -1 else '&') + params
params = ""
# insert local cookies in request
headers = {
"Cookie": '; '.join([key + '=' + self.cookies[key] for key in self.cookies.keys()])
}
if method == "POST":
headers["Content-type"] = "application/x-www-form-urlencoded"
logging.debug("Request URL: %s:/%s > %s # %s", url,
uri, unquote(params), headers["Cookie"])
conn = httplib.HTTPSConnection(url)
conn.request(method, uri, params, headers)
response = conn.getresponse()
data = response.read()
conn.close()
logging.debug("Response : %s > %s",
response.status,
response.getheaders())
result = tools.Struct(status=response.status,
location=response.getheader('location', None),
data=data)
# update local cookies
sk = Cookie.SimpleCookie(response.getheader("Set-Cookie", ""))
for key in sk:
self.cookies[key] = sk[key].value
# delete cookies whose content is "deleteme"
for key in self.cookies.keys():
if self.cookies[key] == "deleteme":
del self.cookies[key]
return result
def process_queue():
""""simulate to perform some work (for testing)"""
while True:
if msgqueue.empty():
logging.debug("empty queue")
else:
msg = msgqueue.get()
msgqueue.task_done()
logging.info("tcp-msg: %s" % str(msg))
if "exit" in msg:
break
time.sleep(1)
logging.info("finished")
def _searchNote(self, note):
note = tools.strip(note)
# load search result
result = self.getStorage().getSearch()
if result and tools.checkIsInt(note) and 1 <= int(note) <= len(result.notes):
note = result.notes[int(note) - 1]
else:
request = self._createSearchRequest(search=note)
logging.debug("Search notes: %s" % request)
result = self.getEvernote().findNotes(request, 20)
logging.debug("Search notes result: %s" % str(result))
if result.totalNotes == 0:
out.failureMessage("Notes have not been found.")
return tools.exitErr()
elif result.totalNotes == 1 or self.selectFirstOnUpdate:
note = result.notes[0]
else:
logging.debug("Choose notes: %s" % str(result.notes))
note = out.SelectSearchResult(result.notes)
logging.debug("Selected note: %s" % str(note))
return note
def doPingTestFor(lst_id: int, server_lst: list, dbname: str):
try:
begin_time = time.time()
name, _, proto, addr, family = config.serverList[lst_id]
logging.debug('Started ping test for ID#%d, name: %s' % (lst_id, name))
min_time, avg_time, max_time, std_dev = pingtest(
config.ping_interval, config.ping_batchnum, config.ping_timeout,
proto, addr, family)
db = database(dbname)
db.insert_record(name, begin_time, min_time, avg_time, max_time,
std_dev)
logging.debug('Ping test ended ID#%d, name: %s' % (lst_id, name))
except BaseException as e:
logging.error(e)
def do_GET(self):
global g_countererror
host, port = self.client_address
ok,text= g_dot_geoip(host)
logging.info("IP : "+str(host)+":"+str(port)+" :"+ text)
if ok == True:
logging.debug("GET request,\nPath: %s\nHeaders:\n%s\n", str(self.path), str(self.headers))
self._set_response(404)
logging.info("IP : "+str(host)+": "+ "404")
g_countererror=g_countererror +1
else:
self._set_response(403)
logging.info("IP : "+str(host)+": "+ text)
g_countererror=g_countererror +1
def set_coeffs(self, sol):
'''
Set found numerical values for the independent parameters of each spline.
This method is used to get the actual splines by using the numerical
solutions to set up the coefficients of the polynomial spline parts of
every created spline.
Parameters
----------
sol : numpy.ndarray
The solution vector for the free parameters, i.e. the independent coefficients.
'''
# TODO: look for bugs here!
logging.debug("Set spline coefficients")
sol_bak = sol.copy()
subs = dict()
for k, v in sorted(self.indep_coeffs.items(), key=lambda (k, v): k):
i = len(v)
subs[k] = sol[:i]
sol = sol[i:]
if self._parameters['use_chains']:
for var in self.sys.states + self.sys.inputs:
for ic in self._chains:
if var in ic:
subs[var] = subs[ic.upper]
# set numerical coefficients for each spline and derivative
for k in self.splines.keys():
self.splines[k].set_coefficients(free_coeffs=subs[k])
# yet another dictionary for solution and coeffs
coeffs_sol = dict()
# used for indexing
i = 0
j = 0
for k, v in sorted(self.indep_coeffs.items(), key=lambda (k, v): k):
j += len(v)
coeffs_sol[k] = sol_bak[i:j]
i = j
self.coeffs_sol = coeffs_sol
def createNote(self, title, content, tags=None, created=None, notebook=None, resources=None, reminder=None):
note = Types.Note()
note.title = title
try:
note.content = content.encode('utf-8')
except UnicodeDecodeError:
note.content = content
if tags:
note.tagNames = tags
note.created = created
if notebook:
note.notebookGuid = notebook
if resources:
""" make EverNote API resources """
note.resources = map(make_resource, resources)
""" add to content """
resource_nodes = ""
for resource in note.resources:
resource_nodes += '<en-media type="%s" hash="%s" />' % (resource.mime, resource.data.bodyHash)
note.content = note.content.replace("</en-note>", resource_nodes + "</en-note>")
# Allow creating a completed reminder (for task tracking purposes),
# skip reminder creation steps if we have a DELETE
if reminder and reminder != config.REMINDER_DELETE:
now = int(round(time.time() * 1000))
note.attributes = Types.NoteAttributes()
if reminder == config.REMINDER_NONE:
note.attributes.reminderOrder = now
elif reminder == config.REMINDER_DONE:
note.attributes.reminderOrder = now
note.attributes.reminderDoneTime = now
else: # we have an actual reminder time stamp
if reminder > now: # future reminder only
note.attributes.reminderOrder = now
note.attributes.reminderTime = reminder
else:
out.failureMessage("Error: reminder must be in the future.")
tools.exitErr()
logging.debug("New note : %s", note)
return self.getNoteStore().createNote(self.authToken, note)
def set_coeffs(self, sol):
'''
Set found numerical values for the independent parameters of each spline.
This method is used to get the actual splines by using the numerical
solutions to set up the coefficients of the polynomial spline parts of
every created spline.
Parameters
----------
sol : numpy.ndarray
The solution vector for the free parameters, i.e. the independent coefficients.
'''
# TODO: look for bugs here!
logging.debug("Set spline coefficients")
sol_bak = sol.copy()
subs = dict()
for k, v in sorted(self.indep_coeffs.items(), key=lambda (k, v): k):
i = len(v)
subs[k] = sol[:i]
sol = sol[i:]
if self._parameters['use_chains']:
for var in self.sys.states + self.sys.inputs:
for ic in self._chains:
if var in ic:
subs[var] = subs[ic.upper]
# set numerical coefficients for each spline and derivative
for k in self.splines.keys():
self.splines[k].set_coefficients(free_coeffs=subs[k])
# yet another dictionary for solution and coeffs
coeffs_sol = dict()
# used for indexing
i = 0
j = 0
for k, v in sorted(self.indep_coeffs.items(), key=lambda (k, v): k):
j += len(v)
coeffs_sol[k] = sol_bak[i:j]
i = j
self.coeffs_sol = coeffs_sol
def createNote(self, title, content, tags=None, notebook=None):
note = Types.Note()
note.title = title
note.content = content
if tags:
note.tagNames = tags
if notebook:
note.notebookGuid = notebook
logging.debug("New note : %s", note)
self.getNoteStore().createNote(self.authToken, note)
return True
def getOAuthToken(self):
response = self.loadPage(self.url['base'], self.url['token'], "GET",
self.getTokenRequestData(oauth_token=self.tmpOAuthToken, oauth_verifier=self.verifierToken))
if response.status != 200:
logging.error("Unexpected response status on getting oauth token 200 != %s", response.status)
tools.exit()
responseData = self.parseResponse(response.data)
if not responseData.has_key('oauth_token'):
logging.error("OAuth token not found")
tools.exit()
logging.debug("OAuth token take : %s", responseData['oauth_token'])
self.OAuthToken = responseData['oauth_token']
def createNote(self, title, content, tags=None, notebook=None):
note = Types.Note()
note.title = title
note.content = content
if tags:
note.tagNames = tags
if notebook:
note.notebookGuid = notebook
logging.debug("New note : %s", note)
self.getNoteStore().createNote(self.authToken, note)
return True
def solve(self):
'''
This is just a wrapper to call the chosen algorithm for solving the
collocation equation system.
'''
if (self.method == 'leven'):
logging.debug("Run Levenberg-Marquardt method")
self.leven()
if (self.sol is None):
logging.warning("Wrong solver, returning initial value.")
return self.x0
else:
return self.sol
def solve(self):
'''
This is just a wrapper to call the chosen algorithm for solving the
collocation equation system.
'''
if (self.method == 'leven'):
logging.debug("Run Levenberg-Marquardt method")
self.leven()
if (self.sol is None):
logging.warning("Wrong solver, returning initial value.")
return self.x0
else:
return self.sol
def run(self):
while True:
if self.RepoLock.acquire():
if len(self.AllRepos) == 0:
self.RepoLock.release()
break
oneRepo = self.AllRepos[0]
del self.AllRepos[0]
self.RepoLock.release()
logging.debug("========================thread[%s] begin to get another Repo file;left Repo num[%d]=====================" %(self.name, len(self.AllRepos)))
self.FileListParser(oneRepo['path'], oneRepo['owner'])
logging.debug("*****thread[%s] exit" %(self.name))
def getOAuthToken(self):
response = self.loadPage(self.url['base'], self.url['token'], "GET",
self.getTokenRequestData(oauth_token=self.tmpOAuthToken, oauth_verifier=self.verifierToken))
if response.status != 200:
logging.error("Unexpected response status on getting oauth token 200 != %s", response.status)
tools.exit()
responseData = self.parseResponse(response.data)
if not responseData.has_key('oauth_token'):
logging.error("OAuth token not found")
tools.exit()
logging.debug("OAuth token take : %s", responseData['oauth_token'])
self.OAuthToken = responseData['oauth_token']
def getTmpOAuthToken(self):
response = self.loadPage(self.url['base'], self.url['token'], "GET",
self.getTokenRequestData(oauth_callback="https://"+self.url['base']))
if response.status != 200:
logging.error("Unexpected response status on get temporary oauth_token 200 != %s", response.status)
tools.exit()
responseData = self.parseResponse(response.data)
if not responseData.has_key('oauth_token'):
logging.error("OAuth temporary not found")
tools.exit()
self.tmpOAuthToken = responseData['oauth_token']
logging.debug("Temporary OAuth token : %s", self.tmpOAuthToken)
def getTmpOAuthToken(self):
response = self.loadPage(self.url['base'], self.url['token'], "GET",
self.getTokenRequestData(oauth_callback="https://"+self.url['base']))
if response.status != 200:
logging.error("Unexpected response status on get temporary oauth_token 200 != %s", response.status)
tools.exit()
responseData = self.parseResponse(response.data)
if not responseData.has_key('oauth_token'):
logging.error("OAuth temporary not found")
tools.exit()
self.tmpOAuthToken = responseData['oauth_token']
logging.debug("Temporary OAuth token : %s", self.tmpOAuthToken)
def edit(content=None):
"""
Call the system editor, that types as a default in the system.
Editing goes in markdown format, and then the markdown converts into HTML, before uploading to Evernote.
"""
if content is None:
content = ""
if not isinstance(content, str):
raise Exception(
"Note content must be an instanse of string, '%s' given." %
type(content))
storage = Storage()
userSuffix = storage.getUserprop('suffix')
if not userSuffix:
userSuffix = '.md'
(tmpFileHandler, tmpFileName) = tempfile.mkstemp(suffix=userSuffix)
os.write(tmpFileHandler, ENMLtoText(content))
os.close(tmpFileHandler)
# Try to find default editor in the system.
editor = storage.getUserprop('editor')
if not editor:
# If default editor is not finded, then use nano as a default.
if sys.platform == 'win32':
editor = config.DEF_WIN_EDITOR
else:
editor = config.DEF_UNIX_EDITOR
if not editor:
editor = os.environ.get("editor")
if not editor:
editor = os.environ.get("EDITOR")
# Make a system call to open file for editing.
logging.debug("launch system editor: %s %s" % (editor, tmpFileName))
out.preloader.stop()
os.system(editor + " " + tmpFileName)
out.preloader.launch()
newContent = open(tmpFileName, 'r').read()
return newContent
def find(
self,
search=None,
tags=None,
notebooks=None,
date=None,
exact_entry=None,
content_search=None,
with_url=None,
count=None,
):
request = self._createSearchRequest(search, tags, notebooks, date,
exact_entry, content_search)
if not count:
count = 20
else:
count = int(count)
logging.debug("Search count: %s", count)
createFilter = True if search == "*" else False
result = self.getEvernote().findNotes(request, count, createFilter)
# Reduces the count by the amount of notes already retrieved
update_count = lambda c: max(c - len(result.notes), 0)
count = update_count(count)
# Evernote api will only return so many notes in one go. Checks for more
# notes to come whilst obeying count rules
while ((result.totalNotes != len(result.notes)) and count != 0):
offset = len(result.notes)
result.notes += self.getEvernote().findNotes(
request, count, createFilter, offset).notes
count = update_count(count)
if result.totalNotes == 0:
out.failureMessage("Notes have not been found.")
return tools.exitErr()
# save search result
# print result
self.getStorage().setSearch(result)
out.SearchResult(result.notes, request, showUrl=with_url)
def switch(txdata):
logging.debug("")
# NO OUTPUT SET RETURN NONE
if output is None:
return None
# TEST DOMIN IN FILTER
if dropdns is None:
return None
isblock, rxblock, blockname, dname = dropdns.drop(txdata)
if isblock == True:
rxdata = rxblock
else:
rxdata = output.send(txdata)
return [isblock, rxdata, blockname, dname]
def run(self):
while True:
if self.RepoLock.acquire():
if len(self.AllRepos) == 0:
self.RepoLock.release()
break
oneRepo = self.AllRepos[0]
del self.AllRepos[0]
self.RepoLock.release()
logging.debug(
"========================thread[%s] begin to get another Repo file;left Repo num[%d]====================="
% (self.name, len(self.AllRepos)))
self.FileListParser(oneRepo['path'], oneRepo['owner'])
logging.debug("*****thread[%s] exit" % (self.name))
def login(self):
response = self.loadPage(self.url['base'],
self.url['login'],
"GET",
{'oauth_token': self.tmpOAuthToken})
if response.status != 200:
logging.error("Unexpected response status "
"on login 200 != %s", response.status)
tools.exitErr()
if 'JSESSIONID' not in self.cookies:
logging.error("Not found value JSESSIONID in the response cookies")
tools.exitErr()
# get login/password
self.username, self.password = out.GetUserCredentials()
self.postData['login']['username'] = self.username
self.postData['login']['password'] = self.password
self.postData['login']['targetUrl'] = self.url['oauth'] % self.tmpOAuthToken
response = self.loadPage(self.url['base'],
self.url['login'] + ";jsessionid=" + self.cookies['JSESSIONID'],
"POST",
self.postData['login'])
if not response.location and response.status == 200:
if self.incorrectLogin < 3:
out.preloader.stop()
out.printLine('Sorry, incorrect login or password')
out.preloader.setMessage('Authorize...')
self.incorrectLogin += 1
return self.login()
else:
logging.error("Incorrect login or password")
if not response.location:
logging.error("Target URL was not found in the response on login")
tools.exitErr()
if response.status == 302:
# the user has enabled two factor auth
return self.handleTwoFactor()
logging.debug("Success authorize, redirect to access page")
def find(self,
search=None,
tag=None,
notebooks=None,
date=None,
exact_entry=None,
content_search=None,
with_url=None,
with_tags=None,
with_notebook=None,
count=None,
ignore_completed=None,
reminders_only=None,
guid=None):
request = self._createSearchRequest(search, tag, notebooks, date,
exact_entry, content_search,
ignore_completed, reminders_only)
if not count:
count = 20
else:
count = int(count)
logging.debug("Search count: %s", count)
createFilter = True if search == "*" else False
result = self.getEvernote().findNotes(request, count, createFilter)
if result.totalNotes == 0:
out.failureMessage("Notes have not been found.")
return tools.exitErr()
# save search result
# print result
self.getStorage().setSearch(result)
for note in result.notes:
self.getStorage().setNote(note)
out.SearchResult(result.notes,
request,
showUrl=with_url,
showTags=with_tags,
showNotebook=with_notebook,
showGUID=guid)
def get_page_permissions(self):
"""
Calls API using user access token to retrieve pages user admins. Data is returned
as a list of dictionaries, which includes:
category
name
access_token
id
perms a list of admin roles (e.g. "ADMINISTER", "EDIT_PROFILE", etc.)
This call requires that test user be granted 'manage_pages' permissions.
Returns an empty list if user has no page permissions.
"""
url = FB_HOST + "/%s/accounts" % self.id
params = {'access_token': self.access_token}
response = requests.get(url, params=params)
logging.debug("get_page_data response: %s" % response.text)
return self._get_json(response).get('data')
def allowAccess(self):
self.postData['access']['oauth_token'] = self.tmpOAuthToken
self.postData['access']['oauth_callback'] = "https://"+self.url['base']
response = self.loadPage(self.url['base'], self.url['access'], "POST", self.postData['access'])
if response.status != 302:
logging.error("Unexpected response status on allowing access 302 != %s", response.status)
tools.exit()
responseData = self.parseResponse(response.location)
if not responseData.has_key('oauth_verifier'):
logging.error("OAuth verifier not found")
tools.exit()
self.verifierToken = responseData['oauth_verifier']
logging.debug("OAuth verifier token take")
def __init__(self,switch,geoip,udpserver=None):
logging.debug ("")
self._switch =switch
self._geoip=geoip
if udpserver is None:
self._listen_addr = Config.I_DOTSERVER
self._listen_port = Config.I_UDPPORT
else:
server,port =udpserver
self._listen_addr = server
self._listen_port = port
self._buffersize =1024
self._conterrequests=0
self._conterrequest=0
self._countererror =0
def _parseInput(self,
title=None,
content=None,
tags=None,
notebook=None,
resources=None,
note=None):
result = {
"title": title,
"content": content,
"tags": tags,
"notebook": notebook,
"resources": resources if resources else []
}
result = tools.strip(result)
# if get note without params
if note and title is None and content is None and tags is None and notebook is None:
content = config.EDITOR_OPEN
if title is None and note:
result['title'] = note.title
if content:
if content == config.EDITOR_OPEN:
logging.debug("launch system editor")
if note:
self.getEvernote().loadNoteContent(note)
content = editor.edit(note.content)
else:
content = editor.edit()
elif isinstance(content, str) and os.path.isfile(content):
logging.debug("Load content from the file")
content = open(content, "r").read()
logging.debug("Convert content")
content = editor.textToENML(content)
result['content'] = content
if tags:
result['tags'] = tools.strip(tags.split(','))
if notebook:
notepadGuid = Notebooks().getNoteGUID(notebook)
if notepadGuid is None:
newNotepad = Notebooks().create(notebook)
notepadGuid = newNotepad.guid
result['notebook'] = notepadGuid
logging.debug("Search notebook")
return result
def _decoder_thread(self,conn, addr):
logging.debug ("")
try:
conn.settimeout(self._timeout)
host, port = addr
if self._geoip is None:
ok= True
text= "NO GEOIP"
else:
ok,text = self._geoip(host)
if ok == True:
self._conterrequests=self._conterrequests+1
self._conterrequest=self._conterrequest+1
txdata = conn.recv(self._buffersize)
# clear the first 2 byte for datalen
txdata =txdata[2:]
data =self._switch(txdata)
if data is not None:
isblock,rxdata,blockname,dname = data
# Add 2 byte for len for tcp protokll
rxdata= (len(rxdata)).to_bytes(2, byteorder="big") + rxdata
conn.send(rxdata)
logging.info("IP : "+str(host)+":"+str(port)+" :"+ text)
if isblock==True:
logging.info("Domain : "+str(dname)+ " is block true : blockname " + str(blockname))
else:
logging.info("Domain : "+str(dname)+ " is block false ")
else:
logging.info("IP : "+str(host)+": "+ text)
conn.close()
except OSError as err:
logging.error("OS error: {0}".format(err))
self._countererror=self._countererror +1
self._conterrequest=self._conterrequest-1
def matlab_engine(fun_file, input_list, eng=None):
'''matlab enginer wrapper
return_val = fun(input_list)
'''
import matlab.engine
script_dirname = os.path.abspath(os.path.dirname(fun_file))
fun_name = stem(fun_file)
if eng is None:
eng = matlab.engine.start_matlab('-nodisplay -nojvm -nosplash -nodesktop')
else:
logging.debug("Use opened Matlab session: {}".format(eng))
try:
eng.cd(script_dirname)
func = getattr(eng, fun_name)
result = func(input_list)
except matlab.engine.EngineError, e:
# Use new engine instead
logging.error("Existing engine no response: {}".format(eng))
raise e
def updateNote(self, guid, title=None, content=None, tags=None, notebook=None):
note = Types.Note()
note.guid = guid
if title:
note.title = title
if content:
note.content = content
if tags:
note.tagNames = tags
if notebook:
note.notebookGuid = notebook
logging.debug("Update note : %s", note)
self.getNoteStore().updateNote(self.authToken, note)
return True
def DownloadFunc(self, fileInfo):
filePath = 'fileDir/' + fileInfo['owner']
if os.path.exists(filePath + '/' + fileInfo['fileName']):
logging.debug("file[%s] has been downloaded" %
fileInfo['fileName'])
return
try:
fileData = self.session.get(fileInfo['downloadPath'])
except Exception as e:
logging.error(e)
if self.FileLock.acquire() == True:
self.AllFiles.append(fileInfo)
self.FileLock.release()
return
with open(filePath + '/' + fileInfo['fileName'], "wb") as newfile:
newfile.write(fileData.content)
newfile.close()
logging.debug('===download [%s] end====' % (fileInfo['fileName']))
