def dump_to_json(self, zip_fname):
"""
Dump the contents of the database into a JSON file with the specific format
of DatabasePolygon. Creates a directory '{PROJECT}/tmp' and exports all db
objects to it, then zips the directory and deletes tmp. Returns ``True`` on
success, ``False`` otherwise
:param str zip_fname: name of the zip file
:rtype: bool
"""
session = self.__Session()
# tmp should not previously exist because we don't want files we didn't
# add ourselves
print(PATH)
if os.path.exists(PATH + '/../tmp'):
logger.error('Tmp directory should not exist, will not zip')
return False
logger.info('Creating /tmp and exporting shapes')
os.makedirs(PATH + '/../tmp')
for shape in session.query(DatabasePolygon).order_by(
DatabasePolygon.tag):
self.encode(PATH + '/../tmp/' + shape.tag + '.json', str(shape))
logger.info('Packing /tmp into %s' % zip_fname)
zipf = zipfile.ZipFile(zip_fname, 'w')
zipdir(PATH + '/../tmp', zipf)
zipf.close()
shutil.rmtree(PATH + '/../tmp')
session.close()
return True
def handle_douban_id(redis_con, douban_id):
try:
logger.info('id %s' % douban_id)
url = detail_url % douban_id
# url = '/data/project/self/bookMenu/detail.html'
response_text = get_page_content(url)
# response_text = open('/data/project/self/bookMenu/detail.html', 'r').read()
soup = get_soup(response_text)
info = BookDetailInfo()
info.name = soup.find(attrs={
'property': 'v:itemreviewed'
}).text.strip()
info.author_name = fetch_author_name(soup)
info.press_name = (lambda x: x.next_sibling.strip()
if x is not None else None)(soup.find(
'span', string='出版社:'))
info.origin_book_name = (lambda x: x.next_sibling.strip()
if x is not None else None)(soup.find(
'span', string='原作名:'))
info.translator_name = (
lambda x: x.next_sibling.next_sibling.text.strip()
if x is not None else None)(soup.find('span', string=' 译者'))
info.page_count = (lambda x: x.next_sibling.strip()
if x is not None else None)(soup.find('span',
string='页数:'))
info.price = (lambda x: x.next_sibling.strip()
if x is not None else None)(soup.find('span',
string='定价:'))
tag_a_2 = soup.find(
'a', attrs={'href': re.compile("https://book.douban.com/series/")})
info.series_id = (lambda x: x['href'].split('/')[-1]
if x is not None else None)(tag_a_2)
info.series_name = (lambda x: x.text.strip()
if x is not None else None)(tag_a_2)
info.isbn = (lambda x: x.next_sibling.strip()
if x is not None else None)(soup.find('span',
string='ISBN:'))
info.publish_year = (lambda x: x.next_sibling.strip()
if x is not None else None)(soup.find(
'span', string='出版年:'))
info.score = (lambda x: Decimal(x.text.strip())
if x is not None else None)(
soup.find(attrs={'property': 'v:average'}))
info.menu = (lambda x: x.text.strip() if x is not None else None)(
soup.find(attrs={'id': 'dir_' + douban_id + '_full'}))
info.tags = '|'.join(
map(lambda x: x.text.strip(),
soup.find_all('a', attrs={'href': re.compile("/tag/")})))
info.douban_id = douban_id
logger.info(
json.dumps(info,
sort_keys=True,
indent=2,
cls=AlchemyEncoder,
encoding='utf-8',
ensure_ascii=False))
BookDetailInfo.add(info)
except Exception as e:
logger.error('--->%s', e)
redis_con.lpush(QUEUE_DOUBAN_ID_EXCEPTION, douban_id)
def persist_compare_result(moc_name, db_not_in_configure_para,
configure_not_in_db_para):
if is_record_exist(moc_name):
cmd = 'RMV MOCEXTRAINFO:NAME=%s;' % moc_name
cmd_obj = get_cmd_obj(cmd)
if cmd_obj is not None:
ret, info = cmd_obj.execute()
logger.info(
'persist_compare_result result is %s, extra info is %s...' %
(ret, info))
info = ""
info += compare_result("This_para_is_in_db_but_not_in_xml:",
db_not_in_configure_para)
info += compare_result("This_para_is_in_xml_but_not_in_DB:",
configure_not_in_db_para)
if len(info) == 0:
logger.info("moc_name %s not need to update moc extra info." %
moc_name)
return
cmd = 'ADD MOCEXTRAINFO:NAME=%s, INFO=%s;' % (moc_name.upper(),
info.upper())
cmd_obj = get_cmd_obj(cmd)
if cmd_obj is None:
logger.error("cmd[%s] get None obj." % cmd)
return
ret, info = cmd_obj.execute()
logger.info('persist_compare_result result is %s, extra info is %s...' %
(ret, info))
def render(self):
"""
Errors checks all user entered parameters and calls ``set_plot`` from *Calipso*
"""
'''
if self.plot_type.get() == 0:
logger.error('No plot type set')
tkMessageBox.showerror('toolswindow', 'No plot type specified')
return
'''
if not self.__parent.get_file():
logger.error('No file entered')
tkMessageBox.showerror('toolswindow', 'No file loaded')
return
time_range = ToolsWindow.__check_range(0, 1000, 5,
self.begin_range_entry,
self.end_range_entry)
alt_range = ToolsWindow.__check_range(0, 20, 5,
self.begin_alt_range_entry,
self.end_alt_range_entry)
logger.info('Grabbing range and safe checking')
if time_range is None or alt_range is None:
return
logger.info('Calling plot')
self.__parent.set_plot(self.__parent.plot_type.get(),
xrange_=time_range,
yrange=alt_range)
def plot_point(self, event):
"""
Plot a single point to the screen for the current shape object,
if other points exist, a line is drawn between then until a
polygon is formed
:param event: A ``matplotlib.backend_bases.MouseEvent`` passed object
"""
if self.__current_plot == Plot.baseplot:
logger.warning('Cannot draw to the base plot')
return
if event.xdata and event.ydata:
logger.info('Plotting point at %.5f, %.5f' %
(event.xdata, event.ydata))
check = self.__current_list[-1].plot_point(
event, self.__current_plot, self.__hdf, self.__figure,
ShapeManager.outline_toggle)
self.__drawing = True
if check:
self.__current_list[-1].set_tag(self.generate_tag())
self.__current_list.append(Shape(self.__canvas))
self.__drawing = False
self.__canvas.show()
else:
logger.error("Point to plot is out or range, skipping")
def save_as_json(self, save_all=False):
"""
Save all selected objects on the plot, asking for a filename first
if ``save_all`` is specified and set to ``True``, the function will save **all**
shapes across **all** plots in the program.
"""
logger.info('Notifying JSON to save as')
# Save to a file entered by user, saveAll saves ALL objects across canvas
# and cannot be called as a normal save(must always be save as)
if self.__shapemanager.get_selected_count() > 0:
options = dict()
options['defaultextension'] = '.json'
options['filetypes'] = [('CALIPSO Data files', '*.json'),
('All files', '*')]
f = tkFileDialog.asksaveasfilename(**options)
if f == '':
logger.info("cancelling save as json")
return False
if save_all:
self.__shapemanager.save_all_json(f)
else:
self.__shapemanager.save_json(f)
else:
logger.error('No selected objects found, canceling save')
tkMessageBox.showerror('save as JSON', 'No objects to be saved')
def check_lst_para(cls, para_2_val, key_para_lst):
for para in para_2_val.keys():
if para not in key_para_lst:
logger.error("para %s is not in key_para_lst %s." %
(para, key_para_lst))
return False
return True
def export_json_db():
"""
Export the contents of the database to an archive containing JSON, which can then be
loaded into other databases and have all shapes imported
"""
if tkMessageBox.askyesno(
'Export database', 'Database will be exported to a specified' +
' archive (this operation is a copy, not a move)' +
' continue?'):
options = dict()
options['defaultextension'] = '.zip'
options['filetypes'] = [('ZIP Files', '*.zip'), ('All files', '*')]
options['initialdir'] = CONF.session_db.dir()
fl = tkFileDialog.asksaveasfilename(**options)
if fl != '':
log_fname = fl.rpartition('/')[2]
logger.info('Dumping database to \'%s\'' % log_fname)
success = db.dump_to_json(fl)
if success:
logger.info('Success, JSON file created')
tkMessageBox.showinfo(
'database', 'Database exported to \'%s\'' % log_fname)
else:
logger.error('No objects to be saved')
tkMessageBox.showerror(
'database',
'No objects inside database to export to JSON')
else:
logger.info('Export to database canceled')
def import_json_db():
"""
Import the contents of a JSON file to the database, works hand in hand
with the ``export_json_db`` class method. This will allows users to share
their database without needing to manually move their db file.
:return:
"""
options = dict()
options['defaultextension'] = '.zip'
options['filetypes'] = [('CALIPSO Data Archive', '*.zip'),
('All files', '*')]
options['initialdir'] = CONF.session_db.dir()
fl = tkFileDialog.askopenfilename(**options)
if fl != '':
log_fname = fl.rpartition('/')[2]
logger.info('Importing database from \'%s\'' % log_fname)
success = db.import_from_json(fl)
if success:
logger.info('Success, JSON file imported')
tkMessageBox.showinfo(
'database', 'shapes from %s imported ' % log_fname +
'(note: new tags have been assigned to these shapes!)')
else:
logger.error('Invalid JSON file')
tkMessageBox.showerror('database',
'Invalid JSON file %s' % log_fname)
def handle_tag(redis_con, tag_name):
logger.info('handle%s', tag_name)
try:
BookTag.add_if_absent(tag_name)
i = 0
while True:
start = 20 * i
url = 'https://book.douban.com/tag/%s?start=%s&type=T' % (tag_name,
start)
logger.info(url)
text = get_page_content(url)
if not text.find('没有找到符合条件的图书') < 0:
logger.info('finished:' + tag_name)
break
if text.find('豆瓣图书标签') < 0:
raise Exception('列表页面异常' + text)
soup = get_soup(text)
item_list = []
for item in soup.find_all(class_='subject-item'):
info = BookTableInfo()
info.name = item.find(title=True)['title']
info.douban_id = item.find(title=True)['href'].split('/')[-2]
info.short_info = item.find(class_='pub').text.strip()
info.img_url = item.find('img')['src']
info.from_tag_id = BookTag.get_tag_by_name(tag_name).id
item_list.append(info)
redis_con.lpush(QUEUE_DOUBAN_ID,
map(lambda x: x.douban_id, item_list))
BookTableInfo.add_list(item_list)
i += 1
except Exception as e:
logger.error('---->%s', e)
redis_con.lpush(QUEUE_DOUBAN_TAG_EXCEPTION, tag_name)
def fill_rectangle(self, event):
"""
Informs the correct shape list's blank object to draw a rectangle to the screen
using the provided coordinates
:param event: A backend passed ``matplotlib.backend_bases.MouseEvent`` object
"""
if self.__current_plot == Plot.baseplot:
logger.warning("Cannot draw to BASE_PLOT")
return
if event.xdata and event.ydata:
if len(self.__current_list[-1].get_coordinates()) is 0:
return
logger.debug('Filling: %d, %d' % (event.xdata, event.ydata))
logger.info('Creating rectangle')
self.__current_list[-1].fill_rectangle(event, self.__current_plot,
self.__hdf, self.__figure,
ShapeManager.outline_toggle)
self.__current_list[-1].set_tag(self.generate_tag())
self.__current_list.append(Shape(self.__canvas))
self.__canvas.show()
else:
logger.error('Bounds out of plot range, skipping')
self.__current_list[-1].set_coordinates([])
self.__canvas._tkcanvas.delete(self.__current_list[-1].lastrect)
def get_sql(self, sql_val):
try:
result = self.db.query(sql_val)
return result.all(as_dict=True)
except:
logger.error('数据库错误')
finally:
self.db.close()
def get_para_lst_configure(moc_name):
moc_ins = MocSingleton.get_instance()
moc_obj = moc_ins.get_moc(moc_name)
if moc_obj is None:
logger.error("moc name %s not in moc single ton." % moc_name)
return False, []
return True, moc_obj.get_all_para_name()
def get_fig(self):
"""
Returns the figure that is plotted to the canvas
:rtype: :py:class:`SubplotAxes`
"""
if self.__fig:
return self.__fig
logger.error('Fig does not exist')
def get_request(url):
"""
Gets request by url
"""
try:
return requests.get(url, verify=False)
except (requests.exceptions.Timeout, requests.exceptions.TooManyRedirects,
requests.exceptions.RequestException) as e:
logger.error("{}\n {}".format(e, url))
def get_raw_cmd(cls, cmd):
if cmd.count(':') < 1:
logger.error('cmd has not :, cmd is %s.' % cmd)
return None
m = cmd.find(':')
cmd_name = cmd[:m]
para_info = cmd[m:].strip(':')
cmd_name = re.sub(' +', ' ', cmd_name.strip())
para_info = para_info.replace(' ', '')
return cmd_name + ":" + para_info
def set_attributes(self, attributes_list):
"""
Set the internal list of attributes to a custom passed list
:param list attributes_list:
"""
for i in attributes_list:
if i not in TAGS:
logger.error('Caught invalid attribute for setting \'%s\'' % i)
return
self.__attributes = attributes_list
def add_attribute(self, attr):
"""
Append a passed attribute onto the internal attribute list
:param str attr: An attribute enum
"""
if attr in TAGS:
self.__attributes.append(attr)
self.__saved = False
else:
logger.error('Caught invalid attribute for adding \'%s\'' % attr)
def remove_attribute(self, attr):
"""
Remove an attribute as specified in ``constants.py`` from the internal attributes variable
:param str attr:
"""
if attr in TAGS:
self.__attributes.remove(attr)
self.__saved = False
else:
logger.error('Caught invalid attribute for removal \'%s\'' % attr)
def _execute_rmv(self, tbl_name, key_dict):
try:
tbl_key = self._get_table_key(tbl_name, key_dict)
if self.__connect.exists(tbl_key) == 0:
logger.info("rmv table %s not exist...." % tbl_name)
return False, "rmv table %s not exist...." % tbl_name
self.__connect.delete(tbl_key)
return True, "rmv success...."
except Exception as e:
logger.error("_execute_rmv, info is %s." % str(e))
return False, str(e)
def _execute_mod(self, tbl_name, key_dict, para_dict):
try:
tbl_key = self._get_table_key(tbl_name, key_dict)
if self.__connect.exists(tbl_key) == 0:
logger.info("mod table %s not exist...." % tbl_name)
return False, "mod table %s not exist...." % tbl_name
for key, val in para_dict.items():
self.__connect.hset(tbl_key, key, val)
return True, "mod success...."
except Exception as e:
logger.error("_execute_mod, info is %s." % str(e))
return False, str(e)
def get_request(url):
"""
Gets request by url
"""
try:
return requests.get(url, verify=False)
except (
requests.exceptions.Timeout,
requests.exceptions.TooManyRedirects,
requests.exceptions.RequestException,
) as e:
logger.error("{}\n {}".format(e, url))
def _execute_add(self, tbl_name, key_dict, para_dict):
try:
tbl_key = self._get_table_key(tbl_name, key_dict)
if self.__connect.exists(tbl_key) != 0:
logger.info("add table %s already exist...." % tbl_name)
return False, "add table %s already exist...." % tbl_name
para_dict.update(key_dict)
self.__connect.hmset(tbl_key, para_dict)
return True, "add success...."
except Exception as e:
logger.error("_execute_add, info is %s." % str(e))
return False, str(e)
def anchor_rectangle(self, event):
"""
Informs the correct shape list's blank object to plot a corner of a rectangle.
:param event: A backend passed ``matplotlib.backend_bases.MouseEvent`` object
"""
if self.__current_plot == Plot.baseplot:
logger.warning("Cannot draw to BASE_PLOT")
return
if event.xdata and event.ydata:
logger.info('Anchoring %d, %d' % (event.xdata, event.ydata))
self.__current_list[-1].anchor_rectangle(event)
else:
logger.error('Anchor selected is out of range, skipping')
def check_para_consist(cls, moc_obj, para_2_val):
para_lst = moc_obj.get_all_para_name()
if len(para_lst) != len(para_2_val):
logger.error("para_lst[%s] len is not equal to para_2_val[%s]." %
(para_lst, para_2_val))
return False
for para in para_lst:
if para not in para_2_val:
logger.error("para[%s] not in para_2_val[%s]." %
(para, para_2_val))
return False
return True
def check_key_para_consist(cls, para_2_val, key_para_lst):
if len(para_2_val) != len(key_para_lst):
logger.error(
"para_2_val[%s] len is not equal to key_para_lst[%s]." %
(para_2_val, key_para_lst))
return False
for key_para in key_para_lst:
if key_para not in para_2_val:
logger.error("key_para %s not in para_2_val %s." %
(key_para, para_2_val))
return False
return True
def get_para_lst_db(moc_name):
db_obj = DbSingleton.get_instance().get_db_obj()
ret, query_dict_lst = db_obj.query(moc_name)
if not ret:
logger.error("query moc name %s fail. info is %s." %
(moc_name, query_dict_lst))
return False, []
para_lst = []
for query_dict in query_dict_lst:
for key, _ in query_dict.items():
para_lst.append(key)
return True, list(set(para_lst))
def check_mod_para(cls, para_2_val, key_para_lst, all_para_lst):
for key_para in key_para_lst:
if key_para not in para_2_val:
logger.error("key_para %s not in para_2_val %s." %
(key_para, para_2_val))
return False
for para in para_2_val.keys():
if para not in all_para_lst:
logger.error("para %s not in all_para_lst %s." %
(para, all_para_lst))
return False
return True
def wrapper(request, *args, **kwargs):
user = request.user
if not user.is_authenticated:
return redirect('client:login')
elif user.is_adminkvm:
return func(request, *args, **kwargs)
else:
try:
connect = KeystoneClient(admin=True)
connect.keystone.tokens.validate(user.token_id)
except Exception as e:
logger.error(e)
return redirect('client:login')
return func(request, *args, **kwargs)
def export_db(self, only_selected=False):
"""
Notify the database that a save is taking place, the
db will then save all polygons present on the screen
"""
logger.info('Notifying database to save with select flag %s' %
(str(only_selected)))
success = self.__shapemanager.save_db(only_selected)
if success:
logger.info('Success, saved to db')
tkMessageBox.showinfo('database', 'Objects saved to database')
else:
logger.error('No objects to be saved')
tkMessageBox.showerror('database', 'No objects to be saved')
def do_local_backup(query_dict):
src = query_dict['SRC']
dst = query_dict['DST']
if not os.listdir(src):
logger.error("SRC %s dir is empty..." % src)
return
time_now = time.strftime('%Y_%m_%d_%H_%M_%S', time.localtime(time.time()))
dst_abs = os.path.join(dst, time_now)
if os.path.exists(dst_abs):
shutil.rmtree(dst_abs)
# try to rm too old file
rm_backup_db_7_day_ago(dst)
shutil.copytree(src, dst_abs)
