def __init__(self, item=None):
self.sentiment = 0
self.countries = []
self.title = ""
self.summary = ""
self.link = ""
self.publushed = None
if item:
try:
# Set source, author and title
self.link = item['links'][0]['href']
self.title = item.get('title')
self.published = parser.parse(item.get('published'))
if self.published.tzinfo is None:
self.published = pytz.utc.localize(self.published)
# Set summary, get rid of all the junk at the end
summary = item.get('summary')
summary = summary[:summary.find("\n\n")]
summary = summary[:summary.find("<")]
self.summary = summary
except (TypeError, KeyError):
log.error("Could not find first link: {}".format(
json.dumps(item)))
def parse(title, source, n = None):
'''
Function parses the source search page and returns the .mp3 links in it.
@param title: Search string.
@param source: Search website source. Value can be dilandau, mp3skull, youtube, seekasong.
@return links: .mp3 url links generator.
'''
source = source.lower()
if source == "dilandau":
gen = parse_dilandau(title)
elif source == "mp3skull":
gen = parse_Mp3skull(title)
elif source == "soundcloud":
gen = parse_soundcloud_api2(title)
elif source == 'bandcamp':
gen = parse_bandcamp(title)
elif source == "youtube":
gen = parse_Youtube(title)
else:
log.error('no source "%s". (from parse function in WebParser)' % source)
gen = (x for x in []) # empty generator
if n:
gen = itertools.islice(gen, n)
return gen
def _on_client_close(self, client):
log.info('tcp_server_client_close|id=%s,remote=%s', client.id.encode('hex'), client.remote_address)
if client.id not in self._clients:
log.error('tcp_server_close_conn_not_found|id=%s,remote=%s', client.id.encode('hex'), client.remote_address)
return
self._handle_task(client, GTCP_CMD_DISCONNECT, '')
del self._clients[client.id]
def _on_client_connect(self, stream, address):
client = GTcpConnection(stream, address, self._config, self._on_client_packet, self._on_client_close)
if client.id in self._clients:
log.error('tcp_server_dup_client|id=%s,remote=%s', client.id.encode('hex'), client.remote_address)
self._clients[client.id] = client
self._handle_task(client, GTCP_CMD_CONNECT, '')
log.info('tcp_server_client_connect|id=%s,remote=%s', client.id.encode('hex'), client.remote_address)
def run(self):
random.seed()
self.on_init()
from gtcpclient import GTcpClient
log.info('tcp_worker_start|id=%d', self._id)
self._client = GTcpClient(self._config.WORK_ENDPOINT['address'], self._config.WORK_ENDPOINT['port'], 0)
while True:
try:
request = self._client.receive()
if request is None:
log.warn('tcp_worker_lost_connection|client_id=%s,client=%s', self._client.id.encode('hex'), self._client.remote_address)
self._client.close()
elif len(request) < GTCP_HEADER_SIZE:
log.error('tcp_worker_request_packet_error|client_id=%s,client=%s,request=%s', self._client.id.encode('hex'), self._client.remote_address, request.encode('hex'))
self._client.close()
else:
request_cmd = request[:GTCP_CMD_SIZE]
request_client = TcpEndpoint(request[GTCP_CMD_SIZE:GTCP_HEADER_SIZE])
reply_body = None
if request_cmd == GTCP_CMD_RELAY:
request_body = request[GTCP_HEADER_SIZE:]
reply_body = self.on_packet(request_client, request_body)
elif request_cmd == GTCP_CMD_CONNECT:
reply_body = self.on_client_connect(request_client)
elif request_cmd == GTCP_CMD_DISCONNECT:
self.on_client_disconnect(request_client)
if reply_body is None:
self._client.send(GTCP_CMD_NONE + request_client.client_id)
else:
self._client.send(GTCP_CMD_RELAY + request_client.client_id + reply_body)
except Exception as ex:
log.exception('tcp_worker_exception|id=%u,exception=%s', self._id, ex, exc_info=True)
self._client.close()
def consume_message(self, channel):
# 再订阅消息队列
try:
pubsub = self.rds.pubsub()
pubsub.subscribe(channel)
log.debug('subscribe push channel %s successfully' % channel)
redis_channel = pubsub.listen()
for message in redis_channel:
self.retry_time = 0
if 'kill' == message['data']:
break
else:
self.send_message(message)
except:
# 连接redis不上, 睡眠几秒钟重试连接
if self.retry_time <= self.retry_time_max:
time.sleep(10)
self.retry_time = self.retry_time + 1
log.debug(u'redis cannot connect, retry %d' % self.retry_time)
self.consume_message(channel)
else:
# 这时，需要Email通知管理员了
log.error(u'retry time up, redis gone! help!')
log.debug('i am leaving push')
def log_error(self):
self.rds.hincrby("fail_counter", self.app_key)
log.error('message push fail')
type, value, tb = sys.exc_info()
error_message = traceback.format_exception(type, value, tb)
log.debug(type)
log.error(error_message)
def send(self, to=None, frm=None, body=None, topic=None):
"""
Send messages to recipients contained
in self._rcpt_list. Creates and closes SMTP connection.
:rtype: None
"""
if to and frm and body and topic:
rcpt_list = [{"to": to, "from": frm, "msg": body, "topic": topic}]
elif not to and not frm and not body and not topic:
rcpt_list = self._rcpt_list
else:
log.error('Incorrect params, all or nothing')
raise Exception('Incorrect params, all or nothing')
connection = smtplib.SMTP(self._host, self._port, timeout=20)
connection.ehlo()
if self._tls:
connection.starttls()
connection.login(self._user, self._psw)
count = 0
try:
for rcpt in rcpt_list:
body = MIMEText(rcpt['msg'].encode('utf-8'), _subtype='html', _charset='utf-8')
body.add_header('Subject', rcpt['topic'])
body.add_header('Content-Type', 'text/html; charset=utf-8')
connection.sendmail(rcpt['from'], rcpt['to'], str(body))
count += 1
finally:
sendlog.info('Sending: posted %s emails' % count)
connection.close()
def runCMD(cmd, workDir=None, pipe=None, isShell=False):
tProcData = dict()
usedTmp = False
if pipe is None:
pipeFile = open(_TMP_FILE, "w+")
usedTmp = True
else:
pipeFile = pipe
try:
#log.debug("[Debug] Start process: {0}".format(cmd if type(cmd) != list else " ".join(cmd)))
tProc = Popen(cmd, cwd=workDir, shell=isShell, stdout=pipeFile, stderr=pipeFile)
tProc.wait()
except:
log.error("[Error] Can't run process: {0}".format(cmd))
log.error("[Error] Problem: {0}".format(sys.exc_info()[1]))
tProcData["ret_code"] = None
else:
tProcData["ret_code"] = tProc.returncode
finally:
pipeFile.tell()
pipeFile.seek(0)
tProcData["out"] = pipeFile.read()
pipeFile.close()
if usedTmp:
os.remove(_TMP_FILE)
return tProcData
def _runweb():
"""Запускаем прием писем по HTTP"""
try:
run_simple("0.0.0.0", SRV_PORT, wcgiapp)
except Exception as e:
log.error("HTTP service stopped with message: %s" % e)
def run(self):
self._create_db()
while True:
try:
sql, params, result = self._queue.get()
if sql == '__CLOSE__':
self._db.close()
break
log.info("[DBWrapper] QUERY: %s" % sql)
log.info("[DBWrapper] PARAMS: %s" % str(params))
log.info("[DBWrapper] RESULT: " + str(result))
cursor = self._db.cursor()
if params:
cursor.execute(sql, params)
else:
cursor.execute(sql)
except sqlite3.OperationalError, e:
log.error("[DBWrapper] OperationalError : %s" % e)
if result:
log.info("[DBWrapper] Putting Results")
for row in cursor.fetchall():
result.put(row)
result.put("__END__")
self._db.commit()
def _on_recv_header(self, data):
(body_size,) = struct.unpack('<I', data)
if body_size > self.TCP_MAX_PACKET_SIZE:
log.error('tcp_asnyc_client_body_size_overflow|id=%s,size=%u', self._id, body_size)
self._close()
return
self._stream.read_bytes(body_size, self._on_recv_body)
def _runmailer():
"""Запускаем мониторинг очереди писем"""
try:
Mailer(smtp_conn=smtp_conn).loop()
except Exception as e:
log.error("Mailer stopped with message: %s" % e)
def parse_subcommands(self, argv, gparser=None ):
"""
Parse global arguments, find subcommand from list of subcommand
objects, parse local subcommand arguments and return a tuple of
global options, selected command object, command options, and
command arguments.
Call execute() on the command object to run. The command object has
members 'gopts' and 'opts' set for global and command options
respectively, you don't need to call execute with those but you could
if you wanted to.
"""
gparser = gparser or self.gparser
# parse global options.
gopts, args = gparser.parse_args( argv)
if not args:
gparser.print_help()
raise SystemExit("\nError: you must specify an action.")
# set debug level if specified
if gopts.DEBUG:
console.setLevel(logger.DEBUG)
# load StarClusterConfig into global options
try:
cfg = config.StarClusterConfig(gopts.CONFIG)
cfg.load()
except exception.ConfigNotFound, e:
log.error(e.msg)
e.display_options()
sys.exit(1)
def _single_convert(self, input_file_object):
if input_file_object:
input_file_path = input_file_object.get_input_file_path()
output_file_name = rename_filename_with_extension(
os.path.basename(input_file_path), self.final_format)
intermediate_filename = str(time.time()).replace('.', '') + '.html'
output_file_path = os.path.join(self.tmp_dir, output_file_name)
intermediate_path = os.path.join(
self.tmp_dir, intermediate_filename)
with codecs.open(input_file_path, "r", "utf-8") as f:
cleaned_content = remove_tags(f.read())
with open(intermediate_path, 'w') as w:
w.write(cleaned_content)
converter = CONVERTER_LOCATION.format(
input_file_path=intermediate_path,
output_file_path=output_file_path)
self.execute(converter)
if os.path.isfile(output_file_path):
return output_file_path
else:
self.handle_failed_conversion(input_file_object)
log.error('Conversion failed from HTML => PDF')
return None
def get_dir_index(document_root, path, fullpath):
files = []
dirs = []
def get_descriptor(title):
fullpath = os.path.join(path, title)
return { 'title': title, 'type': tools.get_type(contrib.get_full_path(document_root, fullpath)) }
if not document_root:
pagetype = 'front-page'
for key in contrib.get_virtual_paths():
dir_descriptor = get_descriptor(key)
dirs.append(dir_descriptor)
else:
pagetype = tools.get_type(fullpath)
for f in sorted(os.listdir(fullpath)):
if not f.startswith('.'):
if os.path.isfile(os.path.join(fullpath, f)):
files.append(get_descriptor(f))
else:
f += '/'
dirs.append(get_descriptor(f))
try:
if tools.get_type(fullpath) == 'virtual':
contexts = context.global_settings(fullpath).sections()
else:
contexts = context.get(fullpath).sections()
log.debug(contexts)
except Exception, e:
log.error(e)
contexts = []
def parsexml(self,xml):
log.debug('in parsexml')
# 如果传进来的是文件，则直接读取内容
# 如果是字符串，则先尝试解释，如果不是xml则再尝试作为文件路径，再不行，则抛异常了。
xml_content = ''
if type(xml) == file:
xml_content = xml.read()
xml.close()
elif type(xml) == str or type(xml) == unicode:
log.debug('try to load file')
if os.path.exists(xml):
xml_file = open(xml,'r')
xml_content = xml_file.read()
xml_file.close()
else:
xml_content = xml
else:
log.error('could not init testcase from xml')
raise TypeError,'fromxml need a file instance or a string instance for argument'
log.debug('starting parse xml')
log.debug('xml content: %s'%xml_content)
doc = minidom.parseString(xml_content)
ret = get_child_tags(doc,'test')#this statement return a list
log.debug('child tag len : %d'%len(ret))
if ret:
test_node = ret[0]
return test_node
else:
log.warn('no test node in the xml!')
def load_file(self, filename):
"""loads skeleton information from file"""
if os.path.exists(filename):
log.debug("loading file: " + filename)
self.filename = filename
# try to load the file
try:
tree = etree.parse(filename)
skel = tree.getroot()
for node in self.info_nodes:
keys = node.keys()
attr_name = keys[0]
attr_path = node[attr_name]
node_data = tree.xpath(attr_path)
if node_data:
self.__dict__[attr_name] = node_data[0].text
# get file list
filelist = tree.xpath("/skel/filelist/file")
for fil in filelist:
self.filelist.append(fil.get("name"))
# parameters
params = tree.xpath("/skel/params/param")
for param in params:
self.params.append({param.get("name"): param.get("value")})
except Exception as ex:
log.warning("Could not parse skeleton file: " + str(ex) + " in file: " + filename)
return False
else:
log.error("File does not exist: " + filename)
return False
def loop(self):
while True:
try:
# получаем id`шники записей из БД рандомно для того чтобы не
# получать одни и теже записи, которые не могут быть отправлены
rids = [record.id for record in Message.select().order_by(Random()).limit(10)]
except PeeweeException as e:
# если что-то полшло не так
# пропускаем текущую итерацию
log.error(e)
continue
# перебираем все id`шники
for rid in rids:
try:
message = Message.get(Message.id == rid)
with db.transaction():
# удаляем запись по id
Message.delete().where(Message.id == rid).execute()
# выполняем отправку сообщения
self.smtp_conn.send(message.to, message.frm, message.body, message.topic)
except (OSError, TypeError, PeeweeException, SMTPException) as e:
log.error(e)
continue
sleep(self.sleep_time)
def slot_playlist(self):
dialog = QtGui.QFileDialog()
dialog.setFileMode(QtGui.QFileDialog.ExistingFile)
if config.post_download_playlist_path:
dialog.setDirectory(os.path.dirname(config.post_download_playlist_path))
else:
# The default playlist directory
dialog.setDirectory(r'%s\My Documents\My Music\My Playlists' % utils.get_home_dir())
f = unicode(dialog.getOpenFileName(caption=tr("Open Playlist"), filter=tr("Supported Playlist Files") + " (*.m3u *.wpl)"))
f = f.replace('/','\\')
if f:
try:
log.debug("Adding %s to the %s playlist..." % (self.audio_path, f))
utils.add_item_to_playlist(f, self.audio_path)
self.statusBar_append = tr("; Saved to playlist")
except (IOError, RuntimeError), e:
log.error(str(e))
QtGui.QMessageBox.critical(self, tr("Error"), str(e), QtGui.QMessageBox.Ok)
return
config.post_download_playlist_path = f
if self.saveSelection_CheckBox.isChecked():
config.post_download_action = 'addPlaylist'
self.close()
def assert_file_exists(file):
if not os.path.exists(file):
log.error("file {} does not exist".format(file))
exit()
if os.stat(file).st_size == 0:
log.error("file {} is empty".format(file))
exit()
def __init__(self, tasks, parent=None):
super(MainWin, self).__init__(parent)
self.tasks = tasks
self.songObj = tasks[0].songObj
self.dl_dir = config.dl_dir
self.setWindowTitle(tr("Post Download Window"))
self.setWindowIcon(QtGui.QIcon(os.path.join('pics', 'kmix.png')))
self.video_path = os.path.join(self.dl_dir, self.songObj.GetProperFilename())
self.audio_path = os.path.join(self.dl_dir, self.songObj.GetProperFilename('mp3'))
self.isAudio = config.downloadAudio and os.path.exists(self.audio_path)
self.isVideo = self.video_path != self.audio_path and config.downloadVideo \
and os.path.exists(self.video_path)
# the video_path != audio_path is because GetProperFilename returns mp3
# for a music file, and mp4/flv/webm for a video file. If they both are
# the same, we have only a music file and no video.
log.debug('PostDownloadWindow: self.video_path: %s, self.isVideo: %s' % (self.video_path, str(self.isVideo)))
log.debug('PostDownloadWindow: self.audio_path: %s, self.isAudio: %s' % (self.audio_path, str(self.isAudio)))
if not self.isAudio and not self.isVideo:
log.error("Error: no audio or video files were found.")
QtGui.QMessageBox.critical(self, tr("Error"), tr("no audio or video files were found."), QtGui.QMessageBox.Ok)
self.deleteLater()
self.init_widgets()
def _completer(self):
try:
rimages = self.ec2.registered_images
completion_list = [i.id for i in rimages]
return optcomplete.ListCompleter(completion_list)
except Exception, e:
log.error('something went wrong fix me: %s' % e)
def init():
"Initizing the environment"
# Making sure temp folder is accessable.
if not os.path.exists(config.temp_dir):
try:
os.makedirs(config.temp_dir)
except (IOError, WindowsError):
config.temp_dir = os.path.join(os.environ["Temp"], "iQuality") # if temp folder cannot be created, we should set it to default.
# setting ext_bin directory
if hasattr(sys, "frozen"):
# the ext_bin folder will be one level upper
splitted_dirs = config.ext_bin_path.split('\\')
config.ext_bin_path = "\\".join(splitted_dirs[:-2]+[splitted_dirs[-1]])
if not os.path.exists(config.ext_bin_path):
os.makedirs(config.ext_bin_path)
# Checking internet connection
returncode = utils.launch_without_console('ping 8.8.8.8 -n 1').wait()
if returncode > 0:
log.error("Got NoInternetConnectionException()")
raise NoInternetConnectionException()
returncode = utils.launch_without_console('ping www.google.com -n 1').wait()
if returncode > 0:
log.error("Got NoDnsServerException()")
raise NoDnsServerException()
def get_mime_type(file_path):
try:
mime_type = magic.from_file(
file_path.encode('utf-8'), mime=True)
except IOError, e:
log.error('Error getting MIME type : {}'.format(e))
mime_type = None
def get_filesize(url, timeout=config.get_filesize_timeout):
'''
Function fetches filesize of a file over HTTP.
@param url: url address.
@param timeout: Timeout in seconds.
@return bool: Size in bytes.
'''
if isinstance(url, utils.cls.MetaUrl):
url = url.url
url = url.replace(' ', '%20')
try:
req = urllib2.Request(url, headers=config.generic_http_headers)
u = urllib2.urlopen(req, timeout=timeout)
except (urllib2.HTTPError, urllib2.URLError) as e:
log.error(e)
return 0
meta = u.info()
try:
file_size = int(meta.getheaders("Content-Length")[0])
except IndexError:
return 0
return file_size
def eval(self, func, args=None):
'''Evaluate an uristscript or other callable object.'''
# If the function is actually an urist, make sure we know that
uristinstance = None
if isinstance(func, uristscript):
uristinstance = func
func = uristinstance.func
name = uristinstance.getname()
else:
name = func.__name__
# Actually execute the script
log.info('Running script %s%s.' % (name, ('with args %s' % args) if args else ''))
try:
response = func(self.df, **args) if args else func(self.df) # Call the function
if response is not None:
# Handle success/failure response
log.info(str(response))
(self.successes if response.success else self.failures).append(uristinstance if uristinstance else func)
else:
log.error('Received no response from script %s.' % name)
self.noresponse.append(uristinstance if uristinstance else func)
except Exception:
log.exception('Unhandled exception while running script %s.' % name)
return False
else:
return True
def handleall(self, infos=None):
'''Handle all scripts specified by the session's configuration.'''
if infos is None and self.conf is not None: infos = self.conf.scripts
if infos and len(infos):
for info in infos: self.handle(info)
else:
log.error('No scripts to run.')
def handle(self, info):
'''Handle a single script.'''
funcs = self.funcs(info)
if funcs:
for func in funcs: self.eval(func)
else:
log.error('Found no scripts matching %s.' % info)
def __init__(self, item=None):
self.sentiment = 0
self.countries = []
self.title = ""
self.summary = ""
self.link = ""
self.publushed = None
if item:
try:
# Set source, author and title
self.link = item["links"][0]["href"]
self.title = item.get("title")
self.published = parser.parse(item.get("published"))
if self.published.tzinfo is None:
self.published = pytz.utc.localize(self.published)
# Set summary, get rid of all the junk at the end
summary = item.get("summary")
summary = summary[: summary.find("\n\n")]
summary = summary[: summary.find("<")]
self.summary = summary
except:
log.error("Problem processing article: {}".format(str(item)))
def _read_from_excel(arg, context):
"""
Try to evaluate an argument by reading from the loaded Excel spreadsheet.
"""
# Try handling reading value from an Excel spreadsheet cell.
arg_str = str(arg)
if (("thisworkbook." in arg_str.lower()) and
('("thisworkbook.' not in arg_str.lower()) and
("sheets(" in arg_str.lower()) and
("range(" in arg_str.lower())):
log.debug("Try as Excel cell read...")
# Pull out the sheet name.
tmp_arg_str = arg_str.lower()
start = tmp_arg_str.index("sheets(") + len("sheets(")
end = start + tmp_arg_str[start:].index(")")
sheet_name = arg_str[start:end].strip().replace('"', "").replace("'", "").replace("//", "")
# Pull out the cell index.
start = tmp_arg_str.index("range(") + len("range(")
end = start + tmp_arg_str[start:].index(")")
cell_index = arg_str[start:end].strip().replace('"', "").replace("'", "").replace("//", "")
log.debug("Sheet name = '" + sheet_name + "', cell index = " + cell_index)
try:
# Load the sheet.
sheet = context.loaded_excel.sheet_by_name(sheet_name)
# Pull out the cell column and row.
col = ""
row = ""
for c in cell_index:
if (c.isalpha()):
col += c
else:
row += c
# Convert the row and column to numeric indices for xlrd.
row = int(row) - 1
col = excel_col_letter_to_index(col)
# Pull out the cell value.
val = str(sheet.cell_value(row, col))
# Return the cell value.
log.info("Read cell (" + str(cell_index) + ") from sheet " + str(sheet_name))
log.debug("Cell value = '" + val + "'")
return val
except Exception as e:
log.error("Cannot read cell from Excel spreadsheet. " + str(e))
def _download_binary(binary):
f = urllib.URLopener()
try:
if not os.path.isdir(binary.dir):
_ensure_dir(binary.dir)
fname, headers = f.retrieve(
binary.url, os.path.join(binary.dir, os.path.basename(binary.url)))
os.chmod(fname, 755)
log.debug("chmod set to 755 for {}".format(fname))
except IOError as e:
log.error(e.args)
def main():
if args.delete_all:
delete_all()
else:
try:
tfvars = utils.get_tfvars()
if not args.only_nodes:
try_to_delete_cluster(tfvars)
delete_nodes(tfvars)
except:
log.error("Failed to delete nodes")
def main():
appname = os.path.basename(sys.argv[0])
try:
options, args = getopt.getopt(sys.argv[1:], "fhit:qvV", [
"force", "help", "init", "template=", "quiet", "verbose", "version"
])
except getopt.GetoptError, err:
log.error("%s\n" % err)
print USAGE % appname
sys.exit(1)
def wrapper(*args, **kwargs):
friendsList = []
rows, err = func(*args, **kwargs)
if not err and rows:
for row in rows:
friendsList.append(row[0])
return friendsList
if not rows:
return friendsList
log.error(err)
return None
def process_replacements(self):
for href, enclosure_replacement, pub_key, test_pub_key in self.replacements:
log.debug(
"href:`%s` enclosure_replacement:`%s` pub_key:`%s` test_pub_key:`%s`"
% (href, enclosure_replacement, pub_key, test_pub_key))
errors = []
if not href:
errors.append("href null or empty")
continue
if not enclosure_replacement:
errors.append("enclosure_replacement null or empty")
if not pub_key:
errors.append("pub_key null or empty")
if not test_pub_key:
errors.append("test_pub_key null or empty")
if errors:
log.error("Skipping:%s for: href:%s enclosure_replacement:%s "
"pub_key:%s test_pub_key:%s" %
(",".join(errors), href, enclosure_replacement,
pub_key, test_pub_key))
continue
if self.max_errors_reached():
log.critical("Max errors as been reached for all urls %s" %
pformat(error_tracker))
return
while not self.max_errors_reached():
find = href
good_url = self.get_first_test_result(test_pub_key, pub_key)
if not good_url:
log.error("Unable to get a good url for test_pub_key:%s "
"pub_key:%s" % (test_pub_key, pub_key))
break
log.debug("******* MADE IT good_url:%s" % good_url)
hash_url = good_url.replace(test_pub_key, ":hash")
replace = good_url.replace(test_pub_key, enclosure_replacement)
log.debug("replace:%s" % replace)
if hash_url not in error_tracker:
error_tracker[hash_url] = 0
if hash_url not in reliablity_tracker:
reliablity_tracker[hash_url] = 0
if self.test_download(replace):
reliablity_tracker[hash_url] += 1
self.text = self.text.replace(find, replace)
break
error_tracker[hash_url] += 1
async def exe(ctx, *, arg):
try:
cursor = connection.cursor()
postgres_insert_query = """{}""".format(arg)
cursor.execute(postgres_insert_query)
connection.commit()
count = cursor.rowcount
except (Exception, psycopg2.Error) as error:
if (connection):
log.error(error)
await ctx.channel.send(error)
def eval(self, context, params=None):
# return the and of all the arguments:
try:
log.debug("Compute not " + str(self.arg))
val = self.arg
if (isinstance(val, VBA_Object)):
val = val.eval(context)
return (~ int(val))
except Exception as e:
log.error("Cannot compute Not " + str(self.arg) + ". " + str(e))
return "NULL"
def eval(self, context, params=None):
# return the eqv of all the arguments:
try:
return reduce(lambda a, b: (a & b) | ~(a | b),
coerce_args(eval_args(self.arg, context)))
except (TypeError, ValueError):
log.error('Impossible to Eqv arguments of different types.')
return 0
except RuntimeError as e:
log.error("overflow trying eval Eqv: %r" % self.arg)
raise e
def load_config(self, config_file, updated_settings):
with open(config_file, 'r') as config_io:
try:
self.config = yaml.load(config_io)
self.config.update(self.__filter_nonetypes(updated_settings))
except yaml.YAMLError as exc:
log.error("Load config failed: {}".format(exc))
log.debug("Load config: {}".format(self.config))
if 'log-file' in self.config:
logger.set_log_file(self.config['log-file'])
def __repr__(self):
if (self.op is not None):
if (self.lhs is not None):
return self.lhs.__repr__() + " " + self.op + " " + self.rhs.__repr__()
else:
return self.op + " " + self.rhs.__repr__()
elif (self.lhs is not None):
return self.lhs.__repr__()
else:
log.error("BoolExpr: Improperly parsed.")
return ""
def eval(self, context, params=None):
assert len(params) > 0
# TODO: Actually implement this properly.
val = params[0]
try:
r = int(val)
log.debug("Int: return %r" % r)
return r
except:
log.error("Int(): Invalid call int(%r). Returning ''." % val)
return ''
def main():
log.info('Parsing arguments...')
args = parse_arguments()
if args.team_name_tsv is not None:
team_name_dict = parse_team_name_tsv(args.team_name_tsv)
else:
team_name_dict = None
print(team_name_dict)
syn = synapseclient.login()
t0 = time.perf_counter()
while True:
try:
if not args.update_wiki_only:
evaluation = syn.getEvaluation(args.eval_queue_id)
# init multiprocessing
pool = multiprocessing.Pool(args.nth)
# distribute jobs
ret_vals = []
for submission, status in syn.getSubmissionBundles(
evaluation, status='RECEIVED'):
ret_vals.append(
pool.apply_async(validate_submission,
(submission, status, args, syn)))
# gather
for r in ret_vals:
r.get(BIG_INT)
pool.close()
pool.join()
update_wiki_for_round2(syn, team_name_dict, args)
except Exception as ex1:
st = StringIO()
traceback.print_exc(file=st)
message = st.getvalue()
subject = 'Server error:'
if args.send_msg_to_admin:
send_message(syn, args.admin_id, subject, message)
log.error(message)
log.info('Waiting for new submissions...')
while time.perf_counter() - t0 < args.period:
time.sleep(60)
t0 = time.perf_counter()
log.info('All done')
def create_git_repo(self):
try:
log.debug('creating git repo')
os.environ['PATH'] = os.environ[
'PATH'] + ':' + '/root/api/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin'
log.debug('check if git is seen: ' + os.environ['PATH'])
git.Repo.init(self.table_db_folder)
return True
except Exception as e:
log.error('couldnt create git repo', e)
return False
def train(self, model, data):
if os.path.exists(model):
log.warn("model already exists: {}".format(model))
if not os.path.exists(data):
log.error("data file does not exist: {}".format(data))
options = (config['vw-options'] or "") + VWTrainer.DEFAULT_OPTIONS
log.info("train VW: {}".format(model))
cmd.run("{vw}/vowpalwabbit/vw -f {model} -d {data} -c {options}" \
.format(vw=self.vw, model=model, data=data, options=options))
def eval(self, context, params=None):
# assumption: here the params have already been evaluated by Call_Function beforehand
assert (len(params) == 1)
r = ''
try:
num = float(params[0])
r = math.sin(num)
except:
pass
log.error("Sin: %r returns %r" % (self, r))
return r
def get_assisted_controller_status(kubeconfig):
log.info("Getting controller status")
command = f"oc --insecure-skip-tls-verify --kubeconfig={kubeconfig} --no-headers=true -n assisted-installer " \
f"get pods -l job-name=assisted-installer-controller"
response = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
if response.returncode != 0:
log.error(f'failed to get controller status: {response.stderr}')
return b''
log.info(f'{response.stdout}')
return response.stdout
def search(timestamp, tag='photoslicebot'):
'''
Ищет фотографии по хэштегу
:param timestamp:
:param tag:
:return:
'''
posts = []
url = "https://www.instagram.com/explore/tags/%s/" % tag
payload = {'__a': '1'}
res = requests.get(url, params=payload).json()
edges = res['graphql']['hashtag']['edge_hashtag_to_media']['edges']
#res = ie.tag(tag)
codes = []
for data in edges:
# тут уже сортированные по дате посты
data = data['node']
if data['taken_at_timestamp'] > timestamp:
if not data['is_video']:
codes.append(data['shortcode'])
else:
log.info('CRAWLER: There is a video {}'.format(
data['shortcode']))
for code in codes:
image = ie.media(code)
try:
if 'edge_media_to_caption' in image.data:
if 'edges' in image.data['edge_media_to_caption']:
if len(image.data['edge_sidecar_to_children']
['edges']) > 1:
p = Post(
username=image.data['owner']['username'],
caption=image.data['edge_media_to_caption']
['edges'][0]['node']['text'],
location=image.data['location'],
urls=list(
x['node']['display_url']
for x in image.data['edge_sidecar_to_children']
['edges']),
date=image.data['taken_at_timestamp'],
code=code)
posts.append(p)
else:
log.warning('CRAWLER: Post {} isn\'t a panorama'.format(code))
except Exception, err:
log.error('CRAWLER: Get post {0} info error {1}'.format(code, err))
def try_to_delete_cluster(tfvars):
try:
cluster_id = tfvars.get("cluster_inventory_id")
if cluster_id:
client = bm_inventory_api.create_client(
args.inventory_url, wait_for_url=False
)
client.delete_cluster(cluster_id=cluster_id)
# TODO add different exception validations
except Exception as exc:
log.error("Failed to delete cluster %s", str(exc))
async def http_get(self, url):
timeout = aiohttp.ClientTimeout(total=3)
async with aiohttp.ClientSession(timeout=timeout) as session:
try:
log.info(f"in http get {url}")
async with session.get(url) as r:
log.info(r.status)
if r.status != 200:
return f"URL {url} response abnormal code {r.status}"
except ClientConnectionError as e:
log.error(e)
return f"Unable to reach url {url}"
def get_public_ip():
try:
for service in ["https://api.ipify.org", "http://ip.42.pl/raw"]:
external_ip = get(service).text
if external_ip:
return external_ip
except Exception as e:
# try one more before giving up
try:
return get("http://httpbin.org/ip").json()["origin"].split(",")[0]
except:
log.error("could not check external ip")
def handle_get(self, timeout, runners, controllers):
try:
ctx_timeout, res = self.base_schedule(timeout, runners,
controllers)
except Exception as e:
log.error(e)
else:
if ctx_timeout.state == ctx_timeout.EXECUTED:
if not res:
return []
return [item for item in res if type(item) == dict]
return []
def send_postback(url, data):
try:
response = requests.post(url, data=json.dumps(data), timeout=60)
except Exception as e:
log.error('Postback error {}'.format(e))
post_handler.apply_async((url, data))
for d in data:
log.info('Postback status {} for doc_id={}, signed_url={}'.format(
response.status_code, d.get('doc_id', ''), d.get('signed_url',
'')))
def test_format_change_error(self):
log.enable_timestamp()
time_format = "[%H:%M:%S]"
log.set_log_timestamp_format(time_format)
time = datetime.now()
time_formatted = time.strftime(time_format)
expected_log = f"{self.RESET}{self.RED}{self.BOLD}{time_formatted}[-] Error: {self.RESET}{self.TEXT}{self.RESET}\n"
with patch('sys.stdout', new = StringIO()) as fake_out:
log.error(self.TEXT)
self.assertEqual(fake_out.getvalue(), expected_log)
def network_create_end(self, req=None, **kwargs):
try:
payload = json.loads(req.body)
network_id = payload['network']['id']
network = payload['network']
#self.pool.spawn(self._network_create, network_id, network)
self._network_create(network_id, network)
LOG.debug("network_info:%s", self.cache.get_state())
return 200, "SUCCESS"
except Exception as err:
LOG.error(err)
raise Exception('Error: %s' % err)
def miband4_keys_DB(miband4_MAC_KEY):
with open('miband4_auth_keys.json', 'r') as openfile:
# Reading from json file
miband4_keys_dict = json.load(openfile)
for device in miband4_devices:
try:
miband4_MAC_KEY.append([device, miband4_keys_dict[device]])
except:
log.error(
"The device with the MAC -> [{}] has a invalid key!\n Please check the file <miband4_auth_keys.json>"
.format(device))
def start(self):
"""Start the daemon."""
if self.pidfile.is_running():
self.pidfile.unlock()
LOG.error(('Pidfile %s already exist. Daemon already '
'running?'), self.pidfile)
sys.exit(1)
# Start the daemon
self.daemonize()
self.run()
def connect_db(user, password, server, port, database):
db = None
try:
uri = "mongodb://%s:%s@%s:%d" % (quote_plus(user),
quote_plus(password), server, port)
log.debug("MongoDB URI : " + str(uri))
client = MongoClient(uri)
db = client[database]
except Exception as ex:
log.error("unable to connect to MongoDB: " + str(ex))
db = None
return db
def main(onems_root_path):
"""Main flow"""
general_prop = _read_properties(
os.path.join(onems_root_path, 'conf', 'general.properties'))
timestamp = datetime.now().strftime('%Y%m%d%H%M%S')
target_dir = os.path.join(onems_root_path, 'ftproot', 'pldt')
if not os.path.exists(target_dir):
os.mkdir(target_dir)
try:
ftp_conf_icms = _get_ftp_config(general_prop, 'icms')
icms_file_path = os.path.join('/tmp', 'subscriber_%s.txt' % timestamp)
invalid_file_dir = general_prop['icms.invalid.file.dir']
if not invalid_file_dir:
invalid_file_dir = '/var/log'
save_icms_files(ftp_conf_icms, icms_file_path, invalid_file_dir)
shutil.copy(icms_file_path, target_dir)
except:
log.exception("Error saving icms files")
try:
dhcp_names = set([
s.strip(' \r')
for s in general_prop['dhcp.names'].strip(',').split(',')
])
except:
log.error(
"DHCP names not found, please check \"dhcp.names\" property in /OneMS/conf/general.properties."
)
return
for dhcp_name in dhcp_names:
if not dhcp_name:
continue
try:
ftp_conf_dhcp = _get_ftp_config(general_prop, dhcp_name)
ftp_conf_radius = _get_ftp_config(general_prop, 'radius')
lease_file_path = os.path.join('/tmp', '%s.leases' % dhcp_name)
radius_file_path = os.path.join(
'/tmp', 'radius.log_%s' % (_yesterday().strftime('%Y%m%d')))
download_dhcp_leases(ftp_conf_dhcp, lease_file_path)
download_radius_log(ftp_conf_radius, radius_file_path)
parseDHCPFile(dhcp_name)
parseRadius()
dhcp_file_path = os.path.join(
'/tmp', 'device_%s_%s.txt' % (timestamp, dhcp_name))
save_dhcp_files(dhcp_name, dhcp_file_path)
shutil.copy(dhcp_file_path, target_dir)
os.remove(lease_file_path)
os.remove(radius_file_path)
except:
log.exception("Error getting MAC2NSP files from DHCP server: %s",
dhcp_name)
def getTotalSize(o):
"""
Get approximate object size using dill
:param o: object
:return: approximate bytes size
"""
try:
return len(dill.dumps(o, recurse=True))
except:
log.error("Error while getting total object size!")
return 0
