示例#1
0
 def __init__(self, filename = "/run/media/eea1ee1d-e5c4-4534-9e0b-24308315e271/pynews/stream/clusteringData.db"):
     logger.info("Start building " + self.__class__.__name__)
     self.__mutex = threading.Semaphore()
     data = shelve.open(filename, protocol=-1, flag="r")
     langDetect = LangDetect.instance()
     vectors = [features(item["text"]) for digest, item in data.items() if item["text"] and item["text"] != "ERROR" and langDetect.detect(item["text"]) is "en"]
     self.__maxV = calcDiv(vectors)
     #vectors = normalize(vectors, self.__maxV)
     means = [array([10, 40, 0, 1]), array([30, 340, 2, 30]), array([120, 1500, 15, 50])]
     self.__clusterer = cluster.KMeansClusterer(3, euclidean_distance, initial_means=means, avoid_empty_clusters=True)
     self.__clusterer.cluster(vectors)
     klassIdToSize = {"0": 0, "1": 0, "2": 0}
     klassIdToWordsCount = {"0": 0, "1": 0, "2": 0}
     for item in data.itervalues():
         text = item["text"]
         if text and text != "ERROR":
             feat = features(text)
             #feat = normalize(feat, self.__maxV)
             klass = str(self.__clusterer.classify(feat))
             klassIdToSize[klass] += 1
             klassIdToWordsCount[klass] += len(text.split())
     data.close()
     results = []
     for klassId in ["0", "1", "2"]:
         meanWordsInKlass = klassIdToWordsCount[klassId] / klassIdToSize[klassId] if klassIdToSize[klassId] != 0 else 0
         results.append({"klass": klassId, "mean" : meanWordsInKlass})
     logger.info("Clustering results: " + str(results))
     sortedKlass = sorted(results, lambda x,y: x["mean"] < y["mean"])
     self.__klassIdToLabel = {klassIdWithLabel[0]: klassIdWithLabel[1] for klassIdWithLabel in zip([item["klass"] for item in sortedKlass], ["short", "medium", "long"])}
def run_standalone_test():
    run("service couchbase-server stop", warn_only=True)
    with shell_env(LD_LIBRARY_PATH="{}/forestdb/build".format(args.remote_workdir)):
        with cd(args.remote_workdir):
            run("rm -rf data/")
            run("mkdir data")
            run("ldd ./{}".format(prog_name))
            run("./{}".format(prog_name))
            run("cat incrementalsecondary.txt")

            # Now for internal processing and posting to showfast
            output_text = run("cat incrementalsecondary.txt")
            groups = re.search(
                r"initial index build time[^\d]*(\d*).*?seconds",
                output_text)
            initial_time = int(groups.group(1))

            groups = re.search(
                r"incrmental index build time[^\d]*(\d*).*?seconds",
                output_text)
            incremental_time = int(groups.group(1))
            logger.info("Grepped intial build time {}".format(initial_time))
            logger.info("Grepped incremental build time {}".format(
                incremental_time))
            if initial_time:
                post_initial(initial_time)
            if incremental_time:
                post_incremental(incremental_time)
示例#3
0
 def initialize_project(self):
     logger.info('Intializing local worker environment')
     with quiet():
         local('virtualenv -p python2.7 env')
         local('PATH=/usr/lib/ccache:/usr/lib64/ccache/bin:$PATH '
               'env/bin/pip install '
               '--download-cache /tmp/pip -r requirements.txt')
示例#4
0
    def consume(self):
        _, password = self.cluster_spec.rest_credentials
        for master in self.cluster_spec.yield_masters():
            host = master.split(':')[0]
            for bucket in self.test_config.buckets:
                logger.info(
                    'Reading data via UPR from {}/{}'.format(host, bucket)
                )
                upr_client = UprClient(host=host, port=11210)
                upr_client.sasl_auth_plain(username=bucket, password=password)
                mcd_client = MemcachedClient(host=host, port=11210)
                mcd_client.sasl_auth_plain(user=bucket, password=password)

                op = upr_client.open_producer("stream")
                response = op.next_response()
                if response['status'] != SUCCESS:
                    logger.interrupt('Failed to open producer')

                for vb in range(1024):
                    vb_stats = mcd_client.stats('vbucket-seqno {}'.format(vb))
                    uuid = long(vb_stats['vb_{}:uuid'.format(vb)])
                    high_seqno = long(vb_stats['vb_{}:high_seqno'.format(vb)])

                    op = upr_client.stream_req(vb=vb,
                                               flags=0,
                                               start_seqno=0,
                                               end_seqno=high_seqno,
                                               vb_uuid=uuid,
                                               high_seqno=high_seqno)
                    while op.has_response():
                        response = op.next_response()
                        if response['opcode'] != CMD_STREAM_REQ:
                            break
                    upr_client.close_stream(vbucket=vb)
                upr_client.shutdown()
示例#5
0
    def initialize_project(self):
        for worker, master in zip(self.cluster_spec.workers,
                                  self.cluster_spec.yield_masters()):
            state.env.host_string = worker
            run('killall -9 celery', quiet=True)
            for bucket in self.buckets:
                logger.info('Intializing remote worker environment')

                qname = '{}-{}'.format(master.split(':')[0], bucket)
                temp_dir = '{}-{}'.format(self.temp_dir, qname)

                r = run('test -d {}'.format(temp_dir), warn_only=True, quiet=True)
                if r.return_code == 0:
                    if self.reuse_worker == 'true':
                        return
                    logger.error('Worker env exists, but reuse not specified')
                    sys.exit(1)

                run('mkdir {}'.format(temp_dir))
                with cd(temp_dir):
                    run('git clone {}'.format(REPO))
                with cd('{}/perfrunner'.format(temp_dir)):
                    run('virtualenv -p python2.7 env')
                    run('PATH=/usr/lib/ccache:/usr/lib64/ccache/bin:$PATH '
                        'env/bin/pip install '
                        '--download-cache /tmp/pip -r requirements.txt')
示例#6
0
 def on_step_run(self):
     # `get_step_options` is provided by the `StepOptionsController` mixin.
     options = self.get_step_options()
     if options.run_pstrace and options.script:
         app_values = self.get_app_values()
         exe_path = path(app_values['pstrace_exe'])
         script = path(options.script)
         if not exe_path.isfile():
             logger.error('[PSTraceLauncher] invalid exe-path: %s' %
                          exe_path.abspath())
         elif not script.isfile():
             logger.error('[PSTraceLauncher] invalid script-path: %s' %
                          script.abspath())
         elif os.name != 'nt':
             logger.error('[PSTraceLauncher] This plugin is only supported '
                          'on Windows')
         else:
             pstrace_processes = [p for p in psutil.process_iter() if
                                  safe_psutil_attr(p, 'exe') ==
                                  exe_path.abspath()]
             if not pstrace_processes:
                 if options.delay_ms > 0:
                     logger.info('[PSTraceLauncher] delay: %s ms',
                                 options.delay_ms)
                     gtk.timeout_add(options.delay_ms, self._execute,
                                     exe_path, script)
                 else:
                     self._execute(exe_path, script)
             else:
                 logger.info('[PSTraceLauncher] skipping, since PSTrace is '
                             'already running as process %s',
                             [p.pid for p in pstrace_processes])
     self.complete_step()
示例#7
0
def upload_and_delete_file(path):
    """
    Upload the given file to S3, then remove it from the local filesystem
    """
    upload_file(path)
    logger.info('Upload complete. Deleting %s' % path)
    os.remove(path)
示例#8
0
    def wait_for_indexes_to_become_online(self, host, index_name=None):
        # POLL to ensure the indexes become online
        url = 'http://{}:8093/query/service'.format(host)
        data = {
            'statement': 'SELECT * FROM system:indexes'
        }
        if index_name is not None:
            data = {
                'statement': 'SELECT * FROM system:indexes WHERE name = "{}"'.format(index_name)
            }

        ready = False
        while not ready:
            time.sleep(10)
            resp = requests.Session().post(url=url, data=data)
            if resp.json()['status'] == 'success':
                results = resp.json()['results']
                for result in results:
                    if result['indexes']['state'] == 'online':
                        ready = True
                    else:
                        ready = False
                        break
            else:
                logger.error('Query:{} => Did not return a success!'.format(data['statement']))

        if index_name is None:
            logger.info('All Indexes: ONLINE')
        else:
            logger.info('Index:{} is ONLINE'.format(index_name))
示例#9
0
 def set_data_path(self, host_port, data_path, index_path):
     logger.info('Configuring data paths: {}'.format(host_port))
     api = 'http://{}/nodes/self/controller/settings'.format(host_port)
     data = {
         'path': data_path, 'index_path': index_path
     }
     self.post(url=api, data=data)
示例#10
0
    def get_server_groups(self, host_port):
        logger.info('Getting server groups')

        api = 'http://{}/pools/default/serverGroups'.format(host_port)
        return {
            g['name']: g['addNodeURI'] for g in self.get(url=api).json()['groups']
        }
示例#11
0
 def exec_n1ql_stmnt(self, host, stmnt):
     logger.info('Executing: {}'.format(stmnt))
     api = 'http://{}:8093/query/service'.format(host)
     data = {
         'statement': '{0}'.format(stmnt)
     }
     return self.post(url=api, data=data)
示例#12
0
    def create_bucket(self, host_port, name, ram_quota, replica_number,
                      replica_index, eviction_policy, threads_number,
                      password, proxy_port=None):
        logger.info('Adding new bucket: {}'.format(name))

        api = 'http://{}/pools/default/buckets'.format(host_port)

        data = {
            'name': name,
            'bucketType': 'membase',
            'ramQuotaMB': ram_quota,
            'evictionPolicy': eviction_policy,
            'flushEnabled': 1,
            'replicaNumber': replica_number,
            'replicaIndex': replica_index,
        }

        if proxy_port is None:
            data.update(
                {
                    'authType': 'sasl',
                    'saslPassword': password,
                })
        else:
            data.update(
                {
                    'authType': 'none',
                    'proxyPort': proxy_port,
                })

        logger.info('bucket specification: {}'.format(data))

        if threads_number:
            data.update({'threadsNumber': threads_number})
        self.post(url=api, data=data)
示例#13
0
    def get_version(self, host_port):
        logger.info('Getting Couchbase Server version')

        api = 'http://{}/pools/'.format(host_port)
        r = self.get(url=api).json()
        return r['implementationVersion']\
            .replace('-rel-enterprise', '').replace('-community', '')
示例#14
0
    def __call__(self, environ, start_response):
        """
        If the wsgi PATH_INFO starts with the static contents location, it will be returned.
        Otherwise the wrapped application will be called.
        """
        if environ['REQUEST_METHOD'] == 'GET' and environ['PATH_INFO'].startswith('/%s/' % self._location):
            logger.info('GET from %s: %s' % (environ.get('REMOTE_ADDR', 'unknown'), environ['PATH_INFO']))
            prefix = "/usr/share/skdrepo/"
            path = prefix + environ['PATH_INFO'][1:]
            try:
                f = open(path, 'r')
                data = f.read()
                f.close()
                (mime, encoding) = guess_type(path)
                status = '200 OK'  
                response_headers = [('Content-Type', mime)]
                response_body = [data]
            except IOError, e:
                logger.warning('failed to open file: %s' % path)
                status = '404 Not Found'
                response_headers = [('Content-Type', 'text/plain')]
                response_body = ['404 Not Found - \'%s\'' % path]

            start_response(status, response_headers)
            logger.debug('response to %s: %s, %s' % (environ['REMOTE_ADDR'], status, str(response_headers)))
            return response_body
示例#15
0
文件: bot.py 项目: encorehu/pyqq
 def grouphandler(self, data):
     #logger.debug(data)
     #content_list = data['content'][1]
     content_list = data['content'] # a list contains cface text
     content = ''
     for piece in content_list:
         if type(piece) == list:
             continue
         else:
             content += piece
     content = content.strip()
     if len(content)==0: # cface without text
         pass
     else:
         #logger.debug(content)
         if(content[0] =='@'):
             re = u'命令行可以使用'
             cmdcontent=content[1:]
             cmds = cmdcontent.split(':',2)
             h=self.findHandler(cmds[0])
             if(h!=None):
                 h.grouphandler(data['from_uin'],cmds[1])
             else:
                 self.send_group_msg(data['from_uin'], u'你能说人话么, 我怎么听不懂')
             #re = u"命令:"+cmds[0]+'内容:'+cmds[1]
             #self.send_group_msg(data['from_uin'], re)
         else:
             re = self.bot.reply(content)
             self.send_group_msg(data['from_uin'], re)
         logger.info("IN:%s\nreply group:%s"%(content, re))
示例#16
0
 def send_friend_msg(self, reply_content, uin, msg_id, fail_times=0):
     fix_content = str(reply_content.replace("\\", "\\\\\\\\").replace("\n", "\\\\n").replace("\t", "\\\\t"))
     rsp = ""
     try:
         req_url = "http://d1.web2.qq.com/channel/send_buddy_msg2"
         data = (
             ('r',
              '{{"to":{0}, "face":594, "content":"[\\"{4}\\", [\\"font\\", {{\\"name\\":\\"Arial\\", \\"size\\":\\"10\\", \\"style\\":[0, 0, 0], \\"color\\":\\"000000\\"}}]]", "clientid":{1}, "msg_id":{2}, "psessionid":"{3}"}}'.format(
                      uin, self.client_id, msg_id, self.psessionid, fix_content)),
             ('clientid', self.client_id),
             ('psessionid', self.psessionid)
         )
         rsp = self.client.post(req_url, data, self.smart_qq_refer)
         rsp_json = json.loads(rsp)
         if 'errCode' in rsp_json and rsp_json['errCode'] != 0:
             raise ValueError("reply pmchat error" + str(rsp_json['retcode']))
         logger.info("RUNTIMELOG Reply successfully.")
         logger.debug("RESPONSE Reply response: " + str(rsp))
         return rsp_json
     except:
         if fail_times < 5:
             logger.warning("RUNTIMELOG Response Error.Wait for 2s and Retrying." + str(fail_times))
             logger.debug("RESPONSE " + str(rsp))
             time.sleep(2)
             self.send_friend_msg(reply_content, uin, msg_id, fail_times + 1)
         else:
             logger.warning("RUNTIMELOG Response Error over 5 times.Exit.reply content:" + str(reply_content))
             return False
def run_pre_configuration_commands(itf=None):
    """runs iptables commands to enable vrrp"""
    command_templ_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), KEEPALIVED_PRE_COMMANDS_FILE)
    cmd_templ=None
    cmds=None
    if os.path.exists(command_templ_file):
        logger.info("command file exists")
    else:
        logger.debug("command file doesn't exist")
        exit(1)
    with open(command_templ_file, "r") as f:
        cmd_templ = Template(f.read())
    if itf:
        cmds = cmd_templ.substitute(interface=itf)
    else:
        logger.debug("Interface is None")
    if cmds:
        cmds = cmds.splitlines()
        for line in cmds:
            line.rstrip()
            try:
                cmd = line.split()
                print('cmd:', cmd)
                subprocess.check_call(cmd)
            except:
                logger.debug("ERROR running iptables command")
    else:
        logger.debug("ERROR getting cmds")
示例#18
0
    def get_friend_info(self, tuin):
        """
        获取好友详情信息
        get_friend_info
        {"retcode":0,"result":{"face":0,"birthday":{"month":1,"year":1989,"day":30},"occupation":"","phone":"","allow":1,"college":"","uin":3964575484,"constel":1,"blood":3,"homepage":"http://blog.lovewinne.com","stat":20,"vip_info":0,"country":"中国","city":"","personal":"","nick":" 信","shengxiao":5,"email":"*****@*****.**","province":"山东","gender":"male","mobile":"158********"}}
        :return:dict
        """

        uin = str(tuin)
        if uin not in self.friend_uin_list:
            logger.info("RUNTIMELOG Requesting the account info by uin: {}".format(uin))
            info = json.loads(self.client.get(
                'http://s.web2.qq.com/api/get_friend_info2?tuin={0}&vfwebqq={1}&clientid={2}&psessionid={3}&t={4}'.format(
                    uin,
                    self.vfwebqq,
                    self.client_id,
                    self.psessionid,
                    self.client.get_timestamp()
                )
            ))
            logger.debug("get_friend_info2 html: {}".format(str(info)))
            if info['retcode'] != 0:
                logger.warning('get_friend_info2 retcode unknown: {}'.format(info))
                return None
            info = info['result']
            info['account'] = self.uin_to_account(uin)
            info['longnick'] = self.get_friend_longnick(uin)
            self.friend_uin_list[uin] = info

        try:
            return self.friend_uin_list[uin]
        except:
            logger.warning("RUNTIMELOG get_friend_info return fail.")
            logger.debug("RUNTIMELOG now uin list:    " + str(self.friend_uin_list[uin]))
示例#19
0
 def send_group_msg(self, reply_content, group_code, msg_id, fail_times=0):
     fix_content = str(reply_content.replace("\\", "\\\\\\\\").replace("\n", "\\\\n").replace("\t", "\\\\t"))
     rsp = ""
     try:
         logger.info("Starting send group message: %s" % reply_content)
         req_url = "http://d1.web2.qq.com/channel/send_qun_msg2"
         data = (
             ('r',
              '{{"group_uin":{0}, "face":564,"content":"[\\"{4}\\",[\\"font\\",{{\\"name\\":\\"Arial\\",\\"size\\":\\"10\\",\\"style\\":[0,0,0],\\"color\\":\\"000000\\"}}]]","clientid":{1},"msg_id":{2},"psessionid":"{3}"}}'.format(
                      group_code, self.client_id, msg_id, self.psessionid, fix_content)),
             ('clientid', self.client_id),
             ('psessionid', self.psessionid)
         )
         rsp = self.client.post(req_url, data, self.smart_qq_refer)
         rsp_json = json.loads(rsp)
         if 'retcode' in rsp_json and rsp_json['retcode'] not in MESSAGE_SENT:
             raise ValueError("RUNTIMELOG reply group chat error" + str(rsp_json['retcode']))
         logger.info("RUNTIMELOG send_qun_msg: Reply '{}' successfully.".format(reply_content))
         logger.debug("RESPONSE send_qun_msg: Reply response: " + str(rsp))
         return rsp_json
     except:
         logger.warning("RUNTIMELOG send_qun_msg fail")
         if fail_times < 5:
             logger.warning("RUNTIMELOG send_qun_msg: Response Error.Wait for 2s and Retrying." + str(fail_times))
             logger.debug("RESPONSE send_qun_msg rsp:" + str(rsp))
             time.sleep(2)
             self.send_group_msg(reply_content, group_code, msg_id, fail_times + 1)
         else:
             logger.warning("RUNTIMELOG send_qun_msg: Response Error over 5 times.Exit.reply content:" + str(reply_content))
             return False
示例#20
0
    def uin_to_account(self, tuin):
        """
        将uin转换成用户QQ号
        :param tuin:
        :return:str 用户QQ号
        """
        uin_str = str(tuin)
        try:
            logger.info("RUNTIMELOG Requesting the account by uin:    " + str(tuin))
            info = json.loads(
                self.client.get(
                    'http://s.web2.qq.com/api/get_friend_uin2?tuin={0}&type=1&vfwebqq={1}&t={2}'.format(
                        uin_str,
                        self.vfwebqq,
                        self.client.get_timestamp()
                    ),
                    self.smart_qq_refer
                )
            )
            logger.debug("RESPONSE uin_to_account html:    " + str(info))
            if info['retcode'] != 0:
                raise TypeError('uin_to_account retcode error')
            info = info['result']['account']
            return info

        except Exception:
            logger.exception("RUNTIMELOG uin_to_account fail")
            return None
示例#21
0
    def get_online_friends_list(self):
        """
        获取在线好友列表
        get_online_buddies2
        :return:list
        """
        logger.info("RUNTIMELOG Requesting the online buddies.")
        response = self.client.get(
            'http://d1.web2.qq.com/channel/get_online_buddies2?vfwebqq={0}&clientid={1}&psessionid={2}&t={3}'.format(
                self.vfwebqq,
                self.client_id,
                self.psessionid,
                self.client.get_timestamp(),
            )
        )  # {"result":[],"retcode":0}
        logger.debug("RESPONSE get_online_buddies2 html:{}".format(response))
        try:
            online_buddies = json.loads(response)
        except ValueError:
            logger.warning("get_online_buddies2 response decode as json fail.")
            return None

        if online_buddies['retcode'] != 0:
            logger.warning('get_online_buddies2 retcode is not 0. returning.')
            return None

        online_buddies = online_buddies['result']
        return online_buddies
示例#22
0
def async_reader(conn, loop):
    data = conn.recv()
    logger.debug("RECV DATA: %s", data)
    if data is None:
        return

    if data == b'':
        if conn in FD_MAP:
            forward_server = FD_MAP[conn]
            loop.remove_reader(forward_server.fd)
            forward_server.shutdown()
            del FD_MAP[forward_server]
            del FD_MAP[conn]
        
        loop.remove_reader(conn.fd)
        conn.shutdown()
        return

    if conn not in FD_MAP:
        target_host = find_host(data)
        if target_host is None:
            return

        forward_server = ProxySocket.get_client(host=target_host, port=80)
        forward_server.connect()
        if forward_server and forward_server.fd != -1: 
            FD_MAP[conn] = forward_server
            FD_MAP[forward_server] = conn
            loop.add_reader(forward_server.fd, async_reader, forward_server, loop)
            logger.info("Create a connection to %s", target_host)
        else:
            logger.error("FAIL to connect to target host {0}".format(target_host))
    
    FD_MAP[conn].sendall(data)
示例#23
0
    def add_snapshot(self, name, ts_from, ts_to):
        logger.info("Adding snapshot: {}".format(name))

        url = self.base_url + "/add_snapshot/"
        data = {"cluster": self.settings.cluster, "name": name,
                "ts_from": ts_from, "ts_to": ts_to}
        self.post(url, data)
示例#24
0
 def __init__(self, Finder, Amapping, NSmapping, SOAmapping, servers):
     logger.info("[MapResolver] Init.");
     self.Finder = Finder
     self.Amapping = Amapping
     self.NSmapping = NSmapping
     self.SOAmapping = SOAmapping
     client.Resolver.__init__(self, servers=servers)
示例#25
0
文件: __init__.py 项目: oweidner/soma
def main():

    size = 10000           # lengt of vector v
    ITER = 50              # number of iterations to run for each report


    comm = MPI.COMM_WORLD
    # size = comm.Get_size()

    # Configure and connect to the Kafka data backbone server.
    # This is done only once by the rank 0 MPI process.
    #
    if comm.rank == MASTER:

        # Parse the command line
        #
        try:
            options = parse_commandline()

        except Exception, ex:
            logger.error("Command line parsing error: %s", str(ex))
            comm.Abort(-1)

        # Connect to Kafka
        #
        kw_cfg = KafkaWriterConfig(Host=options.KafkaHost, Topic=options.KafkaTopic)
        try:
            kw = KafkaWriter(kw_cfg)
            kw.connect()
            logger.info('connecting to Kafka backbone via %s', kw_cfg.Host)
        except Exception, ex:
            logger.error("Error connecting to Kafka backbone via %s: %s",  kw_cfg.Host, str(ex) )
            comm.Abort(-1)
示例#26
0
def upload_shot( screenshot ):
    while True:
        try:
            get_upload_form( screenshot )
            opened_file = open( screenshot.filename, 'rb' )
            file_length = len( open( screenshot.filename, 'rb').read() )
            logger.debug('%s: length %d' % (screenshot.filename, file_length ) )
            file_arg = {'file': opened_file }
            r=requests.post( 
                    url = screenshot.form.url, 
                    data=screenshot.form.payload, 
                    files=file_arg,
                    verify=False)
            opened_file.close()
            logger.debug(r.request.headers)
            logger.debug(r.headers)
            screenshot.uploaded = True
            logger.info("Done uploading %s"%(screenshot.filename) )
            return
        except AttributeError as e:
            logger.debug( e )
            logger.info( "Could not upload %s - need upload credentials" % screenshot.filename )
            logger.info( "Waiting 10 seconds then retrying upload")
            sleep(10)
        except requests.ConnectionError as e:
            logger.info( "Could not upload %s - connection error %s " % (screenshot.filename, e) )
            logger.info( "Waiting 10 seconds then retrying upload")
            sleep(10)
示例#27
0
 def start_server(self):
     logger.info('Starting Couchbase Server')
     getosname = run('uname -a|cut -c1-6')
     if(getosname.find("CYGWIN") != -1):
         run('net start CouchbaseServer')
     else:
         run('/etc/init.d/couchbase-server start')
    def __init__(self, resource_obj, ready_to_transfer_input_queue, 
        ready_to_exec_q, ready_to_transfer_output_q, done_q, failed_q):
        """DS
        """

        # Multiprocessing stuff
        multiprocessing.Process.__init__(self)
        self.daemon = True
        self._stop  = False

        # The resource object binds the worker to the public API & callbacks
        self._res_obj = resource_obj

        # BigJob handles
        self._pilot_job = None
        self._pilot_service = None

        self._physical_tasks = []

        # All queue an InputFileTransferWorker can access
        self._tasks_done_q = done_q
        self._tasks_failed_q = failed_q
        self._tasks_ready_to_exec_q = ready_to_exec_q
        self._tasks_ready_to_transfer_output_q = ready_to_transfer_output_q
        self._tasks_ready_to_transfer_input_q = ready_to_transfer_input_queue

        logger.info("Starting BigJobWorker using BigJob version %s" % pilot.version)
示例#29
0
def upload_forever():
    while True:
        screenshot = upload_queue.get()
        if screenshot == messages.QUIT:
            logger.info("Uploader is quitting...")
            sys.exit(0)
        upload_shot( screenshot )
示例#30
0
 def start_samplers(self):
     logger.info('Creating seriesly dbs')
     seriesly = Seriesly(host='{}'.format(self.test_config.gateload_settings.seriesly_host))
     for i, _ in enumerate(self.remote.gateways, start=1):
         seriesly.create_db('gateway_{}'.format(i))
         seriesly.create_db('gateload_{}'.format(i))
     self.remote.start_sampling()
示例#31
0
 def add_entry(self, key, service):
     if not self.get_entry(key):
         logger.info(f"DATABASE:ADD {key}, {service}")
         self.entries.append([key, json.dumps(service)])
示例#32
0
def get_s3_client():
    """Returns an S3 client, if you're credentials are not set via a .aws folder or other way set them here."""
    logger.info("Getting an S3 client...")
    s3_client = boto3.client('s3')

    return s3_client
示例#33
0
    def parse(self, response):
        logger.info("SCRAPING '%s' " % response.url)
        logger.info("RESPONSE: %s" % response.status)

        hxs = Selector(response)
        merchant = None

        try:
            if "amazon.co.uk" in response.url:
                merchant = AmazonUK(hxs, response.url)
            elif "amazon.com" in response.url:
                merchant = Amazon(hxs, response.url)
            elif "keds.com" in response.url:
                merchant = Keds(hxs, response.url)
            elif "6pm.com" in response.url:
                merchant = SixPM(hxs, response.url)
            elif "forever21.com" in response.url:
                merchant = ForeverTwentyOne(hxs, response.url)
            elif "harrods.com" in response.url:
                merchant = Harrods(hxs, response.url)
            elif "karmaloop.com" in response.url:
                merchant = Karmaloop(hxs, response.url)
            elif "adidas.com" in response.url:
                merchant = Adidas(hxs, response.url)
            elif "macys.com" in response.url:
                merchant = Macys(hxs, response.url)
            elif "eastbay.com" in response.url:
                merchant = Eastbay(hxs, response.url)
            elif "footaction.com" in response.url:
                merchant = FootAction(hxs, response.url)
            elif "walmart.com" in response.url:
                merchant = Walmart(hxs, response.url)
            elif "yoox.com" in response.url:
                merchant = Yoox(hxs, response.url)
            elif "zappos.com" in response.url:
                merchant = Zappos(hxs, response.url)
            elif "bowling.com" in response.url:
                merchant = Bowling(hxs, response.url)
            elif "nike.com" in response.url:
                merchant = Nike(hxs, response.url)
            elif "athleta.gap.com" in response.url:
                merchant = Athleta(hxs, response.url)
            elif "nordstrom.com" in response.url:
                merchant = Nordstrom(hxs, response.url)
            elif "sportsdirect.com" in response.url:
                merchant = SportsDirect(hxs, response.url)
            elif "carters.com" in response.url:
                merchant = Carters(hxs, response.url)
            elif "oshkosh.com" in response.url:
                merchant = OshKosh(hxs, response.url)
            elif "babytula.com" in response.url:
                merchant = BabyTula(hxs, response.url)
            elif "case-mate.com" in response.url:
                merchant = CaseMate(hxs, response.url)

            if merchant is not None:
                item = merchant.extractProductData()

            # Parse item
            _encoder = ScrapyJSONEncoder()
            jsonString = json.loads(_encoder.encode(item))
            keywords = jsonString['keywords']

            # Create a CSV file for product-category data
            # with open('train_data.csv', 'ab') as csvfile:
            #     trainwriter = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_ALL)
            #     for keyword in keywords:
            #         trainwriter.writerow([jsonString['category'], str(''.join(keyword['title'])).lower()])

            # Create a CSV file for training data
            with open('train_data.csv', 'ab') as csvfile:
                trainwriter = csv.writer(csvfile,
                                         delimiter=',',
                                         quoting=csv.QUOTE_ALL)
                trainwriter.writerow([
                    jsonString['category'],
                    str(jsonString['keywords']).lower()
                ])
            return item
        except Exception, e:
            print ' '
            print '--------------------BEGIN ERROR-------------------'
            print('Error: %s' % e.message)
            exc_type, exc_value, exc_traceback = sys.exc_info()
            traceback_details = {
                'filename': exc_traceback.tb_frame.f_code.co_filename,
                'lineno': exc_traceback.tb_lineno,
                'name': exc_traceback.tb_frame.f_code.co_name,
                'type': exc_type.__name__,
                'message': exc_value.message,
            }
            del (exc_type, exc_value, exc_traceback)
            print traceback.format_exc()
            print traceback_template % traceback_details
            print '--------------------END ERROR-------------------'
            print ' '
            return item
示例#34
0
l = val_label.asnumpy()
pred = []
ff = []
tt = []
fff = []
ttf = []
ttc = []
allf = []
the = 0.90
for i in range(len(p)):
    pp = p[i]
    mm = pp.argmax()
    if mm != l[i]:
        allf.append(pp[mm])
    if mm != l[i] and pp[mm] < the:
        ff.append(pp[mm])
    else:
        if pp[mm] < the:
            tt.append(pp[mm])
            ttc.append(mm)
logger.info(len(tt), len(ff), len(ttf), len(fff), len(allf))
logger.info((len(tt) + len(ff)) / res_final.shape[0])
logger.info((len(allf) - len(ff)) / res_final.shape[0])
preds = res_final.asnumpy()
preds = preds.argmax(axis=1)
from sklearn.metrics import confusion_matrix, classification_report, f1_score, recall_score, accuracy_score
cm = confusion_matrix(l, preds)
cm_norm = (cm / cm.sum(axis=1)[:, np.newaxis])
from pprint import pprint
logger.info(cm_norm)
logger.info(classification_report(y_pred=preds, y_true=l))
def predict():
    # initialize the data dictionary that will be returned from the
    # view
    data = {"success": False}
    dt = strftime("[%Y-%b-%d %H:%M:%S]")
    # ensure an image was properly uploaded to our endpoint
    if flask.request.method == "POST":
        request_json = flask.request.get_json()

        if request_json["embarked"]:
            embarked = request_json['embarked']

        if request_json["sex"]:
            sex = request_json['sex']

        if request_json["pclass"]:
            pclass = request_json['pclass']

        if request_json["age"]:
            age = request_json['age']

        if request_json["sibsp"]:
            sibsp = request_json['sibsp']

        if request_json["parch"]:
            parch = request_json['parch']

        if request_json["fare"]:
            fare = request_json['fare']

        if request_json["fullname"]:
            fullname = request_json['fullname']

        logger.info(
            f'{dt} Data: fullname={fullname} embarked={embarked}, sex={sex}, pclass={pclass}, age={age}, sibsp={sibsp}, parch={parch}, fare={fare}'
        )

        try:
            preds = model.predict(
                pd.DataFrame({
                    "Embarked": [embarked],
                    "Sex": [sex],
                    "Pclass": [pclass],
                    "Age": [age],
                    "SibSp": [sibsp],
                    "Parch": [parch],
                    "Fare": [fare]
                }))

        except AttributeError as e:
            logger.warning(f'{dt} Exception: {str(e)}')
            data['predictions'] = str(e)
            data['success'] = False
            return flask.jsonify(data)

        data["predictions"] = str(preds[0])

        # indicate that the request was a success
        data["success"] = True

    # return the data dictionary as a JSON response
    return flask.jsonify(data)
示例#36
0
文件: connection.py 项目: sammiej/vpn
    def recv(self, asBytes=False):
        if self.sessionKey:
            iv = self._recvBytes(self.blockSize)

            logger.info("{}: iv received: {}".format(prefix, iv))
            
            cipher = Cipher(algorithms.AES(self.sessionKey), modes.CBC(iv), backend=default_backend())
            decryptor = cipher.decryptor()

            ct = b''
            blk = self._recvBytes(self.blockSize)
            ct += blk
            dblk = decryptor.update(blk)
            gen = self._blkGenerator(dblk)            
            headerBytes, state = self._recvHeaderBytes(generator=gen)
            logger.debug("state: {}".format(state))
            while state != None:
                blk = self._recvBytes(self.blockSize)
                ct += blk
                dblk = decryptor.update(blk)
                gen = self._blkGenerator(dblk)
                temp, state = self._recvHeaderBytes(generator=gen)
                headerBytes += temp

            logger.debug("HeaderBytes: {}".format(headerBytes))
            header = Header(self.blockSize)
            header.update(headerBytes)
            body = b''
            try:
                while True:
                    body += next(gen)
            except StopIteration:
                pass

            leftoverBodySize = header.getSize() - len(body) + header.getPaddingSize()
            leftoverBodyEncrypted = self._recvBytes(leftoverBodySize)
            ct += leftoverBodyEncrypted
            body += decryptor.update(leftoverBodyEncrypted) + decryptor.finalize()

            logger.info("{}: cipher text received: {}".format(prefix, ct))
            
            checkHmac = self._recvBytes(self.hashSize)

            logger.info("{}: received cipher text hmac: {}".format(prefix, checkHmac))
            
            h = hmac.HMAC(self.sessionKey, hashes.SHA256(), backend=default_backend())
            h.update(ct)
            h.verify(checkHmac)

            logger.info("{}: complete received plain-text: {}".format(prefix, headerBytes + body))

            body = body[:len(body) - header.getPaddingSize()]
            if not asBytes:
                body = body.decode()
            return body                  
        else:
            header = Header()
            (headerBytes, _) = self._recvHeaderBytes()
            header.update(headerBytes)
            body = b''
            
            totalSize = header.getSize()
            while len(body) < totalSize:
                body += self.conn.recv(int(totalSize - len(body)))
            
            if not asBytes:
                return body.decode()
            return body
示例#37
0
 def remove_entry(self, key):
     entry = self.get_entry(key)
     if entry:
         logger.info(f"DATABASE:REMOVE {key}")
         self.entries.remove(entry)
示例#38
0
 def __on_event_callback(self, event_json):
     logger.info('Received event')
     decoded_event = json.loads(event_json)
     logger.info('Publishing event {}'.format(decoded_event))
示例#39
0
    def _estimate_internal(
            self, append_system_fn: Callable[['SystemTransition'],
                                             None]) -> None:
        count_triviality = 0
        count_with_unknowns = 0
        self.dump_system('start estimation')

        some_transitions_removed = True
        while some_transitions_removed:
            logger.debug("Start analyse from last transition")
            some_transitions_removed = False
            for x in range(len(self.__transitions) - 1, -1, -1):
                transition = self.__transitions[x]
                logger.debug("Analyse transition {}".format(transition))
                left = transition.get_left_side()
                right = transition.get_right_side()

                assert len(left) > 0 and len(right) > 0

                # 2 sides does not have unknowns
                if not left.contains_unknown() and not right.contains_unknown(
                ):
                    count_triviality += 1
                    logger.debug(
                        "Transition {} is triviality".format(transition))
                    continue

                is_left_non_zero = self._is_side_non_zero(left)
                is_right_non_zero = self._is_side_non_zero(right)

                if is_left_non_zero and is_right_non_zero:
                    logger.debug(
                        "Both sides '{}' and '{}' are not zero".format(
                            left, right))
                    # do nothing, just increase counter
                    count_with_unknowns += 1
                    continue
                elif is_left_non_zero and not is_right_non_zero:
                    logger.debug(
                        "Left side '{}' is NON ZERO. Rigth is undefined '{}'".
                        format(left, right))
                    # fixed left side - not fork
                    nz = Condition.create_non_zero_condition(right.copy())
                    logger.debug("Create non zero condition '{}'".format(nz))
                    self._conds_non_zero.append(nz)
                    count_with_unknowns += 1
                    continue
                elif not is_left_non_zero and is_right_non_zero:
                    logger.debug(
                        "Right side '{}' is NON ZERO. Left is undefined '{}'".
                        format(right, left))
                    # fixed right side - not fork
                    nz = Condition.create_non_zero_condition(left.copy())
                    logger.debug("Create non zero condition: '{}'".format(nz))
                    self._conds_non_zero.append(nz)
                    count_with_unknowns += 1
                    continue
                else:
                    fork = False
                    # both sides not zero
                    # check that they does not contain unkwowns
                    if not left.contains_unknown(
                    ) or not right.contains_unknown():
                        logger.info(
                            'Left or right sides does not contains UNKNOWN')
                        # need divide in two cases: zero and not zero
                        # zero case
                    else:
                        logger.info(
                            "Left and right contains UNKNOWN and sides in undefined. 'Fork' will processing"
                        )
                        fork = True

                    new_system = self.__clone(is_fork=fork)
                    # create non zero condition and add them to new system
                    # logger.debug("Creating new conditions for non zero case")
                    left_nzc = Condition.create_non_zero_condition(left.copy())
                    right_nzc = Condition.create_non_zero_condition(
                        right.copy())
                    logger.debug(
                        "New non zero conditions '{}' and '{}'".format(
                            left_nzc, right_nzc))
                    new_system._conds_non_zero.append(left_nzc)
                    new_system._conds_non_zero.append(right_nzc)
                    append_system_fn(new_system)
                    logger.debug("New system with id {} added to queue".format(
                        new_system._id))

                    # logger.debug("Creating new conditions for zero case")
                    left_zc = Condition.create_zero_condition(left.copy())
                    right_zc = Condition.create_zero_condition(right.copy())
                    logger.debug("New zero conditions '{}' and '{}'".format(
                        left_zc, right_zc))
                    self._use_and_append_zero_cond(left_zc)
                    self._use_and_append_zero_cond(right_zc)
                    some_transitions_removed = False
                    res = 1
                    while res > 0:
                        res = self._remove_empty_transitions()
                        res += self._analyse_and_generate_new_conditions()
                        some_transitions_removed |= bool(res)
                    if some_transitions_removed:
                        break
                    continue

        self._mark = None
        for tr in self.__transitions:
            if self._mark is None:
                self._mark = Symbol(tr.get_probability())
            else:
                self._mark *= Symbol(tr.get_probability())

        N = Symbol('N')
        amount = self._count_special_equal_conds()
        for i in range(amount):
            if self._mark is None:
                self._mark = N
            else:
                self._mark *= N
        if self._mark is not None:
            collector.make_node_leaf(self._node, self._mark)
            self.save_node(self._node)

        self.dump_system('Estimated with mark {}'.format(str(self._mark)))
        return
示例#40
0
def _lambda_handler(event, context):
    logger.info('Event:' + json.dumps(event))
    records = event['Records']
    now = datetime.datetime.utcnow()

    ddb_deserializer = StreamTypeDeserializer()
    cnt_insert = cnt_modify = cnt_remove = 0
    for record in records:
        # Handle both native DynamoDB Streams or Streams data from Kinesis (for manual replay)
        if record.get('eventSource') == 'aws:dynamodb':
            ddb = record['dynamodb']
            ddb_table_name = get_table_name_from_arn(record['eventSourceARN'])
            doc_seq = ddb['SequenceNumber']
        elif record.get('eventSource') == 'aws:kinesis':
            ddb = json.loads(base64.b64decode(record['kinesis']['data']))
            ddb_table_name = ddb['SourceTable']
            doc_seq = record['kinesis']['sequenceNumber']
        else:
            logger.error('Ignoring non-DynamoDB event sources: %s',
                         record.get('eventSource'))
            continue

        # Compute DynamoDB table, type and index for item
        doc_table = DOC_TABLE_FORMAT.format(
            ddb_table_name.lower())  # Use formatter
        doc_type = DOC_TYPE_FORMAT.format(
            ddb_table_name.lower())  # Use formatter
        doc_index = compute_doc_index(ddb['Keys'], ddb_deserializer)

        # Dispatch according to event TYPE
        event_name = record['eventName'].upper()  # INSERT, MODIFY, REMOVE

        items, LastEvaluatedKey = general_storage.scan_items(
            general_storage.get_dynamodb_table('client_configuration'),
            Attr('table_name').eq(doc_table))

        cf = general_config.create_configuration(items[0])

        # Treat events from a Kinesis stream as INSERTs
        if event_name == 'AWS:KINESIS:RECORD':
            event_name = 'INSERT'

        # Update counters
        if event_name == 'INSERT':
            cnt_insert += 1
        elif event_name == 'MODIFY':
            cnt_modify += 1
        elif event_name == 'REMOVE':
            cnt_remove += 1
        else:
            logger.warning('Unsupported event_name: %s', event_name)

        # If DynamoDB INSERT only, send 'item' to RDS
        if event_name == 'INSERT':
            if 'NewImage' not in ddb:
                logger.warning(
                    'Cannot process stream if it does not contain NewImage')
                continue
            # Deserialize DynamoDB type to Python types
            doc_fields = ddb_deserializer.deserialize({'M': ddb['NewImage']})

            # Now only store own post and replies
            if doc_fields['object_type'] == 'post' and str(
                    doc_fields['user_id']) != str(cf.twitter_user_id):
                continue

            # Now only store own post and replies
            if doc_fields['object_type'] == 'comment' and str(
                    doc_fields['asset_id']) != str(cf.twitter_user_id):
                continue

            # Normalize DynamoDB object to Mysql object and write to RDS
            normalizer_mysql.insert_dynamodb_item_into_mysql(cf, doc_fields)
示例#41
0
 def print(self, idx):
     logger.info("Sentence %d, Correctness: %.2f, Loss: %.2f",
                 idx, self.correct_count / self.total_count * 100,
                 self.total_loss)
     self.reset()
        #  logger.debug(features.get_params().keys())
        # pipe.fit(X_train, y_train)
        # scores = cross_val_score(pipe, X_train, y_train, cv=5, scoring='accuracy')
        # logger.info("Validation Accuracy: {:.3f} ± {:.3f}".format(np.mean(scores), 2 * np.std(scores)))

        # clf = pipe.steps[1][1]
        # self.print_importances(clf, X_train)
        # logger.info(format_as_text(explain_weights_lightgbm(lgb=clf, vec=features)))
        return pipe

    def predict(self, pipe, X_test, k=3):
        """

        :param pipe:
        :param X_test:
        :return:
        """
        y_test = pipe.predict(X_test)
        return y_test


if __name__ == '__main__':
    fn = "https://raw.githubusercontent.com/amueller/scipy-2017-sklearn/master/notebooks/datasets/titanic3.csv"
    model = NewPipeline()
    X_train, X_test, y_train, y_test = model.get_data(fn)
    pipe = model.get_pipe()
    pipe.fit(X_train, y_train)
    y_pred = pipe.predict(X_test)
    logger.info(accuracy_score(y_pred, y_test))

示例#43
0
    async def create(self, file: UploadFile, subdir: str = ''):
        '''Creates a file using UploadFile object from fastapi.

        Expected next arguments:

        :param file: An UploadFile object
        :type file: bytes
        
        :param subdir: The optional sub directory for the file.
        :type subdir: str

        :return: Status.done.value and name of the file, if the 
        file was created or status.exists.value and name of the file, 
        if it is already exist.
        :rtype: tuple(str, str)
        '''

        file_type = file.filename.split('.')[-1]

        temp_folder_name = 'tempfolder'
        temp_folder_path = os.path.join(subdir, temp_folder_name)
        logger.info('temp folder path:\n{0}\n'.format(temp_folder_path))
        temp_folder = Folder(temp_folder_path)
        temp_folder.prepare()

        temp_file_name = 'tempfile'
        temp_file_path = os.path.join(temp_folder_path, temp_file_name)
        logger.info('temp file path:\n{0}\n'.format(temp_file_path))
        temp_file = open(temp_file_path, 'wb')
        hash_code = md5()

        while True:

            chunk = await file.read(20000)

            if not chunk:
                break

            hash_code.update(chunk)
            temp_file.write(chunk)

        temp_file.close()
        the_hash = hash_code.hexdigest()
        filename = '.'.join([the_hash, file_type])
        sub_folder_name = filename[:2]
        sub_folder_path = os.path.join(subdir, sub_folder_name)
        logger.info('sub folder path:\n{0}\n'.format(sub_folder_path))

        # If the file already exists?
        if sub_folder_name in os.listdir(subdir):
            if filename in os.listdir(sub_folder_path):
                shutil.rmtree(temp_folder_path)
                return status.exists.value, filename

        # Rename the exiting temp file:
        renamed_file_path = os.path.join(temp_folder_path, filename)
        logger.info('renamed file path:\n{0}\n'.format(renamed_file_path))
        os.rename(os.path.join(temp_folder_path, temp_file_name),
                  renamed_file_path)

        # Rename the temp folder:
        shutil.move(temp_folder_path, sub_folder_path)
        logger.info('renamed dir path:\n{0}\n'.format(sub_folder_path))

        return status.done.value, filename
示例#44
0
    def _use_and_append_zero_cond(self, condition: Condition) -> None:
        self.__apply_condition(condition)
        self.dump_system("System after apply condition {}".format(condition),
                         with_main_conds=False)
        for nzcondition in self._conds_non_zero:
            if nzcondition.update_with(condition):
                self.dump_system(
                    "Conditions after apply condition {}".format(condition),
                    with_trans=False)
            if not nzcondition.is_correct():
                func = inspect.currentframe().f_back.f_code
                raise ConditionException(
                    "[{}:{}] Contradiction detected '{}'".format(
                        func.co_name, func.co_firstlineno, nzcondition))

        new_zero_conds = []
        useless_econds = []
        for econdition in self._conds_equals:
            if econdition.update_with(condition):
                self.dump_system(
                    "[_use_and_append_zero_cond] Conditions after apply condition {}"
                    .format(condition),
                    with_trans=False)
            if not econdition.is_correct():
                func = inspect.currentframe().f_back.f_code
                raise ConditionException(
                    "[{}:{}] Contradiction detected '{}'".format(
                        func.co_name, func.co_firstlineno, econdition))
            if econdition.get_state() == ConditionState.IS_ZERO:
                #  condition was converted
                new_zero_conds.append(econdition)
            if econdition.is_useless():
                useless_econds.append(econdition)
                logger.debug("useless: Condition {} true ".format(econdition))
            else:
                logger.debug("useless: Condition {} false ".format(econdition))

        if condition.get_state() == ConditionState.IS_EQUAL:
            self._conds_equals.append(condition)
        else:
            self._conds_zero.append(condition)
        self.dump_system("Added condition {}".format(condition),
                         with_trans=False)
        #  remove useless condition
        if len(useless_econds):
            for ucondition in useless_econds:
                self._conds_equals.remove(ucondition)
                logger.info(
                    "_use_and_append_zero_cond: remove useless condition from _conds_equals '{}'"
                    .format(ucondition))
            self.dump_system("Clear _conds_equals", with_trans=False)
        # remove and apply new zero conditions
        if len(new_zero_conds) > 0:
            for zcondition in new_zero_conds:
                self._conds_equals.remove(zcondition)
                logger.info(
                    "_use_and_append_zero_cond: remove new zero condition from _conds_equals '{}'"
                    .format(zcondition))
            self.dump_system("Clear _conds_equals 2", with_trans=False)
            for zcondition in new_zero_conds:
                self._use_and_append_zero_cond(zcondition)
 def connect_display(self, id):
     print(self.displays[id])
     logger.info("Connected to remote display " + str(id))
示例#46
0
    def measure(self, client, kvclient, metric, bucket):
        if self.n1ql_op == 'create':
            self.curr_items += 1
            key, ttl = self.new_keys.next(curr_items=self.curr_items)
            key = "stat" + key[5:]

        elif self.n1ql_op == 'delete' or self.n1ql_op == 'update':
            self.curr_items += 1
            key, ttl = self.new_keys.next(curr_items=self.curr_items)
            key = "stat" + key[5:]
            doc = self.new_docs.next(key)
            doc['key'] = key
            doc['bucket'] = bucket
            kvclient.create(key, doc)
            ddoc_name, view_name, query = self.new_queries.next(doc)
            _, latency = client.query(ddoc_name, view_name, query=query)
            return 1000 * latency  # s -> ms

        elif self.n1ql_op == 'rangeupdate':
            if self.smallcappedinit == False:
                logger.info(
                    "Initiating load for rangeupdate latency collection")
                for i in range(100):
                    key, ttl = self.new_keys.next(curr_items=self.curr_items)
                    key = "stat" + key[5:]
                    doc = self.new_docs.next(key)
                    doc['key'] = key
                    doc['bucket'] = bucket
                    doc['capped_small'] = "stat"
                    kvclient.create(key, doc)
                    self.curr_items += 1
                self.smallcappedinit = True
                logger.info(
                    "Completed load for rangeupdate latency collection")
                return 0
            key, ttl = self.new_keys.next(curr_items=self.curr_items)
            key = "stat" + key[5:]
            doc = self.new_docs.next(key)
            doc['capped_small'] = "stat"
            ddoc_name, view_name, query = self.new_queries.next(doc)
            query[
                'statement'] = "UPDATE `bucket-1` SET name = name||'' WHERE capped_small=$1;"
            del query['prepared']
            _, latency = client.query(ddoc_name, view_name, query=query)
            return 1000 * latency  # s -> ms

        elif self.n1ql_op == 'rangedelete':
            if self.smallcappedinit == False:
                logger.info(
                    "Initiating load for range update latency collection")
                for i in range(10000):
                    key, ttl = self.new_keys.next(curr_items=self.curr_items)
                    key = "stat" + key[5:]
                    doc = self.new_docs.next(key)
                    doc['key'] = key
                    doc['bucket'] = bucket
                    doc['capped_small'] = "stat" + str(i / 100)
                    kvclient.create(key, doc)
                    self.curr_items += 1
                self.smallcappedinit = True
                logger.info(
                    "Completed load for range delete latency collection")
                return 0
            key, ttl = self.new_keys.next(curr_items=self.curr_items)
            key = "stat" + key[5:]
            doc = self.new_docs.next(key)
            doc['capped_small'] = "stat" + str(self.cappedcounter)
            ddoc_name, view_name, query = self.new_queries.next(doc)
            _, latency = client.query(ddoc_name, view_name, query=query)
            self.cappedcounter += 1
            return 1000 * latency  # s -> ms

        elif self.n1ql_op == 'merge':
            doc = {}
            ddoc_name, view_name, query = self.new_queries.next(doc)
            _, latency = client.query(ddoc_name, view_name, query=query)
            flushpath = '/pools/default/buckets/bucket-2/controller/doFlush'
            self.post_http(path=flushpath)
            return latency  # s -> ms

        else:
            key = self.existing_keys.next(curr_items=self.items,
                                          curr_deletes=0)
        doc = self.new_docs.next(key)
        doc['key'] = key
        doc['bucket'] = bucket
        ddoc_name, view_name, query = self.new_queries.next(doc)

        _, latency = client.query(ddoc_name, view_name, query=query)
        return 1000 * latency  # s -> ms
示例#47
0
 def terminate(self):
     logger.info('Terminating Celery workers')
     local.kill_process('celery')
 def share_display(self, id):
     print(self.displays[id])
     logger.info("Shared display " + str(id))
示例#49
0
 def start(self):
     logger.info('Starting local Celery worker')
     local.start_celery_worker(queue=self.next_worker())
    def add_new_display(self):
        # cannot have more than 5 sessions
        if len(self.displays) >= 5:
            logger.warning("You have already 5 displays")
            return

        display_win = QDisplayDialog(list(self.displays.keys()))
        display_win.setModal(True)

        if display_win.exec() != 1:
            return

        display_hor_layout = QHBoxLayout()
        display_hor_layout.setContentsMargins(0, 2, 0, 2)
        display_hor_layout.setSpacing(2)

        display_ver_layout = QVBoxLayout()
        display_ver_layout.setContentsMargins(0, 0, 0, 0)
        display_ver_layout.setSpacing(0)

        display_ver_layout.addLayout(display_hor_layout)

        display_widget = QWidget()
        display_widget.setLayout(display_ver_layout)

        id = display_win.display_name
        print(id)

        name = QLabel()
        name.setText(str(id)[:16])
        display_hor_layout.addWidget(name)

        status = QLabel()
        status.setText("Pending...")
        display_hor_layout.addWidget(status)

        time = QLabel()
        time.setText("24H")
        display_hor_layout.addWidget(time)

        resources = QLabel()
        resources.setText("1 Node")
        display_hor_layout.addWidget(resources)

        connect_btn = QPushButton()
        connect_btn.setIcon(self.connect_ico)
        connect_btn.setToolTip('Connect to the remote display')
        connect_btn.clicked.connect(lambda: self.connect_display(id))
        display_hor_layout.addWidget(connect_btn)

        share_btn = QPushButton()
        share_btn.setIcon(self.share_ico)
        share_btn.setToolTip('Share the remote display via file')
        share_btn.clicked.connect(lambda: self.share_display(id))
        display_hor_layout.addWidget(share_btn)

        kill_btn = QPushButton()
        kill_btn.setIcon(self.kill_ico)
        kill_btn.setToolTip('Kill the remote display')
        kill_btn.clicked.connect(lambda: self.kill_display(id))
        display_hor_layout.addWidget(kill_btn)

        separator = QFrame()
        separator.setFrameShape(QFrame.HLine)
        separator.setFrameShadow(QFrame.Sunken)
        display_ver_layout.addWidget(separator)

        self.rows_ver_layout.addWidget(display_widget)

        self.displays[id] = display_widget
        logger.info("Added new display")
示例#51
0
 def wait_for_workers(self):
     logger.info('Waiting for all tasks to finish')
     for async_result in self.async_results:
         async_result.get()
     logger.info('All tasks are done')
示例#52
0
 def abort(self):
     logger.info('Interrupting Celery workers')
     os.kill(self.pid, signal.SIGTERM)
     self.wait_for_workers()
示例#53
0
 def terminate(self):
     logger.info('Terminating local Celery workers')
     with quiet():
         local('killall -9 celery')
         for db in self.SQLITE_DBS:
             local('rm -fr {}'.format(db))
示例#54
0
 def terminate(self):
     logger.info('Terminating Celery workers')
     if self.dynamic_infra:
         self.remote.terminate_client_pods(self.worker_path)
     else:
         self.remote.terminate_client_processes()
示例#55
0
 def get(self, q=""):
     self.q = q
     logger.info("query for '%s'" % q)
     self.set_header('Content-Type', 'application/json')
     self.write(json.dumps(self.get_papers()))
示例#56
0
 def start(self):
     logger.info('Initializing remote worker environment')
     if self.dynamic_infra:
         self.start_kubernetes_workers()
     else:
         self.start_remote_workers()
示例#57
0
    def update_offerings(self, slug, justwatch_id):
        if slug not in self.films:
            logger.warning('Could not update "%s", not in watchlist' % (slug))
            return None

        # Get offerings
        logger.info('Getting offerings for "%s" using JustWatch id=%s' %
                    (slug, justwatch_id))

        try:
            providers = {
                p['id']: p['clear_name']
                for p in self.justwatch.get_providers()
            }
            justwatch = self.justwatch.get_title(title_id=justwatch_id)
            print dumps(justwatch, indent=4)
            offers = justwatch.get('offers', [])
            justwatch_id = justwatch['id']
            justwatch_url = justwatch.get('full_paths',
                                          {}).get('MOVIE_DETAIL_OVERVIEW')
        except:
            logger.exception(
                'No offerings found for "%s" using JustWatch id=%s' %
                (slug, justwatch_id))
            return {}

        # if not offers:
        #     logger.error('No offerings found for "%s" using JustWatch id=%s' % (slug, justwatch_id))
        #     return {}

        # Parse JustWatch data
        try:
            # Offerings
            offerings = {}

            for offer in offers:
                if offer.get('provider_id') not in offerings:
                    offerings[offer.get('provider_id')] = {
                        'name': providers.get(offer.get('provider_id')),
                        'offers': [],
                        'offer_types': [],
                    }

                offerings[offer.get('provider_id')]['offers'].append({
                    'date_created':
                    offer.get('date_created'),
                    'monetization_type':
                    offer.get('monetization_type'),
                    'presentation_type':
                    offer.get('presentation_type'),
                    # 'provider_id': offer.get('provider_id'),
                    'urls':
                    offer.get('urls', {}),
                    'price':
                    offer.get('retail_price'),
                    'currency':
                    offer.get('currency'),
                })
                if offer.get('monetization_type') not in offerings[offer.get(
                        'provider_id')]['offer_types']:
                    offerings[offer.get('provider_id')]['offer_types'].append(
                        offer.get('monetization_type'))

            # Scoring
            tomato_id = None
            scoring = {}
            average_score = None
            scores = []

            for score in justwatch.get('scoring', []):
                if ':id' not in score['provider_type']:
                    key = score['provider_type'].replace(':', '_')
                    scoring[key] = score['value']

                    if key == 'imdb_score':
                        scores.append(float(score['value']))
                    if key == 'tmdb_score':
                        scores.append(float(score['value']))
                    if key == 'tomato_score':
                        scores.append((float(score['value']) / 10))
                    if key == 'metacritic_score':
                        scores.append((float(score['value']) / 10))

                if score['provider_type'] == 'tomato:id':
                    tomato_id = score['value']

            # Calculate average
            if len(scores) > 0:
                average_score = (float(sum(scores)) / len(scores))
                average_score = round(average_score, 2)

        except:
            logger.exception('Could not parse metadata for %s' % (slug))
            return {}

        # Update film
        logger.info('Updating offerings for "%s"' % (slug))

        self.films[slug]['ids']['justwatch'] = justwatch_id
        self.films[slug]['ids']['tomato'] = tomato_id
        self.films[slug]['offerings'] = offerings
        self.films[slug]['offerings_updated'] = time()
        self.films[slug]['offerings_updated_str'] = datetime.now().strftime(
            '%Y-%m-%d')
        self.films[slug]['justwatch_url'] = justwatch_url
        self.films[slug]['scoring'] = scoring
        self.films[slug]['scoring']['average'] = average_score
        self.save()

        return offerings
示例#58
0
def sendMsg(html, receiver_email, failure_num=1, is_path=True, is_send=True):
    """
    发送邮件
    :param html: 邮件正文用到的html文件路径,或者html
    :param receiver_email: 收件人邮箱地址的txt文件路径
    :param failure_num: 失败的用例数
    :param is_path: bool,True表示html是一个路径,False表示html是html
    :param is_send: bool,是否发邮件,仅用于第一次发送失败后,再次发送
    :return: 
    """
    flag = 0
    is_email = int(cfg.getConfig('is_email'))
    if is_email:
        if is_email == 1:
            flag = 1
        if is_email == 2:
            if failure_num > 0:
                flag = 1
            else:
                logger.info('所有用例执行成功,不发送邮件,已跳过。')
        if is_email == 3:
            if failure_num == 0:
                flag = 1
            else:
                logger.info('有执行失败的用例,不发送邮件,已跳过。')
    else:
        logger.info('设置为不自动发送邮件,已跳过。')

    if flag:
        try:
            receive_name = re.findall('email_(.*?).txt',
                                      receiver_email)[0]  # 提取收件人姓名
            with open(receiver_email, 'r', encoding='utf-8') as f:
                sends = f.readlines()
            subject = sends[0]
            send_to = sends[1]
            logger.info('开始发送邮件,收件人{}'.format(send_to))
            message = MIMEMultipart()
            message['From'] = Header(cfg.getConfig('sender_name'))  # 发件人名字
            message['To'] = Header(receive_name)  # 收件人名字
            message['Subject'] = Header(subject, 'utf-8')  # 邮件主题

            if is_path:
                with open(html, 'r', encoding='utf-8') as f:
                    fail_case = f.read()
            else:
                fail_case = html

            email_text = MIMEText(fail_case, 'html', 'utf-8')
            message.attach(email_text)  # 添加邮件正文

            try:
                server = smtplib.SMTP_SSL(cfg.getConfig('smtp'), 465)
                # server.connect(cfg.getConfig('smtp'))
            except Exception as err:
                logger.error(err)
                server = smtplib.SMTP(cfg.getConfig('smtp'), 25)
                # server.connect(cfg.getConfig('smtp'))

            server.login(cfg.getConfig('sender_email'), '123456')  # 登陆邮箱
            server.sendmail(cfg.getConfig('sender_email'), send_to.split(','),
                            message.as_string())  # 发送邮件
            server.quit()
            del fail_case, email_text, message, server
            logger.info('邮件发送成功')
        except Exception as err:
            logger.error(err)
            if is_send:
                sendMsg(html, receiver_email, is_path=is_path,
                        is_send=False)  # 发送失败后,重发一次
示例#59
0
    def update_metadata(self, slug, tmdb_id):
        if slug not in self.films:
            logger.warning('Could not update "%s", not in watchlist' % (slug))
            return None

        # Get metadata
        logger.info('Getting metadata for "%s" using TMDb id=%s' %
                    (slug, tmdb_id))

        details = self.tmdb.details(tmdb_id)

        if not details or details.get('status_code'):
            raise Exception('No metadata found for %s' % (slug))

        # Parse TMDb details
        try:
            # Details
            year = details.get('release_date')
            year = int(year.split('-')[0]) if year else None
            credits = details.get('credits', {})
            crew = credits.get('crew', [])

            metadata = {
                'title':
                details.get('title'),
                'original_title':
                details.get('original_title'),
                'year':
                year,
                'overview':
                details.get('overview'),
                'genres': [g['name'] for g in details.get('genres', [])],
                'runtime':
                details.get('runtime'),
                'original_language':
                details.get('original_language'),
                'spoken_languages':
                [l['name'] for l in details.get('spoken_languages', [])],
                'directors':
                [p['name'] for p in crew if p['job'] == 'Director'],
                'writers': [p['name'] for p in crew if p['job'] == 'Writer'],
            }

            # Images
            if details.get('backdrop_path') and not path.isfile(
                    path.join(BACKDROPS_PATH, '%s.jpg' % (slug))):
                try:
                    backdrop_url = TMDB_BACKDROP_URL % (
                        details.get('backdrop_path'))

                    logger.info('Fetching backdrop for "%s", url=%s' %
                                (slug, backdrop_url))

                    r = get(backdrop_url, stream=True)
                    r.raise_for_status()

                    with open(path.join(BACKDROPS_PATH, '%s.jpg' % (slug)),
                              'wb') as f:
                        r.raw.decode_content = True
                        copyfileobj(r.raw, f)
                except:
                    logger.exception('Could not save backdrop image')
            else:
                logger.warning('No backdrop found for "%s"' % (slug))
        except:
            logger.exception('TMDb parse error')
            raise Exception('Could not parse metadata for %s' % (slug))

        # Update film
        logger.info('Updating metadata for "%s"' % (slug))

        self.films[slug]['ids']['tmdb'] = details.get('id')
        self.films[slug]['ids']['imdb'] = details.get('imdb_id')
        self.films[slug]['metadata'] = metadata
        self.save()

        return metadata
示例#60
0
    def sync(self):
        logger.info('Syncing watchlist')

        results = {'new': {}, 'removed': []}

        # Fetch Letterboxd watchlist
        logger.info('> Existing films: %d' % (len(self.films.keys())))

        lb_watchlist = self.letterboxd.watchlist()

        logger.info('> Got %d films from Letterboxd' %
                    (len(lb_watchlist.keys())))

        logger.info('Updating watchlist')

        for slug, metadata in lb_watchlist.iteritems():
            if slug in self.films:
                # Update
                self.films[slug]['ids']['letterboxd'] = metadata['id']
                self.films[slug]['title'] = metadata['title']
                # self.films[slug]['year'] = metadata['year']

            else:
                # Create
                self.films[slug] = {
                    'ids': {
                        'letterboxd': metadata['id']
                    },
                    'title': metadata['title'],
                    # 'year': metadata['year'],
                }

                results['new'][slug] = self.films[slug]

                logger.info('> Added %s' % (slug))

        # Find removed
        removed = [
            f for f in self.films.keys() if f not in lb_watchlist.keys()
        ]

        for slug in removed:
            logger.info('> Removed %s' % (slug))

            del self.films[slug]

        results['removed'] = removed

        # Save
        self.save()

        return results