예제 #1
0
def test_set_info():
    file = next(get_file(name="Eric", unit_test=True))
    info = get_info(file)
    new_info = str(np.random.rand())
    info["test"] = new_info
    set_info(file, info)
    cur_info = get_info(file)
    assert cur_info["test"] == new_info
예제 #2
0
def home(id):
    info = utils.get_info(id)
    choices1 = info[1].split("##")
    return render_template("poll.html",
                           sname=info[3],
                           creator=info[2],
                           choices=choices1)
예제 #3
0
def tally_group(bot, update, args):
    user_name, user_id, group = utils.get_info(update)

    data = {
        'group': group,
        'set_date': False if len(args) == 0 else True
    }
    r = requests.post(API_URL + 'tally_group', json=data)

    if r.status_code == 200:
        users = r.json()
        output = ''
        total = 0
        most_social = ''
        least_portions = sys.maxsize
        if len(users) == 0:
            bot.sendMessage(group, 'No information to display')
            return

        for user, portions in users.items():
            output += user + ' - ' + str(portions) + '\n'
            total += portions
            if portions < least_portions:
                # Need to handle the case where multiple people tie
                most_social = user
                least_portions = portions
        output += str(total) + ' portions consumed\n'
        output += most_social + ' has the most life'

        bot.sendMessage(group, text=output)
    else:
        bot.sendMessage(group, text='Unable to retrieve information')
예제 #4
0
def get_start_end(file):
    print(utils.get_name(file))
    result = {}
    fitfile = FitFile(file)

    # Get all data messages that are of type record
    for i, record in enumerate(fitfile.get_messages("record")):
        # Go through all the data entries in this record
        for record_data in record:
            if not str(record_data.name) in list(result.keys()):
                result[str(record_data.name)] = {}

            result[record_data.name][i] = record_data.value
        result = fill_na(result, i)

    df = pd.DataFrame().from_dict(result)
    good = False
    while not good:
        ax = df.loc[:, ["cadence", "power", "heart_rate"]].plot()
        plt.show()
        start = int(input("start? "))
        end = int(input("end? "))
        ax = df.loc[start:end, ["cadence", "power", "heart_rate"]].plot()
        plt.show()
        good = input("good? (y/n) ") == "y"
    info = utils.get_info(file)
    info["start(sec)"] = str(start)
    info["end(sec)"] = str(end)
    utils.set_info(file, info)
    df.to_csv(file.replace(".fit", ".csv"))
    sns.lmplot(x="power", y="heart_rate", data=df)
예제 #5
0
def calculate(expr):
    stack = []
    n = len(expr)
    for cur in range(n):
        element = expr[cur]
        if isinstance(element, list):
            neg = element[1]
            num, posting_head = utils.get_info(element)
            vec = list2vec(neg, posting_head)
            stack.append(vec)
        else:
            if expr[cur] == 3:
                op = stack.pop()
                stack.append(calc_not(op))
            else:
                op1 = stack.pop()
                op2 = stack.pop()
                res = None
                if expr[cur] == 2:
                    res = calc_and(op1, op2)
                else:
                    res = calc_or(op1, op2)
                stack.append(res)
    assert(len(stack) == 1)
    # print(stack[0])
    return stack[0]
예제 #6
0
def meal_info(bot, update, args):
    user_name, user_id, group = utils.get_info(update)

    # Argument handling
    meal_id = -1
    if len(args) == 1:
        meal_id = args[0]

    # Request
    data = {
        'group': group,
        'meal_id': meal_id
    }
    r = requests.post(API_URL + 'meal_info', json=data)

    if r.status_code == 200:
        meal_participations = r.json()
        output = ''
        total_portions = 0
        if len(meal_participations) == 0:
            bot.sendMessage(group, 'No information to display')
            return

        for mp in meal_participations:
            output += str(mp['user_name']) + ' '
            output += str(mp['portions']) + (' portion' if mp['portions'] == 1 else ' portions')
            output += ' - cooking' if mp['cooked'] else ''
            output += '\n'
            total_portions += mp['portions']
        output += str(total_portions) + (' portion' if total_portions == 1 else ' portions') + ' required'
        bot.sendMessage(group, text=output)
    else:
        bot.sendMessage(group, text='Unable to retrieve meal information')
예제 #7
0
def fetch_info(request):
    if "url" in request.GET:
        info = get_info(request.GET['url'])
        if not info:
            return HttpResponse(status=404)
        dummy_link = {
            "url": info['url'],
            "thumbnail_url": info.get("image"),
            "title": info.get("title"),
            "description": info.get("description"),
            "player": info.get("player")
        }
        return HttpResponse(
            simplejson.dumps({
                "html": render_to_string(
                    "links/attachment.html",
                    {
                        "link": dummy_link,
                        "attachment_editable": True
                    }
                ),
                "info": info
            }), 'application/javascript'
        )
    else:
        return HttpResponse(status=400)
예제 #8
0
    def card(self, data, request):
        """
            Save card information's to elastic search cluster
        """
        # get the ip from the request and make it the id
        _id = request.access_route[0]

        # check if the documents exists
        if self.es.exists(id=_id, index=INDEX_CARD, doc_type=DOC_CARD):
            # the documents is already exists , only update document
            self.es.update(id=_id,
                           index=INDEX_CARD,
                           doc_type=DOC_CARD,
                           body={"doc": data})
        else:
            # the document not exists , now we created it.
            data.update(get_info(request))

            # add timestap to the data
            data['timestamp'] = datetime.now()

            # create document and save data to cluster
            self.es.index(id=_id,
                          index=INDEX_CARD,
                          doc_type=DOC_CARD,
                          body=data)
예제 #9
0
def define_words(a_list):
    '''
    :param a_list: a list of words
    :return: alist of semicolon separated information about each word
        - word, type of word, example usage
    '''
    a_result = []
    with open('words_wiki_500.txt', 'w') as out_file:
        for word in a_list:
            '''stahni stranku z cambridge
               najdi jednotlive casti pomoci regexpu
               sloz vysledek
               pridej do resultu
            '''
            clean_word = word.strip()
            logger.debug('word: %s' % clean_word)

            utils.download(get_page(clean_word), get_file_name(clean_word),
                           logger)

            word_type = utils.get_info(
                get_file_name(clean_word),
                'span class="headword">.*?%s.*?span class="pos".*?>(.*?)<' %
                clean_word, logger)
            out_line = '%s\t%s\n' % (clean_word, word_type)
            logger.debug(out_line)
            out_file.write(out_line)
    out_file.close()
예제 #10
0
파일: main.py 프로젝트: init10xa/ParserBot
def announce():
    # Get all news from site.
    fresh_news = utils.get_info(fpmi_url)
    # Load dumped news from .pkl file.
    loaded_news = utils.load_news()
    # Find difference between fresh and dumped news.
    news_list = utils.diff_news(fresh_news, loaded_news)

    news_list = utils.get_sorted(news_list)

    # If there is at least one fresh announcement -- post it.
    if len(news_list) == 0:
        print('-> nothing new :(')
    else:
        print('-> announce')
        to_announce = news_list[0]
        try:
            if to_announce not in loaded_news:
                text = utils.compose_announcement(to_announce)
                # Make announcement.
                bot.send_message(chat_id=channel_id, text=text, parse_mode=telegram.ParseMode.MARKDOWN, timeout=9999)
                # Save updated news.

                loaded_news.append(to_announce)
                print('-> appended')
                utils.save_news(loaded_news)
                print('-> done')
            else:
                print('-> duplicate :(')
        except telegram.error.TimedOut as e:
            print('-> skipping this event due to Timed Out')
            print(e)
예제 #11
0
def change_portions(bot, update, args):
    user_name, user_id, group = utils.get_info(update)

    meal_id = None
    portions = None

    if len(args) == 2:
        meal_id = int(args[0])
        portions = int(args[1])
    else:
        utils.throw_error(bot, group, errors.WRONG_N_ARGS)
        return

    data = {
        'user_id': user_id,
        'meal_id': meal_id,
        'portions': portions
    }

    r = requests.post(API_URL + 'change_portions', json=data)

    if r.status_code == 200:
        bot.sendMessage(group, text='Update successful')
    else:
        bot.sendMessage(group, text='Update unsuccessful')
예제 #12
0
def get_meals(bot, update, args):
    user_name, user_id, group = utils.get_info(update)

    # Argument handling
    number = 5
    if len(args) > 0:
        number = args[0]

    data = {
        'group': group,
        'number': number
    }
    r = requests.post(API_URL + 'meals', json=data)

    if r.status_code == 200:
        meals = r.json()
        output = ''
        if len(meals) == 0:
            bot.sendMessage(group, 'No information to display')
            return

        for meal in meals:
            output += 'ID: ' + str(meal[0]) + '     '
            output += 'Portions: ' + str(meal[1])
            output += '\n'
        output += str(len(meals)) + ' meals retrieved'
        bot.sendMessage(group, text=output)
    else:
        bot.sendMessage(group, text='Unable to retrieve information')
예제 #13
0
def define_words(a_list):
    """
    :param a_list: a list of words
    :return: alist of semicolon separated information about each word
        - word, type of word, example usage
    """
    a_result = []
    with open("words_wiki_500.txt", "w") as out_file:
        for word in a_list:
            """stahni stranku z cambridge
               najdi jednotlive casti pomoci regexpu
               sloz vysledek
               pridej do resultu
            """
            clean_word = word.strip()
            logger.debug("word: %s" % clean_word)

            utils.download(get_page(clean_word), get_file_name(clean_word), logger)

            word_type = utils.get_info(
                get_file_name(clean_word),
                'span class="headword">.*?%s.*?span class="pos".*?>(.*?)<' % clean_word,
                logger,
            )
            out_line = "%s\t%s\n" % (clean_word, word_type)
            logger.debug(out_line)
            out_file.write(out_line)
    out_file.close()
예제 #14
0
def start_meal(bot, update, args):
    user_name, user_id, group = utils.get_info(update)

    # TODO: Check the number of args

    if len(args) == 0:
        bot.sendMessage(group, text=user_name + ", please indicate meal type, for example: '/start lunch'")
        return
    else:
        meal_type = args[0].strip()

    # Request
    data = {
        'group': group,
        'meal_type': meal_type
    }
    r = requests.post(API_URL + 'add_meal', json=data)

    # Response handling
    if r.status_code == 200:
        data2 = {
            'user_id': user_id,
            'group': group,
            'portions': 1,
            'cooked': True
        }
        r2 = requests.post(API_URL + 'eating', json=data2)
        bot.sendMessage(group, text='Thanks for cooking, ' + user_name + '! You are eating 1 portion')
        bot.sendMessage(group, text='Who is eating ' + meal_type + '?')
    else:
        bot.sendMessage(group, text='Another meal is running / unable to start meal')
예제 #15
0
    def run_experiment(self):
        if self.onEuler:
            self.run_euler_experiment(self.nethz_username)
            if self.nethz_username == 'xxx':
                self.processs_euler_experiment()
            return

        self.re_init_data()
        files = sorted([
            os.path.join(CNF_FOLDER, f) for f in os.listdir(CNF_FOLDER) if
            os.path.isfile(os.path.join(CNF_FOLDER, f)) and f.endswith('.cnf')
        ])
        for s in EXECUTABLES:
            exe = os.path.join(parent_parent, "./{}_main".format(s))
            delete_files_in_folder(CNF_FOLDER, 'time')
            self.data[s] = {}
            for f in files:
                print('main: {}  file: {}'.format(s, f))
                self.data[s][f] = {}
                self.data[s][f]['info'] = get_info(f).__dict__
                command = 'gtimeout {} mpirun -np 4 {} {}'.format(
                    TIMEOUT, exe, f)
                run_n_times(command, REPETITIONS)
                timing_file = f[:-3] + 'time'
                times = get_results(timing_file)
                self.data[s][f]['time'] = times
예제 #16
0
def add_weight(file):
    info = utils.get_info(file)
    kg = input("wat is het gewicht van " + utils.get_name(file) + " ? ")

    if str(kg) == "0":
        kg = "Onbekend"
    info["gewicht(kg)"] = kg
    utils.set_info(file, info)
예제 #17
0
def get_info():
    info = dict()
    info["os_info"] = u.get_info()
    info["disk_info"] = u.get_disk_info()
    info["virtual_memory_info"] = u.get_virtual_memory_info()
    info["cpu_usage"] = u.psutil.cpu_percent()
    info["processes"] = u.get_process_info()
    info["timestamp"] = str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
    return info
예제 #18
0
    def _show_info(self, widget):
        desc, teacher, mimetype =\
          utils.get_info(self._sftp,
                         self._subject, widget.title)

        dialog = InfoDialog(widget.title, desc, teacher,
                            self._subject, mimetype)
        dialog.connect('save-document', lambda w: utils.save_document(
                self._sftp, self._subject, widget.title, mimetype))
예제 #19
0
def gender(file):
    info = utils.get_info(file)
    while "sex" not in info:
        sex = input("what is the sex of " + utils.get_name(file) + "? (m/w) ")
        if sex == "m" or sex == "w":
            info["sex"] = sex
        else:
            print("please enter m or w ")
    utils.set_info(file, info)
예제 #20
0
    def echo(self, bot, update):
        user_name, user_id, group = utils.get_info(update)
        text = update.message.text
        parsed_text = self.en_nlp(text)
        output = ''
        for token in parsed_text:
            output += token.orth_ + ' ' + token.pos_ + ' ' + token.lemma_ + '\n'

        if output:
            bot.sendMessage(group, text=output)
예제 #21
0
 def test_get_info():
     mapping = {'name': '.test1', 'value': '.asdfg'}
     default = None
     output = u.get_info(basic_soup, mapping)
     expected_output = {}
     for key, selector in mapping.items():
         expected_output[key] = u.get_first_element(basic_soup,
                                                    selector,
                                                    default=default)
     assert output == expected_output
예제 #22
0
def get_flag(user, priv_cubes):
    for cube in priv_cubes:
        r = remote(*ADDR)
        try:
            login(r, user, cube)
        except Exception:
            r.close()
            continue
        return get_info(r)
        _exit(r)
        r.close()
    print(user, 'Not found')
예제 #23
0
def register_group(bot, update):
    user_name, user_id, group = utils.get_info(update)

    # Server request
    data = {
        'group': group
    }
    r = requests.post(API_URL + 'register_group', json=data)

    # Response handling
    if r.status_code == 200:
        bot.sendMessage(group, text='Group registered')
    else:
        bot.sendMessage(group, text='Group already registered / registration failed')
예제 #24
0
    def processs_euler_experiment(self):
        self.re_init_data()
        files = sorted([
            os.path.join(CNF_FOLDER, f) for f in os.listdir(CNF_FOLDER) if
            os.path.isfile(os.path.join(CNF_FOLDER, f)) and f.endswith('.cnf')
        ])
        for s in EXECUTABLES:
            self.data[s] = {}
            for f in files:
                time_file = f.replace('.cnf', '_{}.time'.format(s))
                print('main: {}  file: {}'.format(s, time_file))

                self.data[s][f] = {}
                self.data[s][f]['info'] = get_info(f).__dict__
                times = get_results(time_file)
                self.data[s][f]['time'] = times
예제 #25
0
def end_meal(bot, update, args):
    user_name, user_id, group = utils.get_info(update)

    # Request
    data = {
        'group': group
    }
    r = requests.post(API_URL + 'end_meal', json=data)

    # Response handling
    # Do not end the meal immediately, should trigger some handlers for warnings
    # E.g. 15min / 10min
    if r.status_code == 200:
        bot.sendMessage(group, text='Meal ended')
    else:
        bot.sendMessage(group, text='Unable to end meal')
예제 #26
0
    def analytics(self, data, request):
        """
            Save analytics metrics to elastic search cluster
        """

        # get the ip of client make the request
        # and make it the id of document
        _id = request.access_route[0]

        # get request type from data
        request_type = data['type']

        # check if the documents exists
        if self.es.exists(id=_id, index=INDEX_ANALYTICS, doc_type=INDEX_ANALYTICS):
            # the documents is already exists we make only update
            # for request_type field
            self.es.update(
                id=_id,
                index=INDEX_ANALYTICS,
                doc_type=INDEX_ANALYTICS,
                body={"doc": {
                    request_type: datetime.now(),
                    "page": data['page']
                }}
            )
        else:
            # the document is not exists we create a new document
            # and we save it.
            doc = {
                "timestamp": datetime.now(),
                request_type: datetime.now(),
                "page": data['page']
            }
            # add extra information's to the document, like
            # browser type and platform , and geo localisation of
            # the ip address
            doc.update(get_info(request))

            # create document in elasticsearch database
            self.es.index(
                id=_id,
                index=INDEX_ANALYTICS,
                doc_type=INDEX_ANALYTICS,
                body=doc

            )
예제 #27
0
def register(bot, update):
    user_name, user_id, group = utils.get_info(update)

    # Server request
    data = {
        'id': user_id,
        'username': user_name,
        'groupID': group
    }
    r = requests.post(API_URL + 'register', json=data)

    print(r.text)

    # Response handling
    if r.status_code == 200:
        bot.sendMessage(group, text=user_name + ': Successful registration')
    else:
        bot.sendMessage(group, text=user_name + ': Registration failed / already registered')
예제 #28
0
def featurize(fn):
    savep = "{}/{}".format(path.basename(path.dirname(fn)), path.basename(fn))
    savep = path.join(base, savep) + '.npz'
    os.makedirs(path.dirname(savep), exist_ok=True)
    if args.skip_existing and path.exists(savep):
        logging.info("Found existing {}, skipping".format(savep))
        return True

    if not os.path.exists(fn):
        raise RuntimeError("No such replay")
    rep = replayer.load(fn)
    map = rep.getMap()
    map_size = map['walkability'].shape
    sl = make_start_loc(map['start_locations'], map_size)

    batch = []
    for i in range(0, len(rep)):
        batch.append(rep.getFrame(i))
    reduced = featurizer.reduce(batch)
    map_features = np.stack([
        map['walkability'],
        map['buildability'],
        map['ground_height'],
        sl,
    ],
                            axis=0).astype(dtype='uint8')

    info_hash = chain(*get_info(rep))  # 6 ints, p0race, p0slx, p0sly, p1...
    map_hash = hash_map(map)  # 4 ints for the map hash

    info = np.asarray(list(chain(info_hash, map_hash)), dtype=np.int32)

    mfdata = io.BytesIO()
    with gzip.GzipFile(fileobj=mfdata, mode='w') as f:
        np.save(f, map_features)
    to_save = {
        'reduced': reduced,
        'map': mfdata.getvalue(),
        'info': info,
    }
    # TODO: Both reduced data and overall data may be compressed
    np.savez(savep, **to_save)

    return True
예제 #29
0
def process(user, message):

	user_id, text = ut.get_info(user, message)

	if "start" in text and "workout" in text:

		#=====[ Start guided workout ]=====
		if "guided" in text:
			workout_template = True
			workout_log.start(user, workout_template)

		#=====[ Manually log workout ]=====
		else:
			workout_log.start(user)

	#=====[ Send default message ]=====
	else:

		ut.send_response(DEFAULT_IDLE_MESSAGE, user_id)
예제 #30
0
def index():
    form = NameForm()
    message = ""
    if form.validate_on_submit():
        option = int(form.language.data)
        main_uri = get_info()['containers']['MainController']['uri']
        obj = Pyro4.Proxy(main_uri)
        file = form.input_data.data
        file_path = path.join(get_shared_folder(), 'input.json')
        file.save(file_path)
        try:
            response = obj.run(action=option, input_json=file_path)
            print(response)
            # response_path = save_json(response, path.join(get_shared_folder(), 'response.json'))
        except Exception as e:
            return render_template('index.html', form=form, message=str(e), async_mode=socketio.async_mode)
        return send_file(response,
                         attachment_filename="response.json", mimetype="text/json", as_attachment=True)

    return render_template('index.html', form=form, message=message, async_mode=socketio.async_mode)
예제 #31
0
def tally_user(bot, update, args):
    user_name, user_id, group = utils.get_info(update)

    data = {
        'group': group,
        'user_id': user_id
    }
    r = requests.post(API_URL + 'tally_user', json=data)

    if r.status_code == 200:
        output = ''
        meals = r.json()
        print(meals)

        for meal in meals:
            output += meal['type'] + ' ' + meal['date'] + '\n'
        output += 'Involved in ' + str(len(meals)) + ' meals'
        bot.sendMessage(group, text=output)
    else:
        bot.sendMessage(group, text='Unable to retrieve information')
예제 #32
0
def get_pharma_info(drugName, clusterEndpoint=CLUSTER_CONFIG_ENDPOINT):
    # Retrieves pharmaceutical usage information from cache for the given drug
    # name. Set the cache if not available
    pharma_info = None
    try:
        mclient = memcache.Client([clusterEndpoint], debug=0)
        pharma_info = get_cache_item(mclient, drugName)
        if not pharma_info:
            pharma_info = dynamoDB_manager.get_info(drugName)
            if not pharma_info:
                print("Given drug name not available in the table")
                return None
            set_pharma_item(mclient, drugName, pharma_info)
            pharma_info = get_cache_item(mclient, drugName)
            if not pharma_info:
                print(
                    "Unable to set the cache for the given DrugName:{0}".format(drugName))
    except Exception as err:
        print("Error message: {0}".format(err))
    return pharma_info
    def featurize(fn):
        raise RuntimeError("This doesn't work anymore, add in game name, visibility")
        rep = replayer.load(fn)
        map = rep.getMap()
        map_size = map['walkability'].shape
        race = [x[0] for x in get_info(rep)]
        sl = make_start_loc(map['start_locations'], map_size)

        batch = []
        fromt = args.from_time if args.until_time != 0 else 0
        until = args.from_time if args.until_time != 0 else len(rep)
        for i in range(fromt, min(len(rep), until), args.skip_frames):
            frames = [rep.getFrame(k) for k in
                      range(i, min(i + args.combine_frames, len(rep)))]
            batch.append(frames)

        featurized = (
            # map
            th.from_numpy(
                np.stack([
                    map['walkability'],
                    map['buildability'],
                    map['ground_height'],
                    sl,
                ], axis=0)[np.newaxis, :]).type(th.FloatTensor),
            # race
            th.LongTensor([race if pers == 0 else list(reversed(race))]),
            # input
            th.from_numpy(
                normalize(featurizer.featurize(
                    batch, map_size, perspective=pers
                ).transpose(0, 3, 1, 2))[:-1]).type(th.FloatTensor),
            # targets
            th.from_numpy(
                normalize(featurizer.featurize(
                    batch, map_size, perspective=pers, full=True
                ).transpose(0, 3, 1, 2))[1:]).type(th.FloatTensor),
        )
        assert featurized[2].size(0) > 1

        return featurized
예제 #34
0
파일: views.py 프로젝트: BDGL-Hacks/Docker
def container_details(request):
	if request.method == 'GET':
		if 'id' in request.GET:
			if 'save' in request.GET:
				# Call function to convert existing container to image
				utils.branch_container(request.GET.get('id'), request.GET.get('id')+"branch")
				return HttpResponseRedirect('/manager/images/')
			elif 'start' in request.GET:
				# Call function to start a stopped or paused container
				utils.resume_container(request.GET.get('id'))
				pass
			elif 'pause' in request.GET:
				# Call function to pause a started container
				utils.stop_container(request.GET.get('id'))
				pass
			elif 'stop' in request.GET:
				# Call function to stop a started or paused container
				utils.remove_container(request.GET.get('id'))
				pass

			# Get detailed information for container
			info = utils.get_info(request.GET['id'])[0]

			# Most useful information
			container_details = {}
			container_details['cpu_shares'] = info['Config']['CpuShares']
			container_details['memory'] = info['Config']['Memory']
			container_details['memory_swap'] = info['Config']['MemorySwap']
			container_details['created_time'] = utils.convert_time(info['Created'])
			container_details['id'] = info['Id'][:12]  		# Use first 12 digits
			container_details['image'] = info['Image'][:12] # Use first 12 digits
			container_details['name'] = info['Name'][1:]	# First char is always '/'
			container_details['ip'] = info['NetworkSettings']['IPAddress']
			container_details['is_running'] = info['State']['Running']
			container_details['start_time'] = utils.convert_time(info['State']['StartedAt'])
			container_details['is_paused'] = info['State']['Paused']
			container_details['finish_time'] = utils.convert_time(info['State']['FinishedAt'])

			return render_to_response('manager/container_details.html', { 'details': container_details, 'id': request.GET['id'] })

	return HttpResponseRedirect('/manager/containers/')
def crawl_by_category(cateName):
    day = 1
    data = []
    while True:
        # print(day)
        text = make_request(generate_url(cateName, day))
        # print(text)
        article_rg_str = '(?=<h3).*?(?<=</h3>)'
        article_regex = re.compile(article_rg_str, flags=re.I | re.S)
        articles = re.findall(article_regex, text)
        if day >= 30:
            break
        if not articles:
            day += 1
            continue

        for article in articles:
            content = get_info(article)
            content["category"] = cateName
            data.append(content)
        day += 1
    return data
예제 #36
0
 def set_path(self, sftp, subject="."):
     self.clear()
     self._sftp = sftp
     self._subject = subject
     self._documents = []
     if not utils.get_documents(sftp, subject):
         label = gtk.Label(
             '<span font_desc="12"><i>%s</i></span>' % 'No hay documentos')
         label.set_use_markup(True)
         self._vbox.pack_start(label, False, True, 5)
     for document in utils.get_documents(sftp, subject):
         mimetype = utils.get_info(self._sftp,
                                   self._subject,
                                   document,
                                   only_mime=True)
         item = ListItem(document, mimetype)
         item.connect("show-info", self._show_info)
         item.connect("selected", self._selected)
         item.connect("unselected", self._unselected)
         self._vbox.pack_start(item, False, True, 5)
         item.show()
     self._activity.show_all()
예제 #37
0
def meal_participation(bot, update, args, cooked):
    user_name, user_id, group = utils.get_info(update)

    # Argument handling
    if len(args) == 1:
        # TODO: Check arg value
        if not representsInt(args[0]):
            bot.sendMessage(group, text="Zzz. Please type an integer as the argument, or nothing at all.")
            return
        elif int(args[0]) < 0:
            bot.sendMessage(group, text="Nice try...but nope.")
            return
        portion = int(args[0])
    elif len(args) == 0:
        portion = 1
    else:
        utils.throw_error(bot, group, errors.WRONG_N_ARGS)
        return

    portion_text = str(portion) + (' portion' if portion == 1 else ' portions')

    # Request
    data = {
        'user_id': user_id,
        'group': group,
        'portions': portion,
        'cooked': cooked
    }
    r = requests.post(API_URL + 'eating', json=data)

    if r.status_code == 200:
        if portion == 0:
            bot.sendMessage(group, text=user_name + ' removed from meal')
        elif cooked:
            bot.sendMessage(group, text='Thanks for cooking, ' + user_name + '! You are eating ' + portion_text)
        else:
            bot.sendMessage(group, text=user_name + ' eating ' + portion_text)
    else:
        bot.sendMessage(group, text='Unable to update, is a meal started?')
예제 #38
0
파일: pull_repo.py 프로젝트: ravisvi/OSCM
def download_sec(sec, config_file):
    v = "0"
    if config_file.has_option(sec, "version"):
        v = config_file.get(sec, "version")
    OS = utils.get_os()
    arch = utils.get_arch()
    d = utils.get_info({
        "name": sec,
        "os": OS,
        "version": v,
        "arch": arch
    })
    for software in d:
        if len(software['command']) > 0:
            subprocess.call(software['command'], shell=True)
        elif len(software["url"]) > 0:
            if os.path.isfile(os.path.basename(software["url"])):
                print 'Downloading ' + sec + '...'
                r = requests.get(software["url"], stream=True)
                if r.status_code == 200:
                    with open(os.path.basename(software["url"]), 'wb') as f:
                        for chunk in r:
                            f.write(chunk)
                        if tarfile.is_tarfile(f.name):
                            tfile = tarfile.open(
                                os.path.basename(software["url"]), "r:gz")
                            tfile.extractall(sec)
                        elif zipfile.is_zipfile(f.name):
                            z = zipfile.ZipFile(f)
                            z.extractall(sec)
                else:
                    print 'Error downloading package, Please download ' + sec + ' on your own!'
            else:
                print sec + ' already present in folder, extracting...'
            print 'Running command ' + str(
                ['tar', '-xvf', os.path.basename(software["url"])])
            subprocess.call(['tar', '-xvf', os.path.basename(software["url"])])
예제 #39
0
def process(user, message):

	user_id, text, status_state = ut.get_info(user, message, state=True)
	workout = user.current_workout if user.status_state == FREE_WORKOUT else user.workout_guider.workout
	clear_timers(user)

	#=====[ End Workout ]=====
	if "end" in text and "workout" in text:
		
		end_user_workout(user, user_id, workout)
		feedback.start(user)

	#======================================================================================#
	#																				       #
	#               		 Code for Manually loggin ad-hoc workout 		   			   #
	#																					   #
	########################################################################################

	elif status_state == FREE_WORKOUT:

		#=====[ If user is moving on to next exercise ]=====
		if ('done' in text or 'end' in text) and ('exercise' in text or 'circuit' in text):

			workout.add_subroutine(workout.curr_subroutine)

			total_seconds = workout.curr_subroutine.get_total_set_time()
			time = str(int(total_seconds/60)) + ' min ' + str(total_seconds % 60) + ' sec' if total_seconds > 60 else str(total_seconds) + ' sec'

			#=====[ Notify user of duration of previous subroutine ]=====
			ut.send_response('Your ' + ', '.join(workout.curr_subroutine.exercises) + ' exercise took you a total of ' + time, user_id)

			workout.curr_subroutine = None

			ut.update(user_id, user)

		#=====[ If starting a new circuit ]=====
		elif "circuit" in text:

			#=====[ Get exercises for circuit ]=====
			exercises = [x.strip() for x in (text.split(':')[1]).split(',')]

			#=====[ If currently in a subroutine, add to workout ]=====
			if workout.curr_subroutine:
				workout.add_subroutine(workout.curr_subroutine)

			#=====[ Making new subroutine in workout ]=====
			workout.new_subroutine('circuit', exercises)

			user.curr_workout = workout 
			ut.update(user_id, user)

		#=====[ Extract exercise ]=====
		else:

			curr_set = ut.extract_exercise(text)

			#=====[ Update current workout if exercise extracted ]=====
			if curr_set:
				log_set(curr_set, workout, user, user_id)

			#=====[ If no exercise extracted, notify user ]=====
			else:

				ut.send_response(NO_EXERCISE_EXTRACTED_MESSAGE, user_id)

	#======================================================================================#
	#																				       #
	#                		Code for logging guided workout 							   #
	#																					   #
	########################################################################################

	elif status_state == GUIDED_WORKOUT:

		workout_guider = user.workout_guider

		#=====[ Log set and move to next set in workout ]=====
		cur_set_feedback_message, next_set = workout_guider.process(text, user, user_id)

		next_set_feedback_message = True

		#=====[ If there is feedback for the current set, then we were able to successfully extract a log and we move 
		#=====[ on to the next set
		end_subroutine = False
		if next_set:
			next_set_feedback_message, end_subroutine = workout_guider.next_set(user_id)

		#=====[ If there is not feedback for the next set, then that means there are no more sets to perform, and we
		#=====[ end the workout ]=====
		if not next_set_feedback_message:
			end_user_workout(user, user_id, workout)
			workout_guider = None

		elif not end_subroutine:
			message = cur_set_feedback_message.strip()if type(cur_set_feedback_message) == str else ''
			message += ' ' + next_set_feedback_message.strip() if type(next_set_feedback_message) == str else ''
			ut.send_response(message, user_id)

		user.workout_guider = workout_guider
		ut.update(user_id, user)
    print('\t\t0\t1')
    print('Actual\t0\t%d\t%d'%(cm[0,0],cm[0,1]))
    print('\t1\t%d\t%d'%(cm[1,0],cm[1,1]))

# this is kludgey but it works
sys.path.append('../utils')
import utils
import crossvalidation
importlib.reload(utils)
importlib.reload(crossvalidation)

#warnings.filterwarnings("ignore") # only turn this on in production mode
                                  # to keep log files from overflowing

dataset='Discovery_9-26-16'
basedir=utils.get_info('base_directory')
derived_dir=os.path.join(basedir,'Data/Derived_Data/%s'%dataset)

if not os.path.exists('surveypred'):
    os.mkdir('surveypred')

binary_vars=["Sex","ArrestedChargedLifeCount","DivorceCount","GamblingProblem","ChildrenNumber",
            "CreditCardDebt","RentOwn","RetirementAccount","TrafficTicketsLastYearCount","Obese",
             "TrafficAccidentsLifeCount","CaffienatedSodaCansPerDay","Nervous",
             'Hopeless', 'RestlessFidgety', 'Depressed',
             'EverythingIsEffort', 'Worthless','CigsPerDay','LifetimeSmoke100Cigs',
             'CannabisPast6Months']
# try:
#     binary_vars=[sys.argv[1]]
# except:
#     print('specify variable as command line argument')
예제 #41
0
def get_info(prob_code):
    if prob_code != None:
        prob_dir=os.path.join(os.path.join(Config.get_root(),'spoj'),prob_code)
    else:
        prob_dir=os.getcwd()
    return utils.get_info(prob_dir)
예제 #42
0
def process(user, message, instructions=False):
	""" Introduces a user to the workout chatbot and to logging workouts """

	user_id, text, status_state = ut.get_info(user, message, True)

	if status_state == 0:

		#=====[ Send introduction messages to user ]=====
		ut.send_response(WELCOME_MESSAGE, user_id)
		ut.send_response(COMMAND_INTRO, user_id)
		ut.send_response(WORKOUT_INTRO, user_id)

		user.status_state = 1

	#=====[ Begins teaching a user how to log a workout ]=====	
	elif status_state == 1:

		#=====[ Check if user said yes to learning about working out ]=====
		for word in yes_words:			
			if word in text:
				instructions = True
				break

		if instructions:
			#=====[ Send instructions for starting workout, logging sets, and ask if they want to learn about logging circuits ]=====
			ut.send_response(WORKOUT_START_INSTRUCTIONS, user_id)
			ut.send_response(WORKOUT_SET_INITIALIZATION, user_id)
			ut.send_response(WORKOUT_SET_CONTINUATION, user_id)
			ut.send_response(LEARN_ABOUT_CIRCUITS, user_id)

			user.status_state = 2

		else:
			
			user.status = "idle"
			user.status_state = 0
			ut.send_response(DONE_INSTRUCTIONS, user_id)


	#=====[ Processes message when user is learning about logging workouts. Currently, just checks to see if user wants to learn 
	#=====[ about logging circuits ]=====
	elif status_state == 2:

		user_id, text = ut.get_info(user, message)

		#=====[ Check if user gave a 'yes' word. If so gives information about logging circuits ]=====
		for word in yes_words:
			
			if word in text:
			
				#=====[ Teach user how to log circuits ]=====
				ut.send_response(WORKOUT_CIRCUIT_INITIALIZATION, user_id)
				ut.send_response(WORKOUT_CIRCUIT_CONTINUATION, user_id)

		ut.send_response(ASK_SEE_EXAMPLE, user_id)
		user.status_state = 3

	elif status_state == 3:

		#=====[ Update user into idle mode regardless of whether they wish to learn about circuits or not ]=====
		user.status = "idle"
		user.status_state = 0
		ut.update(user_id, user)

		#=====[ Check if user gave a 'yes' word. If so gives information about logging circuits ]=====
		for word in yes_words:
			
			if word in text:
			
				#=====[ Teach user how to log circuits ]=====
				ut.send_response(WORKOUT_EXAMPLE, user_id)
				return

		#=====[ Checks if user gave a 'no' word ]=====
		for word in no_words:

			if word in text:

				ut.send_response(DONE_INSTRUCTIONS, user_id)
				return

		#=====[ If no 'yes' or 'no' word found, then just assume user doesn't want to learn about logging circuits ]=====
		ut.send_response(ASSUME_DONE_INSTRUCTIONS, user_id)

	#=====[ update user (primarily state) in db ]=====
	ut.update(user_id, user)
예제 #43
0
def process(user, message):

	user_id, text = ut.get_info(user, message)

	#==========[ Command used to list all super commands for the workout logger ]==========#
	#																				       #
	#	usage: " list commands "									   					   #
	#																					   #
	########################################################################################

	if ('list' in text and 'commands' in text) or text == 'ls':

		ut.send_response('Here are a list of commands:\n\n' + '\n'.join(command_list.keys()), user_id)


	#=====[ Command used to log thoughts/improvements while using the workout logger ]=====#
	#																				       #
	#	usage: " sudo log: [suggestion to improve] "									   #
	#																					   #
	########################################################################################

	elif 'sudo' in text and 'log' in text and ':' in text:

		with open('to_improve_log','a') as f:
			f.write(text.split(':')[1].strip() + '\n\n')

		ut.send_response(MESSAGE_LOGGED, user_id)

	############################################################################
	############################################################################
	#################TODO TODO TODO TODO TODO TODO TODO#########################
	####################IMPLEMENT LISTING WORKOUTS##############################
	############################################################################

	elif 'list' in text and 'template' in text and 'workouts' in text:

		saved_workouts = '1. Chest Gains | Sets: 24\nTime: 1 hour\nMuscles: Chest, Triceps\n\n2. Ab Attack | Sets: 15\nTime: 20 minutes\nMuscles: Abs, Obliques\n\n3. Leg Day | Sets: 28\nTime: 1 hour 20 minutes\nMuscles: Quads, Hamstrings, Calves'
		ut.send_response(saved_workouts, user_id)


	#=========================[ Command used list recent workouts ]========================#
	#																				       #
	#	usage: " list workouts "									   					   #
	#																					   #
	########################################################################################

	elif 'list' in text and 'workouts' in text:

		ut.send_response('\n'.join([str(idx + 1) + '. ' + str(workout) for idx, workout in enumerate(reversed(user.workouts))]), user_id)


	#=====================[ Command used to review most recent workout ]===================#
	#																				       #
	#	usage: " review last workout "									   				   #
	#																					   #
	########################################################################################

	elif 'review' in text and 'last' in text and 'workout' in text:

		workout = user.workouts[-1]

		ut.send_response(workout.get_summary(), user_id)
		ut.send_response(workout.get_stats(),user_id)

		ut.send_response(workout.summarize_muscle_groups(4), user_id)
		# index = index if len(workout.muscle_groups) >= index else len(workout.muscle_groups)

		# if generate_spider(user_id, dict(workout.muscle_groups.most_common(index))):
		# 	ut.send_response('Check out the muscles you targeted:\nfile:///Users/Brandon/Desktop/Projects/workout_logger/spider.png', user_id)

	
	#==================[ Command used to review a particular workout ]=====================#
	#																				       #
	#	usage: " review workout: [index] "									   			   #
	#																					   #
	########################################################################################

	elif 'review' in text and 'workout' in text:

		try: 
			
			idx = int(ut.extract_int(text))
			workout = user.workouts[-idx]

			ut.send_response(user.workouts[-idx].get_summary(), user_id)
			ut.send_response(user.workouts[-idx].get_stats(), user_id)

			ut.send_response(workout.summarize_muscle_groups(4), user_id)

			# index = 4 if len(workout.muscle_groups) > 3 else len(workout.muscle_groups)

			# if generate_spider(user_id, dict(workout.muscle_groups.most_common(index))):
			# 	ut.send_response('Check out the muscles you targeted:\nfile:///Users/Brandon/Desktop/Projects/workout_logger/spider.png', user_id)

		except Exception as e:
			
			print e


	#==================[ Command used to remove a particular workout ]=====================#
	#																				       #
	#	usage: " remove workout: [index] "									   			   #
	#																					   #
	########################################################################################

	elif 'remove' in text and 'workout' in text:

		try: 
			
			idx = int(ut.extract_int(text))
			workout = user.workouts[-idx]

			ut.send_response('Deleted workout: ' + str(workout), user_id)

			del user.workouts[-idx]

			ut.update(user_id, user)

		except Exception as e:
			
			print e


	#=============[ Command used to review a weeks worth of workout stats ]================#
	#																				       #
	#	usage: " review week "									   			   			   #
	#																					   #
	########################################################################################

	elif 'review' in text and 'week' in text:

		info = 'Here\'s your week in review: \n\n'

		info += 'Total Volume: ' + str(user.get_volume(7)) + '\n'
		info += 'Total Sets: ' + str(user.get_num_sets(7)) + '\n' 
		info += 'Total Time: ' + "{0:.2f}".format(float(user.get_total_set_time(7)/3600.0)) + ' hours' + '\n'
		info += 'Avg. Set Time: ' + "{0:.2f}".format(user.get_avg_set_time(7)) + '\n'

		ut.send_response(info, user_id)

		ut.send_response(user.get_muscle_groups(7, 7), user_id)
		
		if hasattr(user, 'goals'):
			goal.review(user, user_id, 7, 7)

		# index = 6 if len(workout.muscle_groups) > 5 else len(workout.muscle_groups)

		# if generate_spider(user_id, dict(user.get_muscle_groups(7).most_common(index))):
		# 	ut.send_response('Check out the muscles you targeted most:\nfile:///Users/Brandon/Desktop/Projects/workout_logger/spider.png', user_id)
		
	elif 'q:' in text or 'query:' in text:
		exercise = text.split(':')[1].strip()
		info = 'You queried for: %s\n\n' % exercise
		print 'before query'
		sets = user.query_exercise(exercise)
		print 'after query'
		if sets:
			for date in sets:
				workout_sets = sets[date]

				info += "Workout on %s:\n" % date

				for xset in workout_sets:
					info += "%s reps of %s @ %s\n" % (xset.reps, xset.exercise, xset.weight)

			ut.send_response(info, user_id)
		else:
			ut.send_response("Sorry, I couldn't find any sets of " + exercise, user_id)


	#==================[ Command used to drop a user from the database ]===================#
	#																				       #
	#	usage: " reset db "									   			   			   	   #
	#																					   #
	########################################################################################

	elif text == 'reset db':
		
		ut.send_response('BYE FOREVER! :( ', user_id)
		ut.remove_user(user_id)


	#==================[ Command used set goals for targeted muscles ]=====================#
	#																				       #
	#	usage: " set goals: muslce1, muscle2, muscle3"			   			   			   #
	#																					   #
	########################################################################################

	elif 'set' in text and 'goal' in text:

		muscle_groups = text.split(':')

		#=====[ Check to make sure there are muscle groups specified after a colon ]=====
		if len(muscle_groups) == 1:
			ut.send_response(SET_GOALS, user_id)

		#=====[ set goals for user ]=====
		goal.set(user, user_id, muscle_groups[1])

	#================[ Command used set timers at specified intervals ]====================#
	#																				       #
	#	usage: " set timer for 45 and 60 seconds"			   			   			       #
	#																					   #
	########################################################################################

	elif 'set' in text and 'timer' in text:

		times = re.split(',|and',text)
		found_time = False

		#=====[ Checks each comma separated value for a time ]=====
		for time_candidate in times:

			time = extract_int(time_candidate)

			#=====[ If time for timer extracted, save info ]=====
			if time:
				
				found_time = True

				if hasattr(user,'timer'):
					user.timer.append(time)
				else:
					user.timer = [time]

		#=====[ Tell user how to set timer ]=====
		if not found_time:
			ut.send_response(HOW_TO_SET_TIMER, user_id)
		else:
			ut.send_response(SET_TIMER + ', '.join([str(x) for x in user.timer]) + ' seconds', user_id	)
			ut.update(user_id, user)

	#========================[ Command used to clear all timers ]==========================#
	#																				       #
	#	usage: " clear timer "			   			   			       					   #
	#																					   #
	########################################################################################

	elif 'clear' in text and 'timer' in text:

		user.timer = []
		ut.send_response(CLEARED_TIMER, user_id)
		ut.update(user_id, user)

	#========================[ Command used to exit to idle mode ]=========================#
	#																				       #
	#	usage: " exit "			   			   			       					  	 	   #
	#																					   #
	########################################################################################

	elif text == 'exit':
		
		user.status = 'idle'
		ut.send_response(IDLE_MODE, user_id)
		ut.update(user_id, user)

	#=====================[ Command used to get help for any command ]=====================#
	#																				       #
	#	usage: " help [command] "			   			   			       				   #
	#																					   #
	########################################################################################

	elif 'help' in text:

		user_command = text.replace('help','').strip()

		#=====[ Check if user command is in our command shortcut list. If so, send appropriate help response ]=====
		for command in command_shortcuts:
			
			if user_command in command_shortcuts[command]:
				
				ut.send_response(command_list[command], user_id)
				return True

		#=====[ Tell user that we couldn't find specified command and send list of commands ]=====
		ut.send_response(HELP_COMMAND, user_id)
		ut.send_response('Here are a list of commands:\n\n' + '\n'.join(command_list.keys()), user_id)

	#===================[ Command used to learn how to log a workout ]=====================#
	#																				       #
	#	usage: " how to log?" | "how do I log a workout? "  			       				   #
	#																					   #
	########################################################################################

	elif 'how' in text and 'log' in text:

		user.status = "intro"
		user.status_state = 1
		intro.process(user, message, instructions=True)

	#==========================[ Command used to greet user ]==============================#
	#																				       #
	#	usage: Greet Tony: 'Hi'  			       			   							   #
	#																					   #
	########################################################################################
	
	elif text in greetings:

		ut.send_response(GREETING, user_id)

	elif 'thank' in text:

		ut.send_response(NO_PROBLEM, user_id)
		ut.send_response(np.random.choice(INSPIRATIONAL),user_id)

	else:

		return False

	return True
예제 #44
0
def process(user, message):

	user_id, text = ut.get_info(user, message)
	workout = user.get_last_workout()

	rating = extract_int(text)
	print rating

	#On feedback question
	if user.status_state == 0:

		if "yes" in text or "ok" in text:
			ut.send_response(RATING_QUESTION, user_id)
			user.status_state = 1

		elif "no" in text:
			ut.send_response(NO_FEEDBACK, user_id)
			user.status = "idle"
			user.status_state = 0

		else:
			ut.send_response(FEEDBACK_CLARIFY, user_id)
			ut.send_response(FEEDBACK_QUESTION, user_id)

		ut.update(user_id, user)


	# On rating question
	elif user.status_state == 1:

		if rating:

			if rating >= 1 and rating <= 10:
				workout.rating = rating
				ut.send_response(QUESTION_END, user_id)
				user.status_state = 2
				ut.update(user_id, user)
				ut.send_response(DIFFICULTY_QUESTION, user_id)
				return

		# If anything goes wrong, send a clarifying message
		ut.send_response(RATING_CLARIFY, user_id)
		ut.send_response(RATING_QUESTION, user_id)

		


	# On difficulty question
	elif user.status_state == 2:
		
		if rating:
			if rating >= 1 and rating <= 5:
				workout.rating = rating
				ut.send_response(QUESTION_END, user_id)
				user.status_state = 3
				ut.update(user_id, user)
				ut.send_response(TIREDNESS_QUESTION, user_id)
				return

		# If anything goes wrong, send a clarifying message
		ut.send_response(DIFF_TIRED_CLARIFY, user_id)
		ut.send_response(DIFFICULTY_QUESTION, user_id)


	# On tiredness question
	elif user.status_state == 3:
		
		if rating:

			if rating >= 1 and rating <= 5:
				workout.rating = rating
				ut.send_response(FEEDBACK_END, user_id)
				user.status = "idle"
				user.status_state = 0
				ut.update(user_id, user)
				return

		# If anything goes wrong, send a clarifying message
		ut.send_response(DIFF_TIRED_CLARIFY, user_id)
		ut.send_response(TIREDNESS_QUESTION, user_id)
"""

import os,glob,sys
import numpy,pandas
import json

from sklearn.preprocessing import scale
from sklearn.decomposition import FactorAnalysis
from sklearn import cross_validation

# this is kludgey but it works
sys.path.append('../utils')
from utils import get_info,get_survey_data

dataset='Discovery_9-26-16'
basedir=get_info('base_directory')
derived_dir=os.path.join(basedir,'Data/Derived_Data/%s'%dataset)


data,surveykey=get_survey_data(dataset)

cdata=data.values


kf = cross_validation.KFold(cdata.shape[0], n_folds=4)

max_components=30

sc=numpy.zeros((max_components,4))

for n_components in range(1,max_components):
예제 #46
0
파일: pull_repo.py 프로젝트: ravisvi/OSCM
def parse_config(config_file):
    secs = config_file.sections()
    v = "0"
    for sec in secs:
        if config_file.has_option(sec, "version"):
            v = config_file.get(sec, "version")
        OS = utils.get_os()
        arch = utils.get_arch()
        d = utils.get_info({
            "name": sec,
            "os": OS,
            "version": v,
            "arch": arch
        })
        print d, str({"name": sec, "os": OS, "version": v, "arch": arch})
        for software in d:
            if len(software.get('command', '')) > 0:
                subprocess.call(software['command'], shell=True)
            elif len(software["url"]) > 0:
                print software["url"]
                if not os.path.isfile(os.path.basename(software["url"])):
                    print 'Downloading ' + sec + '...'
                    r = requests.get(software["url"], stream=True)
                    if r.status_code == 200:
                        with open(os.path.basename(software["url"]), 'wb') as f:
                            for chunk in r:
                                f.write(chunk)
                            # if tarfile.is_tarfile(f.name):
                            #     tfile = tarfile.open(os.path.basename(software["url"]), "r:gz")
                            #     tfile.extractall(sec)
                            # elif zipfile.is_zipfile(f.name):
                            #     z = zipfile.ZipFile(f)
                            #     z.extractall(sec)
                            subprocess.call('tar -xvf ' + f.name, shell=True)
                    else:
                        print 'Error downloading package, Please download ' + sec + ' on your own!'
                else:
                    print sec + ' already present in folder, extracting...'
                subprocess.call('tar -xvf ' + os.path.basename(software["url"]),
                                shell=True)
                # if tarfile.is_tarfile(os.path.basename(software["url"])):
                #     tfile = tarfile.open(os.path.basename(software["url"]), "r:gz")
                #     tfile.extractall(sec)
                # elif zipfile.is_zipfile(os.path.basename(software["url"])):
                #     z = zipfile.ZipFile(os.path.basename(software["url"]))
                #     z.extractall(sec)
        if not os.path.exists(sec):
            os.mkdir(sec)
        if sec == 'eclipse':
            if config_file.has_option(sec, "plugins"):
                plugins = config_file.get(sec, "plugins").split(",")
                if os.path.isdir(os.path.join(os.getcwd(), sec, "dropins")):
                    for plugin in plugins:
                        f = download_file(plugin)
                        if len(f) > 0 and zipfile.is_zipfile(f):
                            z = zipfile.ZipFile(open(f, "rb"))
                            path = os.path.join(os.getcwd(), "eclipse",
                                                "dropins",
                                                os.path.splitext(f)[0])
                            if not os.path.exists(path):
                                os.makedirs(path)
                            z.extractall(path)
from joblib import Parallel, delayed
import multiprocessing

num_cores = multiprocessing.cpu_count()
print('using %d cores'%num_cores)

from sklearn.preprocessing import scale
from sklearn.decomposition import FactorAnalysis
from sklearn import cross_validation

# this is kludgey but it works
sys.path.append('../utils')
from utils import get_info,get_behav_data,get_survey_data
from r_to_py_utils import missForest

dataset=get_info('dataset')
print('using dataset:',dataset)
basedir=get_info('base_directory')
derived_dir=os.path.join(basedir,'Data/Derived_Data/%s'%dataset)

alldata=get_behav_data(dataset)
surveydata=pandas.DataFrame()
for k in alldata.columns:
    if k.find('survey')>-1:
        surveydata[k]=alldata[k]

data=pandas.read_csv(os.path.join(derived_dir,'taskdata_clean_cutoff3.00IQR_imputed.csv'))

assert all(data.index == surveydata.index)

alldata = surveydata.merge(data,'inner',right_index=True,left_index=True)