Esempio n. 1
0
def send_day(day):
    send_data(day)#自当年元旦开始的天(个位)、分隔标志、天(十位)、基准标志
    #天(百位)、分隔标志、未编码位、基准标志
    send_bits(bcd(day//100))
    utils.divide()
    utils.vacancy()
    utils.p_unit()
Esempio n. 2
0
 def test_divide(self):
     self.assertEquals(utils.divide([]), 1)
     self.assertAlmostEquals(utils.divide([2.1]), 0.47619047619047616)
     self.assertAlmostEquals(utils.divide([2.1, 4.3, 2.1]), 
                             0.052734272003374986)
     self.assertAlmostEquals(utils.divide([-0.5, 10.1, 3]), 
                             -0.06600660066006601)
Esempio n. 3
0
    def __init__(self, attention_mask_func, hidden_size, num_attention_heads,
                 attention_dropout):
        super(ParallelSelfAttention, self).__init__()

        self.attention_mask_func = attention_mask_func

        # Per attention head and per partition values.
        world_size = torch.distributed.get_world_size()
        self.hidden_size_per_partition = divide(hidden_size, world_size)
        self.hidden_size_per_attention_head = divide(hidden_size,
                                                     num_attention_heads)
        self.num_attention_heads_per_partition = divide(
            num_attention_heads, world_size)

        # Strided linear layer.
        self.query_key_value = ColumnParallelLinear(  # column linear
            hidden_size,
            3 * hidden_size,
            gather_output=False)

        self.norm_factor = math.sqrt(self.hidden_size_per_attention_head)

        self.scale_mask_softmax = ScaleMaskSoftmax(
            mask_func=self.attention_mask_func, scale=None)

        # Dropout. Note that for a single iteration, this layer will generate
        # different outputs on different number of parallel partitions but
        # on average it should not be partition dependent.
        self.attention_dropout = torch.nn.Dropout(attention_dropout)

        # Output.
        self.dense = RowParallelLinear(input_size=hidden_size,
                                       output_size=hidden_size,
                                       input_is_parallel=True)
Esempio n. 4
0
def send_data(data,digit=2):
    dot=1
    for i in range(digit):
        dot*=10
        unit=data%dot*10//dot#单位数字
        send_bits(bcd(unit))
        if i <digit-1:
            utils.divide()
    utils.p_unit()
Esempio n. 5
0
    def test_divide(self):
        """Expectation: divide one number by another using the correct template"""
        with self.subTest("Testing divide() using integers"):
            computed = utils.divide(self.i3, self.i4)
            self.assertIsInstance(computed, int)
            self.assertEqual(computed, int(self.i3 / self.i4))

        with self.subTest("Testing divide() using floats"):
            computed = utils.divide(self.f3, self.f4)
            self.assertIsInstance(computed, float)
            self.assertEqual(computed, self.f3 / self.f4)
Esempio n. 6
0
def calculate_predictions(true_positive, false_positive, true_negative, false_negative):
    """
    Calculate predictions based on the number of true/false positive/negative's
    """
    sens_denom = true_positive + false_negative
    spec_denom = true_negative + false_positive
    sensitivity = divide(true_positive, sens_denom)
    specificity = divide(true_negative, spec_denom)
    accuracy = divide(true_positive + true_negative, sens_denom + spec_denom)
    tpr = sensitivity  # Calculate the true positive rate
    fpr = 1 - specificity  # Calculate the false positive rate
    return accuracy, tpr, fpr
Esempio n. 7
0
def send_control():
    """
    未编码位、分隔标志、未编码位、基准标志+时间质量标志、校验位、未编码位、基准标志
    """
    utils.vacancy()
    utils.divide()
    utils.vacancy()
    utils.p_unit()
    utils.time_quality()
    utils.verify()
    utils.vacancy()
    utils.time_quality()
Esempio n. 8
0
 def compute(self):
   if not(self.graph.processed()): self.graph.process()
   user_count = len(self.graph)
   self.analyze_group()
   self.statistics.update({
     'subgroups' : len(self.graph.groups),
     'user_count': user_count,
     'like_avg': divide(self.statistics['likes_total'], user_count),
     'age_avg': divide(self.statistics['age_total'], user_count),
     'friends_avg': divide(len(self.friends), user_count),
     'friends_age_avg':divide(self.statistics['friends_age_total'], len(self.friends)),
   })
   return self.statistics
Esempio n. 9
0
 def hydrate(self, ids):
     ids = divide(list(ids), 100)
     results = []
     for id in ids:
         result = data_from_id(self.tw, map(str, id))
         results.append(result)
     return results
Esempio n. 10
0
    def __init__(self, input_size, output_size, input_is_parallel=False):
        super(RowParallelLinear, self).__init__()

        # Keep input parameters
        self.input_size = input_size
        self.output_size = output_size
        self.input_is_parallel = input_is_parallel
        # Divide the weight matrix along the last dimension.
        world_size = torch.distributed.get_world_size()
        self.input_size_per_partition = divide(input_size, world_size)

        # Parameters.
        # Note: torch.nn.functional.linear performs XA^T + b and as a result
        # we allocate the transpose.
        # Initialize weight.
        self.weight = Parameter(
            torch.empty(self.output_size,
                        self.input_size_per_partition,
                        device=torch.cuda.current_device(),
                        dtype=torch.float))
        torch.nn.init.xavier_normal_(self.weight)

        self.bias = Parameter(
            torch.empty(self.output_size,
                        device=torch.cuda.current_device(),
                        dtype=torch.float))
        # Always initialize bias to zero.
        with torch.no_grad():
            self.bias.zero_()
Esempio n. 11
0
 def __init__(self, pattern = "", seeds_dict = {}):
     self.pattern = pattern
     self.strlen = len(pattern)
     self.num_of_seeds = len(seeds_dict)
     self.seeds_dict = seeds_dict
     self.max_seed = max(self.seeds_dict.values())
     self.avg_seed = utils.divide(utils.sum(self.seeds_dict.values()), 
                                                 self.num_of_seeds)
Esempio n. 12
0
    def features(self):
        return {
            # Device
            #'device': self.device,

            # Product
            #'product': self.merchandise,

            #Country
            'country':
            self.country,

            # Price
            # 'price': self.price,
            'price_fr':
            utils.short_float(utils.divide(self.price, self.last_price)),

            # IP
            #'ip_3oct': self.ip_3oct,
            #'ip_2oct': self.ip_2oct,

            # Time
            # 'time': self.time,
            # 'time_from_start': self.time_from_start,
            # 'time_to_end': self.time_to_end,
            # format _perc to have 2 digist after .
            'time_from_start_perc':
            utils.short_float(utils.divide(self.time, self.auc_length)),
            'time_to_end_perc':
            utils.short_float(utils.divide(self.time_to_end, self.auc_length)),
            # 'time_from_day_start': self.time_from_day_start,
            # 'time_to_prev_bid': self.time_to_prev_bid,

            # Order
            #'is_first': self.is_first,
            #'is_last': self.is_last,
            #'day': self.day,
            'unique':
            self.prev_unique,
            'unique_50':
            self.prev_unique_50,
            # how many immediate prev bids are made by the same user?

            # Url
            #'ref': self.url,
        }
Esempio n. 13
0
def EStep(pose, log_var, log_activation, vote):
    normal_vote = utils.divide(tf.square(vote - pose), 2 * tf.exp(log_var))
    log_probs = normal_vote + utils.log(2 * np.pi) + log_var
    log_probs = -0.5 * tf.reduce_sum(log_probs, axis=-1, keepdims=True)
    log_act_logit = log_activation + log_probs
    log_act_logit = log_probs
    log_R = log_act_logit - tf.reduce_logsumexp(log_act_logit, axis=-2, keepdims=True)
    return log_R
Esempio n. 14
0
def use_my_utils(a, b):
    differenceof = utils.differenceof(a, b)
    sumof = utils.sumof(a, b)
    multiplyof = utils.multiplyof(a, b)
    divide = utils.divide(a, b)
    print(differenceof)
    print(sumof)
    print(multiplyof)
    print(divide)
Esempio n. 15
0
  def analyze_group(self):
    for k, group in enumerate(self.graph.groups):
      shared_songs = 0
      friendships = 0
      for edge in self.graph.edges:
        if edge[0] in group or edge[1] in group:
          friendships += 1
          shared_songs += int(edge[2]['songs'])

      self.statistics['shared_songs_total_group'][k] = shared_songs
      self.statistics['shared_songs_avg_group'][k] = divide(shared_songs, friendships) if shared_songs else 0
Esempio n. 16
0
    def migrate_node(self, src_node):
        nodes = [n for n in self.masters if n.name != src_node.name]
        slot_count = len(src_node.slots)
        if slot_count <= 0:
            return
        slots = divide(slot_count, len(nodes))

        nodes.sort(key=lambda x: len(x.slots))

        for node, count in zip(nodes, slots):
            src, dst = (src_node, node)
            self.migrate(src, dst, count)
Esempio n. 17
0
    def fill_slots(self):
        masters = self.masters
        slots = itertools.chain(*[n.slots for n in masters])
        missing = list(set(range(self.CLUSTER_HASH_SLOTS)).difference(slots))

        div = divide(len(missing), len(masters))
        masters.sort(key=lambda x: len(x.slots))

        i = 0
        for count, node in zip(div, masters):
            node.add_slots(*missing[i:count + i])
            i += count
Esempio n. 18
0
    def features(self):
        return {
            # Device
            #'device': self.device,

            # Product
            #'product': self.merchandise,

            #Country
            'country': self.country,

            # Price
            # 'price': self.price,
            'price_fr': utils.short_float(utils.divide(self.price, self.last_price)),

            # IP
            #'ip_3oct': self.ip_3oct,
            #'ip_2oct': self.ip_2oct,

            # Time
            # 'time': self.time,
            # 'time_from_start': self.time_from_start,
            # 'time_to_end': self.time_to_end,
            # format _perc to have 2 digist after .
            'time_from_start_perc': utils.short_float(utils.divide(self.time, self.auc_length)),
            'time_to_end_perc': utils.short_float(utils.divide(self.time_to_end, self.auc_length)),
            # 'time_from_day_start': self.time_from_day_start,
            # 'time_to_prev_bid': self.time_to_prev_bid,

            # Order
            #'is_first': self.is_first,
            #'is_last': self.is_last,
            #'day': self.day,
            'unique': self.prev_unique,
            'unique_50': self.prev_unique_50,
            # how many immediate prev bids are made by the same user?

            # Url
            #'ref': self.url,
        }
Esempio n. 19
0
    def bind_slots_force(self):
        masters = self.masters
        slots = itertools.chain(*[n.slots for n in masters])
        missing = list(set(range(self.CLUSTER_HASH_SLOTS)).difference(slots))

        div = divide(len(missing), len(masters))
        masters.sort(key=lambda x: len(x.slots))

        i = 0
        for count, node in zip(div, masters):
            for slot in missing[i:count + i]:
                self.update_slot_mapping(slot, node.name)
            i += count
Esempio n. 20
0
 def count_proximity_over_cooc(self, list, cache=False):
     """
     list - check proximity over cooccurrence for the given list of terms
     
     """
     prox = self.count_proximity(list, cache)
     cooc = self.count_cooccurrence(list, cache)
     result = utils.divide(prox, cooc)
     if self.debug:
         print "Calculated proximity count for terms %s - %0.5f (cooc - %d)" % (list, 
                                                                               result, 
                                                                               cooc)
     return result
def compute_metrics(stats_authors):
    #Keep only the possible candidates:
    stats_authors = {
        s: stats_authors[s]
        for s in stats_authors if stats_authors[s]["Candidate"]
    }
    #Compute (more complex) metrics:
    for authorName, authorStats in stats_authors.items():
        stats_authors[authorName]["Following-Followers Ratio"] = divide(
            authorStats["Following"], authorStats["Followers"])
        stats_authors[authorName]["TotViews"] = sum(authorStats["ViewsSerie"])
        stats_authors[authorName]["Views-Followers Ratio"] = divide(
            authorStats["TotViews"],
            authorStats["Followers"],
            round_result=False)
        stats_authors[authorName]["AverageLikes"] = divide(
            authorStats["NLikes"], authorStats["NVideos"])
        stats_authors[authorName]["AverageViews"] = divide(
            sum(authorStats["ViewsSerie"]), authorStats["NVideos"])

    save_metrics(stats_authors)

    return stats_authors
Esempio n. 22
0
def split_tensor_along_last_dim(tensor,
                                num_partitions,
                                contiguous_split_chunks=False):
    """Split a tensor along its last dimension.
    Arguments:
        tensor: input tensor.
        num_partitions: number of partitions to split the tensor
        contiguous_split_chunks: If True, make each chunk contiguous in memory.
    """
    last_dim = tensor.dim() - 1
    last_dim_size = divide(tensor.size()[last_dim], num_partitions)

    # Split
    tensor_list = torch.split(tensor,
                              split_size_or_sections=last_dim_size,
                              dim=last_dim)

    return tensor_list
Esempio n. 23
0
    def choose_solo_notes(self):
        rest_beginning = random.choice([True, False, False, False, False])
        rest_middle = random.choice([True, True, False])
        rest_end = random.choice([True, True, True, False, False])

        rests = [rest_beginning, rest_middle, rest_end]
        num_rests = rests.count(True)

        min_num_divs = 3 + num_rests

        num_divs = random.randint(min_num_divs, 9)

        divs = divide(16, num_divs)

        notes = [{'pitch': None, 'duration': div / 4.0} for div in divs]

        if rest_beginning:
            notes[0]['pitch'] = 'rest'

        if rest_end:
            notes[-1]['pitch'] = 'rest'

        if rest_middle:
            if rest_beginning:
                start = 2
            else:
                start = 1
            if rest_end:
                end = -2
            else:
                end = -1
            middle_rest_index = random.choice(range(len(notes))[start:end])
            notes[middle_rest_index]['pitch'] = 'rest'


        for note in notes:
            if note['pitch'] != 'rest':
                note['pitch'] = ps = random.choice(self.soloist_shared_notes)

                self.soloist_shared_notes = [p for p in frange(ps - 2, ps + 3) if p in self.all_soloists_shared_notes]

        self.soloist_shared_notes = [p for p in frange(ps - 5, ps + 6) if p in self.all_soloists_shared_notes]

        return notes
Esempio n. 24
0
    async def list_category(self, ctx):
        embeds = []

        for categories in divide(list(self.bot.categoryDB.find()), 10):
            embed = Embed(title='카테고리 목록', color=Color.green())

            for category in categories:
                embed.add_field(name=category['name'],
                                value=f'`{category["description"]}`')

            embeds.append(embed)

        if embeds:
            msg = await ctx.send(embed=embeds[0])
            await Paginator(self.bot, msg, embeds=embeds).start()
        else:
            await ctx.send(embed=Embed(
                title='카테고리 목록', description='카테고리가 없습니다', color=Color.green())
                           )
Esempio n. 25
0
    def __init__(self, g: Graph, levels_to_build):
        super().__init__()

        self.g = g
        self.division = divide(g)

        self.levels_to_build = levels_to_build

        self.final = []
        self.start = FactorGraph(g, sorted([CongruenceClass([node]) for node in g.nodes]), '∆')
        self.add_node(self.start)

        self.levels_count = g.number_of_nodes() - 1
        if self.levels_to_build is not None:
            self.levels_count = min(self.levels_to_build, self.levels_count)
        self.levels = [[] for _ in range(self.levels_count)]
        self.levels_set = [set() for _ in range(self.levels_count)]
        self.nodes_levels = {}
        self._build()
Esempio n. 26
0
    async def help(self, ctx):
        embeds = [
            Embed(title='도움',
                  description='`이모지`로 페이지를 넘기세요',
                  color=Color.green())
        ]

        for i in range(len(self.bot.cogs)):
            cog = self.bot.cogs[list(self.bot.cogs.keys())[i]]
            embeds[0].add_field(name=f'{i + 1}페이지  |  {cog.name}',
                                value=f'`{cog.description}`',
                                inline=True)

            for cmds in divide(cog.get_commands(), 10):
                embed = Embed(title=f'{cog.name} 도움', color=Color.green())
                embed.set_footer(text=f'Page {i + 1}')

                for cmd in cmds:
                    if 'commands' not in dir(cmd):
                        embed.add_field(
                            name=f'{self.bot.command_prefix}{cmd.name}'
                            if not cmd.usage else
                            f'{self.bot.command_prefix}{cmd.usage}',
                            value=cmd.help or '설명 없음',
                            inline=True)
                    else:
                        for child_cmd in cmd.commands:
                            embed.add_field(
                                name=
                                f'{self.bot.command_prefix}{cmd.name} {child_cmd.name}'
                                if not child_cmd.usage else
                                f'{self.bot.command_prefix}{cmd.name} {child_cmd.usage}',
                                value=child_cmd.help or '설명 없음',
                                inline=True)

                embeds.append(embed)

        msg = await ctx.send(embed=embeds[0])
        await Paginator(self.bot, msg, embeds=embeds).start()
Esempio n. 27
0
    async def list_post(self, ctx, *, query: str = None):
        if not query:
            postlist = list(self.bot.postDB.find())
            title = '글 목록'
            no_description = '글이 없습니다'

        else:
            postlist = []
            _idlist = []

            _postlist = self.bot.postDB.get({'title': {'$regex': f'.*{query}.*'}}) + self.bot.postDB.get({'content': {'$regex': f'.*{query}.*'}})

            for data in _postlist:
                if data['_id'] in _idlist:
                    continue
                _idlist.append(data['_id'])
                postlist.append(data)

            title = f'"{query}" 검색 결과'
            no_description = '검색 결과가 없습니다'

        embeds = []

        for categories in divide(postlist, 10):
            embed = Embed(
                title=title,
                color=Color.green()
            )

            for post in categories:
                embed.add_field(name=f'{post["title"]} ({post["_id"]})', value=f'by `{self.bot.get_user(int(post["authorID"]))}` :heart: `{len(post["hearts"])}` :speech_balloon: `{len(self.bot.commentDB.get({"postID": post["_id"]}))}`', inline=False)

            embeds.append(embed)

        if embeds:
            msg = await ctx.send(embed=embeds[0])
            await Paginator(self.bot, msg, embeds=embeds).start()
        else:
            await ctx.send(embed=Embed(title=title, description=no_description, color=Color.green()))
Esempio n. 28
0
    def slot_balance(self, seq):
        amt = self.CLUSTER_HASH_SLOTS
        seq.sort(key=lambda x: x['count'], reverse=True)
        chunks = divide(amt, len(seq))
        pairs = list(zip(seq, chunks))

        i, j = 0, len(pairs) - 1
        while i < j:
            m, count = pairs[i]
            more = m['count'] - count
            if more <= 0:
                i += 1
                continue

            n, count = pairs[j]
            need = count - n['count']
            if need <= 0:
                j -= 1
                continue

            if need < more:
                n['need'].append((m['node'], need))
                n['count'] += need
                m['count'] -= need
                j -= 1
            elif need > more:
                n['need'].append((m['node'], more))
                n['count'] += more
                m['count'] -= more
                i += 1
            else:
                n['need'].append((m['node'], need))
                n['count'] += need
                m['count'] -= more
                j -= 1
                i += 1

        return seq
Esempio n. 29
0
from numpy import asarray
from numpy import save
import straw
from scipy.sparse import csr_matrix
import utils

highres = utils.matrix_extract(
    18, 10000,
    "https://hicfiles.s3.amazonaws.com/hiseq/gm12878/in-situ/combined.hic")

print('dividing, filtering and downsampling files...')

highres_sub, index = utils.divide(highres)

print("highres shape: ", highres_sub.shape)

lowres = utils.genDownsample(highres, 1 / float(16))
lowres_sub, index = utils.divide(lowres)
print("lowres shape: ", lowres_sub.shape)

save('lowres_ch18.npy', lowres_sub)
save('highres_ch18.npy', highres_sub)
Esempio n. 30
0
                             str(args['resolution']))
    low_chr_mat = low_cool.matrix(
        balance=False).fetch("chr" + str(args['chr_num'])).astype(float)
    low_chr_mat[np.isnan(low_chr_mat)] = 0
    chr_frames, chr_indices = utils.divide2(low_chr_mat, args['chr_num'])
    enhanced_chr_mat = low_cool.matrix(
        balance=False).fetch("chr" + str(args['chr_num'])).astype(float)
    enhanced_chr_mat[np.isnan(enhanced_chr_mat)] = 0
    """
    average_chr_mat = low_cool.matrix(balance = False).fetch("chr" + str(args['chr_num'])).astype(float)
    average_chr_mat[np.isnan(average_chr_mat)] = 0
    """

else:
    chr_frames, chr_indices = utils.divide(args['LowRes_matrix_path'],
                                           args['chr_num'], args['resolution'],
                                           args['genome_type'])
    low_chr_mat = np.load(args['LowRes_matrix_path'] + '_npy_form_tmp.npy')
    enhanced_chr_mat = np.load(args['LowRes_matrix_path'] +
                               '_npy_form_tmp.npy')
    # average_chr_mat = np.load(args['LowRes_matrix_path'] + '_npy_form_tmp.npy')
# applying model on frames
chr_frames = np.stack(chr_frames, axis=0)
chr_indices = np.stack(chr_indices, axis=0)
chr_frames = np.expand_dims(chr_frames, axis=1)
lowres_set = torch.from_numpy(chr_frames).float()
enhanced_set = Net(Variable(lowres_set))
enhanced_set = enhanced_set.data.cpu().numpy()
enhanced_set = np.reshape(
    enhanced_set,
    (enhanced_set.shape[0], enhanced_set.shape[2], enhanced_set.shape[3]))
Esempio n. 31
0
use_gpu = 1

conv2d1_filters_numbers = 8
conv2d1_filters_size = 9
conv2d2_filters_numbers = 8
conv2d2_filters_size = 1
conv2d3_filters_numbers = 1
conv2d3_filters_size = 5

down_sample_ratio = 16
epochs = 10
HiC_max_value = 100

# This block is the actual training data used in the training. The training data is too large to put on Github, so only toy data is used.
input_file = '/home/zhangyan/Desktop/chr21.10kb.matrix'
low_resolution_samples, index = utils.divide(input_file)

low_resolution_samples = np.minimum(HiC_max_value, low_resolution_samples)

batch_size = low_resolution_samples.shape[0]

# Reshape the high-quality Hi-C sample as the target value of the training.
sample_size = low_resolution_samples.shape[-1]
padding = conv2d1_filters_size + conv2d2_filters_size + conv2d3_filters_size - 3
half_padding = padding / 2
output_length = sample_size - padding

print low_resolution_samples.shape

lowres_set = data.TensorDataset(
    torch.from_numpy(low_resolution_samples),
Esempio n. 32
0
 def __init__(self, encoder, decoder, n_samples=1, q=6 / 5, n_lambda=29):
     super(particle_flow, self).__init__()
     self.n_samples = n_samples
     self.intervals = divide(q, n_lambda)
     self.encoder = encoder
     self.decoder = decoder
Esempio n. 33
0
    if args.ci_mode and not args.ci_threshold:
        parser.error("CI mode requires a threshold to be set")

    cwd = os.getcwd()
    master = load_tool(args.mutation_tool, cwd)

    if not args.benchmark:
        print("Checking project compatibility with {0}...".format(args.mutation_tool))
        if not master.check():
            sys.exit("Selected mutation tool reports that it doesn't support the current project.")

    print("Creating mutants...")
    mdir, mutants = master.mutate()

    print("Scoring mutants in parallel...")
    divided_mutants = divide(mutants, args.scorers)

    # functools.partial is instead of lamda below, as the latter can't be pickled
    toolfun = functools.partial(load_tool, args.mutation_tool)
    scorefun = functools.partial(local_scorer.create_and_score, toolfun, cwd, mdir)

    with Pool(processes=args.scorers) as pool:
        nested_results = pool.map(scorefun, divided_mutants, 1)

    results = ScoringResult(flatten_list(nested_results))

    if not args.benchmark:
        print("Loading mutant metadata from the filesystem...")
        results.add_metadata(cwd, mdir)

    if args.ci_mode:
Esempio n. 34
0
    def features(self):
        # global human_cnt
        # if self.is_robot or (self.is_human and human_cnt > 0):
        #     if self.is_human:
        #         human_cnt -= 1
        #     self.find_increment_patterns()
            #self.find_inc_price_patterns()

        increments = self.get_all_increments()
        bids_count = len(self.bids)
        true_bids_count = len(increments)

        # Auctions
        auctions_count = len(self.auctions)
        sim_auctions = self.get_sim_auctions()
        won_auctions_count = len(self.win_bids)
        auction_rank = 0.0
        for auc, bids in self.auctions.iteritems():
            auction_rank += shared.auction_rank[auc]
        if self.auctions:
            auction_rank /= len(self.auctions)

        # Price
        human_price_rmse_per_auction = self.get_price_rmse(self.last_bids, shared.human_median_price_per_auction, 'auction')
        human_price_rmse_per_product = self.get_price_rmse(self.last_bids, shared.human_median_price_per_product, 'merchandise')

        robot_price_rmse_per_auction = self.get_price_rmse(self.last_bids, shared.robot_median_price_per_auction, 'auction')
        robot_price_rmse_per_product = self.get_price_rmse(self.last_bids, shared.robot_median_price_per_product, 'merchandise')

        # Stats per auction
        avg_countries_per_auction = self.get_per_auction(np.average, self.counties_per_auction)
        median_countries_per_auction = self.get_per_auction(np.median, self.counties_per_auction)
        std_countries_per_auction = self.get_per_auction(np.std, self.counties_per_auction)

        avg_devices_per_auction = self.get_per_auction(np.average, self.devices_per_auction)
        median_devices_per_auction = self.get_per_auction(np.median, self.devices_per_auction)
        std_devices_per_auction = self.get_per_auction(np.std, self.devices_per_auction)

        avg_referrals_per_auction = self.get_per_auction(np.average, self.referrals_per_auction)
        median_referrals_per_auction = self.get_per_auction(np.median, self.referrals_per_auction)
        std_referrals_per_auction = self.get_per_auction(np.std, self.referrals_per_auction)

        avg_ips_per_auction = self.get_per_auction(np.average, self.ips_per_auction)
        median_ips_per_auction = self.get_per_auction(np.median, self.ips_per_auction)
        std_ips_per_auction = self.get_per_auction(np.std, self.ips_per_auction)

        # avg_inc_per_auction = self.get_per_auction(np.average, self.increments_per_auction)
        # median_inc_per_auction = self.get_per_auction(np.median, self.increments_per_auction)
        # std_inc_per_auction = self.get_per_auction(np.std, self.increments_per_auction)

        # IP
        avg_ips_per_increment = utils.divide(sum([inc.ips_count for inc in increments]), len(increments))
        frequent_ip = '.'.join(self.get_frequent(self.ips).split('.')[:2])
        ip_octets = defaultdict(int)

        for ip, count in self.ips.iteritems():
            octets = ip.split('.')
            ip_octets['.'.join(octets[:1])] += count
            ip_octets['.'.join(octets[:2])] += count
            ip_octets['.'.join(octets[:3])] += count
            ip_octets['.'.join(octets[:4])] += count

        sorted_octets = sorted(ip_octets.items(), key=operator.itemgetter(1), reverse=True)

        # Countries
        cnt_mask = 0
        seen_countries = set(self.countries.keys())
        for country in seen_countries:
            i = shared.countries.index(country)
            cnt_mask |= (1<<(i+1))
        cnt_mask = str(cnt_mask)

        countries_inc = defaultdict(int)
        for inc in self.bids:
            countries_inc[inc.country] += 1

        country_rank = 0
        for country, count in countries_inc.iteritems():
            country_rank += shared.country_rank[country] * count
        if len(countries_inc):
            country_rank = float(country_rank) / sum(countries_inc.values())

        regions_mask = 0
        seen_regions = set([shared.country_to_region[c] for c in seen_countries])
        for region in seen_regions:
            i = shared.regions.index(region)
            regions_mask |= (1<<(i+1))
        regions_mask = str(regions_mask)

        # Products
        all_products = ['mobile', 'jewelry', 'home goods', 'sporting goods', 'auto parts', 'office equipment', 'computers', 'books and music', 'furniture', 'clothing']
        products_mask = 0  # compute all products user bidded on as a number
        for p in self.products.keys():
            i = all_products.index(p)+1
            if not i:
                continue
            products_mask |= (1<<i)
        products_mask = str(products_mask)

        # Bids
        bid_on_unpopular = -1.0
        for bid in self.bids:
            if bid.merchandise in ["auto parts", "clothing", "furniture"]:
                bid_on_unpopular = 1.0
                break

        # Generate
        # TODO: add count of times user reached max price in auction (measure greediness)
        labels = ["country", "device", "product", "ip", "ref", "auction", "bids", "increments", "won_auctions", "sim_auctions"]
        values = [len(self.countries), len(self.devices), len(self.products), len(self.ips), len(self.referrals), auctions_count, len(self.bids), true_bids_count, won_auctions_count, sim_auctions]
        #ops = ["+", "-", "*", "/"]
        ops = ["/"]
        generated_features = dict()

        for op in ops:
            for i in xrange(len(labels)):
                al = labels[i]
                av = values[i]
                for j in xrange(i+1, len(labels)):
                    bl = labels[j]
                    bv = values[j]
                    if op == "+":
                        generated_features[al+op+bl] = av+bv
                    elif op == "-":
                        generated_features[al+op+bl] = av-bv
                    elif op == "*":
                        generated_features[al+op+bl] = av*bv
                    elif op == "/":
                        generated_features[al+op+bl] = utils.divide(av,bv)

        # Time
        time_hist_prob = 0
        for bid in self.bids:
            i = bisect.bisect_left(shared.human_hist_bins, bid.time)
            time_hist_prob += shared.human_hist[i-1]

        if self.bids:
            time_hist_prob /= float(len(self.bids))


        features = {
            "set_0": self.bidder_id in shared.set0,
            # Address
            # "addr_1": self.get_addr_1(),
            # "addr_2": self.get_addr_2(),
            # "is_addr_1_unique": len(shared.addr_1[self.get_addr_1()]) == 1,
            # "is_addr_2_unique": len(shared.addr_2[self.get_addr_2()]) == 1,
            # "addr_1=a3d2de7675556553a5f08e4c88d2c228": self.get_addr_1() == 'a3d2de7675556553a5f08e4c88d2c228',

            # Devices
            # "devices": len(self.devices),
            # TODO: devices per last X bids
            #"avg_devices_per_auction": avg_devices_per_auction,
            #"median_devices_per_auction": median_devices_per_auction,
            #"std_devices_per_auction": std_devices_per_auction,

            # Time
            "time_hist_prob": time_hist_prob,

            # Country
            #"unique_countries_count": len(self.countries),
            # "frequent_country": self.get_frequent(self.countries),
            "avg_countries_per_auction": avg_countries_per_auction,
            "median_countries_per_auction": median_countries_per_auction,
            "std_countries_per_auction": std_countries_per_auction,
            # "seen_countries": cnt_mask,
            #"country_rank": country_rank,
            # "seen_regions": regions_mask,
            #"regions_count": len(seen_regions),
            "country_change": self.get_change_rate('country'),

            # Products
            # "frequent_product": self.get_frequent(self.products),
            "bid_on_unpopular": bid_on_unpopular,
            # "products_mask": products_mask,

            # IP
            #"unique_ips": len(self.ips),
            "avg_ips_per_auction": avg_ips_per_auction,
            "median_ips_per_auction": median_ips_per_auction,
            "std_ips_per_auction": std_ips_per_auction,
            #"avg_unique_ips": utils.divide(len(self.ips), bids_count),
            #"avg_ips": utils.divide(sum(self.ips.values()), bids_count),
            "avg_ips_per_increment": avg_ips_per_increment,
            # "frequent_ip": frequent_ip,
            #"frequent_ip_class": self.get_ip_class(frequent_ip),  # http://www.vlsm-calc.net/ipclasses.php
            # "most_popular_octet": sorted_octets[0][0] if sorted_octets else '',
            #"ip_rank": self.get_ip_rank(self.bids),
            "ip_change": self.get_change_rate('ip_pref'),

            # Referrals
            # "frequent_referral": self.get_frequent(self.referrals),
            #"referrals_count": len(self.referrals),
            "avg_referrals_per_auction": avg_referrals_per_auction,
            "median_referrals_per_auction": median_referrals_per_auction,
            "std_referrals_per_auction": std_referrals_per_auction,
            "referral_change": self.get_change_rate('url'),

            # Auctions
            # "auctions_count": auctions_count,
            #"won_auctions_count": won_auctions_count,
            #"sim_auctions": sim_auctions,
            # "auction_rank": auction_rank,

            # Payment
            # "payment_type": self.get_payment_type(),
            # "payment_acct": self.get_payment_acct(),
            # "is_pmt_type_unique": len(shared.pmt_type[self.get_payment_type()]) == 1,
            # "is_pmt_acct_unique": len(shared.pmt_accnt[self.get_payment_acct()]) == 1,
            #"payment_type=addr_1": self.get_payment_type() == self.get_addr_1(),
            #"payment_acct=addr_2": self.get_payment_acct() == self.get_addr_2(),

            # Bids
            #"true_bids_count": true_bids_count,
            # "avg_inc_per_auction": avg_inc_per_auction,
            # "median_inc_per_auction": median_inc_per_auction,
            # "std_inc_per_auction": std_inc_per_auction,

            # Price
            # rmse is calculated based on won_price and measure user price threshold/estimate
            # expect to have it bigger for humans and smaller for robots
            #"human_price_rmse_per_auction": human_price_rmse_per_auction,
            #"human_price_rmse_per_auction": human_price_rmse_per_auction,

            #"robot_price_rmse_per_auction": robot_price_rmse_per_auction,
            #"robot_price_rmse_per_product": robot_price_rmse_per_product,
        }

        features.update(generated_features)
        # features.update(buckets_dict)

        return features
def train():

    # Initialize torch.distributed
    init_distributed()

    print_rank_0('AutoMP: training ParallelTransformerLayer...')

    batch_size = args.batch_size
    sequence_length = args.sequence_length
    hidden_size = args.hidden_size
    vocab_size = args.vocab_size
    hidden_dropout = args.hidden_dropout
    attention_dropout = args.attention_dropout
    num_layers = args.num_layers
    layernorm_epsilon = args.layernorm_epsilon
    num_attention_heads = args.num_attention_heads

    input_indices = torch.randint(low=0,
                                  high=vocab_size,
                                  size=(batch_size, sequence_length))
    input_indices = input_indices.to(torch.cuda.current_device())
    labels = torch.randint(low=0,
                           high=vocab_size,
                           size=(batch_size, sequence_length))
    labels = labels.to(torch.cuda.current_device())
    position_indices = torch.tile(torch.arange(start=0, end=sequence_length),
                                  (batch_size, 1))
    position_indices = position_indices.to(torch.cuda.current_device())

    def init_method_normal(tensor):
        return torch.nn.init.normal_(tensor, mean=0.0, std=1.0)

    def gpt2_attention_mask_func(attention_scores, ltor_mask):
        attention_scores.masked_fill_(ltor_mask, -10000.0)
        return attention_scores

    def init_method_normal(tensor):
        return torch.nn.init.normal_(tensor, mean=0.0, std=1.0)

    embedding = Embedding(hidden_size=hidden_size,
                          vocab_size=vocab_size,
                          max_sequence_length=sequence_length,
                          embedding_dropout_prob=hidden_dropout,
                          init_method=init_method_normal)
    embedding_output = embedding.forward(input_indices, position_indices)

    transformer_layer = ParallelTransformerLayer(
        attention_mask_func=gpt2_attention_mask_func,
        layer_number=0,
        hidden_size=hidden_size,
        layernorm_epsilon=layernorm_epsilon,
        num_attention_heads=num_attention_heads,
        attention_dropout=attention_dropout,
        hidden_dropout=hidden_dropout)

    # attention_mask, loss_mask, position_ids = get_ltor_masks_and_position_ids(input_indices, vocab_size - 1)
    attention_mask = (torch.randint(
        low=0,
        high=2,
        size=(sequence_length,
              divide(num_attention_heads, torch.distributed.get_world_size()),
              batch_size, batch_size)) < 0).cuda()

    optimizer = torch.optim.SGD(transformer_layer.parameters(), lr=0.01)

    profiler = Profiler(os.path.join('benchmark', args.exp_name))

    num_epochs = 5
    tot_time = 0
    nproc = torch.distributed.get_world_size()
    for epoch in range(num_epochs):
        input_ = torch.rand(size=embedding_output.size()).cuda()

        overall_name = f'transformer_layer_np-{nproc}_hs-{hidden_size}_nah-{num_attention_heads}_bsz-{batch_size}'
        profiler.start(overall_name)

        fname = f'transformer_layer_forward_np-{nproc}_hs-{hidden_size}_nah-{num_attention_heads}_bsz-{batch_size}'
        # Forward pass
        profiler.start(fname)
        loss = transformer_layer.forward(input_, attention_mask)
        train_loss = torch.mean(loss)
        # print(train_loss)
        torch.cuda.synchronize()
        profiler.stop(fname)
        # Backward pass
        bname = f'transformer_layer_backward_np-{nproc}_hs-{hidden_size}_nah-{num_attention_heads}_bsz-{batch_size}'
        profiler.start(bname)
        optimizer.zero_grad()
        train_loss.backward()
        optimizer.step()
        torch.cuda.synchronize()
        profiler.stop(bname)

        profiler.stop(overall_name)
Esempio n. 36
0
    """
    # this works if format of file names in a COO_folder is not like chr[chr_num].txt but we should care about sequence of filenames to be same in
    # folder of high resolution files and folder of low resolution files
    chr_files_list = [f for f in os.listdir(COO_folder_path) if (not f.startswith('.')) & (not f.endswith('_npy_form_tmp.npy'))]
    chr_num_list = []
    for f in chr_files_list:
        m = re.search('chr(\d+|x)', f, re.IGNORECASE)
        chr_num_list.append(int(m.group(1)))
    """
    frames_data = []
    index_data = []
    for i in range(args['min_chrN'], args['max_chrN'] + 1):
        chr_COO_file_name = "chr" + str(i) + ".txt"
        chr_data_path = os.path.join(COO_folder_path, chr_COO_file_name)
        temp_frames, temp_index = utils.divide(chr_data_path, i,
                                               args['resolution'],
                                               args['genome_type'],
                                               args['COO_format'])
        frames_data.extend(temp_frames)
        index_data.extend(temp_index)
        print("chr" + str(i) + " is done!")

frames_data = np.stack(frames_data, axis=0)
index_data = np.stack(index_data, axis=0)
if not os.path.exists(args['output_folder_path']):
    os.makedirs(args['output_folder_path'])
np.save(
    os.path.join(args['output_folder_path'],
                 args['output_file_name'] + ".npy"), frames_data)
np.save(
    os.path.join(args['output_folder_path'],
                 args['output_file_name'] + "-index.npy"), index_data)
Esempio n. 37
0
delimiter = opt.delimiter
expRes = 10000
## need to make resolution adjustable.
length = chrs_length[chrN - 1] / expRes

# divide the input matrix into sub-matrixes.

inputMatrix = utils.readFiles(input_file, length + 1, expRes, delimiter)
print("inputMatrix is symmetric?")
print(is_symmetric(inputMatrix))

compareMatrix = utils.readFiles(compare_matrix, length + 1, expRes, delimiter)
print("compareMatrix is symmetric?")
print(is_symmetric(compareMatrix))

low_resolution_samples, index = utils.divide(inputMatrix, chrN)

low_resolution_samples = np.minimum(
    HiC_max_value, low_resolution_samples
)  # why use HiC_max_value, in this way, low_resolution_samples will not change.

batch_size = low_resolution_samples.shape[0]  #256
# batch_size=256

print("batch_size:", batch_size)

# Reshape the high-quality Hi-C sample as the target value of the training.
sample_size = low_resolution_samples.shape[-1]
padding = conv2d1_filters_size + conv2d2_filters_size + conv2d3_filters_size - 3
half_padding = padding / 2
output_length = sample_size - padding
Esempio n. 38
0
 def test_process(self):
     numSets = [[1,-1,2,4,3.2, 4.1], [1], [-1,2]]
     for nums in numSets:
         self.assertAlmostEquals(self.node._processReturn(nums, 1), 
                                 utils.divide(nums))
         self.assertTimeIndependent(nums)
Esempio n. 39
0
 def count_cooc_over_min_freq(self, list, cache=False):
     
    cooc, frequencies = self.count_cooc_and_freq(list, cache)
    return utils.divide(cooc, min(frequencies))
Esempio n. 40
0
 def generate_diff_player_shots_closest_defender(
         self, player_daily_shots_closest_defender,
         player_total_shots_closest_defender):
     player_daily_shots_closest_defender = player_daily_shots_closest_defender[
         ['player_id', 'fg3m', 'fg3a', 'def_dist']]
     player_total_shots_closest_defender = player_total_shots_closest_defender[
         ['player_id', 'fg3m', 'fg3a', 'def_dist']]
     # Tight
     player_daily_shots_closest_defender_tight = player_daily_shots_closest_defender[
         (player_daily_shots_closest_defender['def_dist'] ==
          '0-2 Feet - Very Tight') |
         (player_daily_shots_closest_defender['def_dist'] ==
          '2-4 Feet - Tight')].drop('def_dist',
                                    axis=1).groupby(['player_id'
                                                     ]).sum().reset_index()
     player_daily_shots_closest_defender_tight[
         'fg3_pct'] = player_daily_shots_closest_defender_tight.apply(
             lambda row: divide(row.fg3m, row.fg3a), axis=1)
     player_daily_shots_closest_defender_tight.columns = [
         str(col) + '_daily_tight' if col != 'player_id' else col
         for col in player_daily_shots_closest_defender_tight
     ]
     player_total_shots_closest_defender_tight = player_total_shots_closest_defender[
         (player_total_shots_closest_defender['def_dist'] ==
          '0-2 Feet - Very Tight') |
         (player_total_shots_closest_defender['def_dist'] ==
          '2-4 Feet - Tight')].drop('def_dist',
                                    axis=1).groupby(['player_id'
                                                     ]).sum().reset_index()
     player_total_shots_closest_defender_tight[
         'fg3_pct'] = player_total_shots_closest_defender_tight.apply(
             lambda row: divide(row.fg3m, row.fg3a), axis=1)
     player_total_shots_closest_defender_tight.columns = [
         str(col) + '_total_tight' if col != 'player_id' else col
         for col in player_total_shots_closest_defender_tight
     ]
     merged_df_tight = player_daily_shots_closest_defender_tight.merge(
         player_total_shots_closest_defender_tight,
         how='inner',
         on='player_id')
     merged_df_tight['fg3m_diff_tight'] = merged_df_tight.apply(
         lambda row: row.fg3m_daily_tight - row.fg3m_total_tight, axis=1)
     merged_df_tight['fg3a_diff_tight'] = merged_df_tight.apply(
         lambda row: row.fg3a_daily_tight - row.fg3a_total_tight, axis=1)
     merged_df_tight['fg3_pct_diff_tight'] = merged_df_tight.apply(
         lambda row: row.fg3_pct_daily_tight - row.fg3_pct_total_tight,
         axis=1)
     # Open
     player_daily_shots_closest_defender_open = player_daily_shots_closest_defender[
         (player_daily_shots_closest_defender['def_dist'] ==
          '4-6 Feet - Open') |
         (player_daily_shots_closest_defender['def_dist'] ==
          '6+ Feet - Wide Open')].drop('def_dist', axis=1).groupby(
              ['player_id']).sum().reset_index()
     player_daily_shots_closest_defender_open[
         'fg3_pct'] = player_daily_shots_closest_defender_open.apply(
             lambda row: divide(row.fg3m, row.fg3a), axis=1)
     player_daily_shots_closest_defender_open.columns = [
         str(col) + '_daily_open' if col != 'player_id' else col
         for col in player_daily_shots_closest_defender_open
     ]
     player_total_shots_closest_defender_open = player_total_shots_closest_defender[
         (player_total_shots_closest_defender['def_dist'] ==
          '4-6 Feet - Open') |
         (player_total_shots_closest_defender['def_dist'] ==
          '6+ Feet - Wide Open')].drop('def_dist', axis=1).groupby(
              ['player_id']).sum().reset_index()
     player_total_shots_closest_defender_open[
         'fg3_pct'] = player_total_shots_closest_defender_open.apply(
             lambda row: divide(row.fg3m, row.fg3a), axis=1)
     player_total_shots_closest_defender_open.columns = [
         str(col) + '_total_open' if col != 'player_id' else col
         for col in player_total_shots_closest_defender_open
     ]
     merged_df_open = player_daily_shots_closest_defender_open.merge(
         player_total_shots_closest_defender_open,
         how='inner',
         on='player_id')
     merged_df_open['fg3m_diff_open'] = merged_df_open.apply(
         lambda row: row.fg3m_daily_open - row.fg3m_total_open, axis=1)
     merged_df_open['fg3a_diff_open'] = merged_df_open.apply(
         lambda row: row.fg3a_daily_open - row.fg3a_total_open, axis=1)
     merged_df_open['fg3_pct_diff_open'] = merged_df_open.apply(
         lambda row: row.fg3_pct_daily_open - row.fg3_pct_total_open,
         axis=1)
     return merged_df_tight.merge(merged_df_open,
                                  how='inner',
                                  on='player_id')