def parse_from_kinopoisk(kinopoisk_id, name=None, film=None): ''' Parse page of film with kinopoisk_id from kinopoisk.ru site ''' if film is None: try: film = Films.objects.get(kinopoisk_id=kinopoisk_id) except Films.DoesNotExist: for word in APP_FILM_DIRTY_WORDS: if word in name: name = name.replace(word, '').strip() msg = u"Creating new film object with name {name} and kinopoisk_id: {id}" print msg.format(name=name, id=kinopoisk_id) params = { 'kinopoisk_id': kinopoisk_id, 'name': name if name else u' ', 'type': APP_FILM_FULL_FILM, 'release_date': datetime.utcfromtimestamp(0), 'world_release_date': datetime.utcfromtimestamp(0), } film = Films(**params) film.save() try: page_dump = acquire_page(film.kinopoisk_id) facts = extract_facts_from_dump(page_dump) process_film_facts(film, facts) except Exception, e: msg = u"Caught Exception in parse_from_kinopoisk" traceback_own(e, msg=msg)
def get_time_stamp(time_object): epoch = datetime.utcfromtimestamp(0) def unix_time_millis(dt): return (dt - epoch).total_seconds() * 1000.0 return int(unix_time_millis(time_object))
def parse_from_kinopoisk(kinopoisk_id, name=None, film=None): ''' Parse page of film with kinopoisk_id from kinopoisk.ru site ''' if film is None: try: film = Films.objects.get(kinopoisk_id=kinopoisk_id) except Films.DoesNotExist: print u"Creating new film object with name {} and kinopoisk_id {}".format(name, kinopoisk_id) film = Films(kinopoisk_id=kinopoisk_id, name=name if name else u' ', type=APP_FILM_FULL_FILM, release_date=datetime.utcfromtimestamp(0)) film.save() try: page_dump = acquire_page(film.kinopoisk_id) facts = extract_facts_from_dump(page_dump) process_film_facts(film, facts) except Exception, exception: print "Caught Exception in parse_from_kinopoisk" import traceback traceback.print_exc()
def micro_timestamp_datetime(self, ts): return dt.utcfromtimestamp(int(ts) / 1000).replace(tzinfo=pytz.utc)
def utcfromtimestamp(t): utc = pytz.utc return datetime_z.utcfromtimestamp(t).replace(tzinfo=utc)
def utcfromtimestamp(ts): return _datetime.utcfromtimestamp(ts).replace(tzinfo=UTC)
def socrata(catalog_record, url): o = urlparse(url) client = ModdedSocrata(o.netloc, None) uparts = o.path.split('/') looks_like_ids = list(filter(lambda x: '-' in x, uparts)) if looks_like_ids: identifier = looks_like_ids[-1] else: identifier = uparts[-1] md = client.get_metadata(identifier) from pprint import pprint pprint(md.keys()) assert 'columns' in md, 'columns not found in json response' assert 'name' in md, 'name not found in json response' if not catalog_record.id: catalog_record.save() for column in md['columns']: rc = RecordColumn.objects.get_or_create( catalog_record=catalog_record, field_name=column['fieldName'])[0] rc.data_type = column['dataTypeName'] rc.label = column['name'] #rc.position = column['position'] rc.catalog_record_order = column['position'] rc.render_type = column['renderTypeName'] rc.save() dataset = Dataset.objects.get_or_create(catalog_record=catalog_record)[0] dataset.last_sync = datetime.now() dataset.sourced_meta_data = md dataset.title = md['name'] dataset.description = md.get('description', '') dataset.issued = datetime.utcfromtimestamp(md['publicationDate']) dataset.modified = datetime.utcfromtimestamp(md['indexUpdatedAt']) dataset.identifier = md['id'] if 'tags' in md: dataset.keyword = ', '.join(md['tags']) dataset.theme = md.get('category', '') dataset.save() #only record one distriubtion: JSON dist = Distribution.objects.get_or_create(dataset=dataset)[0] dist.license = md.get('license', '') #dist.rights = md['rights'] #wrong kind of rights dist.download_url = url dist.access_url = catalog_record.landing_page or catalog_record.distribution_fields dist.media_type = 'application/json' dist.format = 'json' if 'blobFileSize' in md: dist.byte_size = md['blobFileSize'] if 'blobMimeType' in md: dist.media_type = md['blobMimeType'] #blobFilename? dist.save() #Sometimes the API doesn't report the columns fields with an old api request if not md['columns'] and not md.get('blobId'): try: fields, types = client.get_columns(identifier) except HTTPError: #may not be available pass else: for field_name, field_type in zip(fields, types): rc = RecordColumn.objects.get_or_create( catalog_record=catalog_record, field_name=field_name)[0] rc.data_type = field_type if not rc.label: rc.label = field_name rc.save()
def process(self, charge, subscription): # Grab the payment card ID from the subscription's payment source list (there will only be one) card_id = subscription.customer.sources.data[0].id # Confirm the payment method was migrated to Braintree by looking up # BT Payment Methods using the Stripe card ID payment_method = gateway.payment_method.find(card_id) # Ensure we found a Payment Method, and if not, report it to Sentry as an error if not payment_method: message = f'The payment method was not migrated to' \ f' Braintree for this charge: {charge.id}' print(message) logger.error(message, exc_info=True) return # Determine the Braintree plan, merchant account, and price # for the subscription based on the subscription's attributes BT_plan_id = get_plan_id(charge.currency) BT_merchant_account = get_merchant_account_id_for_card(charge.currency) BT_price = Decimal( zero_decimal_currency_fix(charge.amount, charge.currency)) BT_first_billing_date = datetime.utcfromtimestamp( subscription.current_period_end) # Create the Braintree subscription and record the result for analysis BT_create_subscription_result = gateway.subscription.create({ 'payment_method_token': payment_method.token, 'plan_id': BT_plan_id, 'merchant_account_id': BT_merchant_account, 'price': BT_price, 'first_billing_date': BT_first_billing_date }) # If the subscription creation failed, record the error in Sentry if not BT_create_subscription_result.is_success: message = f'Failed to create a Braintree subscription from Stripe subscription {subscription.id}' print(message) logger.error(message, exc_info=True) logger.error(BT_create_subscription_result.message) # this might not exist if BT_create_subscription_result.errors: logger.error(BT_create_subscription_result.errors) return # Cancel Stripe subscription try: stripe.Subscription.delete(subscription.id) except stripe.error.StripeError: message = f'Failed to cancel the stripe subscription ' \ f'{subscription.id}, cancelling the Braintree subscription' print(message) logger.error(message, exc_info=True) # Cancel the subscription that we just created since the Stripe subscription cancellation failed. BT_cancel_subscription_result = gateway.subscription.cancel( BT_create_subscription_result.subscription.id) # if this fails, we're really screwed, so make it a critical log now that the # customer has two active subscriptions if not BT_cancel_subscription_result.is_success: message = f'Failed to cancel the Braintree Subscription, ' \ f'customer now has two active subscriptions - ' \ f'Braintree: {BT_create_subscription_result.subscription.id}, ' \ f'Stripe: {subscription.id}' print(message) logger.critical(message)
def mofo_donation_receipt_datetime(ts): # convert unix timestamp to e.g. "Thursday, Feb 11, 2021 at 4:20pm (GMT-08:00)" ds = datetime.utcfromtimestamp(float(ts)) return ds - timedelta(hours=8)