def parse_from_kinopoisk(kinopoisk_id, name=None, film=None):
'''
Parse page of film with kinopoisk_id from kinopoisk.ru site
'''
if film is None:
try:
film = Films.objects.get(kinopoisk_id=kinopoisk_id)
except Films.DoesNotExist:
for word in APP_FILM_DIRTY_WORDS:
if word in name:
name = name.replace(word, '').strip()
msg = u"Creating new film object with name {name} and kinopoisk_id: {id}"
print msg.format(name=name, id=kinopoisk_id)
params = {
'kinopoisk_id': kinopoisk_id,
'name': name if name else u' ',
'type': APP_FILM_FULL_FILM,
'release_date': datetime.utcfromtimestamp(0),
'world_release_date': datetime.utcfromtimestamp(0),
}
film = Films(**params)
film.save()
try:
page_dump = acquire_page(film.kinopoisk_id)
facts = extract_facts_from_dump(page_dump)
process_film_facts(film, facts)
except Exception, e:
msg = u"Caught Exception in parse_from_kinopoisk"
traceback_own(e, msg=msg)
def get_time_stamp(time_object):
epoch = datetime.utcfromtimestamp(0)
def unix_time_millis(dt):
return (dt - epoch).total_seconds() * 1000.0
return int(unix_time_millis(time_object))
def parse_from_kinopoisk(kinopoisk_id, name=None, film=None):
'''
Parse page of film with kinopoisk_id from kinopoisk.ru site
'''
if film is None:
try:
film = Films.objects.get(kinopoisk_id=kinopoisk_id)
except Films.DoesNotExist:
print u"Creating new film object with name {} and kinopoisk_id {}".format(name, kinopoisk_id)
film = Films(kinopoisk_id=kinopoisk_id,
name=name if name else u' ',
type=APP_FILM_FULL_FILM,
release_date=datetime.utcfromtimestamp(0))
film.save()
try:
page_dump = acquire_page(film.kinopoisk_id)
facts = extract_facts_from_dump(page_dump)
process_film_facts(film, facts)
except Exception, exception:
print "Caught Exception in parse_from_kinopoisk"
import traceback
traceback.print_exc()
def micro_timestamp_datetime(self, ts):
return dt.utcfromtimestamp(int(ts) / 1000).replace(tzinfo=pytz.utc)
def utcfromtimestamp(t):
utc = pytz.utc
return datetime_z.utcfromtimestamp(t).replace(tzinfo=utc)
def utcfromtimestamp(ts):
return _datetime.utcfromtimestamp(ts).replace(tzinfo=UTC)
def socrata(catalog_record, url):
o = urlparse(url)
client = ModdedSocrata(o.netloc, None)
uparts = o.path.split('/')
looks_like_ids = list(filter(lambda x: '-' in x, uparts))
if looks_like_ids:
identifier = looks_like_ids[-1]
else:
identifier = uparts[-1]
md = client.get_metadata(identifier)
from pprint import pprint
pprint(md.keys())
assert 'columns' in md, 'columns not found in json response'
assert 'name' in md, 'name not found in json response'
if not catalog_record.id:
catalog_record.save()
for column in md['columns']:
rc = RecordColumn.objects.get_or_create(
catalog_record=catalog_record,
field_name=column['fieldName'])[0]
rc.data_type = column['dataTypeName']
rc.label = column['name']
#rc.position = column['position']
rc.catalog_record_order = column['position']
rc.render_type = column['renderTypeName']
rc.save()
dataset = Dataset.objects.get_or_create(catalog_record=catalog_record)[0]
dataset.last_sync = datetime.now()
dataset.sourced_meta_data = md
dataset.title = md['name']
dataset.description = md.get('description', '')
dataset.issued = datetime.utcfromtimestamp(md['publicationDate'])
dataset.modified = datetime.utcfromtimestamp(md['indexUpdatedAt'])
dataset.identifier = md['id']
if 'tags' in md:
dataset.keyword = ', '.join(md['tags'])
dataset.theme = md.get('category', '')
dataset.save()
#only record one distriubtion: JSON
dist = Distribution.objects.get_or_create(dataset=dataset)[0]
dist.license = md.get('license', '')
#dist.rights = md['rights'] #wrong kind of rights
dist.download_url = url
dist.access_url = catalog_record.landing_page or catalog_record.distribution_fields
dist.media_type = 'application/json'
dist.format = 'json'
if 'blobFileSize' in md:
dist.byte_size = md['blobFileSize']
if 'blobMimeType' in md:
dist.media_type = md['blobMimeType']
#blobFilename?
dist.save()
#Sometimes the API doesn't report the columns fields with an old api request
if not md['columns'] and not md.get('blobId'):
try:
fields, types = client.get_columns(identifier)
except HTTPError: #may not be available
pass
else:
for field_name, field_type in zip(fields, types):
rc = RecordColumn.objects.get_or_create(
catalog_record=catalog_record,
field_name=field_name)[0]
rc.data_type = field_type
if not rc.label:
rc.label = field_name
rc.save()
def process(self, charge, subscription):
# Grab the payment card ID from the subscription's payment source list (there will only be one)
card_id = subscription.customer.sources.data[0].id
# Confirm the payment method was migrated to Braintree by looking up
# BT Payment Methods using the Stripe card ID
payment_method = gateway.payment_method.find(card_id)
# Ensure we found a Payment Method, and if not, report it to Sentry as an error
if not payment_method:
message = f'The payment method was not migrated to' \
f' Braintree for this charge: {charge.id}'
print(message)
logger.error(message, exc_info=True)
return
# Determine the Braintree plan, merchant account, and price
# for the subscription based on the subscription's attributes
BT_plan_id = get_plan_id(charge.currency)
BT_merchant_account = get_merchant_account_id_for_card(charge.currency)
BT_price = Decimal(
zero_decimal_currency_fix(charge.amount, charge.currency))
BT_first_billing_date = datetime.utcfromtimestamp(
subscription.current_period_end)
# Create the Braintree subscription and record the result for analysis
BT_create_subscription_result = gateway.subscription.create({
'payment_method_token':
payment_method.token,
'plan_id':
BT_plan_id,
'merchant_account_id':
BT_merchant_account,
'price':
BT_price,
'first_billing_date':
BT_first_billing_date
})
# If the subscription creation failed, record the error in Sentry
if not BT_create_subscription_result.is_success:
message = f'Failed to create a Braintree subscription from Stripe subscription {subscription.id}'
print(message)
logger.error(message, exc_info=True)
logger.error(BT_create_subscription_result.message)
# this might not exist
if BT_create_subscription_result.errors:
logger.error(BT_create_subscription_result.errors)
return
# Cancel Stripe subscription
try:
stripe.Subscription.delete(subscription.id)
except stripe.error.StripeError:
message = f'Failed to cancel the stripe subscription ' \
f'{subscription.id}, cancelling the Braintree subscription'
print(message)
logger.error(message, exc_info=True)
# Cancel the subscription that we just created since the Stripe subscription cancellation failed.
BT_cancel_subscription_result = gateway.subscription.cancel(
BT_create_subscription_result.subscription.id)
# if this fails, we're really screwed, so make it a critical log now that the
# customer has two active subscriptions
if not BT_cancel_subscription_result.is_success:
message = f'Failed to cancel the Braintree Subscription, ' \
f'customer now has two active subscriptions - ' \
f'Braintree: {BT_create_subscription_result.subscription.id}, ' \
f'Stripe: {subscription.id}'
print(message)
logger.critical(message)
def mofo_donation_receipt_datetime(ts):
# convert unix timestamp to e.g. "Thursday, Feb 11, 2021 at 4:20pm (GMT-08:00)"
ds = datetime.utcfromtimestamp(float(ts))
return ds - timedelta(hours=8)
